├── Functions ├── host.json ├── .vscode │ ├── extensions.json │ ├── settings.json │ ├── launch.json │ └── tasks.json ├── Services │ ├── PipelineServiceType.cs │ ├── Returns │ │ ├── PipelineDescription.cs │ │ ├── PipelineErrorDetail.cs │ │ └── PipelineRunStatus.cs │ └── PipelineService.cs ├── template_local.settings.json ├── Helpers │ ├── InvalidRequestException.cs │ ├── PipelineRunRequest.cs │ ├── KeyVaultClient.cs │ ├── SMTPClient.cs │ └── BodyReader.cs ├── Functions.csproj └── Functions │ ├── CancelPipeline.cs │ ├── ExecutePipeline.cs │ ├── ValidatePipeline.cs │ ├── CheckPipelineStatus.cs │ └── GetActivityErrors.cs ├── Synapse ├── factory │ └── FrameworkFactory.json ├── dataflow │ └── MDF _Order_Count.json ├── linkedService │ ├── covid-tracking.json │ ├── Keys.json │ ├── procfwkforsynapse-WorkspaceDefaultStorage.json │ ├── SupportDatabase.json │ ├── procfwkforsynapse-WorkspaceDefaultSqlServer.json │ └── FrameworkFunctions.json ├── integrationRuntime │ └── AutoResolveIntegrationRuntime.json ├── dataset │ └── GetSetMetadata.json ├── trigger │ └── FunctionalTestingTrigger.json └── pipeline │ ├── Wait 1.json │ ├── Wait 2.json │ ├── Wait 3.json │ ├── Wait 4.json │ ├── Wait 5.json │ ├── Wait 6.json │ ├── Wait 7.json │ ├── Wait 8.json │ ├── Wait 10.json │ ├── Wait 9.json │ ├── 01-Grandparent.json │ ├── Throw Exception.json │ └── Email Sender.json ├── .gitattributes ├── MetadataDB ├── Scripts │ ├── Metadata │ │ ├── Pipelines.sql │ │ ├── Properties.sql │ │ ├── Recipients.sql │ │ ├── AlertOutcomes.sql │ │ ├── PipelineParams.sql │ │ ├── PipelineDependencies.sql │ │ ├── RecipientAlertsLink.sql │ │ ├── Stages.sql │ │ ├── Orchestrators.sql │ │ ├── ReplaceDataFactorys.sql │ │ ├── TransferReportingObjects.sql │ │ ├── TransferHelperObjects.sql │ │ ├── DropLegacyObjects.sql │ │ └── DropLegacyTables.sql │ ├── Alter Database Scale.sql │ ├── LogData │ │ ├── ErrorLogBackup.sql │ │ └── ExecutionLogBackup.sql │ ├── Handy Selects.sql │ ├── Script.PreDeployment.sql │ └── Script.PostDeployment.sql ├── Security │ ├── procfwk.sql │ ├── procfwkHelpers.sql │ ├── procfwkTesting.sql │ ├── procfwkReporting.sql │ └── procfwkuser Role.sql ├── procfwk │ ├── Synonyms │ │ ├── PipelineProcesses.sql │ │ ├── DataFactoryDetails.sql │ │ ├── LastExecution.sql │ │ ├── ProcessingStageDetails.sql │ │ ├── AddProperty.sql │ │ ├── AverageStageDuration.sql │ │ ├── CheckForValidURL.sql │ │ ├── CompleteExecutionLog.sql │ │ ├── LastExecutionSummary.sql │ │ ├── AddServicePrincipal.sql │ │ ├── GetExecutionDetails.sql │ │ ├── AddPipelineDependant.sql │ │ ├── CurrentExecutionSummary.sql │ │ ├── DeleteRecipientAlerts.sql │ │ ├── AddServicePrincipalUrls.sql │ │ ├── CompleteExecutionErrorLog.sql │ │ ├── DeleteServicePrincipal.sql │ │ ├── PipelineDependencyChains.sql │ │ ├── WorkerParallelismOverTime.sql │ │ ├── AddRecipientPipelineAlerts.sql │ │ ├── AddServicePrincipalWrapper.sql │ │ └── CheckStageAndPiplineIntegrity.sql │ ├── Views │ │ ├── CurrentProperties.sql │ │ ├── DataFactorys.sql │ │ └── PipelineParameterDataSizes.sql │ ├── Tables │ │ ├── Tenants.sql │ │ ├── Batches.sql │ │ ├── Stages.sql │ │ ├── AlertOutcomes.sql │ │ ├── Subscriptions.sql │ │ ├── BatchExecution.sql │ │ ├── Properties.sql │ │ ├── BatchStageLink.sql │ │ ├── Recipients.sql │ │ ├── PipelineParameters.sql │ │ ├── ErrorLog.sql │ │ ├── PipelineAlertLink.sql │ │ ├── Orchestrators.sql │ │ ├── PipelineAuthLink.sql │ │ ├── PipelineDependencies.sql │ │ ├── Pipelines.sql │ │ ├── ExecutionLog.sql │ │ └── CurrentExecution.sql │ ├── Functions │ │ └── GetPropertyValueInternal.sql │ └── Stored Procedures │ │ ├── SetLogPipelineChecking.sql │ │ ├── SetLogStagePreparing.sql │ │ ├── SetLogPipelineValidating.sql │ │ ├── SetLogPipelineLastStatusCheck.sql │ │ ├── SetLogPipelineRunId.sql │ │ ├── GetPipelinesInStage.sql │ │ ├── GetWorkerPipelineDetails.sql │ │ ├── SetLogPipelineSuccess.sql │ │ ├── SetExecutionBlockDependants.sql │ │ ├── SetLogPipelineRunning.sql │ │ ├── ExecutePrecursorProcedure.sql │ │ ├── GetStages.sql │ │ ├── GetFrameworkOrchestratorDetails.sql │ │ ├── GetPropertyValue.sql │ │ ├── SetErrorLogDetails.sql │ │ ├── CheckForEmailAlerts.sql │ │ ├── GetPipelineParameters.sql │ │ └── GetWorkerDetailsWrapper.sql ├── procfwkTesting │ └── Stored Procedures │ │ ├── CleanUpMetadata.sql │ │ ├── ResetMetadata.sql │ │ ├── Add300WorkerPipelineBatches.sql │ │ └── GetRunIdWhenAvailable.sql ├── dbo │ ├── Stored Procedures │ │ ├── FailProcedure.sql │ │ └── ExampleCustomExecutionPrecursor.sql │ └── Tables │ │ └── ServicePrincipals.sql ├── procfwkReporting │ └── Views │ │ ├── CurrentExecutionSummary.sql │ │ ├── CompleteExecutionLog.sql │ │ ├── LastExecutionSummary.sql │ │ ├── AverageStageDuration.sql │ │ ├── LastExecution.sql │ │ ├── CompleteExecutionErrorLog.sql │ │ └── WorkerParallelismOverTime.sql ├── procfwkHelpers │ ├── Stored Procedures │ │ ├── SetDefaultAlertOutcomes.sql │ │ ├── SetDefaultRecipientPipelineAlerts.sql │ │ ├── SetDefaultPipelineDependants.sql │ │ ├── SetDefaultBatchStageLink.sql │ │ ├── DeleteMetadataWithoutIntegrity.sql │ │ ├── SetDefaultTenant.sql │ │ ├── SetDefaultBatches.sql │ │ ├── SetDefaultSubscription.sql │ │ ├── SetDefaultRecipients.sql │ │ ├── SetDefaultStages.sql │ │ ├── AddServicePrincipalWrapper.sql │ │ ├── SetDefaultPipelineParameters.sql │ │ ├── DeleteRecipientAlerts.sql │ │ ├── SetDefaultPipelines.sql │ │ ├── AddPipelineDependant.sql │ │ └── GetExecutionDetails.sql │ ├── Views │ │ └── PipelineDependencyChains.sql │ └── Functions │ │ └── CheckForValidURL.sql └── MetadataDB.refactorlog ├── .github ├── FUNDING.yml └── ISSUE_TEMPLATE │ ├── feature-request.md │ ├── help---support-request.md │ └── bug-found.md ├── Images ├── Activity Chain.png ├── Database Diagram.png ├── procfwk Designs.vsdx └── Repo Social Media Image.png ├── MetadataDBTests ├── tSQLt.dacpac ├── _TestClasses │ ├── procfwk_GetPropertyValue.sql │ └── procfwk_GetPropertyValueInternal.sql ├── procfwk_GetPropertyValue │ ├── setup.sql │ ├── test WHEN property name is null THEN error raised.sql │ ├── test WHEN property does not exist THEN error raised.sql │ ├── test WHEN property invalidated THEN error raised.sql │ └── test WHEN property exists THEN property value returned.sql ├── procfwk_GetPropertyValueInternal │ ├── test WHEN property does not exist THEN empty string returned.sql │ ├── test WHEN property name is null THEN empty string returned.sql │ └── test WHEN property exists THEN property value returned.sql └── Scripts │ ├── Script.PostDeployment.sql │ └── Script.PreDeployment.sql ├── Reporting ├── Scoping-Template-LDI-Blank.docx └── PowerBI │ ├── Executions Overview.pbit │ └── Executions Overview.pbix ├── .vscode ├── extensions.json ├── settings.json └── launch.json ├── DeploymentTools ├── DataFactory │ ├── config-all.csv │ ├── ProcFwkComponents.json │ ├── GlobalVars.ps1 │ ├── Get Pipelines.ps1 │ └── PopulatePipelinesInDb.ps1 └── DeploymentTools.deployproj ├── CONTRIBUTING.md ├── DataFactory ├── factory │ └── FrameworkFactory.json ├── linkedService │ ├── Keys.json │ ├── SupportDatabase.json │ └── FrameworkFunctions.json ├── integrationRuntime │ └── AzureIR-UKSouth.json ├── dataset │ └── GetSetMetadata.json ├── trigger │ └── FunctionalTestingTrigger.json └── pipeline │ ├── Wait 1.json │ ├── Wait 2.json │ ├── Wait 3.json │ ├── Wait 4.json │ ├── Wait 5.json │ ├── Wait 6.json │ ├── Wait 7.json │ ├── Wait 8.json │ ├── Wait 9.json │ ├── Wait 10.json │ ├── 01-Grandparent.json │ ├── Email Sender.json │ ├── Throw Exception.json │ └── Intentional Error.json ├── Notebooks └── Databricks - Throw Exception.scala ├── FactoryTesting ├── dev.runsettings ├── test.runsettings ├── multi.runsettings ├── Pipelines │ ├── Utilities │ │ ├── UtilitiesHelper.cs │ │ ├── GivenThrowException.cs │ │ ├── GivenEmailSender.cs │ │ └── GivenCheckForRunningPipeline.cs │ ├── 02-Parent │ │ ├── GivenDisabledPipelines.cs │ │ ├── GivenDisabledStages.cs │ │ ├── GivenDisabledBatches.cs │ │ ├── GivenOneExecutionStage.cs │ │ ├── GivenNoPipelineParameters.cs │ │ ├── GivenNoErrorsAndSPNStoredInDatabase.cs │ │ ├── GivenNoErrorsAndSPNStoredInKeyVault.cs │ │ ├── GivenBatchExecutionsForSingleBatch.cs │ │ ├── GivenAlreadyRunning.cs │ │ ├── GivenNoFailureHandling.cs │ │ ├── GivenSimpleFailureHandlingAndRestart.cs │ │ ├── GivenDependencyChainFailureHandlingAndRestart.cs │ │ ├── GivenCancelledWorkerAndRestart.cs │ │ ├── GivenCancelledWorkerThatDoesntBlock.cs │ │ ├── GivenCancelledWorkerInOneExecutionStage.cs │ │ └── GivenSimpleFailureHandling.cs │ └── 01-Grandparent │ │ ├── GivenOneWorkerPipeline.cs │ │ └── Given300WorkerPipelines.cs ├── FactoryTesting.csproj └── Helpers │ └── SettingsHelper.cs └── LICENSE /Functions/host.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0" 3 | } -------------------------------------------------------------------------------- /Synapse/factory/FrameworkFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FrameworkFactory" 3 | } -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/Pipelines.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultPipelines]; -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/Properties.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultProperties]; -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/Recipients.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultRecipients]; -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/AlertOutcomes.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultAlertOutcomes]; -------------------------------------------------------------------------------- /MetadataDB/Security/procfwk.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [procfwk] 2 | AUTHORIZATION [dbo]; 3 | 4 | -------------------------------------------------------------------------------- /MetadataDB/Security/procfwkHelpers.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [procfwkHelpers] 2 | AUTHORIZATION [dbo]; -------------------------------------------------------------------------------- /MetadataDB/Security/procfwkTesting.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [procfwkTesting] 2 | AUTHORIZATION [dbo]; -------------------------------------------------------------------------------- /MetadataDB/Security/procfwkReporting.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA [procfwkReporting] 2 | AUTHORIZATION [dbo]; -------------------------------------------------------------------------------- /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | # These are supported funding model platforms 2 | 3 | github: [mrpaulandrew] 4 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/PipelineParams.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultPipelineParameters]; -------------------------------------------------------------------------------- /Images/Activity Chain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Images/Activity Chain.png -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/PipelineDependencies.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultPipelineDependants]; -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/RecipientAlertsLink.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultRecipientPipelineAlerts] -------------------------------------------------------------------------------- /Images/Database Diagram.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Images/Database Diagram.png -------------------------------------------------------------------------------- /Images/procfwk Designs.vsdx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Images/procfwk Designs.vsdx -------------------------------------------------------------------------------- /MetadataDBTests/tSQLt.dacpac: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDBTests/tSQLt.dacpac -------------------------------------------------------------------------------- /Images/Repo Social Media Image.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Images/Repo Social Media Image.png -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/PipelineProcesses.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[PipelineProcesses] 2 | FOR [procfwk].[Pipelines]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/DataFactoryDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[DataFactoryDetails] 2 | FOR [procfwk].[Orchestrators]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/LastExecution.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[LastExecution] 2 | FOR [procfwkReporting].[LastExecution]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/ProcessingStageDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[ProcessingStageDetails] 2 | FOR [procfwk].[Stages]; 3 | -------------------------------------------------------------------------------- /Reporting/Scoping-Template-LDI-Blank.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Reporting/Scoping-Template-LDI-Blank.docx -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddProperty.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddProperty.sql -------------------------------------------------------------------------------- /Reporting/PowerBI/Executions Overview.pbit: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Reporting/PowerBI/Executions Overview.pbit -------------------------------------------------------------------------------- /Reporting/PowerBI/Executions Overview.pbix: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/Reporting/PowerBI/Executions Overview.pbix -------------------------------------------------------------------------------- /.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-vscode.csharp" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Alter Database Scale.sql: -------------------------------------------------------------------------------- 1 | ALTER DATABASE [FrameworkMetadataDev] MODIFY (EDITION ='Standard', SERVICE_OBJECTIVE = 'S2', MAXSIZE = 50 GB); -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AverageStageDuration.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[AverageStageDuration] 2 | FOR [procfwkReporting].[AverageStageDuration]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/CheckForValidURL.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/CheckForValidURL.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/CompleteExecutionLog.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[CompleteExecutionLog] 2 | FOR [procfwkReporting].[CompleteExecutionLog]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/LastExecutionSummary.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[LastExecutionSummary] 2 | FOR [procfwkReporting].[LastExecutionSummary]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddServicePrincipal.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddServicePrincipal.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/GetExecutionDetails.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/GetExecutionDetails.sql -------------------------------------------------------------------------------- /DeploymentTools/DataFactory/config-all.csv: -------------------------------------------------------------------------------- 1 | type,name,path,value 2 | linkedService,SupportDatabase,typeProperties.connectionString.secretName,$($Env:SQLDatabase) -------------------------------------------------------------------------------- /Functions/.vscode/extensions.json: -------------------------------------------------------------------------------- 1 | { 2 | "recommendations": [ 3 | "ms-azuretools.vscode-azurefunctions", 4 | "ms-dotnettools.csharp" 5 | ] 6 | } 7 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddPipelineDependant.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddPipelineDependant.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/CurrentExecutionSummary.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[CurrentExecutionSummary] 2 | FOR [procfwkReporting].[CurrentExecutionSummary]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/DeleteRecipientAlerts.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/DeleteRecipientAlerts.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddServicePrincipalUrls.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddServicePrincipalUrls.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/CompleteExecutionErrorLog.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[CompleteExecutionErrorLog] 2 | FOR [procfwkReporting].[CompleteExecutionErrorLog]; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/DeleteServicePrincipal.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/DeleteServicePrincipal.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/PipelineDependencyChains.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[PipelineDependencyChains] 2 | FOR [procfwkHelpers].[PipelineDependencyChains]; 3 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/WorkerParallelismOverTime.sql: -------------------------------------------------------------------------------- 1 | CREATE SYNONYM [procfwk].[WorkerParallelismOverTime] 2 | FOR [procfwkReporting].[WorkerParallelismOverTime]; -------------------------------------------------------------------------------- /Functions/Services/PipelineServiceType.cs: -------------------------------------------------------------------------------- 1 | namespace mrpaulandrew.azure.procfwk 2 | { 3 | public enum PipelineServiceType 4 | { 5 | ADF, SYN 6 | } 7 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddRecipientPipelineAlerts.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddRecipientPipelineAlerts.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/AddServicePrincipalWrapper.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/AddServicePrincipalWrapper.sql -------------------------------------------------------------------------------- /MetadataDB/procfwk/Synonyms/CheckStageAndPiplineIntegrity.sql: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mrpaulandrew/procfwk/HEAD/MetadataDB/procfwk/Synonyms/CheckStageAndPiplineIntegrity.sql -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/Stages.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultBatches]; 2 | EXEC [procfwkHelpers].[SetDefaultStages]; 3 | EXEC [procfwkHelpers].[SetDefaultBatchStageLink]; -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/Orchestrators.sql: -------------------------------------------------------------------------------- 1 | EXEC [procfwkHelpers].[SetDefaultTenant]; 2 | EXEC [procfwkHelpers].[SetDefaultSubscription]; 3 | EXEC [procfwkHelpers].[SetDefaultOrchestrators]; -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | If you'd like to contribute towards this code project please contact me. I kindly request a minimum of 5 years experience working with Azure Data Platform services. 2 | 3 | Thanks 4 | 5 | Paul 6 | -------------------------------------------------------------------------------- /DataFactory/factory/FrameworkFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FrameworkFactory", 3 | "properties": { 4 | "globalConfigurations": { 5 | "PipelineBillingEnabled": "true" 6 | } 7 | }, 8 | "location": "uksouth" 9 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Views/CurrentProperties.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwk].[CurrentProperties] 2 | AS 3 | 4 | SELECT 5 | [PropertyName], 6 | [PropertyValue] 7 | FROM 8 | [procfwk].[Properties] 9 | WHERE 10 | [ValidTo] IS NULL; -------------------------------------------------------------------------------- /MetadataDB/procfwkTesting/Stored Procedures/CleanUpMetadata.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkTesting].[CleanUpMetadata] 2 | AS 3 | BEGIN 4 | EXEC [procfwkHelpers].[DeleteMetadataWithIntegrity]; 5 | EXEC [procfwkHelpers].[DeleteMetadataWithoutIntegrity]; 6 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Tenants.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Tenants] 2 | ( 3 | [TenantId] [UNIQUEIDENTIFIER] NOT NULL, 4 | [Name] [NVARCHAR](200) NOT NULL, 5 | [Description] [NVARCHAR](MAX) NULL, 6 | CONSTRAINT [PK_Tenants] PRIMARY KEY CLUSTERED ([TenantId] ASC) 7 | ) -------------------------------------------------------------------------------- /Synapse/dataflow/MDF _Order_Count.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MDF _Order_Count", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [], 7 | "sinks": [], 8 | "transformations": [], 9 | "script": "" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /MetadataDBTests/_TestClasses/procfwk_GetPropertyValue.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA procfwk_GetPropertyValue 2 | AUTHORIZATION [dbo]; 3 | GO 4 | 5 | EXEC sp_addextendedproperty @name = N'tSQLt.TestClass', @value = 1, @level0type = N'SCHEMA', @level0name = N'procfwk_GetPropertyValue' 6 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Tell me about your idea to enhance the procfwk please 4 | title: '' 5 | labels: enhancement 6 | assignees: mrpaulandrew 7 | 8 | --- 9 | 10 | **Share Your Idea** 11 | All features considered. 12 | -------------------------------------------------------------------------------- /MetadataDB/dbo/Stored Procedures/FailProcedure.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [dbo].[FailProcedure] 2 | ( 3 | @RaiseError VARCHAR(50) 4 | ) 5 | AS 6 | BEGIN 7 | IF(@RaiseError = 'true') 8 | BEGIN 9 | RAISERROR('The Stored Procedure intentionally failed.',16,1); 10 | RETURN 0; 11 | END 12 | END; -------------------------------------------------------------------------------- /MetadataDBTests/_TestClasses/procfwk_GetPropertyValueInternal.sql: -------------------------------------------------------------------------------- 1 | CREATE SCHEMA procfwk_GetPropertyValueInternal 2 | AUTHORIZATION [dbo]; 3 | GO 4 | 5 | EXEC sp_addextendedproperty @name = N'tSQLt.TestClass', @value = 1, @level0type = N'SCHEMA', @level0name = N'procfwk_GetPropertyValueInternal' 6 | -------------------------------------------------------------------------------- /Functions/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "bin/Release/netcoreapp3.1/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~3", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish" 7 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "azureFunctions.deploySubpath": "PipelineExecutor/bin/Release/netcoreapp3.1/publish", 3 | "azureFunctions.projectLanguage": "C#", 4 | "azureFunctions.projectRuntime": "~3", 5 | "debug.internalConsoleOptions": "neverOpen", 6 | "azureFunctions.preDeployTask": "publish" 7 | } -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/CurrentExecutionSummary.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[CurrentExecutionSummary] 2 | AS 3 | 4 | SELECT 5 | ISNULL([PipelineStatus], 'Not Started') AS 'PipelineStatus', 6 | COUNT(0) AS 'RecordCount' 7 | FROM 8 | [procfwk].[CurrentExecution] 9 | GROUP BY 10 | [PipelineStatus] -------------------------------------------------------------------------------- /Functions/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Attach to .NET Functions", 6 | "type": "coreclr", 7 | "request": "attach", 8 | "processId": "${command:azureFunctions.pickProcess}" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Views/DataFactorys.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwk].[DataFactorys] 2 | AS 3 | SELECT 4 | [OrchestratorId] AS DataFactoryId, 5 | [OrchestratorName] AS DataFactoryName, 6 | [ResourceGroupName], 7 | [SubscriptionId], 8 | [Description] 9 | FROM 10 | [procfwk].[Orchestrators] 11 | WHERE 12 | [OrchestratorType] = 'ADF'; -------------------------------------------------------------------------------- /MetadataDB/Scripts/LogData/ErrorLogBackup.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID(N'[dbo].[ErrorLogBackup]') IS NOT NULL DROP TABLE [dbo].[ErrorLogBackup]; 2 | 3 | IF OBJECT_ID(N'[procfwk].[ErrorLog]') IS NOT NULL --check for new deployments 4 | BEGIN 5 | SELECT 6 | * 7 | INTO 8 | [dbo].[ErrorLogBackup] 9 | FROM 10 | [procfwk].[ErrorLog]; 11 | END; -------------------------------------------------------------------------------- /Synapse/linkedService/covid-tracking.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "covid-tracking", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "sasUri": "https://pandemicdatalake.blob.core.windows.net/public/curated/covid-19/covid_tracking/latest/covid_tracking.parquet?\"\"" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Batches.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Batches]( 2 | [BatchId] UNIQUEIDENTIFIER DEFAULT(NEWID()) NOT NULL, 3 | [BatchName] [VARCHAR](255) NOT NULL, 4 | [BatchDescription] [VARCHAR](4000) NULL, 5 | [Enabled] [BIT] DEFAULT(0) NOT NULL, 6 | CONSTRAINT [PK_Batches] PRIMARY KEY CLUSTERED 7 | ( 8 | [BatchId] ASC 9 | ) 10 | ) -------------------------------------------------------------------------------- /MetadataDB/Scripts/LogData/ExecutionLogBackup.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID(N'[dbo].[ExecutionLogBackup]') IS NOT NULL DROP TABLE [dbo].[ExecutionLogBackup]; 2 | 3 | IF OBJECT_ID(N'[procfwk].[ExecutionLog]') IS NOT NULL --check for new deployments 4 | BEGIN 5 | SELECT 6 | * 7 | INTO 8 | [dbo].[ExecutionLogBackup] 9 | FROM 10 | [procfwk].[ExecutionLog]; 11 | END; -------------------------------------------------------------------------------- /Notebooks/Databricks - Throw Exception.scala: -------------------------------------------------------------------------------- 1 | // Databricks notebook source 2 | import scala.util.Try 3 | 4 | dbutils.widgets.text("RaiseError", "","") 5 | 6 | 7 | // COMMAND ---------- 8 | 9 | val raiseError = Try(dbutils.widgets.get("RaiseError").toBoolean).getOrElse(false) 10 | 11 | if(raiseError) 12 | { 13 | throw new Exception("The Notebook intentionally failed.") 14 | } 15 | -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValue/setup.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValue.[setup] 2 | AS 3 | 4 | DECLARE @propertyName NVARCHAR(128) = 'TestProperty' 5 | 6 | EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' 7 | 8 | INSERT INTO procfwk.CurrentProperties ( 9 | [PropertyName] 10 | , [PropertyValue] 11 | ) VALUES ( 12 | @propertyName 13 | , 'TestPropertyValue' 14 | ) 15 | -------------------------------------------------------------------------------- /Synapse/linkedService/Keys.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Keys", 3 | "properties": { 4 | "annotations": [ 5 | "procfwk" 6 | ], 7 | "type": "AzureKeyVault", 8 | "typeProperties": { 9 | "baseUrl": "https://FrameworkKeys.vault.azure.net/" 10 | }, 11 | "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework." 12 | } 13 | } -------------------------------------------------------------------------------- /DataFactory/linkedService/Keys.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Keys", 3 | "properties": { 4 | "annotations": [ 5 | "procfwk" 6 | ], 7 | "type": "AzureKeyVault", 8 | "typeProperties": { 9 | "baseUrl": "https://FrameworkKeys.vault.azure.net/" 10 | }, 11 | "description": "Connection to Key Vault for all other ADF linked service credentials required to run the processing framework." 12 | } 13 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Stages.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Stages] ( 2 | [StageId] INT IDENTITY (1, 1) NOT NULL, 3 | [StageName] VARCHAR (225) NOT NULL, 4 | [StageDescription] VARCHAR (4000) NULL, 5 | [Enabled] BIT CONSTRAINT [DF_Stages_Enabled] DEFAULT ((1)) NOT NULL, 6 | CONSTRAINT [PK_Stages] PRIMARY KEY CLUSTERED ([StageId] ASC) 7 | ); 8 | 9 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/AlertOutcomes.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[AlertOutcomes] 2 | ( 3 | [OutcomeBitPosition] INT IDENTITY(0,1) NOT NULL, 4 | [PipelineOutcomeStatus] NVARCHAR(200) NOT NULL, 5 | [BitValue] AS (POWER((2),[OutcomeBitPosition])), 6 | CONSTRAINT [PK_AlertOutcomes] PRIMARY KEY CLUSTERED ([OutcomeBitPosition] ASC), 7 | CONSTRAINT [UK_PipelineOutcomeStatus] UNIQUE ([PipelineOutcomeStatus]) 8 | ) -------------------------------------------------------------------------------- /MetadataDB/procfwk/Views/PipelineParameterDataSizes.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwk].[PipelineParameterDataSizes] 2 | AS 3 | 4 | SELECT 5 | [PipelineId], 6 | SUM( 7 | (CAST( 8 | DATALENGTH( 9 | STRING_ESCAPE([ParameterName] + [ParameterValue],'json')) AS DECIMAL) 10 | /1024) --KB 11 | /1024 --MB 12 | ) AS Size 13 | FROM 14 | [procfwk].[PipelineParameters] 15 | GROUP BY 16 | [PipelineId]; -------------------------------------------------------------------------------- /Synapse/integrationRuntime/AutoResolveIntegrationRuntime.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AutoResolveIntegrationRuntime", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 8, 11 | "timeToLive": 0 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /DataFactory/integrationRuntime/AzureIR-UKSouth.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureIR-UKSouth", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "UK South", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 8, 11 | "timeToLive": 10, 12 | "cleanup": false 13 | } 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultAlertOutcomes.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultAlertOutcomes] 2 | AS 3 | BEGIN 4 | TRUNCATE TABLE [procfwk].[AlertOutcomes]; 5 | 6 | INSERT INTO [procfwk].[AlertOutcomes] 7 | ( 8 | [PipelineOutcomeStatus] 9 | ) 10 | VALUES 11 | ('All'), 12 | ('Success'), 13 | ('Failed'), 14 | ('Unknown'), 15 | ('Cancelled'); 16 | END; -------------------------------------------------------------------------------- /MetadataDB/Security/procfwkuser Role.sql: -------------------------------------------------------------------------------- 1 | /* 2 | CREATE USER [##Data Factory Name (Managed Identity)##] 3 | FROM EXTERNAL PROVIDER; 4 | */ 5 | 6 | CREATE ROLE [procfwkuser] 7 | GO 8 | 9 | GRANT 10 | EXECUTE, 11 | SELECT, 12 | CONTROL, 13 | ALTER 14 | ON SCHEMA::[procfwk] TO [procfwkuser] 15 | GO 16 | 17 | /* 18 | ALTER ROLE [procfwkuser] 19 | ADD MEMBER [##Data Factory Name (Managed Identity)##]; 20 | */ -------------------------------------------------------------------------------- /Functions/Services/Returns/PipelineDescription.cs: -------------------------------------------------------------------------------- 1 | namespace mrpaulandrew.azure.procfwk.Services 2 | { 3 | public class PipelineDescription 4 | { 5 | public string PipelineExists { get; set; } 6 | public string PipelineName { get; set; } 7 | public string PipelineId { get; set; } 8 | public string PipelineType { get; set; } 9 | public int ActivityCount { get; set; } 10 | } 11 | } -------------------------------------------------------------------------------- /Functions/template_local.settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "IsEncrypted": false, 3 | "Values": { 4 | "AzureWebJobsStorage": "UseDevelopmentStorage=true", 5 | "FUNCTIONS_WORKER_RUNTIME": "dotnet", 6 | "AppSettingSmtpHost": "my.smtpservice.com", 7 | "AppSettingSmtpPort": 1234, 8 | "AppSettingSmtpUser": "AlertMailboxUser", 9 | "AppSettingSmtpPass": "Passw0rd123!", 10 | "AppSettingFromEmail": "noreply@adfprocfwk.com" 11 | } 12 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Subscriptions.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Subscriptions] 2 | ( 3 | [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, 4 | [Name] NVARCHAR(200) NOT NULL, 5 | [Description] NVARCHAR(MAX) NULL, 6 | [TenantId] UNIQUEIDENTIFIER NOT NULL, 7 | CONSTRAINT [PK_Subscriptions] PRIMARY KEY CLUSTERED ([SubscriptionId] ASC), 8 | CONSTRAINT [FK_Subscriptions_Tenants] FOREIGN KEY([TenantId]) REFERENCES [procfwk].[Tenants] ([TenantId]) 9 | ) -------------------------------------------------------------------------------- /MetadataDB/dbo/Tables/ServicePrincipals.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [dbo].[ServicePrincipals]( 2 | [CredentialId] INT IDENTITY(1,1) NOT NULL, 3 | [PrincipalName] NVARCHAR(256) NULL, 4 | [PrincipalId] UNIQUEIDENTIFIER NULL, 5 | [PrincipalSecret] VARBINARY(256) NULL, 6 | [PrincipalIdUrl] NVARCHAR(MAX) NULL, 7 | [PrincipalSecretUrl] NVARCHAR(MAX) NULL, 8 | CONSTRAINT [PK_ServicePrincipals] PRIMARY KEY CLUSTERED ([CredentialId] ASC) 9 | ) 10 | GO 11 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Functions/GetPropertyValueInternal.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [procfwk].[GetPropertyValueInternal] 2 | ( 3 | @PropertyName VARCHAR(128) 4 | ) 5 | RETURNS NVARCHAR(MAX) 6 | AS 7 | BEGIN 8 | DECLARE @PropertyValue NVARCHAR(MAX) 9 | 10 | SELECT 11 | @PropertyValue = ISNULL([PropertyValue],'') 12 | FROM 13 | [procfwk].[CurrentProperties] 14 | WHERE 15 | [PropertyName] = @PropertyName 16 | 17 | RETURN @PropertyValue 18 | END; 19 | -------------------------------------------------------------------------------- /Synapse/dataset/GetSetMetadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GetSetMetadata", 3 | "properties": { 4 | "description": "Single generic dataset used to get and set all database metadata.", 5 | "linkedServiceName": { 6 | "referenceName": "SupportDatabase", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "folder": { 10 | "name": "_ProcFwk" 11 | }, 12 | "annotations": [ 13 | "procfwk" 14 | ], 15 | "type": "AzureSqlTable", 16 | "schema": [] 17 | } 18 | } -------------------------------------------------------------------------------- /DataFactory/dataset/GetSetMetadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "GetSetMetadata", 3 | "properties": { 4 | "description": "Single generic dataset used to get and set all database metadata.", 5 | "linkedServiceName": { 6 | "referenceName": "SupportDatabase", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "folder": { 10 | "name": "_ProcFwk" 11 | }, 12 | "annotations": [ 13 | "procfwk" 14 | ], 15 | "type": "AzureSqlTable", 16 | "schema": [] 17 | } 18 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineChecking.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetLogPipelineChecking] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | UPDATE 12 | [procfwk].[CurrentExecution] 13 | SET 14 | [PipelineStatus] = 'Checking' 15 | WHERE 16 | [LocalExecutionId] = @ExecutionId 17 | AND [StageId] = @StageId 18 | AND [PipelineId] = @PipelineId 19 | END; 20 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogStagePreparing.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetLogStagePreparing] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT 5 | ) 6 | AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | UPDATE 11 | [procfwk].[CurrentExecution] 12 | SET 13 | [PipelineStatus] = 'Preparing' 14 | WHERE 15 | [LocalExecutionId] = @ExecutionId 16 | AND [StageId] = @StageId 17 | AND [StartDateTime] IS NULL 18 | AND [IsBlocked] <> 1; 19 | END; 20 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineValidating.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetLogPipelineValidating] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | UPDATE 12 | [procfwk].[CurrentExecution] 13 | SET 14 | [PipelineStatus] = 'Validating' 15 | WHERE 16 | [LocalExecutionId] = @ExecutionId 17 | AND [StageId] = @StageId 18 | AND [PipelineId] = @PipelineId 19 | END; 20 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/BatchExecution.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[BatchExecution]( 2 | [BatchId] [UNIQUEIDENTIFIER] NOT NULL, 3 | [ExecutionId] [UNIQUEIDENTIFIER] NOT NULL, 4 | [BatchName] VARCHAR(255) NOT NULL, 5 | [BatchStatus] [NVARCHAR](200) NOT NULL, 6 | [StartDateTime] [DATETIME] NOT NULL, 7 | [EndDateTime] [DATETIME] NULL, 8 | CONSTRAINT [PK_BatchExecution] PRIMARY KEY CLUSTERED 9 | ( 10 | [BatchId] ASC, 11 | [ExecutionId] ASC 12 | ) 13 | ) 14 | 15 | 16 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineLastStatusCheck.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetLogPipelineLastStatusCheck] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | UPDATE 12 | [procfwk].[CurrentExecution] 13 | SET 14 | [LastStatusCheckDateTime] = GETUTCDATE() 15 | WHERE 16 | [LocalExecutionId] = @ExecutionId 17 | AND [StageId] = @StageId 18 | AND [PipelineId] = @PipelineId 19 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Properties.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Properties] 2 | ( 3 | [PropertyId] [int] IDENTITY (1, 1) NOT NULL, 4 | [PropertyName] [varchar](128) NOT NULL, 5 | [PropertyValue] [nvarchar](MAX) NOT NULL, 6 | [Description] [nvarchar](MAX) NULL, 7 | [ValidFrom] [datetime] CONSTRAINT [DF_Properties_ValidFrom] DEFAULT (GETDATE()) NOT NULL, 8 | [ValidTo] [datetime] NULL, 9 | CONSTRAINT [PK_Properties] PRIMARY KEY CLUSTERED ([PropertyId] ASC, [PropertyName] ASC) 10 | ) 11 | GO -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineRunId.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetLogPipelineRunId] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT, 6 | @RunId UNIQUEIDENTIFIER = NULL 7 | ) 8 | AS 9 | BEGIN 10 | SET NOCOUNT ON; 11 | 12 | UPDATE 13 | [procfwk].[CurrentExecution] 14 | SET 15 | [PipelineRunId] = LOWER(@RunId) 16 | WHERE 17 | [LocalExecutionId] = @ExecutionId 18 | AND [StageId] = @StageId 19 | AND [PipelineId] = @PipelineId 20 | END; -------------------------------------------------------------------------------- /Synapse/linkedService/procfwkforsynapse-WorkspaceDefaultStorage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "procfwkforsynapse-WorkspaceDefaultStorage", 3 | "type": "Microsoft.Synapse/workspaces/linkedservices", 4 | "properties": { 5 | "typeProperties": { 6 | "url": "https://frameworkdatalake01.dfs.core.windows.net" 7 | }, 8 | "type": "AzureBlobFS", 9 | "connectVia": { 10 | "referenceName": "AutoResolveIntegrationRuntime", 11 | "type": "IntegrationRuntimeReference" 12 | }, 13 | "annotations": [] 14 | } 15 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetPipelinesInStage.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetPipelinesInStage] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT 5 | ) 6 | AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | SELECT 11 | [PipelineId] 12 | FROM 13 | [procfwk].[CurrentExecution] 14 | WHERE 15 | [LocalExecutionId] = @ExecutionId 16 | AND [StageId] = @StageId 17 | AND ISNULL([PipelineStatus],'') <> 'Success' 18 | AND [IsBlocked] <> 1 19 | ORDER BY 20 | [PipelineId] ASC; 21 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/BatchStageLink.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[BatchStageLink] 2 | ( 3 | [BatchId] [UNIQUEIDENTIFIER] NOT NULL, 4 | [StageId] [INT] NOT NULL, 5 | CONSTRAINT [PK_BatchStageLink] PRIMARY KEY CLUSTERED 6 | ( 7 | [BatchId] ASC, 8 | [StageId] ASC 9 | ), 10 | CONSTRAINT [FK_BatchStageLink_Batches] FOREIGN KEY([BatchId]) REFERENCES [procfwk].[Batches] ([BatchId]), 11 | CONSTRAINT [FK_BatchStageLink_Stages] FOREIGN KEY([StageId]) REFERENCES [procfwk].[Stages] ([StageId]) 12 | ) 13 | -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/CompleteExecutionLog.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[CompleteExecutionLog] 2 | AS 3 | 4 | SELECT 5 | [LogId], 6 | [LocalExecutionId], 7 | [StageId], 8 | [PipelineId], 9 | [CallingOrchestratorName], 10 | [ResourceGroupName], 11 | [OrchestratorType], 12 | [OrchestratorName], 13 | [PipelineName], 14 | [StartDateTime], 15 | [PipelineStatus], 16 | [EndDateTime], 17 | DATEDIFF(MINUTE, [StartDateTime], [EndDateTime]) 'RunDurationMinutes' 18 | FROM 19 | [procfwk].[ExecutionLog] -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetWorkerPipelineDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetWorkerPipelineDetails] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | SELECT 12 | [PipelineName], 13 | [OrchestratorName], 14 | [OrchestratorType], 15 | [ResourceGroupName] 16 | FROM 17 | [procfwk].[CurrentExecution] 18 | WHERE 19 | [LocalExecutionId] = @ExecutionId 20 | AND [StageId] = @StageId 21 | AND [PipelineId] = @PipelineId; 22 | END; -------------------------------------------------------------------------------- /Synapse/linkedService/SupportDatabase.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SupportDatabase", 3 | "properties": { 4 | "description": "Connection between ADF and processing framework metadata SQLDB.", 5 | "annotations": [ 6 | "procfwk" 7 | ], 8 | "type": "AzureSqlDatabase", 9 | "typeProperties": { 10 | "connectionString": { 11 | "type": "AzureKeyVaultSecret", 12 | "store": { 13 | "referenceName": "Keys", 14 | "type": "LinkedServiceReference" 15 | }, 16 | "secretName": "FrameworkMetadataDev" 17 | } 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /DataFactory/linkedService/SupportDatabase.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SupportDatabase", 3 | "properties": { 4 | "description": "Connection between ADF and processing framework metadata SQLDB.", 5 | "annotations": [ 6 | "procfwk" 7 | ], 8 | "type": "AzureSqlDatabase", 9 | "typeProperties": { 10 | "connectionString": { 11 | "type": "AzureKeyVaultSecret", 12 | "store": { 13 | "referenceName": "Keys", 14 | "type": "LinkedServiceReference" 15 | }, 16 | "secretName": "FrameworkMetadataDev" 17 | } 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Recipients.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Recipients] 2 | ( 3 | [RecipientId] INT IDENTITY(1,1) NOT NULL, 4 | [Name] VARCHAR(255) NULL, 5 | [EmailAddress] NVARCHAR(500) NOT NULL, 6 | [MessagePreference] CHAR(3) NOT NULL DEFAULT ('TO'), 7 | CONSTRAINT [MessagePreferenceValue] CHECK ([MessagePreference] IN ('TO','CC','BCC')), 8 | [Enabled] BIT NOT NULL DEFAULT 1, 9 | CONSTRAINT [PK_Recipients] PRIMARY KEY CLUSTERED ([RecipientId] ASC), 10 | CONSTRAINT [UK_EmailAddressMessagePreference] UNIQUE ([EmailAddress],[MessagePreference]) 11 | ); 12 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/PipelineParameters.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[PipelineParameters] ( 2 | [ParameterId] INT IDENTITY (1, 1) NOT NULL, 3 | [PipelineId] INT NOT NULL, 4 | [ParameterName] VARCHAR (128) NOT NULL, 5 | [ParameterValue] NVARCHAR(MAX) NULL, 6 | [ParameterValueLastUsed] NVARCHAR(MAX) NULL, 7 | CONSTRAINT [PK_PipelineParameters] PRIMARY KEY CLUSTERED ([ParameterId] ASC), 8 | CONSTRAINT [FK_PipelineParameters_Pipelines] FOREIGN KEY ([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]) 9 | ); 10 | 11 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/ErrorLog.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[ErrorLog] 2 | ( 3 | [LogId] [int] IDENTITY(1,1) NOT NULL, 4 | [LocalExecutionId] [uniqueidentifier] NOT NULL, 5 | [PipelineRunId] [uniqueidentifier] NOT NULL, 6 | [ActivityRunId] [uniqueidentifier] NOT NULL, 7 | [ActivityName] [varchar](100) NOT NULL, 8 | [ActivityType] [varchar](100) NOT NULL, 9 | [ErrorCode] VARCHAR(100) NOT NULL, 10 | [ErrorType] [varchar](100) NOT NULL, 11 | [ErrorMessage] [nvarchar](MAX) NULL, 12 | CONSTRAINT [PK_ErrorLog] PRIMARY KEY CLUSTERED 13 | ( 14 | [LogId] ASC 15 | ) 16 | ) -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineSuccess.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk.SetLogPipelineSuccess 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | UPDATE 12 | [procfwk].[CurrentExecution] 13 | SET 14 | --case for clean up runs 15 | [EndDateTime] = CASE WHEN [EndDateTime] IS NULL THEN GETUTCDATE() ELSE [EndDateTime] END, 16 | [PipelineStatus] = 'Success' 17 | WHERE 18 | [LocalExecutionId] = @ExecutionId 19 | AND [StageId] = @StageId 20 | AND [PipelineId] = @PipelineId 21 | END; 22 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetExecutionBlockDependants.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetExecutionBlockDependants] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER = NULL, 4 | @PipelineId INT 5 | ) 6 | AS 7 | BEGIN 8 | --update dependents status 9 | UPDATE 10 | ce 11 | SET 12 | ce.[PipelineStatus] = 'Blocked', 13 | ce.[IsBlocked] = 1 14 | FROM 15 | [procfwk].[PipelineDependencies] pe 16 | INNER JOIN [procfwk].[CurrentExecution] ce 17 | ON pe.[DependantPipelineId] = ce.[PipelineId] 18 | WHERE 19 | ce.[LocalExecutionId] = @ExecutionId 20 | AND pe.[PipelineId] = @PipelineId 21 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetLogPipelineRunning.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk.SetLogPipelineRunning 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | SET NOCOUNT ON; 10 | 11 | UPDATE 12 | [procfwk].[CurrentExecution] 13 | SET 14 | --case for clean up runs 15 | [StartDateTime] = CASE WHEN [StartDateTime] IS NULL THEN GETUTCDATE() ELSE [StartDateTime] END, 16 | [PipelineStatus] = 'Running' 17 | WHERE 18 | [LocalExecutionId] = @ExecutionId 19 | AND [StageId] = @StageId 20 | AND [PipelineId] = @PipelineId 21 | END; 22 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/PipelineAlertLink.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[PipelineAlertLink] 2 | ( 3 | [AlertId] INT IDENTITY(1,1) NOT NULL, 4 | [PipelineId] INT NOT NULL, 5 | [RecipientId] INT NOT NULL, 6 | [OutcomesBitValue] INT NOT NULL, 7 | [Enabled] BIT NOT NULL DEFAULT 1, 8 | CONSTRAINT [PK_PipelineAlertLink] PRIMARY KEY CLUSTERED ([AlertId] ASC), 9 | CONSTRAINT [FK_PipelineAlertLink_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), 10 | CONSTRAINT [FK_PipelineAlertLink_Recipients] FOREIGN KEY([RecipientId]) REFERENCES [procfwk].[Recipients] ([RecipientId]) 11 | ); -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property does not exist THEN empty string returned.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property name does not exist THEN empty string returned] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = 'TestProperty' 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' 8 | 9 | DECLARE @expected NVARCHAR(4000) = '' 10 | 11 | -- ACT 12 | DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) 13 | 14 | -- ASSERT 15 | EXEC tSQLt.AssertEquals 16 | @Expected = @expected 17 | , @Actual = @actual 18 | -------------------------------------------------------------------------------- /FactoryTesting/dev.runsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /FactoryTesting/test.runsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultRecipientPipelineAlerts.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultRecipientPipelineAlerts] 2 | AS 3 | BEGIN 4 | EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] 5 | @RecipientName = N'Test User 1', 6 | @AlertForStatus = 'All'; 7 | 8 | EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] 9 | @RecipientName = N'Test User 2', 10 | @PipelineName = 'Intentional Error', 11 | @AlertForStatus = 'Failed'; 12 | 13 | EXEC [procfwkHelpers].[AddRecipientPipelineAlerts] 14 | @RecipientName = N'Test User 3', 15 | @PipelineName = 'Wait 1', 16 | @AlertForStatus = 'Success, Failed, Cancelled'; 17 | END; -------------------------------------------------------------------------------- /Synapse/linkedService/procfwkforsynapse-WorkspaceDefaultSqlServer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "procfwkforsynapse-WorkspaceDefaultSqlServer", 3 | "type": "Microsoft.Synapse/workspaces/linkedservices", 4 | "properties": { 5 | "typeProperties": { 6 | "connectionString": "Data Source=tcp:procfwkforsynapse.sql.azuresynapse.net,1433;Initial Catalog=@{linkedService().DBName}" 7 | }, 8 | "parameters": { 9 | "DBName": { 10 | "type": "String" 11 | } 12 | }, 13 | "type": "AzureSqlDW", 14 | "connectVia": { 15 | "referenceName": "AutoResolveIntegrationRuntime", 16 | "type": "IntegrationRuntimeReference" 17 | }, 18 | "annotations": [] 19 | } 20 | } -------------------------------------------------------------------------------- /FactoryTesting/multi.runsettings: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Handy Selects.sql: -------------------------------------------------------------------------------- 1 | SELECT * FROM [procfwk].[BatchExecution] 2 | 3 | SELECT * FROM [procfwk].[CurrentExecution] 4 | 5 | SELECT * FROM [procfwkReporting].[CurrentExecutionSummary] 6 | 7 | SELECT * FROM [procfwk].[CurrentProperties] 8 | 9 | SELECT * FROM [procfwk].[Batches] 10 | 11 | SELECT * FROM [procfwk].[BatchStageLink] 12 | 13 | SELECT * FROM [procfwk].[Orchestrators] 14 | 15 | SELECT * FROM [procfwk].[Stages] 16 | 17 | SELECT * FROM [procfwk].[Pipelines] 18 | 19 | --SELECT * FROM [procfwk].[PipelineParameters] 20 | 21 | SELECT * FROM [procfwk].[ExecutionLog] 22 | 23 | SELECT * FROM [procfwk].[ErrorLog] 24 | 25 | --SELECT * FROM [dbo].[ServicePrincipals] -------------------------------------------------------------------------------- /DeploymentTools/DataFactory/ProcFwkComponents.json: -------------------------------------------------------------------------------- 1 | { 2 | "linkedServices": [ 3 | "/linkedService/Keys.json", 4 | "/linkedService/FrameworkFunctions.json", 5 | "/linkedService/SupportDatabase.json" 6 | ], 7 | "datasets": [ 8 | "/dataset/GetSetMetadata.json" 9 | ], 10 | "pipelines": [ 11 | "/pipeline/Throw Exception.json", 12 | "/pipeline/Check For Running Pipeline.json", 13 | "/pipeline/Email Sender.json", 14 | "/pipeline/04-Infant.json", 15 | "/pipeline/03-Child.json", 16 | "/pipeline/02-Parent.json", 17 | "/pipeline/01-Grandparent.json" 18 | ], 19 | "triggers": [ 20 | "/trigger/FunctionalTestingTrigger.json" 21 | ] 22 | } 23 | -------------------------------------------------------------------------------- /MetadataDB/MetadataDB.refactorlog: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelineDependants.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelineDependants] 2 | AS 3 | BEGIN 4 | EXEC [procfwkHelpers].[AddPipelineDependant] 5 | @PipelineName = 'Intentional Error', 6 | @DependantPipelineName = 'Wait 5'; 7 | 8 | EXEC [procfwkHelpers].[AddPipelineDependant] 9 | @PipelineName = 'Intentional Error', 10 | @DependantPipelineName = 'Wait 6'; 11 | 12 | EXEC [procfwkHelpers].[AddPipelineDependant] 13 | @PipelineName = 'Wait 6', 14 | @DependantPipelineName = 'Wait 9'; 15 | 16 | EXEC [procfwkHelpers].[AddPipelineDependant] 17 | @PipelineName = 'Wait 9', 18 | @DependantPipelineName = 'Wait 10'; 19 | END; -------------------------------------------------------------------------------- /DataFactory/trigger/FunctionalTestingTrigger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FunctionalTestingTrigger", 3 | "properties": { 4 | "description": "Used for functional testing of the framework in a dedicated environment.", 5 | "annotations": [ 6 | "procfwk" 7 | ], 8 | "runtimeState": "Stopped", 9 | "pipelines": [ 10 | { 11 | "pipelineReference": { 12 | "referenceName": "01-Grandparent", 13 | "type": "PipelineReference" 14 | } 15 | } 16 | ], 17 | "type": "ScheduleTrigger", 18 | "typeProperties": { 19 | "recurrence": { 20 | "frequency": "Hour", 21 | "interval": 2, 22 | "startTime": "2020-04-06T15:00:00Z", 23 | "timeZone": "UTC" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Synapse/trigger/FunctionalTestingTrigger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FunctionalTestingTrigger", 3 | "properties": { 4 | "description": "Used for functional testing of the framework in a dedicated environment.", 5 | "annotations": [ 6 | "procfwk" 7 | ], 8 | "runtimeState": "Stopped", 9 | "pipelines": [ 10 | { 11 | "pipelineReference": { 12 | "referenceName": "01-Grandparent", 13 | "type": "PipelineReference" 14 | } 15 | } 16 | ], 17 | "type": "ScheduleTrigger", 18 | "typeProperties": { 19 | "recurrence": { 20 | "frequency": "Hour", 21 | "interval": 2, 22 | "startTime": "2020-04-06T15:00:00Z", 23 | "timeZone": "UTC" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Synapse/linkedService/FrameworkFunctions.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FrameworkFunctions", 3 | "properties": { 4 | "annotations": [ 5 | "procfwk" 6 | ], 7 | "type": "AzureFunction", 8 | "typeProperties": { 9 | "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", 10 | "functionKey": { 11 | "type": "AzureKeyVaultSecret", 12 | "store": { 13 | "referenceName": "Keys", 14 | "type": "LinkedServiceReference" 15 | }, 16 | "secretName": "FrameworkFunctionsKey" 17 | } 18 | }, 19 | "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level." 20 | } 21 | } -------------------------------------------------------------------------------- /DataFactory/linkedService/FrameworkFunctions.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FrameworkFunctions", 3 | "properties": { 4 | "annotations": [ 5 | "procfwk" 6 | ], 7 | "type": "AzureFunction", 8 | "typeProperties": { 9 | "functionAppUrl": "https://frameworksupportfunctions.azurewebsites.net", 10 | "functionKey": { 11 | "type": "AzureKeyVaultSecret", 12 | "store": { 13 | "referenceName": "Keys", 14 | "type": "LinkedServiceReference" 15 | }, 16 | "secretName": "FrameworkFunctionsKey" 17 | } 18 | }, 19 | "description": "Interact with the Azure Functions App used as middle ware when making requests to Worker pipelines. Authentication done at the Function App level." 20 | } 21 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Orchestrators.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Orchestrators] 2 | ( 3 | [OrchestratorId] [int] IDENTITY(1,1) NOT NULL, 4 | [OrchestratorName] NVARCHAR(200) NOT NULL, 5 | [OrchestratorType] CHAR(3) NOT NULL, 6 | [IsFrameworkOrchestrator] BIT NOT NULL DEFAULT(0), 7 | [ResourceGroupName] NVARCHAR(200) NOT NULL, 8 | [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, 9 | [Description] NVARCHAR(MAX) NULL, 10 | CONSTRAINT [OrchestratorType] CHECK ([OrchestratorType] IN ('ADF','SYN')), 11 | CONSTRAINT [FK_Orchestrators_Subscriptions] FOREIGN KEY([SubscriptionId]) REFERENCES [procfwk].[Subscriptions] ([SubscriptionId]), 12 | CONSTRAINT [PK_Orchestrators] PRIMARY KEY CLUSTERED ([OrchestratorId] ASC) 13 | ) -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/PipelineAuthLink.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[PipelineAuthLink] 2 | ( 3 | [AuthId] [int] IDENTITY(1,1) NOT NULL, 4 | [PipelineId] [int] NOT NULL, 5 | [OrchestratorId] [int] NOT NULL, 6 | [CredentialId] [int] NOT NULL, 7 | CONSTRAINT [PK_PipelineAuthLink] PRIMARY KEY CLUSTERED ([AuthId] ASC), 8 | CONSTRAINT [FK_PipelineAuthLink_Orchestrators] FOREIGN KEY([OrchestratorId]) REFERENCES [procfwk].[Orchestrators] ([OrchestratorId]), 9 | CONSTRAINT [FK_PipelineAuthLink_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), 10 | CONSTRAINT [FK_PipelineAuthLink_ServicePrincipals] FOREIGN KEY([CredentialId]) REFERENCES [dbo].[ServicePrincipals] ([CredentialId]) 11 | ); -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultBatchStageLink.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultBatchStageLink] 2 | AS 3 | BEGIN 4 | TRUNCATE TABLE [procfwk].[BatchStageLink] 5 | 6 | INSERT INTO [procfwk].[BatchStageLink] 7 | ( 8 | [BatchId], 9 | [StageId] 10 | ) 11 | SELECT 12 | b.[BatchId], 13 | s.[StageId] 14 | FROM 15 | [procfwk].[Batches] b 16 | INNER JOIN [procfwk].[Stages] s 17 | ON s.[StageName] <> 'Speed' 18 | WHERE 19 | b.[BatchName] = 'Daily' 20 | 21 | UNION ALL 22 | 23 | SELECT 24 | b.[BatchId], 25 | s.[StageId] 26 | FROM 27 | [procfwk].[Batches] b 28 | INNER JOIN [procfwk].[Stages] s 29 | ON s.[StageName] = 'Speed' 30 | WHERE 31 | b.[BatchName] = 'Hourly' 32 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/ExecutePrecursorProcedure.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[ExecutePrecursorProcedure] 2 | AS 3 | BEGIN 4 | DECLARE @SQL VARCHAR(MAX) 5 | DECLARE @ErrorDetail NVARCHAR(MAX) 6 | 7 | IF OBJECT_ID([procfwk].[GetPropertyValueInternal]('ExecutionPrecursorProc')) IS NOT NULL 8 | BEGIN 9 | BEGIN TRY 10 | SET @SQL = [procfwk].[GetPropertyValueInternal]('ExecutionPrecursorProc'); 11 | EXEC(@SQL); 12 | END TRY 13 | BEGIN CATCH 14 | SELECT 15 | @ErrorDetail = 'Precursor procedure failed with error: ' + ERROR_MESSAGE(); 16 | 17 | RAISERROR(@ErrorDetail,16,1); 18 | END CATCH 19 | END; 20 | ELSE 21 | BEGIN 22 | PRINT 'Precursor object not found in database.'; 23 | END; 24 | END; -------------------------------------------------------------------------------- /Functions/Helpers/InvalidRequestException.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Runtime.Serialization; 3 | 4 | namespace mrpaulandrew.azure.procfwk.Helpers 5 | { 6 | [Serializable] 7 | internal class InvalidRequestException : Exception 8 | { 9 | public InvalidRequestException() 10 | { 11 | } 12 | 13 | public InvalidRequestException(string message) : base(message) 14 | { 15 | } 16 | 17 | public InvalidRequestException(string message, Exception innerException) : base(message, innerException) 18 | { 19 | } 20 | 21 | protected InvalidRequestException(SerializationInfo info, StreamingContext context) : base(info, context) 22 | { 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/ReplaceDataFactorys.sql: -------------------------------------------------------------------------------- 1 | IF EXISTS 2 | ( 3 | SELECT 4 | * 5 | FROM 6 | sys.objects o 7 | INNER JOIN sys.schemas s 8 | ON o.[schema_id] = s.[schema_id] 9 | WHERE 10 | o.[name] = 'DataFactorys' 11 | AND s.[name] = 'procfwk' 12 | AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability 13 | ) 14 | BEGIN 15 | DROP TABLE [procfwk].[DataFactorys]; 16 | 17 | EXEC('CREATE VIEW [procfwk].[DataFactorys] 18 | AS 19 | SELECT 20 | [OrchestratorId] AS DataFactoryId, 21 | [OrchestratorName] AS DataFactoryName, 22 | [ResourceGroupName], 23 | [SubscriptionId], 24 | [Description] 25 | FROM 26 | [procfwk].[Orchestrators] 27 | WHERE 28 | [OrchestratorType] = ''ADF'';') 29 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/LastExecutionSummary.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[LastExecutionSummary] 2 | AS 3 | 4 | WITH maxLog AS 5 | ( 6 | SELECT 7 | MAX([LogId]) AS 'MaxLogId' 8 | FROM 9 | [procfwk].[ExecutionLog] 10 | ), 11 | lastExecutionId AS 12 | ( 13 | SELECT 14 | [LocalExecutionId] 15 | FROM 16 | [procfwk].[ExecutionLog] el1 17 | INNER JOIN maxLog 18 | ON maxLog.[MaxLogId] = el1.[LogId] 19 | ) 20 | SELECT 21 | el2.[LocalExecutionId], 22 | DATEDIFF(MINUTE, MIN(el2.[StartDateTime]), MAX(el2.[EndDateTime])) 'RunDurationMinutes' 23 | FROM 24 | [procfwk].[ExecutionLog] el2 25 | INNER JOIN lastExecutionId 26 | ON el2.[LocalExecutionId] = lastExecutionId.[LocalExecutionId] 27 | GROUP BY 28 | el2.[LocalExecutionId] -------------------------------------------------------------------------------- /DeploymentTools/DataFactory/GlobalVars.ps1: -------------------------------------------------------------------------------- 1 | # Set global variables as required: 2 | $resourceGroupName = "ADF.procfwk" 3 | $dataFactoryName = "FrameworkFactory" 4 | $region = "uksouth" 5 | 6 | # ADF deployment from PS script 7 | .\DeploymentTools\DataFactory\DeployProcFwkComponents.ps1 ` 8 | -resourceGroupName "$resourceGroupName" ` 9 | -dataFactoryName "$dataFactoryName" ` 10 | -region "$region" 11 | 12 | 13 | .\DeploymentTools\DataFactory\PopulatePipelinesInDb.ps1 ` 14 | -SqlServerName '*****.database.windows.net' ` 15 | -SqlDatabaseName 'adfprocfwk' ` 16 | -SqlUser 'adm' ` 17 | -SqlPass '******' ` 18 | -resourceGroupName "$resourceGroupName" ` 19 | -dataFactoryName "$dataFactoryName" ` 20 | -region "$region" 21 | 22 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/PipelineDependencies.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[PipelineDependencies] 2 | ( 3 | [DependencyId] [INT] IDENTITY(1,1) NOT NULL, 4 | [PipelineId] [INT] NOT NULL, 5 | [DependantPipelineId] [INT] NOT NULL, 6 | CONSTRAINT [PK_PipelineDependencies] PRIMARY KEY CLUSTERED ([DependencyId] ASC), 7 | CONSTRAINT [FK_PipelineDependencies_Pipelines] FOREIGN KEY([PipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), 8 | CONSTRAINT [FK_PipelineDependencies_Pipelines1] FOREIGN KEY([DependantPipelineId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]), 9 | CONSTRAINT [UK_PipelinesToDependantPipelines] UNIQUE ([PipelineId],[DependantPipelineId]), 10 | CONSTRAINT [EQ_PipelineIdDependantPipelineId] CHECK ([PipelineId] <> [DependantPipelineId]) 11 | ) -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValue/test WHEN property name is null THEN error raised.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property name is null THEN error raised] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = NULL 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.Properties' 8 | 9 | INSERT INTO procfwk.Properties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | ) VALUES ( 13 | 'TestProperty' 14 | , 'TestPropertyValue' 15 | ) 16 | 17 | -- EXPECT 18 | EXEC tSQLt.ExpectException @ExpectedMessagePattern = '%Invalid property name %' 19 | 20 | -- ACT 21 | CREATE TABLE #actual ( 22 | PropertyValue NVARCHAR(4000) 23 | ) 24 | 25 | INSERT INTO #actual ( 26 | PropertyValue 27 | ) 28 | EXEC procfwk.GetPropertyValue @propertyName 29 | -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property name is null THEN empty string returned.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property name is null THEN empty string returned] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = NULL 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' 8 | 9 | INSERT INTO procfwk.CurrentProperties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | ) VALUES ( 13 | 'TestProperty' 14 | , 'TestPropertyValue' 15 | ) 16 | 17 | DECLARE @expected NVARCHAR(4000) = '' 18 | 19 | -- ACT 20 | DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) 21 | 22 | -- ASSERT 23 | EXEC tSQLt.AssertEquals 24 | @Expected = @expected 25 | , @Actual = @actual 26 | -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/AverageStageDuration.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[AverageStageDuration] 2 | AS 3 | 4 | WITH stageStartEnd AS 5 | ( 6 | SELECT 7 | [LocalExecutionId], 8 | [StageId], 9 | MIN([StartDateTime]) AS 'StageStart', 10 | MAX([EndDateTime]) AS 'StageEnd' 11 | FROM 12 | [procfwk].[ExecutionLog] 13 | GROUP BY 14 | [LocalExecutionId], 15 | [StageId] 16 | ) 17 | 18 | SELECT 19 | s.[StageId], 20 | s.[StageName], 21 | s.[StageDescription], 22 | AVG(DATEDIFF(MINUTE, stageStartEnd.[StageStart], stageStartEnd.[StageEnd])) 'AvgStageRunDurationMinutes' 23 | FROM 24 | stageStartEnd 25 | INNER JOIN [procfwk].[Stages] s 26 | ON stageStartEnd.[StageId] = s.[StageId] 27 | GROUP BY 28 | s.[StageId], 29 | s.[StageName], 30 | s.[StageDescription] -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValue/test WHEN property does not exist THEN error raised.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property does not exist THEN error raised] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = 'OtherProperty' 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.Properties' 8 | 9 | INSERT INTO procfwk.Properties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | ) VALUES ( 13 | 'TestProperty' 14 | , 'TestPropertyValue' 15 | ) 16 | 17 | -- EXPECT 18 | EXEC tSQLt.ExpectException @ExpectedMessagePattern = '%Invalid property name %' 19 | 20 | -- ACT 21 | CREATE TABLE #actual ( 22 | PropertyValue NVARCHAR(4000) 23 | ) 24 | 25 | INSERT INTO #actual ( 26 | PropertyValue 27 | ) 28 | EXEC procfwk.GetPropertyValue @propertyName 29 | -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValueInternal/test WHEN property exists THEN property value returned.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValueInternal.[test WHEN property exists THEN property value returned] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = 'TestProperty' 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.CurrentProperties' 8 | 9 | INSERT INTO procfwk.CurrentProperties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | ) VALUES ( 13 | 'TestProperty' 14 | , 'TestPropertyValue' 15 | ) 16 | 17 | DECLARE @expected NVARCHAR(4000) = 'TestPropertyValue' 18 | 19 | -- ACT 20 | DECLARE @actual NVARCHAR(4000) = procfwk.GetPropertyValueInternal(@propertyName) 21 | 22 | -- ASSERT 23 | EXEC tSQLt.AssertEquals 24 | @Expected = @expected 25 | , @Actual = @actual 26 | -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 1", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait1", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 2", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait2", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 3", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait3", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 4", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait4", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 5", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait5", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 6", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait6", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 7.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 7", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait7", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 8.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 8", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait8", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 1", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait1", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 2", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait2", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 3", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait3", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 4", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait4", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 5", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait5", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 6", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait6", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 7.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 7", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait7", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 8.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 8", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait8", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 9.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 9", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait9", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 15 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 10", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait10", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /Synapse/pipeline/Wait 9.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 9", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait9", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 15 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Wait 10.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Wait 10", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait10", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | } 19 | ], 20 | "parameters": { 21 | "WaitTime": { 22 | "type": "int", 23 | "defaultValue": 5 24 | } 25 | }, 26 | "folder": { 27 | "name": "_Workers" 28 | }, 29 | "annotations": [ 30 | "_ProcFwkWorker" 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetStages.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetStages] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER 4 | ) 5 | AS 6 | BEGIN 7 | SET NOCOUNT ON; 8 | 9 | --defensive check 10 | IF NOT EXISTS 11 | ( 12 | SELECT 13 | 1 14 | FROM 15 | [procfwk].[CurrentExecution] 16 | WHERE 17 | [LocalExecutionId] = @ExecutionId 18 | AND ISNULL([PipelineStatus],'') <> 'Success' 19 | ) 20 | BEGIN 21 | RAISERROR('Requested execution run does not contain any enabled stages/pipelines.',16,1); 22 | RETURN 0; 23 | END; 24 | 25 | SELECT DISTINCT 26 | [StageId] 27 | FROM 28 | [procfwk].[CurrentExecution] 29 | WHERE 30 | [LocalExecutionId] = @ExecutionId 31 | AND ISNULL([PipelineStatus],'') <> 'Success' 32 | ORDER BY 33 | [StageId] ASC 34 | END; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/Utilities/UtilitiesHelper.cs: -------------------------------------------------------------------------------- 1 | using FactoryTesting.Helpers; 2 | using System; 3 | using System.Data.SqlClient; 4 | using System.Threading; 5 | using System.Threading.Tasks; 6 | using System.Xml; 7 | 8 | namespace FactoryTesting.Pipelines.Utilities 9 | { 10 | class UtilitiesHelper : CoverageHelper 11 | { 12 | public UtilitiesHelper WithBasicMetadata() 13 | { 14 | AddBasicMetadata(); 15 | return this; 16 | } 17 | 18 | public UtilitiesHelper WithTenantAndSubscriptionIds() 19 | { 20 | AddTenantAndSubscription(); 21 | return this; 22 | } 23 | 24 | public override void TearDown() 25 | { 26 | base.TearDown(); 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Views/PipelineDependencyChains.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkHelpers].[PipelineDependencyChains] 2 | AS 3 | 4 | SELECT 5 | ps.[StageName] AS PredecessorStage, 6 | pp.[PipelineName] AS PredecessorPipeline, 7 | ds.[StageName] AS DependantStage, 8 | dp.[PipelineName] AS DependantPipeline 9 | FROM 10 | [procfwk].[PipelineDependencies] pd --pipeline dependencies 11 | INNER JOIN [procfwk].[Pipelines] pp --predecessor pipelines 12 | ON pd.[PipelineId] = pp.[PipelineId] 13 | INNER JOIN [procfwk].[Pipelines] dp --dependant pipelines 14 | ON pd.[DependantPipelineId] = dp.[PipelineId] 15 | INNER JOIN [procfwk].[Stages] ps --predecessor stage 16 | ON pp.[StageId] = ps.[StageId] 17 | INNER JOIN [procfwk].[Stages] ds --dependant stage 18 | ON dp.[StageId] = ds.[StageId]; -------------------------------------------------------------------------------- /MetadataDB/procfwkTesting/Stored Procedures/ResetMetadata.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkTesting].[ResetMetadata] 2 | AS 3 | BEGIN 4 | EXEC [procfwkHelpers].[SetDefaultProperties]; 5 | EXEC [procfwkHelpers].[SetDefaultTenant]; 6 | EXEC [procfwkHelpers].[SetDefaultSubscription]; 7 | EXEC [procfwkHelpers].[SetDefaultOrchestrators]; 8 | EXEC [procfwkHelpers].[SetDefaultBatches]; 9 | EXEC [procfwkHelpers].[SetDefaultStages]; 10 | EXEC [procfwkHelpers].[SetDefaultBatchStageLink]; 11 | EXEC [procfwkHelpers].[SetDefaultPipelines]; 12 | EXEC [procfwkHelpers].[SetDefaultPipelineParameters]; 13 | EXEC [procfwkHelpers].[SetDefaultPipelineDependants]; 14 | EXEC [procfwkHelpers].[SetDefaultRecipients]; 15 | EXEC [procfwkHelpers].[SetDefaultAlertOutcomes]; 16 | EXEC [procfwkHelpers].[SetDefaultRecipientPipelineAlerts]; 17 | END; -------------------------------------------------------------------------------- /MetadataDBTests/Scripts/Script.PostDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Post-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be appended to the build script. 5 | Use SQLCMD syntax to include a file in the post-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the post-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | 13 | :r ..\..\MetadataDB\Scripts\Script.PostDeployment.sql -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValue/test WHEN property invalidated THEN error raised.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property invalidated THEN error raised] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = 'TestProperty' 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.Properties' 8 | 9 | INSERT INTO procfwk.Properties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | , ValidTo 13 | ) VALUES ( 14 | 'TestProperty' 15 | , 'TestPropertyValue' 16 | , GETDATE() 17 | ) 18 | 19 | -- EXPECT 20 | EXEC tSQLt.ExpectException @ExpectedMessagePattern = '% does not have a current valid version %' 21 | 22 | -- ACT 23 | CREATE TABLE #actual ( 24 | PropertyValue NVARCHAR(4000) 25 | ) 26 | 27 | INSERT INTO #actual ( 28 | PropertyValue 29 | ) 30 | EXEC procfwk.GetPropertyValue @propertyName 31 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/TransferReportingObjects.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID(N'tempdb..#TransferReportingObjects') IS NOT NULL DROP PROCEDURE #TransferReportingObjects; 2 | GO 3 | 4 | CREATE PROCEDURE #TransferReportingObjects 5 | ( 6 | @ObjectName NVARCHAR(128), 7 | @ObjectType CHAR(2) 8 | ) 9 | AS 10 | BEGIN 11 | IF EXISTS 12 | ( 13 | SELECT 14 | * 15 | FROM 16 | sys.objects o 17 | INNER JOIN sys.schemas s 18 | ON o.[schema_id] = s.[schema_id] 19 | WHERE 20 | o.[Name] = @ObjectName 21 | AND s.[name] = 'procfwk' 22 | AND o.[type] = @ObjectType 23 | ) 24 | BEGIN 25 | PRINT 'Transferring: ' + @ObjectName; 26 | EXEC('ALTER SCHEMA [procfwkHelpers] TRANSFER [procfwk].[' + @ObjectName + '];') 27 | END; 28 | END; 29 | GO 30 | 31 | EXEC #TransferReportingObjects 'PipelineDependencyChains', 'V'; 32 | -------------------------------------------------------------------------------- /MetadataDBTests/Scripts/Script.PreDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Pre-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be executed before the build script. 5 | Use SQLCMD syntax to include a file in the pre-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the pre-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | 13 | :r ..\..\MetadataDB\Scripts\Script.PreDeployment.sql 14 | 15 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/Pipelines.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[Pipelines] ( 2 | [PipelineId] INT IDENTITY (1, 1) NOT NULL, 3 | [OrchestratorId] INT NOT NULL, 4 | [StageId] INT NOT NULL, 5 | [PipelineName] NVARCHAR (200) NOT NULL, 6 | [LogicalPredecessorId] INT NULL, 7 | [Enabled] BIT CONSTRAINT [DF_Pipelines_Enabled] DEFAULT ((1)) NOT NULL, 8 | CONSTRAINT [PK_Pipelines] PRIMARY KEY CLUSTERED ([PipelineId] ASC), 9 | CONSTRAINT [FK_Pipelines_Stages] FOREIGN KEY ([StageId]) REFERENCES [procfwk].[Stages] ([StageId]), 10 | CONSTRAINT [FK_Pipelines_Orchestrators] FOREIGN KEY([OrchestratorId]) REFERENCES [procfwk].[Orchestrators] ([OrchestratorId]), 11 | CONSTRAINT [FK_Pipelines_Pipelines] FOREIGN KEY([LogicalPredecessorId]) REFERENCES [procfwk].[Pipelines] ([PipelineId]) 12 | ); 13 | 14 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/help---support-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Help & Support Request 3 | about: Tell me about the problem or error you are facing when using the procfwk 4 | title: '' 5 | labels: help wanted 6 | assignees: mrpaulandrew 7 | 8 | --- 9 | 10 | **Describe the error** 11 | A clear and concise description of what the bug is. 12 | 13 | **Error message** 14 | An output of the error message presented. 15 | 16 | **Affected services** 17 | Which resource within the processing framework does this affect? 18 | * Data Factory/Synapse 19 | * SQL Database 20 | * Functions 21 | * All of them 22 | * Other 23 | 24 | **To Reproduce** 25 | Steps to reproduce the behavior: 26 | 1. Go to '...' 27 | 2. Click on '....' 28 | 3. Scroll down to '....' 29 | 4. See error 30 | 31 | **Screenshots** 32 | If applicable, add screenshots to help explain your problem. 33 | -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/LastExecution.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[LastExecution] 2 | AS 3 | 4 | WITH maxLog AS 5 | ( 6 | SELECT 7 | MAX([LogId]) AS 'MaxLogId' 8 | FROM 9 | [procfwk].[ExecutionLog] 10 | ), 11 | lastExecutionId AS 12 | ( 13 | SELECT 14 | [LocalExecutionId] 15 | FROM 16 | [procfwk].[ExecutionLog] el1 17 | INNER JOIN maxLog 18 | ON maxLog.[MaxLogId] = el1.[LogId] 19 | ) 20 | SELECT 21 | el2.[LogId], 22 | el2.[StageId], 23 | el2.[PipelineId], 24 | el2.[PipelineName], 25 | el2.[StartDateTime], 26 | el2.[PipelineStatus], 27 | el2.[EndDateTime], 28 | DATEDIFF(MINUTE, el2.[StartDateTime], el2.[EndDateTime]) AS RunDurationMinutes 29 | FROM 30 | [procfwk].[ExecutionLog] el2 31 | INNER JOIN lastExecutionId 32 | ON el2.[LocalExecutionId] = lastExecutionId.[LocalExecutionId] 33 | WHERE 34 | el2.[EndDateTime] IS NOT NULL; -------------------------------------------------------------------------------- /MetadataDBTests/procfwk_GetPropertyValue/test WHEN property exists THEN property value returned.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE procfwk_GetPropertyValue.[test WHEN property exists THEN property value returned] 2 | AS 3 | 4 | -- ARRANGE 5 | DECLARE @propertyName NVARCHAR(128) = 'TestProperty' 6 | 7 | EXEC tSQLt.FakeTable 'procfwk.Properties' 8 | 9 | INSERT INTO procfwk.Properties ( 10 | [PropertyName] 11 | , [PropertyValue] 12 | ) VALUES ( 13 | 'TestProperty' 14 | , 'TestPropertyValue' 15 | ) 16 | 17 | SELECT N'TestPropertyValue' AS PropertyValue 18 | INTO #expected 19 | 20 | -- ACT 21 | CREATE TABLE #actual ( 22 | PropertyValue NVARCHAR(4000) 23 | ) 24 | 25 | INSERT INTO #actual ( 26 | PropertyValue 27 | ) 28 | EXEC procfwk.GetPropertyValue @propertyName 29 | 30 | -- ASSERT 31 | EXEC tSQLt.AssertEqualsTable 32 | @Expected = '#expected' 33 | , @Actual = '#actual' 34 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Script.PreDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Pre-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be executed before the build script. 5 | Use SQLCMD syntax to include a file in the pre-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the pre-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | 13 | --data 14 | :r .\LogData\ExecutionLogBackup.sql 15 | :r .\LogData\ErrorLogBackup.sql 16 | 17 | --delete all 18 | :r .\Metadata\DropLegacyTables.sql 19 | :r .\Metadata\DropLegacyObjects.sql 20 | :r .\Metadata\DeleteAll.sql -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-found.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Found 3 | about: Tell me about the bug you've found in the procfwk please 4 | title: '' 5 | labels: bug 6 | assignees: mrpaulandrew 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **Affected services** 14 | Which resource within the processing framework does this affect? 15 | * Data Factory/Synapse 16 | * SQL Database 17 | * Functions 18 | * All of them 19 | * Other 20 | 21 | **To Reproduce** 22 | Steps to reproduce the behavior: 23 | 1. Go to '...' 24 | 2. Click on '....' 25 | 3. Scroll down to '....' 26 | 4. See error 27 | 28 | **Expected behaviour** 29 | A clear and concise description of what you expected to happen. 30 | 31 | **Screenshots** 32 | If applicable, add screenshots to help explain your problem. 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/Utilities/GivenThrowException.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Utilities 7 | { 8 | public class GivenThrowException 9 | { 10 | private UtilitiesHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new UtilitiesHelper() 16 | .WithParameter("Message","NUnit Test"); 17 | 18 | await _helper.RunPipeline("Throw Exception"); 19 | } 20 | 21 | [Test] 22 | public void ThenPipelineOutcomeIsFailed() 23 | { 24 | _helper.RunOutcome.Should().Be("Failed"); 25 | } 26 | 27 | [OneTimeTearDown] 28 | public void TearDown() 29 | { 30 | _helper?.TearDown(); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Functions/Helpers/PipelineRunRequest.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using Microsoft.Extensions.Logging; 3 | 4 | namespace mrpaulandrew.azure.procfwk.Helpers 5 | { 6 | public class PipelineRunRequest : PipelineRequest 7 | { 8 | public string RunId { get; set; } 9 | 10 | public bool RecursivePipelineCancel = true; //might provide this as part of the request later 11 | 12 | public DateTime ActivityQueryStart = DateTime.Now.AddDays(-7); //max duration for eventual RunFilterParameters 13 | public DateTime ActivityQueryEnd = DateTime.Now; 14 | 15 | public override void Validate(ILogger logger) 16 | { 17 | base.Validate(logger); 18 | 19 | // ensure properties not null 20 | if (RunId == null) 21 | ReportInvalidBody(logger); 22 | 23 | //other validation 24 | if (!CheckGuid(RunId)) ReportInvalidBody(logger, "Expected Run Id to be a GUID."); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/CompleteExecutionErrorLog.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[CompleteExecutionErrorLog] 2 | AS 3 | 4 | SELECT 5 | exeLog.[LogId] AS ExecutionLogId, 6 | errLog.[LogId] AS ErrorLogId, 7 | exeLog.[LocalExecutionId], 8 | exeLog.[StartDateTime] AS ProcessingDateTime, 9 | exeLog.[CallingOrchestratorName], 10 | exeLog.[OrchestratorType] AS WorkerOrchestartorType, 11 | exeLog.[OrchestratorName] AS WorkerOrchestrator, 12 | exeLog.[PipelineName] AS WorkerPipelineName, 13 | exeLog.[PipelineStatus], 14 | errLog.[ActivityRunId], 15 | errLog.[ActivityName], 16 | errLog.[ActivityType], 17 | errLog.[ErrorCode], 18 | errLog.[ErrorType], 19 | errLog.[ErrorMessage] 20 | FROM 21 | [procfwk].[ExecutionLog] exeLog 22 | INNER JOIN [procfwk].[ErrorLog] errLog 23 | ON exeLog.[LocalExecutionId] = errLog.[LocalExecutionId] 24 | AND exeLog.[PipelineRunId] = errLog.[PipelineRunId] 25 | INNER JOIN [procfwk].[Stages] stgs 26 | ON exeLog.[StageId] = stgs.[StageId] 27 | ; -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/DeleteMetadataWithoutIntegrity.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[DeleteMetadataWithoutIntegrity] 2 | AS 3 | BEGIN 4 | 5 | DECLARE @SQL NVARCHAR(MAX) = '' 6 | 7 | ;WITH procfwkTables AS 8 | ( 9 | SELECT 10 | QUOTENAME(s.[name]) + '.' + QUOTENAME(o.[name]) AS FullName 11 | FROM 12 | sys.objects o 13 | INNER JOIN sys.schemas s 14 | ON o.[schema_id] = s.[schema_id] 15 | WHERE 16 | s.[name] LIKE 'procfwk%' 17 | AND o.[type] = 'U' 18 | 19 | UNION 20 | 21 | SELECT 22 | QUOTENAME(s.[name]) + '.' + QUOTENAME(o.[name]) AS FullName 23 | FROM 24 | sys.objects o 25 | INNER JOIN sys.schemas s 26 | ON o.[schema_id] = s.[schema_id] 27 | WHERE 28 | o.[name] = 'ServicePrincipals' 29 | AND o.[type] = 'U' 30 | ) 31 | 32 | SELECT --tables must exist or wouldnt appear in sys.objects query 33 | @SQL += 'DELETE FROM ' + [FullName] + ';' + CHAR(13) 34 | FROM 35 | procfwkTables 36 | 37 | EXEC(@SQL); 38 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/ExecutionLog.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[ExecutionLog] 2 | ( 3 | [LogId] INT IDENTITY (1, 1) NOT NULL, 4 | [LocalExecutionId] UNIQUEIDENTIFIER NOT NULL, 5 | [StageId] INT NOT NULL, 6 | [PipelineId] INT NOT NULL, 7 | [CallingOrchestratorName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), 8 | [ResourceGroupName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), 9 | [OrchestratorType] CHAR(3) NOT NULL DEFAULT('N/A'), 10 | [OrchestratorName] NVARCHAR(200) NOT NULL DEFAULT ('Unknown'), 11 | [PipelineName] NVARCHAR (200) NOT NULL, 12 | [StartDateTime] DATETIME NULL, 13 | [PipelineStatus] NVARCHAR (200) NULL, 14 | [EndDateTime] DATETIME NULL, 15 | [PipelineRunId] UNIQUEIDENTIFIER NULL, 16 | [PipelineParamsUsed] NVARCHAR(MAX) NULL DEFAULT ('None'), 17 | CONSTRAINT [PK_ExecutionLog] PRIMARY KEY CLUSTERED ([LogId] ASC) 18 | ); 19 | 20 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetFrameworkOrchestratorDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetFrameworkOrchestratorDetails] 2 | ( 3 | @CallingOrchestratorName NVARCHAR(200) 4 | ) 5 | AS 6 | BEGIN 7 | SET NOCOUNT ON; 8 | 9 | DECLARE @FrameworkOrchestrator NVARCHAR(200) 10 | 11 | --defensive check 12 | SELECT 13 | @FrameworkOrchestrator = UPPER([OrchestratorName]), 14 | @CallingOrchestratorName = UPPER(@CallingOrchestratorName) 15 | FROM 16 | [procfwk].[Orchestrators] 17 | WHERE 18 | [IsFrameworkOrchestrator] = 1; 19 | 20 | IF(@FrameworkOrchestrator <> @CallingOrchestratorName) 21 | BEGIN 22 | RAISERROR('Orchestrator mismatch. Calling orchestrator does not match expected IsFrameworkOrchestrator name.',16,1); 23 | RETURN 0; 24 | END 25 | 26 | --orchestrator detials 27 | SELECT 28 | [SubscriptionId], 29 | [ResourceGroupName], 30 | [OrchestratorName], 31 | [OrchestratorType] 32 | FROM 33 | [procfwk].[Orchestrators] 34 | WHERE 35 | [IsFrameworkOrchestrator] = 1; 36 | END; -------------------------------------------------------------------------------- /DeploymentTools/DataFactory/Get Pipelines.ps1: -------------------------------------------------------------------------------- 1 | # Set global variables as required: 2 | $resourceGroupName = "ADF.procfwk" 3 | $dataFactoryName = "FrameworkFactory" 4 | $region = "uksouth" 5 | 6 | #SPN for deploying ADF: 7 | $tenantId = [System.Environment]::GetEnvironmentVariable('AZURE_TENANT_ID') 8 | $subscriptionId = [System.Environment]::GetEnvironmentVariable('AZURE_SUBSCRIPTION_ID') 9 | $spId = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_ID') 10 | $spKey = [System.Environment]::GetEnvironmentVariable('AZURE_CLIENT_SECRET') 11 | 12 | #Modules 13 | Import-Module -Name "Az" 14 | Import-Module -Name "Az.DataFactory" 15 | 16 | # Login as a Service Principal 17 | $passwd = ConvertTo-SecureString $spKey -AsPlainText -Force 18 | $pscredential = New-Object System.Management.Automation.PSCredential($spId, $passwd) 19 | Connect-AzAccount -ServicePrincipal -Credential $pscredential -TenantId $tenantId | Out-Null 20 | 21 | Get-AzDataFactoryV2Pipeline -DataFactoryName $dataFactoryName -ResourceGroupName $resourceGroupName 22 | 23 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultTenant.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultTenant] 2 | AS 3 | BEGIN 4 | DECLARE @Tenants TABLE 5 | ( 6 | [TenantId] UNIQUEIDENTIFIER NOT NULL, 7 | [Name] NVARCHAR(200) NOT NULL, 8 | [Description] NVARCHAR(MAX) NULL 9 | ) 10 | 11 | INSERT INTO @Tenants 12 | ( 13 | [TenantId], 14 | [Name], 15 | [Description] 16 | ) 17 | VALUES 18 | ('12345678-1234-1234-1234-012345678910', 'Default', 'Example value for development environment.'); 19 | 20 | MERGE INTO [procfwk].[Tenants] AS tgt 21 | USING 22 | @Tenants AS src 23 | ON tgt.[TenantId] = src.[TenantId] 24 | WHEN MATCHED THEN 25 | UPDATE 26 | SET 27 | tgt.[Name] = src.[Name], 28 | tgt.[Description] = src.[Description] 29 | WHEN NOT MATCHED BY TARGET THEN 30 | INSERT 31 | ( 32 | [TenantId], 33 | [Name], 34 | [Description] 35 | ) 36 | VALUES 37 | ( 38 | src.[TenantId], 39 | src.[Name], 40 | src.[Description] 41 | ) 42 | WHEN NOT MATCHED BY SOURCE THEN 43 | DELETE; 44 | END; -------------------------------------------------------------------------------- /Functions/Services/Returns/PipelineErrorDetail.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace mrpaulandrew.azure.procfwk.Services 4 | { 5 | public class PipelineErrorDetail : PipelineRunStatus 6 | { 7 | public PipelineErrorDetail() 8 | { 9 | Errors = new List(); 10 | } 11 | public int ResponseCount { get; set; } 12 | public int ResponseErrorCount 13 | { 14 | get 15 | { 16 | return Errors.Count; 17 | } 18 | } 19 | public List Errors { get; } 20 | } 21 | 22 | public class FailedActivity 23 | { 24 | public string ActivityRunId { get; internal set; } 25 | public string ActivityName { get; internal set; } 26 | public string ActivityType { get; internal set; } 27 | public string ErrorCode { get; internal set; } 28 | public string ErrorType { get; internal set; } 29 | public string ErrorMessage { get; internal set; } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetPropertyValue.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetPropertyValue] 2 | ( 3 | @PropertyName VARCHAR(128) 4 | ) 5 | AS 6 | BEGIN 7 | DECLARE @ErrorDetail NVARCHAR(4000) = '' 8 | 9 | --defensive checks 10 | IF NOT EXISTS 11 | ( 12 | SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName 13 | ) 14 | BEGIN 15 | SET @ErrorDetail = 'Invalid property name provided. Property does not exist.' 16 | RAISERROR(@ErrorDetail, 16, 1); 17 | RETURN 0; 18 | END 19 | ELSE IF NOT EXISTS 20 | ( 21 | SELECT * FROM [procfwk].[Properties] WHERE [PropertyName] = @PropertyName AND [ValidTo] IS NULL 22 | ) 23 | BEGIN 24 | SET @ErrorDetail = 'Property name provided does not have a current valid version of the required value.' 25 | RAISERROR(@ErrorDetail, 16, 1); 26 | RETURN 0; 27 | END 28 | --get valid property value 29 | ELSE 30 | BEGIN 31 | SELECT 32 | [PropertyValue] 33 | FROM 34 | [procfwk].[CurrentProperties] 35 | WHERE 36 | [PropertyName] = @PropertyName 37 | END 38 | END; -------------------------------------------------------------------------------- /Synapse/pipeline/01-Grandparent.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01-Grandparent", 3 | "properties": { 4 | "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", 5 | "activities": [ 6 | { 7 | "name": "procfwk", 8 | "description": "Call procfwk", 9 | "type": "ExecutePipeline", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "pipeline": { 14 | "referenceName": "02-Parent", 15 | "type": "PipelineReference" 16 | }, 17 | "waitOnCompletion": true, 18 | "parameters": { 19 | "BatchName": { 20 | "value": "@pipeline().parameters.BatchName", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } 26 | ], 27 | "parameters": { 28 | "BatchName": { 29 | "type": "string", 30 | "defaultValue": "NotUsed" 31 | } 32 | }, 33 | "folder": { 34 | "name": "_ProcFwk" 35 | }, 36 | "annotations": [ 37 | "procfwk", 38 | "Grandparent" 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/01-Grandparent.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "01-Grandparent", 3 | "properties": { 4 | "description": "procfwk grandparent pipeline used optionally to bootstrap any wider processes in your Data Factory that then calls the processing framework.", 5 | "activities": [ 6 | { 7 | "name": "procfwk", 8 | "description": "Call procfwk", 9 | "type": "ExecutePipeline", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "pipeline": { 14 | "referenceName": "02-Parent", 15 | "type": "PipelineReference" 16 | }, 17 | "waitOnCompletion": true, 18 | "parameters": { 19 | "BatchName": { 20 | "value": "@pipeline().parameters.BatchName", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } 26 | ], 27 | "parameters": { 28 | "BatchName": { 29 | "type": "string", 30 | "defaultValue": "NotUsed" 31 | } 32 | }, 33 | "folder": { 34 | "name": "_ProcFwk" 35 | }, 36 | "annotations": [ 37 | "procfwk", 38 | "Grandparent" 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultBatches.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultBatches] 2 | AS 3 | BEGIN 4 | DECLARE @Batches TABLE 5 | ( 6 | [BatchName] [VARCHAR](225) NOT NULL, 7 | [BatchDescription] [VARCHAR](4000) NULL, 8 | [Enabled] [BIT] NOT NULL 9 | ) 10 | 11 | INSERT @Batches 12 | ( 13 | [BatchName], 14 | [BatchDescription], 15 | [Enabled] 16 | ) 17 | VALUES 18 | ('Daily', N'Daily Worker Pipelines.', 1), 19 | ('Hourly', N'Hourly Worker Pipelines.', 1); 20 | 21 | MERGE INTO [procfwk].[Batches] AS tgt 22 | USING 23 | @Batches AS src 24 | ON tgt.[BatchName] = src.[BatchName] 25 | WHEN MATCHED THEN 26 | UPDATE 27 | SET 28 | tgt.[BatchDescription] = src.[BatchDescription], 29 | tgt.[Enabled] = src.[Enabled] 30 | WHEN NOT MATCHED BY TARGET THEN 31 | INSERT 32 | ( 33 | [BatchName], 34 | [BatchDescription], 35 | [Enabled] 36 | ) 37 | VALUES 38 | ( 39 | src.[BatchName], 40 | src.[BatchDescription], 41 | src.[Enabled] 42 | ) 43 | WHEN NOT MATCHED BY SOURCE THEN 44 | DELETE; 45 | END; -------------------------------------------------------------------------------- /Functions/Services/Returns/PipelineRunStatus.cs: -------------------------------------------------------------------------------- 1 | using System.Collections.Generic; 2 | 3 | namespace mrpaulandrew.azure.procfwk.Services 4 | { 5 | public class PipelineRunStatus 6 | { 7 | private const string Running = "Running"; 8 | private const string Complete = "Complete"; 9 | 10 | public string PipelineName { get; set; } 11 | public string RunId { get; set; } 12 | public string ActualStatus { get; set; } 13 | 14 | public string SimpleStatus 15 | { 16 | get 17 | { 18 | return ConvertPipelineStatus(ActualStatus); 19 | } 20 | } 21 | 22 | private string ConvertPipelineStatus(string actualStatus) 23 | { 24 | string simpleStatus = actualStatus switch 25 | { 26 | "Queued" => Running, 27 | "InProgress" => Running, 28 | "Canceling" => Running, //microsoft typo 29 | "Cancelling" => Running, 30 | _ => Complete, 31 | }; 32 | return simpleStatus; 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /DeploymentTools/DataFactory/PopulatePipelinesInDb.ps1: -------------------------------------------------------------------------------- 1 | Param( 2 | [Parameter(Mandatory)] 3 | [string]$SqlServerName, 4 | [Parameter(Mandatory)] 5 | [string]$SqlDatabaseName, 6 | [Parameter(Mandatory)] 7 | [string]$SqlUser, 8 | [Parameter(Mandatory)] 9 | [string]$SqlPass, 10 | [Parameter(Mandatory)] 11 | [string]$resourceGroupName, 12 | [Parameter(Mandatory)] 13 | [string]$dataFactoryName, 14 | [Parameter(Mandatory)] 15 | [string]$region 16 | ) 17 | 18 | $ErrorActionPreference = 'Stop' 19 | 20 | #Install-Module -Name SqlServer -AllowClobber 21 | Import-Module -Name SqlServer 22 | 23 | $pipelines = Get-AzDataFactoryV2Pipeline -DataFactoryName $dataFactoryName -ResourceGroupName $resourceGroupName 24 | 25 | foreach ($p in $pipelines) { 26 | Write-Host $p.name 27 | $query = "EXEC [procfwkHelpers].[AddPipelineViaPowerShell] '$resourceGroupName', '$dataFactoryName', '$($p.Name)';" 28 | Invoke-Sqlcmd -ServerInstance "$SqlServerName" -Database "$SqlDatabaseName" -Query "$query" -Username "$SqlUser" -Password "$SqlPass" 29 | } 30 | 31 | Write-Host "List of ADF pipelines has been populated into database." 32 | 33 | 34 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenDisabledPipelines.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | class GivenDisabledPipelines 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithPipelinesDisabled(); 21 | await _helper.RunPipeline(); 22 | } 23 | 24 | #region Integration tests 25 | 26 | [Test, Order(1)] 27 | public void ThenPipelineOutcomeIsFailed() 28 | { 29 | _helper.RunOutcome.Should().Be("Failed"); 30 | } 31 | 32 | #endregion 33 | 34 | [OneTimeTearDown] 35 | public void TearDown() 36 | { 37 | _helper?.TearDown(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenDisabledStages.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenDisabledStages 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithStagesDisabled(); 21 | await _helper.RunPipeline(); 22 | } 23 | 24 | #region Integration tests 25 | 26 | [Test, Order(1)] 27 | public void ThenPipelineOutcomeIsFailed() 28 | { 29 | _helper.RunOutcome.Should().Be("Failed"); 30 | } 31 | 32 | #endregion 33 | 34 | [OneTimeTearDown] 35 | public void TearDown() 36 | { 37 | _helper?.TearDown(); 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /MetadataDB/dbo/Stored Procedures/ExampleCustomExecutionPrecursor.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [dbo].[ExampleCustomExecutionPrecursor] 2 | AS 3 | BEGIN 4 | 5 | --set random Worker pipeline parameter wait times for development environment 6 | ;WITH cte AS 7 | ( 8 | SELECT 9 | [PipelineId], 10 | LEFT(ABS(CAST(CAST(NEWID() AS VARBINARY(192)) AS INT)),2) AS NewValue 11 | FROM 12 | [procfwk].[PipelineParameters] 13 | ) 14 | UPDATE 15 | pp 16 | SET 17 | pp.[ParameterValue] = cte.[NewValue] 18 | FROM 19 | [procfwk].[PipelineParameters] pp 20 | INNER JOIN cte 21 | ON pp.[PipelineId] = cte.[PipelineId] 22 | INNER JOIN [procfwk].[Pipelines] p 23 | ON pp.[PipelineId] = p.[PipelineId] 24 | WHERE 25 | pp.[ParameterName] LIKE 'Wait%' 26 | AND p.[Enabled] = 1; 27 | 28 | 29 | --disable certain Workers if running at the weekend... 30 | -- YOUR CODE HERE 31 | 32 | --enable certain Workers if running on the 10th day of the month... 33 | -- YOUR CODE HERE 34 | 35 | --disable certain Stages if running on Friday... 36 | -- YOUR CODE HERE 37 | 38 | --set Worker pipeline parameters to new value based on ______ .... 39 | -- YOUR CODE HERE 40 | 41 | --etc 42 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/SetErrorLogDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[SetErrorLogDetails] 2 | ( 3 | @LocalExecutionId UNIQUEIDENTIFIER, 4 | @JsonErrorDetails VARCHAR(MAX) 5 | ) 6 | AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | INSERT INTO [procfwk].[ErrorLog] 11 | ( 12 | [LocalExecutionId], 13 | [PipelineRunId], 14 | [ActivityRunId], 15 | [ActivityName], 16 | [ActivityType], 17 | [ErrorCode], 18 | [ErrorType], 19 | [ErrorMessage] 20 | ) 21 | SELECT 22 | @LocalExecutionId, 23 | Base.[RunId], 24 | ErrorDetail.[ActivityRunId], 25 | ErrorDetail.[ActivityName], 26 | ErrorDetail.[ActivityType], 27 | ErrorDetail.[ErrorCode], 28 | ErrorDetail.[ErrorType], 29 | ErrorDetail.[ErrorMessage] 30 | FROM 31 | OPENJSON(@JsonErrorDetails) WITH 32 | ( 33 | [RunId] UNIQUEIDENTIFIER, 34 | [Errors] NVARCHAR(MAX) AS JSON 35 | ) AS Base 36 | CROSS APPLY OPENJSON (Base.[Errors]) WITH 37 | ( 38 | [ActivityRunId] UNIQUEIDENTIFIER, 39 | [ActivityName] VARCHAR(100), 40 | [ActivityType] VARCHAR(100), 41 | [ErrorCode] VARCHAR(100), 42 | [ErrorType] VARCHAR(100), 43 | [ErrorMessage] VARCHAR(MAX) 44 | ) AS ErrorDetail 45 | END; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenDisabledBatches.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenDisabledBatches 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithBatchExecutionHandling() 21 | .WithBatchesDisabled(); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Integration tests 26 | 27 | [Test] 28 | public void ThenPipelineOutcomeIsFailed() 29 | { 30 | _helper.RunOutcome.Should().Be("Failed"); 31 | } 32 | 33 | #endregion 34 | 35 | [OneTimeTearDown] 36 | public void TearDown() 37 | { 38 | _helper?.TearDown(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/01-Grandparent/GivenOneWorkerPipeline.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Grandparent 7 | { 8 | public class GivenOneWorkerPipeline 9 | { 10 | private GrandparentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new GrandparentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithSimpleFailureHandling() 21 | .WithOneWorkerPipelineEnabled(); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Integration tests 26 | 27 | [Test] 28 | public void ThenPipelineOutcomeIsSucceeded() 29 | { 30 | _helper.RunOutcome.Should().Be("Succeeded"); 31 | } 32 | 33 | #endregion 34 | 35 | [OneTimeTearDown] 36 | public void TearDown() 37 | { 38 | _helper?.TearDown(); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenOneExecutionStage.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | class GivenOneExecutionStage 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutSimulatedError() 21 | .WithFailureHandling("Simple") 22 | .WithSingleExecutionStage(); 23 | await _helper.RunPipeline(); 24 | } 25 | 26 | #region Integration tests 27 | 28 | [Test] 29 | public void ThenPipelineOutcomeIsSucceeded() 30 | { 31 | _helper.RunOutcome.Should().Be("Succeeded"); 32 | } 33 | 34 | #endregion 35 | 36 | [OneTimeTearDown] 37 | public void TearDown() 38 | { 39 | _helper?.TearDown(); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Functions/Helpers/KeyVaultClient.cs: -------------------------------------------------------------------------------- 1 | using Azure.Identity; 2 | using Azure.Security.KeyVault.Secrets; 3 | using System; 4 | 5 | namespace mrpaulandrew.azure.procfwk.Helpers 6 | { 7 | internal class KeyVaultClient 8 | { 9 | private static readonly DefaultAzureCredential defaultCred = new DefaultAzureCredential(); 10 | 11 | public static string GetSecretFromUri(string secretString) 12 | { 13 | return GetSecretFromUri(new Uri(secretString)); 14 | } 15 | 16 | public static string GetSecretFromUri(Uri secretUri) 17 | { 18 | string keyVaultURL = "https://" + secretUri.Host.ToString(); 19 | string secretName = secretUri.LocalPath.ToString().Replace("secrets/", "").Replace("/", ""); 20 | return CreateKeyVaultClient(keyVaultURL).GetSecret(secretName).Value.Value; 21 | } 22 | 23 | public static string GetSecretFromName(string keyVaultURL, string secretName) 24 | { 25 | return CreateKeyVaultClient(keyVaultURL).GetSecret(secretName).Value.Value; 26 | } 27 | 28 | private static SecretClient CreateKeyVaultClient(string keyVaultURL) 29 | { 30 | return new SecretClient(new Uri(keyVaultURL), defaultCred); 31 | } 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenNoPipelineParameters.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | class GivenNoPipelineParameters 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutSimulatedError() 21 | .WithFailureHandling("Simple"); ; 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Integration tests 26 | 27 | [Test] 28 | public void ThenPipelineOutcomeIsSucceeded() 29 | { 30 | _helper.RunOutcome.Should().Be("Succeeded"); 31 | } 32 | 33 | #endregion 34 | 35 | #region Functional tests 36 | 37 | #endregion 38 | 39 | [OneTimeTearDown] 40 | public void TearDown() 41 | { 42 | _helper?.TearDown(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Script.PostDeployment.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Post-Deployment Script Template 3 | -------------------------------------------------------------------------------------- 4 | This file contains SQL statements that will be appended to the build script. 5 | Use SQLCMD syntax to include a file in the post-deployment script. 6 | Example: :r .\myfile.sql 7 | Use SQLCMD syntax to reference a variable in the post-deployment script. 8 | Example: :setvar TableName MyTable 9 | SELECT * FROM [$(TableName)] 10 | -------------------------------------------------------------------------------------- 11 | */ 12 | --load default metadata 13 | :r .\Metadata\Properties.sql 14 | :r .\Metadata\Orchestrators.sql 15 | :r .\Metadata\Stages.sql 16 | :r .\Metadata\Pipelines.sql 17 | :r .\Metadata\PipelineParams.sql 18 | :r .\Metadata\PipelineDependencies.sql 19 | :r .\Metadata\Recipients.sql 20 | :r .\Metadata\AlertOutcomes.sql 21 | :r .\Metadata\RecipientAlertsLink.sql 22 | 23 | --restore log data 24 | :r .\LogData\ExecutionLogRestore.sql 25 | :r .\LogData\ErrorLogRestore.sql 26 | 27 | --object transfers 28 | :r .\Metadata\TransferHelperObjects.sql 29 | :r .\Metadata\TransferReportingObjects.sql 30 | 31 | --replace old objects 32 | :r .\Metadata\ReplaceDataFactorys.sql -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultSubscription.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultSubscription] 2 | AS 3 | BEGIN 4 | DECLARE @Subscriptions TABLE 5 | ( 6 | [SubscriptionId] UNIQUEIDENTIFIER NOT NULL, 7 | [Name] NVARCHAR(200) NOT NULL, 8 | [Description] NVARCHAR(MAX) NULL, 9 | [TenantId] UNIQUEIDENTIFIER NOT NULL 10 | ) 11 | 12 | INSERT INTO @Subscriptions 13 | ( 14 | [SubscriptionId], 15 | [Name], 16 | [Description], 17 | [TenantId] 18 | ) 19 | VALUES 20 | ('12345678-1234-1234-1234-012345678910', 'Default', 'Example value for development environment.', '12345678-1234-1234-1234-012345678910'); 21 | 22 | MERGE INTO [procfwk].[Subscriptions] AS tgt 23 | USING 24 | @Subscriptions AS src 25 | ON tgt.[SubscriptionId] = src.[SubscriptionId] 26 | WHEN MATCHED THEN 27 | UPDATE 28 | SET 29 | tgt.[Name] = src.[Name], 30 | tgt.[Description] = src.[Description], 31 | tgt.[TenantId] = src.[TenantId] 32 | WHEN NOT MATCHED BY TARGET THEN 33 | INSERT 34 | ( 35 | [SubscriptionId], 36 | [Name], 37 | [Description], 38 | [TenantId] 39 | ) 40 | VALUES 41 | ( 42 | src.[SubscriptionId], 43 | src.[Name], 44 | src.[Description], 45 | src.[TenantId] 46 | ) 47 | WHEN NOT MATCHED BY SOURCE THEN 48 | DELETE; 49 | END; -------------------------------------------------------------------------------- /Functions/Helpers/SMTPClient.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Net.Mail; 3 | 4 | namespace mrpaulandrew.azure.procfwk.Helpers 5 | { 6 | internal class SMTPClient 7 | { 8 | public static string FromEmail { get; set; } 9 | 10 | public static SmtpClient CreateSMTPClient() 11 | { 12 | string smtpHost = Environment.GetEnvironmentVariable("AppSettingSmtpHost"); 13 | int smtpPort = int.Parse(Environment.GetEnvironmentVariable("AppSettingSmtpPort")); 14 | string smtpUser = Environment.GetEnvironmentVariable("AppSettingSmtpUser"); 15 | string smtpPass = Environment.GetEnvironmentVariable("AppSettingSmtpPass"); 16 | 17 | FromEmail = Environment.GetEnvironmentVariable("AppSettingFromEmail"); 18 | 19 | SmtpClient emailClient = new SmtpClient 20 | { 21 | EnableSsl = true, 22 | UseDefaultCredentials = false, //order properties are set is important 23 | Credentials = new System.Net.NetworkCredential(smtpUser, smtpPass), 24 | DeliveryMethod = SmtpDeliveryMethod.Network, 25 | Host = smtpHost, 26 | Port = smtpPort 27 | }; 28 | 29 | return emailClient; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultRecipients.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultRecipients] 2 | AS 3 | BEGIN 4 | DECLARE @Recipients TABLE 5 | ( 6 | [Name] [VARCHAR](255) NULL, 7 | [EmailAddress] [NVARCHAR](500) NOT NULL, 8 | [MessagePreference] [CHAR](3) NOT NULL, 9 | [Enabled] [BIT] NOT NULL 10 | ) 11 | 12 | INSERT INTO @Recipients 13 | ( 14 | [Name], 15 | [EmailAddress], 16 | [MessagePreference], 17 | [Enabled] 18 | ) 19 | VALUES 20 | ('Test User 1','test.user1@adfprocfwk.com', 'TO', 1), 21 | ('Test User 2','test.user2@adfprocfwk.com', 'CC', 1), 22 | ('Test User 3','test.user3@adfprocfwk.com', 'BCC', 1); 23 | 24 | MERGE INTO [procfwk].[Recipients] AS tgt 25 | USING 26 | @Recipients AS src 27 | ON tgt.[Name] = src.[Name] 28 | WHEN MATCHED THEN 29 | UPDATE 30 | SET 31 | tgt.[EmailAddress] = src.[EmailAddress], 32 | tgt.[MessagePreference] = src.[MessagePreference], 33 | tgt.[Enabled] = src.[Enabled] 34 | WHEN NOT MATCHED BY TARGET THEN 35 | INSERT 36 | ( 37 | [Name], 38 | [EmailAddress], 39 | [MessagePreference], 40 | [Enabled] 41 | ) 42 | VALUES 43 | ( 44 | src.[Name], 45 | src.[EmailAddress], 46 | src.[MessagePreference], 47 | src.[Enabled] 48 | ) 49 | WHEN NOT MATCHED BY SOURCE THEN 50 | DELETE; 51 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultStages.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultStages] 2 | AS 3 | BEGIN 4 | DECLARE @Stages TABLE 5 | ( 6 | [StageName] [VARCHAR](225) NOT NULL, 7 | [StageDescription] [VARCHAR](4000) NULL, 8 | [Enabled] [BIT] NOT NULL 9 | ) 10 | 11 | INSERT @Stages 12 | ( 13 | [StageName], 14 | [StageDescription], 15 | [Enabled] 16 | ) 17 | VALUES 18 | ('Extract', N'Ingest all data from source systems.', 1), 19 | ('Transform', N'Transform ingested data and apply business logic.', 1), 20 | ('Load', N'Load transformed data into data warehouse layer.', 1), 21 | ('Serve', N'Load transformed data into semantic layer.', 1), 22 | ('Speed', N'Regular loading of frequently used data.', 0); 23 | 24 | MERGE INTO [procfwk].[Stages] AS tgt 25 | USING 26 | @Stages AS src 27 | ON tgt.[StageName] = src.[StageName] 28 | WHEN MATCHED THEN 29 | UPDATE 30 | SET 31 | tgt.[StageDescription] = src.[StageDescription], 32 | tgt.[Enabled] = src.[Enabled] 33 | WHEN NOT MATCHED BY TARGET THEN 34 | INSERT 35 | ( 36 | [StageName], 37 | [StageDescription], 38 | [Enabled] 39 | ) 40 | VALUES 41 | ( 42 | src.[StageName], 43 | src.[StageDescription], 44 | src.[Enabled] 45 | ) 46 | WHEN NOT MATCHED BY SOURCE THEN 47 | DELETE; 48 | END; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/Utilities/GivenEmailSender.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Utilities 7 | { 8 | public class GivenEmailSender 9 | { 10 | private UtilitiesHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new UtilitiesHelper() 16 | .WithParameter("Recipients", "paul@mrpaulandrew.com") 17 | .WithParameter("Subject", "NUnit Test") 18 | .WithParameter("Body", "NUnit Test"); ; 19 | 20 | await _helper.RunPipeline("Email Sender"); 21 | } 22 | 23 | [Test] 24 | public void ThenPipelineOutcomeIsSucceeded() 25 | { 26 | _helper.RunOutcome.Should().Be("Succeeded"); 27 | } 28 | 29 | [Test] 30 | public async Task ThenActivityShouldReturnEmailSentTrue() 31 | { 32 | var sentState = await _helper.GetActivityOutput("Send Email", "$.EmailSent"); 33 | bool.Parse(sentState).Should().Be(true); 34 | } 35 | 36 | [OneTimeTearDown] 37 | public void TearDown() 38 | { 39 | _helper?.TearDown(); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Tables/CurrentExecution.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE [procfwk].[CurrentExecution] ( 2 | [LocalExecutionId] UNIQUEIDENTIFIER NOT NULL, 3 | [StageId] INT NOT NULL, 4 | [PipelineId] INT NOT NULL, 5 | [CallingOrchestratorName] NVARCHAR(200) NOT NULL, 6 | [ResourceGroupName]NVARCHAR (200) NOT NULL, 7 | [OrchestratorType] CHAR(3) NOT NULL, 8 | [OrchestratorName] NVARCHAR (200) NOT NULL, 9 | [PipelineName] NVARCHAR (200) NOT NULL, 10 | [StartDateTime] DATETIME NULL, 11 | [PipelineStatus] NVARCHAR (200) NULL, 12 | [LastStatusCheckDateTime] DATETIME NULL, 13 | [EndDateTime] DATETIME NULL, 14 | [IsBlocked] BIT NOT NULL DEFAULT 0, 15 | [PipelineRunId] UNIQUEIDENTIFIER NULL, 16 | [PipelineParamsUsed] NVARCHAR(MAX) NULL, 17 | CONSTRAINT [PK_CurrentExecution] PRIMARY KEY CLUSTERED ([LocalExecutionId] ASC, [StageId] ASC, [PipelineId] ASC) 18 | ); 19 | GO 20 | 21 | CREATE NONCLUSTERED INDEX [IDX_GetPipelinesInStage] ON [procfwk].[CurrentExecution] 22 | ( 23 | [LocalExecutionId], 24 | [StageId], 25 | [PipelineStatus] 26 | ) 27 | INCLUDE 28 | ( 29 | [PipelineId], 30 | [PipelineName], 31 | [OrchestratorType], 32 | [OrchestratorName], 33 | [ResourceGroupName] 34 | ) 35 | GO 36 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/CheckForEmailAlerts.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[CheckForEmailAlerts] 2 | ( 3 | @PipelineId INT 4 | ) 5 | AS 6 | BEGIN 7 | SET NOCOUNT ON; 8 | 9 | DECLARE @SendAlerts BIT 10 | DECLARE @AlertingEnabled BIT 11 | 12 | --get property 13 | SELECT 14 | @AlertingEnabled = [procfwk].[GetPropertyValueInternal]('UseFrameworkEmailAlerting'); 15 | 16 | --based on global property 17 | IF (@AlertingEnabled = 1) 18 | BEGIN 19 | --based on piplines to recipients link 20 | IF EXISTS 21 | ( 22 | SELECT pal.AlertId 23 | FROM procfwk.CurrentExecution AS ce 24 | INNER JOIN procfwk.AlertOutcomes AS ao 25 | ON ao.PipelineOutcomeStatus = ce.PipelineStatus 26 | INNER JOIN procfwk.PipelineAlertLink AS pal 27 | ON pal.PipelineId = ce.PipelineId 28 | INNER JOIN procfwk.Recipients AS r 29 | ON r.RecipientId = pal.RecipientId 30 | WHERE ce.PipelineId = @PipelineId 31 | AND ( 32 | ao.BitValue & pal.OutcomesBitValue <> 0 33 | OR pal.OutcomesBitValue & 1 <> 0 --all 34 | ) 35 | AND pal.[Enabled] = 1 36 | AND r.[Enabled] = 1 37 | ) 38 | BEGIN 39 | SET @SendAlerts = 1; 40 | END; 41 | ELSE 42 | BEGIN 43 | SET @SendAlerts = 0; 44 | END; 45 | END 46 | ELSE 47 | BEGIN 48 | SET @SendAlerts = 0; 49 | END; 50 | 51 | SELECT @SendAlerts AS SendAlerts 52 | END; -------------------------------------------------------------------------------- /Functions/Functions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | netcoreapp3.1 4 | v3 5 | mrpaulandrew.azure.procfwk 6 | mrpaulandrew.azure.procfwk 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | PreserveNewest 21 | 22 | 23 | Always 24 | 25 | 26 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Functions/CheckForValidURL.sql: -------------------------------------------------------------------------------- 1 | CREATE FUNCTION [procfwkHelpers].[CheckForValidURL] (@Url NVARCHAR(MAX)) 2 | RETURNS INT 3 | AS 4 | BEGIN 5 | DECLARE @Ending VARCHAR(50) 6 | DECLARE @TempString VARCHAR(50) 7 | 8 | --check URL start 9 | IF CHARINDEX('https://', @Url) <> 1 10 | BEGIN 11 | RETURN 0; 12 | END 13 | 14 | --check for expected sub domains 15 | IF CHARINDEX('vault.azure.net', @Url) = 0 16 | BEGIN 17 | RETURN 0; 18 | END 19 | 20 | --check for expected value type 21 | IF CHARINDEX('secrets', @Url) = 0 22 | BEGIN 23 | RETURN 0; 24 | END 25 | 26 | --attempt to check for secret version 27 | SELECT 28 | @Ending = 29 | CASE 30 | WHEN RIGHT(@Url,1) = '/' THEN REVERSE(LEFT(@Url,LEN(@Url)-1)) 31 | ELSE REVERSE(@Url) 32 | END, 33 | @Ending = REVERSE(LEFT(@Ending,CHARINDEX('/',@Ending)-1)) 34 | 35 | IF LEN(@Ending) = 32 36 | BEGIN 37 | SET @TempString = 38 | SUBSTRING(@Ending, 1, 8) + '-' + 39 | SUBSTRING(@Ending, 9, 4) + '-' + 40 | SUBSTRING(@Ending, 13, 4) + '-' + 41 | SUBSTRING(@Ending, 13, 4) + '-' + 42 | SUBSTRING(@Ending, 20, 12) 43 | END 44 | 45 | IF TRY_CAST(@TempString AS UNIQUEIDENTIFIER) IS NOT NULL 46 | BEGIN 47 | RETURN 0; 48 | END; 49 | 50 | -- It is a valid URL 51 | RETURN 1; 52 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/AddServicePrincipalWrapper.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[AddServicePrincipalWrapper] 2 | ( 3 | @OrchestratorName NVARCHAR(200), 4 | @OrchestratorType CHAR(3), 5 | @PrincipalIdValue NVARCHAR(MAX), 6 | @PrincipalSecretValue NVARCHAR(MAX), 7 | @SpecificPipelineName NVARCHAR(200) = NULL, 8 | @PrincipalName NVARCHAR(256) = NULL 9 | ) 10 | AS 11 | BEGIN 12 | 13 | IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInDatabase' 14 | BEGIN 15 | EXEC [procfwk].[AddServicePrincipal] 16 | @OrchestratorName = @OrchestratorName, 17 | @OrchestratorType = @OrchestratorType, 18 | @PrincipalId = @PrincipalIdValue, 19 | @PrincipalSecret = @PrincipalSecretValue, 20 | @PrincipalName = @PrincipalName, 21 | @SpecificPipelineName = @SpecificPipelineName 22 | END 23 | ELSE IF ([procfwk].[GetPropertyValueInternal]('SPNHandlingMethod')) = 'StoreInKeyVault' 24 | BEGIN 25 | EXEC [procfwk].[AddServicePrincipalUrls] 26 | @OrchestratorName = @OrchestratorName, 27 | @OrchestratorType = @OrchestratorType, 28 | @PrincipalIdUrl = @PrincipalIdValue, 29 | @PrincipalSecretUrl = @PrincipalSecretValue, 30 | @PrincipalName = @PrincipalName, 31 | @SpecificPipelineName = @SpecificPipelineName 32 | END 33 | ELSE 34 | BEGIN 35 | RAISERROR('Unknown SPN insert method.',16,1); 36 | RETURN 0; 37 | END 38 | END; 39 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelineParameters.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelineParameters] 2 | AS 3 | BEGIN 4 | DECLARE @PipelineParameters TABLE 5 | ( 6 | [PipelineId] [INT] NOT NULL, 7 | [ParameterName] [VARCHAR](128) NOT NULL, 8 | [ParameterValue] [NVARCHAR](MAX) NULL 9 | ) 10 | 11 | INSERT @PipelineParameters 12 | ( 13 | [PipelineId], 14 | [ParameterName], 15 | [ParameterValue] 16 | ) 17 | VALUES 18 | (1, 'WaitTime', '3'), 19 | (2, 'WaitTime', '6'), 20 | (6, 'WaitTime', '9'), 21 | (4, 'WaitTime', '5'), 22 | (5, 'WaitTime', '2'), 23 | (3, 'RaiseErrors', 'false'), 24 | (3, 'WaitTime', '10'), 25 | (7, 'WaitTime', '3'), 26 | (8, 'WaitTime', '5'), 27 | (9, 'WaitTime', '7'), 28 | (11, 'WaitTime', '10'); 29 | 30 | MERGE INTO [procfwk].[PipelineParameters] AS tgt 31 | USING 32 | @PipelineParameters AS src 33 | ON tgt.[PipelineId] = src.[PipelineId] 34 | AND tgt.[ParameterName] = src.[ParameterName] 35 | WHEN MATCHED THEN 36 | UPDATE 37 | SET 38 | tgt.[ParameterValue] = src.[ParameterValue] 39 | WHEN NOT MATCHED BY TARGET THEN 40 | INSERT 41 | ( 42 | [PipelineId], 43 | [ParameterName], 44 | [ParameterValue] 45 | ) 46 | VALUES 47 | ( 48 | src.[PipelineId], 49 | src.[ParameterName], 50 | src.[ParameterValue] 51 | ) 52 | WHEN NOT MATCHED BY SOURCE THEN 53 | DELETE; 54 | END; -------------------------------------------------------------------------------- /MetadataDB/procfwkTesting/Stored Procedures/Add300WorkerPipelineBatches.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkTesting].[Add300WorkerPipelineBatches] 2 | AS 3 | BEGIN 4 | SET NOCOUNT ON; 5 | 6 | --clear default metadata 7 | DELETE FROM [procfwk].[BatchStageLink]; 8 | DELETE FROM [procfwk].[Batches]; 9 | 10 | --add batch details 11 | ;WITH sourceData AS 12 | ( 13 | SELECT 14 | '0to300' AS BatchName, 15 | 'The first 300.' AS BatchDescription, 16 | 1 AS [Enabled] 17 | UNION SELECT 18 | '301to600', 19 | 'The second 300.', 20 | 1 21 | ) 22 | MERGE INTO [procfwk].[Batches] AS tgt 23 | USING 24 | sourceData AS src 25 | ON tgt.[BatchName] = src.[BatchName] 26 | WHEN MATCHED THEN 27 | UPDATE 28 | SET 29 | tgt.[BatchDescription] = src.[BatchDescription], 30 | tgt.[Enabled] = src.[Enabled] 31 | WHEN NOT MATCHED BY TARGET THEN 32 | INSERT 33 | ( 34 | [BatchName], 35 | [BatchDescription], 36 | [Enabled] 37 | ) 38 | VALUES 39 | ( 40 | src.[BatchName], 41 | src.[BatchDescription], 42 | src.[Enabled] 43 | ) 44 | WHEN NOT MATCHED BY SOURCE THEN 45 | DELETE; 46 | 47 | --link batches to stages 48 | INSERT INTO [procfwk].[BatchStageLink] 49 | ( 50 | [BatchId], 51 | [StageId] 52 | ) 53 | SELECT 54 | b.[BatchId], 55 | s.[StageId] 56 | FROM 57 | [procfwk].[Batches] b 58 | INNER JOIN [procfwk].[Stages] s 59 | ON s.[Enabled] = 1; 60 | END; 61 | -------------------------------------------------------------------------------- /Synapse/pipeline/Throw Exception.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Throw Exception", 3 | "properties": { 4 | "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", 5 | "activities": [ 6 | { 7 | "name": "Raise Error", 8 | "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", 9 | "type": "Lookup", 10 | "dependsOn": [], 11 | "policy": { 12 | "timeout": "0.00:10:00", 13 | "retry": 0, 14 | "retryIntervalInSeconds": 30, 15 | "secureOutput": false, 16 | "secureInput": false 17 | }, 18 | "userProperties": [], 19 | "typeProperties": { 20 | "source": { 21 | "type": "AzureSqlSource", 22 | "sqlReaderQuery": { 23 | "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", 24 | "type": "Expression" 25 | }, 26 | "queryTimeout": "02:00:00", 27 | "partitionOption": "None" 28 | }, 29 | "dataset": { 30 | "referenceName": "GetSetMetadata", 31 | "type": "DatasetReference" 32 | }, 33 | "firstRowOnly": false 34 | } 35 | } 36 | ], 37 | "parameters": { 38 | "Message": { 39 | "type": "string" 40 | } 41 | }, 42 | "folder": { 43 | "name": "_ProcFwk/_ProcFwkUtils" 44 | }, 45 | "annotations": [ 46 | "procfwk", 47 | "Utils" 48 | ] 49 | } 50 | } -------------------------------------------------------------------------------- /Functions/Helpers/BodyReader.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.Collections.Generic; 3 | using System.IO; 4 | using System.Text; 5 | using System.Threading.Tasks; 6 | using Microsoft.AspNetCore.Http; 7 | using Microsoft.Extensions.Logging; 8 | using Newtonsoft.Json; 9 | 10 | namespace mrpaulandrew.azure.procfwk.Helpers 11 | { 12 | public class BodyReader 13 | { 14 | public string Body; 15 | public BodyReader(HttpRequest httpRequest) 16 | { 17 | Body = new StreamReader(httpRequest.Body).ReadToEnd(); 18 | } 19 | 20 | public Task GetRequestBody() 21 | { 22 | PipelineRequest request = JsonConvert.DeserializeObject(Body); 23 | return Task.FromResult(request); 24 | } 25 | 26 | public async Task GetRequestBodyAsync() 27 | { 28 | PipelineRequest request = await GetRequestBody(); 29 | return request; 30 | } 31 | 32 | public Task GetRunRequestBody() 33 | { 34 | PipelineRunRequest request = JsonConvert.DeserializeObject(Body); 35 | return Task.FromResult(request); 36 | } 37 | 38 | public async Task GetRunRequestBodyAsync() 39 | { 40 | PipelineRunRequest request = await GetRunRequestBody(); 41 | return request; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /FactoryTesting/FactoryTesting.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | all 26 | runtime; build; native; contentfiles; analyzers; buildtransitive 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/TransferHelperObjects.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID(N'tempdb..#TransferHelperObjects') IS NOT NULL DROP PROCEDURE #TransferHelperObjects; 2 | GO 3 | 4 | CREATE PROCEDURE #TransferHelperObjects 5 | ( 6 | @ObjectName NVARCHAR(128), 7 | @ObjectType CHAR(2) 8 | ) 9 | AS 10 | BEGIN 11 | IF EXISTS 12 | ( 13 | SELECT 14 | * 15 | FROM 16 | sys.objects o 17 | INNER JOIN sys.schemas s 18 | ON o.[schema_id] = s.[schema_id] 19 | WHERE 20 | o.[Name] = @ObjectName 21 | AND s.[name] = 'procfwk' 22 | AND o.[type] = @ObjectType 23 | ) 24 | BEGIN 25 | PRINT 'Transferring: ' + @ObjectName; 26 | EXEC('ALTER SCHEMA [procfwkHelpers] TRANSFER [procfwk].[' + @ObjectName + '];') 27 | END; 28 | END; 29 | GO 30 | 31 | EXEC #TransferHelperObjects 'AddProperty', 'P'; 32 | EXEC #TransferHelperObjects 'GetExecutionDetails', 'P'; 33 | EXEC #TransferHelperObjects 'AddRecipientPipelineAlerts', 'P'; 34 | EXEC #TransferHelperObjects 'DeleteRecipientAlerts', 'P'; 35 | EXEC #TransferHelperObjects 'CheckStageAndPiplineIntegrity', 'P'; 36 | EXEC #TransferHelperObjects 'AddPipelineDependant', 'P'; 37 | EXEC #TransferHelperObjects 'AddServicePrincipalWrapper', 'P'; 38 | EXEC #TransferHelperObjects 'AddServicePrincipalUrls', 'P'; 39 | EXEC #TransferHelperObjects 'AddServicePrincipal', 'P'; 40 | EXEC #TransferHelperObjects 'DeleteServicePrincipal', 'P'; 41 | EXEC #TransferHelperObjects 'CheckForValidURL', 'FN'; 42 | EXEC #TransferHelperObjects 'PipelineDependencyChains', 'V'; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Paul Andrew (mrpaulandrew.com) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | THE SOFTWARE IS ALSO IN NO WAY ASOCIATED WITH OR SUPPORTED BY: 24 | - Altius Consulting Limited 25 | - Cloudpoint Limited 26 | - Farah Bidco Limited 27 | - Farah Midco Limited 28 | - Farah Topco Limited 29 | - Avanade UK Ltd 30 | - Avanade Europe Services Ltd 31 | - Avanade Europe Holdings Ltd 32 | - Avanade International Corporation -------------------------------------------------------------------------------- /Functions/Functions/CancelPipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Azure.WebJobs.Extensions.Http; 7 | using Microsoft.AspNetCore.Http; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using mrpaulandrew.azure.procfwk.Helpers; 11 | using mrpaulandrew.azure.procfwk.Services; 12 | 13 | namespace mrpaulandrew.azure.procfwk 14 | { 15 | public static class CancelPipeline 16 | { 17 | [FunctionName("CancelPipeline")] 18 | public static async Task Run( 19 | [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest httpRequest, 20 | ILogger logger) 21 | { 22 | logger.LogInformation("CancelPipeline Function triggered by HTTP request."); 23 | 24 | logger.LogInformation("Parsing body from request."); 25 | PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); 26 | request.Validate(logger); 27 | 28 | using (var service = PipelineService.GetServiceForRequest(request, logger)) 29 | { 30 | PipelineRunStatus result = service.CancelPipeline(request); 31 | logger.LogInformation("CancelPipeline Function complete."); 32 | return new OkObjectResult(JsonConvert.SerializeObject(result)); 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Functions/Functions/ExecutePipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Azure.WebJobs.Extensions.Http; 7 | using Microsoft.AspNetCore.Http; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using mrpaulandrew.azure.procfwk.Helpers; 11 | using mrpaulandrew.azure.procfwk.Services; 12 | 13 | namespace mrpaulandrew.azure.procfwk 14 | { 15 | public static class ExecutePipeline 16 | { 17 | [FunctionName("ExecutePipeline")] 18 | public static async Task Run( 19 | [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequest httpRequest, 20 | ILogger logger) 21 | { 22 | logger.LogInformation("ExecutePipeline Function triggered by HTTP request."); 23 | 24 | logger.LogInformation("Parsing body from request."); 25 | PipelineRequest request = await new BodyReader(httpRequest).GetRequestBodyAsync(); 26 | request.Validate(logger); 27 | 28 | using (var service = PipelineService.GetServiceForRequest(request, logger)) 29 | { 30 | PipelineRunStatus result = service.ExecutePipeline(request); 31 | logger.LogInformation("ExecutePipeline Function complete."); 32 | return new OkObjectResult(JsonConvert.SerializeObject(result)); 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/DeleteRecipientAlerts.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[DeleteRecipientAlerts] 2 | ( 3 | @EmailAddress NVARCHAR(500), 4 | @SoftDeleteOnly BIT = 1 5 | ) 6 | AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | --defensive check 11 | IF NOT EXISTS 12 | ( 13 | SELECT [RecipientId] FROM [procfwk].[Recipients] WHERE [EmailAddress] = @EmailAddress 14 | ) 15 | BEGIN 16 | RAISERROR('Recipient email address does not exists in [procfwk].[Recipients] table.',16,1); 17 | RETURN 0; 18 | END; 19 | 20 | --update/delete 21 | IF @SoftDeleteOnly = 1 22 | BEGIN 23 | --disable links 24 | UPDATE 25 | al 26 | SET 27 | al.[Enabled] = 0 28 | FROM 29 | [procfwk].[PipelineAlertLink] al 30 | INNER JOIN [procfwk].[Recipients] r 31 | ON al.[RecipientId] = r.[RecipientId] 32 | WHERE 33 | r.[EmailAddress] = @EmailAddress; 34 | 35 | --disable recipient(s) 36 | UPDATE 37 | [procfwk].[Recipients] 38 | SET 39 | [Enabled] = 0 40 | WHERE 41 | [EmailAddress] = @EmailAddress; 42 | 43 | END 44 | ELSE 45 | BEGIN 46 | --delete links 47 | DELETE 48 | al 49 | FROM 50 | [procfwk].[PipelineAlertLink] al 51 | INNER JOIN [procfwk].[Recipients] r 52 | ON al.[RecipientId] = r.[RecipientId] 53 | WHERE 54 | r.[EmailAddress] = @EmailAddress; 55 | 56 | --delete recipient(s) 57 | DELETE FROM 58 | [procfwk].[Recipients] 59 | WHERE 60 | [EmailAddress] = @EmailAddress; 61 | END; 62 | END; -------------------------------------------------------------------------------- /Functions/Functions/ValidatePipeline.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Azure.WebJobs.Extensions.Http; 7 | using Microsoft.AspNetCore.Http; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using mrpaulandrew.azure.procfwk.Helpers; 11 | using mrpaulandrew.azure.procfwk.Services; 12 | 13 | namespace mrpaulandrew.azure.procfwk 14 | { 15 | public static class ValidatePipeline 16 | { 17 | [FunctionName("ValidatePipeline")] 18 | public static async Task Run( 19 | [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, 20 | ILogger logger) 21 | { 22 | logger.LogInformation("ValidatePipeline Function triggered by HTTP request."); 23 | 24 | logger.LogInformation("Parsing body from request."); 25 | PipelineRequest request = await new BodyReader(httpRequest).GetRequestBodyAsync(); 26 | request.Validate(logger); 27 | 28 | using (var service = PipelineService.GetServiceForRequest(request, logger)) 29 | { 30 | PipelineDescription result = service.ValidatePipeline(request); 31 | logger.LogInformation("ValidatePipeline Function complete."); 32 | return new OkObjectResult(JsonConvert.SerializeObject(result)); 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/01-Grandparent/Given300WorkerPipelines.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Grandparent 7 | { 8 | public class Given300WorkerPipelines 9 | { 10 | private GrandparentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new GrandparentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .With300WorkerPipelinesEnabled() 19 | .WithSPNInKeyVault("WorkersFactory"); 20 | 21 | await _helper.RunPipeline(); 22 | } 23 | 24 | #region Integration tests 25 | 26 | [Test] 27 | public void ThenPipelineOutcomeIsSucceeded() 28 | { 29 | _helper.RunOutcome.Should().Be("Succeeded"); 30 | } 31 | [Test] 32 | public void ThenCurrentExecutionTableIsEmpty() 33 | { 34 | _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); 35 | } 36 | 37 | [Test] 38 | public void Then300ExecutionLogRecords() 39 | { 40 | _helper.RowCount("procfwk.ExecutionLog").Should().Be(300); 41 | } 42 | 43 | #endregion 44 | 45 | [OneTimeTearDown] 46 | public void TearDown() 47 | { 48 | _helper?.TearDown(); 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /MetadataDB/procfwkTesting/Stored Procedures/GetRunIdWhenAvailable.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkTesting].[GetRunIdWhenAvailable] 2 | ( 3 | @PipelineName NVARCHAR(200) = NULL 4 | ) 5 | AS 6 | BEGIN 7 | IF @PipelineName IS NULL 8 | BEGIN 9 | WHILE 1=1 10 | BEGIN 11 | IF EXISTS 12 | ( 13 | SELECT TOP 1 14 | [PipelineRunId] 15 | FROM 16 | [procfwk].[CurrentExecution] 17 | WHERE 18 | [PipelineRunId] IS NOT NULL 19 | ) 20 | BEGIN 21 | BREAK; 22 | END 23 | 24 | WAITFOR DELAY '00:00:10'; 25 | END; 26 | 27 | SELECT TOP 1 28 | CAST([PipelineRunId] AS VARCHAR(36)) AS RunId 29 | FROM 30 | [procfwk].[CurrentExecution] 31 | WHERE 32 | [PipelineRunId] IS NOT NULL 33 | END 34 | ELSE IF @PipelineName IS NOT NULL 35 | BEGIN 36 | WHILE 1=1 37 | BEGIN 38 | IF EXISTS 39 | ( 40 | SELECT TOP 1 41 | [PipelineRunId] 42 | FROM 43 | [procfwk].[CurrentExecution] 44 | WHERE 45 | [PipelineRunId] IS NOT NULL 46 | AND [PipelineName] = @PipelineName 47 | ) 48 | BEGIN 49 | BREAK; 50 | END 51 | 52 | WAITFOR DELAY '00:00:10'; 53 | END; 54 | 55 | SELECT TOP 1 56 | CAST([PipelineRunId] AS VARCHAR(36)) AS RunId 57 | FROM 58 | [procfwk].[CurrentExecution] 59 | WHERE 60 | [PipelineRunId] IS NOT NULL 61 | AND [PipelineName] = @PipelineName 62 | END 63 | ELSE 64 | BEGIN 65 | RAISERROR('Unknown use of testing procedure.',16,1); 66 | END 67 | END; -------------------------------------------------------------------------------- /Functions/Functions/CheckPipelineStatus.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Azure.WebJobs.Extensions.Http; 7 | using Microsoft.AspNetCore.Http; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using mrpaulandrew.azure.procfwk.Helpers; 11 | using mrpaulandrew.azure.procfwk.Services; 12 | 13 | namespace mrpaulandrew.azure.procfwk 14 | { 15 | public static class CheckPipelineStatus 16 | { 17 | [FunctionName("CheckPipelineStatus")] 18 | public static async Task Run( 19 | [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, 20 | ILogger logger) 21 | { 22 | logger.LogInformation("CheckPipelineStatus Function triggered by HTTP request."); 23 | 24 | logger.LogInformation("Parsing body from request."); 25 | PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); 26 | request.Validate(logger); 27 | 28 | using (var service = PipelineService.GetServiceForRequest(request, logger)) 29 | { 30 | PipelineRunStatus result = service.GetPipelineRunStatus(request); 31 | logger.LogInformation("CheckPipelineStatus Function complete."); 32 | return new OkObjectResult(JsonConvert.SerializeObject(result)); 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Functions/Functions/GetActivityErrors.cs: -------------------------------------------------------------------------------- 1 | using System; 2 | using System.IO; 3 | using System.Threading.Tasks; 4 | using Microsoft.AspNetCore.Mvc; 5 | using Microsoft.Azure.WebJobs; 6 | using Microsoft.Azure.WebJobs.Extensions.Http; 7 | using Microsoft.AspNetCore.Http; 8 | using Microsoft.Extensions.Logging; 9 | using Newtonsoft.Json; 10 | using mrpaulandrew.azure.procfwk.Helpers; 11 | using mrpaulandrew.azure.procfwk.Services; 12 | 13 | namespace mrpaulandrew.azure.procfwk 14 | { 15 | public static class GetActivityErrors 16 | { 17 | [FunctionName("GetActivityErrors")] 18 | public static async Task Run( 19 | [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest httpRequest, 20 | ILogger logger) 21 | { 22 | logger.LogInformation("GetActivityErrors Function triggered by HTTP request."); 23 | 24 | logger.LogInformation("Parsing body from request."); 25 | PipelineRunRequest request = await new BodyReader(httpRequest).GetRunRequestBodyAsync(); 26 | request.Validate(logger); 27 | 28 | using (var service = PipelineService.GetServiceForRequest(request, logger)) 29 | { 30 | PipelineErrorDetail result = service.GetPipelineRunActivityErrors(request); 31 | logger.LogInformation("GetActivityErrors Function complete."); 32 | return new OkObjectResult(JsonConvert.SerializeObject(result)); 33 | } 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /FactoryTesting/Helpers/SettingsHelper.cs: -------------------------------------------------------------------------------- 1 | using Azure.Identity; 2 | using Azure.Security.KeyVault.Secrets; 3 | using NUnit.Framework; 4 | using System; 5 | 6 | namespace FactoryTesting.Helpers 7 | { 8 | public class SettingsHelper 9 | { 10 | public string GetSetting(string settingName) 11 | { 12 | // return environment variable "settingName", if present 13 | var value = Environment.GetEnvironmentVariable(settingName); 14 | if (value?.Length > 0) 15 | return value; 16 | 17 | // return value of runsettings parameter "settingName", if present 18 | value = TestContext.Parameters[settingName]; 19 | if (value?.Length > 0) 20 | return value; 21 | 22 | // if a key vault is specified, return the value of secret "settingName", if present 23 | if (_keyVaultClient != null) 24 | { 25 | value = _keyVaultClient.GetSecret(settingName).Value.Value; 26 | if (value?.Length > 0) 27 | return value; 28 | } 29 | 30 | throw new Exception($"Test setting '{settingName}' not found"); 31 | } 32 | 33 | private readonly SecretClient _keyVaultClient; 34 | 35 | public SettingsHelper() 36 | { 37 | var kvUrl = TestContext.Parameters["KeyVaultUrl"]; 38 | if (kvUrl?.Length > 0) 39 | _keyVaultClient = new SecretClient(new Uri(kvUrl), new DefaultAzureCredential()); 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /Functions/.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "label": "clean", 6 | "command": "dotnet", 7 | "args": [ 8 | "clean", 9 | "/property:GenerateFullPaths=true", 10 | "/consoleloggerparameters:NoSummary" 11 | ], 12 | "type": "process", 13 | "problemMatcher": "$msCompile" 14 | }, 15 | { 16 | "label": "build", 17 | "command": "dotnet", 18 | "args": [ 19 | "build", 20 | "/property:GenerateFullPaths=true", 21 | "/consoleloggerparameters:NoSummary" 22 | ], 23 | "type": "process", 24 | "dependsOn": "clean", 25 | "group": { 26 | "kind": "build", 27 | "isDefault": true 28 | }, 29 | "problemMatcher": "$msCompile" 30 | }, 31 | { 32 | "label": "clean release", 33 | "command": "dotnet", 34 | "args": [ 35 | "clean", 36 | "--configuration", 37 | "Release", 38 | "/property:GenerateFullPaths=true", 39 | "/consoleloggerparameters:NoSummary" 40 | ], 41 | "type": "process", 42 | "problemMatcher": "$msCompile" 43 | }, 44 | { 45 | "label": "publish", 46 | "command": "dotnet", 47 | "args": [ 48 | "publish", 49 | "--configuration", 50 | "Release", 51 | "/property:GenerateFullPaths=true", 52 | "/consoleloggerparameters:NoSummary" 53 | ], 54 | "type": "process", 55 | "dependsOn": "clean release", 56 | "problemMatcher": "$msCompile" 57 | }, 58 | { 59 | "type": "func", 60 | "dependsOn": "build", 61 | "options": { 62 | "cwd": "${workspaceFolder}/bin/Debug/netcoreapp3.1" 63 | }, 64 | "command": "host start", 65 | "isBackground": true, 66 | "problemMatcher": "$func-watch" 67 | } 68 | ] 69 | } -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInDatabase.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenNoErrorsAndSPNStoredInDatabase 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutSimulatedError() 21 | .WithFailureHandling("Simple"); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Integration tests 26 | 27 | [Test] 28 | public void ThenPipelineOutcomeIsSucceeded() 29 | { 30 | _helper.RunOutcome.Should().Be("Succeeded"); 31 | } 32 | 33 | #endregion 34 | 35 | #region Functional tests 36 | 37 | [Test] 38 | public void ThenCurrentExecutionTableIsEmpty() 39 | { 40 | _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); 41 | } 42 | 43 | [Test] 44 | public void ThenElevenExecutionLogRecords() 45 | { 46 | _helper.RowCount("procfwk.ExecutionLog").Should().Be(11); 47 | } 48 | 49 | #endregion 50 | 51 | [OneTimeTearDown] 52 | public void TearDown() 53 | { 54 | _helper?.TearDown(); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenNoErrorsAndSPNStoredInKeyVault.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenNoErrorsAndSPNStoredInKeyVault 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInKeyVault("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutSimulatedError() 21 | .WithFailureHandling("Simple"); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Integration tests 26 | 27 | [Test, Order(1)] 28 | public void ThenPipelineOutcomeIsSucceeded() 29 | { 30 | _helper.RunOutcome.Should().Be("Succeeded"); 31 | } 32 | 33 | #endregion 34 | 35 | #region Functional tests 36 | 37 | [Test, Order(2)] 38 | public void ThenCurrentExecutionTableIsEmpty() 39 | { 40 | _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); 41 | } 42 | 43 | [Test, Order(3)] 44 | public void ThenElevenExecutionLogRecords() 45 | { 46 | _helper.RowCount("procfwk.ExecutionLog").Should().Be(11); 47 | } 48 | 49 | #endregion 50 | 51 | [OneTimeTearDown] 52 | public void TearDown() 53 | { 54 | _helper?.TearDown(); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Functions/Services/PipelineService.cs: -------------------------------------------------------------------------------- 1 | using mrpaulandrew.azure.procfwk.Helpers; 2 | using Microsoft.Extensions.Logging; 3 | using System; 4 | 5 | namespace mrpaulandrew.azure.procfwk.Services 6 | { 7 | public abstract class PipelineService : IDisposable 8 | { 9 | public const int internalWaitDuration = 5000; //ms 10 | 11 | public static PipelineService GetServiceForRequest(PipelineRequest pr, ILogger logger) 12 | { 13 | if (pr.OrchestratorType == PipelineServiceType.ADF) 14 | return new AzureDataFactoryService(pr, logger); 15 | 16 | if (pr.OrchestratorType == PipelineServiceType.SYN) 17 | return new AzureSynapseService(pr, logger); 18 | 19 | throw new InvalidRequestException ("Unsupported orchestrator type: " + (pr.OrchestratorType?.ToString() ?? "")); 20 | } 21 | 22 | public abstract PipelineDescription ValidatePipeline(PipelineRequest request); 23 | 24 | public abstract PipelineRunStatus ExecutePipeline(PipelineRequest request); 25 | 26 | public abstract PipelineRunStatus CancelPipeline(PipelineRunRequest request); 27 | 28 | public abstract PipelineRunStatus GetPipelineRunStatus(PipelineRunRequest request); 29 | 30 | public abstract PipelineErrorDetail GetPipelineRunActivityErrors(PipelineRunRequest request); 31 | 32 | protected void PipelineNameCheck(string requestName, string foundName) 33 | { 34 | if (requestName.ToUpper() != foundName.ToUpper()) 35 | { 36 | throw new InvalidRequestException($"Pipeline name mismatch. Provided pipeline name does not match the provided Run Id. Expected name: {foundName}"); 37 | } 38 | } 39 | 40 | public abstract void Dispose(); 41 | } 42 | } -------------------------------------------------------------------------------- /MetadataDB/procfwkReporting/Views/WorkerParallelismOverTime.sql: -------------------------------------------------------------------------------- 1 | CREATE VIEW [procfwkReporting].[WorkerParallelismOverTime] 2 | AS 3 | 4 | WITH numbers AS 5 | ( 6 | SELECT TOP 500 7 | ROW_NUMBER() OVER (ORDER BY s1.[object_id]) - 1 AS 'Number' 8 | FROM 9 | sys.all_columns AS s1 10 | CROSS JOIN sys.all_columns AS s2 11 | ), 12 | executionBoundaries AS 13 | ( 14 | SELECT 15 | [LocalExecutionId], 16 | CAST(CONVERT(VARCHAR(16), MIN([StartDateTime]), 120) AS DATETIME) AS 'ExecutionStart', 17 | CAST(CONVERT(VARCHAR(16), MAX([EndDateTime]), 120) AS DATETIME) AS 'ExecutionEnd' 18 | FROM 19 | [procfwk].[ExecutionLog] 20 | --WHERE 21 | -- [LocalExecutionId] = '2BB02783-2A2C-4970-9BEA-0543013BFD5E' 22 | GROUP BY 23 | [LocalExecutionId] 24 | ), 25 | wallclockRunning AS 26 | ( 27 | SELECT 28 | CAST(DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) AS DATE) AS 'WallclockDate', 29 | CAST(DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) AS TIME) AS 'WallclockTime', 30 | el.[LocalExecutionId], 31 | el.[PipelineId], 32 | el.[PipelineName], 33 | s.[StageName] 34 | FROM 35 | executionBoundaries eB 36 | CROSS JOIN numbers n 37 | INNER JOIN [procfwk].[ExecutionLog] eL 38 | ON eB.[LocalExecutionId] = eL.[LocalExecutionId] 39 | AND DATEADD(MINUTE, n.[Number], eB.[ExecutionStart]) 40 | BETWEEN eL.[StartDateTime] AND eL.[EndDateTime] 41 | INNER JOIN [procfwk].[Stages] s 42 | ON eL.[StageId] = s.[StageId] 43 | ) 44 | 45 | SELECT 46 | [WallclockDate], 47 | [WallclockTime], 48 | [LocalExecutionId], 49 | [StageName], 50 | STRING_AGG(ISNULL([PipelineName],' '),', ') As 'PipelineName', 51 | COUNT([PipelineId]) AS 'WorkerCount' 52 | FROM 53 | wallclockRunning 54 | GROUP BY 55 | [WallclockDate], 56 | [WallclockTime], 57 | [LocalExecutionId], 58 | [StageName] 59 | GO 60 | 61 | 62 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetPipelineParameters.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetPipelineParameters] 2 | ( 3 | @PipelineId INT 4 | ) 5 | AS 6 | BEGIN 7 | SET NOCOUNT ON; 8 | 9 | DECLARE @Json VARCHAR(MAX) = '' 10 | 11 | --get parameters if required for worker pipeline 12 | IF NOT EXISTS 13 | ( 14 | SELECT 15 | [ParameterId] 16 | FROM 17 | [procfwk].[PipelineParameters] 18 | WHERE 19 | [PipelineId] = @PipelineId 20 | ) 21 | BEGIN 22 | SET @Json = '' --Can't return NULL. Would break ADF expression. 23 | END 24 | ELSE 25 | BEGIN 26 | SELECT 27 | @Json += 28 | CASE 29 | WHEN [ParameterValue] IS NULL THEN '' --don't add pair so ADF uses default 30 | ELSE '"' + [ParameterName] + '": "' + STRING_ESCAPE([ParameterValue],'json') + '",' 31 | END 32 | FROM 33 | [procfwk].[PipelineParameters] 34 | WHERE 35 | [PipelineId] = @PipelineId; 36 | 37 | --handle parameter(s) with a NULL values 38 | IF LEN(@Json) > 0 39 | BEGIN 40 | --JSON snippet gets injected into Azure Function body request via Orchestrator expressions. 41 | --Comma used to support Orchestrator expression. 42 | SET @Json = ',"pipelineParameters": {' + LEFT(@Json,LEN(@Json)-1) + '}' 43 | 44 | --update current execution log if this is a runtime request 45 | UPDATE 46 | [procfwk].[CurrentExecution] 47 | SET 48 | --add extra braces to make JSON string valid in logs 49 | [PipelineParamsUsed] = '{ ' + RIGHT(@Json,LEN(@Json)-1) + ' }' 50 | WHERE 51 | [PipelineId] = @PipelineId; 52 | 53 | --set last values values 54 | UPDATE 55 | [procfwk].[PipelineParameters] 56 | SET 57 | [ParameterValueLastUsed] = [ParameterValue] 58 | WHERE 59 | [PipelineId] = @PipelineId; 60 | END; 61 | END; 62 | 63 | --return JSON snippet 64 | SELECT @Json AS Params 65 | END; -------------------------------------------------------------------------------- /DeploymentTools/DeploymentTools.deployproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Debug 6 | AnyCPU 7 | 8 | 9 | Release 10 | AnyCPU 11 | 12 | 13 | 14 | 3aead846-dbe4-45ad-97dd-37e94be009fd 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | False 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /MetadataDB/procfwk/Stored Procedures/GetWorkerDetailsWrapper.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwk].[GetWorkerDetailsWrapper] 2 | ( 3 | @ExecutionId UNIQUEIDENTIFIER, 4 | @StageId INT, 5 | @PipelineId INT 6 | ) 7 | AS 8 | BEGIN 9 | /* 10 | Created this proc just to reduce and refactor the number of pipeline activity 11 | calls needed due to the Microsoft enforced limit of 40 activities per pipeline. 12 | */ 13 | SET NOCOUNT ON; 14 | 15 | DECLARE @WorkerAuthDetails TABLE 16 | ( 17 | [tenantId] UNIQUEIDENTIFIER NULL, 18 | [applicationId] NVARCHAR(MAX) NULL, 19 | [authenticationKey] NVARCHAR(MAX) NULL, 20 | [subscriptionId] UNIQUEIDENTIFIER NULL 21 | ) 22 | 23 | DECLARE @WorkerDetails TABLE 24 | ( 25 | [resourceGroupName] NVARCHAR(200) NULL, 26 | [orchestratorName] NVARCHAR(200) NULL, 27 | [orchestratorType] CHAR(3) NULL, 28 | [pipelineName] NVARCHAR(200) NULL 29 | ) 30 | 31 | --get work auth details 32 | INSERT INTO @WorkerAuthDetails 33 | ( 34 | [tenantId], 35 | [subscriptionId], 36 | [applicationId], 37 | [authenticationKey] 38 | ) 39 | EXEC [procfwk].[GetWorkerAuthDetails] 40 | @ExecutionId = @ExecutionId, 41 | @StageId = @StageId, 42 | @PipelineId = @PipelineId; 43 | 44 | --get main worker details 45 | INSERT INTO @WorkerDetails 46 | ( 47 | [pipelineName], 48 | [orchestratorName], 49 | [orchestratorType], 50 | [resourceGroupName] 51 | ) 52 | EXEC [procfwk].[GetWorkerPipelineDetails] 53 | @ExecutionId = @ExecutionId, 54 | @StageId = @StageId, 55 | @PipelineId = @PipelineId; 56 | 57 | --return all details 58 | SELECT 59 | ad.[tenantId], 60 | ad.[applicationId], 61 | ad.[authenticationKey], 62 | ad.[subscriptionId], 63 | d.[resourceGroupName], 64 | d.[orchestratorName], 65 | d.[orchestratorType], 66 | d.[pipelineName] 67 | FROM 68 | @WorkerDetails d 69 | CROSS JOIN @WorkerAuthDetails ad; 70 | END; -------------------------------------------------------------------------------- /Synapse/pipeline/Email Sender.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Email Sender", 3 | "properties": { 4 | "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", 5 | "activities": [ 6 | { 7 | "name": "Send Email", 8 | "description": "Use an Azure Function to perform an SMTP client email send operation.", 9 | "type": "AzureFunctionActivity", 10 | "dependsOn": [], 11 | "policy": { 12 | "timeout": "0.00:10:00", 13 | "retry": 0, 14 | "retryIntervalInSeconds": 30, 15 | "secureOutput": false, 16 | "secureInput": false 17 | }, 18 | "userProperties": [], 19 | "typeProperties": { 20 | "functionName": "SendEmail", 21 | "method": "POST", 22 | "body": { 23 | "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", 24 | "type": "Expression" 25 | } 26 | }, 27 | "linkedServiceName": { 28 | "referenceName": "FrameworkFunctions", 29 | "type": "LinkedServiceReference" 30 | } 31 | } 32 | ], 33 | "parameters": { 34 | "Recipients": { 35 | "type": "string" 36 | }, 37 | "CcRecipients": { 38 | "type": "string" 39 | }, 40 | "BccRecipients": { 41 | "type": "string" 42 | }, 43 | "Subject": { 44 | "type": "string" 45 | }, 46 | "Body": { 47 | "type": "string" 48 | }, 49 | "Importance": { 50 | "type": "string" 51 | } 52 | }, 53 | "folder": { 54 | "name": "_ProcFwk/_ProcFwkUtils" 55 | }, 56 | "annotations": [ 57 | "procfwk", 58 | "Utils" 59 | ] 60 | } 61 | } -------------------------------------------------------------------------------- /DataFactory/pipeline/Email Sender.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Email Sender", 3 | "properties": { 4 | "description": "Provide a simple abstract over the send email function with request body item exposed as pipeline parameters.", 5 | "activities": [ 6 | { 7 | "name": "Send Email", 8 | "description": "Use an Azure Function to perform an SMTP client email send operation.", 9 | "type": "AzureFunctionActivity", 10 | "dependsOn": [], 11 | "policy": { 12 | "timeout": "0.00:10:00", 13 | "retry": 0, 14 | "retryIntervalInSeconds": 30, 15 | "secureOutput": false, 16 | "secureInput": false 17 | }, 18 | "userProperties": [], 19 | "typeProperties": { 20 | "functionName": "SendEmail", 21 | "method": "POST", 22 | "body": { 23 | "value": "{\n\"emailRecipients\": \"@{pipeline().parameters.Recipients}\",\n\"emailCcRecipients\": \"@{pipeline().parameters.CcRecipients}\",\n\"emailBccRecipients\": \"@{pipeline().parameters.BccRecipients}\",\n\"emailSubject\": \"@{pipeline().parameters.Subject}\",\n\"emailBody\": \"@{pipeline().parameters.Body}\",\n\"emailImportance\": \"@{pipeline().parameters.Importance}\"\n}", 24 | "type": "Expression" 25 | } 26 | }, 27 | "linkedServiceName": { 28 | "referenceName": "FrameworkFunctions", 29 | "type": "LinkedServiceReference" 30 | } 31 | } 32 | ], 33 | "parameters": { 34 | "Recipients": { 35 | "type": "string" 36 | }, 37 | "CcRecipients": { 38 | "type": "string" 39 | }, 40 | "BccRecipients": { 41 | "type": "string" 42 | }, 43 | "Subject": { 44 | "type": "string" 45 | }, 46 | "Body": { 47 | "type": "string" 48 | }, 49 | "Importance": { 50 | "type": "string" 51 | } 52 | }, 53 | "folder": { 54 | "name": "_ProcFwk/_ProcFwkUtils" 55 | }, 56 | "annotations": [ 57 | "procfwk", 58 | "Utils" 59 | ] 60 | } 61 | } -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenBatchExecutionsForSingleBatch.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenBatchExecutionsForSingleBatch 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithBatchExecutionHandling() 21 | .WithStagesEnabled() 22 | .WithPipelinesEnabled() 23 | .WithParameter("BatchName", "Hourly"); 24 | await _helper.RunPipeline(); 25 | } 26 | 27 | #region Functional tests 28 | 29 | [Test] 30 | public void ThenPipelineOutcomeIsSucceeded() 31 | { 32 | _helper.RunOutcome.Should().Be("Succeeded"); 33 | } 34 | 35 | [Test] 36 | public void ThenCurrentExecutionTableIsEmpty() 37 | { 38 | _helper.RowCount("procfwk.CurrentExecution").Should().Be(0); 39 | } 40 | 41 | [Test] 42 | public void ThenFourExecutionLogSuccessRecords() 43 | { 44 | _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(4); 45 | } 46 | 47 | [Test] 48 | public void ThenOneBatchExecutionSuccessRecord() 49 | { 50 | _helper.RowCount("procfwk.BatchExecution", where: "BatchStatus", equals: "Success").Should().Be(1); 51 | } 52 | #endregion 53 | 54 | [OneTimeTearDown] 55 | public void TearDown() 56 | { 57 | _helper?.TearDown(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenAlreadyRunning.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenAlreadyRunning 9 | { 10 | private ParentHelper _helperRunOne; 11 | private ParentHelper _helperRunTwo; 12 | 13 | [OneTimeSetUp] 14 | public async Task WhenPipelineRun() 15 | { 16 | _helperRunOne = new ParentHelper() 17 | .WithBasicMetadata() 18 | .WithTenantAndSubscriptionIds() 19 | .WithSPNInDatabase("FrameworkFactory") 20 | .WithEmptyExecutionTables() 21 | .WithoutSimulatedError() 22 | .WithFailureHandling("Simple"); 23 | 24 | _helperRunTwo = new ParentHelper(); 25 | 26 | var firstRun = _helperRunOne.RunPipeline(); 27 | var secondRun = _helperRunTwo.RunPipeline(15000); //15 second delay in run 28 | 29 | await Task.WhenAll(firstRun, secondRun); 30 | } 31 | 32 | [Test] 33 | public void ThenFirstPipelineOutcomeIsSucceeded() 34 | { 35 | _helperRunOne.RunOutcome.Should().Be("Succeeded"); 36 | } 37 | 38 | [Test] 39 | public void ThenSecondPipelineOutcomeIsFailed() 40 | { 41 | _helperRunTwo.RunOutcome.Should().Be("Failed"); 42 | } 43 | 44 | [Test] 45 | public void ThenCurrentExecutionTableIsEmpty() 46 | { 47 | _helperRunOne.RowCount("procfwk.CurrentExecution").Should().Be(0); 48 | } 49 | 50 | [Test] 51 | public void ThenElevenExecutionLogRecords() 52 | { 53 | _helperRunOne.RowCount("procfwk.ExecutionLog").Should().Be(11); 54 | } 55 | 56 | [OneTimeTearDown] 57 | public void TearDown() 58 | { 59 | _helperRunOne?.TearDown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/SetDefaultPipelines.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[SetDefaultPipelines] 2 | AS 3 | BEGIN 4 | DECLARE @Pipelines TABLE 5 | ( 6 | [OrchestratorId] [INT] NOT NULL, 7 | [StageId] [INT] NOT NULL, 8 | [PipelineName] [NVARCHAR](200) NOT NULL, 9 | [LogicalPredecessorId] [INT] NULL, 10 | [Enabled] [BIT] NOT NULL 11 | ) 12 | 13 | INSERT @Pipelines 14 | ( 15 | [OrchestratorId], 16 | [StageId], 17 | [PipelineName], 18 | [LogicalPredecessorId], 19 | [Enabled] 20 | ) 21 | VALUES 22 | (1,1 ,'Wait 1' ,NULL ,1), 23 | (1,1 ,'Wait 2' ,NULL ,1), 24 | (1,1 ,'Intentional Error' ,NULL ,1), 25 | (1,1 ,'Wait 3' ,NULL ,1), 26 | (1,2 ,'Wait 4' ,NULL ,1), 27 | (1,2 ,'Wait 5' ,1 ,1), 28 | (1,2 ,'Wait 6' ,1 ,1), 29 | (1,2 ,'Wait 7' ,NULL ,1), 30 | (1,3 ,'Wait 8' ,1 ,1), 31 | (1,3 ,'Wait 9' ,6 ,1), 32 | (1,4 ,'Wait 10' ,9 ,1), 33 | --speed 34 | (1,5 ,'Wait 1' ,NULL ,0), 35 | (1,5 ,'Wait 2' ,NULL ,0), 36 | (1,5 ,'Wait 3' ,NULL ,0), 37 | (1,5 ,'Wait 4' ,NULL ,0), 38 | --synapse 39 | (5,1 ,'Wait 1' ,NULL ,1), 40 | (5,1 ,'Wait 2' ,NULL ,1), 41 | (5,1 ,'Wait 3' ,NULL ,1), 42 | (5,1 ,'Wait 4' ,NULL ,1); 43 | 44 | MERGE INTO [procfwk].[Pipelines] AS tgt 45 | USING 46 | @Pipelines AS src 47 | ON tgt.[OrchestratorId] = src.[OrchestratorId] 48 | AND tgt.[PipelineName] = src.[PipelineName] 49 | AND tgt.[StageId] = src.[StageId] 50 | WHEN MATCHED THEN 51 | UPDATE 52 | SET 53 | tgt.[LogicalPredecessorId] = src.[LogicalPredecessorId], 54 | tgt.[Enabled] = src.[Enabled] 55 | WHEN NOT MATCHED BY TARGET THEN 56 | INSERT 57 | ( 58 | [OrchestratorId], 59 | [StageId], 60 | [PipelineName], 61 | [LogicalPredecessorId], 62 | [Enabled] 63 | ) 64 | VALUES 65 | ( 66 | src.[OrchestratorId], 67 | src.[StageId], 68 | src.[PipelineName], 69 | src.[LogicalPredecessorId], 70 | src.[Enabled] 71 | ) 72 | WHEN NOT MATCHED BY SOURCE THEN 73 | DELETE; 74 | END; -------------------------------------------------------------------------------- /DataFactory/pipeline/Throw Exception.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Throw Exception", 3 | "properties": { 4 | "description": "Provide a simple way of throwing an exception within Data Factory using TSQL error handling.", 5 | "activities": [ 6 | { 7 | "name": "Raise Error Backup", 8 | "description": "Using a SQL database to raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", 9 | "type": "Lookup", 10 | "dependsOn": [ 11 | { 12 | "activity": "Raise Error", 13 | "dependencyConditions": [ 14 | "Failed" 15 | ] 16 | } 17 | ], 18 | "policy": { 19 | "timeout": "0.00:10:00", 20 | "retry": 0, 21 | "retryIntervalInSeconds": 30, 22 | "secureOutput": false, 23 | "secureInput": false 24 | }, 25 | "userProperties": [], 26 | "typeProperties": { 27 | "source": { 28 | "type": "AzureSqlSource", 29 | "sqlReaderQuery": { 30 | "value": "RAISERROR('@{pipeline().parameters.Message}',16,1);", 31 | "type": "Expression" 32 | }, 33 | "queryTimeout": "02:00:00", 34 | "partitionOption": "None" 35 | }, 36 | "dataset": { 37 | "referenceName": "GetSetMetadata", 38 | "type": "DatasetReference" 39 | }, 40 | "firstRowOnly": false 41 | } 42 | }, 43 | { 44 | "name": "Raise Error", 45 | "description": "Using newer native activity raise an error/exception but wrapped up as a data factory pipeline. Error message information exposed as a pipeline parameter.", 46 | "type": "Fail", 47 | "dependsOn": [], 48 | "userProperties": [], 49 | "typeProperties": { 50 | "message": { 51 | "value": "@pipeline().parameters.Message", 52 | "type": "Expression" 53 | }, 54 | "errorCode": "16" 55 | } 56 | } 57 | ], 58 | "parameters": { 59 | "Message": { 60 | "type": "string" 61 | } 62 | }, 63 | "folder": { 64 | "name": "_ProcFwk/_ProcFwkUtils" 65 | }, 66 | "annotations": [ 67 | "procfwk", 68 | "Utils" 69 | ] 70 | } 71 | } -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenNoFailureHandling.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenNoFailureHandling 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithSimulatedError() 21 | .WithFailureHandling("None"); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Functional tests 26 | 27 | [Test, Order(1)] 28 | public void ThenPipelineOutcomeIsFailed() 29 | { 30 | _helper.RunOutcome.Should().Be("Failed"); 31 | } 32 | 33 | [Test, Order(2)] 34 | public void ThenTenExecutionsSucceeded() 35 | { 36 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(10); 37 | } 38 | 39 | [Test, Order(3)] 40 | public void ThenOneExecutionFailed() 41 | { 42 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); 43 | } 44 | 45 | [Test, Order(4)] 46 | public void ThenOneExecutionLogRecord() 47 | { 48 | _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); 49 | } 50 | 51 | [Test, Order(5)] 52 | public void ThenTwoErrorLogRecords() 53 | { 54 | _helper.RowCount("procfwk.ErrorLog").Should().Be(2); 55 | } 56 | 57 | #endregion 58 | 59 | [OneTimeTearDown] 60 | public void TearDown() 61 | { 62 | _helper?.TearDown(); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/DropLegacyObjects.sql: -------------------------------------------------------------------------------- 1 | IF OBJECT_ID(N'tempdb..#DropLegacyObjects') IS NOT NULL DROP PROCEDURE #DropLegacyObjects; 2 | GO 3 | 4 | CREATE PROCEDURE #DropLegacyObjects 5 | ( 6 | @ObjectName NVARCHAR(128), 7 | @ObjectType CHAR(2) 8 | ) 9 | AS 10 | BEGIN 11 | DECLARE @DDLType NVARCHAR(128) 12 | 13 | IF EXISTS 14 | ( 15 | SELECT 16 | * 17 | FROM 18 | sys.objects o 19 | INNER JOIN sys.schemas s 20 | ON o.[schema_id] = s.[schema_id] 21 | WHERE 22 | o.[Name] = @ObjectName 23 | AND s.[name] = 'procfwk' 24 | AND o.[type] = @ObjectType 25 | ) 26 | BEGIN 27 | SELECT 28 | @DDLType = CASE @ObjectType 29 | WHEN 'P' THEN 'PROCEDURE' 30 | WHEN 'V' THEN 'VIEW' 31 | WHEN 'FN' THEN 'FUNCTION' 32 | END; 33 | 34 | EXEC('DROP ' + @DDLType + ' [procfwk].[' + @ObjectName + '];') 35 | END; 36 | END; 37 | GO 38 | 39 | EXEC #DropLegacyObjects 'AddProperty', 'P'; 40 | EXEC #DropLegacyObjects 'GetExecutionDetails', 'P'; 41 | EXEC #DropLegacyObjects 'AddRecipientPipelineAlerts', 'P'; 42 | EXEC #DropLegacyObjects 'DeleteRecipientAlerts', 'P'; 43 | EXEC #DropLegacyObjects 'CheckStageAndPiplineIntegrity', 'P'; 44 | EXEC #DropLegacyObjects 'AddPipelineDependant', 'P'; 45 | EXEC #DropLegacyObjects 'AddServicePrincipalWrapper', 'P'; 46 | EXEC #DropLegacyObjects 'AddServicePrincipalUrls', 'P'; 47 | EXEC #DropLegacyObjects 'AddServicePrincipal', 'P'; 48 | EXEC #DropLegacyObjects 'DeleteServicePrincipal', 'P'; 49 | EXEC #DropLegacyObjects 'CheckForValidURL', 'FN'; 50 | EXEC #DropLegacyObjects 'PipelineDependencyChains', 'V'; 51 | EXEC #DropLegacyObjects 'AverageStageDuration', 'V'; 52 | EXEC #DropLegacyObjects 'CompleteExecutionErrorLog', 'V'; 53 | EXEC #DropLegacyObjects 'CompleteExecutionLog', 'V'; 54 | EXEC #DropLegacyObjects 'CurrentExecutionSummary', 'V'; 55 | EXEC #DropLegacyObjects 'LastExecution', 'V'; 56 | EXEC #DropLegacyObjects 'LastExecutionSummary', 'V'; 57 | EXEC #DropLegacyObjects 'WorkerParallelismOverTime', 'V'; 58 | 59 | --replaced with new precursor concept in v1.8.5: 60 | IF OBJECT_ID(N'[dbo].[SetRandomWaitValues]') IS NOT NULL DROP PROCEDURE [dbo].[SetRandomWaitValues]; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandlingAndRestart.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenSimpleFailureHandlingAndRestart 9 | { 10 | private ParentHelper _helperFirstRun; 11 | private ParentHelper _helperRestartRun; 12 | 13 | [OneTimeSetUp] 14 | public async Task WhenPipelineRun() 15 | { 16 | _helperFirstRun = new ParentHelper() 17 | .WithBasicMetadata() 18 | .WithTenantAndSubscriptionIds() 19 | .WithSPNInDatabase("FrameworkFactory") 20 | .WithEmptyExecutionTables() 21 | .WithSimulatedError() 22 | .WithFailureHandling("Simple"); 23 | await _helperFirstRun.RunPipeline(); 24 | 25 | _helperRestartRun = new ParentHelper() 26 | .WithoutSimulatedError(); 27 | await _helperRestartRun.RunPipeline(); 28 | } 29 | 30 | #region Functional tests 31 | 32 | [Test] 33 | public void ThenPipelineOutcomeIsSucceeded() 34 | { 35 | _helperRestartRun.RunOutcome.Should().Be("Succeeded"); 36 | } 37 | 38 | [Test] 39 | public void ThenOneExecutionLogFailedRecord() 40 | { 41 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); 42 | } 43 | [Test] 44 | public void ThenElevenExecutionLogSuccessRecord() 45 | { 46 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); 47 | } 48 | 49 | [Test] 50 | public void ThenTwoErrorLogRecords() 51 | { 52 | _helperRestartRun.RowCount("procfwk.ErrorLog").Should().Be(2); 53 | } 54 | #endregion 55 | 56 | [OneTimeTearDown] 57 | public void TearDown() 58 | { 59 | _helperRestartRun?.TearDown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/AddPipelineDependant.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[AddPipelineDependant] 2 | ( 3 | @PipelineName NVARCHAR(200), 4 | @DependantPipelineName NVARCHAR(200) 5 | ) 6 | AS 7 | BEGIN 8 | SET NOCOUNT ON; 9 | 10 | DECLARE @PipelineId INT; 11 | DECLARE @DependantPipelineId INT; 12 | 13 | --get pipeline ids 14 | SELECT 15 | @PipelineId = [PipelineId] 16 | FROM 17 | [procfwk].[Pipelines] 18 | WHERE 19 | [PipelineName] = @PipelineName; 20 | 21 | SELECT 22 | @DependantPipelineId = [PipelineId] 23 | FROM 24 | [procfwk].[Pipelines] 25 | WHERE 26 | [PipelineName] = @DependantPipelineName; 27 | 28 | --defensive checks 29 | IF @PipelineId IS NULL 30 | BEGIN 31 | RAISERROR('Pipeline not found in pipelines table.', 16,1); 32 | RETURN 0; 33 | END; 34 | 35 | IF @DependantPipelineId IS NULL 36 | BEGIN 37 | RAISERROR('Dependant pipeline not found in pipelines table.', 16,1); 38 | RETURN 0; 39 | END; 40 | 41 | IF @PipelineId = @DependantPipelineId 42 | BEGIN 43 | RAISERROR('Pipeline cannot be dependant on itself.', 16,1); 44 | RETURN 0; 45 | END; 46 | 47 | IF EXISTS 48 | ( 49 | SELECT 50 | * 51 | FROM 52 | [procfwk].[Pipelines] pp 53 | INNER JOIN [procfwk].[Pipelines] dp 54 | ON dp.[PipelineId] = @DependantPipelineId 55 | WHERE 56 | pp.[PipelineId] = @PipelineId 57 | AND pp.[StageId] = dp.[StageId] 58 | ) 59 | BEGIN 60 | RAISERROR('Pipeline and dependent pipeline cannot be in the same execution stage.', 16,1); 61 | RETURN 0; 62 | END; 63 | 64 | --final soft check and insert 65 | IF EXISTS 66 | ( 67 | SELECT 68 | * 69 | FROM 70 | [procfwk].[PipelineDependencies] 71 | WHERE 72 | [PipelineId] = @PipelineId 73 | AND [DependantPipelineId] = @DependantPipelineId 74 | ) 75 | BEGIN 76 | PRINT 'Dependency already exists. Nothing added.' 77 | RETURN 0; 78 | END 79 | ELSE 80 | BEGIN 81 | INSERT INTO [procfwk].[PipelineDependencies] 82 | ( 83 | [PipelineId], 84 | [DependantPipelineId] 85 | ) 86 | VALUES 87 | ( 88 | @PipelineId, 89 | @DependantPipelineId 90 | ) 91 | END; 92 | END; 93 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenDependencyChainFailureHandlingAndRestart.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenDependencyChainFailureHandlingAndRestart 9 | { 10 | private ParentHelper _helperFirstRun; 11 | private ParentHelper _helperRestartRun; 12 | 13 | [OneTimeSetUp] 14 | public async Task WhenPipelineRun() 15 | { 16 | _helperFirstRun = new ParentHelper() 17 | .WithBasicMetadata() 18 | .WithTenantAndSubscriptionIds() 19 | .WithSPNInDatabase("FrameworkFactory") 20 | .WithEmptyExecutionTables() 21 | .WithSimulatedError() 22 | .WithFailureHandling("DependencyChain"); 23 | await _helperFirstRun.RunPipeline(); 24 | 25 | _helperRestartRun = new ParentHelper() 26 | .WithoutSimulatedError(); 27 | await _helperRestartRun.RunPipeline(); 28 | } 29 | 30 | #region Functional tests 31 | 32 | [Test] 33 | public void ThenPipelineOutcomeIsSucceeded() 34 | { 35 | _helperRestartRun.RunOutcome.Should().Be("Succeeded"); 36 | } 37 | 38 | [Test] 39 | public void ThenOneExecutionLogFailedRecord() 40 | { 41 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); 42 | } 43 | [Test] 44 | public void ThenElevenExecutionLogSuccessRecord() 45 | { 46 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); 47 | } 48 | 49 | [Test] 50 | public void ThenTwoErrorLogRecords() 51 | { 52 | _helperRestartRun.RowCount("procfwk.ErrorLog").Should().Be(2); 53 | } 54 | #endregion 55 | 56 | [OneTimeTearDown] 57 | public void TearDown() 58 | { 59 | _helperFirstRun?.TearDown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/Utilities/GivenCheckForRunningPipeline.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | using FactoryTesting.Helpers; 6 | 7 | namespace FactoryTesting.Pipelines.Utilities 8 | { 9 | public class GivenCheckForRunningPipeline 10 | { 11 | private UtilitiesHelper _helper; 12 | 13 | [OneTimeSetUp] 14 | public async Task WhenPipelineRun() 15 | { 16 | _helper = new UtilitiesHelper() 17 | .WithBasicMetadata() 18 | .WithTenantAndSubscriptionIds() 19 | .WithParameter("PipelineName", "Check For Running Pipeline"); 20 | 21 | await _helper.RunPipeline("Check For Running Pipeline"); 22 | } 23 | 24 | [Test] 25 | public void ThenPipelineOutcomeIsFailed() 26 | { 27 | _helper.RunOutcome.Should().Be("Failed"); 28 | } 29 | 30 | [Test] 31 | public async Task ThenActivityShouldReturnOneFilteredItemCount() 32 | { 33 | var filteredCount = await _helper.GetActivityOutput("Filter Running Pipelines", "$.FilteredItemsCount"); 34 | int.Parse(filteredCount).Should().Be(1); 35 | } 36 | 37 | [Test] 38 | public async Task ThenActivityShouldReturnMatchingSubscriptionId() 39 | { 40 | string subSetting = _helper.GetSetting("AZURE_SUBSCRIPTION_ID"); 41 | 42 | var subscriptionId = await _helper.GetActivityOutput("Set Subscription Id", "$.value"); 43 | subscriptionId.Should().Equals(subSetting.ToString()); 44 | } 45 | 46 | [Test] 47 | public async Task ThenActivityShouldReturnMatchingResourceGroup() 48 | { 49 | string rgSsetting = _helper.GetSetting("DataFactoryResourceGroup"); 50 | 51 | var resourceGroupName = await _helper.GetActivityOutput("Set Resource Group Name", "$.value"); 52 | resourceGroupName.Should().Equals(rgSsetting.ToString()); 53 | } 54 | 55 | [OneTimeTearDown] 56 | public void TearDown() 57 | { 58 | _helper?.TearDown(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /MetadataDB/procfwkHelpers/Stored Procedures/GetExecutionDetails.sql: -------------------------------------------------------------------------------- 1 | CREATE PROCEDURE [procfwkHelpers].[GetExecutionDetails] 2 | ( 3 | @LocalExecutionId UNIQUEIDENTIFIER = NULL 4 | ) 5 | AS 6 | BEGIN 7 | 8 | --Get last execution ID 9 | IF @LocalExecutionId IS NULL 10 | BEGIN 11 | WITH maxLog AS 12 | ( 13 | SELECT 14 | MAX([LogId]) AS MaxLogId 15 | FROM 16 | [procfwk].[ExecutionLog] 17 | ) 18 | SELECT 19 | @LocalExecutionId = el1.[LocalExecutionId] 20 | FROM 21 | [procfwk].[ExecutionLog] el1 22 | INNER JOIN maxLog 23 | ON maxLog.[MaxLogId] = el1.[LogId]; 24 | END; 25 | 26 | --Execution Summary 27 | SELECT 28 | CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName] AS Stage, 29 | COUNT(0) AS RecordCount, 30 | DATEDIFF(MINUTE, MIN(el2.[StartDateTime]), MAX(el2.[EndDateTime])) DurationMinutes 31 | FROM 32 | [procfwk].[ExecutionLog] el2 33 | INNER JOIN [procfwk].[Stages] stgs 34 | ON el2.[StageId] = stgs.[StageId] 35 | WHERE 36 | el2.[LocalExecutionId] = @LocalExecutionId 37 | GROUP BY 38 | CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName] 39 | ORDER BY 40 | CAST(el2.[StageId] AS VARCHAR(5)) + ' - ' + stgs.[StageName]; 41 | 42 | --Full execution details 43 | SELECT 44 | el3.[LogId], 45 | el3.[LocalExecutionId], 46 | el3.[OrchestratorType], 47 | el3.[OrchestratorName], 48 | el3.[StageId], 49 | stgs.[StageName], 50 | el3.[PipelineId], 51 | el3.[PipelineName], 52 | el3.[StartDateTime], 53 | el3.[EndDateTime], 54 | ISNULL(DATEDIFF(MINUTE, el3.[StartDateTime], el3.[EndDateTime]),0) AS DurationMinutes, 55 | el3.[PipelineStatus], 56 | el3.[PipelineRunId], 57 | el3.[PipelineParamsUsed], 58 | errLog.[ActivityRunId], 59 | errLog.[ActivityName], 60 | errLog.[ActivityType], 61 | errLog.[ErrorCode], 62 | errLog.[ErrorType], 63 | errLog.[ErrorMessage] 64 | FROM 65 | [procfwk].[ExecutionLog] el3 66 | LEFT OUTER JOIN [procfwk].[ErrorLog] errLog 67 | ON el3.[LocalExecutionId] = errLog.[LocalExecutionId] 68 | AND el3.[PipelineRunId] = errLog.[PipelineRunId] 69 | INNER JOIN [procfwk].[Stages] stgs 70 | ON el3.[StageId] = stgs.[StageId] 71 | WHERE 72 | el3.[LocalExecutionId] = @LocalExecutionId 73 | ORDER BY 74 | el3.[PipelineId], 75 | el3.[StartDateTime]; 76 | END; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerAndRestart.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenCancelledWorkerAndRestart 9 | { 10 | private ParentHelper _helperFirstRun; 11 | private ParentHelper _helperRestartRun; 12 | 13 | [OneTimeSetUp] 14 | public async Task WhenPipelineRun() 15 | { 16 | _helperFirstRun = new ParentHelper() 17 | .WithBasicMetadata() 18 | .WithTenantAndSubscriptionIds() 19 | .WithSPNInDatabase("FrameworkFactory") 20 | .WithEmptyExecutionTables() 21 | .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits 22 | .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time 23 | .WithCancelledWorkersBlock(true) 24 | .WithFailureHandling("Simple"); 25 | 26 | var runOrchestrator = _helperFirstRun.RunPipeline(); 27 | var cancelWorker = _helperFirstRun.CancelAnyWorkerPipeline(); 28 | 29 | await Task.WhenAll(runOrchestrator, cancelWorker); 30 | 31 | _helperRestartRun = new ParentHelper(); 32 | await _helperRestartRun.RunPipeline(); 33 | } 34 | #region Integration tests 35 | 36 | [Test] 37 | public void ThenPipelineOutcomeIsSucceeded() 38 | { 39 | _helperRestartRun.RunOutcome.Should().Be("Succeeded"); 40 | } 41 | 42 | [Test] 43 | public void ThenOneExecutionLogRecord() 44 | { 45 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); 46 | } 47 | [Test] 48 | public void ThenElevenExecutionsSucceeded() 49 | { 50 | _helperRestartRun.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Success").Should().Be(11); 51 | } 52 | #endregion 53 | 54 | [OneTimeTearDown] 55 | public void TearDown() 56 | { 57 | _helperRestartRun?.TearDown(); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerThatDoesntBlock.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenCancelledWorkerThatDoesntBlock 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits 21 | .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time 22 | .WithCancelledWorkersBlock(false) 23 | .WithFailureHandling("Simple"); 24 | 25 | var runOrchestrator = _helper.RunPipeline(); 26 | var cancelWorker = _helper.CancelIntentionalErrorWorkerPipeline(); //specific worker 27 | 28 | await Task.WhenAll(runOrchestrator, cancelWorker); 29 | } 30 | #region Integration tests 31 | 32 | [Test] 33 | public void ThenPipelineOutcomeIsFailed() 34 | { 35 | _helper.RunOutcome.Should().Be("Failed"); 36 | } 37 | 38 | [Test] 39 | public void ThenOneExecutionsCancelled() 40 | { 41 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); 42 | } 43 | 44 | [Test] 45 | public void ThenOneExecutionLogRecord() 46 | { 47 | _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); 48 | } 49 | [Test] 50 | public void ThenTenExecutionsSucceeded() 51 | { 52 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(10); 53 | } 54 | #endregion 55 | 56 | [OneTimeTearDown] 57 | public void TearDown() 58 | { 59 | _helper?.TearDown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /MetadataDB/Scripts/Metadata/DropLegacyTables.sql: -------------------------------------------------------------------------------- 1 | --PipelineProcesses 2 | IF EXISTS 3 | ( 4 | SELECT 5 | * 6 | FROM 7 | sys.objects o 8 | INNER JOIN sys.schemas s 9 | ON o.[schema_id] = s.[schema_id] 10 | WHERE 11 | o.[name] = 'PipelineProcesses' 12 | AND s.[name] = 'procfwk' 13 | AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability 14 | ) 15 | BEGIN 16 | --drop just to avoid constraints 17 | IF OBJECT_ID(N'[procfwk].[PipelineParameters]') IS NOT NULL DROP TABLE [procfwk].[PipelineParameters]; 18 | IF OBJECT_ID(N'[procfwk].[PipelineAuthLink]') IS NOT NULL DROP TABLE [procfwk].[PipelineAuthLink]; 19 | 20 | SELECT * INTO [dbo].[zz_PipelineProcesses] FROM [procfwk].[PipelineProcesses]; 21 | 22 | DROP TABLE [procfwk].[PipelineProcesses]; 23 | END 24 | 25 | --ProcessingStageDetails 26 | IF EXISTS 27 | ( 28 | SELECT 29 | * 30 | FROM 31 | sys.objects o 32 | INNER JOIN sys.schemas s 33 | ON o.[schema_id] = s.[schema_id] 34 | WHERE 35 | o.[name] = 'ProcessingStageDetails' 36 | AND s.[name] = 'procfwk' 37 | AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability 38 | ) 39 | BEGIN 40 | SELECT * INTO [dbo].[zz_ProcessingStageDetails] FROM [procfwk].[ProcessingStageDetails]; 41 | 42 | DROP TABLE [procfwk].[ProcessingStageDetails]; 43 | END; 44 | 45 | --DataFactoryDetails 46 | IF EXISTS 47 | ( 48 | SELECT 49 | * 50 | FROM 51 | sys.objects o 52 | INNER JOIN sys.schemas s 53 | ON o.[schema_id] = s.[schema_id] 54 | WHERE 55 | o.[name] = 'DataFactoryDetails' 56 | AND s.[name] = 'procfwk' 57 | AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability 58 | ) 59 | BEGIN 60 | SELECT * INTO [dbo].[zz_DataFactoryDetails] FROM [procfwk].[DataFactoryDetails]; 61 | 62 | DROP TABLE [procfwk].[DataFactoryDetails]; 63 | END; 64 | 65 | --DataFactorys 66 | IF EXISTS 67 | ( 68 | SELECT 69 | * 70 | FROM 71 | sys.objects o 72 | INNER JOIN sys.schemas s 73 | ON o.[schema_id] = s.[schema_id] 74 | WHERE 75 | o.[name] = 'DataFactorys' 76 | AND s.[name] = 'procfwk' 77 | AND o.[type] = 'U' --Check for tables as created synonyms to support backwards compatability 78 | ) 79 | BEGIN 80 | SELECT * INTO [dbo].[zz_DataFactorys] FROM [procfwk].[DataFactorys]; 81 | END; -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenCancelledWorkerInOneExecutionStage.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenCancelledWorkerInOneExecutionStage 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithoutPrecursorObject() //done to ensure 2min waits are used, not example precursor waits 21 | .With2MinWaitsOnWorkers() //to ensure the cancel call has enough time 22 | .WithCancelledWorkersBlock(false) 23 | .WithOnlyStageOneEnabled(); 24 | 25 | var runOrchestrator = _helper.RunPipeline(); 26 | var cancelWorker = _helper.CancelAnyWorkerPipeline(); //any worker 27 | 28 | await Task.WhenAll(runOrchestrator, cancelWorker); 29 | } 30 | #region Integration tests 31 | 32 | [Test] 33 | public void ThenPipelineOutcomeIsFailed() 34 | { 35 | _helper.RunOutcome.Should().Be("Failed"); 36 | } 37 | 38 | [Test] 39 | public void ThenOneExecutionsCancelled() 40 | { 41 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); 42 | } 43 | 44 | [Test] 45 | public void ThenOneExecutionLogRecord() 46 | { 47 | _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Cancelled").Should().Be(1); 48 | } 49 | [Test] 50 | public void ThenThreeExecutionsSucceeded() 51 | { 52 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); 53 | } 54 | #endregion 55 | 56 | [OneTimeTearDown] 57 | public void TearDown() 58 | { 59 | _helper?.TearDown(); 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /FactoryTesting/Pipelines/02-Parent/GivenSimpleFailureHandling.cs: -------------------------------------------------------------------------------- 1 | using FluentAssertions; 2 | using NUnit.Framework; 3 | using NUnit.Framework.Internal; 4 | using System.Threading.Tasks; 5 | 6 | namespace FactoryTesting.Pipelines.Parent 7 | { 8 | public class GivenSimpleFailureHandling 9 | { 10 | private ParentHelper _helper; 11 | 12 | [OneTimeSetUp] 13 | public async Task WhenPipelineRun() 14 | { 15 | _helper = new ParentHelper() 16 | .WithBasicMetadata() 17 | .WithTenantAndSubscriptionIds() 18 | .WithSPNInDatabase("FrameworkFactory") 19 | .WithEmptyExecutionTables() 20 | .WithSimulatedError() 21 | .WithFailureHandling("Simple"); 22 | await _helper.RunPipeline(); 23 | } 24 | 25 | #region Functional tests 26 | 27 | [Test, Order(1)] 28 | public void ThenPipelineOutcomeIsFailed() 29 | { 30 | _helper.RunOutcome.Should().Be("Failed"); 31 | } 32 | 33 | [Test, Order(2)] 34 | public void ThenThreeExecutionsSucceeded() 35 | { 36 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Success").Should().Be(3); 37 | } 38 | 39 | [Test, Order(3)] 40 | public void ThenOneExecutionFailed() 41 | { 42 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Failed").Should().Be(1); 43 | } 44 | 45 | [Test, Order(4)] 46 | public void ThenSevenExecutionsBlocked() 47 | { 48 | _helper.RowCount("procfwk.CurrentExecution", where: "PipelineStatus", equals: "Blocked").Should().Be(7); 49 | } 50 | 51 | [Test, Order(5)] 52 | public void ThenOneExecutionLogRecord() 53 | { 54 | _helper.RowCount("procfwk.ExecutionLog", where: "PipelineStatus", equals: "Failed").Should().Be(1); 55 | } 56 | 57 | [Test, Order(6)] 58 | public void ThenTwoErrorLogRecords() 59 | { 60 | _helper.RowCount("procfwk.ErrorLog").Should().Be(2); 61 | } 62 | #endregion 63 | 64 | [OneTimeTearDown] 65 | public void TearDown() 66 | { 67 | _helper?.TearDown(); 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /DataFactory/pipeline/Intentional Error.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Intentional Error", 3 | "properties": { 4 | "description": "Used just so the procfwk has something to call during development.", 5 | "activities": [ 6 | { 7 | "name": "Wait1", 8 | "description": "Framework development worker simulator.", 9 | "type": "Wait", 10 | "dependsOn": [], 11 | "userProperties": [], 12 | "typeProperties": { 13 | "waitTimeInSeconds": { 14 | "value": "@pipeline().parameters.WaitTime", 15 | "type": "Expression" 16 | } 17 | } 18 | }, 19 | { 20 | "name": "Raise Errors or Not", 21 | "description": "Framework development worker simulator.", 22 | "type": "IfCondition", 23 | "dependsOn": [ 24 | { 25 | "activity": "Wait1", 26 | "dependencyConditions": [ 27 | "Succeeded" 28 | ] 29 | } 30 | ], 31 | "userProperties": [], 32 | "typeProperties": { 33 | "expression": { 34 | "value": "@equals(pipeline().parameters.RaiseErrors,'true')", 35 | "type": "Expression" 36 | }, 37 | "ifTrueActivities": [ 38 | { 39 | "name": "Call Fail Procedure", 40 | "type": "SqlServerStoredProcedure", 41 | "dependsOn": [], 42 | "policy": { 43 | "timeout": "0.00:10:00", 44 | "retry": 0, 45 | "retryIntervalInSeconds": 30, 46 | "secureOutput": false, 47 | "secureInput": false 48 | }, 49 | "userProperties": [], 50 | "typeProperties": { 51 | "storedProcedureName": "[dbo].[FailProcedure]", 52 | "storedProcedureParameters": { 53 | "RaiseError": { 54 | "value": { 55 | "value": "@pipeline().parameters.RaiseErrors", 56 | "type": "Expression" 57 | }, 58 | "type": "String" 59 | } 60 | } 61 | }, 62 | "linkedServiceName": { 63 | "referenceName": "SupportDatabase", 64 | "type": "LinkedServiceReference" 65 | } 66 | } 67 | ] 68 | } 69 | } 70 | ], 71 | "parameters": { 72 | "RaiseErrors": { 73 | "type": "string", 74 | "defaultValue": "false" 75 | }, 76 | "WaitTime": { 77 | "type": "int", 78 | "defaultValue": 5 79 | } 80 | }, 81 | "folder": { 82 | "name": "_Workers" 83 | }, 84 | "annotations": [ 85 | "_ProcFwkWorker" 86 | ] 87 | } 88 | } --------------------------------------------------------------------------------