├── README.md ├── bentalebdf ├── globalParameters │ └── bentalebdf_GlobalParameters.json ├── ARMTemplateParametersForFactory.json ├── linkedTemplates │ ├── ArmTemplateParameters_master.json │ ├── ArmTemplate_master.json │ └── ArmTemplate_0.json └── ARMTemplateForFactory.json ├── publish_config.json ├── bentalebdatafactory ├── globalParameters │ └── bentalebdatafactory_GlobalParameters.json ├── ARMTemplateParametersForFactory.json ├── linkedTemplates │ ├── ArmTemplateParameters_master.json │ ├── ArmTemplate_master.json │ └── ArmTemplate_0.json └── ARMTemplateForFactory.json ├── factory └── bentalebdatafactory.json ├── trigger └── AutomatisationTrigger.json ├── linkedService └── bentalebDataBricksLink.json └── pipeline └── automateTransforming.json /README.md: -------------------------------------------------------------------------------- 1 | # AzurePublicTransportAutomatisation -------------------------------------------------------------------------------- /bentalebdf/globalParameters/bentalebdf_GlobalParameters.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /publish_config.json: -------------------------------------------------------------------------------- 1 | {"publishBranch":"main","enableGitComment":true} -------------------------------------------------------------------------------- /bentalebdatafactory/globalParameters/bentalebdatafactory_GlobalParameters.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /factory/bentalebdatafactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bentalebdatafactory", 3 | "location": "eastus2", 4 | "identity": { 5 | "type": "SystemAssigned", 6 | "principalId": "f2d8d299-b035-430f-b9a1-fec1029ae2ae", 7 | "tenantId": "a2e466aa-4f86-4545-b5b8-97da7c8febf3" 8 | } 9 | } -------------------------------------------------------------------------------- /bentalebdf/ARMTemplateParametersForFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "value": "bentalebdf" 7 | }, 8 | "bentalebDataBricksLink_accessToken": { 9 | "value": "" 10 | }, 11 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 12 | "value": "0925-074124-82mpisbm" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /bentalebdatafactory/ARMTemplateParametersForFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "value": "bentalebdatafactory" 7 | }, 8 | "bentalebDataBricksLink_accessToken": { 9 | "value": "" 10 | }, 11 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 12 | "value": "0927-073616-deozl5ap" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /bentalebdf/linkedTemplates/ArmTemplateParameters_master.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "value": "bentalebdf" 7 | }, 8 | "bentalebDataBricksLink_accessToken": { 9 | "value": "" 10 | }, 11 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 12 | "value": "0925-074124-82mpisbm" 13 | }, 14 | "containerUri": { 15 | "value": "" 16 | }, 17 | "containerSasToken": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /bentalebdatafactory/linkedTemplates/ArmTemplateParameters_master.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "value": "bentalebdatafactory" 7 | }, 8 | "bentalebDataBricksLink_accessToken": { 9 | "value": "" 10 | }, 11 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 12 | "value": "0927-073616-deozl5ap" 13 | }, 14 | "containerUri": { 15 | "value": "" 16 | }, 17 | "containerSasToken": { 18 | "value": "" 19 | } 20 | } 21 | } -------------------------------------------------------------------------------- /trigger/AutomatisationTrigger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AutomatisationTrigger", 3 | "properties": { 4 | "annotations": [], 5 | "runtimeState": "Started", 6 | "pipelines": [ 7 | { 8 | "pipelineReference": { 9 | "referenceName": "automateTransforming", 10 | "type": "PipelineReference" 11 | } 12 | } 13 | ], 14 | "type": "ScheduleTrigger", 15 | "typeProperties": { 16 | "recurrence": { 17 | "frequency": "Minute", 18 | "interval": 1, 19 | "startTime": "2023-09-27T13:27:00", 20 | "endTime": "2023-09-27T13:55:00", 21 | "timeZone": "Morocco Standard Time" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /linkedService/bentalebDataBricksLink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bentalebDataBricksLink", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureDatabricks", 7 | "typeProperties": { 8 | "domain": "https://adb-7197147741827979.19.azuredatabricks.net", 9 | "existingClusterId": "0927-073616-deozl5ap", 10 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQEMxOUQ0NTUyLUYxNjQtNERFNy05NzU3LTVBQTYzMTI0RjM2Ql81MTkxMGIzNC03YTQ3LTQyNDctOGI1Ny0wZjY1MTdiY2M0Y2IiDQp9" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /pipeline/automateTransforming.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "automateTransforming", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "DataTransformation", 7 | "type": "DatabricksNotebook", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "0.12:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "notebookPath": "/Repos/sbentaleb.ext@simplonformations.onmicrosoft.com/AzurePublicTransportDataIntegration/ETL/integration" 19 | }, 20 | "linkedServiceName": { 21 | "referenceName": "bentalebDataBricksLink", 22 | "type": "LinkedServiceReference" 23 | } 24 | }, 25 | { 26 | "name": "Run log", 27 | "type": "SetVariable", 28 | "dependsOn": [ 29 | { 30 | "activity": "DataTransformation", 31 | "dependencyConditions": [ 32 | "Succeeded" 33 | ] 34 | } 35 | ], 36 | "policy": { 37 | "secureOutput": true, 38 | "secureInput": false 39 | }, 40 | "userProperties": [], 41 | "typeProperties": { 42 | "variableName": "transformationInfo", 43 | "value": "@activity('DataTransformation').output.runOutput" 44 | } 45 | } 46 | ], 47 | "variables": { 48 | "transformationInfo": { 49 | "type": "String" 50 | } 51 | }, 52 | "annotations": [] 53 | } 54 | } -------------------------------------------------------------------------------- /bentalebdf/linkedTemplates/ArmTemplate_master.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdf" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0925-074124-82mpisbm" 17 | }, 18 | "containerUri": { 19 | "type": "string", 20 | "metadata": "URI to storage account containing linked ARM templates" 21 | }, 22 | "containerSasToken": { 23 | "type": "string", 24 | "metadata": "Sas token to storage account containing linked ARM templates" 25 | } 26 | }, 27 | "variables": { 28 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 29 | }, 30 | "resources": [ 31 | { 32 | "name": "bentalebdf_ArmTemplate_0", 33 | "type": "Microsoft.Resources/deployments", 34 | "apiVersion": "2018-02-01", 35 | "properties": { 36 | "mode": "incremental", 37 | "templateLink": { 38 | "uri": "[concat(parameters('containerUri'), '/ArmTemplate_0.json', parameters('containerSasToken'))]", 39 | "contentVersion": "1.0.0.0" 40 | }, 41 | "parameters": { 42 | "factoryName": { 43 | "value": "[parameters('factoryName')]" 44 | }, 45 | "bentalebDataBricksLink_accessToken": { 46 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 47 | }, 48 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 49 | "value": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 50 | } 51 | } 52 | }, 53 | "dependsOn": [] 54 | } 55 | ] 56 | } -------------------------------------------------------------------------------- /bentalebdatafactory/linkedTemplates/ArmTemplate_master.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdatafactory" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0927-073616-deozl5ap" 17 | }, 18 | "containerUri": { 19 | "type": "string", 20 | "metadata": "URI to storage account containing linked ARM templates" 21 | }, 22 | "containerSasToken": { 23 | "type": "string", 24 | "metadata": "Sas token to storage account containing linked ARM templates" 25 | } 26 | }, 27 | "variables": { 28 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 29 | }, 30 | "resources": [ 31 | { 32 | "name": "bentalebdatafactory_ArmTemplate_0", 33 | "type": "Microsoft.Resources/deployments", 34 | "apiVersion": "2018-02-01", 35 | "properties": { 36 | "mode": "incremental", 37 | "templateLink": { 38 | "uri": "[concat(parameters('containerUri'), '/ArmTemplate_0.json', parameters('containerSasToken'))]", 39 | "contentVersion": "1.0.0.0" 40 | }, 41 | "parameters": { 42 | "factoryName": { 43 | "value": "[parameters('factoryName')]" 44 | }, 45 | "bentalebDataBricksLink_accessToken": { 46 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 47 | }, 48 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 49 | "value": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 50 | } 51 | } 52 | }, 53 | "dependsOn": [] 54 | } 55 | ] 56 | } -------------------------------------------------------------------------------- /bentalebdf/ARMTemplateForFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdf" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0925-074124-82mpisbm" 17 | } 18 | }, 19 | "variables": { 20 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 21 | }, 22 | "resources": [ 23 | { 24 | "name": "[concat(parameters('factoryName'), '/automateTransforming')]", 25 | "type": "Microsoft.DataFactory/factories/pipelines", 26 | "apiVersion": "2018-06-01", 27 | "properties": { 28 | "activities": [ 29 | { 30 | "name": "Notebook1", 31 | "type": "DatabricksNotebook", 32 | "dependsOn": [], 33 | "policy": { 34 | "timeout": "0.12:00:00", 35 | "retry": 0, 36 | "retryIntervalInSeconds": 30, 37 | "secureOutput": false, 38 | "secureInput": false 39 | }, 40 | "userProperties": [], 41 | "typeProperties": { 42 | "notebookPath": "/Repos/sbentaleb.ext@simplonformations.onmicrosoft.com/AzurePublicTransportDataIntegration/ETL/integration" 43 | }, 44 | "linkedServiceName": { 45 | "referenceName": "bentalebDataBricksLink", 46 | "type": "LinkedServiceReference" 47 | } 48 | } 49 | ], 50 | "policy": { 51 | "elapsedTimeMetric": {}, 52 | "cancelAfter": {} 53 | }, 54 | "annotations": [] 55 | }, 56 | "dependsOn": [ 57 | "[concat(variables('factoryId'), '/linkedServices/bentalebDataBricksLink')]" 58 | ] 59 | }, 60 | { 61 | "name": "[concat(parameters('factoryName'), '/bentalebDataBricksLink')]", 62 | "type": "Microsoft.DataFactory/factories/linkedServices", 63 | "apiVersion": "2018-06-01", 64 | "properties": { 65 | "annotations": [], 66 | "type": "AzureDatabricks", 67 | "typeProperties": { 68 | "domain": "https://adb-2184495587427117.17.azuredatabricks.net", 69 | "accessToken": { 70 | "type": "SecureString", 71 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 72 | }, 73 | "existingClusterId": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 74 | } 75 | }, 76 | "dependsOn": [] 77 | }, 78 | { 79 | "name": "[concat(parameters('factoryName'), '/batchProcTrigger')]", 80 | "type": "Microsoft.DataFactory/factories/triggers", 81 | "apiVersion": "2018-06-01", 82 | "properties": { 83 | "annotations": [], 84 | "runtimeState": "Started", 85 | "pipelines": [ 86 | { 87 | "pipelineReference": { 88 | "referenceName": "automateTransforming", 89 | "type": "PipelineReference" 90 | }, 91 | "parameters": {} 92 | } 93 | ], 94 | "type": "ScheduleTrigger", 95 | "typeProperties": { 96 | "recurrence": { 97 | "frequency": "Minute", 98 | "interval": 3, 99 | "startTime": "2023-09-25T15:21:00", 100 | "endTime": "2023-09-25T16:28:00", 101 | "timeZone": "Morocco Standard Time" 102 | } 103 | } 104 | }, 105 | "dependsOn": [ 106 | "[concat(variables('factoryId'), '/pipelines/automateTransforming')]" 107 | ] 108 | } 109 | ] 110 | } -------------------------------------------------------------------------------- /bentalebdf/linkedTemplates/ArmTemplate_0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdf" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0925-074124-82mpisbm" 17 | } 18 | }, 19 | "variables": { 20 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 21 | }, 22 | "resources": [ 23 | { 24 | "name": "[concat(parameters('factoryName'), '/bentalebDataBricksLink')]", 25 | "type": "Microsoft.DataFactory/factories/linkedServices", 26 | "apiVersion": "2018-06-01", 27 | "properties": { 28 | "annotations": [], 29 | "type": "AzureDatabricks", 30 | "typeProperties": { 31 | "domain": "https://adb-2184495587427117.17.azuredatabricks.net", 32 | "accessToken": { 33 | "type": "SecureString", 34 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 35 | }, 36 | "existingClusterId": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 37 | } 38 | }, 39 | "dependsOn": [] 40 | }, 41 | { 42 | "name": "[concat(parameters('factoryName'), '/automateTransforming')]", 43 | "type": "Microsoft.DataFactory/factories/pipelines", 44 | "apiVersion": "2018-06-01", 45 | "properties": { 46 | "activities": [ 47 | { 48 | "name": "Notebook1", 49 | "type": "DatabricksNotebook", 50 | "dependsOn": [], 51 | "policy": { 52 | "timeout": "0.12:00:00", 53 | "retry": 0, 54 | "retryIntervalInSeconds": 30, 55 | "secureOutput": false, 56 | "secureInput": false 57 | }, 58 | "userProperties": [], 59 | "typeProperties": { 60 | "notebookPath": "/Repos/sbentaleb.ext@simplonformations.onmicrosoft.com/AzurePublicTransportDataIntegration/ETL/integration" 61 | }, 62 | "linkedServiceName": { 63 | "referenceName": "bentalebDataBricksLink", 64 | "type": "LinkedServiceReference" 65 | } 66 | } 67 | ], 68 | "policy": { 69 | "elapsedTimeMetric": {}, 70 | "cancelAfter": {} 71 | }, 72 | "annotations": [] 73 | }, 74 | "dependsOn": [ 75 | "[concat(variables('factoryId'), '/linkedServices/bentalebDataBricksLink')]" 76 | ] 77 | }, 78 | { 79 | "name": "[concat(parameters('factoryName'), '/batchProcTrigger')]", 80 | "type": "Microsoft.DataFactory/factories/triggers", 81 | "apiVersion": "2018-06-01", 82 | "properties": { 83 | "annotations": [], 84 | "runtimeState": "Started", 85 | "pipelines": [ 86 | { 87 | "pipelineReference": { 88 | "referenceName": "automateTransforming", 89 | "type": "PipelineReference" 90 | }, 91 | "parameters": {} 92 | } 93 | ], 94 | "type": "ScheduleTrigger", 95 | "typeProperties": { 96 | "recurrence": { 97 | "frequency": "Minute", 98 | "interval": 3, 99 | "startTime": "2023-09-25T15:21:00", 100 | "endTime": "2023-09-25T16:28:00", 101 | "timeZone": "Morocco Standard Time" 102 | } 103 | } 104 | }, 105 | "dependsOn": [ 106 | "[concat(variables('factoryId'), '/pipelines/automateTransforming')]" 107 | ] 108 | } 109 | ] 110 | } -------------------------------------------------------------------------------- /bentalebdatafactory/ARMTemplateForFactory.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdatafactory" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0927-073616-deozl5ap" 17 | } 18 | }, 19 | "variables": { 20 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 21 | }, 22 | "resources": [ 23 | { 24 | "name": "[concat(parameters('factoryName'), '/automateTransforming')]", 25 | "type": "Microsoft.DataFactory/factories/pipelines", 26 | "apiVersion": "2018-06-01", 27 | "properties": { 28 | "activities": [ 29 | { 30 | "name": "DataTransformation", 31 | "type": "DatabricksNotebook", 32 | "dependsOn": [], 33 | "policy": { 34 | "timeout": "0.12:00:00", 35 | "retry": 0, 36 | "retryIntervalInSeconds": 30, 37 | "secureOutput": false, 38 | "secureInput": false 39 | }, 40 | "userProperties": [], 41 | "typeProperties": { 42 | "notebookPath": "/Repos/sbentaleb.ext@simplonformations.onmicrosoft.com/AzurePublicTransportDataIntegration/ETL/integration" 43 | }, 44 | "linkedServiceName": { 45 | "referenceName": "bentalebDataBricksLink", 46 | "type": "LinkedServiceReference" 47 | } 48 | }, 49 | { 50 | "name": "Run log", 51 | "type": "SetVariable", 52 | "dependsOn": [ 53 | { 54 | "activity": "DataTransformation", 55 | "dependencyConditions": [ 56 | "Succeeded" 57 | ] 58 | } 59 | ], 60 | "policy": { 61 | "secureOutput": true, 62 | "secureInput": false 63 | }, 64 | "userProperties": [], 65 | "typeProperties": { 66 | "variableName": "transformationInfo", 67 | "value": { 68 | "value": "@activity('DataTransformation').output.runOutput", 69 | "type": "Expression" 70 | } 71 | } 72 | } 73 | ], 74 | "policy": { 75 | "elapsedTimeMetric": {}, 76 | "cancelAfter": {} 77 | }, 78 | "variables": { 79 | "transformationInfo": { 80 | "type": "String" 81 | } 82 | }, 83 | "annotations": [] 84 | }, 85 | "dependsOn": [ 86 | "[concat(variables('factoryId'), '/linkedServices/bentalebDataBricksLink')]" 87 | ] 88 | }, 89 | { 90 | "name": "[concat(parameters('factoryName'), '/bentalebDataBricksLink')]", 91 | "type": "Microsoft.DataFactory/factories/linkedServices", 92 | "apiVersion": "2018-06-01", 93 | "properties": { 94 | "annotations": [], 95 | "type": "AzureDatabricks", 96 | "typeProperties": { 97 | "domain": "https://adb-7197147741827979.19.azuredatabricks.net", 98 | "accessToken": { 99 | "type": "SecureString", 100 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 101 | }, 102 | "existingClusterId": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 103 | } 104 | }, 105 | "dependsOn": [] 106 | }, 107 | { 108 | "name": "[concat(parameters('factoryName'), '/AutomatisationTrigger')]", 109 | "type": "Microsoft.DataFactory/factories/triggers", 110 | "apiVersion": "2018-06-01", 111 | "properties": { 112 | "annotations": [], 113 | "runtimeState": "Started", 114 | "pipelines": [ 115 | { 116 | "pipelineReference": { 117 | "referenceName": "automateTransforming", 118 | "type": "PipelineReference" 119 | }, 120 | "parameters": {} 121 | } 122 | ], 123 | "type": "ScheduleTrigger", 124 | "typeProperties": { 125 | "recurrence": { 126 | "frequency": "Minute", 127 | "interval": 1, 128 | "startTime": "2023-09-27T13:27:00", 129 | "endTime": "2023-09-27T13:55:00", 130 | "timeZone": "Morocco Standard Time" 131 | } 132 | } 133 | }, 134 | "dependsOn": [ 135 | "[concat(variables('factoryId'), '/pipelines/automateTransforming')]" 136 | ] 137 | } 138 | ] 139 | } -------------------------------------------------------------------------------- /bentalebdatafactory/linkedTemplates/ArmTemplate_0.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "factoryName": { 6 | "type": "string", 7 | "metadata": "Data Factory name", 8 | "defaultValue": "bentalebdatafactory" 9 | }, 10 | "bentalebDataBricksLink_accessToken": { 11 | "type": "secureString", 12 | "metadata": "Secure string for 'accessToken' of 'bentalebDataBricksLink'" 13 | }, 14 | "bentalebDataBricksLink_properties_typeProperties_existingClusterId": { 15 | "type": "string", 16 | "defaultValue": "0927-073616-deozl5ap" 17 | } 18 | }, 19 | "variables": { 20 | "factoryId": "[concat('Microsoft.DataFactory/factories/', parameters('factoryName'))]" 21 | }, 22 | "resources": [ 23 | { 24 | "name": "[concat(parameters('factoryName'), '/bentalebDataBricksLink')]", 25 | "type": "Microsoft.DataFactory/factories/linkedServices", 26 | "apiVersion": "2018-06-01", 27 | "properties": { 28 | "annotations": [], 29 | "type": "AzureDatabricks", 30 | "typeProperties": { 31 | "domain": "https://adb-7197147741827979.19.azuredatabricks.net", 32 | "accessToken": { 33 | "type": "SecureString", 34 | "value": "[parameters('bentalebDataBricksLink_accessToken')]" 35 | }, 36 | "existingClusterId": "[parameters('bentalebDataBricksLink_properties_typeProperties_existingClusterId')]" 37 | } 38 | }, 39 | "dependsOn": [] 40 | }, 41 | { 42 | "name": "[concat(parameters('factoryName'), '/automateTransforming')]", 43 | "type": "Microsoft.DataFactory/factories/pipelines", 44 | "apiVersion": "2018-06-01", 45 | "properties": { 46 | "activities": [ 47 | { 48 | "name": "DataTransformation", 49 | "type": "DatabricksNotebook", 50 | "dependsOn": [], 51 | "policy": { 52 | "timeout": "0.12:00:00", 53 | "retry": 0, 54 | "retryIntervalInSeconds": 30, 55 | "secureOutput": false, 56 | "secureInput": false 57 | }, 58 | "userProperties": [], 59 | "typeProperties": { 60 | "notebookPath": "/Repos/sbentaleb.ext@simplonformations.onmicrosoft.com/AzurePublicTransportDataIntegration/ETL/integration" 61 | }, 62 | "linkedServiceName": { 63 | "referenceName": "bentalebDataBricksLink", 64 | "type": "LinkedServiceReference" 65 | } 66 | }, 67 | { 68 | "name": "Run log", 69 | "type": "SetVariable", 70 | "dependsOn": [ 71 | { 72 | "activity": "DataTransformation", 73 | "dependencyConditions": [ 74 | "Succeeded" 75 | ] 76 | } 77 | ], 78 | "policy": { 79 | "secureOutput": true, 80 | "secureInput": false 81 | }, 82 | "userProperties": [], 83 | "typeProperties": { 84 | "variableName": "transformationInfo", 85 | "value": { 86 | "value": "@activity('DataTransformation').output.runOutput", 87 | "type": "Expression" 88 | } 89 | } 90 | } 91 | ], 92 | "policy": { 93 | "elapsedTimeMetric": {}, 94 | "cancelAfter": {} 95 | }, 96 | "variables": { 97 | "transformationInfo": { 98 | "type": "String" 99 | } 100 | }, 101 | "annotations": [] 102 | }, 103 | "dependsOn": [ 104 | "[concat(variables('factoryId'), '/linkedServices/bentalebDataBricksLink')]" 105 | ] 106 | }, 107 | { 108 | "name": "[concat(parameters('factoryName'), '/AutomatisationTrigger')]", 109 | "type": "Microsoft.DataFactory/factories/triggers", 110 | "apiVersion": "2018-06-01", 111 | "properties": { 112 | "annotations": [], 113 | "runtimeState": "Started", 114 | "pipelines": [ 115 | { 116 | "pipelineReference": { 117 | "referenceName": "automateTransforming", 118 | "type": "PipelineReference" 119 | }, 120 | "parameters": {} 121 | } 122 | ], 123 | "type": "ScheduleTrigger", 124 | "typeProperties": { 125 | "recurrence": { 126 | "frequency": "Minute", 127 | "interval": 1, 128 | "startTime": "2023-09-27T13:27:00", 129 | "endTime": "2023-09-27T13:55:00", 130 | "timeZone": "Morocco Standard Time" 131 | } 132 | } 133 | }, 134 | "dependsOn": [ 135 | "[concat(variables('factoryId'), '/pipelines/automateTransforming')]" 136 | ] 137 | } 138 | ] 139 | } --------------------------------------------------------------------------------