├── README.md └── Samples ├── dataflow ├── DataVault_Customer.json ├── DataVault_Geography.json ├── DataVault_InternetSales.json ├── DataVault_Product.json ├── NYC_Taxi_Distance_Trip_wdf.json ├── TaxiDemo.json ├── dataflow_SQL_SAT_ROC.json ├── dataflow_SQL_SAT_df.json ├── df_PASS_ADF_DataVault.json ├── df_flatten_task_cakes_JSON.json ├── df_stacking_datasets.json ├── wdf_expand_json_column.json └── wdf_json_file.json ├── dataset ├── ADLS_Gen2.json ├── ADLS_dvs_internetsales_init_ds.json ├── ADLS_dvs_product_init_ds.json ├── ASQL_DV_HUB_Customer.json ├── ASQL_DV_HUB_Geography.json ├── ASQL_DV_HUB_Product.json ├── ASQL_DV_LINK_CustomerGeography.json ├── ASQL_DV_LINK_InternetSales.json ├── ASQL_DV_SAT_Customer.json ├── ASQL_DV_SAT_Customer_Latest.json ├── ASQL_DV_SAT_Geography.json ├── ASQL_DV_SAT_Geography_Latest.json ├── ASQL_DV_SAT_InternetSales.json ├── ASQL_DV_SAT_InternetSales_Latest.json ├── ASQL_DV_SAT_Product.json ├── ASQL_DV_SAT_Product_Latest.json ├── AzureBlobStorageSourceFileStore.json ├── AzureBlob_bgdata_daily.json ├── AzureBlob_bgdata_monthly.json ├── AzureBlob_storesales_store01.json ├── AzureBlob_storesales_store01_single.json ├── AzureBlob_storesales_store02.json ├── AzureBlob_storesales_store03.json ├── AzureBlob_storesales_storesall.json ├── AzureDataLakeStoreSinkFileStore.json ├── AzureSqlTable_bgdata_daily.json ├── AzureSqlTable_bgdata_monthly.json ├── AzureSqlTable_storesales.json ├── BLOB_MusicNotes.json ├── BLOB_dvs_customer_incr_ds.json ├── BLOB_dvs_geography_incr_ds.json ├── BLOB_dvs_internetsales_incr_ds.json ├── BLOB_dvs_product_incr_ds.json ├── TaxiSink.json ├── Template_FileShare_ds.json ├── Template_wasb_delete.json ├── Template_wasb_delete_time.json ├── Template_wasb_single_sink_ds.json ├── Template_wasb_single_source_ds.json ├── Template_wasb_sink_ds.json ├── Template_wasb_source_ds.json ├── TripData.json ├── TripFares.json ├── ds_adls_adf_dataflows.json ├── ds_adls_generic_folder.json ├── ds_azsql_generic.json ├── ds_dataflow_output.json ├── ds_dataflow_sample.json ├── ds_df_temperature_data.json ├── ds_df_temperature_header.json ├── ds_df_temperature_output.json ├── ds_json_dataflow_sample.json ├── ds_json_output_temp.json ├── ds_json_temp.json ├── ls_json_flatten_data.json └── validation_adls_source_ds.json ├── linkedService ├── AzureBlobStorage_LS.json ├── AzureDataLakeStorage_Gen2.json ├── AzureDataLakeStore_ls.json ├── AzureKeyVault_LS.json ├── AzureSqlDatabase_DV_LS.json ├── AzureSqlDatabase_LS.json ├── FileSystem_ls.json ├── Spark1.json ├── azuredatabricks_bigdata_ls.json ├── ls_af.json └── ls_af_gettime.json ├── pipeline ├── CopyFiles_DeleteFiles_Blob.json ├── CopyFiles_Message_Email.json ├── CopyFiles_Message_ServiceBus.json ├── DV_IncrementalLoad_pl.json ├── DV_InitialLoad_pl.json ├── Delete files with Azure Data Factory FileSystem.json ├── Delete files with Azure Data Factory Time.json ├── ValidationActivity_CopyDeleteFiles.json ├── Workflow_IF_activity_pl.json ├── Workflow_Switch_activity_pl.json ├── adf_arrays_sample_pl.json ├── bdata_adf_email_notify_pl.json ├── bdata_adf_logging_pl.json ├── bdata_adf_variable_append_pl.json ├── bdata_adf_variable_array_first_element.json ├── bdata_adf_variable_array_many_elements.json ├── bdata_adf_variable_set_pl.json ├── bdata_adf_variable_set_single_pl.json ├── dataflow_flatten_pl.json ├── pl_CreateArray_UseCase.json ├── pl_PASS_ADF_DataVault.json ├── pl_SQL_SP_optional_parameter.json ├── pl_azurefunction_test.json ├── pl_constant_dynamic_JSON.json ├── pl_constant_dynamic_JSON_SQL.json ├── pl_data_json_flatten.json ├── pl_data_json_flatten_corrected.json ├── pl_default_value_array.json ├── pl_default_value_array_child.json ├── pl_default_value_array_variable.json ├── pl_durable_function_loop_polling.json ├── pl_durable_function_simple_polling.json ├── pl_durable_function_simple_polling_safe.json ├── pl_fail_activity.json ├── pl_fail_activity_old.json ├── pl_general_email_attachement.json ├── pl_html_formatted_email.json ├── pl_json_arrays.json ├── pl_json_arrays_sub.json ├── pl_passing_array_between_pipelines.json └── pl_passing_array_child.json └── templates └── Copy Files with Email Notification ├── Copy Files with Email Notification.json └── manifest.json /README.md: -------------------------------------------------------------------------------- 1 | # Azure-DataFactory 2 | -------------------------------------------------------------------------------- /Samples/dataflow/DataVault_Customer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DataVault_Customer", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_customer_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceCustomer" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ASQL_DV_HUB_Customer", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "HubCustomer" 20 | }, 21 | { 22 | "dataset": { 23 | "referenceName": "ASQL_DV_SAT_Customer_Latest", 24 | "type": "DatasetReference" 25 | }, 26 | "name": "SatCustomer" 27 | }, 28 | { 29 | "dataset": { 30 | "referenceName": "ASQL_DV_LINK_CustomerGeography", 31 | "type": "DatasetReference" 32 | }, 33 | "name": "LinkCustomerGeography" 34 | } 35 | ], 36 | "sinks": [ 37 | { 38 | "dataset": { 39 | "referenceName": "ASQL_DV_HUB_Customer", 40 | "type": "DatasetReference" 41 | }, 42 | "name": "SinkHubCustomer" 43 | }, 44 | { 45 | "dataset": { 46 | "referenceName": "ASQL_DV_SAT_Customer", 47 | "type": "DatasetReference" 48 | }, 49 | "name": "SinkSatCustomer" 50 | }, 51 | { 52 | "dataset": { 53 | "referenceName": "ASQL_DV_LINK_CustomerGeography", 54 | "type": "DatasetReference" 55 | }, 56 | "name": "SinkLinkCustomerGeography" 57 | } 58 | ], 59 | "transformations": [ 60 | { 61 | "name": "HashColumns" 62 | }, 63 | { 64 | "name": "CheckNewCustomer" 65 | }, 66 | { 67 | "name": "NewHubCustomer" 68 | }, 69 | { 70 | "name": "CheckNewSatCustomer" 71 | }, 72 | { 73 | "name": "NewSatCustomer" 74 | }, 75 | { 76 | "name": "CheckNewLinkCustomerGeography" 77 | }, 78 | { 79 | "name": "NewLinkCustomerGeography" 80 | } 81 | ], 82 | "script": "\n\nsource(output(\n\t\tCustomerAlternateKey as string,\n\t\tGeographyAlternateKey as string,\n\t\tTitle as string,\n\t\tFirstName as string,\n\t\tMiddleName as string,\n\t\tLastName as string,\n\t\tNameStyle as string,\n\t\tBirthDate as string,\n\t\tMaritalStatus as string,\n\t\tSuffix as string,\n\t\tGender as string,\n\t\tEmailAddress as string,\n\t\tYearlyIncome as string,\n\t\tTotalChildren as string,\n\t\tNumberChildrenAtHome as string,\n\t\tHouseOwnerFlag as string,\n\t\tNumberCarsOwned as string,\n\t\tAddressLine1 as string,\n\t\tAddressLine2 as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceCustomer\nsource(output(\n\t\tHUB_Customer_HK as string,\n\t\tCustomerAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> HubCustomer\nsource(output(\n\t\tHUB_Customer_HK as string,\n\t\tSAT_Customer_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SatCustomer\nsource(output(\n\t\tLINK_CustomerGeography_HK as string,\n\t\tHUB_Customer_HK as string,\n\t\tHUB_Geography_HK as string,\n\t\tCustomerAlternateKey as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> LinkCustomerGeography\nSourceCustomer derive(LINK_CustomerGeography_HK = sha2(512, CustomerAlternateKey, GeographyAlternateKey),\n\t\tHUB_Customer_HK = sha2(512, CustomerAlternateKey),\n\t\tHUB_Geography_HK = sha2(512, GeographyAlternateKey),\n\t\tSAT_Customer_HDIFF = sha2(512, CustomerAlternateKey, GeographyAlternateKey,Title,FirstName,MiddleName,LastName,NameStyle,BirthDate,MaritalStatus,Suffix,Gender,EmailAddress,YearlyIncome,TotalChildren,NumberChildrenAtHome,HouseOwnerFlag,NumberCarsOwned,AddressLine1,AddressLine2),\n\t\tLOAD_TS = currentTimestamp()) ~> HashColumns\nHashColumns, HubCustomer exists(HashColumns@HUB_Customer_HK == HubCustomer@HUB_Customer_HK,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewCustomer\nCheckNewCustomer select(mapColumn(\n\t\tCustomerAlternateKey,\n\t\tHUB_Customer_HK,\n\t\tLOAD_TS\n\t)) ~> NewHubCustomer\nHashColumns, SatCustomer exists(HashColumns@HUB_Customer_HK == SatCustomer@HUB_Customer_HK\n\t&& HashColumns@SAT_Customer_HDIFF == SatCustomer@SAT_Customer_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatCustomer\nCheckNewSatCustomer select(mapColumn(\n\t\tTitle,\n\t\tFirstName,\n\t\tMiddleName,\n\t\tLastName,\n\t\tNameStyle,\n\t\tBirthDate,\n\t\tMaritalStatus,\n\t\tSuffix,\n\t\tGender,\n\t\tEmailAddress,\n\t\tYearlyIncome,\n\t\tTotalChildren,\n\t\tNumberChildrenAtHome,\n\t\tHouseOwnerFlag,\n\t\tNumberCarsOwned,\n\t\tAddressLine1,\n\t\tAddressLine2,\n\t\tHUB_Customer_HK,\n\t\tSAT_Customer_HDIFF,\n\t\tLOAD_TS\n\t)) ~> NewSatCustomer\nHashColumns, LinkCustomerGeography exists(HashColumns@LINK_CustomerGeography_HK == LinkCustomerGeography@LINK_CustomerGeography_HK,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewLinkCustomerGeography\nCheckNewLinkCustomerGeography select(mapColumn(\n\t\tCustomerAlternateKey,\n\t\tGeographyAlternateKey,\n\t\tLINK_CustomerGeography_HK,\n\t\tHUB_Customer_HK,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS\n\t)) ~> NewLinkCustomerGeography\nNewHubCustomer sink(input(\n\t\tHUB_Customer_HK as string,\n\t\tCustomerAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkHubCustomer\nNewSatCustomer sink(input(\n\t\tHUB_Customer_HK as string,\n\t\tTitle as string,\n\t\tFirstName as string,\n\t\tMiddleName as string,\n\t\tLastName as string,\n\t\tNameStyle as boolean,\n\t\tBirthDate as date,\n\t\tMaritalStatus as string,\n\t\tSuffix as string,\n\t\tGender as string,\n\t\tEmailAddress as string,\n\t\tYearlyIncome as decimal(19,4),\n\t\tTotalChildren as integer,\n\t\tNumberChildrenAtHome as integer,\n\t\tHouseOwnerFlag as string,\n\t\tNumberCarsOwned as integer,\n\t\tAddressLine1 as string,\n\t\tAddressLine2 as string,\n\t\tSAT_Customer_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkSatCustomer\nNewLinkCustomerGeography sink(input(\n\t\tLINK_CustomerGeography_HK as string,\n\t\tHUB_Customer_HK as string,\n\t\tHUB_Geography_HK as string,\n\t\tCustomerAlternateKey as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkLinkCustomerGeography" 83 | } 84 | } 85 | } -------------------------------------------------------------------------------- /Samples/dataflow/DataVault_Geography.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DataVault_Geography", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_geography_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceGeogrpahy" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ASQL_DV_HUB_Geography", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "HubGeography" 20 | }, 21 | { 22 | "dataset": { 23 | "referenceName": "ASQL_DV_SAT_Geography_Latest", 24 | "type": "DatasetReference" 25 | }, 26 | "name": "SatGeography" 27 | } 28 | ], 29 | "sinks": [ 30 | { 31 | "dataset": { 32 | "referenceName": "ASQL_DV_HUB_Geography", 33 | "type": "DatasetReference" 34 | }, 35 | "name": "SinkHubGeography" 36 | }, 37 | { 38 | "dataset": { 39 | "referenceName": "ASQL_DV_SAT_Geography", 40 | "type": "DatasetReference" 41 | }, 42 | "name": "SinkSatGeography" 43 | } 44 | ], 45 | "transformations": [ 46 | { 47 | "name": "HashColumns" 48 | }, 49 | { 50 | "name": "CheckNewGeography" 51 | }, 52 | { 53 | "name": "NewHubGeography" 54 | }, 55 | { 56 | "name": "CheckNewSatGeography" 57 | }, 58 | { 59 | "name": "NewSatGeography" 60 | } 61 | ], 62 | "script": "\n\nsource(output(\n\t\tGeographyAlternateKey as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceGeogrpahy\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> HubGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tSAT_Geography_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SatGeography\nSourceGeogrpahy derive(HUB_Geography_HK = sha2(512, GeographyAlternateKey),\n\t\tLOAD_TS = currentTimestamp(),\n\t\tSAT_Geography_HDIFF = sha2(512, GeographyAlternateKey, City, StateProvinceCode, StateProvinceName, CountryRegionCode, EnglishCountryRegionName, PostalCode)) ~> HashColumns\nHashColumns, HubGeography exists(HashColumns@HUB_Geography_HK == HubGeography@HUB_Geography_HK,\n\tnegate:true,\n\tbroadcast: 'both')~> CheckNewGeography\nCheckNewGeography select(mapColumn(\n\t\tGeographyAlternateKey,\n\t\tLOAD_TS,\n\t\tHUB_Geography_HK\n\t)) ~> NewHubGeography\nHashColumns, SatGeography exists(HashColumns@HUB_Geography_HK == SatGeography@HUB_Geography_HK\n\t&& HashColumns@SAT_Geography_HDIFF == SatGeography@SAT_Geography_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatGeography\nCheckNewSatGeography select(mapColumn(\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS,\n\t\tSAT_Geography_HDIFF\n\t)) ~> NewSatGeography\nNewHubGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkHubGeography\nNewSatGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string,\n\t\tSAT_Geography_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tmapColumn(\n\t\tHUB_Geography_HK,\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tSAT_Geography_HDIFF,\n\t\tLOAD_TS\n\t)) ~> SinkSatGeography" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /Samples/dataflow/DataVault_InternetSales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DataVault_InternetSales", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_internetsales_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceInternetSales", 13 | "script": "source(output(\n\t\tProductAlternateKey as string,\n\t\tCustomerAlternateKey as string,\n\t\tSalesOrderNumber as string,\n\t\tSalesOrderLineNumber as integer,\n\t\tRevisionNumber as integer,\n\t\tOrderQuantity as integer,\n\t\tUnitPrice as double,\n\t\tExtendedAmount as double,\n\t\tUnitPriceDiscountPct as double,\n\t\tDiscountAmount as double,\n\t\tProductStandardCost as double,\n\t\tTotalProductCost as double,\n\t\tSalesAmount as double,\n\t\tTaxAmt as double,\n\t\tFreight as double,\n\t\tCarrierTrackingNumber as string,\n\t\tCustomerPONumber as string,\n\t\tOrderDate as date,\n\t\tDueDate as date,\n\t\tShipDate as date\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceInternetSales" 14 | }, 15 | { 16 | "dataset": { 17 | "referenceName": "ASQL_DV_LINK_InternetSales", 18 | "type": "DatasetReference" 19 | }, 20 | "name": "LinkInternetSales", 21 | "script": "source(output(\n\t\tLINK_InternetSales_HK as string,\n\t\tHUB_Customer_HK as string,\n\t\tHUB_Product_HK as string,\n\t\tProductAlternateKey as string,\n\t\tCustomerAlternateKey as string,\n\t\tSalesOrderNumber as string,\n\t\tSalesOrderLineNumber as integer,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table') ~> LinkInternetSales" 22 | }, 23 | { 24 | "dataset": { 25 | "referenceName": "ASQL_DV_SAT_InternetSales_Latest", 26 | "type": "DatasetReference" 27 | }, 28 | "name": "SatInternetSales", 29 | "script": "source(output(\n\t\tLINK_InternetSales_HK as string,\n\t\tSAT_InternetSales_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table') ~> SatInternetSales" 30 | } 31 | ], 32 | "sinks": [ 33 | { 34 | "dataset": { 35 | "referenceName": "ASQL_DV_LINK_InternetSales", 36 | "type": "DatasetReference" 37 | }, 38 | "name": "SinkLinkInternetSales", 39 | "script": "NewLinkInternetSales sink(input(\n\t\tLINK_InternetSales_HK as string,\n\t\tHUB_Customer_HK as string,\n\t\tHUB_Product_HK as string,\n\t\tProductAlternateKey as string,\n\t\tCustomerAlternateKey as string,\n\t\tSalesOrderNumber as string,\n\t\tSalesOrderLineNumber as integer,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkLinkInternetSales" 40 | }, 41 | { 42 | "dataset": { 43 | "referenceName": "ASQL_DV_SAT_InternetSales", 44 | "type": "DatasetReference" 45 | }, 46 | "name": "SinkSatInternetSales", 47 | "script": "NewSatProduct sink(input(\n\t\tLINK_InternetSales_HK as string,\n\t\tRevisionNumber as integer,\n\t\tOrderQuantity as integer,\n\t\tUnitPrice as decimal(19,4),\n\t\tExtendedAmount as decimal(19,4),\n\t\tUnitPriceDiscountPct as double,\n\t\tDiscountAmount as double,\n\t\tProductStandardCost as decimal(19,4),\n\t\tTotalProductCost as decimal(19,4),\n\t\tSalesAmount as decimal(19,4),\n\t\tTaxAmt as decimal(19,4),\n\t\tFreight as decimal(19,4),\n\t\tCarrierTrackingNumber as string,\n\t\tCustomerPONumber as string,\n\t\tOrderDate as timestamp,\n\t\tDueDate as timestamp,\n\t\tShipDate as timestamp,\n\t\tSAT_InternetSales_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkSatInternetSales" 48 | } 49 | ], 50 | "transformations": [ 51 | { 52 | "name": "HashColumns", 53 | "script": "SourceInternetSales derive(LINK_InternetSales_HK = sha2(512, ProductAlternateKey,CustomerAlternateKey,SalesOrderNumber,SalesOrderLineNumber),\n\t\tHUB_Customer_HK = sha2(512, CustomerAlternateKey),\n\t\tLOAD_TS = currentTimestamp(),\n\t\tHUB_Product_HK = sha2(512, ProductAlternateKey),\n\t\tSAT_InternetSales_HDIFF = sha2(512, ProductAlternateKey,CustomerAlternateKey,SalesOrderNumber,SalesOrderLineNumber,RevisionNumber,OrderQuantity,UnitPrice,ExtendedAmount,UnitPriceDiscountPct,DiscountAmount,ProductStandardCost,TotalProductCost,SalesAmount,TaxAmt,Freight,CarrierTrackingNumber,CustomerPONumber,OrderDate,DueDate)) ~> HashColumns" 54 | }, 55 | { 56 | "name": "CheckNewLinkInternetSales", 57 | "script": "HashColumns, LinkInternetSales exists(HashColumns@LINK_InternetSales_HK == LinkInternetSales@LINK_InternetSales_HK,\n\tnegate:true,\n\tbroadcast: 'both')~> CheckNewLinkInternetSales" 58 | }, 59 | { 60 | "name": "NewLinkInternetSales", 61 | "script": "CheckNewLinkInternetSales select(mapColumn(\n\t\tProductAlternateKey,\n\t\tCustomerAlternateKey,\n\t\tSalesOrderNumber,\n\t\tSalesOrderLineNumber,\n\t\tLINK_InternetSales_HK,\n\t\tHUB_Customer_HK,\n\t\tLOAD_TS,\n\t\tHUB_Product_HK\n\t))~> NewLinkInternetSales" 62 | }, 63 | { 64 | "name": "CheckNewSatInternetSales", 65 | "script": "HashColumns, SatInternetSales exists(HashColumns@LINK_InternetSales_HK == SatInternetSales@LINK_InternetSales_HK\n\t&& HashColumns@SAT_InternetSales_HDIFF == SatInternetSales@SAT_InternetSales_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatInternetSales" 66 | }, 67 | { 68 | "name": "NewSatProduct", 69 | "script": "CheckNewSatInternetSales select(mapColumn(\n\t\tRevisionNumber,\n\t\tOrderQuantity,\n\t\tUnitPrice,\n\t\tExtendedAmount,\n\t\tUnitPriceDiscountPct,\n\t\tDiscountAmount,\n\t\tProductStandardCost,\n\t\tTotalProductCost,\n\t\tSalesAmount,\n\t\tTaxAmt,\n\t\tFreight,\n\t\tCarrierTrackingNumber,\n\t\tCustomerPONumber,\n\t\tOrderDate,\n\t\tDueDate,\n\t\tShipDate,\n\t\tLINK_InternetSales_HK,\n\t\tLOAD_TS,\n\t\tSAT_InternetSales_HDIFF\n\t))~> NewSatProduct" 70 | } 71 | ] 72 | } 73 | } 74 | } -------------------------------------------------------------------------------- /Samples/dataflow/DataVault_Product.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DataVault_Product", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_product_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceProduct", 13 | "script": "source(output(\n\t\tProductAlternateKey as string,\n\t\tWeightUnitMeasureCode as string,\n\t\tSizeUnitMeasureCode as string,\n\t\tEnglishProductName as string,\n\t\tStandardCost as double,\n\t\tFinishedGoodsFlag as boolean,\n\t\tColor as string,\n\t\tSafetyStockLevel as short,\n\t\tReorderPoint as short,\n\t\tListPrice as double,\n\t\tSize as string,\n\t\tSizeRange as string,\n\t\tWeight as double,\n\t\tModelName as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceProduct" 14 | }, 15 | { 16 | "dataset": { 17 | "referenceName": "ASQL_DV_HUB_Product", 18 | "type": "DatasetReference" 19 | }, 20 | "name": "HubProduct", 21 | "script": "source(output(\n\t\tHUB_Product_HK as string,\n\t\tProductAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table') ~> HubProduct" 22 | }, 23 | { 24 | "dataset": { 25 | "referenceName": "ASQL_DV_SAT_Product_Latest", 26 | "type": "DatasetReference" 27 | }, 28 | "name": "SatProduct", 29 | "script": "source(output(\n\t\tHUB_Product_HK as string,\n\t\tSAT_Product_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table') ~> SatProduct" 30 | } 31 | ], 32 | "sinks": [ 33 | { 34 | "dataset": { 35 | "referenceName": "ASQL_DV_HUB_Product", 36 | "type": "DatasetReference" 37 | }, 38 | "name": "SinkHubProduct", 39 | "script": "NewHubProduct sink(input(\n\t\tHUB_Product_HK as string,\n\t\tProductAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkHubProduct" 40 | }, 41 | { 42 | "dataset": { 43 | "referenceName": "ASQL_DV_SAT_Product", 44 | "type": "DatasetReference" 45 | }, 46 | "name": "SinkSatProduct", 47 | "script": "NewSatProduct sink(input(\n\t\tHUB_Product_HK as string,\n\t\tWeightUnitMeasureCode as string,\n\t\tSizeUnitMeasureCode as string,\n\t\tEnglishProductName as string,\n\t\tStandardCost as decimal(19,4),\n\t\tFinishedGoodsFlag as boolean,\n\t\tColor as string,\n\t\tSafetyStockLevel as integer,\n\t\tReorderPoint as integer,\n\t\tListPrice as decimal(19,4),\n\t\tSize as string,\n\t\tSizeRange as string,\n\t\tWeight as double,\n\t\tModelName as string,\n\t\tSAT_Product_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> SinkSatProduct" 48 | } 49 | ], 50 | "transformations": [ 51 | { 52 | "name": "HashColumns", 53 | "script": "SourceProduct derive(HUB_Product_HK = sha2(512, ProductAlternateKey),\n\t\tSAT_Product_HDIFF = sha2(512, ProductAlternateKey\r\n ,WeightUnitMeasureCode\r\n ,SizeUnitMeasureCode\r\n ,EnglishProductName\r\n ,StandardCost\r\n ,FinishedGoodsFlag\r\n ,Color\r\n ,SafetyStockLevel\r\n ,ReorderPoint\r\n ,ListPrice\r\n ,Size\r\n ,SizeRange\r\n ,Weight\r\n ,ModelName),\n\t\tLOAD_TS = currentTimestamp()) ~> HashColumns" 54 | }, 55 | { 56 | "name": "CheckNewProduct", 57 | "script": "HashColumns, HubProduct exists(HashColumns@HUB_Product_HK == HubProduct@HUB_Product_HK,\n\tnegate:true,\n\tbroadcast: 'both')~> CheckNewProduct" 58 | }, 59 | { 60 | "name": "NewHubProduct", 61 | "script": "CheckNewProduct select(mapColumn(\n\t\tProductAlternateKey,\n\t\tHUB_Product_HK,\n\t\tLOAD_TS\n\t))~> NewHubProduct" 62 | }, 63 | { 64 | "name": "CheckNewSatProduct", 65 | "script": "HashColumns, SatProduct exists(HashColumns@HUB_Product_HK == SatProduct@HUB_Product_HK\n\t&& HashColumns@SAT_Product_HDIFF == SatProduct@SAT_Product_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatProduct" 66 | }, 67 | { 68 | "name": "NewSatProduct", 69 | "script": "CheckNewSatProduct select(mapColumn(\n\t\tWeightUnitMeasureCode,\n\t\tSizeUnitMeasureCode,\n\t\tEnglishProductName,\n\t\tStandardCost,\n\t\tFinishedGoodsFlag,\n\t\tColor,\n\t\tSafetyStockLevel,\n\t\tReorderPoint,\n\t\tListPrice,\n\t\tSize,\n\t\tSizeRange,\n\t\tWeight,\n\t\tModelName,\n\t\tHUB_Product_HK,\n\t\tSAT_Product_HDIFF,\n\t\tLOAD_TS\n\t))~> NewSatProduct" 70 | } 71 | ] 72 | } 73 | } 74 | } -------------------------------------------------------------------------------- /Samples/dataflow/NYC_Taxi_Distance_Trip_wdf.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "NYC_Taxi_Distance_Trip_wdf", 3 | "properties": { 4 | "type": "WranglingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "name": "TripData", 9 | "script": "", 10 | "dataset": { 11 | "referenceName": "TripData", 12 | "type": "DatasetReference" 13 | }, 14 | "typeProperties": { 15 | "readMode": null 16 | } 17 | } 18 | ], 19 | "sinks": [ 20 | { 21 | "name": "Sink1", 22 | "dataset": { 23 | "referenceName": "TaxiSink", 24 | "type": "DatasetReference" 25 | }, 26 | "script": "sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\ttruncate: true,\n\tquoteAll: true) ~> Sink1" 27 | } 28 | ], 29 | "script": "section Section1; shared #\"TripData\" = let AdfDoc = Web.Contents(\"https://bdataadlsgen2.dfs.core.windows.net/nyctaxidata-staging/trip_data/trip_data_1.csv?sv=2018-03-28&sig=On4A%2BKZBoXW0Ro4Q3x%2FWTJjNuIoB6f2SKAptnVXioWU%3D&spr=https&se=2019-11-08T20%3A49%3A38Z&srt=sco&ss=bf&sp=rwl\"),Csv = Csv.Document(AdfDoc, [Delimiter = \",\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv]), PromotedHeaders = Table.PromoteHeaders(Csv, [PromoteAllScalars = true]), TrimmedHeaders = Table.TransformColumnNames(PromotedHeaders, (columnName as text) as text => Text.Trim(columnName)) in TrimmedHeaders;\r\nshared UserQuery = let\r\n Source = TripData,\r\n ChangedType = Table.TransformColumnTypes(Source, {{\"trip_distance\", type number}, {\"pickup_longitude\", type number}, {\"pickup_latitude\", type number}, {\"dropoff_longitude\", type number}, {\"dropoff_latitude\", type number}}),\r\n Lat1_Rad = Table.AddColumn(ChangedType, \"Lat1_Rad\", each ([pickup_latitude] / 180) * Number.PI),\r\n Lon1_Rad = Table.AddColumn(Lat1_Rad, \"Lon1_Rad\", each ([pickup_longitude] / 180) * Number.PI),\r\n Lat2_Rad = Table.AddColumn(Lon1_Rad, \"Lat2_Rad\", each ([dropoff_latitude] / 180) * Number.PI),\r\n Lon2_Rad = Table.AddColumn(Lat2_Rad, \"Lon2_Rad\", each ([dropoff_longitude] / 180) * Number.PI),\r\n Distance_Mi = Table.AddColumn(Lon2_Rad, \"Distance_Mi\", each Number.Acos(Number.Sin([Lat1_Rad]) * Number.Sin([Lat2_Rad]) + Number.Cos([Lat1_Rad]) * Number.Cos([Lat2_Rad]) * Number.Cos([Lon2_Rad]-[Lon1_Rad])) * 3959),\r\n RemovedColumns = Table.RemoveColumns(Distance_Mi, {\"Lat1_Rad\", \"Lon1_Rad\", \"Lat2_Rad\", \"Lon2_Rad\"})\r\nin\r\n RemovedColumns;" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /Samples/dataflow/TaxiDemo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiDemo", 3 | "properties": { 4 | "type": "WranglingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "name": "TripData", 9 | "script": "", 10 | "dataset": { 11 | "referenceName": "TripData", 12 | "type": "DatasetReference" 13 | }, 14 | "typeProperties": { 15 | "readMode": null 16 | } 17 | }, 18 | { 19 | "name": "TripFares", 20 | "script": "", 21 | "dataset": { 22 | "referenceName": "TripFares", 23 | "type": "DatasetReference" 24 | }, 25 | "typeProperties": { 26 | "readMode": null 27 | } 28 | } 29 | ], 30 | "sinks": [ 31 | { 32 | "name": "Sink1", 33 | "dataset": { 34 | "referenceName": "TaxiSink", 35 | "type": "DatasetReference" 36 | }, 37 | "script": "sink(allowSchemaDrift: true,\n\tvalidateSchema: false) ~> Sink1" 38 | } 39 | ], 40 | "script": "section Section1; shared #\"TripData\" = let AdfDoc = Web.Contents(\"https://bdataadlsgen2.dfs.core.windows.net/nyctaxidata-staging/trip_data/trip_data_1.csv?sv=2018-03-28&sig=GqfnzpNsitPfmZH8AwnhXmymQUcC4VSLFZAg9z%2BFJuA%3D&spr=https&se=2019-11-08T19%3A15%3A35Z&srt=sco&ss=bf&sp=rwl\"),Csv = Csv.Document(AdfDoc, [Delimiter = \",\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv]), PromotedHeaders = Table.PromoteHeaders(Csv, [PromoteAllScalars = true]), TrimmedHeaders = Table.TransformColumnNames(PromotedHeaders, (columnName as text) as text => Text.Trim(columnName)) in TrimmedHeaders;shared #\"TripFares\" = let AdfDoc = Web.Contents(\"https://bdataadlsgen2.dfs.core.windows.net/nyctaxidata-staging/trip_fare/trip_fare_1.csv?sv=2018-03-28&sig=1p0MCkz9EoTNb5RmDfw1HL%2F2Sl2QKvOllBw0kbSgbGs%3D&spr=https&se=2019-11-08T19%3A15%3A37Z&srt=sco&ss=bf&sp=rwl\"),Csv = Csv.Document(AdfDoc, [Delimiter = \",\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv]), PromotedHeaders = Table.PromoteHeaders(Csv, [PromoteAllScalars = true]), TrimmedHeaders = Table.TransformColumnNames(PromotedHeaders, (columnName as text) as text => Text.Trim(columnName)) in TrimmedHeaders;\r\nshared UserQuery = let\r\n Source = TripData,\r\n #\"Inner Join\" = Table.NestedJoin(Source, {\"medallion\", \"hack_license\", \"vendor_id\", \"pickup_datetime\"}, TripFares, {\"medallion\", \"hack_license\", \"vendor_id\", \"pickup_datetime\"}, \"TripFares\", JoinKind.Inner),\r\n #\"Expanding Joined Tables\" = Table.ExpandTableColumn(#\"Inner Join\", \"TripFares\", {\"medallion\", \"hack_license\", \"vendor_id\", \"pickup_datetime\", \"payment_type\", \"fare_amount\", \"surcharge\", \"mta_tax\", \"tip_amount\", \"tolls_amount\", \"total_amount\"}, {\"TripFares.medallion\", \"TripFares.hack_license\", \"TripFares.vendor_id\", \"TripFares.pickup_datetime\", \"TripFares.payment_type\", \"TripFares.fare_amount\", \"TripFares.surcharge\", \"TripFares.mta_tax\", \"TripFares.tip_amount\", \"TripFares.tolls_amount\", \"TripFares.total_amount\"}),\r\n #\"Converting to Decimal Types\" = Table.TransformColumnTypes(#\"Expanding Joined Tables\", {{\"TripFares.total_amount\", type number}, {\"passenger_count\", type number}, {\"trip_time_in_secs\", type number}, {\"trip_distance\", type number}}),\r\n #\"Grouped rows\" = Table.Group(#\"Converting to Decimal Types\", {\"vendor_id\"}, {{\"total_passenger_count\", each List.Sum([passenger_count]), type number}, {\"total_trip_time_in_secs\", each List.Sum([trip_time_in_secs]), type number}, {\"total_trip_distance\", each List.Sum([trip_distance]), type number}, {\"total_trip_fare\", each List.Sum([TripFares.total_amount]), type number}})\r\nin\r\n #\"Grouped rows\";" 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /Samples/dataflow/dataflow_SQL_SAT_ROC.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflow_SQL_SAT_ROC", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_geography_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceGeography" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ASQL_DV_HUB_Geography", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "SourceHubGeography" 20 | }, 21 | { 22 | "dataset": { 23 | "referenceName": "ASQL_DV_SAT_Geography_Latest", 24 | "type": "DatasetReference" 25 | }, 26 | "name": "SourceSatGeography" 27 | } 28 | ], 29 | "sinks": [ 30 | { 31 | "dataset": { 32 | "referenceName": "ASQL_DV_HUB_Geography", 33 | "type": "DatasetReference" 34 | }, 35 | "name": "SinkHubGeography" 36 | }, 37 | { 38 | "dataset": { 39 | "referenceName": "ASQL_DV_SAT_Geography", 40 | "type": "DatasetReference" 41 | }, 42 | "name": "sinkSatGeography" 43 | } 44 | ], 45 | "transformations": [ 46 | { 47 | "name": "HashColumns" 48 | }, 49 | { 50 | "name": "CheckNewGeography" 51 | }, 52 | { 53 | "name": "SelectNewGeography" 54 | }, 55 | { 56 | "name": "CheckNewSatgeogrphy" 57 | }, 58 | { 59 | "name": "SelectNewSatGeography" 60 | } 61 | ], 62 | "script": "source(output(\n\t\tGeographyAlternateKey as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SourceHubGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tSAT_Geography_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SourceSatGeography\nSourceGeography derive(HUB_Geography_HK = sha2(512, GeographyAlternateKey),\n\t\tLOAD_TS = currentTimestamp(),\n\t\tSAT_Geography_HDIFF = sha2(512, GeographyAlternateKey,City,StateProvinceCode,StateProvinceName,CountryRegionCode,EnglishCountryRegionName,PostalCode)) ~> HashColumns\nHashColumns, SourceHubGeography exists(HashColumns@HUB_Geography_HK == SourceHubGeography@HUB_Geography_HK,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewGeography\nCheckNewGeography select(mapColumn(\n\t\tGeographyAlternateKey,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectNewGeography\nHashColumns, SourceSatGeography exists(HashColumns@HUB_Geography_HK == SourceSatGeography@HUB_Geography_HK\n\t&& HashColumns@SAT_Geography_HDIFF == SourceSatGeography@SAT_Geography_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatgeogrphy\nCheckNewSatgeogrphy select(mapColumn(\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS,\n\t\tSAT_Geography_HDIFF\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectNewSatGeography\nSelectNewGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table',\n\tmapColumn(\n\t\tHUB_Geography_HK,\n\t\tGeographyAlternateKey,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SinkHubGeography\nSelectNewSatGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string,\n\t\tSAT_Geography_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table',\n\tmapColumn(\n\t\tHUB_Geography_HK,\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tSAT_Geography_HDIFF,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sinkSatGeography" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /Samples/dataflow/dataflow_SQL_SAT_df.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflow_SQL_SAT_df", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_geography_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceGeography" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ASQL_DV_HUB_Geography", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "HUBGeography" 20 | }, 21 | { 22 | "dataset": { 23 | "referenceName": "ASQL_DV_SAT_Geography_Latest", 24 | "type": "DatasetReference" 25 | }, 26 | "name": "SourceLastSatGeography" 27 | } 28 | ], 29 | "sinks": [ 30 | { 31 | "dataset": { 32 | "referenceName": "ASQL_DV_HUB_Geography", 33 | "type": "DatasetReference" 34 | }, 35 | "name": "SinkGeography" 36 | }, 37 | { 38 | "dataset": { 39 | "referenceName": "ASQL_DV_SAT_Geography", 40 | "type": "DatasetReference" 41 | }, 42 | "name": "SinkSatGeography" 43 | } 44 | ], 45 | "transformations": [ 46 | { 47 | "name": "HashColumns" 48 | }, 49 | { 50 | "name": "CheckNewGeography" 51 | }, 52 | { 53 | "name": "NewGeography" 54 | }, 55 | { 56 | "name": "CheckNewSatGeography" 57 | }, 58 | { 59 | "name": "NewSatGeography" 60 | } 61 | ], 62 | "script": "\n\nsource(output(\n\t\tGeographyAlternateKey as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> HUBGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tSAT_Geography_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SourceLastSatGeography\nSourceGeography derive(HUB_Geography_HK = sha2(512, GeographyAlternateKey),\n\t\tLOAD_TS = currentTimestamp(),\n\t\tSAT_Geography_HDIFF = sha2(512, GeographyAlternateKey,City,StateProvinceCode,StateProvinceName,CountryRegionCode,EnglishCountryRegionName,PostalCode)) ~> HashColumns\nHashColumns, HUBGeography exists(HashColumns@HUB_Geography_HK == HUBGeography@HUB_Geography_HK,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewGeography\nCheckNewGeography select(mapColumn(\n\t\tGeographyAlternateKey,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> NewGeography\nHashColumns, SourceLastSatGeography exists(HashColumns@HUB_Geography_HK == SourceLastSatGeography@HUB_Geography_HK\n\t&& HashColumns@SAT_Geography_HDIFF == SourceLastSatGeography@SAT_Geography_HDIFF,\n\tnegate:true,\n\tbroadcast: 'none')~> CheckNewSatGeography\nCheckNewSatGeography select(mapColumn(\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS,\n\t\tSAT_Geography_HDIFF\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> NewSatGeography\nNewGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table',\n\tmapColumn(\n\t\tHUB_Geography_HK,\n\t\tGeographyAlternateKey,\n\t\tLOAD_TS\n\t)) ~> SinkGeography\nNewSatGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string,\n\t\tSAT_Geography_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table') ~> SinkSatGeography" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /Samples/dataflow/df_PASS_ADF_DataVault.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "df_PASS_ADF_DataVault", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "BLOB_dvs_geography_incr_ds", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceGeography" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ASQL_DV_HUB_Geography", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "SourceHUBGeography" 20 | }, 21 | { 22 | "dataset": { 23 | "referenceName": "ASQL_DV_SAT_Geography_Latest", 24 | "type": "DatasetReference" 25 | }, 26 | "name": "SourcSatGeographyLatest" 27 | } 28 | ], 29 | "sinks": [ 30 | { 31 | "dataset": { 32 | "referenceName": "ASQL_DV_HUB_Geography", 33 | "type": "DatasetReference" 34 | }, 35 | "name": "SinkHUbGeography" 36 | }, 37 | { 38 | "dataset": { 39 | "referenceName": "ASQL_DV_SAT_Geography", 40 | "type": "DatasetReference" 41 | }, 42 | "name": "SinkSatGeography" 43 | } 44 | ], 45 | "transformations": [ 46 | { 47 | "name": "HashColumns" 48 | }, 49 | { 50 | "name": "CheckNewGeography" 51 | }, 52 | { 53 | "name": "SelectHubGeography" 54 | }, 55 | { 56 | "name": "CheckNewSatGeography" 57 | }, 58 | { 59 | "name": "SelectNewSatGeography" 60 | } 61 | ], 62 | "script": "source(output(\n\t\tGeographyAlternateKey as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> SourceGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SourceHUBGeography\nsource(output(\n\t\tHUB_Geography_HK as string,\n\t\tSAT_Geography_HDIFF as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tformat: 'table') ~> SourcSatGeographyLatest\nSourceGeography derive(HUB_Geography_HK = sha2(256, GeographyAlternateKey),\n\t\tSAT_Geography_HDIFF = sha2(256, GeographyAlternateKey,City,StateProvinceCode,StateProvinceName,CountryRegionCode,EnglishCountryRegionName,PostalCode),\n\t\tLOAD_TS = currentTimestamp()) ~> HashColumns\nHashColumns, SourceHUBGeography exists(HashColumns@HUB_Geography_HK == SourceHUBGeography@HUB_Geography_HK,\n\tnegate:true,\n\tbroadcast: 'auto')~> CheckNewGeography\nCheckNewGeography select(mapColumn(\n\t\tGeographyAlternateKey,\n\t\tHUB_Geography_HK,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectHubGeography\nHashColumns, SourcSatGeographyLatest exists(HashColumns@HUB_Geography_HK == SourcSatGeographyLatest@HUB_Geography_HK\n\t&& HashColumns@SAT_Geography_HDIFF == SourcSatGeographyLatest@SAT_Geography_HDIFF,\n\tnegate:true,\n\tbroadcast: 'auto')~> CheckNewSatGeography\nCheckNewSatGeography select(mapColumn(\n\t\tCity,\n\t\tStateProvinceCode,\n\t\tStateProvinceName,\n\t\tCountryRegionCode,\n\t\tEnglishCountryRegionName,\n\t\tPostalCode,\n\t\tHUB_Geography_HK,\n\t\tSAT_Geography_HDIFF,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SelectNewSatGeography\nSelectHubGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tGeographyAlternateKey as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table',\n\tmapColumn(\n\t\tHUB_Geography_HK,\n\t\tGeographyAlternateKey,\n\t\tLOAD_TS\n\t),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SinkHUbGeography\nSelectNewSatGeography sink(input(\n\t\tHUB_Geography_HK as string,\n\t\tCity as string,\n\t\tStateProvinceCode as string,\n\t\tStateProvinceName as string,\n\t\tCountryRegionCode as string,\n\t\tEnglishCountryRegionName as string,\n\t\tPostalCode as string,\n\t\tSAT_Geography_HDIFF as string,\n\t\tLOAD_TS as timestamp\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SinkSatGeography" 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /Samples/dataflow/df_flatten_task_cakes_JSON.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "df_flatten_task_cakes_JSON", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "ds_json_dataflow_sample", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceJSONCakes" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "ds_dataflow_output", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "SinkFlattenOutput" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "FlattenTopping" 27 | }, 28 | { 29 | "name": "FlattenBatter" 30 | } 31 | ], 32 | "script": "source(output(\n\t\tbatters as (batter as (id as string, type as string)[]),\n\t\tid as string,\n\t\tname as string,\n\t\tppu as string,\n\t\ttopping as (id as string, type as string)[],\n\t\ttype as string\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: true,\n\tignoreNoFilesFound: false,\n\tlimit: 100,\n\tdocumentForm: 'arrayOfDocuments',\n\twildcardPaths:['ManyJSON_Objects.json']) ~> SourceJSONCakes\nSourceJSONCakes foldDown(unroll(topping),\n\tmapColumn(\n\t\tid,\n\t\tcake_type = type,\n\t\tcake_name = name,\n\t\tppu,\n\t\ttopping_id = topping.id,\n\t\ttopping_type = topping.type,\n\t\tbatters\n\t),\n\tskipDuplicateMapInputs: false,\n\tskipDuplicateMapOutputs: false) ~> FlattenTopping\nFlattenTopping foldDown(unroll(batters.batter, batters.batter),\n\tmapColumn(\n\t\tid,\n\t\tcake_type,\n\t\tcake_name,\n\t\tppu,\n\t\ttopping_id,\n\t\ttopping_type,\n\t\tbatter_id = batters.batter.id,\n\t\tbatter_type = batters.batter.type\n\t),\n\tskipDuplicateMapInputs: false,\n\tskipDuplicateMapOutputs: false) ~> FlattenBatter\nFlattenBatter sink(input(\n\t\tColumn_1 as string,\n\t\tColumn_2 as string\n\t),\n\tallowSchemaDrift: false,\n\tvalidateSchema: false,\n\tpartitionFileNames:['flatten_output.csv'],\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SinkFlattenOutput" 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /Samples/dataflow/df_stacking_datasets.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "df_stacking_datasets", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "ds_df_temperature_data", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "SourceTemperature" 13 | }, 14 | { 15 | "dataset": { 16 | "referenceName": "ds_df_temperature_header", 17 | "type": "DatasetReference" 18 | }, 19 | "name": "SourceHeader" 20 | } 21 | ], 22 | "sinks": [ 23 | { 24 | "dataset": { 25 | "referenceName": "ds_df_temperature_output", 26 | "type": "DatasetReference" 27 | }, 28 | "name": "SingOutputDataset" 29 | } 30 | ], 31 | "transformations": [ 32 | { 33 | "name": "FilterCanadaData" 34 | }, 35 | { 36 | "name": "CombineDatasets" 37 | }, 38 | { 39 | "name": "DataSortColumn" 40 | }, 41 | { 42 | "name": "SortOutput" 43 | } 44 | ], 45 | "script": "source(output(\n\t\tdt as string,\n\t\tAverageTemperature as string,\n\t\tAverageTemperatureUncertainty as string,\n\t\tCity as string,\n\t\tCountry as string,\n\t\tLatitude as string,\n\t\tLongitude as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceTemperature\nsource(output(\n\t\tColumn_1 as string,\n\t\tColumn_2 as string,\n\t\tColumn_3 as string,\n\t\tColumn_4 as string,\n\t\tColumn_5 as string,\n\t\tColumn_6 as string,\n\t\tColumn_7 as string,\n\t\tColumn_8 as integer\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> SourceHeader\nSourceTemperature filter(Country == 'Canada') ~> FilterCanadaData\nSourceHeader, DataSortColumn union(byName: false)~> CombineDatasets\nFilterCanadaData derive(row_id = 2) ~> DataSortColumn\nCombineDatasets sort(asc(Column_8, true)) ~> SortOutput\nSortOutput sink(input(\n\t\tColumn_1 as string,\n\t\tColumn_2 as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['GlobalLandTemperaturesByMajorCity-Canada.csv'],\n\tmapColumn(\n\t\tColumn_1,\n\t\tColumn_2,\n\t\tColumn_3,\n\t\tColumn_4,\n\t\tColumn_5,\n\t\tColumn_6,\n\t\tColumn_7\n\t),\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> SingOutputDataset" 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /Samples/dataflow/wdf_expand_json_column.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wdf_expand_json_column", 3 | "properties": { 4 | "type": "WranglingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "name": "ds_json_temp", 9 | "script": "", 10 | "dataset": { 11 | "referenceName": "ds_json_temp", 12 | "type": "DatasetReference" 13 | } 14 | } 15 | ], 16 | "sinks": [ 17 | { 18 | "name": "Sink1", 19 | "dataset": { 20 | "referenceName": "ds_json_output_temp", 21 | "type": "DatasetReference" 22 | }, 23 | "script": "sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['Test_JSON_output.txt'],\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> Sink1" 24 | } 25 | ], 26 | "script": "section Section1;\r\nshared ds_json_temp = let\r\n AdfDoc = AzureStorage.DataLakeContents(\"https://azuesu2edusa.dfs.core.windows.net/jsondata/Test_JSON.txt\"),\r\n Csv = Csv.Document(AdfDoc, [Delimiter = \"|\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv]),\r\n PromotedHeaders = Table.PromoteHeaders(Csv, [PromoteAllScalars = true])\r\nin\r\n PromotedHeaders;\r\nshared UserQuery = let\r\n Source = ds_json_temp,\r\n #\"Parsed JSON\" = Table.TransformColumns(Source, {{\"Detail\", each Json.Document(_), type any}}),\r\n #\"Expanded Detail\" = Table.ExpandRecordColumn(#\"Parsed JSON\", \"Detail\", {\"id\", \"type\", \"name\"}, {\"Detail.id\", \"Detail.type\", \"Detail.name\"})\r\nin\r\n #\"Expanded Detail\";\r\n" 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /Samples/dataflow/wdf_json_file.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wdf_json_file", 3 | "properties": { 4 | "type": "WranglingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "name": "ds_dataflow_sample", 9 | "script": "", 10 | "dataset": { 11 | "referenceName": "ds_dataflow_sample", 12 | "type": "DatasetReference" 13 | } 14 | } 15 | ], 16 | "sinks": [ 17 | { 18 | "name": "Sink1", 19 | "dataset": { 20 | "referenceName": "ds_dataflow_output", 21 | "type": "DatasetReference" 22 | }, 23 | "script": "sink(input(\n\t\tColumn_1 as string,\n\t\tColumn_2 as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['adf_wdf_output.csv'],\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\tpartitionBy('hash', 1)) ~> Sink1" 24 | } 25 | ], 26 | "script": "section Section1;\r\nshared ds_dataflow_sample = let\r\n AdfDoc = AzureStorage.BlobContents(\"https://bdatablob.blob.core.windows.net/dataflows/ManyJSON_Objects - CSV.csv\"),\r\n Csv = Csv.Document(AdfDoc, [Delimiter = \"|\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv])\r\nin\r\n Csv;\r\nshared UserQuery = let\r\n Source = ds_dataflow_sample,\r\n #\"Renamed columns\" = Table.RenameColumns(Source, {{\"Column1\", \"column\"}}),\r\n #\"Parse JSON\" = Table.TransformColumns(#\"Renamed columns\",{},Json.Document),\r\n #\"Expand JSON\" = Table.ExpandRecordColumn(#\"Parse JSON\", \"column\", {\"id\", \"type\", \"name\", \"ppu\", \"batters\", \"topping\"}, {\"id\", \"type\", \"name\", \"ppu\", \"batters\", \"topping\"}),\r\n #\"Unroll topping\" = Table.ExpandListColumn(#\"Expand JSON\", \"topping\"),\r\n #\"Expand topping\" = Table.ExpandRecordColumn(#\"Unroll topping\", \"topping\", {\"id\", \"type\"}, {\"topping.id\", \"topping.type\"}),\r\n #\"Expand batters\" = Table.ExpandRecordColumn(#\"Expand topping\", \"batters\", {\"batter\"}, {\"batter\"}),\r\n #\"Unroll batter\" = Table.ExpandListColumn(#\"Expand batters\", \"batter\"),\r\n #\"Expand batter\" = Table.ExpandRecordColumn(#\"Unroll batter\", \"batter\", {\"id\", \"type\"}, {\"batter.id\", \"batter.type\"})\r\nin\r\n #\"Expand batter\";\r\n" 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /Samples/dataset/ADLS_Gen2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ADLS_Gen2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "folder": { 10 | "type": "string" 11 | }, 12 | "filename": { 13 | "type": "string" 14 | } 15 | }, 16 | "folder": { 17 | "name": "TaxiDemo" 18 | }, 19 | "annotations": [], 20 | "type": "DelimitedText", 21 | "typeProperties": { 22 | "location": { 23 | "type": "AzureBlobFSLocation", 24 | "fileName": { 25 | "value": "@dataset().filename", 26 | "type": "Expression" 27 | }, 28 | "fileSystem": { 29 | "value": "@dataset().folder", 30 | "type": "Expression" 31 | } 32 | }, 33 | "columnDelimiter": ",", 34 | "escapeChar": "\\", 35 | "quoteChar": "\"" 36 | }, 37 | "schema": [] 38 | } 39 | } -------------------------------------------------------------------------------- /Samples/dataset/ADLS_dvs_internetsales_init_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ADLS_dvs_internetsales_init_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStore_ls", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureDataLakeStoreLocation", 15 | "fileName": "dvs_internetsales.csv", 16 | "folderPath": "AdventureWorksDataVault/InitialLoad" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [] 24 | } 25 | } -------------------------------------------------------------------------------- /Samples/dataset/ADLS_dvs_product_init_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ADLS_dvs_product_init_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStore_ls", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureDataLakeStoreLocation", 15 | "fileName": "dvs_product.csv", 16 | "folderPath": "AdventureWorksDataVault/InitialLoad" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [ 24 | { 25 | "name": "ProductAlternateKey", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "WeightUnitMeasureCode", 30 | "type": "String" 31 | }, 32 | { 33 | "name": "SizeUnitMeasureCode", 34 | "type": "String" 35 | }, 36 | { 37 | "name": "EnglishProductName", 38 | "type": "String" 39 | }, 40 | { 41 | "name": "StandardCost", 42 | "type": "String" 43 | }, 44 | { 45 | "name": "FinishedGoodsFlag", 46 | "type": "String" 47 | }, 48 | { 49 | "name": "Color", 50 | "type": "String" 51 | }, 52 | { 53 | "name": "SafetyStockLevel", 54 | "type": "String" 55 | }, 56 | { 57 | "name": "ReorderPoint", 58 | "type": "String" 59 | }, 60 | { 61 | "name": "ListPrice", 62 | "type": "String" 63 | }, 64 | { 65 | "name": "Size", 66 | "type": "String" 67 | }, 68 | { 69 | "name": "SizeRange", 70 | "type": "String" 71 | }, 72 | { 73 | "name": "Weight", 74 | "type": "String" 75 | }, 76 | { 77 | "name": "ModelName", 78 | "type": "String" 79 | }, 80 | { 81 | "name": "StartDate", 82 | "type": "String" 83 | }, 84 | { 85 | "name": "EndDate", 86 | "type": "String" 87 | }, 88 | { 89 | "name": "Status", 90 | "type": "String" 91 | }, 92 | { 93 | "name": "HUB_Product_HK", 94 | "type": "String" 95 | }, 96 | { 97 | "name": "SAT_Product_HDIFF", 98 | "type": "String" 99 | }, 100 | { 101 | "name": "LOAD_TS", 102 | "type": "String" 103 | } 104 | ] 105 | } 106 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_HUB_Customer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_HUB_Customer", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Customer_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "CustomerAlternateKey", 20 | "type": "nvarchar" 21 | }, 22 | { 23 | "name": "LOAD_TS", 24 | "type": "datetime2", 25 | "scale": 7 26 | } 27 | ], 28 | "typeProperties": { 29 | "schema": "dbo", 30 | "table": "HUB_Customer" 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_HUB_Geography.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_HUB_Geography", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Geography_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "GeographyAlternateKey", 20 | "type": "nvarchar" 21 | }, 22 | { 23 | "name": "LOAD_TS", 24 | "type": "datetime2", 25 | "scale": 7 26 | } 27 | ], 28 | "typeProperties": { 29 | "schema": "dbo", 30 | "table": "HUB_Geography" 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_HUB_Product.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_HUB_Product", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "HUB_Product_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "ProductAlternateKey", 19 | "type": "nvarchar" 20 | }, 21 | { 22 | "name": "LOAD_TS", 23 | "type": "datetime2", 24 | "scale": 7 25 | } 26 | ], 27 | "typeProperties": { 28 | "tableName": "[dbo].[HUB_Product]" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_LINK_CustomerGeography.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_LINK_CustomerGeography", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "LINK_CustomerGeography_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "HUB_Customer_HK", 20 | "type": "varchar" 21 | }, 22 | { 23 | "name": "HUB_Geography_HK", 24 | "type": "varchar" 25 | }, 26 | { 27 | "name": "CustomerAlternateKey", 28 | "type": "nvarchar" 29 | }, 30 | { 31 | "name": "GeographyAlternateKey", 32 | "type": "nvarchar" 33 | }, 34 | { 35 | "name": "LOAD_TS", 36 | "type": "datetime2", 37 | "scale": 7 38 | } 39 | ], 40 | "typeProperties": { 41 | "schema": "dbo", 42 | "table": "LINK_CustomerGeography" 43 | } 44 | } 45 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_LINK_InternetSales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_LINK_InternetSales", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "LINK_InternetSales_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "HUB_Customer_HK", 19 | "type": "varchar" 20 | }, 21 | { 22 | "name": "HUB_Product_HK", 23 | "type": "varchar" 24 | }, 25 | { 26 | "name": "ProductAlternateKey", 27 | "type": "nvarchar" 28 | }, 29 | { 30 | "name": "CustomerAlternateKey", 31 | "type": "nvarchar" 32 | }, 33 | { 34 | "name": "SalesOrderNumber", 35 | "type": "nvarchar" 36 | }, 37 | { 38 | "name": "SalesOrderLineNumber", 39 | "type": "tinyint", 40 | "precision": 3 41 | }, 42 | { 43 | "name": "LOAD_TS", 44 | "type": "datetime2", 45 | "scale": 7 46 | } 47 | ], 48 | "typeProperties": { 49 | "tableName": "[dbo].[LINK_InternetSales]" 50 | } 51 | } 52 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Customer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Customer", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Customer_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "Title", 20 | "type": "nvarchar" 21 | }, 22 | { 23 | "name": "FirstName", 24 | "type": "nvarchar" 25 | }, 26 | { 27 | "name": "MiddleName", 28 | "type": "nvarchar" 29 | }, 30 | { 31 | "name": "LastName", 32 | "type": "nvarchar" 33 | }, 34 | { 35 | "name": "NameStyle", 36 | "type": "bit" 37 | }, 38 | { 39 | "name": "BirthDate", 40 | "type": "date" 41 | }, 42 | { 43 | "name": "MaritalStatus", 44 | "type": "nchar" 45 | }, 46 | { 47 | "name": "Suffix", 48 | "type": "nvarchar" 49 | }, 50 | { 51 | "name": "Gender", 52 | "type": "nvarchar" 53 | }, 54 | { 55 | "name": "EmailAddress", 56 | "type": "nvarchar" 57 | }, 58 | { 59 | "name": "YearlyIncome", 60 | "type": "money", 61 | "precision": 19, 62 | "scale": 4 63 | }, 64 | { 65 | "name": "TotalChildren", 66 | "type": "tinyint", 67 | "precision": 3 68 | }, 69 | { 70 | "name": "NumberChildrenAtHome", 71 | "type": "tinyint", 72 | "precision": 3 73 | }, 74 | { 75 | "name": "HouseOwnerFlag", 76 | "type": "nchar" 77 | }, 78 | { 79 | "name": "NumberCarsOwned", 80 | "type": "tinyint", 81 | "precision": 3 82 | }, 83 | { 84 | "name": "AddressLine1", 85 | "type": "nvarchar" 86 | }, 87 | { 88 | "name": "AddressLine2", 89 | "type": "nvarchar" 90 | }, 91 | { 92 | "name": "SAT_Customer_HDIFF", 93 | "type": "varchar" 94 | }, 95 | { 96 | "name": "LOAD_TS", 97 | "type": "datetime2", 98 | "scale": 7 99 | } 100 | ], 101 | "typeProperties": { 102 | "schema": "dbo", 103 | "table": "SAT_Customer" 104 | } 105 | } 106 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Customer_Latest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Customer_Latest", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Customer_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "SAT_Customer_HDIFF", 20 | "type": "varchar" 21 | } 22 | ], 23 | "typeProperties": { 24 | "schema": "dbo", 25 | "table": "vw_latest_sat_customer" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Geography.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Geography", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Geography_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "City", 20 | "type": "nvarchar" 21 | }, 22 | { 23 | "name": "StateProvinceCode", 24 | "type": "nvarchar" 25 | }, 26 | { 27 | "name": "StateProvinceName", 28 | "type": "nvarchar" 29 | }, 30 | { 31 | "name": "CountryRegionCode", 32 | "type": "nvarchar" 33 | }, 34 | { 35 | "name": "EnglishCountryRegionName", 36 | "type": "nvarchar" 37 | }, 38 | { 39 | "name": "PostalCode", 40 | "type": "nvarchar" 41 | }, 42 | { 43 | "name": "SAT_Geography_HDIFF", 44 | "type": "varchar" 45 | }, 46 | { 47 | "name": "LOAD_TS", 48 | "type": "datetime2", 49 | "scale": 7 50 | } 51 | ], 52 | "typeProperties": { 53 | "schema": "dbo", 54 | "table": "SAT_Geography" 55 | } 56 | } 57 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Geography_Latest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Geography_Latest", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "AzureSqlTable", 13 | "schema": [ 14 | { 15 | "name": "HUB_Geography_HK", 16 | "type": "varchar" 17 | }, 18 | { 19 | "name": "SAT_Geography_HDIFF", 20 | "type": "varchar" 21 | } 22 | ], 23 | "typeProperties": { 24 | "schema": "dbo", 25 | "table": "vw_latest_sat_geography" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_InternetSales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_InternetSales", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "LINK_InternetSales_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "RevisionNumber", 19 | "type": "tinyint", 20 | "precision": 3 21 | }, 22 | { 23 | "name": "OrderQuantity", 24 | "type": "smallint", 25 | "precision": 5 26 | }, 27 | { 28 | "name": "UnitPrice", 29 | "type": "money", 30 | "precision": 19, 31 | "scale": 4 32 | }, 33 | { 34 | "name": "ExtendedAmount", 35 | "type": "money", 36 | "precision": 19, 37 | "scale": 4 38 | }, 39 | { 40 | "name": "UnitPriceDiscountPct", 41 | "type": "float", 42 | "precision": 15 43 | }, 44 | { 45 | "name": "DiscountAmount", 46 | "type": "float", 47 | "precision": 15 48 | }, 49 | { 50 | "name": "ProductStandardCost", 51 | "type": "money", 52 | "precision": 19, 53 | "scale": 4 54 | }, 55 | { 56 | "name": "TotalProductCost", 57 | "type": "money", 58 | "precision": 19, 59 | "scale": 4 60 | }, 61 | { 62 | "name": "SalesAmount", 63 | "type": "money", 64 | "precision": 19, 65 | "scale": 4 66 | }, 67 | { 68 | "name": "TaxAmt", 69 | "type": "money", 70 | "precision": 19, 71 | "scale": 4 72 | }, 73 | { 74 | "name": "Freight", 75 | "type": "money", 76 | "precision": 19, 77 | "scale": 4 78 | }, 79 | { 80 | "name": "CarrierTrackingNumber", 81 | "type": "nvarchar" 82 | }, 83 | { 84 | "name": "CustomerPONumber", 85 | "type": "nvarchar" 86 | }, 87 | { 88 | "name": "OrderDate", 89 | "type": "datetime", 90 | "precision": 23, 91 | "scale": 3 92 | }, 93 | { 94 | "name": "DueDate", 95 | "type": "datetime", 96 | "precision": 23, 97 | "scale": 3 98 | }, 99 | { 100 | "name": "ShipDate", 101 | "type": "datetime", 102 | "precision": 23, 103 | "scale": 3 104 | }, 105 | { 106 | "name": "SAT_InternetSales_HDIFF", 107 | "type": "varchar" 108 | }, 109 | { 110 | "name": "LOAD_TS", 111 | "type": "datetime2", 112 | "scale": 7 113 | } 114 | ], 115 | "typeProperties": { 116 | "tableName": "[dbo].[SAT_InternetSales]" 117 | } 118 | } 119 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_InternetSales_Latest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_InternetSales_Latest", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "LINK_InternetSales_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "SAT_InternetSales_HDIFF", 19 | "type": "varchar" 20 | } 21 | ], 22 | "typeProperties": { 23 | "tableName": "[dbo].[vw_latest_sat_internetsales]" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Product.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Product", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "HUB_Product_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "WeightUnitMeasureCode", 19 | "type": "nchar" 20 | }, 21 | { 22 | "name": "SizeUnitMeasureCode", 23 | "type": "nchar" 24 | }, 25 | { 26 | "name": "EnglishProductName", 27 | "type": "nvarchar" 28 | }, 29 | { 30 | "name": "StandardCost", 31 | "type": "money", 32 | "precision": 19, 33 | "scale": 4 34 | }, 35 | { 36 | "name": "FinishedGoodsFlag", 37 | "type": "bit" 38 | }, 39 | { 40 | "name": "Color", 41 | "type": "nvarchar" 42 | }, 43 | { 44 | "name": "SafetyStockLevel", 45 | "type": "smallint", 46 | "precision": 5 47 | }, 48 | { 49 | "name": "ReorderPoint", 50 | "type": "smallint", 51 | "precision": 5 52 | }, 53 | { 54 | "name": "ListPrice", 55 | "type": "money", 56 | "precision": 19, 57 | "scale": 4 58 | }, 59 | { 60 | "name": "Size", 61 | "type": "nvarchar" 62 | }, 63 | { 64 | "name": "SizeRange", 65 | "type": "nvarchar" 66 | }, 67 | { 68 | "name": "Weight", 69 | "type": "float", 70 | "precision": 15 71 | }, 72 | { 73 | "name": "ModelName", 74 | "type": "nvarchar" 75 | }, 76 | { 77 | "name": "SAT_Product_HDIFF", 78 | "type": "varchar" 79 | }, 80 | { 81 | "name": "LOAD_TS", 82 | "type": "datetime2", 83 | "scale": 7 84 | } 85 | ], 86 | "typeProperties": { 87 | "tableName": "[dbo].[SAT_Product]" 88 | } 89 | } 90 | } -------------------------------------------------------------------------------- /Samples/dataset/ASQL_DV_SAT_Product_Latest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ASQL_DV_SAT_Product_Latest", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_DV_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "AzureSqlTable", 12 | "schema": [ 13 | { 14 | "name": "HUB_Product_HK", 15 | "type": "varchar" 16 | }, 17 | { 18 | "name": "SAT_Product_HDIFF", 19 | "type": "varchar" 20 | } 21 | ], 22 | "typeProperties": { 23 | "tableName": "[dbo].[vw_latest_sat_product]" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlobStorageSourceFileStore.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobStorageSourceFileStore", 3 | "properties": { 4 | "description": "Connection to your source data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | }, 13 | "filename": { 14 | "type": "string" 15 | } 16 | }, 17 | "annotations": [], 18 | "type": "AzureBlob", 19 | "typeProperties": { 20 | "compression": { 21 | "type": "ZipDeflate" 22 | }, 23 | "fileName": { 24 | "value": "@dataset().filename", 25 | "type": "Expression" 26 | }, 27 | "folderPath": { 28 | "value": "@dataset().folder", 29 | "type": "Expression" 30 | } 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_bgdata_daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_bgdata_daily", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "FilePath": { 10 | "type": "String" 11 | }, 12 | "FileName": { 13 | "type": "String" 14 | } 15 | }, 16 | "folder": { 17 | "name": "Sample Datasets" 18 | }, 19 | "type": "AzureBlob", 20 | "structure": [ 21 | { 22 | "name": "Date", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Account ID", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Sales", 31 | "type": "String" 32 | } 33 | ], 34 | "typeProperties": { 35 | "format": { 36 | "type": "TextFormat", 37 | "columnDelimiter": "\t", 38 | "rowDelimiter": "", 39 | "nullValue": "\\N", 40 | "encodingName": null, 41 | "treatEmptyAsNull": true, 42 | "skipLineCount": 0, 43 | "firstRowAsHeader": true 44 | }, 45 | "fileName": { 46 | "value": "@dataset().FileName", 47 | "type": "Expression" 48 | }, 49 | "folderPath": { 50 | "value": "@dataset().FilePath", 51 | "type": "Expression" 52 | } 53 | } 54 | }, 55 | "type": "Microsoft.DataFactory/factories/datasets" 56 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_bgdata_monthly.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_bgdata_monthly", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "FileName": { 10 | "type": "String" 11 | }, 12 | "FilePath": { 13 | "type": "String" 14 | } 15 | }, 16 | "folder": { 17 | "name": "Sample Datasets" 18 | }, 19 | "type": "AzureBlob", 20 | "structure": [ 21 | { 22 | "name": "Year", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Month", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Account ID", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "Total Sales", 35 | "type": "String" 36 | } 37 | ], 38 | "typeProperties": { 39 | "format": { 40 | "type": "TextFormat", 41 | "columnDelimiter": ",", 42 | "nullValue": "\\N", 43 | "treatEmptyAsNull": true, 44 | "skipLineCount": 0, 45 | "firstRowAsHeader": true 46 | }, 47 | "fileName": { 48 | "value": "@dataset().FileName", 49 | "type": "Expression" 50 | }, 51 | "folderPath": { 52 | "value": "@dataset().FilePath", 53 | "type": "Expression" 54 | } 55 | } 56 | }, 57 | "type": "Microsoft.DataFactory/factories/datasets" 58 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_storesales_store01.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_storesales_store01", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "encodingName": null, 19 | "treatEmptyAsNull": true, 20 | "skipLineCount": 0, 21 | "firstRowAsHeader": true 22 | }, 23 | "fileName": "", 24 | "folderPath": "storesales/store_01" 25 | } 26 | }, 27 | "type": "Microsoft.DataFactory/factories/datasets" 28 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_storesales_store01_single.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_storesales_store01_single", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "store_id", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "date_id", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "product_id", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "stock_amt", 27 | "type": "String" 28 | } 29 | ], 30 | "typeProperties": { 31 | "format": { 32 | "type": "TextFormat", 33 | "columnDelimiter": ",", 34 | "rowDelimiter": "", 35 | "quoteChar": "\"", 36 | "nullValue": "\\N", 37 | "encodingName": null, 38 | "treatEmptyAsNull": true, 39 | "skipLineCount": 0, 40 | "firstRowAsHeader": true 41 | }, 42 | "fileName": "storeinventory_01_20181027.csv", 43 | "folderPath": "storesales-source/store_01" 44 | } 45 | } 46 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_storesales_store02.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_storesales_store02", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "encodingName": null, 19 | "treatEmptyAsNull": true, 20 | "skipLineCount": 0, 21 | "firstRowAsHeader": true 22 | }, 23 | "fileName": "", 24 | "folderPath": "storesales/store_02" 25 | } 26 | }, 27 | "type": "Microsoft.DataFactory/factories/datasets" 28 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_storesales_store03.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_storesales_store03", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "encodingName": null, 19 | "treatEmptyAsNull": true, 20 | "skipLineCount": 0, 21 | "firstRowAsHeader": true 22 | }, 23 | "fileName": "", 24 | "folderPath": "storesales/store_03" 25 | } 26 | }, 27 | "type": "Microsoft.DataFactory/factories/datasets" 28 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureBlob_storesales_storesall.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob_storesales_storesall", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "FilePath": { 10 | "type": "String" 11 | }, 12 | "FileName": { 13 | "type": "String" 14 | } 15 | }, 16 | "folder": { 17 | "name": "Sample Datasets" 18 | }, 19 | "type": "AzureBlob", 20 | "structure": [ 21 | { 22 | "name": "store_id", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "date_id", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "product_id", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "stock_amt", 35 | "type": "String" 36 | } 37 | ], 38 | "typeProperties": { 39 | "format": { 40 | "type": "TextFormat", 41 | "columnDelimiter": ",", 42 | "rowDelimiter": "", 43 | "nullValue": "\\N", 44 | "encodingName": null, 45 | "treatEmptyAsNull": true, 46 | "skipLineCount": 0, 47 | "firstRowAsHeader": true 48 | }, 49 | "fileName": { 50 | "value": "@dataset().FileName", 51 | "type": "Expression" 52 | }, 53 | "folderPath": { 54 | "value": "@dataset().FilePath", 55 | "type": "Expression" 56 | } 57 | } 58 | }, 59 | "type": "Microsoft.DataFactory/factories/datasets" 60 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureDataLakeStoreSinkFileStore.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStoreSinkFileStore", 3 | "properties": { 4 | "description": "Connection to your destination data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureDataLakeStore_ls", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "AzureDataLakeStoreFile", 16 | "typeProperties": { 17 | "fileName": "", 18 | "folderPath": { 19 | "value": "@dataset().folder", 20 | "type": "Expression" 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureSqlTable_bgdata_daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable_bgdata_daily", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "Date", 15 | "type": "DateTime" 16 | }, 17 | { 18 | "name": "Account_ID", 19 | "type": "Int32" 20 | }, 21 | { 22 | "name": "Sales", 23 | "type": "Int32" 24 | } 25 | ], 26 | "typeProperties": { 27 | "tableName": "[dbo].[DailySales]" 28 | } 29 | }, 30 | "type": "Microsoft.DataFactory/factories/datasets" 31 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureSqlTable_bgdata_monthly.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable_bgdata_monthly", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "Year", 15 | "type": "Int32" 16 | }, 17 | { 18 | "name": "Month", 19 | "type": "Int32" 20 | }, 21 | { 22 | "name": "Account_ID", 23 | "type": "Int32" 24 | }, 25 | { 26 | "name": "Total_Sales", 27 | "type": "Int32" 28 | } 29 | ], 30 | "typeProperties": { 31 | "tableName": "[dbo].[MonthlySales]" 32 | } 33 | }, 34 | "type": "Microsoft.DataFactory/factories/datasets" 35 | } -------------------------------------------------------------------------------- /Samples/dataset/AzureSqlTable_storesales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable_storesales", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample Datasets" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "store_id", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "date_id", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "product_id", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "stock_amt", 27 | "type": "String" 28 | } 29 | ], 30 | "typeProperties": { 31 | "tableName": "[dbo].[storeinventory]" 32 | } 33 | }, 34 | "type": "Microsoft.DataFactory/factories/datasets" 35 | } -------------------------------------------------------------------------------- /Samples/dataset/BLOB_MusicNotes.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BLOB_MusicNotes", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Other Datasets" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureBlobStorageLocation", 15 | "fileName": "Notes.csv", 16 | "container": "otherfiles" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [ 24 | { 25 | "name": "ID", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "Note", 30 | "type": "String" 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /Samples/dataset/BLOB_dvs_customer_incr_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BLOB_dvs_customer_incr_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobStorageLocation", 16 | "fileName": "dvs_customer.csv", 17 | "container": "adventureworksdv" 18 | }, 19 | "columnDelimiter": ",", 20 | "escapeChar": "\\", 21 | "firstRowAsHeader": true, 22 | "quoteChar": "\"" 23 | }, 24 | "schema": [ 25 | { 26 | "name": "CustomerAlternateKey", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "GeographyAlternateKey", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "Title", 35 | "type": "String" 36 | }, 37 | { 38 | "name": "FirstName", 39 | "type": "String" 40 | }, 41 | { 42 | "name": "MiddleName", 43 | "type": "String" 44 | }, 45 | { 46 | "name": "LastName", 47 | "type": "String" 48 | }, 49 | { 50 | "name": "NameStyle", 51 | "type": "String" 52 | }, 53 | { 54 | "name": "BirthDate", 55 | "type": "String" 56 | }, 57 | { 58 | "name": "MaritalStatus", 59 | "type": "String" 60 | }, 61 | { 62 | "name": "Suffix", 63 | "type": "String" 64 | }, 65 | { 66 | "name": "Gender", 67 | "type": "String" 68 | }, 69 | { 70 | "name": "EmailAddress", 71 | "type": "String" 72 | }, 73 | { 74 | "name": "YearlyIncome", 75 | "type": "String" 76 | }, 77 | { 78 | "name": "TotalChildren", 79 | "type": "String" 80 | }, 81 | { 82 | "name": "NumberChildrenAtHome", 83 | "type": "String" 84 | }, 85 | { 86 | "name": "HouseOwnerFlag", 87 | "type": "String" 88 | }, 89 | { 90 | "name": "NumberCarsOwned", 91 | "type": "String" 92 | }, 93 | { 94 | "name": "AddressLine1", 95 | "type": "String" 96 | }, 97 | { 98 | "name": "AddressLine2", 99 | "type": "String" 100 | } 101 | ] 102 | } 103 | } -------------------------------------------------------------------------------- /Samples/dataset/BLOB_dvs_geography_incr_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BLOB_dvs_geography_incr_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobStorageLocation", 16 | "container": "adventureworksdv" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [ 24 | { 25 | "name": "GeographyAlternateKey", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "City", 30 | "type": "String" 31 | }, 32 | { 33 | "name": "StateProvinceCode", 34 | "type": "String" 35 | }, 36 | { 37 | "name": "StateProvinceName", 38 | "type": "String" 39 | }, 40 | { 41 | "name": "CountryRegionCode", 42 | "type": "String" 43 | }, 44 | { 45 | "name": "EnglishCountryRegionName", 46 | "type": "String" 47 | }, 48 | { 49 | "name": "PostalCode", 50 | "type": "String" 51 | } 52 | ] 53 | } 54 | } -------------------------------------------------------------------------------- /Samples/dataset/BLOB_dvs_internetsales_incr_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BLOB_dvs_internetsales_incr_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureBlobStorageLocation", 15 | "fileName": "dvs_internetsales_big.csv", 16 | "container": "adventureworksdv" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [ 24 | { 25 | "name": "ProductAlternateKey", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "CustomerAlternateKey", 30 | "type": "String" 31 | }, 32 | { 33 | "name": "SalesOrderNumber", 34 | "type": "String" 35 | }, 36 | { 37 | "name": "SalesOrderLineNumber", 38 | "type": "String" 39 | }, 40 | { 41 | "name": "RevisionNumber", 42 | "type": "String" 43 | }, 44 | { 45 | "name": "OrderQuantity", 46 | "type": "String" 47 | }, 48 | { 49 | "name": "UnitPrice", 50 | "type": "String" 51 | }, 52 | { 53 | "name": "ExtendedAmount", 54 | "type": "String" 55 | }, 56 | { 57 | "name": "UnitPriceDiscountPct", 58 | "type": "String" 59 | }, 60 | { 61 | "name": "DiscountAmount", 62 | "type": "String" 63 | }, 64 | { 65 | "name": "ProductStandardCost", 66 | "type": "String" 67 | }, 68 | { 69 | "name": "TotalProductCost", 70 | "type": "String" 71 | }, 72 | { 73 | "name": "SalesAmount", 74 | "type": "String" 75 | }, 76 | { 77 | "name": "TaxAmt", 78 | "type": "String" 79 | }, 80 | { 81 | "name": "Freight", 82 | "type": "String" 83 | }, 84 | { 85 | "name": "CarrierTrackingNumber", 86 | "type": "String" 87 | }, 88 | { 89 | "name": "CustomerPONumber", 90 | "type": "String" 91 | }, 92 | { 93 | "name": "OrderDate", 94 | "type": "String" 95 | }, 96 | { 97 | "name": "DueDate", 98 | "type": "String" 99 | }, 100 | { 101 | "name": "ShipDate", 102 | "type": "String" 103 | } 104 | ] 105 | } 106 | } -------------------------------------------------------------------------------- /Samples/dataset/BLOB_dvs_product_incr_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BLOB_dvs_product_incr_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "DataVaultTesting" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureBlobStorageLocation", 15 | "fileName": "dvs_product_big.csv", 16 | "container": "adventureworksdv" 17 | }, 18 | "columnDelimiter": ",", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [ 24 | { 25 | "name": "ProductAlternateKey", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "WeightUnitMeasureCode", 30 | "type": "String" 31 | }, 32 | { 33 | "name": "SizeUnitMeasureCode", 34 | "type": "String" 35 | }, 36 | { 37 | "name": "EnglishProductName", 38 | "type": "String" 39 | }, 40 | { 41 | "name": "StandardCost", 42 | "type": "String" 43 | }, 44 | { 45 | "name": "FinishedGoodsFlag", 46 | "type": "String" 47 | }, 48 | { 49 | "name": "Color", 50 | "type": "String" 51 | }, 52 | { 53 | "name": "SafetyStockLevel", 54 | "type": "String" 55 | }, 56 | { 57 | "name": "ReorderPoint", 58 | "type": "String" 59 | }, 60 | { 61 | "name": "ListPrice", 62 | "type": "String" 63 | }, 64 | { 65 | "name": "Size", 66 | "type": "String" 67 | }, 68 | { 69 | "name": "SizeRange", 70 | "type": "String" 71 | }, 72 | { 73 | "name": "Weight", 74 | "type": "String" 75 | }, 76 | { 77 | "name": "ModelName", 78 | "type": "String" 79 | } 80 | ] 81 | } 82 | } -------------------------------------------------------------------------------- /Samples/dataset/TaxiSink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiSink", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "TaxiDemo" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "folderPath": "taxi_output", 17 | "fileSystem": "nyctaxidata-staging" 18 | }, 19 | "columnDelimiter": ",", 20 | "escapeChar": "\\", 21 | "firstRowAsHeader": true, 22 | "quoteChar": "\"" 23 | }, 24 | "schema": [] 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_FileShare_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_FileShare_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "FileSystem_ls", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Template Datasets" 10 | }, 11 | "type": "FileShare", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "*.txt", 23 | "folderPath": "Move" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_delete.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_delete", 3 | "properties": { 4 | "description": "Connection to your source data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | } 13 | }, 14 | "folder": { 15 | "name": "Template Datasets" 16 | }, 17 | "type": "AzureBlob", 18 | "typeProperties": { 19 | "fileName": "*", 20 | "folderPath": { 21 | "value": "@dataset().folder", 22 | "type": "Expression" 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_delete_time.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_delete_time", 3 | "properties": { 4 | "description": "Connection to your source data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | } 13 | }, 14 | "folder": { 15 | "name": "Template Datasets" 16 | }, 17 | "type": "AzureBlob", 18 | "typeProperties": { 19 | "folderPath": { 20 | "value": "@dataset().folder", 21 | "type": "Expression" 22 | }, 23 | "modifiedDatetimeEnd": { 24 | "value": "@addseconds(pipeline().TriggerTime, -10)", 25 | "type": "Expression" 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_single_sink_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_single_sink_ds", 3 | "properties": { 4 | "description": "Connection to your destination data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | }, 13 | "filename": { 14 | "type": "String" 15 | } 16 | }, 17 | "folder": { 18 | "name": "Template Datasets" 19 | }, 20 | "type": "AzureBlob", 21 | "typeProperties": { 22 | "fileName": { 23 | "value": "@dataset().filename", 24 | "type": "Expression" 25 | }, 26 | "folderPath": { 27 | "value": "@dataset().folder", 28 | "type": "Expression" 29 | } 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_single_source_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_single_source_ds", 3 | "properties": { 4 | "description": "Connection to your source data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | }, 13 | "filename": { 14 | "type": "String" 15 | } 16 | }, 17 | "folder": { 18 | "name": "Template Datasets" 19 | }, 20 | "type": "AzureBlob", 21 | "typeProperties": { 22 | "fileName": { 23 | "value": "@dataset().filename", 24 | "type": "Expression" 25 | }, 26 | "folderPath": { 27 | "value": "@dataset().folder", 28 | "type": "Expression" 29 | } 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_sink_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_sink_ds", 3 | "properties": { 4 | "description": "Connection to your destination data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | } 13 | }, 14 | "folder": { 15 | "name": "Template Datasets" 16 | }, 17 | "type": "AzureBlob", 18 | "typeProperties": { 19 | "fileName": "", 20 | "folderPath": { 21 | "value": "@dataset().folder", 22 | "type": "Expression" 23 | } 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /Samples/dataset/Template_wasb_source_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Template_wasb_source_ds", 3 | "properties": { 4 | "description": "Connection to your source data store.", 5 | "linkedServiceName": { 6 | "referenceName": "AzureBlobStorage_LS", 7 | "type": "LinkedServiceReference" 8 | }, 9 | "parameters": { 10 | "folder": { 11 | "type": "String" 12 | } 13 | }, 14 | "folder": { 15 | "name": "Template Datasets" 16 | }, 17 | "type": "AzureBlob", 18 | "typeProperties": { 19 | "folderPath": { 20 | "value": "@dataset().folder", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /Samples/dataset/TripData.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TripData", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "TaxiDemo" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "fileName": "trip_data_1.csv", 17 | "folderPath": "trip_data", 18 | "fileSystem": "nyctaxidata-staging" 19 | }, 20 | "columnDelimiter": ",", 21 | "escapeChar": "\\", 22 | "firstRowAsHeader": true, 23 | "quoteChar": "\"" 24 | }, 25 | "schema": [ 26 | { 27 | "name": "medallion", 28 | "type": "String" 29 | }, 30 | { 31 | "name": "hack_license", 32 | "type": "String" 33 | }, 34 | { 35 | "name": "vendor_id", 36 | "type": "String" 37 | }, 38 | { 39 | "name": "rate_code", 40 | "type": "String" 41 | }, 42 | { 43 | "name": "store_and_fwd_flag", 44 | "type": "String" 45 | }, 46 | { 47 | "name": "pickup_datetime", 48 | "type": "String" 49 | }, 50 | { 51 | "name": "dropoff_datetime", 52 | "type": "String" 53 | }, 54 | { 55 | "name": "passenger_count", 56 | "type": "String" 57 | }, 58 | { 59 | "name": "trip_time_in_secs", 60 | "type": "String" 61 | }, 62 | { 63 | "name": "trip_distance", 64 | "type": "String" 65 | }, 66 | { 67 | "name": "pickup_longitude", 68 | "type": "String" 69 | }, 70 | { 71 | "name": "pickup_latitude", 72 | "type": "String" 73 | }, 74 | { 75 | "name": "dropoff_longitude", 76 | "type": "String" 77 | }, 78 | { 79 | "name": "dropoff_latitude", 80 | "type": "String" 81 | } 82 | ] 83 | } 84 | } -------------------------------------------------------------------------------- /Samples/dataset/TripFares.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TripFares", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "TaxiDemo" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "fileName": "trip_fare_1.csv", 17 | "folderPath": "trip_fare", 18 | "fileSystem": "nyctaxidata-staging" 19 | }, 20 | "columnDelimiter": ",", 21 | "escapeChar": "\\", 22 | "firstRowAsHeader": true, 23 | "quoteChar": "\"" 24 | }, 25 | "schema": [ 26 | { 27 | "name": "medallion", 28 | "type": "String" 29 | }, 30 | { 31 | "name": "hack_license", 32 | "type": "String" 33 | }, 34 | { 35 | "name": "vendor_id", 36 | "type": "String" 37 | }, 38 | { 39 | "name": "pickup_datetime", 40 | "type": "String" 41 | }, 42 | { 43 | "name": "payment_type", 44 | "type": "String" 45 | }, 46 | { 47 | "name": "fare_amount", 48 | "type": "String" 49 | }, 50 | { 51 | "name": "surcharge", 52 | "type": "String" 53 | }, 54 | { 55 | "name": "mta_tax", 56 | "type": "String" 57 | }, 58 | { 59 | "name": "tip_amount", 60 | "type": "String" 61 | }, 62 | { 63 | "name": "tolls_amount", 64 | "type": "String" 65 | }, 66 | { 67 | "name": "total_amount", 68 | "type": "String" 69 | } 70 | ] 71 | } 72 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_adls_adf_dataflows.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_adls_adf_dataflows", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "file_name": { 10 | "type": "string", 11 | "defaultValue": "DtcInstall.log" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "DelimitedText", 16 | "typeProperties": { 17 | "location": { 18 | "type": "AzureBlobFSLocation", 19 | "fileName": { 20 | "value": "@dataset().file_name", 21 | "type": "Expression" 22 | }, 23 | "fileSystem": "adf-dataflows" 24 | }, 25 | "columnDelimiter": ",", 26 | "escapeChar": "\\", 27 | "quoteChar": "\"" 28 | }, 29 | "schema": [ 30 | { 31 | "type": "String" 32 | }, 33 | { 34 | "type": "String" 35 | }, 36 | { 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_adls_generic_folder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_adls_generic_folder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "file_folder": { 10 | "type": "string" 11 | } 12 | }, 13 | "annotations": [], 14 | "type": "DelimitedText", 15 | "typeProperties": { 16 | "location": { 17 | "type": "AzureBlobFSLocation", 18 | "folderPath": { 19 | "value": "@dataset().file_folder", 20 | "type": "Expression" 21 | } 22 | }, 23 | "columnDelimiter": ",", 24 | "escapeChar": "\\", 25 | "quoteChar": "\"" 26 | }, 27 | "schema": [] 28 | } 29 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_azsql_generic.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_azsql_generic", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [] 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_dataflow_output.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_dataflow_output", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "dataflows" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": false, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "type": "String" 23 | }, 24 | { 25 | "type": "String" 26 | } 27 | ] 28 | } 29 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_dataflow_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_dataflow_sample", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "ManyJSON_Objects - CSV.csv", 14 | "container": "jsondata" 15 | }, 16 | "columnDelimiter": "|", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "name": "{\"id\":\"0001\",\"type\":\"donut\",\"name\":\"Cake\",\"ppu\":0.55,\"batters\":{\"batter\":[{\"id\":\"1001\",\"type\":\"Regular\"},{\"id\":\"1002\",\"type\":\"Chocolate\"},{\"id\":\"1003\",\"type\":\"Blueberry\"},{\"id\":\"1004\",\"type\":\"Devil's Food\"}]},\"topping\":[{\"id\":\"5001\",\"type\":\"None\"},{\"id\":\"5002\",\"type\":\"Glazed\"},{\"id\":\"5005\",\"type\":\"Sugar\"},{\"id\":\"5007\",\"type\":\"Powdered Sugar\"},{\"id\":\"5006\",\"type\":\"Chocolate with Sprinkles\"},{\"id\":\"5003\",\"type\":\"Chocolate\"},{\"id\":\"5004\",\"type\":\"Maple\"}]}", 24 | "type": "String" 25 | } 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_df_temperature_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_df_temperature_data", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "GlobalLandTemperaturesByMajorCity.csv", 14 | "container": "dataflows" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "name": "dt", 24 | "type": "String" 25 | }, 26 | { 27 | "name": "AverageTemperature", 28 | "type": "String" 29 | }, 30 | { 31 | "name": "AverageTemperatureUncertainty", 32 | "type": "String" 33 | }, 34 | { 35 | "name": "City", 36 | "type": "String" 37 | }, 38 | { 39 | "name": "Country", 40 | "type": "String" 41 | }, 42 | { 43 | "name": "Latitude", 44 | "type": "String" 45 | }, 46 | { 47 | "name": "Longitude", 48 | "type": "String" 49 | } 50 | ] 51 | } 52 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_df_temperature_header.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_df_temperature_header", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "GlobalLandTemperaturesByMajorCity-Header.csv", 14 | "container": "dataflows" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "type": "String" 23 | }, 24 | { 25 | "type": "String" 26 | }, 27 | { 28 | "type": "String" 29 | }, 30 | { 31 | "type": "String" 32 | }, 33 | { 34 | "type": "String" 35 | }, 36 | { 37 | "type": "String" 38 | }, 39 | { 40 | "type": "String" 41 | }, 42 | { 43 | "type": "String" 44 | } 45 | ] 46 | } 47 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_df_temperature_output.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_df_temperature_output", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "dataflows" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [ 20 | { 21 | "type": "String" 22 | }, 23 | { 24 | "type": "String" 25 | } 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_json_dataflow_sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_json_dataflow_sample", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "ManyJSON_Objects.json", 14 | "container": "jsondata" 15 | } 16 | }, 17 | "schema": {} 18 | } 19 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_json_output_temp.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_json_output_temp", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileSystem": "jsondata" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /Samples/dataset/ds_json_temp.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ds_json_temp", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage_Gen2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "Test_JSON.txt", 14 | "fileSystem": "jsondata" 15 | }, 16 | "columnDelimiter": "|", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "name": "EmpId", 24 | "type": "String" 25 | }, 26 | { 27 | "name": "EmpName", 28 | "type": "String" 29 | }, 30 | { 31 | "name": "Detail", 32 | "type": "String" 33 | } 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /Samples/dataset/ls_json_flatten_data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ls_json_flatten_data", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage_LS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "json_data_file.json", 14 | "container": "dataflows" 15 | } 16 | }, 17 | "schema": { 18 | "type": "object", 19 | "properties": { 20 | "events": { 21 | "type": "array", 22 | "items": { 23 | "type": "object", 24 | "properties": { 25 | "process": { 26 | "type": "string" 27 | }, 28 | "start_time": { 29 | "type": "string" 30 | }, 31 | "stop_time": { 32 | "type": "string" 33 | }, 34 | "last_update": { 35 | "type": "string" 36 | } 37 | } 38 | } 39 | } 40 | } 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /Samples/dataset/validation_adls_source_ds.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "validation_adls_source_ds", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStore_ls", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "FileFolder": { 10 | "type": "String" 11 | } 12 | }, 13 | "folder": { 14 | "name": "Other Datasets" 15 | }, 16 | "type": "AzureDataLakeStoreFile", 17 | "typeProperties": { 18 | "format": { 19 | "type": "TextFormat", 20 | "columnDelimiter": ",", 21 | "rowDelimiter": "", 22 | "nullValue": "\\N", 23 | "treatEmptyAsNull": true, 24 | "skipLineCount": 0, 25 | "firstRowAsHeader": false 26 | }, 27 | "folderPath": { 28 | "value": "@dataset().FileFolder", 29 | "type": "Expression" 30 | } 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureBlobStorage_LS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobStorage_LS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobStorage", 7 | "typeProperties": { 8 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=azueus2edusa;EndpointSuffix=core.windows.net;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkRBVEFGQUNUT1JZQDYzRkNGOTVELTk3Q0QtNEJFNC1CMDZELTFGNUEzOTc2RjI4Rl9lYWI5MzU0MS03NDY2LTRiMjctOTY5NC03MTc5NmFiNGU5ZDUiDQp9" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureDataLakeStorage_Gen2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage_Gen2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobFS", 7 | "typeProperties": { 8 | "url": "https://azuesu2edusa.dfs.core.windows.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFaVS1FVVMyLURBVEFGQUNUT1JZXzdmOTQ0NGU2LWZlMTctNDJmZC05NjUyLWExOTExM2VmMzMyYSINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureDataLakeStore_ls.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStore_ls", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureDataLakeStore", 6 | "typeProperties": { 7 | "dataLakeStoreUri": "https://azueus2devadlsbdatalake.azuredatalakestore.net/webhdfs/v1", 8 | "tenant": "31d5acce-99ba-40b7-b2a1-f329e04fe742", 9 | "subscriptionId": "6c405d27-de5e-483c-9a68-0aaaad20b363", 10 | "resourceGroupName": "big_data" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureKeyVault_LS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureKeyVault_LS", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureKeyVault", 6 | "typeProperties": { 7 | "baseUrl": "https://azu-eus2-edu-kvt.vault.azure.net/" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureSqlDatabase_DV_LS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase_DV_LS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDatabase", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=azu-eus2-edusqlserver.database.windows.net;initial catalog=azu-eus2-edusqldb;user id=rayis", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFaVS1FVVMyLURBVEFGQUNUT1JZXzc3YTA4ZTBmLTliNjktNGM1NS1iNDIwLTUzMjMwODg2ZDFiOSINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/AzureSqlDatabase_LS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase_LS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDatabase", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=azu-eus2-edusqlserver.database.windows.net;initial catalog=azu-eus2-edusqldb;user id=rayis", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFaVS1FVVMyLURBVEFGQUNUT1JZXzk4MGJmMzE1LWUxNjQtNGE5ZS04Nzc1LTA1NTA4Y2I1ODFmYSINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/FileSystem_ls.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FileSystem_ls", 3 | "properties": { 4 | "type": "FileServer", 5 | "typeProperties": { 6 | "host": "C:\\Temp", 7 | "userId": "corp\\rayis.imayev", 8 | "password": { 9 | "type": "AzureKeyVaultSecret", 10 | "store": { 11 | "referenceName": "AzureKeyVault_LS", 12 | "type": "LinkedServiceReference" 13 | }, 14 | "secretName": "local-machine-account-password" 15 | } 16 | }, 17 | "annotations": [] 18 | }, 19 | "type": "Microsoft.DataFactory/factories/linkedservices" 20 | } -------------------------------------------------------------------------------- /Samples/linkedService/Spark1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Spark1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "Spark", 7 | "typeProperties": { 8 | "host": "eastus2.azuredatabricks.net", 9 | "port": 443, 10 | "serverType": "SparkThriftServer", 11 | "thriftTransportProtocol": "HTTP", 12 | "authenticationType": "UsernameAndPassword", 13 | "username": "token", 14 | "httpPath": "sql/protocolv1/o/1381793306306739/1214-144601-sedan385", 15 | "enableSsl": true, 16 | "useSystemTrustStore": false, 17 | "allowHostNameCNMismatch": false, 18 | "allowSelfSignedServerCert": false, 19 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkJEQVRBRkFDVE9SWV8xOTQ0NTYyZi1mNjMxLTRkMGEtYjhkMi0wMTU2N2MyZWJiOGMiDQp9" 20 | } 21 | } 22 | } -------------------------------------------------------------------------------- /Samples/linkedService/azuredatabricks_bigdata_ls.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "azuredatabricks_bigdata_ls", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureDatabricks", 6 | "typeProperties": { 7 | "domain": "https://eastus.azuredatabricks.net", 8 | "existingClusterId": "0402-023740-highs255", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkJEQVRBRkFDVE9SWV84OGVjNmFlMC1mOGQyLTQ4ZTMtOTkwMi00MGVlNGQ2ZGNlYTAiDQp9" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/ls_af.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ls_af", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureFunction", 7 | "typeProperties": { 8 | "functionAppUrl": "https://azu-eus-edu-fapp.azurewebsites.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkFaVS1FVVMyLURBVEFGQUNUT1JZX2U3ODBhMjViLWRkYTAtNDY2YS05MTIyLTMzMGVhZmM0MjhlZCINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/linkedService/ls_af_gettime.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ls_af_gettime", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureFunction", 7 | "typeProperties": { 8 | "functionAppUrl": "https://bd-eus2-fapp.azurewebsites.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkJEQVRBRkFDVE9SWV9mMGM2MDI1Yy0wYzhkLTQxNzctOGUyNy1kMmFlM2Q5YWIwOGIiDQp9" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /Samples/pipeline/CopyFiles_DeleteFiles_Blob.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CopyFiles_DeleteFiles_Blob", 3 | "properties": { 4 | "description": "Copy all files across multiple containers or buckets from one file-based storage store to another and Email Notification", 5 | "activities": [ 6 | { 7 | "name": "EnumerateContainersList", 8 | "description": "Scan the file path on your source storage store and get the container list.", 9 | "type": "GetMetadata", 10 | "policy": { 11 | "timeout": "7.00:00:00", 12 | "retry": 0, 13 | "retryIntervalInSeconds": 30, 14 | "secureOutput": false, 15 | "secureInput": false 16 | }, 17 | "typeProperties": { 18 | "dataset": { 19 | "referenceName": "Template_wasb_source_ds", 20 | "type": "DatasetReference", 21 | "parameters": { 22 | "folder": { 23 | "value": "@pipeline().parameters.SourceFilePath", 24 | "type": "Expression" 25 | } 26 | } 27 | }, 28 | "fieldList": [ 29 | "childItems" 30 | ] 31 | } 32 | }, 33 | { 34 | "name": "ForEachContainer", 35 | "description": "Get the container list from Metadata activity and then iterates each of them to Copy activity.", 36 | "type": "ForEach", 37 | "dependsOn": [ 38 | { 39 | "activity": "EnumerateContainersList", 40 | "dependencyConditions": [ 41 | "Succeeded" 42 | ] 43 | } 44 | ], 45 | "typeProperties": { 46 | "items": { 47 | "value": "@activity('EnumerateContainersList').output.childItems", 48 | "type": "Expression" 49 | }, 50 | "batchCount": 4, 51 | "activities": [ 52 | { 53 | "name": "CopyOneContainer", 54 | "description": "Copy each container from source storage store to destination store.", 55 | "type": "Copy", 56 | "policy": { 57 | "timeout": "7.00:00:00", 58 | "retry": 0, 59 | "retryIntervalInSeconds": 30, 60 | "secureOutput": false, 61 | "secureInput": false 62 | }, 63 | "userProperties": [ 64 | { 65 | "name": "Source", 66 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)" 67 | }, 68 | { 69 | "name": "Destination", 70 | "value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)" 71 | } 72 | ], 73 | "typeProperties": { 74 | "source": { 75 | "type": "BlobSource", 76 | "recursive": true 77 | }, 78 | "sink": { 79 | "type": "BlobSink", 80 | "copyBehavior": "PreserveHierarchy" 81 | }, 82 | "enableStaging": false, 83 | "dataIntegrationUnits": 0 84 | }, 85 | "inputs": [ 86 | { 87 | "referenceName": "Template_wasb_source_ds", 88 | "type": "DatasetReference", 89 | "parameters": { 90 | "folder": { 91 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)", 92 | "type": "Expression" 93 | } 94 | } 95 | } 96 | ], 97 | "outputs": [ 98 | { 99 | "referenceName": "Template_wasb_sink_ds", 100 | "type": "DatasetReference", 101 | "parameters": { 102 | "folder": { 103 | "value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)", 104 | "type": "Expression" 105 | } 106 | } 107 | } 108 | ] 109 | }, 110 | { 111 | "name": "DeleteFiles", 112 | "type": "Delete", 113 | "dependsOn": [ 114 | { 115 | "activity": "CopyOneContainer", 116 | "dependencyConditions": [ 117 | "Succeeded" 118 | ] 119 | } 120 | ], 121 | "policy": { 122 | "timeout": "7.00:00:00", 123 | "retry": 0, 124 | "retryIntervalInSeconds": 30, 125 | "secureOutput": false, 126 | "secureInput": false 127 | }, 128 | "typeProperties": { 129 | "dataset": { 130 | "referenceName": "Template_wasb_delete", 131 | "type": "DatasetReference", 132 | "parameters": { 133 | "folder": { 134 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)", 135 | "type": "Expression" 136 | } 137 | } 138 | }, 139 | "recursive": true, 140 | "logStorageSettings": { 141 | "linkedServiceName": { 142 | "referenceName": "AzureBlobStorage_LS", 143 | "type": "LinkedServiceReference" 144 | }, 145 | "path": "logging" 146 | }, 147 | "enableLogging": true 148 | } 149 | } 150 | ] 151 | } 152 | } 153 | ], 154 | "parameters": { 155 | "SourceFilePath": { 156 | "type": "String", 157 | "defaultValue": "/storesales" 158 | }, 159 | "DestinationFilePath": { 160 | "type": "String", 161 | "defaultValue": "/storesales-staging" 162 | }, 163 | "par_email_address": { 164 | "type": "String", 165 | "defaultValue": "rimayev@hotmail.com" 166 | } 167 | }, 168 | "folder": { 169 | "name": "Copy Activity" 170 | } 171 | } 172 | } -------------------------------------------------------------------------------- /Samples/pipeline/CopyFiles_Message_Email.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CopyFiles_Message_Email", 3 | "properties": { 4 | "description": "Copy all files across multiple containers or buckets from one file-based storage store to another and Email Notification", 5 | "activities": [ 6 | { 7 | "name": "EnumerateContainersList", 8 | "description": "Scan the file path on your source storage store and get the container list.", 9 | "type": "GetMetadata", 10 | "policy": { 11 | "timeout": "7.00:00:00", 12 | "retry": 0, 13 | "retryIntervalInSeconds": 30, 14 | "secureOutput": false, 15 | "secureInput": false 16 | }, 17 | "typeProperties": { 18 | "dataset": { 19 | "referenceName": "Template_wasb_source_ds", 20 | "type": "DatasetReference", 21 | "parameters": { 22 | "folder": { 23 | "value": "@pipeline().parameters.SourceFilePath", 24 | "type": "Expression" 25 | } 26 | } 27 | }, 28 | "fieldList": [ 29 | "childItems" 30 | ] 31 | } 32 | }, 33 | { 34 | "name": "ForEachContainer", 35 | "description": "Get the container list from Metadata activity and then iterates each of them to Copy activity.", 36 | "type": "ForEach", 37 | "dependsOn": [ 38 | { 39 | "activity": "EnumerateContainersList", 40 | "dependencyConditions": [ 41 | "Succeeded" 42 | ] 43 | } 44 | ], 45 | "typeProperties": { 46 | "items": { 47 | "value": "@activity('EnumerateContainersList').output.childItems", 48 | "type": "Expression" 49 | }, 50 | "batchCount": 4, 51 | "activities": [ 52 | { 53 | "name": "CopyOneContainer", 54 | "description": "Copy each container from source storage store to destination store.", 55 | "type": "Copy", 56 | "policy": { 57 | "timeout": "7.00:00:00", 58 | "retry": 0, 59 | "retryIntervalInSeconds": 30, 60 | "secureOutput": false, 61 | "secureInput": false 62 | }, 63 | "userProperties": [ 64 | { 65 | "name": "Source", 66 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)" 67 | }, 68 | { 69 | "name": "Destination", 70 | "value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)" 71 | } 72 | ], 73 | "typeProperties": { 74 | "source": { 75 | "type": "BlobSource", 76 | "recursive": true 77 | }, 78 | "sink": { 79 | "type": "BlobSink", 80 | "copyBehavior": "PreserveHierarchy" 81 | }, 82 | "enableStaging": false, 83 | "dataIntegrationUnits": 0 84 | }, 85 | "inputs": [ 86 | { 87 | "referenceName": "Template_wasb_source_ds", 88 | "type": "DatasetReference", 89 | "parameters": { 90 | "folder": { 91 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)", 92 | "type": "Expression" 93 | } 94 | } 95 | } 96 | ], 97 | "outputs": [ 98 | { 99 | "referenceName": "Template_wasb_sink_ds", 100 | "type": "DatasetReference", 101 | "parameters": { 102 | "folder": { 103 | "value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)", 104 | "type": "Expression" 105 | } 106 | } 107 | } 108 | ] 109 | } 110 | ] 111 | } 112 | }, 113 | { 114 | "name": "Email Notification", 115 | "type": "WebActivity", 116 | "dependsOn": [ 117 | { 118 | "activity": "ForEachContainer", 119 | "dependencyConditions": [ 120 | "Succeeded" 121 | ] 122 | } 123 | ], 124 | "policy": { 125 | "timeout": "7.00:00:00", 126 | "retry": 0, 127 | "retryIntervalInSeconds": 30, 128 | "secureOutput": false, 129 | "secureInput": false 130 | }, 131 | "typeProperties": { 132 | "url": "https://prod-31.eastus.logic.azure.com:443/workflows/001cc7857fac4b41bd535a8ca3ec9e3e/triggers/manual/paths/invoke?api-version=2016-10-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=f-qhwimTZz8R7fgJ3nz5VTqx0sPOD4akHLeBIDDt5Mg", 133 | "method": "POST", 134 | "headers": { 135 | "Content-Type": "application/json" 136 | }, 137 | "body": { 138 | "DataFactoryName": "@{pipeline().DataFactory}", 139 | "PipelineName": "@{pipeline().Pipeline}", 140 | "Message": "Copy Operation finished.", 141 | "EmailTo": "@pipeline().parameters.par_email_address" 142 | } 143 | } 144 | } 145 | ], 146 | "parameters": { 147 | "SourceFilePath": { 148 | "type": "String", 149 | "defaultValue": "/storesales" 150 | }, 151 | "DestinationFilePath": { 152 | "type": "String", 153 | "defaultValue": "/storesales-staging" 154 | }, 155 | "par_email_address": { 156 | "type": "String", 157 | "defaultValue": "rimayev@hotmail.com" 158 | } 159 | }, 160 | "folder": { 161 | "name": "Copy Activity" 162 | } 163 | } 164 | } -------------------------------------------------------------------------------- /Samples/pipeline/CopyFiles_Message_ServiceBus.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CopyFiles_Message_ServiceBus", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "List of files", 7 | "description": "Scan the file path on your source storage store and get the container list.", 8 | "type": "GetMetadata", 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "typeProperties": { 17 | "dataset": { 18 | "referenceName": "Template_wasb_source_ds", 19 | "type": "DatasetReference", 20 | "parameters": { 21 | "folder": { 22 | "value": "@pipeline().parameters.SourceFilePath", 23 | "type": "Expression" 24 | } 25 | } 26 | }, 27 | "fieldList": [ 28 | "childItems" 29 | ] 30 | } 31 | }, 32 | { 33 | "name": "ForEachContainer", 34 | "description": "Get the container list from Metadata activity and then iterates each of them to Copy activity.", 35 | "type": "ForEach", 36 | "dependsOn": [ 37 | { 38 | "activity": "List of files", 39 | "dependencyConditions": [ 40 | "Succeeded" 41 | ] 42 | } 43 | ], 44 | "typeProperties": { 45 | "items": { 46 | "value": "@activity('List of files').output.childItems", 47 | "type": "Expression" 48 | }, 49 | "batchCount": 1, 50 | "activities": [ 51 | { 52 | "name": "CopyOneContainer", 53 | "description": "Copy each container from source storage store to destination store.", 54 | "type": "Copy", 55 | "policy": { 56 | "timeout": "7.00:00:00", 57 | "retry": 0, 58 | "retryIntervalInSeconds": 30, 59 | "secureOutput": false, 60 | "secureInput": false 61 | }, 62 | "userProperties": [ 63 | { 64 | "name": "Source", 65 | "value": "@concat(pipeline().parameters.SourceFilePath,'/',item().name)" 66 | }, 67 | { 68 | "name": "Destination", 69 | "value": "@concat(pipeline().parameters.DestinationFilePath,'/',item().name)" 70 | } 71 | ], 72 | "typeProperties": { 73 | "source": { 74 | "type": "BlobSource", 75 | "recursive": true 76 | }, 77 | "sink": { 78 | "type": "BlobSink", 79 | "copyBehavior": "PreserveHierarchy" 80 | }, 81 | "enableStaging": false, 82 | "dataIntegrationUnits": 0 83 | }, 84 | "inputs": [ 85 | { 86 | "referenceName": "Template_wasb_single_source_ds", 87 | "type": "DatasetReference", 88 | "parameters": { 89 | "folder": { 90 | "value": "@pipeline().parameters.SourceFilePath", 91 | "type": "Expression" 92 | }, 93 | "filename": { 94 | "value": "@item().name", 95 | "type": "Expression" 96 | } 97 | } 98 | } 99 | ], 100 | "outputs": [ 101 | { 102 | "referenceName": "Template_wasb_single_sink_ds", 103 | "type": "DatasetReference", 104 | "parameters": { 105 | "folder": { 106 | "value": "@pipeline().parameters.DestinationFilePath", 107 | "type": "Expression" 108 | }, 109 | "filename": { 110 | "value": "@item().name", 111 | "type": "Expression" 112 | } 113 | } 114 | } 115 | ] 116 | }, 117 | { 118 | "name": "Message ID", 119 | "type": "SetVariable", 120 | "dependsOn": [ 121 | { 122 | "activity": "CopyOneContainer", 123 | "dependencyConditions": [ 124 | "Succeeded" 125 | ] 126 | } 127 | ], 128 | "typeProperties": { 129 | "variableName": "MessageID", 130 | "value": { 131 | "value": "@concat(utcnow(),'_',pipeline().RunId)", 132 | "type": "Expression" 133 | } 134 | } 135 | }, 136 | { 137 | "name": "Queue Message", 138 | "type": "SetVariable", 139 | "dependsOn": [ 140 | { 141 | "activity": "Message ID", 142 | "dependencyConditions": [ 143 | "Succeeded" 144 | ] 145 | } 146 | ], 147 | "typeProperties": { 148 | "variableName": "Message", 149 | "value": { 150 | "value": "@concat('{\"MessageID\":\"', variables('MessageID'), '\",\"FileName\":\"', item().name, '\",\"FileSize\":\"', activity('CopyOneContainer').output.dataWritten,'\"}')", 151 | "type": "Expression" 152 | } 153 | } 154 | }, 155 | { 156 | "name": "Send Queue Message", 157 | "type": "WebActivity", 158 | "dependsOn": [ 159 | { 160 | "activity": "Queue Message", 161 | "dependencyConditions": [ 162 | "Succeeded" 163 | ] 164 | } 165 | ], 166 | "policy": { 167 | "timeout": "7.00:00:00", 168 | "retry": 0, 169 | "retryIntervalInSeconds": 30, 170 | "secureOutput": false, 171 | "secureInput": false 172 | }, 173 | "typeProperties": { 174 | "url": "https://prod-14.eastus2.logic.azure.com:443/workflows/6a0dc1f33be54f2b8f75ca9c835344b8/triggers/manual/paths/invoke?api-version=2016-10-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=GkGC-3VSmcQEN6CC4yF8blYGg6mqO-URjSCBb0Ig38c", 175 | "method": "POST", 176 | "headers": { 177 | "Content-Type": "application/json" 178 | }, 179 | "body": { 180 | "Message": "@variables('Message')", 181 | "SessionID": "@pipeline().RunId" 182 | } 183 | } 184 | } 185 | ] 186 | } 187 | } 188 | ], 189 | "parameters": { 190 | "SourceFilePath": { 191 | "type": "String", 192 | "defaultValue": "/storesalesfiles" 193 | }, 194 | "DestinationFilePath": { 195 | "type": "String", 196 | "defaultValue": "/storesalesfiles-staging" 197 | } 198 | }, 199 | "variables": { 200 | "MessageID": { 201 | "type": "String" 202 | }, 203 | "Message": { 204 | "type": "String" 205 | } 206 | }, 207 | "folder": { 208 | "name": "Copy Activity" 209 | } 210 | } 211 | } -------------------------------------------------------------------------------- /Samples/pipeline/DV_IncrementalLoad_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DV_IncrementalLoad_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "DV_Product", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [ 9 | { 10 | "activity": "DV_Geography", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "policy": { 17 | "timeout": "7.00:00:00", 18 | "retry": 0, 19 | "retryIntervalInSeconds": 30, 20 | "secureOutput": false, 21 | "secureInput": false 22 | }, 23 | "userProperties": [], 24 | "typeProperties": { 25 | "dataflow": { 26 | "referenceName": "DataVault_Product", 27 | "type": "DataFlowReference" 28 | } 29 | } 30 | }, 31 | { 32 | "name": "DV_InternetSales", 33 | "type": "ExecuteDataFlow", 34 | "dependsOn": [ 35 | { 36 | "activity": "DV_Product", 37 | "dependencyConditions": [ 38 | "Succeeded" 39 | ] 40 | } 41 | ], 42 | "policy": { 43 | "timeout": "7.00:00:00", 44 | "retry": 0, 45 | "retryIntervalInSeconds": 30, 46 | "secureOutput": false, 47 | "secureInput": false 48 | }, 49 | "userProperties": [], 50 | "typeProperties": { 51 | "dataflow": { 52 | "referenceName": "DataVault_InternetSales", 53 | "type": "DataFlowReference" 54 | } 55 | } 56 | }, 57 | { 58 | "name": "DV_Geography", 59 | "type": "ExecuteDataFlow", 60 | "dependsOn": [ 61 | { 62 | "activity": "DV_Customer", 63 | "dependencyConditions": [ 64 | "Succeeded" 65 | ] 66 | } 67 | ], 68 | "policy": { 69 | "timeout": "7.00:00:00", 70 | "retry": 0, 71 | "retryIntervalInSeconds": 30, 72 | "secureOutput": false, 73 | "secureInput": false 74 | }, 75 | "userProperties": [], 76 | "typeProperties": { 77 | "dataflow": { 78 | "referenceName": "DataVault_Geography", 79 | "type": "DataFlowReference" 80 | } 81 | } 82 | }, 83 | { 84 | "name": "DV_Customer", 85 | "type": "ExecuteDataFlow", 86 | "dependsOn": [], 87 | "policy": { 88 | "timeout": "7.00:00:00", 89 | "retry": 0, 90 | "retryIntervalInSeconds": 30, 91 | "secureOutput": false, 92 | "secureInput": false 93 | }, 94 | "userProperties": [], 95 | "typeProperties": { 96 | "dataflow": { 97 | "referenceName": "DataVault_Customer", 98 | "type": "DataFlowReference" 99 | } 100 | } 101 | } 102 | ], 103 | "folder": { 104 | "name": "DataVaultTesting" 105 | }, 106 | "annotations": [] 107 | } 108 | } -------------------------------------------------------------------------------- /Samples/pipeline/Delete files with Azure Data Factory FileSystem.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Delete files with Azure Data Factory FileSystem", 3 | "properties": { 4 | "description": "Please input a folder path as the parameter for this pipeline, and the pipeline will delete files which were last modified more than 30 days ago under the particular folder path.", 5 | "activities": [ 6 | { 7 | "name": "DeleteFiles", 8 | "type": "Delete", 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "typeProperties": { 17 | "dataset": { 18 | "referenceName": "Template_FileShare_ds", 19 | "type": "DatasetReference" 20 | }, 21 | "recursive": true, 22 | "enableLogging": false 23 | } 24 | } 25 | ], 26 | "parameters": { 27 | "FolderPath": { 28 | "type": "String", 29 | "defaultValue": "///" 30 | } 31 | }, 32 | "folder": { 33 | "name": "Delete Activity" 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /Samples/pipeline/Delete files with Azure Data Factory Time.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Delete files with Azure Data Factory Time", 3 | "properties": { 4 | "description": "Please input a folder path as the parameter for this pipeline, and the pipeline will delete files which were last modified more than 30 days ago under the particular folder path.", 5 | "activities": [ 6 | { 7 | "name": "DeleteFiles", 8 | "type": "Delete", 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "typeProperties": { 17 | "dataset": { 18 | "referenceName": "Template_wasb_delete_time", 19 | "type": "DatasetReference", 20 | "parameters": { 21 | "folder": { 22 | "value": "@pipeline().parameters.FolderPath", 23 | "type": "Expression" 24 | } 25 | } 26 | }, 27 | "recursive": true, 28 | "enableLogging": false 29 | } 30 | } 31 | ], 32 | "parameters": { 33 | "FolderPath": { 34 | "type": "String", 35 | "defaultValue": "/storesales/store_01/" 36 | } 37 | }, 38 | "folder": { 39 | "name": "Delete Activity" 40 | } 41 | } 42 | } -------------------------------------------------------------------------------- /Samples/pipeline/adf_arrays_sample_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adf_arrays_sample_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Lookup Notes_Syllabic", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "DelimitedTextSource", 20 | "storeSettings": { 21 | "type": "AzureBlobStorageReadSettings", 22 | "recursive": true 23 | }, 24 | "formatSettings": { 25 | "type": "DelimitedTextReadSettings" 26 | } 27 | }, 28 | "dataset": { 29 | "referenceName": "BLOB_MusicNotes", 30 | "type": "DatasetReference" 31 | }, 32 | "firstRowOnly": false 33 | } 34 | }, 35 | { 36 | "name": "Set_Notes_Alphabet", 37 | "type": "SetVariable", 38 | "dependsOn": [], 39 | "userProperties": [], 40 | "typeProperties": { 41 | "variableName": "Notes_Alphabet", 42 | "value": { 43 | "value": "@split(variables('Notes_Alphabet_String'),'-')", 44 | "type": "Expression" 45 | } 46 | } 47 | }, 48 | { 49 | "name": "Combing Notes", 50 | "type": "ForEach", 51 | "dependsOn": [ 52 | { 53 | "activity": "Lookup Notes_Syllabic", 54 | "dependencyConditions": [ 55 | "Succeeded" 56 | ] 57 | }, 58 | { 59 | "activity": "Set_Notes_Alphabet", 60 | "dependencyConditions": [ 61 | "Succeeded" 62 | ] 63 | }, 64 | { 65 | "activity": "ForEach_Array_Value", 66 | "dependencyConditions": [ 67 | "Succeeded" 68 | ] 69 | } 70 | ], 71 | "userProperties": [], 72 | "typeProperties": { 73 | "items": { 74 | "value": "@activity('Lookup Notes_Syllabic').output.value", 75 | "type": "Expression" 76 | }, 77 | "isSequential": true, 78 | "activities": [ 79 | { 80 | "name": "Append Music Notes", 81 | "type": "AppendVariable", 82 | "dependsOn": [], 83 | "userProperties": [], 84 | "typeProperties": { 85 | "variableName": "Notes_Combined", 86 | "value": { 87 | "value": "@concat(item().ID,'-',item().Note,'-',variables('Notes_Alphabet')[add(int(item().ID),-1)])", 88 | "type": "Expression" 89 | } 90 | } 91 | } 92 | ] 93 | } 94 | }, 95 | { 96 | "name": "Notes_Combined_View", 97 | "type": "SetVariable", 98 | "dependsOn": [ 99 | { 100 | "activity": "Combing Notes", 101 | "dependencyConditions": [ 102 | "Succeeded" 103 | ] 104 | } 105 | ], 106 | "userProperties": [], 107 | "typeProperties": { 108 | "variableName": "Notes_Combined_View", 109 | "value": { 110 | "value": "@variables('Notes_Combined')", 111 | "type": "Expression" 112 | } 113 | } 114 | }, 115 | { 116 | "name": "ForEach_Array_Value", 117 | "type": "ForEach", 118 | "dependsOn": [], 119 | "userProperties": [], 120 | "typeProperties": { 121 | "items": { 122 | "value": "@variables('Var_Array_Number')", 123 | "type": "Expression" 124 | }, 125 | "isSequential": true, 126 | "activities": [ 127 | { 128 | "name": "Wait 1 second", 129 | "type": "Wait", 130 | "dependsOn": [], 131 | "userProperties": [], 132 | "typeProperties": { 133 | "waitTimeInSeconds": 1 134 | } 135 | } 136 | ] 137 | } 138 | } 139 | ], 140 | "variables": { 141 | "Notes_Syllabic": { 142 | "type": "Array" 143 | }, 144 | "Notes_Alphabet": { 145 | "type": "Array" 146 | }, 147 | "Notes_Alphabet_String": { 148 | "type": "String", 149 | "defaultValue": "C-D-E-F-G-A-B" 150 | }, 151 | "Notes_Combined": { 152 | "type": "Array" 153 | }, 154 | "Notes_Combined_View": { 155 | "type": "Array" 156 | }, 157 | "Var_Array_Text": { 158 | "type": "Array", 159 | "defaultValue": [ 160 | "C", 161 | "D", 162 | "E", 163 | "F", 164 | "G", 165 | "A", 166 | "B" 167 | ] 168 | }, 169 | "Var_Array_Number": { 170 | "type": "Array", 171 | "defaultValue": [ 172 | 1, 173 | 2, 174 | 3, 175 | 4, 176 | 5, 177 | 6, 178 | 7 179 | ] 180 | } 181 | }, 182 | "folder": { 183 | "name": "Working with Arrays" 184 | }, 185 | "annotations": [] 186 | } 187 | } -------------------------------------------------------------------------------- /Samples/pipeline/bdata_adf_variable_append_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bdata_adf_variable_append_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get FeedType", 7 | "type": "SetVariable", 8 | "typeProperties": { 9 | "variableName": "FeedType", 10 | "value": { 11 | "value": "@replace(pipeline().parameters.FileName, 'Sales.csv','')", 12 | "type": "Expression" 13 | } 14 | } 15 | }, 16 | { 17 | "name": "If Daily DataFeed", 18 | "type": "IfCondition", 19 | "dependsOn": [ 20 | { 21 | "activity": "Get FeedType", 22 | "dependencyConditions": [ 23 | "Succeeded" 24 | ] 25 | } 26 | ], 27 | "typeProperties": { 28 | "expression": { 29 | "value": "@equals(variables('FeedType'), 'Daily')", 30 | "type": "Expression" 31 | }, 32 | "ifTrueActivities": [ 33 | { 34 | "name": "Copy Data Daily", 35 | "type": "Copy", 36 | "policy": { 37 | "timeout": "7.00:00:00", 38 | "retry": 0, 39 | "retryIntervalInSeconds": 30, 40 | "secureOutput": false, 41 | "secureInput": false 42 | }, 43 | "typeProperties": { 44 | "source": { 45 | "type": "BlobSource", 46 | "recursive": true 47 | }, 48 | "sink": { 49 | "type": "SqlSink", 50 | "writeBatchSize": 10000 51 | }, 52 | "enableStaging": false, 53 | "dataIntegrationUnits": 0, 54 | "translator": { 55 | "type": "TabularTranslator", 56 | "columnMappings": { 57 | "Date": "Date", 58 | "Account ID": "Account_ID", 59 | "Sales": "Sales" 60 | } 61 | } 62 | }, 63 | "inputs": [ 64 | { 65 | "referenceName": "AzureBlob_bgdata_daily", 66 | "type": "DatasetReference", 67 | "parameters": { 68 | "FilePath": { 69 | "value": "@pipeline().parameters.FilePath", 70 | "type": "Expression" 71 | }, 72 | "FileName": { 73 | "value": "@pipeline().parameters.FileName", 74 | "type": "Expression" 75 | } 76 | } 77 | } 78 | ], 79 | "outputs": [ 80 | { 81 | "referenceName": "AzureSqlTable_bgdata_daily", 82 | "type": "DatasetReference" 83 | } 84 | ] 85 | } 86 | ] 87 | } 88 | }, 89 | { 90 | "name": "If Monthly DataFeed", 91 | "type": "IfCondition", 92 | "dependsOn": [ 93 | { 94 | "activity": "Get FeedType", 95 | "dependencyConditions": [ 96 | "Succeeded" 97 | ] 98 | } 99 | ], 100 | "typeProperties": { 101 | "expression": { 102 | "value": "@equals(variables('FeedType'), 'Monthly')", 103 | "type": "Expression" 104 | }, 105 | "ifTrueActivities": [ 106 | { 107 | "name": "Copy Data Monthly", 108 | "type": "Copy", 109 | "policy": { 110 | "timeout": "7.00:00:00", 111 | "retry": 0, 112 | "retryIntervalInSeconds": 30, 113 | "secureOutput": false, 114 | "secureInput": false 115 | }, 116 | "typeProperties": { 117 | "source": { 118 | "type": "BlobSource", 119 | "recursive": true 120 | }, 121 | "sink": { 122 | "type": "SqlSink", 123 | "writeBatchSize": 10000 124 | }, 125 | "enableStaging": false, 126 | "dataIntegrationUnits": 0, 127 | "translator": { 128 | "type": "TabularTranslator", 129 | "columnMappings": { 130 | "Year": "Year", 131 | "Month": "Month", 132 | "Account ID": "Account_ID", 133 | "Total Sales": "Total_Sales" 134 | } 135 | } 136 | }, 137 | "inputs": [ 138 | { 139 | "referenceName": "AzureBlob_bgdata_monthly", 140 | "type": "DatasetReference", 141 | "parameters": { 142 | "FileName": { 143 | "value": "@pipeline().parameters.FileName", 144 | "type": "Expression" 145 | }, 146 | "FilePath": { 147 | "value": "@pipeline().parameters.FilePath", 148 | "type": "Expression" 149 | } 150 | } 151 | } 152 | ], 153 | "outputs": [ 154 | { 155 | "referenceName": "AzureSqlTable_bgdata_monthly", 156 | "type": "DatasetReference" 157 | } 158 | ] 159 | } 160 | ] 161 | } 162 | } 163 | ], 164 | "parameters": { 165 | "FilePath": { 166 | "type": "String", 167 | "defaultValue": "salesdata/" 168 | }, 169 | "FileName": { 170 | "type": "String", 171 | "defaultValue": "DailySales.csv" 172 | } 173 | }, 174 | "variables": { 175 | "FeedType": { 176 | "type": "String" 177 | } 178 | }, 179 | "folder": { 180 | "name": "Sample Pipelines" 181 | } 182 | }, 183 | "type": "Microsoft.DataFactory/factories/pipelines" 184 | } -------------------------------------------------------------------------------- /Samples/pipeline/bdata_adf_variable_array_first_element.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bdata_adf_variable_array_first_element", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set FileList", 7 | "type": "SetVariable", 8 | "typeProperties": { 9 | "variableName": "FileList", 10 | "value": { 11 | "value": "@createArray('File', 'Folder')", 12 | "type": "Expression" 13 | } 14 | } 15 | }, 16 | { 17 | "name": "CheckFirstElement", 18 | "type": "IfCondition", 19 | "dependsOn": [ 20 | { 21 | "activity": "Array First Element", 22 | "dependencyConditions": [ 23 | "Succeeded" 24 | ] 25 | } 26 | ], 27 | "typeProperties": { 28 | "expression": { 29 | "value": "@equals(variables('FirstArrayElement'),'Folder')", 30 | "type": "Expression" 31 | }, 32 | "ifFalseActivities": [ 33 | { 34 | "name": "Wait Files 2 sec", 35 | "type": "Wait", 36 | "typeProperties": { 37 | "waitTimeInSeconds": 1 38 | } 39 | } 40 | ], 41 | "ifTrueActivities": [ 42 | { 43 | "name": "Wait Folder 1 sec", 44 | "type": "Wait", 45 | "typeProperties": { 46 | "waitTimeInSeconds": 1 47 | } 48 | } 49 | ] 50 | } 51 | }, 52 | { 53 | "name": "Array First Element", 54 | "type": "SetVariable", 55 | "dependsOn": [ 56 | { 57 | "activity": "Set FileList", 58 | "dependencyConditions": [ 59 | "Succeeded" 60 | ] 61 | } 62 | ], 63 | "typeProperties": { 64 | "variableName": "FirstArrayElement", 65 | "value": { 66 | "value": "@variables('FileList')[0]", 67 | "type": "Expression" 68 | } 69 | } 70 | } 71 | ], 72 | "variables": { 73 | "FileList": { 74 | "type": "Array" 75 | }, 76 | "FirstArrayElement": { 77 | "type": "String" 78 | } 79 | }, 80 | "folder": { 81 | "name": "Sample Pipelines" 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /Samples/pipeline/bdata_adf_variable_array_many_elements.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bdata_adf_variable_array_many_elements", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Array Variable", 7 | "type": "SetVariable", 8 | "typeProperties": { 9 | "variableName": "ArrayVariable", 10 | "value": { 11 | "value": "@createArray('A', 'B')", 12 | "type": "Expression" 13 | } 14 | } 15 | }, 16 | { 17 | "name": "First Element", 18 | "type": "SetVariable", 19 | "dependsOn": [ 20 | { 21 | "activity": "Set Array Variable", 22 | "dependencyConditions": [ 23 | "Succeeded" 24 | ] 25 | } 26 | ], 27 | "typeProperties": { 28 | "variableName": "ArrayElement_1", 29 | "value": { 30 | "value": "@variables('ArrayVariable')[0]", 31 | "type": "Expression" 32 | } 33 | } 34 | }, 35 | { 36 | "name": "Second Element", 37 | "type": "SetVariable", 38 | "dependsOn": [ 39 | { 40 | "activity": "Set Array Variable", 41 | "dependencyConditions": [ 42 | "Succeeded" 43 | ] 44 | } 45 | ], 46 | "typeProperties": { 47 | "variableName": "ArrayElement_1", 48 | "value": { 49 | "value": "@variables('ArrayVariable')[0]", 50 | "type": "Expression" 51 | } 52 | } 53 | }, 54 | { 55 | "name": "Check Out of Range", 56 | "type": "SetVariable", 57 | "dependsOn": [ 58 | { 59 | "activity": "Set Array Variable", 60 | "dependencyConditions": [ 61 | "Succeeded" 62 | ] 63 | } 64 | ], 65 | "typeProperties": { 66 | "variableName": "ArrayElement_OutOfRange", 67 | "value": { 68 | "value": "@variables('ArrayVariable')[2]", 69 | "type": "Expression" 70 | } 71 | } 72 | }, 73 | { 74 | "name": "Add New Value", 75 | "type": "AppendVariable", 76 | "dependsOn": [ 77 | { 78 | "activity": "Check Out of Range", 79 | "dependencyConditions": [ 80 | "Failed" 81 | ] 82 | } 83 | ], 84 | "typeProperties": { 85 | "variableName": "ArrayVariable", 86 | "value": "C" 87 | } 88 | } 89 | ], 90 | "variables": { 91 | "ArrayVariable": { 92 | "type": "Array" 93 | }, 94 | "ArrayElement_1": { 95 | "type": "String" 96 | }, 97 | "ArrayElement_2": { 98 | "type": "String" 99 | }, 100 | "ArrayElement_OutOfRange": { 101 | "type": "String" 102 | }, 103 | "ErrorMessage": { 104 | "type": "String" 105 | } 106 | }, 107 | "folder": { 108 | "name": "Sample Pipelines" 109 | } 110 | } 111 | } -------------------------------------------------------------------------------- /Samples/pipeline/bdata_adf_variable_set_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bdata_adf_variable_set_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Metadata Store 02", 7 | "type": "GetMetadata", 8 | "policy": { 9 | "timeout": "7.00:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "dataset": { 17 | "referenceName": "AzureBlob_storesales_store02", 18 | "type": "DatasetReference" 19 | }, 20 | "fieldList": [ 21 | "childItems" 22 | ] 23 | } 24 | }, 25 | { 26 | "name": "Filenames Store 02", 27 | "type": "ForEach", 28 | "dependsOn": [ 29 | { 30 | "activity": "Metadata Store 02", 31 | "dependencyConditions": [ 32 | "Succeeded" 33 | ] 34 | } 35 | ], 36 | "typeProperties": { 37 | "items": { 38 | "value": "@activity('Metadata Store 02').output.childitems", 39 | "type": "Expression" 40 | }, 41 | "activities": [ 42 | { 43 | "name": "Build filename list 02", 44 | "type": "AppendVariable", 45 | "typeProperties": { 46 | "variableName": "var_file_list", 47 | "value": { 48 | "value": "@concat('store_02/',item().name)", 49 | "type": "Expression" 50 | } 51 | } 52 | } 53 | ] 54 | } 55 | }, 56 | { 57 | "name": "Validate filename list", 58 | "type": "SetVariable", 59 | "dependsOn": [ 60 | { 61 | "activity": "Filenames Store 02", 62 | "dependencyConditions": [ 63 | "Succeeded" 64 | ] 65 | }, 66 | { 67 | "activity": "Filenames Store 03", 68 | "dependencyConditions": [ 69 | "Succeeded" 70 | ] 71 | }, 72 | { 73 | "activity": "Filenames Store 01", 74 | "dependencyConditions": [ 75 | "Succeeded" 76 | ] 77 | } 78 | ], 79 | "typeProperties": { 80 | "variableName": "var_file_list_check", 81 | "value": { 82 | "value": "@variables('var_file_list')", 83 | "type": "Expression" 84 | } 85 | } 86 | }, 87 | { 88 | "name": "Metadata Store 03", 89 | "type": "GetMetadata", 90 | "policy": { 91 | "timeout": "7.00:00:00", 92 | "retry": 0, 93 | "retryIntervalInSeconds": 30, 94 | "secureOutput": false, 95 | "secureInput": false 96 | }, 97 | "typeProperties": { 98 | "dataset": { 99 | "referenceName": "AzureBlob_storesales_store03", 100 | "type": "DatasetReference" 101 | }, 102 | "fieldList": [ 103 | "childItems" 104 | ] 105 | } 106 | }, 107 | { 108 | "name": "Filenames Store 03", 109 | "type": "ForEach", 110 | "dependsOn": [ 111 | { 112 | "activity": "Metadata Store 03", 113 | "dependencyConditions": [ 114 | "Succeeded" 115 | ] 116 | } 117 | ], 118 | "typeProperties": { 119 | "items": { 120 | "value": "@activity('Metadata Store 03').output.childitems", 121 | "type": "Expression" 122 | }, 123 | "activities": [ 124 | { 125 | "name": "Build filename list 03", 126 | "type": "AppendVariable", 127 | "typeProperties": { 128 | "variableName": "var_file_list", 129 | "value": { 130 | "value": "@concat('store_03/',item().name)", 131 | "type": "Expression" 132 | } 133 | } 134 | } 135 | ] 136 | } 137 | }, 138 | { 139 | "name": "Load data into SQL", 140 | "type": "ForEach", 141 | "dependsOn": [ 142 | { 143 | "activity": "Validate filename list", 144 | "dependencyConditions": [ 145 | "Succeeded" 146 | ] 147 | } 148 | ], 149 | "typeProperties": { 150 | "items": { 151 | "value": "@variables('var_file_list')", 152 | "type": "Expression" 153 | }, 154 | "activities": [ 155 | { 156 | "name": "Copy Data Blob to SQL", 157 | "type": "Copy", 158 | "policy": { 159 | "timeout": "7.00:00:00", 160 | "retry": 0, 161 | "retryIntervalInSeconds": 30, 162 | "secureOutput": false, 163 | "secureInput": false 164 | }, 165 | "typeProperties": { 166 | "source": { 167 | "type": "BlobSource", 168 | "recursive": true 169 | }, 170 | "sink": { 171 | "type": "SqlSink", 172 | "writeBatchSize": 10000 173 | }, 174 | "enableStaging": false, 175 | "dataIntegrationUnits": 0, 176 | "translator": { 177 | "type": "TabularTranslator", 178 | "columnMappings": { 179 | "store_id": "store_id", 180 | "date_id": "date_id", 181 | "product_id": "product_id", 182 | "stock_amt": "stock_amt" 183 | } 184 | } 185 | }, 186 | "inputs": [ 187 | { 188 | "referenceName": "AzureBlob_storesales_storesall", 189 | "type": "DatasetReference", 190 | "parameters": { 191 | "FilePath": "storesales", 192 | "FileName": { 193 | "value": "@item()", 194 | "type": "Expression" 195 | } 196 | } 197 | } 198 | ], 199 | "outputs": [ 200 | { 201 | "referenceName": "AzureSqlTable_storesales", 202 | "type": "DatasetReference" 203 | } 204 | ] 205 | } 206 | ] 207 | } 208 | }, 209 | { 210 | "name": "Metadata Store 01", 211 | "type": "GetMetadata", 212 | "policy": { 213 | "timeout": "7.00:00:00", 214 | "retry": 0, 215 | "retryIntervalInSeconds": 30, 216 | "secureOutput": false, 217 | "secureInput": false 218 | }, 219 | "typeProperties": { 220 | "dataset": { 221 | "referenceName": "AzureBlob_storesales_store01", 222 | "type": "DatasetReference" 223 | }, 224 | "fieldList": [ 225 | "childItems" 226 | ] 227 | } 228 | }, 229 | { 230 | "name": "Filenames Store 01", 231 | "type": "ForEach", 232 | "dependsOn": [ 233 | { 234 | "activity": "Metadata Store 01", 235 | "dependencyConditions": [ 236 | "Succeeded" 237 | ] 238 | } 239 | ], 240 | "typeProperties": { 241 | "items": { 242 | "value": "@activity('Metadata Store 01').output.childitems", 243 | "type": "Expression" 244 | }, 245 | "activities": [ 246 | { 247 | "name": "Build filename list 01", 248 | "type": "AppendVariable", 249 | "typeProperties": { 250 | "variableName": "var_file_list", 251 | "value": { 252 | "value": "@concat('store_01/',item().name)", 253 | "type": "Expression" 254 | } 255 | } 256 | } 257 | ] 258 | } 259 | } 260 | ], 261 | "variables": { 262 | "var_file_list": { 263 | "type": "Array" 264 | }, 265 | "var_file_list_check": { 266 | "type": "Array" 267 | } 268 | }, 269 | "folder": { 270 | "name": "Sample Pipelines" 271 | } 272 | }, 273 | "type": "Microsoft.DataFactory/factories/pipelines" 274 | } -------------------------------------------------------------------------------- /Samples/pipeline/bdata_adf_variable_set_single_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bdata_adf_variable_set_single_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set filename", 7 | "type": "SetVariable", 8 | "dependsOn": [ 9 | { 10 | "activity": "Metadata Store 01", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "typeProperties": { 17 | "variableName": "var_file_name", 18 | "value": { 19 | "value": "@activity('Metadata Store 01').output.itemName", 20 | "type": "Expression" 21 | } 22 | } 23 | }, 24 | { 25 | "name": "Metadata Store 01", 26 | "type": "GetMetadata", 27 | "policy": { 28 | "timeout": "7.00:00:00", 29 | "retry": 0, 30 | "retryIntervalInSeconds": 30, 31 | "secureOutput": false, 32 | "secureInput": false 33 | }, 34 | "typeProperties": { 35 | "dataset": { 36 | "referenceName": "AzureBlob_storesales_store01_single", 37 | "type": "DatasetReference" 38 | }, 39 | "fieldList": [ 40 | "itemName" 41 | ] 42 | } 43 | }, 44 | { 45 | "name": "Copy Data to SQL - V", 46 | "type": "Copy", 47 | "dependsOn": [ 48 | { 49 | "activity": "Set filename", 50 | "dependencyConditions": [ 51 | "Succeeded" 52 | ] 53 | }, 54 | { 55 | "activity": "Copy Data to SQL - D", 56 | "dependencyConditions": [ 57 | "Succeeded" 58 | ] 59 | } 60 | ], 61 | "policy": { 62 | "timeout": "7.00:00:00", 63 | "retry": 0, 64 | "retryIntervalInSeconds": 30, 65 | "secureOutput": false, 66 | "secureInput": false 67 | }, 68 | "typeProperties": { 69 | "source": { 70 | "type": "BlobSource", 71 | "recursive": true 72 | }, 73 | "sink": { 74 | "type": "SqlSink", 75 | "writeBatchSize": 10000 76 | }, 77 | "enableStaging": false, 78 | "dataIntegrationUnits": 0, 79 | "translator": { 80 | "type": "TabularTranslator", 81 | "columnMappings": { 82 | "store_id": "store_id", 83 | "date_id": "date_id", 84 | "product_id": "product_id", 85 | "stock_amt": "stock_amt" 86 | } 87 | } 88 | }, 89 | "inputs": [ 90 | { 91 | "referenceName": "AzureBlob_storesales_storesall", 92 | "type": "DatasetReference", 93 | "parameters": { 94 | "FilePath": "storesales-source/store_01", 95 | "FileName": { 96 | "value": "@variables('var_file_name')", 97 | "type": "Expression" 98 | } 99 | } 100 | } 101 | ], 102 | "outputs": [ 103 | { 104 | "referenceName": "AzureSqlTable_storesales", 105 | "type": "DatasetReference" 106 | } 107 | ] 108 | }, 109 | { 110 | "name": "Copy Data to SQL - D", 111 | "description": "Direct data copy from a file to a sql server table", 112 | "type": "Copy", 113 | "policy": { 114 | "timeout": "7.00:00:00", 115 | "retry": 0, 116 | "retryIntervalInSeconds": 30, 117 | "secureOutput": false, 118 | "secureInput": false 119 | }, 120 | "typeProperties": { 121 | "source": { 122 | "type": "BlobSource", 123 | "recursive": true 124 | }, 125 | "sink": { 126 | "type": "SqlSink", 127 | "writeBatchSize": 10000 128 | }, 129 | "enableStaging": false, 130 | "translator": { 131 | "type": "TabularTranslator", 132 | "columnMappings": { 133 | "store_id": "store_id", 134 | "date_id": "date_id", 135 | "product_id": "product_id", 136 | "stock_amt": "stock_amt" 137 | } 138 | } 139 | }, 140 | "inputs": [ 141 | { 142 | "referenceName": "AzureBlob_storesales_store01_single", 143 | "type": "DatasetReference" 144 | } 145 | ], 146 | "outputs": [ 147 | { 148 | "referenceName": "AzureSqlTable_storesales", 149 | "type": "DatasetReference" 150 | } 151 | ] 152 | } 153 | ], 154 | "variables": { 155 | "var_file_name": { 156 | "type": "String" 157 | } 158 | }, 159 | "folder": { 160 | "name": "Sample Pipelines" 161 | } 162 | }, 163 | "type": "Microsoft.DataFactory/factories/pipelines" 164 | } -------------------------------------------------------------------------------- /Samples/pipeline/dataflow_flatten_pl.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflow_flatten_pl", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Data Flow Flatten JSON with cakes", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "df_flatten_task_cakes_JSON", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "folder": { 30 | "name": "JSON data" 31 | }, 32 | "annotations": [] 33 | } 34 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_CreateArray_UseCase.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_CreateArray_UseCase", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Job Title", 7 | "type": "SetVariable", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "variableName": "JobTitle", 12 | "value": "Design Engineer" 13 | } 14 | }, 15 | { 16 | "name": "Check Job Title", 17 | "type": "IfCondition", 18 | "dependsOn": [ 19 | { 20 | "activity": "Set Job Title", 21 | "dependencyConditions": [ 22 | "Succeeded" 23 | ] 24 | } 25 | ], 26 | "userProperties": [], 27 | "typeProperties": { 28 | "expression": { 29 | "value": "@contains(createArray('Design Engineer', 'Tool Designer', 'Marketing Assistant'),variables('JobTitle'))", 30 | "type": "Expression" 31 | }, 32 | "ifFalseActivities": [ 33 | { 34 | "name": "Wait 5 Second", 35 | "type": "Wait", 36 | "dependsOn": [], 37 | "userProperties": [], 38 | "typeProperties": { 39 | "waitTimeInSeconds": 5 40 | } 41 | } 42 | ], 43 | "ifTrueActivities": [ 44 | { 45 | "name": "Wait 1 Second", 46 | "type": "Wait", 47 | "dependsOn": [], 48 | "userProperties": [], 49 | "typeProperties": { 50 | "waitTimeInSeconds": 1 51 | } 52 | } 53 | ] 54 | } 55 | } 56 | ], 57 | "variables": { 58 | "JobTitle": { 59 | "type": "String" 60 | } 61 | }, 62 | "folder": { 63 | "name": "Working with Arrays" 64 | }, 65 | "annotations": [] 66 | } 67 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_PASS_ADF_DataVault.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_PASS_ADF_DataVault", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "df_PASS_ADF_DataVault", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "df_PASS_ADF_DataVault", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "folder": { 30 | "name": "DataVaultTesting" 31 | }, 32 | "annotations": [] 33 | } 34 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_SQL_SP_optional_parameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_SQL_SP_optional_parameter", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Exec Azure SQL SP", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "AzureSqlSource", 20 | "sqlReaderStoredProcedureName": "[dbo].[sp_get_week_days]", 21 | "storedProcedureParameters": { 22 | "week_number": { 23 | "type": "Int32", 24 | "value": "1" 25 | } 26 | }, 27 | "queryTimeout": "02:00:00", 28 | "partitionOption": "None" 29 | }, 30 | "dataset": { 31 | "referenceName": "ds_azsql_generic", 32 | "type": "DatasetReference" 33 | }, 34 | "firstRowOnly": false 35 | } 36 | } 37 | ], 38 | "folder": { 39 | "name": "SQL" 40 | }, 41 | "annotations": [] 42 | } 43 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_azurefunction_test.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_azurefunction_test", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Web Activity", 7 | "type": "WebActivity", 8 | "dependsOn": [ 9 | { 10 | "activity": "WebHook", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "policy": { 17 | "timeout": "7.00:00:00", 18 | "retry": 0, 19 | "retryIntervalInSeconds": 30, 20 | "secureOutput": false, 21 | "secureInput": false 22 | }, 23 | "userProperties": [], 24 | "typeProperties": { 25 | "url": "https://bd-eus2-fapp.azurewebsites.net/api/GetTime?code=WPbrtWZpOERh8z8dl2UjJ/Xu8XsaRHk6I49KKikRWzr38B8n6fax4w==", 26 | "method": "POST", 27 | "body": "{\n timezone: \"Eastern Standard Time\"\n}" 28 | } 29 | }, 30 | { 31 | "name": "Azure Function", 32 | "type": "AzureFunctionActivity", 33 | "dependsOn": [], 34 | "policy": { 35 | "timeout": "7.00:00:00", 36 | "retry": 0, 37 | "retryIntervalInSeconds": 30, 38 | "secureOutput": false, 39 | "secureInput": false 40 | }, 41 | "userProperties": [], 42 | "typeProperties": { 43 | "functionName": "GetTime-AF ", 44 | "method": "POST", 45 | "body": "{\n timezone: \"Eastern Standard Time\"\n}" 46 | }, 47 | "linkedServiceName": { 48 | "referenceName": "ls_af_gettime", 49 | "type": "LinkedServiceReference" 50 | } 51 | }, 52 | { 53 | "name": "WebHook", 54 | "type": "WebHook", 55 | "dependsOn": [ 56 | { 57 | "activity": "Azure Function", 58 | "dependencyConditions": [ 59 | "Succeeded" 60 | ] 61 | } 62 | ], 63 | "userProperties": [], 64 | "typeProperties": { 65 | "url": "https://bd-eus2-fapp.azurewebsites.net/api/GetTime-WH?code=EzOt81JcICeAzV2c4OqFBSy0Hrar9aCHEygHaClJJm7iHds1wYtfKg==", 66 | "method": "POST", 67 | "headers": { 68 | "Content-Type": "application/json" 69 | }, 70 | "body": { 71 | "timezone": "Eastern Standard Time" 72 | }, 73 | "timeout": "00:09:00", 74 | "reportStatusOnCallBack": false 75 | } 76 | } 77 | ], 78 | "folder": { 79 | "name": "Azure Functions" 80 | }, 81 | "annotations": [] 82 | } 83 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_constant_dynamic_JSON.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_constant_dynamic_JSON", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get List of Files", 7 | "type": "GetMetadata", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataset": { 19 | "referenceName": "ds_adls_generic_folder", 20 | "type": "DatasetReference", 21 | "parameters": { 22 | "file_folder": "files/system/logs" 23 | } 24 | }, 25 | "fieldList": [ 26 | "childItems" 27 | ], 28 | "storeSettings": { 29 | "type": "AzureBlobFSReadSettings", 30 | "enablePartitionDiscovery": false 31 | }, 32 | "formatSettings": { 33 | "type": "DelimitedTextReadSettings" 34 | } 35 | } 36 | }, 37 | { 38 | "name": "Collect Initial File List", 39 | "type": "SetVariable", 40 | "dependsOn": [ 41 | { 42 | "activity": "Get List of Files", 43 | "dependencyConditions": [ 44 | "Succeeded" 45 | ] 46 | } 47 | ], 48 | "userProperties": [], 49 | "typeProperties": { 50 | "variableName": "var_file_list_initial", 51 | "value": { 52 | "value": "@activity('Get List of Files').output.childItems", 53 | "type": "Expression" 54 | } 55 | } 56 | }, 57 | { 58 | "name": "Merge File List with Constant Values", 59 | "type": "SetVariable", 60 | "dependsOn": [ 61 | { 62 | "activity": "Collect Initial File List", 63 | "dependencyConditions": [ 64 | "Succeeded" 65 | ] 66 | } 67 | ], 68 | "userProperties": [], 69 | "typeProperties": { 70 | "variableName": "var_file_list_merged", 71 | "value": { 72 | "value": "@json(\n replace(\n concat('[',\n replace(\n replace(\n join(\n variables('var_file_list_initial'),\n concat('*', \n variables('var_constant_json'),'}')\n )\n , '}*', ', ')\n , '}{', '},{')\n , ']')\n , '}]', concat(',', variables('var_constant_json'),'}') \n )\n)", 73 | "type": "Expression" 74 | } 75 | } 76 | } 77 | ], 78 | "variables": { 79 | "var_file_list_initial": { 80 | "type": "Array" 81 | }, 82 | "var_file_list_merged": { 83 | "type": "Array" 84 | }, 85 | "var_constant_json": { 86 | "type": "String", 87 | "defaultValue": "\"filetype\":\"log\", \"path\":\"files/system/logs\"" 88 | } 89 | }, 90 | "folder": { 91 | "name": "JSON data" 92 | }, 93 | "annotations": [] 94 | } 95 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_constant_dynamic_JSON_SQL.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_constant_dynamic_JSON_SQL", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Lookup1", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "AzureSqlSource", 20 | "sqlReaderQuery": { 21 | "value": "declare @JSON nvarchar(max) = '@{pipeline().parameters.Input}';\n\nselect\n\t(\n\tselect\n\t\tjson_value(@JSON, 'strict $.name') as name,\n\t\t(\n\t\t\tselect\n\t\t\t\tname,\n\t\t\t\ttype,\n\t\t\t\t'log' as filetype,\n\t\t\t\t'files/system/logs' as path\n\t\t\tfrom\n\t\t\t\topenjson(@JSON, 'strict $.value')\n\t\t\t\t\twith (\n\t\t\t\t\t\tname nvarchar(max) 'strict $.name',\n\t\t\t\t\t\ttype nvarchar(max) 'strict $.type'\n\t\t\t\t\t)\n\t\t\tfor\n\t\t\t\tjson path\n\t\t) as value\n\tfor\n\t\tjson path,\n\t\twithout_array_wrapper\n\t) as Object\n;", 22 | "type": "Expression" 23 | }, 24 | "queryTimeout": "02:00:00", 25 | "partitionOption": "None" 26 | }, 27 | "dataset": { 28 | "referenceName": "ds_azsql_generic", 29 | "type": "DatasetReference" 30 | } 31 | } 32 | }, 33 | { 34 | "name": "Append variable1", 35 | "type": "AppendVariable", 36 | "dependsOn": [ 37 | { 38 | "activity": "Lookup1", 39 | "dependencyConditions": [ 40 | "Succeeded" 41 | ] 42 | } 43 | ], 44 | "userProperties": [], 45 | "typeProperties": { 46 | "variableName": "Output", 47 | "value": { 48 | "value": "@json(activity('Lookup1').output.firstRow.Object)", 49 | "type": "Expression" 50 | } 51 | } 52 | } 53 | ], 54 | "parameters": { 55 | "Input": { 56 | "type": "object", 57 | "defaultValue": { 58 | "name": "var_file_list_initial", 59 | "value": [ 60 | { 61 | "name": "01-Monday.log", 62 | "type": "File" 63 | }, 64 | { 65 | "name": "01-Tuesday.log", 66 | "type": "File" 67 | }, 68 | { 69 | "name": "01-Wednesday.log", 70 | "type": "File" 71 | }, 72 | { 73 | "name": "01-Thursday.log", 74 | "type": "File" 75 | }, 76 | { 77 | "name": "01-Friday.log", 78 | "type": "File" 79 | } 80 | ] 81 | } 82 | } 83 | }, 84 | "variables": { 85 | "Output": { 86 | "type": "Array", 87 | "defaultValue": [] 88 | } 89 | }, 90 | "folder": { 91 | "name": "JSON data" 92 | }, 93 | "annotations": [] 94 | } 95 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_data_json_flatten.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_data_json_flatten", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get JSON data", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "JsonSource", 20 | "storeSettings": { 21 | "type": "AzureBlobStorageReadSettings", 22 | "recursive": true 23 | } 24 | }, 25 | "dataset": { 26 | "referenceName": "ls_json_flatten_data", 27 | "type": "DatasetReference" 28 | } 29 | } 30 | }, 31 | { 32 | "name": "For Each Events", 33 | "type": "ForEach", 34 | "dependsOn": [ 35 | { 36 | "activity": "Get JSON data", 37 | "dependencyConditions": [ 38 | "Succeeded" 39 | ] 40 | } 41 | ], 42 | "userProperties": [], 43 | "typeProperties": { 44 | "items": { 45 | "value": "@activity('Get JSON data').output.firstRow.events", 46 | "type": "Expression" 47 | }, 48 | "activities": [ 49 | { 50 | "name": "Log Event", 51 | "type": "SqlServerStoredProcedure", 52 | "dependsOn": [], 53 | "policy": { 54 | "timeout": "7.00:00:00", 55 | "retry": 0, 56 | "retryIntervalInSeconds": 30, 57 | "secureOutput": false, 58 | "secureInput": false 59 | }, 60 | "userProperties": [], 61 | "typeProperties": { 62 | "storedProcedureName": "[dbo].[sp_adf_pipeline_log_event]", 63 | "storedProcedureParameters": { 64 | "process": { 65 | "value": { 66 | "value": "@item().process", 67 | "type": "Expression" 68 | }, 69 | "type": "String" 70 | }, 71 | "start_time": { 72 | "value": { 73 | "value": "@item().start_time", 74 | "type": "Expression" 75 | }, 76 | "type": "DateTime" 77 | }, 78 | "stop_time": { 79 | "value": { 80 | "value": "@item().stop_time", 81 | "type": "Expression" 82 | }, 83 | "type": "DateTime" 84 | }, 85 | "last_update": { 86 | "value": { 87 | "value": "@item().last_update", 88 | "type": "Expression" 89 | }, 90 | "type": "Datetime" 91 | } 92 | } 93 | }, 94 | "linkedServiceName": { 95 | "referenceName": "AzureSqlDatabase_LS", 96 | "type": "LinkedServiceReference" 97 | } 98 | } 99 | ] 100 | } 101 | } 102 | ], 103 | "variables": { 104 | "event": { 105 | "type": "Array" 106 | } 107 | }, 108 | "folder": { 109 | "name": "JSON data" 110 | }, 111 | "annotations": [] 112 | } 113 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_data_json_flatten_corrected.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_data_json_flatten_corrected", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get JSON data", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "JsonSource", 20 | "storeSettings": { 21 | "type": "AzureBlobStorageReadSettings", 22 | "recursive": true 23 | } 24 | }, 25 | "dataset": { 26 | "referenceName": "ls_json_flatten_data", 27 | "type": "DatasetReference" 28 | } 29 | } 30 | }, 31 | { 32 | "name": "For Each Events", 33 | "type": "ForEach", 34 | "dependsOn": [ 35 | { 36 | "activity": "Get JSON data", 37 | "dependencyConditions": [ 38 | "Succeeded" 39 | ] 40 | } 41 | ], 42 | "userProperties": [], 43 | "typeProperties": { 44 | "items": { 45 | "value": "@activity('Get JSON data').output.firstRow.events", 46 | "type": "Expression" 47 | }, 48 | "activities": [ 49 | { 50 | "name": "Log Event", 51 | "type": "SqlServerStoredProcedure", 52 | "dependsOn": [], 53 | "policy": { 54 | "timeout": "7.00:00:00", 55 | "retry": 0, 56 | "retryIntervalInSeconds": 30, 57 | "secureOutput": false, 58 | "secureInput": false 59 | }, 60 | "userProperties": [], 61 | "typeProperties": { 62 | "storedProcedureName": "[dbo].[sp_adf_pipeline_log_event]", 63 | "storedProcedureParameters": { 64 | "process": { 65 | "value": { 66 | "value": "@item().process", 67 | "type": "Expression" 68 | }, 69 | "type": "String" 70 | }, 71 | "start_time": { 72 | "value": { 73 | "value": "@item().start_time", 74 | "type": "Expression" 75 | }, 76 | "type": "DateTime" 77 | }, 78 | "stop_time": { 79 | "value": { 80 | "value": "@if(contains(string(item()),'\"stop_time\":'),item().stop_time,null)", 81 | "type": "Expression" 82 | }, 83 | "type": "DateTime" 84 | }, 85 | "last_update": { 86 | "value": { 87 | "value": "@if(contains(string(item()),'\"last_update\":'),item().last_update,utcNow())", 88 | "type": "Expression" 89 | }, 90 | "type": "Datetime" 91 | } 92 | } 93 | }, 94 | "linkedServiceName": { 95 | "referenceName": "AzureSqlDatabase_LS", 96 | "type": "LinkedServiceReference" 97 | } 98 | } 99 | ] 100 | } 101 | } 102 | ], 103 | "variables": { 104 | "event": { 105 | "type": "Array" 106 | } 107 | }, 108 | "folder": { 109 | "name": "JSON data" 110 | }, 111 | "annotations": [] 112 | } 113 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_default_value_array.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_default_value_array", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Each Parameter Meal", 7 | "type": "ForEach", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "items": { 12 | "value": "@pipeline().parameters.par_meal_array", 13 | "type": "Expression" 14 | }, 15 | "activities": [ 16 | { 17 | "name": "pl_meal_array_child", 18 | "type": "ExecutePipeline", 19 | "dependsOn": [], 20 | "userProperties": [], 21 | "typeProperties": { 22 | "pipeline": { 23 | "referenceName": "pl_default_value_array_child", 24 | "type": "PipelineReference" 25 | }, 26 | "waitOnCompletion": true, 27 | "parameters": { 28 | "par_meal_array": { 29 | "value": "@item().meal_items", 30 | "type": "Expression" 31 | } 32 | } 33 | } 34 | } 35 | ] 36 | } 37 | } 38 | ], 39 | "parameters": { 40 | "par_meal_array": { 41 | "type": "array", 42 | "defaultValue": [ 43 | { 44 | "meal_type": "Breakfast", 45 | "meal_time": "07:00:00", 46 | "meal_items": [ 47 | "Egg", 48 | "Greek Yogurt", 49 | "Coffee" 50 | ] 51 | }, 52 | { 53 | "meal_type": "Lunch", 54 | "meal_time": "12:00:00", 55 | "meal_items": [ 56 | "Greek Salad", 57 | "Salmon Bagels", 58 | "Tea" 59 | ] 60 | }, 61 | { 62 | "meal_type": "Dinner", 63 | "meal_time": "18:00:00", 64 | "meal_items": [ 65 | "Kale Salad", 66 | "Lentil Soup", 67 | "Water" 68 | ] 69 | } 70 | ] 71 | } 72 | }, 73 | "folder": { 74 | "name": "Working with Arrays" 75 | }, 76 | "annotations": [] 77 | } 78 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_default_value_array_child.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_default_value_array_child", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Each Parameter Meal", 7 | "type": "ForEach", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "items": { 12 | "value": "@pipeline().parameters.par_meal_array", 13 | "type": "Expression" 14 | }, 15 | "activities": [ 16 | { 17 | "name": "Wait 1 second", 18 | "type": "Wait", 19 | "dependsOn": [], 20 | "userProperties": [], 21 | "typeProperties": { 22 | "waitTimeInSeconds": 1 23 | } 24 | } 25 | ] 26 | } 27 | } 28 | ], 29 | "parameters": { 30 | "par_meal_array": { 31 | "type": "array", 32 | "defaultValue": [ 33 | "Egg", 34 | "Greek Yogurt", 35 | "Coffee" 36 | ] 37 | } 38 | }, 39 | "folder": { 40 | "name": "Working with Arrays" 41 | }, 42 | "annotations": [] 43 | } 44 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_default_value_array_variable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_default_value_array_variable", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Each Variable Meal", 7 | "type": "ForEach", 8 | "dependsOn": [ 9 | { 10 | "activity": "Set Array", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "userProperties": [], 17 | "typeProperties": { 18 | "items": { 19 | "value": "@variables('var_array')", 20 | "type": "Expression" 21 | }, 22 | "activities": [ 23 | { 24 | "name": "Wait 1 second_copy1", 25 | "type": "Wait", 26 | "dependsOn": [], 27 | "userProperties": [], 28 | "typeProperties": { 29 | "waitTimeInSeconds": 1 30 | } 31 | } 32 | ] 33 | } 34 | }, 35 | { 36 | "name": "Set Array", 37 | "type": "SetVariable", 38 | "dependsOn": [], 39 | "userProperties": [], 40 | "typeProperties": { 41 | "variableName": "var_array", 42 | "value": { 43 | "value": "@split(pipeline().parameters.par_meal_string,',')", 44 | "type": "Expression" 45 | } 46 | } 47 | } 48 | ], 49 | "parameters": { 50 | "par_meal_string": { 51 | "type": "string", 52 | "defaultValue": "Egg,Greek Yogurt,Coffee" 53 | } 54 | }, 55 | "variables": { 56 | "var_array": { 57 | "type": "Array" 58 | } 59 | }, 60 | "folder": { 61 | "name": "Working with Arrays" 62 | }, 63 | "annotations": [] 64 | } 65 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_durable_function_loop_polling.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_durable_function_loop_polling", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Azure Durable Function call", 7 | "type": "AzureFunctionActivity", 8 | "dependsOn": [ 9 | { 10 | "activity": "Set Input for Function App", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "policy": { 17 | "timeout": "7.00:00:00", 18 | "retry": 0, 19 | "retryIntervalInSeconds": 30, 20 | "secureOutput": false, 21 | "secureInput": false 22 | }, 23 | "userProperties": [], 24 | "typeProperties": { 25 | "functionName": "orchestrators/DurableFunctions-Orchestrator", 26 | "method": "POST", 27 | "body": { 28 | "value": "@variables('var_input')", 29 | "type": "Expression" 30 | } 31 | }, 32 | "linkedServiceName": { 33 | "referenceName": "ls_af", 34 | "type": "LinkedServiceReference" 35 | } 36 | }, 37 | { 38 | "name": "Check Completed Status", 39 | "type": "Until", 40 | "dependsOn": [ 41 | { 42 | "activity": "Azure Durable Function call", 43 | "dependencyConditions": [ 44 | "Succeeded" 45 | ] 46 | } 47 | ], 48 | "userProperties": [], 49 | "typeProperties": { 50 | "expression": { 51 | "value": "@not(or(equals(activity('Get Current Function Status').output.runtimeStatus, 'Pending'), equals(activity('Get Current Function Status').output.runtimeStatus, 'Running')))\n\n", 52 | "type": "Expression" 53 | }, 54 | "activities": [ 55 | { 56 | "name": "Wait 5 seconds", 57 | "type": "Wait", 58 | "dependsOn": [], 59 | "userProperties": [], 60 | "typeProperties": { 61 | "waitTimeInSeconds": 1 62 | } 63 | }, 64 | { 65 | "name": "Get Current Function Status", 66 | "type": "WebActivity", 67 | "dependsOn": [ 68 | { 69 | "activity": "Wait 5 seconds", 70 | "dependencyConditions": [ 71 | "Succeeded" 72 | ] 73 | } 74 | ], 75 | "policy": { 76 | "timeout": "7.00:00:00", 77 | "retry": 0, 78 | "retryIntervalInSeconds": 30, 79 | "secureOutput": false, 80 | "secureInput": false 81 | }, 82 | "userProperties": [], 83 | "typeProperties": { 84 | "url": { 85 | "value": "@activity('Azure Durable Function call').output.statusQueryGetUri\n", 86 | "type": "Expression" 87 | }, 88 | "method": "GET" 89 | } 90 | } 91 | ], 92 | "timeout": "7.00:00:00" 93 | } 94 | }, 95 | { 96 | "name": "Set Input for Function App", 97 | "type": "SetVariable", 98 | "dependsOn": [], 99 | "userProperties": [], 100 | "typeProperties": { 101 | "variableName": "var_input", 102 | "value": "{\"timezone\":\"Eastern Standard Time\"}" 103 | } 104 | }, 105 | { 106 | "name": "Get Function App Output", 107 | "type": "SetVariable", 108 | "dependsOn": [ 109 | { 110 | "activity": "Check Completed Status", 111 | "dependencyConditions": [ 112 | "Succeeded" 113 | ] 114 | } 115 | ], 116 | "userProperties": [], 117 | "typeProperties": { 118 | "variableName": "var_output", 119 | "value": { 120 | "value": "@activity('Get Current Function Status').output.output.Response", 121 | "type": "Expression" 122 | } 123 | } 124 | } 125 | ], 126 | "variables": { 127 | "var_input": { 128 | "type": "String" 129 | }, 130 | "var_output": { 131 | "type": "String" 132 | } 133 | }, 134 | "folder": { 135 | "name": "Azure Functions" 136 | }, 137 | "annotations": [] 138 | } 139 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_durable_function_simple_polling.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_durable_function_simple_polling", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Call Azure Durable Function", 7 | "type": "AzureFunctionActivity", 8 | "dependsOn": [ 9 | { 10 | "activity": "Set Input for Function App", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "policy": { 17 | "timeout": "7.00:00:00", 18 | "retry": 0, 19 | "retryIntervalInSeconds": 30, 20 | "secureOutput": false, 21 | "secureInput": false 22 | }, 23 | "userProperties": [], 24 | "typeProperties": { 25 | "functionName": "orchestrators/DurableFunctions-Orchestrator", 26 | "method": "POST", 27 | "body": { 28 | "value": "@variables('var_input')", 29 | "type": "Expression" 30 | } 31 | }, 32 | "linkedServiceName": { 33 | "referenceName": "ls_af", 34 | "type": "LinkedServiceReference" 35 | } 36 | }, 37 | { 38 | "name": "Set Input for Function App", 39 | "type": "SetVariable", 40 | "dependsOn": [], 41 | "userProperties": [], 42 | "typeProperties": { 43 | "variableName": "var_input", 44 | "value": "{\"timezone\":\"Eastern Standard Time\"}" 45 | } 46 | }, 47 | { 48 | "name": "Get Function App Output", 49 | "type": "SetVariable", 50 | "dependsOn": [ 51 | { 52 | "activity": "Get Current Function Status", 53 | "dependencyConditions": [ 54 | "Succeeded" 55 | ] 56 | } 57 | ], 58 | "userProperties": [], 59 | "typeProperties": { 60 | "variableName": "var_output", 61 | "value": { 62 | "value": "@activity('Get Current Function Status').output.output.Response", 63 | "type": "Expression" 64 | } 65 | } 66 | }, 67 | { 68 | "name": "Get Current Function Status", 69 | "type": "WebActivity", 70 | "dependsOn": [ 71 | { 72 | "activity": "Call Azure Durable Function", 73 | "dependencyConditions": [ 74 | "Succeeded" 75 | ] 76 | } 77 | ], 78 | "policy": { 79 | "timeout": "7.00:00:00", 80 | "retry": 0, 81 | "retryIntervalInSeconds": 30, 82 | "secureOutput": false, 83 | "secureInput": false 84 | }, 85 | "userProperties": [], 86 | "typeProperties": { 87 | "url": { 88 | "value": "@activity('Call Azure Durable Function').output.statusQueryGetUri\n", 89 | "type": "Expression" 90 | }, 91 | "method": "GET" 92 | } 93 | } 94 | ], 95 | "variables": { 96 | "var_input": { 97 | "type": "String" 98 | }, 99 | "var_output": { 100 | "type": "String" 101 | } 102 | }, 103 | "folder": { 104 | "name": "Azure Functions" 105 | }, 106 | "annotations": [] 107 | } 108 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_durable_function_simple_polling_safe.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_durable_function_simple_polling_safe", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Input for Function App", 7 | "type": "SetVariable", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "variableName": "var_input", 12 | "value": "{\"timezone\":\"Eastern Standard Time\"}" 13 | } 14 | }, 15 | { 16 | "name": "Get Function App Output", 17 | "type": "SetVariable", 18 | "dependsOn": [ 19 | { 20 | "activity": "Azure Function Execution", 21 | "dependencyConditions": [ 22 | "Succeeded" 23 | ] 24 | } 25 | ], 26 | "userProperties": [], 27 | "typeProperties": { 28 | "variableName": "var_output", 29 | "value": { 30 | "value": "@activity('Get Current Function Status').output.output.Response", 31 | "type": "Expression" 32 | } 33 | } 34 | }, 35 | { 36 | "name": "Azure Function Execution", 37 | "type": "Until", 38 | "dependsOn": [ 39 | { 40 | "activity": "Set Input for Function App", 41 | "dependencyConditions": [ 42 | "Succeeded" 43 | ] 44 | } 45 | ], 46 | "userProperties": [], 47 | "typeProperties": { 48 | "expression": { 49 | "value": "@equals(activity('Get Current Function Status').output.output.Status, 'OK')\n", 50 | "type": "Expression" 51 | }, 52 | "activities": [ 53 | { 54 | "name": "Call Azure Durable Function", 55 | "type": "AzureFunctionActivity", 56 | "dependsOn": [], 57 | "policy": { 58 | "timeout": "7.00:00:00", 59 | "retry": 0, 60 | "retryIntervalInSeconds": 30, 61 | "secureOutput": false, 62 | "secureInput": false 63 | }, 64 | "userProperties": [], 65 | "typeProperties": { 66 | "functionName": "orchestrators/DurableFunctions-Orchestrator", 67 | "method": "POST", 68 | "body": { 69 | "value": "@variables('var_input')", 70 | "type": "Expression" 71 | } 72 | }, 73 | "linkedServiceName": { 74 | "referenceName": "ls_af", 75 | "type": "LinkedServiceReference" 76 | } 77 | }, 78 | { 79 | "name": "Get Current Function Status", 80 | "type": "WebActivity", 81 | "dependsOn": [ 82 | { 83 | "activity": "Call Azure Durable Function", 84 | "dependencyConditions": [ 85 | "Succeeded" 86 | ] 87 | } 88 | ], 89 | "policy": { 90 | "timeout": "7.00:00:00", 91 | "retry": 0, 92 | "retryIntervalInSeconds": 30, 93 | "secureOutput": false, 94 | "secureInput": false 95 | }, 96 | "userProperties": [], 97 | "typeProperties": { 98 | "url": { 99 | "value": "@activity('Call Azure Durable Function').output.statusQueryGetUri\n", 100 | "type": "Expression" 101 | }, 102 | "method": "GET" 103 | } 104 | } 105 | ], 106 | "timeout": "0.01:00:00" 107 | } 108 | } 109 | ], 110 | "variables": { 111 | "var_input": { 112 | "type": "String" 113 | }, 114 | "var_output": { 115 | "type": "String" 116 | } 117 | }, 118 | "folder": { 119 | "name": "Azure Functions" 120 | }, 121 | "annotations": [] 122 | } 123 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_fail_activity.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_fail_activity", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "File Name", 7 | "type": "SetVariable", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "variableName": "var_filename", 12 | "value": "''" 13 | } 14 | }, 15 | { 16 | "name": "Folder Name", 17 | "type": "SetVariable", 18 | "dependsOn": [], 19 | "userProperties": [], 20 | "typeProperties": { 21 | "variableName": "var_foldername", 22 | "value": "files" 23 | } 24 | }, 25 | { 26 | "name": "Copy data", 27 | "type": "Copy", 28 | "dependsOn": [ 29 | { 30 | "activity": "Check Files", 31 | "dependencyConditions": [ 32 | "Succeeded" 33 | ] 34 | } 35 | ], 36 | "policy": { 37 | "timeout": "7.00:00:00", 38 | "retry": 0, 39 | "retryIntervalInSeconds": 30, 40 | "secureOutput": false, 41 | "secureInput": false 42 | }, 43 | "userProperties": [], 44 | "typeProperties": { 45 | "source": { 46 | "type": "DelimitedTextSource", 47 | "storeSettings": { 48 | "type": "AzureBlobFSReadSettings", 49 | "recursive": true, 50 | "enablePartitionDiscovery": false 51 | }, 52 | "formatSettings": { 53 | "type": "DelimitedTextReadSettings" 54 | } 55 | }, 56 | "sink": { 57 | "type": "DelimitedTextSink", 58 | "storeSettings": { 59 | "type": "AzureBlobFSWriteSettings" 60 | }, 61 | "formatSettings": { 62 | "type": "DelimitedTextWriteSettings", 63 | "quoteAllText": true, 64 | "fileExtension": ".txt" 65 | } 66 | }, 67 | "enableStaging": false, 68 | "translator": { 69 | "type": "TabularTranslator", 70 | "typeConversion": true, 71 | "typeConversionSettings": { 72 | "allowDataTruncation": true, 73 | "treatBooleanAsNumber": false 74 | } 75 | } 76 | }, 77 | "inputs": [ 78 | { 79 | "referenceName": "ADLS_Gen2", 80 | "type": "DatasetReference", 81 | "parameters": { 82 | "folder": { 83 | "value": "@variables('var_foldername')", 84 | "type": "Expression" 85 | }, 86 | "filename": { 87 | "value": "@variables('var_filename')", 88 | "type": "Expression" 89 | } 90 | } 91 | } 92 | ], 93 | "outputs": [ 94 | { 95 | "referenceName": "ADLS_Gen2", 96 | "type": "DatasetReference", 97 | "parameters": { 98 | "folder": { 99 | "value": "@variables('var_foldername')", 100 | "type": "Expression" 101 | }, 102 | "filename": { 103 | "value": "@variables('var_filename')", 104 | "type": "Expression" 105 | } 106 | } 107 | } 108 | ] 109 | }, 110 | { 111 | "name": "File not Found", 112 | "type": "Fail", 113 | "dependsOn": [ 114 | { 115 | "activity": "Check Files", 116 | "dependencyConditions": [ 117 | "Failed" 118 | ] 119 | } 120 | ], 121 | "userProperties": [], 122 | "typeProperties": { 123 | "message": "Source file not found", 124 | "errorCode": "3000" 125 | } 126 | }, 127 | { 128 | "name": "Check Files", 129 | "type": "Validation", 130 | "dependsOn": [ 131 | { 132 | "activity": "File Name", 133 | "dependencyConditions": [ 134 | "Succeeded" 135 | ] 136 | }, 137 | { 138 | "activity": "Folder Name", 139 | "dependencyConditions": [ 140 | "Succeeded" 141 | ] 142 | } 143 | ], 144 | "userProperties": [], 145 | "typeProperties": { 146 | "dataset": { 147 | "referenceName": "ADLS_Gen2", 148 | "type": "DatasetReference", 149 | "parameters": { 150 | "folder": { 151 | "value": "@variables('var_foldername')", 152 | "type": "Expression" 153 | }, 154 | "filename": { 155 | "value": "@variables('var_filename')", 156 | "type": "Expression" 157 | } 158 | } 159 | }, 160 | "timeout": "0.00:00:10", 161 | "sleep": 2 162 | } 163 | } 164 | ], 165 | "variables": { 166 | "var_filename": { 167 | "type": "String" 168 | }, 169 | "var_foldername": { 170 | "type": "String" 171 | } 172 | }, 173 | "folder": { 174 | "name": "Fail Activity" 175 | }, 176 | "annotations": [] 177 | } 178 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_fail_activity_old.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_fail_activity_old", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "File Name", 7 | "type": "SetVariable", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "variableName": "var_filename", 12 | "value": "''" 13 | } 14 | }, 15 | { 16 | "name": "Folder Name", 17 | "type": "SetVariable", 18 | "dependsOn": [], 19 | "userProperties": [], 20 | "typeProperties": { 21 | "variableName": "var_foldername", 22 | "value": "files" 23 | } 24 | }, 25 | { 26 | "name": "Copy data", 27 | "type": "Copy", 28 | "dependsOn": [ 29 | { 30 | "activity": "Check Files", 31 | "dependencyConditions": [ 32 | "Succeeded" 33 | ] 34 | } 35 | ], 36 | "policy": { 37 | "timeout": "7.00:00:00", 38 | "retry": 0, 39 | "retryIntervalInSeconds": 30, 40 | "secureOutput": false, 41 | "secureInput": false 42 | }, 43 | "userProperties": [], 44 | "typeProperties": { 45 | "source": { 46 | "type": "DelimitedTextSource", 47 | "storeSettings": { 48 | "type": "AzureBlobFSReadSettings", 49 | "recursive": true, 50 | "enablePartitionDiscovery": false 51 | }, 52 | "formatSettings": { 53 | "type": "DelimitedTextReadSettings" 54 | } 55 | }, 56 | "sink": { 57 | "type": "DelimitedTextSink", 58 | "storeSettings": { 59 | "type": "AzureBlobFSWriteSettings" 60 | }, 61 | "formatSettings": { 62 | "type": "DelimitedTextWriteSettings", 63 | "quoteAllText": true, 64 | "fileExtension": ".txt" 65 | } 66 | }, 67 | "enableStaging": false, 68 | "translator": { 69 | "type": "TabularTranslator", 70 | "typeConversion": true, 71 | "typeConversionSettings": { 72 | "allowDataTruncation": true, 73 | "treatBooleanAsNumber": false 74 | } 75 | } 76 | }, 77 | "inputs": [ 78 | { 79 | "referenceName": "ADLS_Gen2", 80 | "type": "DatasetReference", 81 | "parameters": { 82 | "folder": { 83 | "value": "@variables('var_foldername')", 84 | "type": "Expression" 85 | }, 86 | "filename": { 87 | "value": "@variables('var_filename')", 88 | "type": "Expression" 89 | } 90 | } 91 | } 92 | ], 93 | "outputs": [ 94 | { 95 | "referenceName": "ADLS_Gen2", 96 | "type": "DatasetReference", 97 | "parameters": { 98 | "folder": { 99 | "value": "@variables('var_foldername')", 100 | "type": "Expression" 101 | }, 102 | "filename": { 103 | "value": "@variables('var_filename')", 104 | "type": "Expression" 105 | } 106 | } 107 | } 108 | ] 109 | }, 110 | { 111 | "name": "Check Files", 112 | "type": "Validation", 113 | "dependsOn": [ 114 | { 115 | "activity": "File Name", 116 | "dependencyConditions": [ 117 | "Succeeded" 118 | ] 119 | }, 120 | { 121 | "activity": "Folder Name", 122 | "dependencyConditions": [ 123 | "Succeeded" 124 | ] 125 | } 126 | ], 127 | "userProperties": [], 128 | "typeProperties": { 129 | "dataset": { 130 | "referenceName": "ADLS_Gen2", 131 | "type": "DatasetReference", 132 | "parameters": { 133 | "folder": { 134 | "value": "@variables('var_foldername')", 135 | "type": "Expression" 136 | }, 137 | "filename": { 138 | "value": "@variables('var_filename')", 139 | "type": "Expression" 140 | } 141 | } 142 | }, 143 | "timeout": "0.00:00:10", 144 | "sleep": 2 145 | } 146 | } 147 | ], 148 | "variables": { 149 | "var_filename": { 150 | "type": "String" 151 | }, 152 | "var_foldername": { 153 | "type": "String" 154 | } 155 | }, 156 | "folder": { 157 | "name": "Fail Activity" 158 | }, 159 | "annotations": [] 160 | } 161 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_html_formatted_email.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_html_formatted_email", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Execute Azure SQL SP with Output", 7 | "type": "Lookup", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "source": { 19 | "type": "AzureSqlSource", 20 | "sqlReaderStoredProcedureName": "[dbo].[sp_top_10_selling_products]", 21 | "queryTimeout": "02:00:00" 22 | }, 23 | "dataset": { 24 | "referenceName": "ds_azsql_generic", 25 | "type": "DatasetReference" 26 | }, 27 | "firstRowOnly": false 28 | } 29 | }, 30 | { 31 | "name": "Set Email Body", 32 | "type": "SetVariable", 33 | "dependsOn": [ 34 | { 35 | "activity": "Execute Azure SQL SP with Output", 36 | "dependencyConditions": [ 37 | "Succeeded" 38 | ] 39 | } 40 | ], 41 | "userProperties": [], 42 | "typeProperties": { 43 | "variableName": "var_email_body", 44 | "value": { 45 | "value": "@activity('Execute Azure SQL SP with Output').output.value[0].output_result", 46 | "type": "Expression" 47 | } 48 | } 49 | }, 50 | { 51 | "name": "Set Email Recipient", 52 | "type": "SetVariable", 53 | "dependsOn": [], 54 | "userProperties": [], 55 | "typeProperties": { 56 | "variableName": "var_email_to", 57 | "value": "rimayev@hotmail.com" 58 | } 59 | }, 60 | { 61 | "name": "Set Email Subject", 62 | "type": "SetVariable", 63 | "dependsOn": [], 64 | "userProperties": [], 65 | "typeProperties": { 66 | "variableName": "var_email_subject", 67 | "value": "Top 10 Selling Products" 68 | } 69 | }, 70 | { 71 | "name": "Email Notfication", 72 | "type": "WebActivity", 73 | "dependsOn": [ 74 | { 75 | "activity": "Set Email Recipient", 76 | "dependencyConditions": [ 77 | "Succeeded" 78 | ] 79 | }, 80 | { 81 | "activity": "Set Email Body", 82 | "dependencyConditions": [ 83 | "Succeeded" 84 | ] 85 | }, 86 | { 87 | "activity": "Set Email Subject", 88 | "dependencyConditions": [ 89 | "Succeeded" 90 | ] 91 | } 92 | ], 93 | "policy": { 94 | "timeout": "7.00:00:00", 95 | "retry": 0, 96 | "retryIntervalInSeconds": 30, 97 | "secureOutput": false, 98 | "secureInput": false 99 | }, 100 | "userProperties": [], 101 | "typeProperties": { 102 | "url": "https://prod-48.eastus2.logic.azure.com:443/workflows/3e3e32bccb5843e8b5c1ef0821498737/triggers/manual/paths/invoke?api-version=2016-10-01&sp=%2Ftriggers%2Fmanual%2Frun&sv=1.0&sig=IKBjhnmOVsuPxor_PYtuyjK9o38fLUB4WNorY-5THro", 103 | "method": "POST", 104 | "headers": { 105 | "Content-Type": "application/json" 106 | }, 107 | "body": { 108 | "EmailTo": "@variables('var_email_to')", 109 | "EmailSubject": "@variables('var_email_subject')", 110 | "EmailBody": "@variables('var_email_body')" 111 | } 112 | } 113 | } 114 | ], 115 | "variables": { 116 | "var_email_body": { 117 | "type": "String" 118 | }, 119 | "var_email_to": { 120 | "type": "String" 121 | }, 122 | "var_email_subject": { 123 | "type": "String" 124 | } 125 | }, 126 | "folder": { 127 | "name": "Azure Functions" 128 | }, 129 | "annotations": [] 130 | } 131 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_json_arrays.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_json_arrays", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "List of files and folders", 7 | "type": "GetMetadata", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataset": { 19 | "referenceName": "validation_adls_source_ds", 20 | "type": "DatasetReference", 21 | "parameters": { 22 | "FileFolder": "storesales" 23 | } 24 | }, 25 | "fieldList": [ 26 | "childItems", 27 | "itemName", 28 | "itemType", 29 | "lastModified" 30 | ] 31 | } 32 | }, 33 | { 34 | "name": "Set Complete Task output", 35 | "type": "SetVariable", 36 | "dependsOn": [ 37 | { 38 | "activity": "List of files and folders", 39 | "dependencyConditions": [ 40 | "Succeeded" 41 | ] 42 | } 43 | ], 44 | "userProperties": [], 45 | "typeProperties": { 46 | "variableName": "var_json_payload", 47 | "value": { 48 | "value": "@array(activity('List of files and folders').output)", 49 | "type": "Expression" 50 | } 51 | } 52 | }, 53 | { 54 | "name": "Set Sub Array", 55 | "type": "SetVariable", 56 | "dependsOn": [ 57 | { 58 | "activity": "List of files and folders", 59 | "dependencyConditions": [ 60 | "Succeeded" 61 | ] 62 | } 63 | ], 64 | "userProperties": [], 65 | "typeProperties": { 66 | "variableName": "var_json_subarray", 67 | "value": { 68 | "value": "@activity('List of files and folders').output.childitems", 69 | "type": "Expression" 70 | } 71 | } 72 | }, 73 | { 74 | "name": "Reading JSON file", 75 | "type": "Lookup", 76 | "dependsOn": [], 77 | "policy": { 78 | "timeout": "7.00:00:00", 79 | "retry": 0, 80 | "retryIntervalInSeconds": 30, 81 | "secureOutput": false, 82 | "secureInput": false 83 | }, 84 | "userProperties": [], 85 | "typeProperties": { 86 | "source": { 87 | "type": "JsonSource", 88 | "storeSettings": { 89 | "type": "AzureBlobStorageReadSettings", 90 | "recursive": true 91 | } 92 | }, 93 | "dataset": { 94 | "referenceName": "ds_json_dataflow_sample", 95 | "type": "DatasetReference" 96 | }, 97 | "firstRowOnly": false 98 | } 99 | }, 100 | { 101 | "name": "For Each JSON element", 102 | "type": "ForEach", 103 | "dependsOn": [ 104 | { 105 | "activity": "Reading JSON file", 106 | "dependencyConditions": [ 107 | "Succeeded" 108 | ] 109 | } 110 | ], 111 | "userProperties": [], 112 | "typeProperties": { 113 | "items": { 114 | "value": "@activity('Reading JSON file').output.value", 115 | "type": "Expression" 116 | }, 117 | "isSequential": false, 118 | "activities": [ 119 | { 120 | "name": "Execute Baking pipeline", 121 | "type": "ExecutePipeline", 122 | "dependsOn": [], 123 | "userProperties": [], 124 | "typeProperties": { 125 | "pipeline": { 126 | "referenceName": "pl_json_arrays_sub", 127 | "type": "PipelineReference" 128 | }, 129 | "waitOnCompletion": true, 130 | "parameters": { 131 | "param_baking_payload": { 132 | "value": "@array(item())", 133 | "type": "Expression" 134 | } 135 | } 136 | } 137 | } 138 | ] 139 | } 140 | } 141 | ], 142 | "variables": { 143 | "var_json_payload": { 144 | "type": "Array" 145 | }, 146 | "var_json_subarray": { 147 | "type": "Array" 148 | } 149 | }, 150 | "folder": { 151 | "name": "JSON data" 152 | }, 153 | "annotations": [] 154 | } 155 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_json_arrays_sub.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_json_arrays_sub", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Get Toppings", 7 | "type": "SetVariable", 8 | "dependsOn": [ 9 | { 10 | "activity": "Get Baking Recepie", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "userProperties": [], 17 | "typeProperties": { 18 | "variableName": "var_toppings", 19 | "value": { 20 | "value": "@variables('var_baking_payload')[0].topping", 21 | "type": "Expression" 22 | } 23 | } 24 | }, 25 | { 26 | "name": "Get Batters", 27 | "type": "SetVariable", 28 | "dependsOn": [ 29 | { 30 | "activity": "Get Baking Recepie", 31 | "dependencyConditions": [ 32 | "Succeeded" 33 | ] 34 | } 35 | ], 36 | "userProperties": [], 37 | "typeProperties": { 38 | "variableName": "var_batters", 39 | "value": { 40 | "value": "@variables('var_baking_payload')[0].batters.batter", 41 | "type": "Expression" 42 | } 43 | } 44 | }, 45 | { 46 | "name": "Get Baking Recepie", 47 | "type": "SetVariable", 48 | "dependsOn": [], 49 | "userProperties": [], 50 | "typeProperties": { 51 | "variableName": "var_baking_payload", 52 | "value": { 53 | "value": "@pipeline().parameters.param_baking_payload", 54 | "type": "Expression" 55 | } 56 | } 57 | } 58 | ], 59 | "parameters": { 60 | "param_baking_payload": { 61 | "type": "array" 62 | } 63 | }, 64 | "variables": { 65 | "var_toppings": { 66 | "type": "Array" 67 | }, 68 | "var_batters": { 69 | "type": "Array" 70 | }, 71 | "var_baking_payload": { 72 | "type": "Array" 73 | } 74 | }, 75 | "folder": { 76 | "name": "JSON data" 77 | }, 78 | "annotations": [] 79 | } 80 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_passing_array_between_pipelines.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_passing_array_between_pipelines", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Array Variable", 7 | "type": "SetVariable", 8 | "dependsOn": [ 9 | { 10 | "activity": "pl_passing_array_between_pipelines_child", 11 | "dependencyConditions": [ 12 | "Succeeded" 13 | ] 14 | } 15 | ], 16 | "userProperties": [], 17 | "typeProperties": { 18 | "variableName": "var_meal_array", 19 | "value": { 20 | "value": "@json('[\"Egg\",\"Toast\",\"Coffee\"]')", 21 | "type": "Expression" 22 | } 23 | } 24 | }, 25 | { 26 | "name": "pl_passing_array_between_pipelines_child", 27 | "type": "ExecutePipeline", 28 | "dependsOn": [], 29 | "userProperties": [], 30 | "typeProperties": { 31 | "pipeline": { 32 | "referenceName": "pl_passing_array_child", 33 | "type": "PipelineReference" 34 | }, 35 | "waitOnCompletion": true, 36 | "parameters": { 37 | "par_meal_array": "@json('[\"Egg\",\"Toast\",\"Coffee\"]')" 38 | } 39 | } 40 | } 41 | ], 42 | "parameters": { 43 | "par_meal_array": { 44 | "type": "array", 45 | "defaultValue": [ 46 | "Egg", 47 | "Toast", 48 | "Coffee" 49 | ] 50 | } 51 | }, 52 | "variables": { 53 | "var_meal_array": { 54 | "type": "Array", 55 | "defaultValue": [ 56 | "Egg", 57 | "Toast", 58 | "Coffee" 59 | ] 60 | } 61 | }, 62 | "folder": { 63 | "name": "Working with Arrays" 64 | }, 65 | "annotations": [] 66 | } 67 | } -------------------------------------------------------------------------------- /Samples/pipeline/pl_passing_array_child.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pl_passing_array_child", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Array Variable", 7 | "type": "SetVariable", 8 | "dependsOn": [], 9 | "userProperties": [], 10 | "typeProperties": { 11 | "variableName": "var_meal_array", 12 | "value": { 13 | "value": "@pipeline().parameters.par_meal_array", 14 | "type": "Expression" 15 | } 16 | } 17 | } 18 | ], 19 | "parameters": { 20 | "par_meal_array": { 21 | "type": "array", 22 | "defaultValue": [ 23 | "Egg", 24 | "Toast", 25 | "Coffee" 26 | ] 27 | } 28 | }, 29 | "variables": { 30 | "var_meal_array": { 31 | "type": "Array" 32 | } 33 | }, 34 | "folder": { 35 | "name": "Working with Arrays" 36 | }, 37 | "annotations": [] 38 | } 39 | } --------------------------------------------------------------------------------