├── README.md ├── integrationRuntime ├── selfhostedir.json ├── integrationRuntime1.json ├── integrationRuntime3.json ├── alwaysonIR.json ├── EastUSIR.json ├── newwestus.json ├── computeopt.json ├── dataflowWestUs2.json ├── dataflows.json ├── smallcluster.json ├── dataflowcluster.json ├── integrationRuntime4.json ├── myDataFlowCluster.json ├── integrationRuntime5.json ├── mylargecluster.json ├── integrationRuntime6.json ├── ssis2.json └── integrationRuntime2.json ├── managedVirtualNetwork └── default.json ├── factory └── makadfv2.json ├── linkedService ├── AzureKeyVault1.json ├── AzureKeyVault2.json ├── AzureKeyVault3.json ├── AzureKeyVault4.json ├── AzureDataLakeStorage4MSI.json ├── RestService1.json ├── HttpServer1.json ├── sqldbakv.json ├── AzureSqlDatabase3.json ├── AzureSqlDatabase5KV.json ├── AzureDataLakeStore1.json ├── AzureSqlDatabase2.json ├── AzureFunction1.json ├── AzureDatabricks2.json ├── AzureDatabricks3.json ├── AzureDataLakeStorage3.json ├── MoviesADLS.json ├── AzureDataLakeStorage1.json ├── AzureDataLakeStorage4.json ├── mynewls.json ├── mynewls1.json ├── BaseballLS.json ├── AzureDatabricks1.json ├── CosmosDb2.json ├── CosmosDb3.json ├── AzureBlobStorage2.json ├── AzureBlobStorage3.json ├── CosmosDW.json ├── CosmosDb1.json ├── MarksBlob.json ├── MyAzureBlobLinkedService.json ├── blobParamLS2.json ├── blobparamLS.json ├── CosmosDb4Northwind.json ├── databrickswest.json ├── AzureBlobStorage1.json ├── AzureSqlDW2.json ├── Snowflake1.json ├── DWDimProd.json ├── dwaw.json ├── AzureSqlDW1.json ├── AzureDataLakeStorage2.json ├── AzureSqlDatabase1.json └── AzureSqlDatabase4.json ├── dataset ├── ParamDB.json ├── RestResource1.json ├── DBStage.json ├── soccerDB.json ├── LoadFacts.json ├── AzureSqlTable5.json ├── FactTable2.json ├── cosmoscoke.json ├── DimEmployee2.json ├── cosmosMovies.json ├── AzureSqlDWTable5.json ├── AzureSqlTable6.json ├── cosmos_dimprod.json ├── DimEmp.json ├── cosmos_dimcust.json ├── CosmosDbAwDwMetrics.json ├── FactTable21.json ├── CosmosDbSqlApiCollection1.json ├── CosmosDbSqlApiCollection3.json ├── Json7.json ├── MoviesSQLDB.json ├── CosmosDbSqlApiCollection2.json ├── SQLTweets.json ├── loansDB.json ├── DemoTable.json ├── sqlorders.json ├── AzureSqlTable9KV.json ├── SQLDBTransformer.json ├── AzureSqlDWTable2.json ├── Json1.json ├── DWSabermetrics.json ├── Json2.json ├── newbattingtable.json ├── JsonFolder.json ├── JsonOutput.json ├── Avro1.json ├── Orc1.json ├── Parquet3.json ├── Json6.json ├── partout.json ├── jsonOrders.json ├── Json3.json ├── XmlCustomers1.json ├── loansSink.json ├── Avro1_copy1.json ├── ParqFolder.json ├── ParquetPart.json ├── ParquetPartMovies.json ├── ParquetStage.json ├── dupefolder.json ├── moviesParquet2.json ├── moviesparquet_input.json ├── DelimitedText13.json ├── soccerOutput.json ├── DelimitedText5.json ├── Parquet2.json ├── Parquet4.json ├── dummyContainer.json ├── moviesoutnew_parquet.json ├── AzureSqlDWTable3.json ├── Parquet5.json ├── OutputContainer1.json ├── AzureSqlTable7.json ├── xmlout.json ├── tweets.json ├── DestinationDataset_d2r.json ├── TaxiFiles.json ├── marktabledw.json ├── sampledata.json ├── moviesoutputfolder_parquet.json ├── dbSourceParams.json ├── genericsources.json ├── mytable.json ├── paramsfile.json ├── CADOutput1.json ├── ParquetOutput.json ├── TaxiDemoDayStatsSink1.json ├── dbTargetParams.json ├── moviesoutnew_csv.json ├── H_P.json ├── TaxiDemoVendorStatsSink1.json ├── delimiterout.json ├── CurrencyDatasetCAD1.json ├── ParameterTable.json ├── TaxiDemoTotalByPaymentType1.json ├── adls_msi.json ├── perfout.json ├── readskipped.json ├── ParameterTable1.json ├── adlsoutput_folder.json ├── hadooplog.json ├── others.json ├── DelimitedText2.json ├── adlsoutput_folder1.json ├── folderout.json ├── moviesfolder.json ├── salesskip.json ├── AzureBlob3.json ├── dummysource.json ├── genericfolder.json ├── folderout1.json ├── moviesDriftSource1.json ├── wrangleout.json ├── Loans2.json ├── dateRange.json ├── junktable.json ├── MoviesD21.json ├── AzureDataLakeStorageFile1.json ├── folderout_noheaders.json ├── DelimitedText3.json ├── TaxiSink.json ├── TaxiSink1.json ├── MoviesD2Header.json ├── MoviesDW.json ├── movieszipout.json ├── AzureDataLakeStorageFile2.json ├── fixedwidth.json ├── MoviesD2_fixed.json ├── folderout_copy1.json ├── movie_dataflow_sink1.json ├── AzureSQLIncremental_Changes.json ├── moviescompressed.json ├── CosmosDbSqlApiCollection4.json ├── DelimitedText12.json ├── AzureSqlIncremental_Data.json ├── BattingSink.json ├── CADOutput.json ├── locationMap_dt.json ├── DelimitedText4.json ├── DelimitedText6.json ├── DestinationDataset_cce.json ├── DestinationDataset_5ra.json ├── AzureBlob1.json ├── AzureBlobBaseballOutput.json ├── TaxiDemoVendorStatsSink.json ├── AzureSqlTable2.json ├── salesdrift.json ├── DestinationDataset_hii.json ├── Sales3.json ├── sales1.json ├── sales2.json ├── SalesUpdate.json ├── ParquetInput1.json ├── ParquetCrudOutput1.json ├── dynadelim.json ├── Facts.json ├── blob_params.json ├── MoviesJSON.json ├── CurrencyDatasetUSD1.json ├── JSONMoviesWithSchema.json ├── moviesout.json ├── names.json ├── BlobInputFolder.json ├── BlobInputFolder1.json ├── locationMap.json ├── CurrencyRates.json ├── CurrencyRates1.json ├── Facts1.json ├── Excel1.json ├── salesXLS.json ├── DelimitedText14.json ├── MoviesWithRatings.json ├── DelimitedText11.json ├── moviechanges.json ├── CurrencySales.json ├── CurrencySales1.json ├── jsonblob.json ├── RetailProductsCSV.json ├── moviesADLS.json ├── timesheetCSV.json ├── AzureBlob2.json ├── SQLProducts.json ├── DimEmployee.json ├── moviesDB2.json ├── movies2.json ├── searchLog.json ├── ParquetMovies.json ├── dataflowlogger.json ├── Json5.json ├── moviestage2.json ├── movieupdates_parquet.json ├── CurrencyDatasetUSD.json ├── MoviesD2.json ├── driverscsv.json ├── movie_dataflow_source11.json ├── sqltablev2.json ├── movie_dataflow_source1.json ├── EmployeeFiles1.json ├── SQLProdFromSales.json ├── moviesoutnew_parquet2.json ├── EmployeeDay1.json ├── EmployeeDay2.json ├── AzureSqlTable1.json └── EmployeeFiles.json ├── pipeline ├── Dedupe Pipeline.json ├── LoadFacts2.json ├── updateDB.json ├── soccerPipeline.json ├── Duplicates.json ├── deltapipe.json ├── Hierarchical.json ├── Pipe With Vars.json ├── testpipeline.json ├── CDM_Pipeline.json ├── snowflake.json ├── pipeline1.json ├── Fixed Width Data Flow Pipeline.json ├── summaryStats.json ├── fileUpdates.json ├── dynamicFlows.json ├── SearchLogAnalytics.json ├── paramQueryPipe.json ├── masterPipeline.json ├── Load Multiple Tables.json ├── metadatapipeline.json └── invokedPipeline.json └── dataflow ├── moviesCompressed.json ├── delimparam.json ├── processMultipleTables.json ├── paramWilcards.json ├── skipped.json ├── sinkDemo.json ├── paramQuery.json ├── dynafile.json ├── denormOrders.json ├── LoadSnowflakeMovies.json ├── findNulls.json ├── DeltaFlow1.json ├── WranglingDataFlow5.json ├── dataflow3.json ├── Hierarchical.json └── SQLPerf.json /README.md: -------------------------------------------------------------------------------- 1 | hello world 2 | -------------------------------------------------------------------------------- /integrationRuntime/selfhostedir.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "selfhostedir", 3 | "properties": { 4 | "type": "SelfHosted" 5 | } 6 | } -------------------------------------------------------------------------------- /managedVirtualNetwork/default.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "type": "Microsoft.DataFactory/factories/managedvirtualnetworks" 4 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime1", 3 | "properties": { 4 | "type": "SelfHosted" 5 | } 6 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime3", 3 | "properties": { 4 | "type": "SelfHosted" 5 | } 6 | } -------------------------------------------------------------------------------- /factory/makadfv2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "makadfv2", 3 | "properties": { 4 | "globalConfigurations": { 5 | "dataFlowRuntimeVersion": "Stable" 6 | } 7 | }, 8 | "location": "EastUS" 9 | } -------------------------------------------------------------------------------- /integrationRuntime/alwaysonIR.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "alwaysonIR", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "Central US" 8 | } 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/AzureKeyVault1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureKeyVault1", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureKeyVault", 6 | "typeProperties": { 7 | "baseUrl": "https://makkv001.vault.azure.net/" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /linkedService/AzureKeyVault2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureKeyVault2", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureKeyVault", 6 | "typeProperties": { 7 | "baseUrl": "https://makkv001.vault.azure.net/" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /linkedService/AzureKeyVault3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureKeyVault3", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureKeyVault", 6 | "typeProperties": { 7 | "baseUrl": "https://makkv001.vault.azure.net/" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /linkedService/AzureKeyVault4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureKeyVault4", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureKeyVault", 6 | "typeProperties": { 7 | "baseUrl": "https://makkv001.vault.azure.net/" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStorage4MSI.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage4MSI", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureBlobFS", 6 | "typeProperties": { 7 | "url": "https://makadlsg2b.dfs.core.windows.net" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /dataset/ParamDB.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParamDB", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [] 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/RestResource1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RestResource1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "RestService1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "RestResource", 10 | "schema": [] 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/DBStage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DBStage", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "moviestage" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/soccerDB.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "soccerDB", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "dbo.soccerDB" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /linkedService/RestService1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RestService1", 3 | "properties": { 4 | "annotations": [], 5 | "type": "RestService", 6 | "typeProperties": { 7 | "url": "http://worldtimeapi.org/api/ip\"", 8 | "enableServerCertificateValidation": true, 9 | "authenticationType": "Anonymous" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/LoadFacts.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "LoadFacts", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "dbo.SalesFacts" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable5", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "dbo.DimEmp" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/FactTable2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FactTable2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "dbo.FactTable2" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/cosmoscoke.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cosmoscoke", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb3", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "coke" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/DimEmployee2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DimEmployee2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "[dbo].[DimEmployee]" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/cosmosMovies.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cosmosMovies", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb3", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "orders" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/AzureSqlDWTable5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDWTable5", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDW1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlDWTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "normbattingstats2" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable6", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [], 10 | "typeProperties": { 11 | "tableName": "dbo.reverseBatting" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/cosmos_dimprod.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cosmos_dimprod", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "dimprod" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/DimEmp.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DimEmp", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "tableName": "[dbo].[DimEmp]" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /dataset/cosmos_dimcust.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cosmos_dimcust", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "dimcustomer" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/CosmosDbAwDwMetrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDbAwDwMetrics", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "awdwmetrics" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/FactTable21.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "FactTable21", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "tableName": "dbo.FactTable2" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /dataset/CosmosDbSqlApiCollection1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDbSqlApiCollection1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb3", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "movie" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/CosmosDbSqlApiCollection3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDbSqlApiCollection3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "Items" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/Json7.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json7", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "HttpServer1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "HttpServerLocation" 13 | } 14 | }, 15 | "schema": {} 16 | } 17 | } -------------------------------------------------------------------------------- /dataset/MoviesSQLDB.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesSQLDB", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "tableName": "[dbo].[movietest]" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /integrationRuntime/EastUSIR.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "EastUSIR", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "East US", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 80, 11 | "timeToLive": 60 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/newwestus.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "newwestus", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "West US", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 80, 11 | "timeToLive": 240 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/CosmosDbSqlApiCollection2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDbSqlApiCollection2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "typeProperties": { 11 | "collectionName": "movies" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /dataset/SQLTweets.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SQLTweets", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "tweets" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/loansDB.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "loansDB", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase4", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "loanstest" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/computeopt.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "computeopt", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "ComputeOptimized", 10 | "coreCount": 16, 11 | "timeToLive": 0 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/dataflowWestUs2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflowWestUs2", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "West US 2", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 80, 11 | "timeToLive": 240 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/dataflows.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflows", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 80, 11 | "timeToLive": 240 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/DemoTable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DemoTable", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "DemoTable" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/smallcluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "smallcluster", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "ComputeOptimized", 10 | "coreCount": 32, 11 | "timeToLive": 10 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/sqlorders.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqlorders", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "cosmosdborders" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/dataflowcluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflowcluster", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 80, 11 | "timeToLive": 240 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime4", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 32, 11 | "timeToLive": 60 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable9KV.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable9KV", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase5KV", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "loans" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/myDataFlowCluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "myDataFlowCluster", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 80, 11 | "timeToLive": 60 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /linkedService/HttpServer1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "HttpServer1", 3 | "properties": { 4 | "annotations": [], 5 | "type": "HttpServer", 6 | "typeProperties": { 7 | "url": "http://worldtimeapi.org/api/ip", 8 | "enableServerCertificateValidation": false, 9 | "authenticationType": "Anonymous" 10 | } 11 | }, 12 | "type": "Microsoft.DataFactory/factories/linkedservices" 13 | } -------------------------------------------------------------------------------- /linkedService/sqldbakv.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqldbakv", 3 | "properties": { 4 | "type": "AzureSqlDatabase", 5 | "typeProperties": { 6 | "connectionString": { 7 | "type": "AzureKeyVaultSecret", 8 | "store": { 9 | "referenceName": "AzureKeyVault1", 10 | "type": "LinkedServiceReference" 11 | }, 12 | "secretName": "SQLDB1" 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/SQLDBTransformer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SQLDBTransformer", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [], 11 | "typeProperties": { 12 | "schema": "dbo", 13 | "table": "transformer" 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime5", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 32, 11 | "timeToLive": 0 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /integrationRuntime/mylargecluster.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mylargecluster", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "MemoryOptimized", 10 | "coreCount": 68, 11 | "timeToLiveInMinutes": 60 12 | } 13 | } 14 | } 15 | } 16 | } -------------------------------------------------------------------------------- /dataset/AzureSqlDWTable2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDWTable2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDW1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Misc" 10 | }, 11 | "type": "AzureSqlDWTable", 12 | "typeProperties": { 13 | "tableName": "batting" 14 | } 15 | }, 16 | "type": "Microsoft.DataFactory/factories/datasets" 17 | } -------------------------------------------------------------------------------- /dataset/Json1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output", 14 | "container": "mycontainer" 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDatabase3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase3", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureSqlDatabase", 6 | "typeProperties": { 7 | "connectionString": { 8 | "type": "AzureKeyVaultSecret", 9 | "store": { 10 | "referenceName": "AzureKeyVault2", 11 | "type": "LinkedServiceReference" 12 | }, 13 | "secretName": "mak" 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /dataset/DWSabermetrics.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DWSabermetrics", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDW1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Baseball" 10 | }, 11 | "type": "AzureSqlDWTable", 12 | "typeProperties": { 13 | "tableName": "Sabermetrics" 14 | } 15 | }, 16 | "type": "Microsoft.DataFactory/factories/datasets" 17 | } -------------------------------------------------------------------------------- /dataset/Json2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "filelist.json", 14 | "container": "mycontainer" 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /dataset/newbattingtable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "newbattingtable", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Baseball" 10 | }, 11 | "type": "AzureSqlTable", 12 | "typeProperties": { 13 | "tableName": "batting1" 14 | } 15 | }, 16 | "type": "Microsoft.DataFactory/factories/datasets" 17 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDatabase5KV.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase5KV", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureSqlDatabase", 6 | "typeProperties": { 7 | "connectionString": { 8 | "type": "AzureKeyVaultSecret", 9 | "store": { 10 | "referenceName": "AzureKeyVault4", 11 | "type": "LinkedServiceReference" 12 | }, 13 | "secretName": "SQLDB1" 14 | } 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /dataset/JsonFolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "JsonFolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/json", 14 | "container": "mycontainer" 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /dataset/JsonOutput.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "JsonOutput", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/json", 14 | "container": "mycontainer" 15 | } 16 | } 17 | } 18 | } -------------------------------------------------------------------------------- /dataset/Avro1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Avro1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Avro", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/avro", 14 | "container": "mycontainer" 15 | } 16 | }, 17 | "schema": {} 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/Orc1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Orc1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Orc", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output", 14 | "container": "mycontainer" 15 | }, 16 | "enablePhysicalSchema": true 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/Parquet3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Parquet3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "compressionCodec": "snappy" 16 | }, 17 | "schema": [] 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/Json6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json6", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MarksBlob", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "5G8S002ZF (003).json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/partout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "partout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "output/parts", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": "," 16 | }, 17 | "schema": [] 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/jsonOrders.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jsonOrders", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "orders.json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/Json3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "TaskRun_Config_Sample.json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/XmlCustomers1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "XmlCustomers1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Xml", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "customers.xml", 14 | "folderPath": "SampleData/XML", 15 | "container": "mycontainer" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStore1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStore1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureDataLakeStore", 6 | "typeProperties": { 7 | "dataLakeStoreUri": "https://makadls001.azuredatalakestore.net/webhdfs/v1", 8 | "tenant": "72f988bf-86f1-41af-91ab-2d7cd011db47", 9 | "subscriptionId": "1e42591f-1f0c-4c5a-b7f2-a268f6105ec5", 10 | "resourceGroupName": "makbricksadf" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDatabase2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase2", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureSqlDatabase", 6 | "typeProperties": { 7 | "connectionString": { 8 | "type": "AzureKeyVaultSecret", 9 | "store": { 10 | "referenceName": "AzureKeyVault3", 11 | "type": "LinkedServiceReference" 12 | }, 13 | "secretName": "mak" 14 | } 15 | } 16 | }, 17 | "type": "Microsoft.DataFactory/factories/linkedservices" 18 | } -------------------------------------------------------------------------------- /dataset/loansSink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "loansSink", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output/parts/loans", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/Avro1_copy1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Avro1_copy1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Avro", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/avro", 14 | "container": "mycontainer" 15 | }, 16 | "avroCompressionCodec": "snappy" 17 | }, 18 | "schema": {} 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/ParqFolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParqFolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/bydate", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/ParquetPart.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetPart", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/parts", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime6", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "AutoResolve", 8 | "dataFlowProperties": { 9 | "computeType": "General", 10 | "coreCount": 8, 11 | "timeToLive": 0 12 | } 13 | } 14 | }, 15 | "managedVirtualNetwork": { 16 | "type": "ManagedVirtualNetworkReference", 17 | "referenceName": "default" 18 | } 19 | } 20 | } -------------------------------------------------------------------------------- /linkedService/AzureFunction1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureFunction1", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureFunction", 6 | "typeProperties": { 7 | "functionAppUrl": "https://makfunc.azurewebsites.net", 8 | "functionKey": { 9 | "type": "AzureKeyVaultSecret", 10 | "store": { 11 | "referenceName": "AzureKeyVault1", 12 | "type": "LinkedServiceReference" 13 | }, 14 | "secretName": "mysec12345", 15 | "secretVersion": "" 16 | } 17 | } 18 | } 19 | } -------------------------------------------------------------------------------- /dataset/ParquetPartMovies.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetPartMovies", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output/parts/movies", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/ParquetStage.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetStage", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/DWstaging", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/dupefolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dupefolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "output/dupes", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": true 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/moviesParquet2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesParquet2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/parquet", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/moviesparquet_input.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesparquet_input", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage4", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "movies", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/DelimitedText13.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText13", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileSystem": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/soccerOutput.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "soccerOutput", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "output/soccer", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": true 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /linkedService/AzureDatabricks2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDatabricks2", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureDatabricks", 6 | "typeProperties": { 7 | "domain": "https://adb-860887232373193.13.azuredatabricks.net", 8 | "authentication": "MSI", 9 | "workspaceResourceId": "/subscriptions/1e42591f-1f0c-4c5a-b7f2-a268f6105ec5/resourceGroups/keshADF_test/providers/Microsoft.Databricks/workspaces/keshWorkSpace", 10 | "existingClusterId": "0920-185412-happy305" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /linkedService/AzureDatabricks3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDatabricks3", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureDatabricks", 6 | "typeProperties": { 7 | "domain": "https://adb-860887232373193.13.azuredatabricks.net", 8 | "authentication": "MSI", 9 | "workspaceResourceId": "/subscriptions/1e42591f-1f0c-4c5a-b7f2-a268f6105ec5/resourceGroups/keshADF_test/providers/Microsoft.Databricks/workspaces/keshWorkSpace", 10 | "existingClusterId": "0920-185412-happy305" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /dataset/DelimitedText5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText5", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "container": "mycontainer" 13 | }, 14 | "columnDelimiter": "," 15 | }, 16 | "schema": [ 17 | { 18 | "name": "_c0", 19 | "type": "String" 20 | } 21 | ] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/Parquet2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Parquet2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "Parquet", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "SampleData", 13 | "container": "mycontainer" 14 | }, 15 | "compressionCodec": "snappy" 16 | }, 17 | "schema": [] 18 | }, 19 | "type": "Microsoft.DataFactory/factories/datasets" 20 | } -------------------------------------------------------------------------------- /dataset/Parquet4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Parquet4", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "mark.parquet", 14 | "folderPath": "output/parts/taxi", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "compressionCodec": "snappy" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/dummyContainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dummyContainer", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/moviesoutnew_parquet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesoutnew_parquet", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "Parquet", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobFSLocation", 12 | "fileName": "moviesoutnew.csv", 13 | "folderPath": "movies/output", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [] 19 | } 20 | } -------------------------------------------------------------------------------- /dataset/AzureSqlDWTable3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDWTable3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDW1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Misc" 10 | }, 11 | "type": "AzureSqlDWTable", 12 | "structure": [ 13 | { 14 | "name": "digit", 15 | "type": "Int32" 16 | } 17 | ], 18 | "typeProperties": { 19 | "tableName": "[dbo].[Cardinals]" 20 | } 21 | }, 22 | "type": "Microsoft.DataFactory/factories/datasets" 23 | } -------------------------------------------------------------------------------- /dataset/Parquet5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Parquet5", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "test.parquet", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "compressionCodec": "snappy" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/OutputContainer1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "OutputContainer1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "container": "mycontainer" 13 | }, 14 | "columnDelimiter": ",", 15 | "escapeChar": "\\", 16 | "firstRowAsHeader": true, 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable7.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable7", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "tablename": { 10 | "type": "string" 11 | } 12 | }, 13 | "annotations": [], 14 | "type": "AzureSqlTable", 15 | "schema": [], 16 | "typeProperties": { 17 | "schema": "dbo", 18 | "table": { 19 | "value": "@dataset().tablename", 20 | "type": "Expression" 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/xmlout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "xmlout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /dataset/tweets.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "tweets", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": false, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStorage3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage3", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobFS", 6 | "typeProperties": { 7 | "url": "https://makadlsg2.dfs.core.windows.net", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfOWNlOGRmMjItNWRjZC00YWRlLTg0NjItY2U4NjliYWNjOGJmIg0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/MoviesADLS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesADLS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobFS", 7 | "typeProperties": { 8 | "url": "https://makadlsg2.dfs.core.windows.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzUxM2Q4M2YyLTI2ZmYtNDg0Ny05ODE2LTRiMjhiYmZmYTc1YyINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/DestinationDataset_d2r.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DestinationDataset_d2r", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Copy" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "ParquetFormat" 15 | }, 16 | "fileName": "myout.parquet", 17 | "folderPath": "mycontainer/output/parquet" 18 | } 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/TaxiFiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiFiles", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": true, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /dataset/marktabledw.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "marktabledw", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDW1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlDWTable", 10 | "schema": [ 11 | { 12 | "name": "id", 13 | "type": "int", 14 | "precision": 10 15 | }, 16 | { 17 | "name": "val", 18 | "type": "uniqueidentifier" 19 | } 20 | ], 21 | "typeProperties": { 22 | "schema": "dbo", 23 | "table": "marktable" 24 | } 25 | } 26 | } -------------------------------------------------------------------------------- /dataset/sampledata.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sampledata", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": true, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /dataset/moviesoutputfolder_parquet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesoutputfolder_parquet", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "newmovies.parquet", 14 | "folderPath": "movies", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "compressionCodec": "snappy" 18 | }, 19 | "schema": [] 20 | } 21 | } -------------------------------------------------------------------------------- /dataset/dbSourceParams.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dbSourceParams", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "tablename": { 10 | "type": "string", 11 | "defaultValue": "DemoTable" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "AzureSqlTable", 16 | "schema": [], 17 | "typeProperties": { 18 | "tableName": { 19 | "value": "@dataset().tablename", 20 | "type": "Expression" 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/genericsources.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "genericsources", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": true, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /dataset/mytable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mytable", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [ 11 | { 12 | "name": "aaa", 13 | "type": "int", 14 | "precision": 10 15 | }, 16 | { 17 | "name": "mytime", 18 | "type": "datetime2", 19 | "scale": 7 20 | } 21 | ], 22 | "typeProperties": { 23 | "schema": "dbo", 24 | "table": "mytable1" 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /dataset/paramsfile.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paramsfile", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "params.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": "," 17 | }, 18 | "schema": [ 19 | { 20 | "type": "String" 21 | } 22 | ] 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/CADOutput1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CADOutput1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "output/parts", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": false 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/ParquetOutput.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetOutput", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/parquet", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "none" 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/TaxiDemoDayStatsSink1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiDemoDayStatsSink1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "", 13 | "container": "demo" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": true 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/dbTargetParams.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dbTargetParams", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "mytablename": { 10 | "type": "string", 11 | "defaultValue": "DemoTable" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "AzureSqlTable", 16 | "schema": [], 17 | "typeProperties": { 18 | "tableName": { 19 | "value": "@dataset().mytablename", 20 | "type": "Expression" 21 | } 22 | } 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/moviesoutnew_csv.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesoutnew_csv", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobFSLocation", 12 | "fileName": "moviesoutnew.csv", 13 | "folderPath": "movies/output", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStorage1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobFS", 7 | "typeProperties": { 8 | "url": "https://makadlsg2.dfs.core.windows.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyX2ViNDZmZmM3LWNjYzktNDE1Zi05OWQ4LTdjNGU3ZmE3OTYwMCINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /pipeline/Dedupe Pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Dedupe Pipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Deduplication Data Flow", 7 | "type": "ExecuteDataFlow", 8 | "policy": { 9 | "timeout": "7.00:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "dataflow": { 17 | "referenceName": "dedupeProb2", 18 | "type": "DataFlowReference" 19 | } 20 | } 21 | } 22 | ] 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/H_P.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "H_P", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlob", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "", 20 | "folderPath": "mycontainer/output/H_P" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/TaxiDemoVendorStatsSink1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiDemoVendorStatsSink1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "", 13 | "container": "demo" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": false 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/delimiterout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "delimiterout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/mydataflow123partfiles", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [] 21 | } 22 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStorage4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage4", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobFS", 7 | "typeProperties": { 8 | "url": "https://makadlsg2east.dfs.core.windows.net", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyX2Y3NGMwYzk5LWMzNWMtNDI3Yi1hMjU2LWU2MzI0ZWI1YzMwMiINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/CurrencyDatasetCAD1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencyDatasetCAD1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "Currency_CAD.txt", 13 | "container": "demo" 14 | }, 15 | "columnDelimiter": "\t", 16 | "firstRowAsHeader": false 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/ParameterTable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParameterTable", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "tablename": { 10 | "type": "string", 11 | "defaultValue": "DimProducts" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "AzureSqlTable", 16 | "schema": [], 17 | "typeProperties": { 18 | "schema": "dbo", 19 | "table": { 20 | "value": "@dataset().tablename", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /dataset/TaxiDemoTotalByPaymentType1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiDemoTotalByPaymentType1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "", 13 | "container": "demo" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": false 17 | }, 18 | "schema": [] 19 | }, 20 | "type": "Microsoft.DataFactory/factories/datasets" 21 | } -------------------------------------------------------------------------------- /dataset/adls_msi.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adls_msi", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage4MSI", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/perfout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "perfout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/perf", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/readskipped.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "readskipped", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "salesskip.csv", 14 | "folderPath": "SampleData", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/mynewls.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mynewls", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=123demo;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfNTgxYTE1MTMtNmUxOC00MDE4LTkzNDYtZDgwMzM5MzZjMjRkIg0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /dataset/ParameterTable1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParameterTable1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "tablename": { 10 | "type": "string", 11 | "defaultValue": "DimProducts" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "AzureSqlTable", 16 | "schema": [], 17 | "typeProperties": { 18 | "schema": "dbo", 19 | "table": { 20 | "value": "@dataset().tablename", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } -------------------------------------------------------------------------------- /dataset/adlsoutput_folder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adlsoutput_folder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/hadooplog.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hadooplog", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "log1.txt", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": "", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/others.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "others", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlob", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "", 20 | "folderPath": "mycontainer/output/others" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/mynewls1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "mynewls1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=123demo;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfOTEzODE1OTQtNDAzNC00NGE2LWJlMzctZTc5MmIyM2RjNGNjIg0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /dataset/DelimitedText2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "myparam": { 10 | "type": "String", 11 | "defaultValue": "stuff" 12 | } 13 | }, 14 | "type": "DelimitedText", 15 | "typeProperties": { 16 | "location": { 17 | "type": "AzureBlobStorageLocation" 18 | }, 19 | "columnDelimiter": "," 20 | }, 21 | "schema": [] 22 | }, 23 | "type": "Microsoft.DataFactory/factories/datasets" 24 | } -------------------------------------------------------------------------------- /dataset/adlsoutput_folder1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adlsoutput_folder1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/folderout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "folderout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/demoout1", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/moviesfolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesfolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/salesskip.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "salesskip", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "salesskip.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": "", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/BaseballLS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BaseballLS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfMzVmMjc0NTYtNDJlMS00OWY1LTliNGMtNzljOTQwMWVlYzQ0Ig0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /dataset/AzureBlob3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlob", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "loan.csv", 20 | "folderPath": "mycontainer/SampleData" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/dummysource.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dummysource", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "dummysource.txt", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/genericfolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "genericfolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/AzureDatabricks1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDatabricks1", 3 | "properties": { 4 | "annotations": [], 5 | "type": "AzureDatabricks", 6 | "typeProperties": { 7 | "domain": "https://adb-860887232373193.13.azuredatabricks.net", 8 | "authentication": "MSI", 9 | "workspaceResourceId": "/subscriptions/1e42591f-1f0c-4c5a-b7f2-a268f6105ec5/resourceGroups/keshADF_test/providers/Microsoft.Databricks/workspaces/keshWorkSpace", 10 | "newClusterNodeType": "Standard_DS3_v2", 11 | "newClusterNumOfWorker": "1", 12 | "newClusterVersion": "7.4.x-scala2.12" 13 | } 14 | } 15 | } -------------------------------------------------------------------------------- /linkedService/CosmosDb2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDb2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "CosmosDb", 7 | "typeProperties": { 8 | "connectionString": "AccountEndpoint=https://makcosmosdb001.documents.azure.com:443/;Database=ToDoList;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERjJfY2I5MzBjMmUtMDhhZS00ZWU1LWI1ZTMtYzJhZTczODZlMTJlIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /linkedService/CosmosDb3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDb3", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "CosmosDb", 7 | "typeProperties": { 8 | "connectionString": "AccountEndpoint=https://makcosmosdb001.documents.azure.com:443/;Database=Northwind;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzc1Mzk4NDk0LTNiYmItNDZiMC04YWJkLTBjOWY2MDQxZGE2ZiINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/folderout1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "folderout1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/mydataflow123partfiles", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/AzureBlobStorage2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobStorage2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkJVR0JBU0hGRUIwNV8wN2Q2MDY4Yy0zZGY1LTRhN2YtOTcyNS1lNjNkNTAyNGQyNmIiDQp9" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/AzureBlobStorage3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobStorage3", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfOTJmYzg3ODgtYWQ1NC00YzI0LThkZWQtMjFjZGM3MGQxMWY3Ig0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/CosmosDW.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDW", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "CosmosDb", 7 | "typeProperties": { 8 | "connectionString": "AccountEndpoint=https://makcosmosdb001.documents.azure.com:443/;Database=adventureworks;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzViOWFjZWZkLWU4ZWEtNDYwNS05NWQwLWNkYWY2NDJiZDU5MyINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /linkedService/CosmosDb1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDb1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "CosmosDb", 7 | "typeProperties": { 8 | "connectionString": "AccountEndpoint=https://makcosmosdb001.documents.azure.com:443/;Database=adventureworks;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERjJfNTE0ZDA5OTMtNmMyNC00MGJmLTk2YzctOTViNWY3YjZhYjJjIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/moviesDriftSource1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesDriftSource1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "SampleData/movies", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/wrangleout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "wrangleout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "wrangleout.csv", 14 | "folderPath": "output", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /linkedService/MarksBlob.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MarksBlob", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobStorage", 7 | "typeProperties": { 8 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyX2Y1ZWM4MGMwLTk4ODAtNGZiMC05NTk3LWRjMGVkOTU0NTRjYSINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/Loans2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Loans2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "loans2.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": false, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/dateRange.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dateRange", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "quoteChar": "\"" 18 | }, 19 | "schema": [ 20 | { 21 | "type": "String" 22 | }, 23 | { 24 | "type": "String" 25 | } 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/junktable.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "junktable", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [ 11 | { 12 | "name": "id", 13 | "type": "int", 14 | "precision": 10 15 | }, 16 | { 17 | "name": "genres", 18 | "type": "nvarchar" 19 | }, 20 | { 21 | "name": "name", 22 | "type": "nvarchar" 23 | } 24 | ], 25 | "typeProperties": { 26 | "schema": "dbo", 27 | "table": "junk" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /linkedService/MyAzureBlobLinkedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MyAzureBlobLinkedService", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureStorage", 6 | "typeProperties": { 7 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfZDNhMjQ5YmItMzRkZC00NTVhLTliZGQtOTMzODllYjY2YTIzIg0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/blobParamLS2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "blobParamLS2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobStorage", 7 | "typeProperties": { 8 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERjJfYTJkZDYzYzUtN2RlMy00YWJiLTk2OTYtMmQ1YTM0OThkYzNlIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /linkedService/blobparamLS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "blobparamLS", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobStorage", 7 | "typeProperties": { 8 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERjJfMjViNWMzOWYtMDFjNS00MDAyLTkxMjYtZWNhMDI1YWJhOWIyIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/MoviesD21.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesD21", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /linkedService/CosmosDb4Northwind.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDb4Northwind", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "CosmosDb", 7 | "typeProperties": { 8 | "connectionString": "AccountEndpoint=https://makcosmosdb001.documents.azure.com:443/;Database=Northwind;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzljMzkzZDY3LWIzZDctNDk0OS1iMzNiLWFmN2M0MzliY2VjMiINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/AzureDataLakeStorageFile1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorageFile1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlobFSFile", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "", 20 | "folderPath": "mycontainer/myfolder" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/folderout_noheaders.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "folderout_noheaders", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/mydataflow123partfiles", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": false, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | } 23 | } -------------------------------------------------------------------------------- /linkedService/databrickswest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "databrickswest", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureDatabricks", 7 | "typeProperties": { 8 | "domain": "https://adb-2317442744210201.1.azuredatabricks.net", 9 | "existingClusterId": "0623-065715-ages76", 10 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzZmYmEwMzliLTkxOGItNDNhZi1hNTA5LWYxMmM2MmVlMjI5OCINCn0=" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /dataset/DelimitedText3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "params.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": false 18 | }, 19 | "schema": [ 20 | { 21 | "name": "_c0", 22 | "type": "String" 23 | } 24 | ] 25 | } 26 | } -------------------------------------------------------------------------------- /dataset/TaxiSink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiSink", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "TaxiDemo" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "folderPath": "taxi-output", 17 | "fileSystem": "sample-data" 18 | }, 19 | "columnDelimiter": ",", 20 | "escapeChar": "\\", 21 | "firstRowAsHeader": true, 22 | "quoteChar": "\"" 23 | }, 24 | "schema": [] 25 | } 26 | } -------------------------------------------------------------------------------- /dataset/TaxiSink1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiSink1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "TaxiDemo" 10 | }, 11 | "annotations": [], 12 | "type": "DelimitedText", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "folderPath": "taxi-output", 17 | "fileSystem": "sample-data" 18 | }, 19 | "columnDelimiter": ",", 20 | "escapeChar": "\\", 21 | "firstRowAsHeader": true, 22 | "quoteChar": "\"" 23 | }, 24 | "schema": [] 25 | } 26 | } -------------------------------------------------------------------------------- /dataset/MoviesD2Header.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesD2Header", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": false, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/MoviesDW.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesDW", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [ 10 | { 11 | "name": "year", 12 | "type": "int", 13 | "precision": 10 14 | }, 15 | { 16 | "name": "ratingByYear", 17 | "type": "float", 18 | "precision": 15 19 | }, 20 | { 21 | "name": "yearkey", 22 | "type": "bigint", 23 | "precision": 19 24 | } 25 | ], 26 | "typeProperties": { 27 | "tableName": "[dbo].[moviesdw]" 28 | } 29 | } 30 | } -------------------------------------------------------------------------------- /dataset/movieszipout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movieszipout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "movieszip.gzip", 14 | "folderPath": "output/moviefile", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /linkedService/AzureBlobStorage1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobStorage1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureBlobStorage", 7 | "typeProperties": { 8 | "connectionString": "DefaultEndpointsProtocol=https;AccountName=makadf001;EndpointSuffix=core.windows.net;", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RORVdfYWIyZjRjZTUtYzk5Yi00ODgzLTlmMzktOWVmNTZiNDNlMzc4Ig0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/AzureDataLakeStorageFile2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorageFile2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage2", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlobFSFile", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "employee2.csv", 20 | "folderPath": "mycontainer/myfolder" 21 | } 22 | } 23 | } -------------------------------------------------------------------------------- /dataset/fixedwidth.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fixedwidth", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "fixed.txt", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": "", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "type": "String" 24 | } 25 | ] 26 | } 27 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDW2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDW2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureSqlDW", 6 | "typeProperties": { 7 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=makdw001.database.windows.net;initial catalog=DemoDW;user id=adfdemo", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfZDQ2YzNmYjgtMmQ5MC00MzhjLWJiZTYtMDRlODE3NmQwMzRmIg0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /pipeline/LoadFacts2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "LoadFacts2", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Data Flow Fact Loader", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "DF-FactLoader", 20 | "type": "DataFlowReference" 21 | } 22 | } 23 | } 24 | ], 25 | "annotations": [] 26 | } 27 | } -------------------------------------------------------------------------------- /linkedService/Snowflake1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Snowflake1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "Snowflake", 7 | "typeProperties": { 8 | "connectionString": "jdbc:snowflake://microsoft_partner.east-us-2.azure.snowflakecomputing.com/?user=MAKROMER&db=TEST_DB&warehouse=DEMO_WH&role=", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyX2NhMDE2OGYwLTRjOTctNDcxYy04ZTk5LTI3OTE4NmRlYzQ5ZiINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/MoviesD2_fixed.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesD2_fixed", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "myfile123.csv", 14 | "folderPath": "output/mydataflow123partfiles", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": "", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": false, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /dataset/folderout_copy1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "folderout_copy1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "saleemout.csv", 14 | "folderPath": "output/mydataflow123partfiles", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /linkedService/DWDimProd.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DWDimProd", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureSqlDW", 6 | "typeProperties": { 7 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=makdw001.database.windows.net;initial catalog=AdventureWorksDW;user id=kromerm", 8 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkZJTkFMREFUQUZMT1cwMDFfNTZkMmRiNjAtZTExNi00NTBjLTk3OWQtMjU2NWM5MjUzNTE3Ig0KfQ==" 9 | } 10 | } 11 | } -------------------------------------------------------------------------------- /linkedService/dwaw.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dwaw", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDW", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=makdw001.database.windows.net;initial catalog=AdventureWorksDW;user id=kromerm", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERjJfZGZjMDAxYTItMjAwNi00Y2ExLThlODYtN2IwYTk0OTE1YWMzIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDW1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDW1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDW", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=makdw001.database.windows.net;initial catalog=DemoDW;user id=adfdemo", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERlYyXzFkMTQ3ZjRmLThlZjQtNGMwYi05ZDQ2LTJkMTQwMGE0MzVhOCINCn0=" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /dataset/movie_dataflow_sink1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movie_dataflow_sink1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/moviefile", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": true, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [] 22 | }, 23 | "type": "Microsoft.DataFactory/factories/datasets" 24 | } -------------------------------------------------------------------------------- /linkedService/AzureDataLakeStorage2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureDataLakeStorage2", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "type": "AzureBlobFS", 6 | "typeProperties": { 7 | "url": "https://makadlsg2.dfs.core.windows.net", 8 | "tenant": "72f988bf-86f1-41af-91ab-2d7cd011db47", 9 | "servicePrincipalId": "2e73f317-6696-4e10-9443-7dbc77273686", 10 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIkJVR0JBU0hGRUIwNV9mMWUxMzM2Yi01MjViLTQ5MDMtYTQ1OC00ZDBjMmFlY2Q1ODIiDQp9" 11 | } 12 | } 13 | } -------------------------------------------------------------------------------- /dataset/AzureSQLIncremental_Changes.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSQLIncremental_Changes", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Incremental" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "TableName", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "SYS_CHANGE_VERSION", 19 | "type": "Int64" 20 | } 21 | ], 22 | "typeProperties": { 23 | "tableName": "[dbo].[table_store_ChangeTracking_version]" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/moviescompressed.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviescompressed", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output/moviefile", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "compressionCodec": "gzip", 18 | "compressionLevel": "Fastest", 19 | "escapeChar": "\\", 20 | "firstRowAsHeader": true, 21 | "quoteChar": "\"" 22 | }, 23 | "schema": [] 24 | } 25 | } -------------------------------------------------------------------------------- /pipeline/updateDB.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "updateDB", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "dbUpdates", 7 | "type": "ExecuteDataFlow", 8 | "policy": { 9 | "timeout": "7.00:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "dataflow": { 17 | "referenceName": "dbUpdates", 18 | "type": "DataFlowReference" 19 | }, 20 | "integrationRuntime": { 21 | "referenceName": "myLargeSparkIR", 22 | "type": "IntegrationRuntimeReference" 23 | } 24 | } 25 | } 26 | ] 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/CosmosDbSqlApiCollection4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CosmosDbSqlApiCollection4", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "CosmosDb3", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "CosmosDbSqlApiCollection", 10 | "schema": { 11 | "type": "object", 12 | "properties": { 13 | "title": { 14 | "type": "string" 15 | }, 16 | "year": { 17 | "type": "integer" 18 | }, 19 | "cast": { 20 | "type": "array" 21 | }, 22 | "genres": { 23 | "type": "array" 24 | } 25 | } 26 | }, 27 | "typeProperties": { 28 | "collectionName": "movie" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDatabase1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase1", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDatabase", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=maksqldb.database.windows.net;initial catalog=makaw;user id=kromerm", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfOGVhNGFiZGYtYWZkOC00Mzg5LWI3MDYtNmUwZTQxZmU5ZWMyIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /linkedService/AzureSqlDatabase4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlDatabase4", 3 | "type": "Microsoft.DataFactory/factories/linkedservices", 4 | "properties": { 5 | "annotations": [], 6 | "type": "AzureSqlDatabase", 7 | "typeProperties": { 8 | "connectionString": "integrated security=False;encrypt=True;connection timeout=30;data source=maksqldb.database.windows.net;initial catalog=makaw;user id=kromerm", 9 | "encryptedCredential": "ew0KICAiVmVyc2lvbiI6ICIyMDE3LTExLTMwIiwNCiAgIlByb3RlY3Rpb25Nb2RlIjogIktleSIsDQogICJTZWNyZXRDb250ZW50VHlwZSI6ICJQbGFpbnRleHQiLA0KICAiQ3JlZGVudGlhbElkIjogIk1BS0FERldFU1RfYjJmYjdlOTAtZmNjNC00MzgzLWE2N2ItYThjN2I4YTEwY2NlIg0KfQ==" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /pipeline/soccerPipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "soccerPipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "soccerETL", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 3, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "soccerETL", 20 | "type": "DataFlowReference" 21 | } 22 | } 23 | } 24 | ], 25 | "folder": { 26 | "name": "Demos" 27 | }, 28 | "annotations": [] 29 | } 30 | } -------------------------------------------------------------------------------- /dataset/DelimitedText12.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText12", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "TCISDERG001.digicelgroup.local_sandvine_billing_20190830_041000.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": "|", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": false, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [] 23 | } 24 | } -------------------------------------------------------------------------------- /pipeline/Duplicates.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Duplicates", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Dedupe", 7 | "type": "ExecuteDataFlow", 8 | "policy": { 9 | "timeout": "7.00:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "dataflow": { 17 | "referenceName": "dedupeProb2", 18 | "type": "DataFlowReference" 19 | }, 20 | "compute": { 21 | "computeType": "General", 22 | "coreCount": 8 23 | } 24 | } 25 | } 26 | ], 27 | "folder": { 28 | "name": "Demos" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/AzureSqlIncremental_Data.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlIncremental_Data", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Incremental" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "PersonID", 15 | "type": "Int32" 16 | }, 17 | { 18 | "name": "Name", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "Age", 23 | "type": "Int32" 24 | } 25 | ], 26 | "typeProperties": { 27 | "tableName": "[dbo].[data_source_table]" 28 | } 29 | }, 30 | "type": "Microsoft.DataFactory/factories/datasets" 31 | } -------------------------------------------------------------------------------- /dataset/BattingSink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BattingSink", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Baseball" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "mycontainer/output/baseball" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/CADOutput.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CADOutput", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Demos" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "mycontainer/output/currency" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/locationMap_dt.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "locationMap_dt", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "locationMap.csv", 13 | "folderPath": "SampleData/Soccer", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "Type", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "Name", 26 | "type": "String" 27 | } 28 | ] 29 | } 30 | } -------------------------------------------------------------------------------- /dataset/DelimitedText4.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText4", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "myfile": { 10 | "type": "String", 11 | "defaultValue": "Address.csv" 12 | } 13 | }, 14 | "type": "DelimitedText", 15 | "typeProperties": { 16 | "location": { 17 | "type": "AzureBlobStorageLocation", 18 | "fileName": { 19 | "value": "@dataset().myfile", 20 | "type": "Expression" 21 | }, 22 | "folderPath": "SampleData", 23 | "container": "mycontainer" 24 | }, 25 | "columnDelimiter": "," 26 | }, 27 | "schema": [] 28 | } 29 | } -------------------------------------------------------------------------------- /dataset/DelimitedText6.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText6", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "Currency_ARS.txt", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": "\t" 17 | }, 18 | "schema": [ 19 | { 20 | "type": "String" 21 | }, 22 | { 23 | "type": "String" 24 | }, 25 | { 26 | "type": "String" 27 | }, 28 | { 29 | "type": "String" 30 | } 31 | ] 32 | } 33 | } -------------------------------------------------------------------------------- /pipeline/deltapipe.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "deltapipe", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "delta", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "DeltaFlow4", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "annotations": [] 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/DestinationDataset_cce.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DestinationDataset_cce", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Copy" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "mycontainer/output/json" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/DestinationDataset_5ra.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DestinationDataset_5ra", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Copy" 10 | }, 11 | "type": "AzureBlobFSFile", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "somecontainer/markfolder" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /integrationRuntime/ssis2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ssis2", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "West US", 8 | "nodeSize": "Standard_D2_v3", 9 | "numberOfNodes": 1, 10 | "maxParallelExecutionsPerNode": 2 11 | }, 12 | "ssisProperties": { 13 | "catalogInfo": { 14 | "catalogServerEndpoint": "maksqldb.database.windows.net", 15 | "catalogAdminUserName": "kromerm", 16 | "catalogAdminPassword": { 17 | "type": "SecureString", 18 | "value": "**********" 19 | }, 20 | "catalogPricingTier": "S1" 21 | }, 22 | "edition": "Standard", 23 | "licenseType": "BasePrice" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /pipeline/Hierarchical.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hierarchical", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Hierarchical", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "Hierarchical", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "annotations": [] 30 | } 31 | } -------------------------------------------------------------------------------- /pipeline/Pipe With Vars.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Pipe With Vars", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "Set Variable1", 7 | "type": "SetVariable", 8 | "typeProperties": { 9 | "variableName": "myarray", 10 | "value": [ 11 | "abc" 12 | ] 13 | } 14 | }, 15 | { 16 | "name": "Append Variable1", 17 | "type": "AppendVariable", 18 | "typeProperties": { 19 | "variableName": "myarray", 20 | "value": "abc" 21 | } 22 | } 23 | ], 24 | "variables": { 25 | "myarray": { 26 | "type": "Array" 27 | } 28 | }, 29 | "folder": { 30 | "name": "PipelineController" 31 | } 32 | }, 33 | "type": "Microsoft.DataFactory/factories/pipelines" 34 | } -------------------------------------------------------------------------------- /pipeline/testpipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "testpipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "testdataflow", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "evolvingSchema", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "annotations": [] 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/AzureBlob1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlob", 9 | "typeProperties": { 10 | "format": { 11 | "type": "TextFormat", 12 | "columnDelimiter": ",", 13 | "rowDelimiter": "", 14 | "nullValue": "\\N", 15 | "treatEmptyAsNull": true, 16 | "skipLineCount": 0, 17 | "firstRowAsHeader": false 18 | }, 19 | "fileName": "part-00000-tid-3602434708290458144-2151ae16-5154-4f5e-b6af-9cac5957888f-407-c000.csv", 20 | "folderPath": "mycontainer/output/taxipaymentout" 21 | } 22 | }, 23 | "type": "Microsoft.DataFactory/factories/datasets" 24 | } -------------------------------------------------------------------------------- /dataset/AzureBlobBaseballOutput.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlobBaseballOutput", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Baseball" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "mycontainer/output/baseball" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/TaxiDemoVendorStatsSink.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "TaxiDemoVendorStatsSink", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Demos" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "", 23 | "folderPath": "mycontainer/output/vendorout" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /pipeline/CDM_Pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CDM_Pipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "CDM Data Flow", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "CDM_Transactions", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "annotations": [] 30 | } 31 | } -------------------------------------------------------------------------------- /pipeline/snowflake.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "snowflake", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "SnowflakeActivity", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "SnowflakeAnalytics", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | } 26 | } 27 | } 28 | ], 29 | "annotations": [] 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Misc" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "movieId", 15 | "type": "Int32" 16 | }, 17 | { 18 | "name": "title", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "genres", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "junk", 27 | "type": "String" 28 | } 29 | ], 30 | "typeProperties": { 31 | "tableName": "[dbo].[Movies2]" 32 | } 33 | }, 34 | "type": "Microsoft.DataFactory/factories/datasets" 35 | } -------------------------------------------------------------------------------- /dataset/salesdrift.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "salesdrift", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Drifts" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "encodingName": null, 19 | "treatEmptyAsNull": true, 20 | "skipLineCount": 0, 21 | "firstRowAsHeader": true 22 | }, 23 | "fileName": "", 24 | "folderPath": "mycontainer/SampleData/SchemaDrift" 25 | } 26 | }, 27 | "type": "Microsoft.DataFactory/factories/datasets" 28 | } -------------------------------------------------------------------------------- /dataset/DestinationDataset_hii.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DestinationDataset_hii", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Copy" 10 | }, 11 | "type": "AzureBlob", 12 | "typeProperties": { 13 | "format": { 14 | "type": "TextFormat", 15 | "columnDelimiter": ",", 16 | "rowDelimiter": "", 17 | "nullValue": "\\N", 18 | "treatEmptyAsNull": true, 19 | "skipLineCount": 0, 20 | "firstRowAsHeader": false 21 | }, 22 | "fileName": "_committed_2034928433163805213", 23 | "folderPath": "mycontainer/output/sample" 24 | } 25 | }, 26 | "type": "Microsoft.DataFactory/factories/datasets" 27 | } -------------------------------------------------------------------------------- /dataset/Sales3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Sales3", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "sales3.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "ID", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "Title", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "Cost", 30 | "type": "String" 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /dataset/sales1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sales1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "sales2.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "ID", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "Title", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "Cost", 30 | "type": "String" 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /dataset/sales2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sales2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "sales2.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "ID", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "Title", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "Cost", 30 | "type": "String" 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /pipeline/pipeline1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pipeline1", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "dataLakeETL", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "MyLoansDemo1", 20 | "type": "DataFlowReference" 21 | }, 22 | "compute": { 23 | "coreCount": 8, 24 | "computeType": "General" 25 | }, 26 | "traceLevel": "Fine" 27 | } 28 | } 29 | ], 30 | "annotations": [] 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/SalesUpdate.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SalesUpdate", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "sales4.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "ID", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "Title", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "Cost", 30 | "type": "String" 31 | } 32 | ] 33 | } 34 | } -------------------------------------------------------------------------------- /pipeline/Fixed Width Data Flow Pipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Fixed Width Data Flow Pipeline", 3 | "properties": { 4 | "description": "This is an example of creating a data flow to process fixed-width text files", 5 | "activities": [ 6 | { 7 | "name": "fixedwidth", 8 | "type": "ExecuteDataFlow", 9 | "dependsOn": [], 10 | "policy": { 11 | "timeout": "7.00:00:00", 12 | "retry": 0, 13 | "retryIntervalInSeconds": 30, 14 | "secureOutput": false, 15 | "secureInput": false 16 | }, 17 | "userProperties": [], 18 | "typeProperties": { 19 | "dataflow": { 20 | "referenceName": "Fixed Width", 21 | "type": "DataFlowReference" 22 | } 23 | } 24 | } 25 | ], 26 | "annotations": [] 27 | } 28 | } -------------------------------------------------------------------------------- /pipeline/summaryStats.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "summaryStats", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "ExecuteSummaryStats", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false 14 | }, 15 | "userProperties": [], 16 | "typeProperties": { 17 | "dataflow": { 18 | "referenceName": "summaryStats", 19 | "type": "DataFlowReference" 20 | }, 21 | "compute": { 22 | "coreCount": 8, 23 | "computeType": "General" 24 | } 25 | } 26 | } 27 | ], 28 | "annotations": [] 29 | }, 30 | "type": "Microsoft.DataFactory/factories/pipelines" 31 | } -------------------------------------------------------------------------------- /dataset/ParquetInput1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetInput1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "ParquetCrud" 10 | }, 11 | "annotations": [], 12 | "type": "Parquet", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "folderPath": "parquetCrud/input/tables", 17 | "fileSystem": "sample-data" 18 | }, 19 | "compressionCodec": "snappy" 20 | }, 21 | "schema": [ 22 | { 23 | "name": "PK", 24 | "type": "INT32" 25 | }, 26 | { 27 | "name": "col1", 28 | "type": "UTF8" 29 | }, 30 | { 31 | "name": "col2", 32 | "type": "UTF8" 33 | } 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /dataset/ParquetCrudOutput1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetCrudOutput1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "ParquetCrud" 10 | }, 11 | "annotations": [], 12 | "type": "Parquet", 13 | "typeProperties": { 14 | "location": { 15 | "type": "AzureBlobFSLocation", 16 | "folderPath": "parquetCrud/output", 17 | "fileSystem": "sample-data" 18 | }, 19 | "compressionCodec": "snappy" 20 | }, 21 | "schema": [ 22 | { 23 | "name": "PK", 24 | "type": "INT32" 25 | }, 26 | { 27 | "name": "col1", 28 | "type": "UTF8" 29 | }, 30 | { 31 | "name": "col2", 32 | "type": "UTF8" 33 | } 34 | ] 35 | } 36 | } -------------------------------------------------------------------------------- /dataset/dynadelim.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynadelim", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "delim": { 10 | "type": "string" 11 | } 12 | }, 13 | "annotations": [], 14 | "type": "DelimitedText", 15 | "typeProperties": { 16 | "location": { 17 | "type": "AzureBlobStorageLocation", 18 | "fileName": "moviesDB.csv", 19 | "folderPath": "SampleData", 20 | "container": "mycontainer" 21 | }, 22 | "columnDelimiter": { 23 | "value": "@dataset().delim", 24 | "type": "Expression" 25 | }, 26 | "escapeChar": "\\", 27 | "firstRowAsHeader": true, 28 | "quoteChar": "\"" 29 | }, 30 | "schema": [] 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/Facts.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Facts", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "folderPath": "", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "firstRowAsHeader": true 17 | }, 18 | "schema": [ 19 | { 20 | "name": "employeeID", 21 | "type": "String" 22 | }, 23 | { 24 | "name": "Hours", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Amount", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "ProjDate", 33 | "type": "String" 34 | } 35 | ] 36 | } 37 | } -------------------------------------------------------------------------------- /dataset/blob_params.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "blob_params", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Params" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "Prop_0", 15 | "type": "String" 16 | } 17 | ], 18 | "typeProperties": { 19 | "format": { 20 | "type": "TextFormat", 21 | "columnDelimiter": ",", 22 | "rowDelimiter": "", 23 | "nullValue": "\\N", 24 | "encodingName": null, 25 | "treatEmptyAsNull": true, 26 | "skipLineCount": 0, 27 | "firstRowAsHeader": false 28 | }, 29 | "fileName": "params-red.csv", 30 | "folderPath": "mycontainer/SampleData" 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /dataset/MoviesJSON.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesJSON", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "movies.json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | }, 18 | "schema": { 19 | "type": "object", 20 | "properties": { 21 | "title": { 22 | "type": "string" 23 | }, 24 | "year": { 25 | "type": "integer" 26 | }, 27 | "cast": { 28 | "type": "array" 29 | }, 30 | "genres": { 31 | "type": "array" 32 | } 33 | } 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /dataflow/moviesCompressed.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesCompressed", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "moviescompressed", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "folderout", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "\n\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['movietest.csv'],\n\tpartitionBy('hash', 1)) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/CurrencyDatasetUSD1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencyDatasetUSD1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "Currency_USD.txt", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": "\t", 17 | "firstRowAsHeader": false 18 | }, 19 | "schema": [ 20 | { 21 | "type": "String" 22 | }, 23 | { 24 | "type": "String" 25 | }, 26 | { 27 | "type": "String" 28 | }, 29 | { 30 | "type": "String" 31 | } 32 | ] 33 | }, 34 | "type": "Microsoft.DataFactory/factories/datasets" 35 | } -------------------------------------------------------------------------------- /dataset/JSONMoviesWithSchema.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "JSONMoviesWithSchema", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "movies.json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | }, 18 | "schema": { 19 | "type": "object", 20 | "properties": { 21 | "title": { 22 | "type": "string" 23 | }, 24 | "year": { 25 | "type": "integer" 26 | }, 27 | "cast": { 28 | "type": "array" 29 | }, 30 | "genres": { 31 | "type": "array" 32 | } 33 | } 34 | } 35 | } 36 | } -------------------------------------------------------------------------------- /dataset/moviesout.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesout", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "folderPath": "output", 14 | "fileSystem": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "type": "String" 23 | }, 24 | { 25 | "type": "String" 26 | }, 27 | { 28 | "type": "String" 29 | }, 30 | { 31 | "type": "String" 32 | }, 33 | { 34 | "type": "String" 35 | }, 36 | { 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /dataset/names.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "names", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "names2.csv", 13 | "folderPath": "SampleData", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "firstRowAsHeader": true 18 | }, 19 | "schema": [ 20 | { 21 | "name": "acctnum", 22 | "type": "String" 23 | }, 24 | { 25 | "name": "fullname", 26 | "type": "String" 27 | }, 28 | { 29 | "name": "phone", 30 | "type": "String" 31 | }, 32 | { 33 | "name": "zip", 34 | "type": "String" 35 | } 36 | ] 37 | } 38 | } -------------------------------------------------------------------------------- /pipeline/fileUpdates.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fileUpdates", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "fileUpdates", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "rowcount", 20 | "type": "DataFlowReference", 21 | "parameters": { 22 | "logger": "false()" 23 | } 24 | }, 25 | "compute": { 26 | "coreCount": 8, 27 | "computeType": "General" 28 | }, 29 | "traceLevel": "Fine" 30 | } 31 | } 32 | ], 33 | "annotations": [] 34 | } 35 | } -------------------------------------------------------------------------------- /dataset/BlobInputFolder.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BlobInputFolder", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "filename": { 10 | "type": "string", 11 | "defaultValue": "DimProducts.csv" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "DelimitedText", 16 | "typeProperties": { 17 | "location": { 18 | "type": "AzureBlobStorageLocation", 19 | "fileName": { 20 | "value": "@dataset().filename", 21 | "type": "Expression" 22 | }, 23 | "folderPath": "SampleData/AW", 24 | "container": "mycontainer" 25 | }, 26 | "columnDelimiter": "|", 27 | "encodingName": "UTF-8", 28 | "escapeChar": "\\", 29 | "quoteChar": "\"" 30 | }, 31 | "schema": [] 32 | } 33 | } -------------------------------------------------------------------------------- /dataset/BlobInputFolder1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "BlobInputFolder1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "filename": { 10 | "type": "string", 11 | "defaultValue": "DimProducts.csv" 12 | } 13 | }, 14 | "annotations": [], 15 | "type": "DelimitedText", 16 | "typeProperties": { 17 | "location": { 18 | "type": "AzureBlobStorageLocation", 19 | "fileName": { 20 | "value": "@dataset().filename", 21 | "type": "Expression" 22 | }, 23 | "folderPath": "SampleData/AW", 24 | "container": "mycontainer" 25 | }, 26 | "columnDelimiter": "|", 27 | "encodingName": "UTF-8", 28 | "escapeChar": "\\", 29 | "quoteChar": "\"" 30 | }, 31 | "schema": [] 32 | } 33 | } -------------------------------------------------------------------------------- /dataset/locationMap.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "locationMap", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureBlob", 9 | "structure": [ 10 | { 11 | "name": "Type", 12 | "type": "String" 13 | }, 14 | { 15 | "name": "Name", 16 | "type": "String" 17 | } 18 | ], 19 | "typeProperties": { 20 | "format": { 21 | "type": "TextFormat", 22 | "columnDelimiter": ",", 23 | "rowDelimiter": "", 24 | "quoteChar": "\"", 25 | "nullValue": "\\N", 26 | "encodingName": null, 27 | "treatEmptyAsNull": true, 28 | "skipLineCount": 0, 29 | "firstRowAsHeader": true 30 | }, 31 | "fileName": "locationMap.csv", 32 | "folderPath": "mycontainer/SampleData/Soccer" 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /dataset/CurrencyRates.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencyRates", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "monthly_csv.csv", 14 | "folderPath": "SampleData", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "Date", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Country", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "Value", 33 | "type": "String" 34 | } 35 | ] 36 | } 37 | } -------------------------------------------------------------------------------- /dataset/CurrencyRates1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencyRates1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "monthly_csv.csv", 14 | "folderPath": "SampleData", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "Date", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Country", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "Value", 33 | "type": "String" 34 | } 35 | ] 36 | } 37 | } -------------------------------------------------------------------------------- /dataset/Facts1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Facts1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": true, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "name": "employeeID", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Hours", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Amount", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "ProjDate", 35 | "type": "String" 36 | } 37 | ] 38 | } 39 | } -------------------------------------------------------------------------------- /dataflow/delimparam.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "delimparam", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "dynadelim", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "folderout", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['delimfinal.txt'],\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataflow/processMultipleTables.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "processMultipleTables", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "dbSourceParams", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1", 13 | "script": "source(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table') ~> source1" 14 | } 15 | ], 16 | "sinks": [ 17 | { 18 | "dataset": { 19 | "referenceName": "dbTargetParams", 20 | "type": "DatasetReference" 21 | }, 22 | "name": "sink1", 23 | "script": "source1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'table',\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false) ~> sink1" 24 | } 25 | ] 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/Excel1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Excel1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Excel", 10 | "typeProperties": { 11 | "sheetName": "currencysales1", 12 | "location": { 13 | "type": "AzureBlobStorageLocation", 14 | "fileName": "currencysales.xlsx", 15 | "folderPath": "SampleData", 16 | "container": "mycontainer" 17 | }, 18 | "firstRowAsHeader": true 19 | }, 20 | "schema": [ 21 | { 22 | "name": "SalesID", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Amounts", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Quantity", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "Country", 35 | "type": "String" 36 | } 37 | ] 38 | } 39 | } -------------------------------------------------------------------------------- /dataflow/paramWilcards.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paramWilcards", 3 | "properties": { 4 | "folder": { 5 | "name": "Params" 6 | }, 7 | "type": "MappingDataFlow", 8 | "typeProperties": { 9 | "sources": [ 10 | { 11 | "dataset": { 12 | "referenceName": "dummyContainer", 13 | "type": "DatasetReference" 14 | }, 15 | "name": "source1" 16 | } 17 | ], 18 | "sinks": [ 19 | { 20 | "dataset": { 21 | "referenceName": "folderout", 22 | "type": "DatasetReference" 23 | }, 24 | "name": "sink1" 25 | } 26 | ], 27 | "transformations": [], 28 | "script": "\nparameters{\n\tmywildpath as string\n}\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\twildcardPaths:[('DateFiles/'+$mywildpath)]) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false) ~> sink1" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/salesXLS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "salesXLS", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Excel", 10 | "typeProperties": { 11 | "sheetName": "currencysales1", 12 | "location": { 13 | "type": "AzureBlobStorageLocation", 14 | "fileName": "currencysales.xlsx", 15 | "folderPath": "SampleData", 16 | "container": "mycontainer" 17 | }, 18 | "firstRowAsHeader": true 19 | }, 20 | "schema": [ 21 | { 22 | "name": "SalesID", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Amounts", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Quantity", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "Country", 35 | "type": "String" 36 | } 37 | ] 38 | } 39 | } -------------------------------------------------------------------------------- /dataset/DelimitedText14.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText14", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "type": "String" 24 | }, 25 | { 26 | "type": "String" 27 | }, 28 | { 29 | "type": "String" 30 | }, 31 | { 32 | "type": "String" 33 | }, 34 | { 35 | "type": "String" 36 | }, 37 | { 38 | "type": "String" 39 | } 40 | ] 41 | } 42 | } -------------------------------------------------------------------------------- /dataset/MoviesWithRatings.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesWithRatings", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [ 11 | { 12 | "name": "title", 13 | "type": "nvarchar" 14 | }, 15 | { 16 | "name": "genres", 17 | "type": "nvarchar" 18 | }, 19 | { 20 | "name": "year", 21 | "type": "int", 22 | "precision": 10 23 | }, 24 | { 25 | "name": "Rating", 26 | "type": "int", 27 | "precision": 10 28 | }, 29 | { 30 | "name": "Rotton Tomato", 31 | "type": "int", 32 | "precision": 10 33 | }, 34 | { 35 | "name": "title_by_genre", 36 | "type": "nvarchar" 37 | } 38 | ], 39 | "typeProperties": { 40 | "tableName": "[dbo].[MoviesWithRatings]" 41 | } 42 | } 43 | } -------------------------------------------------------------------------------- /dataflow/skipped.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "skipped", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "readskipped", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "skiplines" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "folderout", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "\nparameters{\n\tskiplines as string\n}\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tskipLines: (toInteger($skiplines))) ~> skiplines\nskiplines sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\temptyLinesAsHeader: (toInteger($skiplines)),\n\tpartitionFileNames:['skippedrows.csv'],\n\tpartitionBy('hash', 1)) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/DelimitedText11.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DelimitedText11", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureDataLakeStorage3", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "parameters": { 9 | "filename": { 10 | "type": "string" 11 | } 12 | }, 13 | "type": "DelimitedText", 14 | "typeProperties": { 15 | "location": { 16 | "type": "AzureBlobFSLocation", 17 | "fileName": { 18 | "value": "dataset().filename", 19 | "type": "Expression" 20 | }, 21 | "folderPath": "myfolder", 22 | "container": "mycontainer" 23 | }, 24 | "columnDelimiter": ",", 25 | "escapeChar": "\\", 26 | "quoteChar": "\"" 27 | }, 28 | "schema": [ 29 | { 30 | "type": "String" 31 | }, 32 | { 33 | "type": "String" 34 | }, 35 | { 36 | "type": "String" 37 | }, 38 | { 39 | "type": "String" 40 | } 41 | ] 42 | } 43 | } -------------------------------------------------------------------------------- /dataset/moviechanges.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviechanges", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobFSLocation", 12 | "fileName": "movieupdates.csv", 13 | "folderPath": "movies", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "firstRowAsHeader": false, 19 | "quoteChar": "\"" 20 | }, 21 | "schema": [ 22 | { 23 | "type": "String" 24 | }, 25 | { 26 | "type": "String" 27 | }, 28 | { 29 | "type": "String" 30 | }, 31 | { 32 | "type": "String" 33 | }, 34 | { 35 | "type": "String" 36 | }, 37 | { 38 | "type": "String" 39 | }, 40 | { 41 | "type": "String" 42 | } 43 | ] 44 | } 45 | } -------------------------------------------------------------------------------- /dataset/CurrencySales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencySales", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "currencysales.csv", 14 | "folderPath": "SampleData", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "SalesID", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Amount", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "Quantity", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "Country", 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /dataset/CurrencySales1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencySales1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "currencysales.csv", 14 | "folderPath": "SampleData", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "SalesID", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Amount", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "Quantity", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "Country", 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /pipeline/dynamicFlows.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamicFlows", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "dataflow", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "dynamicLookup", 20 | "type": "DataFlowReference", 21 | "parameters": { 22 | "joinfield": "'movie'" 23 | }, 24 | "datasetParameters": { 25 | "dynaSource": { 26 | "tablename": "moviestage" 27 | } 28 | } 29 | }, 30 | "compute": { 31 | "coreCount": 8, 32 | "computeType": "General" 33 | } 34 | } 35 | } 36 | ], 37 | "annotations": [] 38 | } 39 | } -------------------------------------------------------------------------------- /pipeline/SearchLogAnalytics.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SearchLogAnalytics", 3 | "properties": { 4 | "description": "This is a sample that takes the U-SQL SearchLog analytics example and turns it into an ADF Data Flow: https://kromerbigdata.com/2019/03/03/u-sql-searchlog-aggregations-as-adf-data-flows/", 5 | "activities": [ 6 | { 7 | "name": "SearchLog", 8 | "type": "ExecuteDataFlow", 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "typeProperties": { 17 | "dataflow": { 18 | "referenceName": "SearchLog", 19 | "type": "DataFlowReference" 20 | }, 21 | "compute": { 22 | "computeType": "General", 23 | "dataTransformationUnits": 4, 24 | "coreCount": 8, 25 | "numberOfNodes": 0 26 | } 27 | } 28 | } 29 | ] 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/jsonblob.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jsonblob", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Misc" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "_id", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "city", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "loc", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "pop", 27 | "type": "Int64" 28 | }, 29 | { 30 | "name": "state", 31 | "type": "String" 32 | } 33 | ], 34 | "typeProperties": { 35 | "format": { 36 | "type": "JsonFormat", 37 | "filePattern": "setOfObjects" 38 | }, 39 | "fileName": "zips.json", 40 | "folderPath": "mycontainer/SampleData" 41 | } 42 | }, 43 | "type": "Microsoft.DataFactory/factories/datasets" 44 | } -------------------------------------------------------------------------------- /dataset/RetailProductsCSV.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "RetailProductsCSV", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "Products.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "id", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "productCode", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "basePrice", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "wholeSaleCost", 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /dataset/moviesADLS.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesADLS", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "movieupdates.csv", 14 | "folderPath": "movies", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": false, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "type": "String" 25 | }, 26 | { 27 | "type": "String" 28 | }, 29 | { 30 | "type": "String" 31 | }, 32 | { 33 | "type": "String" 34 | }, 35 | { 36 | "type": "String" 37 | }, 38 | { 39 | "type": "String" 40 | }, 41 | { 42 | "type": "String" 43 | } 44 | ] 45 | } 46 | } -------------------------------------------------------------------------------- /dataset/timesheetCSV.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "timesheetCSV", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "timesheet.csv", 14 | "folderPath": "SampleData/Drivers", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "driverId", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "week", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "hours-logged", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "miles-logged", 37 | "type": "String" 38 | } 39 | ] 40 | } 41 | } -------------------------------------------------------------------------------- /dataset/AzureBlob2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureBlob2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Misc" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "Prop_0", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "Prop_1", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "Prop_2", 23 | "type": "String" 24 | } 25 | ], 26 | "typeProperties": { 27 | "format": { 28 | "type": "TextFormat", 29 | "columnDelimiter": ",", 30 | "rowDelimiter": "", 31 | "nullValue": "\\N", 32 | "treatEmptyAsNull": true, 33 | "skipLineCount": 0, 34 | "firstRowAsHeader": false 35 | }, 36 | "fileName": "movies*.csv", 37 | "folderPath": "mycontainer/SampleData" 38 | } 39 | }, 40 | "type": "Microsoft.DataFactory/factories/datasets" 41 | } -------------------------------------------------------------------------------- /dataflow/sinkDemo.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sinkDemo", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "folderout", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "source(output(\n\t\tmovie as string,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as string,\n\t\tRating as string,\n\t\tRottenTomato as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\ttruncate: true,\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/SQLProducts.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SQLProducts", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Demos" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "ProductID", 15 | "type": "Int32" 16 | }, 17 | { 18 | "name": "Name", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "ProductNumber", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Color", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "StandardCost", 31 | "type": "Decimal" 32 | }, 33 | { 34 | "name": "ListPrice", 35 | "type": "Decimal" 36 | }, 37 | { 38 | "name": "Size", 39 | "type": "String" 40 | }, 41 | { 42 | "name": "Weight", 43 | "type": "Decimal" 44 | } 45 | ], 46 | "typeProperties": { 47 | "tableName": "salesprod" 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /dataset/DimEmployee.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DimEmployee", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample" 10 | }, 11 | "type": "AzureSqlTable", 12 | "structure": [ 13 | { 14 | "name": "EmpID", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "RegionCode", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "StatusCode", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "FunctionCode", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "GradeLevel", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "RoleCode", 35 | "type": "String" 36 | }, 37 | { 38 | "name": "StartDt", 39 | "type": "String" 40 | }, 41 | { 42 | "name": "EndDt", 43 | "type": "String" 44 | } 45 | ], 46 | "typeProperties": { 47 | "tableName": "DimEmployee" 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /dataset/moviesDB2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesDB2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB2.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "movies", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Title", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "genresgenregenre", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "YEAR", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "Rating", 41 | "type": "String" 42 | } 43 | ] 44 | } 45 | } -------------------------------------------------------------------------------- /integrationRuntime/integrationRuntime2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "integrationRuntime2", 3 | "properties": { 4 | "type": "Managed", 5 | "typeProperties": { 6 | "computeProperties": { 7 | "location": "West US 2", 8 | "nodeSize": "Standard_D4_v2", 9 | "numberOfNodes": 2, 10 | "maxParallelExecutionsPerNode": 1, 11 | "vNetProperties": { 12 | "vNetId": "/subscriptions/1e42591f-1f0c-4c5a-b7f2-a268f6105ec5/resourceGroups/SKResourceGroup/providers/Microsoft.Network/virtualNetworks/MyLinuxVMVNET", 13 | "subnet": "MyLinuxVMSubnet" 14 | } 15 | }, 16 | "ssisProperties": { 17 | "catalogInfo": { 18 | "catalogServerEndpoint": "makssis002.database.windows.net", 19 | "catalogAdminUserName": "kromerm", 20 | "catalogAdminPassword": { 21 | "type": "SecureString", 22 | "value": "**********" 23 | }, 24 | "catalogPricingTier": "S0" 25 | }, 26 | "edition": "Standard", 27 | "licenseType": "BasePrice" 28 | } 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/movies2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movies2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Demos" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "movieId", 15 | "type": "String" 16 | }, 17 | { 18 | "name": "title", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "genres", 23 | "type": "String" 24 | } 25 | ], 26 | "typeProperties": { 27 | "format": { 28 | "type": "TextFormat", 29 | "columnDelimiter": ",", 30 | "rowDelimiter": "", 31 | "quoteChar": "\"", 32 | "nullValue": "\\N", 33 | "encodingName": null, 34 | "treatEmptyAsNull": true, 35 | "skipLineCount": 0, 36 | "firstRowAsHeader": true 37 | }, 38 | "fileName": "movies.csv", 39 | "folderPath": "mycontainer/SampleData" 40 | } 41 | }, 42 | "type": "Microsoft.DataFactory/factories/datasets" 43 | } -------------------------------------------------------------------------------- /dataset/searchLog.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "searchLog", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "DelimitedText", 9 | "typeProperties": { 10 | "location": { 11 | "type": "AzureBlobStorageLocation", 12 | "fileName": "SearchLog.tsv", 13 | "folderPath": "SampleData/USQL", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": "\t" 17 | }, 18 | "schema": [ 19 | { 20 | "name": "_c0", 21 | "type": "String" 22 | }, 23 | { 24 | "name": "_c1", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "_c2", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "_c3", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "_c4", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "_c5", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "_c6", 45 | "type": "String" 46 | } 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /dataflow/paramQuery.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paramQuery", 3 | "properties": { 4 | "folder": { 5 | "name": "Params" 6 | }, 7 | "type": "MappingDataFlow", 8 | "typeProperties": { 9 | "sources": [ 10 | { 11 | "dataset": { 12 | "referenceName": "SQLProducts_new", 13 | "type": "DatasetReference" 14 | }, 15 | "name": "source1" 16 | } 17 | ], 18 | "sinks": [ 19 | { 20 | "dataset": { 21 | "referenceName": "folderout", 22 | "type": "DatasetReference" 23 | }, 24 | "name": "sink1" 25 | } 26 | ], 27 | "transformations": [], 28 | "script": "\nparameters{\n\tprodid as string\n}\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tisolationLevel: 'READ_UNCOMMITTED',\n\tquery: (concat('select * from saleslt.product where productid = ',$prodid)),\n\tformat: 'query') ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tfilePattern:($prodid + '.csv'),\n\tpartitionBy('hash', 1)) ~> sink1" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/ParquetMovies.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ParquetMovies", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "SampleData/movies", 14 | "container": "mycontainer" 15 | }, 16 | "compressionCodec": "snappy" 17 | }, 18 | "schema": [ 19 | { 20 | "name": "movie", 21 | "type": "INT32" 22 | }, 23 | { 24 | "name": "title", 25 | "type": "UTF8" 26 | }, 27 | { 28 | "name": "genres", 29 | "type": "UTF8" 30 | }, 31 | { 32 | "name": "year", 33 | "type": "INT32" 34 | }, 35 | { 36 | "name": "Rating", 37 | "type": "INT32" 38 | }, 39 | { 40 | "name": "RottenTomato", 41 | "type": "INT32" 42 | }, 43 | { 44 | "name": "dateOfETL", 45 | "type": "DATE" 46 | } 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /pipeline/paramQueryPipe.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "paramQueryPipe", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "paramQuery", 7 | "type": "ExecuteDataFlow", 8 | "dependsOn": [], 9 | "policy": { 10 | "timeout": "7.00:00:00", 11 | "retry": 0, 12 | "retryIntervalInSeconds": 30, 13 | "secureOutput": false, 14 | "secureInput": false 15 | }, 16 | "userProperties": [], 17 | "typeProperties": { 18 | "dataflow": { 19 | "referenceName": "paramQuery", 20 | "type": "DataFlowReference", 21 | "parameters": { 22 | "prodid": { 23 | "value": "'@{pipeline().parameters.myarray}'", 24 | "type": "Expression" 25 | } 26 | } 27 | }, 28 | "compute": { 29 | "coreCount": 8, 30 | "computeType": "General" 31 | }, 32 | "traceLevel": "Fine" 33 | } 34 | } 35 | ], 36 | "parameters": { 37 | "myarray": { 38 | "type": "array" 39 | } 40 | }, 41 | "annotations": [] 42 | } 43 | } -------------------------------------------------------------------------------- /dataset/dataflowlogger.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflowlogger", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "folderPath": "output", 14 | "container": "mycontainer" 15 | }, 16 | "columnDelimiter": ",", 17 | "escapeChar": "\\", 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "type": "String" 23 | }, 24 | { 25 | "type": "String" 26 | }, 27 | { 28 | "type": "String" 29 | }, 30 | { 31 | "type": "String" 32 | }, 33 | { 34 | "type": "String" 35 | }, 36 | { 37 | "type": "String" 38 | }, 39 | { 40 | "type": "String" 41 | }, 42 | { 43 | "type": "String" 44 | }, 45 | { 46 | "type": "String" 47 | }, 48 | { 49 | "type": "String" 50 | } 51 | ] 52 | } 53 | } -------------------------------------------------------------------------------- /dataset/Json5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Json5", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Json", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "redacted_sample.json", 14 | "folderPath": "SampleData/json", 15 | "container": "mycontainer" 16 | } 17 | }, 18 | "schema": { 19 | "type": "object", 20 | "properties": { 21 | "Animal": { 22 | "type": "string" 23 | }, 24 | "Colour": { 25 | "type": "string" 26 | }, 27 | "Age": { 28 | "type": "integer" 29 | }, 30 | "Visits": { 31 | "type": "array", 32 | "items": { 33 | "type": "object", 34 | "properties": { 35 | "Reason": { 36 | "type": "string" 37 | }, 38 | "Date": { 39 | "type": "string" 40 | } 41 | } 42 | } 43 | } 44 | } 45 | } 46 | } 47 | } -------------------------------------------------------------------------------- /dataset/moviestage2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviestage2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [ 11 | { 12 | "name": "movie", 13 | "type": "nvarchar" 14 | }, 15 | { 16 | "name": "title", 17 | "type": "nvarchar" 18 | }, 19 | { 20 | "name": "genres", 21 | "type": "nvarchar" 22 | }, 23 | { 24 | "name": "year", 25 | "type": "int", 26 | "precision": 10 27 | }, 28 | { 29 | "name": "Rating", 30 | "type": "int", 31 | "precision": 10 32 | }, 33 | { 34 | "name": "RottenTomato", 35 | "type": "int", 36 | "precision": 10 37 | }, 38 | { 39 | "name": "Rotton Tomato", 40 | "type": "nvarchar" 41 | }, 42 | { 43 | "name": "dateOfETL", 44 | "type": "date" 45 | } 46 | ], 47 | "typeProperties": { 48 | "tableName": "[dbo].[moviestage2]" 49 | } 50 | } 51 | } -------------------------------------------------------------------------------- /dataset/movieupdates_parquet.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movieupdates_parquet", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "userdata1.parquet", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "compressionCodec": "snappy" 18 | }, 19 | "schema": [ 20 | { 21 | "name": "Prop_0", 22 | "type": "UTF8" 23 | }, 24 | { 25 | "name": "Prop_1", 26 | "type": "UTF8" 27 | }, 28 | { 29 | "name": "Prop_2", 30 | "type": "UTF8" 31 | }, 32 | { 33 | "name": "Prop_3", 34 | "type": "UTF8" 35 | }, 36 | { 37 | "name": "Prop_4", 38 | "type": "UTF8" 39 | }, 40 | { 41 | "name": "Prop_5", 42 | "type": "UTF8" 43 | }, 44 | { 45 | "name": "Prop_6", 46 | "type": "UTF8" 47 | } 48 | ] 49 | } 50 | } -------------------------------------------------------------------------------- /pipeline/masterPipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "masterPipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "MyExecutePipelineActivity", 7 | "type": "ExecutePipeline", 8 | "typeProperties": { 9 | "pipeline": { 10 | "referenceName": "invokedPipeline", 11 | "type": "PipelineReference" 12 | }, 13 | "waitOnCompletion": true, 14 | "parameters": { 15 | "sourceBlobContainer": { 16 | "value": "@pipeline().parameters.masterSourceBlobContainer", 17 | "type": "Expression" 18 | }, 19 | "sinkBlobCountainer": { 20 | "value": "@pipeline().parameters.masterSinkBlobContainer", 21 | "type": "Expression" 22 | } 23 | } 24 | } 25 | } 26 | ], 27 | "parameters": { 28 | "masterSourceBlobContainer": { 29 | "type": "String" 30 | }, 31 | "masterSinkBlobContainer": { 32 | "type": "String" 33 | } 34 | }, 35 | "folder": { 36 | "name": "PipelineController" 37 | } 38 | }, 39 | "type": "Microsoft.DataFactory/factories/pipelines" 40 | } -------------------------------------------------------------------------------- /dataset/CurrencyDatasetUSD.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "CurrencyDatasetUSD", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MyAzureBlobLinkedService", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Demos" 10 | }, 11 | "type": "AzureBlob", 12 | "structure": [ 13 | { 14 | "name": "PreviousConversionRate", 15 | "type": "Double" 16 | }, 17 | { 18 | "name": "Country", 19 | "type": "String" 20 | }, 21 | { 22 | "name": "DateTime1", 23 | "type": "DateTime" 24 | }, 25 | { 26 | "name": "CurrentConversionRate", 27 | "type": "Double" 28 | } 29 | ], 30 | "typeProperties": { 31 | "format": { 32 | "type": "TextFormat", 33 | "columnDelimiter": "\t", 34 | "rowDelimiter": "", 35 | "treatEmptyAsNull": true, 36 | "skipLineCount": 0, 37 | "firstRowAsHeader": false 38 | }, 39 | "fileName": "Currency_USD.txt", 40 | "folderPath": "mycontainer/SampleData" 41 | } 42 | }, 43 | "type": "Microsoft.DataFactory/factories/datasets" 44 | } -------------------------------------------------------------------------------- /dataset/MoviesD2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "MoviesD2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "movie", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "title", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "genres", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "year", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "Rating", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "RottenTomato", 45 | "type": "String" 46 | } 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /dataflow/dynafile.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynafile", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "DelimitedText13", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "AzureSqlTable7", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "DerivedColumn1" 27 | } 28 | ], 29 | "script": "\n\nsource(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\trowUrlColumn: 'myfilename',\n\twildcardPaths:['SampleData/trip*.csv']) ~> source1\nsource1 derive(filename = trim(split(myfilename,'.')[1],'/')) ~> DerivedColumn1\nDerivedColumn1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\trecreate:true,\n\tformat: 'table') ~> sink1" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/driverscsv.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "driverscsv", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "drivers.csv", 14 | "folderPath": "SampleData/Drivers", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "driverId", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "name", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "ssn", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "location", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "certified", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "wage-plan", 45 | "type": "String" 46 | } 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /pipeline/Load Multiple Tables.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Load Multiple Tables", 3 | "properties": { 4 | "description": "Use a flexible SQL loading data flow to load many tables in a single activity", 5 | "activities": [ 6 | { 7 | "name": "flexible SQL", 8 | "type": "ExecuteDataFlow", 9 | "dependsOn": [], 10 | "policy": { 11 | "timeout": "7.00:00:00", 12 | "retry": 0, 13 | "retryIntervalInSeconds": 30, 14 | "secureOutput": false, 15 | "secureInput": false 16 | }, 17 | "userProperties": [], 18 | "typeProperties": { 19 | "dataflow": { 20 | "referenceName": "dataflowFlexibleSQL", 21 | "type": "DataFlowReference", 22 | "parameters": { 23 | "filename": "'moviesDB.csv'" 24 | }, 25 | "datasetParameters": { 26 | "sinkParams": { 27 | "mytablename": "MoviesWithRatings" 28 | } 29 | } 30 | }, 31 | "compute": { 32 | "coreCount": 32, 33 | "computeType": "MemoryOptimized" 34 | } 35 | } 36 | } 37 | ], 38 | "annotations": [] 39 | } 40 | } -------------------------------------------------------------------------------- /dataflow/denormOrders.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "denormOrders", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "dummysource", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "folderout", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "DerivedColumn1" 27 | } 28 | ], 29 | "script": "parameters{\n\tmyparam1 as timestamp\n}\nsource(output(\n\t\t{_col0_} as boolean\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false) ~> source1\nsource1 derive(param = $myparam1) ~> DerivedColumn1\nDerivedColumn1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['dummy.csv'],\n\tmapColumn(\n\t\tparam\n\t),\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/movie_dataflow_source11.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movie_dataflow_source11", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "moviesDB.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "movie", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "title", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "genres", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "year", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "Rating", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "Rotton Tomato", 45 | "type": "String" 46 | } 47 | ] 48 | } 49 | } -------------------------------------------------------------------------------- /dataflow/LoadSnowflakeMovies.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "LoadSnowflakeMovies", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "SnowflakeTable1", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "source(output(\n\t\tmovie as integer,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as short,\n\t\tRating as short,\n\t\tRottenTomato as short\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tsingleDocument: true) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\trecreate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataset/sqltablev2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sqltablev2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [ 10 | { 11 | "name": "AddressID", 12 | "type": "int", 13 | "precision": 10 14 | }, 15 | { 16 | "name": "AddressLine1", 17 | "type": "nvarchar" 18 | }, 19 | { 20 | "name": "AddressLine2", 21 | "type": "nvarchar" 22 | }, 23 | { 24 | "name": "City", 25 | "type": "nvarchar" 26 | }, 27 | { 28 | "name": "StateProvince", 29 | "type": "nvarchar" 30 | }, 31 | { 32 | "name": "CountryRegion", 33 | "type": "nvarchar" 34 | }, 35 | { 36 | "name": "PostalCode", 37 | "type": "nvarchar" 38 | }, 39 | { 40 | "name": "rowguid", 41 | "type": "uniqueidentifier" 42 | }, 43 | { 44 | "name": "ModifiedDate", 45 | "type": "datetime", 46 | "precision": 23, 47 | "scale": 3 48 | } 49 | ], 50 | "typeProperties": { 51 | "tableName": "[SalesLT].[Address]" 52 | } 53 | } 54 | } -------------------------------------------------------------------------------- /dataset/movie_dataflow_source1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "movie_dataflow_source1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "loan.csv", 14 | "folderPath": "SampleData", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "movie", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "title", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "genres", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "year", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "Rating", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "Rotton Tomato", 45 | "type": "String" 46 | } 47 | ] 48 | }, 49 | "type": "Microsoft.DataFactory/factories/datasets" 50 | } -------------------------------------------------------------------------------- /dataset/EmployeeFiles1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "EmployeeFiles1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "container": "mycontainer" 14 | }, 15 | "columnDelimiter": ",", 16 | "escapeChar": "\\", 17 | "firstRowAsHeader": true, 18 | "quoteChar": "\"" 19 | }, 20 | "schema": [ 21 | { 22 | "name": "EmpID", 23 | "type": "String" 24 | }, 25 | { 26 | "name": "Region", 27 | "type": "String" 28 | }, 29 | { 30 | "name": "Status", 31 | "type": "String" 32 | }, 33 | { 34 | "name": "Function", 35 | "type": "String" 36 | }, 37 | { 38 | "name": "Level", 39 | "type": "String" 40 | }, 41 | { 42 | "name": "Role", 43 | "type": "String" 44 | }, 45 | { 46 | "name": "StartDate", 47 | "type": "String" 48 | }, 49 | { 50 | "name": "EndDate", 51 | "type": "String" 52 | } 53 | ] 54 | } 55 | } -------------------------------------------------------------------------------- /dataflow/findNulls.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "findNulls", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "name": "sink1" 18 | } 19 | ], 20 | "transformations": [ 21 | { 22 | "name": "CreateColumnArray" 23 | }, 24 | { 25 | "name": "LookForNULLs" 26 | } 27 | ], 28 | "script": "source(output(\n\t\tmovie as string,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as string,\n\t\tRating as string,\n\t\tRottenTomato as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> source1\nsource1 derive(cols = array(columns())) ~> CreateColumnArray\nCreateColumnArray split(contains(cols,isNull(#item)),\n\tdisjoint: false) ~> LookForNULLs@(good, bad)\nLookForNULLs@good sink(skipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true,\n\tkeys:['movie'],\n\tstore: 'cache',\n\tformat: 'inline',\n\tsaveOrder: 1) ~> sink1" 29 | } 30 | } 31 | } -------------------------------------------------------------------------------- /dataset/SQLProdFromSales.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SQLProdFromSales", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "type": "AzureSqlTable", 9 | "schema": [ 10 | { 11 | "name": "ProductID", 12 | "type": "int", 13 | "precision": 10 14 | }, 15 | { 16 | "name": "Name", 17 | "type": "nvarchar" 18 | }, 19 | { 20 | "name": "ProductNumber", 21 | "type": "nvarchar" 22 | }, 23 | { 24 | "name": "Color", 25 | "type": "nvarchar" 26 | }, 27 | { 28 | "name": "StandardCost", 29 | "type": "money", 30 | "precision": 19, 31 | "scale": 4 32 | }, 33 | { 34 | "name": "ListPrice", 35 | "type": "money", 36 | "precision": 19, 37 | "scale": 4 38 | }, 39 | { 40 | "name": "_Inserted", 41 | "type": "datetime", 42 | "precision": 23, 43 | "scale": 3 44 | }, 45 | { 46 | "name": "_LastUpdated", 47 | "type": "datetime", 48 | "precision": 23, 49 | "scale": 3 50 | } 51 | ], 52 | "typeProperties": { 53 | "tableName": "[dbo].[ProductForSales]" 54 | } 55 | } 56 | } -------------------------------------------------------------------------------- /dataflow/DeltaFlow1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "DeltaFlow1", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "linkedService": { 18 | "referenceName": "AzureDataLakeStorage4", 19 | "type": "LinkedServiceReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [], 25 | "script": "source(output(\n\t\tmovie as integer,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as short,\n\t\tRating as short,\n\t\tRottenTomato as short\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionBy('hash', 10,\n\t\tmovie\n\t)) ~> source1\nsource1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tformat: 'delta',\n\tfileSystem: 'mycontainer',\n\tfolderPath: 'Delta1',\n\ttruncate: true,\n\tvacuum: 0,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 26 | } 27 | } 28 | } -------------------------------------------------------------------------------- /dataflow/WranglingDataFlow5.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "WranglingDataFlow5", 3 | "properties": { 4 | "type": "WranglingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "name": "MoviesD2", 9 | "script": "", 10 | "dataset": { 11 | "referenceName": "MoviesD2", 12 | "type": "DatasetReference" 13 | } 14 | } 15 | ], 16 | "sinks": [ 17 | { 18 | "name": "Sink1", 19 | "dataset": { 20 | "referenceName": "folderout", 21 | "type": "DatasetReference" 22 | }, 23 | "script": "sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> Sink1" 24 | } 25 | ], 26 | "script": "section Section1;\r\nshared MoviesD2 = let\r\n AdfDoc = AzureStorage.BlobContents(\"https://makadf001.blob.core.windows.net/mycontainer/SampleData/moviesDB.csv\"),\r\n Csv = Csv.Document(AdfDoc, [Delimiter = \",\", Encoding = TextEncoding.Utf8, QuoteStyle = QuoteStyle.Csv]),\r\n PromotedHeaders = Table.PromoteHeaders(Csv, [PromoteAllScalars = true])\r\nin\r\n PromotedHeaders;\r\nshared UserQuery = let\r\n Source = MoviesD2\r\nin\r\n Source;\r\n" 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /pipeline/metadatapipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "metadatapipeline", 3 | "properties": { 4 | "description": "Sample metadata checker rules", 5 | "activities": [ 6 | { 7 | "name": "metadataChecker", 8 | "type": "ExecuteDataFlow", 9 | "dependsOn": [], 10 | "policy": { 11 | "timeout": "7.00:00:00", 12 | "retry": 0, 13 | "retryIntervalInSeconds": 30, 14 | "secureOutput": false, 15 | "secureInput": false 16 | }, 17 | "userProperties": [], 18 | "typeProperties": { 19 | "dataflow": { 20 | "referenceName": "metadataChecker", 21 | "type": "DataFlowReference", 22 | "parameters": { 23 | "filename": "'SampleData/moviesDB.csv'", 24 | "columns": "'movie,title'", 25 | "dbname": "'products'" 26 | } 27 | }, 28 | "compute": { 29 | "coreCount": 8, 30 | "computeType": "General" 31 | } 32 | } 33 | } 34 | ], 35 | "parameters": { 36 | "filename": { 37 | "type": "string", 38 | "defaultValue": "'SampleData/moviesDB.csv'" 39 | } 40 | }, 41 | "variables": { 42 | "metacheck": { 43 | "type": "String" 44 | } 45 | }, 46 | "annotations": [] 47 | } 48 | } -------------------------------------------------------------------------------- /dataflow/dataflow3.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dataflow3", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "JsonOutput", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "DerivedColumn1" 27 | } 28 | ], 29 | "script": "parameters{\n\tparameter1 as string[]\n}\nsource(output(\n\t\tmovie as string,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as string,\n\t\tRating as string,\n\t\tRottenTomato as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> source1\nsource1 derive(newcol = $parameter1) ~> DerivedColumn1\nDerivedColumn1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['ptest.json'],\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/moviesoutnew_parquet2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "moviesoutnew_parquet2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "MoviesADLS", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "Parquet", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobFSLocation", 13 | "fileName": "moviesoutnew.parquet", 14 | "folderPath": "movies", 15 | "fileSystem": "mycontainer" 16 | }, 17 | "compressionCodec": "snappy" 18 | }, 19 | "schema": [ 20 | { 21 | "name": "MovieIdDupe", 22 | "type": "INT32" 23 | }, 24 | { 25 | "name": "ActionDupe", 26 | "type": "UTF8" 27 | }, 28 | { 29 | "name": "MovieId", 30 | "type": "INT32" 31 | }, 32 | { 33 | "name": "Title", 34 | "type": "UTF8" 35 | }, 36 | { 37 | "name": "Genre", 38 | "type": "UTF8" 39 | }, 40 | { 41 | "name": "Year", 42 | "type": "INT32" 43 | }, 44 | { 45 | "name": "Rating", 46 | "type": "INT32" 47 | }, 48 | { 49 | "name": "RottenTom", 50 | "type": "INT32" 51 | }, 52 | { 53 | "name": "Action", 54 | "type": "UTF8" 55 | } 56 | ] 57 | } 58 | } -------------------------------------------------------------------------------- /dataset/EmployeeDay1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "EmployeeDay1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureBlobStorageLocation", 15 | "fileName": "EmployeeDay1.csv", 16 | "folderPath": "SampleData", 17 | "container": "mycontainer" 18 | }, 19 | "columnDelimiter": ",", 20 | "firstRowAsHeader": true 21 | }, 22 | "schema": [ 23 | { 24 | "name": "EmpID", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "RegionCode", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "StatusCode", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "FunctionCode", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "GradeLevel", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "RoleCode", 45 | "type": "String" 46 | }, 47 | { 48 | "name": "StartDt", 49 | "type": "String" 50 | }, 51 | { 52 | "name": "EndDt", 53 | "type": "String" 54 | } 55 | ] 56 | } 57 | } -------------------------------------------------------------------------------- /dataset/EmployeeDay2.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "EmployeeDay2", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "folder": { 9 | "name": "Sample" 10 | }, 11 | "type": "DelimitedText", 12 | "typeProperties": { 13 | "location": { 14 | "type": "AzureBlobStorageLocation", 15 | "fileName": "EmployeeDay2.csv", 16 | "folderPath": "SampleData", 17 | "container": "mycontainer" 18 | }, 19 | "columnDelimiter": ",", 20 | "firstRowAsHeader": true 21 | }, 22 | "schema": [ 23 | { 24 | "name": "EmpID", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "RegionCode", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "StatusCode", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "FunctionCode", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "GradeLevel", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "RoleCode", 45 | "type": "String" 46 | }, 47 | { 48 | "name": "StartDt", 49 | "type": "String" 50 | }, 51 | { 52 | "name": "EndDt", 53 | "type": "String" 54 | } 55 | ] 56 | } 57 | } -------------------------------------------------------------------------------- /pipeline/invokedPipeline.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "invokedPipeline", 3 | "properties": { 4 | "activities": [ 5 | { 6 | "name": "CopyBlobtoBlob", 7 | "type": "Copy", 8 | "policy": { 9 | "timeout": "7.00:00:00", 10 | "retry": 0, 11 | "retryIntervalInSeconds": 30, 12 | "secureOutput": false, 13 | "secureInput": false 14 | }, 15 | "typeProperties": { 16 | "source": { 17 | "type": "BlobSource", 18 | "recursive": true 19 | }, 20 | "sink": { 21 | "type": "BlobSink" 22 | }, 23 | "enableStaging": false, 24 | "dataIntegrationUnits": 0 25 | }, 26 | "inputs": [ 27 | { 28 | "referenceName": "AzureBlob1", 29 | "type": "DatasetReference" 30 | } 31 | ], 32 | "outputs": [ 33 | { 34 | "referenceName": "CADOutput", 35 | "type": "DatasetReference" 36 | } 37 | ] 38 | } 39 | ], 40 | "parameters": { 41 | "sourceBlobContainer": { 42 | "type": "String" 43 | }, 44 | "sinkBlobContainer": { 45 | "type": "String" 46 | } 47 | }, 48 | "folder": { 49 | "name": "PipelineController" 50 | } 51 | }, 52 | "type": "Microsoft.DataFactory/factories/pipelines" 53 | } -------------------------------------------------------------------------------- /dataflow/Hierarchical.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Hierarchical", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "Orc1", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "R2H" 27 | } 28 | ], 29 | "script": "source(output(\n\t\tmovie as integer,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as short,\n\t\tRating as short,\n\t\tRottenTomato as short\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tignoreNoFilesFound: false) ~> source1\nsource1 derive(movies = @(title=title,\n\t\tgenres=split(genres,'|'),\n\t\tyear=year,\n\t\trating=Rating)) ~> R2H\nR2H sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionFileNames:['movies.orc'],\n\tmapColumn(\n\t\tmovies\n\t),\n\tpartitionBy('hash', 1),\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 30 | } 31 | } 32 | } -------------------------------------------------------------------------------- /dataset/AzureSqlTable1.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "AzureSqlTable1", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureSqlDatabase1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "AzureSqlTable", 10 | "schema": [ 11 | { 12 | "name": "AddressID", 13 | "type": "int", 14 | "precision": 10 15 | }, 16 | { 17 | "name": "AddressLine1", 18 | "type": "nvarchar" 19 | }, 20 | { 21 | "name": "AddressLine2", 22 | "type": "nvarchar" 23 | }, 24 | { 25 | "name": "City", 26 | "type": "nvarchar" 27 | }, 28 | { 29 | "name": "StateProvince", 30 | "type": "nvarchar" 31 | }, 32 | { 33 | "name": "CountryRegion", 34 | "type": "nvarchar" 35 | }, 36 | { 37 | "name": "PostalCode", 38 | "type": "nvarchar" 39 | }, 40 | { 41 | "name": "rowguid", 42 | "type": "uniqueidentifier" 43 | }, 44 | { 45 | "name": "ModifiedDate", 46 | "type": "datetime", 47 | "precision": 23, 48 | "scale": 23 49 | } 50 | ], 51 | "typeProperties": { 52 | "tableName": "[SalesLT].[SalesOrderDetail]" 53 | } 54 | }, 55 | "type": "Microsoft.DataFactory/factories/datasets" 56 | } -------------------------------------------------------------------------------- /dataset/EmployeeFiles.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "EmployeeFiles", 3 | "properties": { 4 | "linkedServiceName": { 5 | "referenceName": "AzureBlobStorage1", 6 | "type": "LinkedServiceReference" 7 | }, 8 | "annotations": [], 9 | "type": "DelimitedText", 10 | "typeProperties": { 11 | "location": { 12 | "type": "AzureBlobStorageLocation", 13 | "fileName": "EmployeeDay1.csv", 14 | "folderPath": "SampleData/Emps", 15 | "container": "mycontainer" 16 | }, 17 | "columnDelimiter": ",", 18 | "escapeChar": "\\", 19 | "firstRowAsHeader": true, 20 | "quoteChar": "\"" 21 | }, 22 | "schema": [ 23 | { 24 | "name": "EmpID", 25 | "type": "String" 26 | }, 27 | { 28 | "name": "Region", 29 | "type": "String" 30 | }, 31 | { 32 | "name": "Status", 33 | "type": "String" 34 | }, 35 | { 36 | "name": "Function", 37 | "type": "String" 38 | }, 39 | { 40 | "name": "Level", 41 | "type": "String" 42 | }, 43 | { 44 | "name": "Role", 45 | "type": "String" 46 | }, 47 | { 48 | "name": "StartDate", 49 | "type": "String" 50 | }, 51 | { 52 | "name": "EndDate", 53 | "type": "String" 54 | } 55 | ] 56 | } 57 | } -------------------------------------------------------------------------------- /dataflow/SQLPerf.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "SQLPerf", 3 | "properties": { 4 | "type": "MappingDataFlow", 5 | "typeProperties": { 6 | "sources": [ 7 | { 8 | "dataset": { 9 | "referenceName": "MoviesD2", 10 | "type": "DatasetReference" 11 | }, 12 | "name": "source1" 13 | } 14 | ], 15 | "sinks": [ 16 | { 17 | "dataset": { 18 | "referenceName": "MoviesSQLDB", 19 | "type": "DatasetReference" 20 | }, 21 | "name": "sink1" 22 | } 23 | ], 24 | "transformations": [ 25 | { 26 | "name": "SurrogateKey1" 27 | } 28 | ], 29 | "script": "source(output(\n\t\tmovie as string,\n\t\ttitle as string,\n\t\tgenres as string,\n\t\tyear as string,\n\t\tRating as string,\n\t\tRottenTomato as string\n\t),\n\tallowSchemaDrift: true,\n\tvalidateSchema: false,\n\tpartitionBy('roundRobin', 5)) ~> source1\nsource1 keyGenerate(output(mysk as long),\n\tstartAt: 1L) ~> SurrogateKey1\nSurrogateKey1 sink(allowSchemaDrift: true,\n\tvalidateSchema: false,\n\tdeletable:false,\n\tinsertable:true,\n\tupdateable:false,\n\tupsertable:false,\n\trecreate:true,\n\tformat: 'table',\n\tskipDuplicateMapInputs: true,\n\tskipDuplicateMapOutputs: true) ~> sink1" 30 | } 31 | } 32 | } --------------------------------------------------------------------------------