├── .github ├── ISSUE_TEMPLATE │ ├── data-accelerator-bug-report.md │ └── data-accelerator-feature-request.md ├── dependabot.yml └── workflows │ └── DataXSparkValidation.yml ├── .gitignore ├── .vscode └── settings.json ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DataProcessing ├── .gitignore ├── CONTRIBUTING.md ├── DataX.Utilities │ ├── .gitignore │ ├── DataX.Utilities.sln │ └── DataX.Utility.CodeSign │ │ ├── DataX.Utility.CodeSign.csproj │ │ └── FinalPublicKey.snk ├── README.md ├── Spark.nuspec ├── datax-core │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── datax │ │ ├── authentication │ │ ├── HttpGetter.scala │ │ └── ManagedIdentity.scala │ │ ├── config │ │ ├── DefaultValue.scala │ │ ├── SettingDictionary.scala │ │ ├── SettingNamespace.scala │ │ ├── SparkEnvVariables.scala │ │ └── UnifiedConfig.scala │ │ ├── constants │ │ ├── BlobProperties.scala │ │ ├── ColumnName.scala │ │ ├── DatasetName.scala │ │ ├── FeatureName.scala │ │ ├── JobArgument.scala │ │ ├── MetricName.scala │ │ ├── NamePrefix.scala │ │ ├── ProcessingPropertyName.scala │ │ └── ProductConstant.scala │ │ ├── exception │ │ └── EngineException.scala │ │ ├── extension │ │ ├── DynamicUDF.scala │ │ ├── PreProjectionProcessor.scala │ │ └── StringNormalizer.scala │ │ ├── host │ │ └── AppHost.scala │ │ ├── service │ │ ├── ConfigService.scala │ │ └── TelemetryService.scala │ │ └── sink │ │ └── package.scala ├── datax-host │ ├── pom.xml │ ├── src │ │ ├── main │ │ │ └── scala │ │ │ │ └── datax │ │ │ │ ├── app │ │ │ │ ├── BatchApp.scala │ │ │ │ ├── BlobStreamingApp.scala │ │ │ │ ├── DirectKafkaStreamingApp.scala │ │ │ │ ├── DirectLocalStreamingApp.scala │ │ │ │ ├── DirectStreamingApp.scala │ │ │ │ ├── KafkaProducer.scala │ │ │ │ └── LocalBatchApp.scala │ │ │ │ ├── checkpoint │ │ │ │ └── EventhubCheckpointer.scala │ │ │ │ ├── classloader │ │ │ │ └── ClassLoaderHost.scala │ │ │ │ ├── client │ │ │ │ ├── cosmosdb │ │ │ │ │ └── CosmosDBBase.scala │ │ │ │ ├── eventhub │ │ │ │ │ ├── EventHubBase.scala │ │ │ │ │ ├── EventHubConf.scala │ │ │ │ │ └── EventHubSender.scala │ │ │ │ ├── http │ │ │ │ │ └── HttpPostSender.scala │ │ │ │ ├── redis │ │ │ │ │ ├── RedisBase.scala │ │ │ │ │ └── RedisConnection.scala │ │ │ │ └── sql │ │ │ │ │ └── SqlConf.scala │ │ │ │ ├── config │ │ │ │ └── ConfigManager.scala │ │ │ │ ├── data │ │ │ │ ├── FileInternal.scala │ │ │ │ └── ProcessResult.scala │ │ │ │ ├── executor │ │ │ │ └── ExecutorHelper.scala │ │ │ │ ├── fs │ │ │ │ └── HadoopClient.scala │ │ │ │ ├── handler │ │ │ │ ├── AzureFunctionHandler.scala │ │ │ │ ├── BuiltInFunctionsHandler.scala │ │ │ │ ├── ExtendedUDFHandler.scala │ │ │ │ ├── InputNormalizerHandler.scala │ │ │ │ ├── JarUDFHandler.scala │ │ │ │ ├── MetricsHandler.scala │ │ │ │ ├── PreProjectionHandler.scala │ │ │ │ ├── ProjectionHandler.scala │ │ │ │ ├── PropertiesHandler.scala │ │ │ │ ├── ReferenceDataHandler.scala │ │ │ │ ├── StateTableHandler.scala │ │ │ │ ├── TimeWindowHandler.scala │ │ │ │ └── TransformHandler.scala │ │ │ │ ├── host │ │ │ │ ├── BlobBatchingHost.scala │ │ │ │ ├── CommonAppHost.scala │ │ │ │ ├── Logger.scala │ │ │ │ ├── SparkJarLoader.scala │ │ │ │ ├── SparkSessionSingleton.scala │ │ │ │ ├── StreamingHost.scala │ │ │ │ └── UdfInitializer.scala │ │ │ │ ├── input │ │ │ │ ├── BatchBlobInputSetting.scala │ │ │ │ ├── BlobPointerInput.scala │ │ │ │ ├── BlobPointerInputSetting.scala │ │ │ │ ├── EventHubInputSetting.scala │ │ │ │ ├── EventHubStreamingFactory.scala │ │ │ │ ├── InputManager.scala │ │ │ │ ├── InputSetting.scala │ │ │ │ ├── KafkaInputSetting.scala │ │ │ │ ├── KafkaStreamingFactory.scala │ │ │ │ ├── LocalInputSetting.scala │ │ │ │ ├── LocalStreamingFactory.scala │ │ │ │ ├── LocalStreamingSource.scala │ │ │ │ ├── RouterSetting.scala │ │ │ │ ├── SchemaFile.scala │ │ │ │ ├── StreamingFactory.scala │ │ │ │ └── StreamingInputSetting.scala │ │ │ │ ├── processor │ │ │ │ ├── BatchBlobProcessor.scala │ │ │ │ ├── BlobPointerProcessor.scala │ │ │ │ ├── CommonProcessor.scala │ │ │ │ ├── CommonProcessorFactory.scala │ │ │ │ ├── DirectKafkaProcessor.scala │ │ │ │ ├── DirectLocalProcessor.scala │ │ │ │ ├── DirectProcessor.scala │ │ │ │ ├── EventHubStructuredStreamingProcessor.scala │ │ │ │ ├── JsonProcessor.scala │ │ │ │ ├── StreamingProcessor.scala │ │ │ │ └── StructuredStreamingProcessor.scala │ │ │ │ ├── securedsetting │ │ │ │ └── KeyVaultClient.scala │ │ │ │ ├── sink │ │ │ │ ├── BlobOutputSetting.scala │ │ │ │ ├── BlobSinker.scala │ │ │ │ ├── CosmosDBOutputSetting.scala │ │ │ │ ├── CosmosDBSinker.scala │ │ │ │ ├── EventHubOutputSetting.scala │ │ │ │ ├── EventHubStreamPoster.scala │ │ │ │ ├── HttpPostOutputSetting.scala │ │ │ │ ├── HttpPoster.scala │ │ │ │ ├── OutputManager.scala │ │ │ │ ├── OutputSetting.scala │ │ │ │ ├── SqlOutputSetting.scala │ │ │ │ └── SqlSinker.scala │ │ │ │ ├── sql │ │ │ │ └── TransformSqlParser.scala │ │ │ │ └── telemetry │ │ │ │ ├── AppInsightLogger.scala │ │ │ │ └── MetricLogger.scala │ │ └── test │ │ │ └── scala │ │ │ └── datax │ │ │ └── test │ │ │ ├── app │ │ │ └── BatchAppTests.scala │ │ │ ├── classloader │ │ │ └── ClassLoaderTests.scala │ │ │ ├── config │ │ │ └── ConfigManagerTests.scala │ │ │ ├── fs │ │ │ └── HadoopClientTests.scala │ │ │ ├── host │ │ │ └── BlobBatchingHost.scala │ │ │ ├── input │ │ │ └── BlobPointerInputTest.scala │ │ │ ├── sink │ │ │ └── RedisTests.scala │ │ │ └── sql │ │ │ └── TransformSQLParserTests.scala │ └── with-dependencies.xml ├── datax-keyvault │ ├── .gitignore │ ├── META-INF │ │ └── MANIFEST.MF │ ├── pom.xml │ ├── src │ │ └── main │ │ │ └── scala │ │ │ └── datax │ │ │ └── keyvault │ │ │ └── KeyVaultMsiAuthenticatorClient.scala │ └── with-dependencies.xml ├── datax-udf-samples │ ├── pom.xml │ └── src │ │ └── main │ │ └── scala │ │ └── datax │ │ └── sample │ │ ├── dynamicudf │ │ └── DynamicUdfHelloWorld.scala │ │ ├── normalizer │ │ └── RemoveInvalidChars.scala │ │ ├── udaf │ │ └── UdafLastThreshold.scala │ │ └── udf │ │ └── UdfHelloWorld.scala ├── datax-utility │ ├── pom.xml │ └── src │ │ ├── main │ │ └── scala │ │ │ └── datax │ │ │ └── utility │ │ │ ├── ArgumentsParser.scala │ │ │ ├── AzureFunctionCaller.scala │ │ │ ├── CSVUtil.scala │ │ │ ├── ConcurrentDateFormat.scala │ │ │ ├── ConverterUtil.scala │ │ │ ├── DataGenerator.scala │ │ │ ├── DataMerger.scala │ │ │ ├── DataNormalization.scala │ │ │ ├── DateTimeUtil.scala │ │ │ ├── FutureUtil.scala │ │ │ ├── GZipHelper.scala │ │ │ ├── GuidUtil.scala │ │ │ ├── MapManipulation.scala │ │ │ ├── SinkerUtil.scala │ │ │ ├── StreamingUtility.scala │ │ │ └── Validation.scala │ │ └── test │ │ └── scala │ │ └── datax │ │ └── utility │ │ └── test │ │ ├── ConcurrentDateFormatTests.scala │ │ ├── DataGeneratorTests.scala │ │ └── MapManipulationTests.scala └── deploy.cmd ├── DeploymentCloud ├── .gitignore ├── Deployment.Common │ ├── Admin │ │ └── adminsteps.parameters.txt │ ├── CosmosDB │ │ ├── commons.json │ │ ├── configgenConfigs.json │ │ ├── flows.json │ │ ├── sparkClusters.json │ │ └── sparkJobs.json │ ├── Helpers │ │ └── UtilityModule.psm1 │ ├── Nuget │ │ └── NuGet.exe │ ├── Resources │ │ ├── Parameters │ │ │ ├── Databricks-Parameter.json │ │ │ ├── Resource-Parameter.json │ │ │ ├── SF-Parameter.json │ │ │ ├── Spark-AutoScale-Parameter.json │ │ │ └── Spark-Parameter.json │ │ └── Templates │ │ │ ├── Databricks-Template.json │ │ │ ├── Resource-Template.json │ │ │ ├── SF-Template.json │ │ │ ├── Spark-AutoScale-Template.json │ │ │ └── Spark-Template.json │ ├── deployResources.ps1 │ └── scripts │ │ ├── msalmsiserver.py │ │ ├── msalmsiserverapp.service │ │ ├── msiserver.py │ │ ├── msiserverapp.service │ │ ├── startmsalmsiserverservice.sh │ │ └── startmsiserverservice.sh ├── Deployment.DataX │ ├── Resources │ │ ├── Parameters │ │ │ ├── IotHub-Parameter.json │ │ │ ├── Kafka-Parameter.json │ │ │ └── SampleOutputs-Parameter.json │ │ └── Templates │ │ │ ├── IotHub-Template.json │ │ │ ├── Kafka-Template.json │ │ │ └── SampleOutputs-Template.json │ ├── Samples │ │ ├── flows │ │ │ ├── eventhub-product.json │ │ │ ├── eventhubbatch-product.json │ │ │ ├── eventhubkafka-product.json │ │ │ ├── iotsample-product.json │ │ │ └── nativekafka-product.json │ │ ├── samples │ │ │ ├── eventHubBatch │ │ │ │ └── eventhubbatch-sample.json │ │ │ ├── eventhub-4C530FCD6194E97955B37A448BAAA78C.json │ │ │ ├── eventhubbatch-F6968C9911243571CD556B0650EA5B19.json │ │ │ ├── eventhubkafka-AF621A4B9DE903CF6518435E867C858A.json │ │ │ ├── iotDevice │ │ │ │ └── iotsample.json │ │ │ ├── iotsample-091610A6B869B8C9318E988F390F46AB.json │ │ │ └── nativekafka-A219AF29664A4375E894A779DD5DCEB7.json │ │ └── usercontent │ │ │ ├── devices.csv │ │ │ ├── queryupdatesample.sql │ │ │ ├── readme.txt │ │ │ └── udfsample.jar │ ├── Services │ │ ├── Parameters │ │ │ ├── DataXFlow.Parameters.json │ │ │ ├── DataXGateway.Parameters.json │ │ │ ├── DataXMetrics.Parameters.json │ │ │ └── DataXSimulatedData.Parameters.json │ │ └── Templates │ │ │ ├── DataXFlow.Template.json │ │ │ ├── DataXGateway.Template.json │ │ │ ├── DataXMetrics.Template.json │ │ │ └── DataXSimulatedData.Template.json │ ├── adminSteps.ps1 │ ├── common.parameters.txt │ ├── deploy.bat │ ├── deployApps.ps1 │ ├── deploySample.ps1 │ └── runAdminSteps.bat ├── Deployment.JobRunner │ ├── README.md │ ├── Resources │ │ ├── Parameters │ │ │ └── jobRunner-Parameter.json │ │ └── Templates │ │ │ └── jobRunner-Template.json │ ├── ScenarioTestDatabricks.json │ ├── ScenarioTestHDInsights.json │ ├── deployResources.ps1 │ └── utilities.psm1 ├── Deployment.Kubernetes │ ├── Dockerfile │ ├── Pod-Identity-Service │ │ ├── deployment.yaml │ │ ├── install-pod-identity-service.sh │ │ ├── managed-service-identity-binding.yaml │ │ ├── managed-service-identity.yaml │ │ ├── reinstall-pod-identity-service.sh │ │ └── uninstall-pod-identity-service.sh │ ├── finalrun.sh │ ├── flow-interactive-query-service.yaml │ ├── flow-live-data-service.yaml │ ├── flow-management-service.yaml │ └── flow-schema-inference-service.yaml ├── Deployment.Tests │ ├── Tests │ │ └── ResourceNameTests.ps1 │ └── runTests.bat └── README.md ├── DeploymentLocal ├── Dockerfile ├── README.md ├── finalrun.sh └── sample │ ├── BasicLocal.json │ └── HomeAutomationLocal.json ├── Docs ├── Data Accelerator Introductory Whitepaper.pdf └── Data Accelerator for Apache Spark.pdf ├── LICENSE ├── README.md ├── SECURITY.md ├── Services ├── .editorconfig ├── .gitignore ├── AspnetCore │ └── DataX.FlowManagement │ │ ├── DataX.FlowManagement.sln │ │ └── DataX.FlowManagement │ │ ├── Controllers │ │ └── FlowManagementControllerDelete.cs │ │ ├── DataX.FlowManagement.csproj │ │ ├── DataXSettings.cs │ │ ├── FinalPublicKey.snk │ │ ├── Program.cs │ │ ├── Properties │ │ └── launchSettings.json │ │ ├── Startup.cs │ │ ├── appsettings.Development.json │ │ └── appsettings.json ├── CONTRIBUTING.md ├── DataX.Config │ ├── .editorconfig │ ├── CommonData.Templates │ │ ├── defaultFlowConfig.json │ │ ├── defaultSparkJob.json │ │ └── flattenerConfig.json │ ├── DataX.Config.CICDTests.playlist │ ├── DataX.Config.ConfigurationProviders │ │ ├── CosmosDbConfigurationProvider.cs │ │ ├── DataX.Config.ConfigurationProviders.csproj │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Config.DatabricksClient.Test │ │ ├── DataX.Config.DatabricksClient.Test.csproj │ │ └── DatabricksClientTest.cs │ ├── DataX.Config.DatabricksClient │ │ ├── ConnectionStringParser.cs │ │ ├── DataX.Config.DatabricksClient.csproj │ │ ├── DatabricksClient.cs │ │ ├── DatabricksClientConnectionInfo.cs │ │ ├── DatabricksClientFactory.cs │ │ ├── DatabricksHttpClient.cs │ │ ├── DatabricksHttpClientFactory.cs │ │ ├── DatabricksHttpResult.cs │ │ ├── DatabricksJobResult.cs │ │ ├── FinalPublicKey.snk │ │ ├── HttpMethod.cs │ │ ├── IDatabricksHttpClient.cs │ │ └── IDatabricksHttpClientFactory.cs │ ├── DataX.Config.Input.EventHub │ │ ├── DataX.Config.Input.EventHub.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ └── Processor │ │ │ └── CreateEventHubConsumerGroup.cs │ ├── DataX.Config.KeyVault │ │ ├── DataX.Config.KeyVault.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── HashGenerator.cs │ │ └── KeyVaultClient.cs │ ├── DataX.Config.LivyClient.Test │ │ ├── DataX.Config.LivyClient.Test.csproj │ │ ├── JobOperationTest.cs │ │ └── Mock │ │ │ ├── HttpClient.cs │ │ │ └── HttpClientFactory.cs │ ├── DataX.Config.LivyClient │ │ ├── ConnectionStringParser.cs │ │ ├── DataX.Config.LivyClient.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── HttpMethod.cs │ │ ├── ILivyHttpClient.cs │ │ ├── ILivyHttpClientFactory.cs │ │ ├── LivyBatchResult.cs │ │ ├── LivyBatchesResult.cs │ │ ├── LivyClient.cs │ │ ├── LivyClientConnectionInfo.cs │ │ ├── LivyClientFactory.cs │ │ ├── LivyHttpClient.cs │ │ ├── LivyHttpClientFactory.cs │ │ └── LivyHttpResult.cs │ ├── DataX.Config.PrimaryTests.playlist │ ├── DataX.Config.Storage │ │ ├── AzureBlobConfigStorage.cs │ │ ├── CosmosDBConfigStorage.cs │ │ ├── DataX.Config.Storage.csproj │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Config.Test.Utility │ │ ├── DataX.Config.Test.Utility.csproj │ │ ├── FinalPublicKey.snk │ │ └── Mock │ │ │ ├── CacheLogger.cs │ │ │ └── MockBase.cs │ ├── DataX.Config.Test │ │ ├── CompositionTest.cs │ │ ├── ConfigHelperTest.cs │ │ ├── CustomRuntimeConfigGenerationTest.cs │ │ ├── DataX.Config.Test.csproj │ │ ├── DatabricksRuntimeConfigGenerationTest.cs │ │ ├── Extension │ │ │ └── CommonDataExtension.cs │ │ ├── FinalPublicKey.snk │ │ ├── FlattenerTest.cs │ │ ├── FlowOperationTest.cs │ │ ├── GlobalSuppressions.cs │ │ ├── JsonConverterTest.cs │ │ ├── LoggerFactory.cs │ │ ├── LoggerTest.cs │ │ ├── Mock │ │ │ ├── ConfigurationProvider.cs │ │ │ ├── DesignTimeStorage.cs │ │ │ ├── EventHubInputConfigGenProcessor.cs │ │ │ ├── KeyVaultClient.cs │ │ │ ├── RuntimeStorage.cs │ │ │ └── SparkJobClientFactory.cs │ │ ├── Resource │ │ │ ├── Flattener │ │ │ │ ├── config.json │ │ │ │ ├── input.json │ │ │ │ └── output.conf │ │ │ ├── batchFlow.json │ │ │ ├── batchFlowDefault.json │ │ │ ├── batchJobConfig.conf │ │ │ ├── batchSparkJobTemplate.json │ │ │ ├── configgentest-combined.txt │ │ │ ├── customFlattenerConfig.json │ │ │ ├── customFlow.json │ │ │ ├── customJobConfig1.conf │ │ │ ├── customJobConfig2.conf │ │ │ ├── databricksFlowSaved.json │ │ │ ├── databricksFlowStarted.json │ │ │ ├── databricksJobConfig.conf │ │ │ ├── databricksJobConfig.json │ │ │ ├── dbconfiggentest-combined.txt │ │ │ ├── flattenerConfig.json │ │ │ ├── flowDefault.json │ │ │ ├── flowSaved.json │ │ │ ├── flowStarted.json │ │ │ ├── guiInput.json │ │ │ ├── jobConfig.conf │ │ │ ├── jobConfig.json │ │ │ ├── jsonTest.json │ │ │ ├── projection.txt │ │ │ ├── schema.json │ │ │ └── sparkJobTemplate.json │ │ ├── RuleDefinitionGeneratorTest.cs │ │ ├── RuntimeConfigGenerationBatchTest.cs │ │ ├── RuntimeConfigGenerationTest.cs │ │ ├── SparkJobOperationTest.cs │ │ └── TokenReplacementTest.cs │ ├── DataX.Config.sln │ ├── DataX.Config │ │ ├── Client │ │ │ ├── IKeyVaultClient.cs │ │ │ ├── ISparkJobClient.cs │ │ │ ├── ISparkJobClientFactory.cs │ │ │ └── SparkJobSyncResult.cs │ │ ├── ConfigDataModel │ │ │ ├── CommonData │ │ │ │ └── CommonDataConfig.cs │ │ │ ├── Constants.cs │ │ │ ├── EntityConfig.cs │ │ │ ├── Flattener │ │ │ │ ├── FlattenerConfig.cs │ │ │ │ ├── FlattenerMappingArray.cs │ │ │ │ ├── FlattenerMappingDictionary.cs │ │ │ │ ├── FlattenerMappingExcludeDefaultValue.cs │ │ │ │ ├── FlattenerMappingObject.cs │ │ │ │ ├── FlattenerMappingProps.cs │ │ │ │ ├── FlattenerMappingScopedObject.cs │ │ │ │ └── FlattenerMappingStringList.cs │ │ │ ├── Flow │ │ │ │ ├── FlowCommonProcessor.cs │ │ │ │ ├── FlowConfig.cs │ │ │ │ └── JobMetadata.cs │ │ │ ├── FlowGui │ │ │ │ ├── FlowGuiConfig.cs │ │ │ │ ├── FlowGuiFunction.cs │ │ │ │ ├── FlowGuiFunctionProperties.cs │ │ │ │ ├── FlowGuiInput.cs │ │ │ │ ├── FlowGuiInputBatchInput.cs │ │ │ │ ├── FlowGuiInputBatchInputProperties.cs │ │ │ │ ├── FlowGuiInputBatchJob.cs │ │ │ │ ├── FlowGuiInputBatchJobProperties.cs │ │ │ │ ├── FlowGuiInputProperties.cs │ │ │ │ ├── FlowGuiJobConfig.cs │ │ │ │ ├── FlowGuiOutput.cs │ │ │ │ ├── FlowGuiOutputProperties.cs │ │ │ │ ├── FlowGuiOutputTemplate.cs │ │ │ │ ├── FlowGuiProcess.cs │ │ │ │ ├── FlowGuiReferenceData.cs │ │ │ │ ├── FlowGuiReferenceDataProperties.cs │ │ │ │ └── FlowGuiRule.cs │ │ │ ├── JsonConfig.cs │ │ │ ├── Metric │ │ │ │ ├── MetricSource.cs │ │ │ │ ├── MetricSourceInput.cs │ │ │ │ ├── MetricSourceInputDataKey.cs │ │ │ │ ├── MetricSourceOutput.cs │ │ │ │ ├── MetricWidget.cs │ │ │ │ └── MetricsConfig.cs │ │ │ ├── PropertiesDictionary.cs │ │ │ ├── RuntimeConfig │ │ │ │ ├── AzureFunctionSpec.cs │ │ │ │ ├── FlowBlobOutputSpec.cs │ │ │ │ ├── FlowCosmosDbOutputSpec.cs │ │ │ │ ├── FlowEventHubOutputSpec.cs │ │ │ │ ├── FlowHttpOutputSpec.cs │ │ │ │ ├── FlowOutputSpec.cs │ │ │ │ ├── FlowSqlOutputSpec.cs │ │ │ │ ├── InputBatchingSpec.cs │ │ │ │ ├── JarUdfSpec.cs │ │ │ │ ├── ReferenceDataSpec.cs │ │ │ │ ├── StateTableSpec.cs │ │ │ │ └── TimeWindowSpec.cs │ │ │ ├── SparkClusterConfig.cs │ │ │ ├── SparkJobConfig.cs │ │ │ └── SparkJobFrontEnd.cs │ │ ├── ConfigGenConfiguration.cs │ │ ├── ConfigGeneration │ │ │ ├── ConfigGenerationException.cs │ │ │ ├── DeploymentSession.cs │ │ │ ├── FlowDeploymentSession.cs │ │ │ ├── IFlowDeploymentProcessor.cs │ │ │ ├── ISequentialProcessor.cs │ │ │ ├── JobConfig.cs │ │ │ ├── JobDeploymentSession.cs │ │ │ └── Processor │ │ │ │ ├── ProcessorBase.cs │ │ │ │ ├── S100_PortConfigurationSettings.cs │ │ │ │ ├── S200_MergeDefaultConfig.cs │ │ │ │ ├── S300_ValidateConfig.cs │ │ │ │ ├── S400_PrepareJobConfigVariables.cs │ │ │ │ ├── S450_PrepareTransformFile.cs │ │ │ │ ├── S500_PrepareProjectionFile.cs │ │ │ │ ├── S500_PrepareSchemaFile.cs │ │ │ │ ├── S500_ResolveAzureFunction.cs │ │ │ │ ├── S500_ResolveJarUDF.cs │ │ │ │ ├── S500_ResolveOutputs.cs │ │ │ │ ├── S500_ResolveReferenceData.cs │ │ │ │ ├── S500_ResolveSparkJobParams.cs │ │ │ │ ├── S500_ResolveStateTable.cs │ │ │ │ ├── S500_ResolveStreamming.cs │ │ │ │ ├── S500_ResolveTimeWindow.cs │ │ │ │ ├── S500_ResolveTimestampColumn.cs │ │ │ │ ├── S550_ResolveBatchInput.cs │ │ │ │ ├── S600_GenerateJobConfig.cs │ │ │ │ ├── S600_GenerateJobConfigBatch.cs │ │ │ │ ├── S610_GenerateTransformFile.cs │ │ │ │ ├── S630_GenerateProjectionFile.cs │ │ │ │ ├── S630_GenerateSchemaFile.cs │ │ │ │ ├── S650_FlattenJobConfig.cs │ │ │ │ ├── S700_DeployJobConfigFile.cs │ │ │ │ ├── S800_DeploySparkJob.cs │ │ │ │ ├── S850_UpdateFlowMetrics.cs │ │ │ │ └── S900_Finalizer.cs │ │ ├── DataX.Config.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── IConfigGenConfigurationProvider.cs │ │ ├── InitialConfiguration.cs │ │ ├── InternalService │ │ │ ├── ConfigFlattener.cs │ │ │ ├── ConfigFlattenerManager.cs │ │ │ ├── DesignTimeCommonDataManager.cs │ │ │ ├── FileBasedCommonDataManager.cs │ │ │ ├── FlowConfigBuilder.cs │ │ │ ├── FlowDataManager.cs │ │ │ ├── ICommonDataManager.cs │ │ │ ├── JobDataManager.cs │ │ │ ├── JobState.cs │ │ │ ├── RuleDefinitionGenerator.cs │ │ │ ├── SparkClusterData.cs │ │ │ ├── SparkClusterManager.cs │ │ │ ├── SparkJobData.cs │ │ │ ├── SparkJobOperation.cs │ │ │ └── SparkJobTemplateManager.cs │ │ ├── PublicService │ │ │ ├── FlowOperation.cs │ │ │ ├── JobOperation.cs │ │ │ └── RuntimeConfigGeneration.cs │ │ ├── Storage │ │ │ ├── IDesignTimeConfigStorage.cs │ │ │ └── IRuntimeConfigStorage.cs │ │ ├── Templating │ │ │ ├── Token.cs │ │ │ ├── TokenDictionary.cs │ │ │ ├── TokenReplacement.cs │ │ │ └── TokenType.cs │ │ └── Utility │ │ │ ├── Comparison.cs │ │ │ ├── ConfigHelper.cs │ │ │ ├── DictionaryExtension.cs │ │ │ ├── GenerationLockDictionary.cs │ │ │ ├── KeyVaultUri.cs │ │ │ ├── LocalUtility.cs │ │ │ ├── LoggerExportDescriptorProvider.cs │ │ │ ├── ResourcePathUtil.cs │ │ │ ├── TaskExtensions.cs │ │ │ └── VersionGeneration.cs │ ├── DataX.Utility.EventHub │ │ ├── ConnectionStringParser.cs │ │ ├── DataX.Utility.EventHub.csproj │ │ ├── EventHubInfo.cs │ │ ├── EventHubManagementRestClient.cs │ │ ├── EventHubUtil.cs │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ └── DataX.Utility.Extensions │ │ ├── DataX.Utility.Extensions.csproj │ │ └── TaskExtensions.cs ├── DataX.Contract │ ├── DataX.Contract.csproj │ ├── Ensure.cs │ ├── Exception │ │ └── GeneralException.cs │ ├── FinalPublicKey.snk │ ├── GlobalSuppressions.cs │ ├── ResourceUtility.cs │ ├── Result │ │ ├── ApiResult.cs │ │ ├── FailedResult.cs │ │ ├── Result.cs │ │ └── SuccessResult.cs │ └── Settings │ │ ├── DataXSettings.cs │ │ └── DataXSettingsConstants.cs ├── DataX.Flow │ ├── DataX.Flow.CodegenRules.Tests │ │ ├── CGenAggAlDerTable.txt │ │ ├── CGenAggRulDerTable.txt │ │ ├── CGenAggWithDot.txt │ │ ├── CGenCreateMetric.txt │ │ ├── CGenCreateMetric2.txt │ │ ├── CGenIoTSample.txt │ │ ├── CGenMixedAlert.txt │ │ ├── CGenMixedAlertWithTick.txt │ │ ├── CGenNoCode.txt │ │ ├── CGenNoPivots.txt │ │ ├── CGenSimAlDerTable.txt │ │ ├── CGenSimRulDerTable.txt │ │ ├── CGenSimRulNonTable.txt │ │ ├── CGenSimWithDot.txt │ │ ├── CGenWithTick.txt │ │ ├── CodegenTests.cs │ │ ├── DataX.Flow.CodegenRules.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── OutputTemplates.xml │ │ ├── QueryTemplates.xml │ │ ├── Rules.json │ │ ├── Rules2.json │ │ ├── UserCodeAggAlDerTable.txt │ │ ├── UserCodeAggRulDerTable.txt │ │ ├── UserCodeAggWithDot.txt │ │ ├── UserCodeCreateMetric.txt │ │ ├── UserCodeCreateMetric2.txt │ │ ├── UserCodeIoTSample.txt │ │ ├── UserCodeSimAlDerTable.txt │ │ ├── UserCodeSimRulDerTable.txt │ │ ├── UserCodeSimRulNonTable.txt │ │ ├── UserCodeSimWithDot.txt │ │ ├── UserCodeWithTick.txt │ │ ├── cgen.txt │ │ ├── cgenDefault.txt │ │ └── usercode.txt │ ├── DataX.Flow.CodegenRules │ │ ├── CodeGen.cs │ │ ├── DataX.Flow.CodegenRules.csproj │ │ ├── Engine.cs │ │ ├── FinalPublicKey.snk │ │ ├── Formatter.cs │ │ ├── GlobalSuppressions.cs │ │ ├── Metrics.cs │ │ ├── OutputTemplates.Designer.cs │ │ ├── OutputTemplates.cs │ │ ├── OutputTemplates.xsc │ │ ├── OutputTemplates.xsd │ │ ├── OutputTemplates.xss │ │ ├── QueryTemplates.cs │ │ ├── Resources │ │ │ ├── defaultOutputTemplate.xml │ │ │ └── defaultQueryTemplate.xml │ │ └── Rule.cs │ ├── DataX.Flow.Common.Tests │ │ ├── DataX.Flow.Common.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GeneratorHelperTests.cs │ │ └── GeneratorTests.cs │ ├── DataX.Flow.Common │ │ ├── DataX.Flow.Common.csproj │ │ ├── EngineEnvironment.cs │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── Helper.cs │ │ ├── Models │ │ │ ├── FlowConfigObject.cs │ │ │ ├── FunctionObject.cs │ │ │ ├── InteractiveQueryObject.cs │ │ │ └── ProductConfigObject.cs │ │ └── TemplateGenerator.cs │ ├── DataX.Flow.DeleteHelper │ │ ├── ConfigDeleter.cs │ │ ├── DataX.Flow.DeleteHelper.csproj │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Flow.InteractiveQuery.Tests │ │ ├── DataX.Flow.InteractiveQuery.Tests.csproj │ │ └── InteractiveQueryTests.cs │ ├── DataX.Flow.InteractiveQuery │ │ ├── DataX.Flow.InteractiveQuery.csproj │ │ ├── Databricks │ │ │ ├── DatabricksKernel.cs │ │ │ └── DatabricksKernelService.cs │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── HDInsight │ │ │ ├── HDInsightKernel.cs │ │ │ └── HDInsightKernelService.cs │ │ ├── IKernel.cs │ │ ├── InteractiveQueryManager.cs │ │ ├── KernelService.cs │ │ └── SetupSteps.cs │ ├── DataX.Flow.Scheduler │ │ ├── DataX.Flow.Scheduler.csproj │ │ ├── FinalPublicKey.snk │ │ └── TimedScheduler.cs │ ├── DataX.Flow.SchemaInference.Tests │ │ ├── DataX.Flow.SchemaInference.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── SchemaTests.cs │ │ ├── SchemaUtilityTests.cs │ │ ├── events.json │ │ ├── events4.json │ │ ├── events5.json │ │ ├── events5errors.txt │ │ └── result.txt │ ├── DataX.Flow.SchemaInference │ │ ├── Blob │ │ │ └── BlobMessageBus.cs │ │ ├── DataX.Flow.SchemaInference.csproj │ │ ├── Engine.cs │ │ ├── EventRaw.cs │ │ ├── Eventhub │ │ │ ├── EventProcessor.cs │ │ │ ├── EventProcessorFactory.cs │ │ │ └── EventhubMessageBus.cs │ │ ├── EventsData.cs │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── IMessageBus.cs │ │ ├── Kafka │ │ │ └── KafkaMessageBus.cs │ │ ├── Schema.cs │ │ ├── SchemaGenerator.cs │ │ └── SchemaInferenceManager.cs │ ├── DataX.Flow.SqlParser.Tests │ │ ├── DataX.Flow.SqlParser.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── SqlParserTests.cs │ │ ├── cgen.txt │ │ ├── expectedComplexTest.txt │ │ ├── inputschema.json │ │ └── usercode.txt │ ├── DataX.Flow.SqlParser │ │ ├── Analyzer.cs │ │ ├── DataX.Flow.SqlParser.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── InputSchema.cs │ │ └── SqlParser.cs │ ├── DataX.Flow.sln │ ├── DataX.Flow │ │ ├── ApplicationPackageRoot │ │ │ └── ApplicationManifest.xml │ │ ├── ApplicationParameters │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── DataX.Flow.sfproj │ │ ├── PublishProfiles │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── Scripts │ │ │ └── Deploy-FabricApplication.ps1 │ │ └── packages.config │ ├── DataX.Utilities.Composition │ │ ├── DataX.Utilities.Composition.csproj │ │ ├── FinalPublicKey.snk │ │ └── InstanceExportDescriptorProvider.cs │ ├── DataX.Utility.ServiceCommunication │ │ ├── DataX.Utility.ServiceCommunication.csproj │ │ ├── FinalPublicKey.snk │ │ └── InterServiceCommunicator.cs │ ├── Flow.InteractiveQueryService │ │ ├── Connected Services │ │ │ └── Application Insights │ │ │ │ └── ConnectedService.json │ │ ├── Controllers │ │ │ └── InteractiveQueryController.cs │ │ ├── FinalPublicKey.snk │ │ ├── Flow.InteractiveQueryService.csproj │ │ ├── FlowInteractiveQueryServiceStartup.cs │ │ ├── GlobalSuppressions.cs │ │ ├── InteractiveQueryService.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── launchSettings.json │ │ ├── ServiceEventSource.cs │ │ ├── app.config │ │ ├── appsettings.Development.json │ │ └── appsettings.json │ ├── Flow.LiveDataService │ │ ├── Connected Services │ │ │ └── Application Insights │ │ │ │ └── ConnectedService.json │ │ ├── Controllers │ │ │ └── LiveDataController.cs │ │ ├── FinalPublicKey.snk │ │ ├── Flow.LiveDataService.csproj │ │ ├── FlowLiveDataServiceStartup.cs │ │ ├── GlobalSuppressions.cs │ │ ├── LiveDataManager.cs │ │ ├── LiveDataService.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── launchSettings.json │ │ ├── ServiceEventSource.cs │ │ ├── app.config │ │ ├── appsettings.Development.json │ │ └── appsettings.json │ ├── Flow.ManagementService │ │ ├── Connected Services │ │ │ └── Application Insights │ │ │ │ └── ConnectedService.json │ │ ├── Controllers │ │ │ ├── FlowManagementController.cs │ │ │ └── FlowManagementControllerDelete.cs │ │ ├── FinalPublicKey.snk │ │ ├── Flow.ManagementService.csproj │ │ ├── FlowManagementService.cs │ │ ├── FlowManagementServiceStartup.cs │ │ ├── GlobalSuppressions.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── launchSettings.json │ │ ├── ServiceCollectionExtensions.cs │ │ ├── ServiceEventSource.cs │ │ ├── app.config │ │ ├── appsettings.Development.json │ │ └── appsettings.json │ ├── Flow.SchemaInferenceService │ │ ├── Connected Services │ │ │ └── Application Insights │ │ │ │ └── ConnectedService.json │ │ ├── Controllers │ │ │ └── SchemaInferenceController.cs │ │ ├── FinalPublicKey.snk │ │ ├── Flow.SchemaInferenceService.csproj │ │ ├── FlowSchemaInferenceServiceStartup.cs │ │ ├── GlobalSuppressions.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── launchSettings.json │ │ ├── SchemaInferenceService.cs │ │ ├── ServiceEventSource.cs │ │ ├── app.config │ │ ├── appsettings.Development.json │ │ └── appsettings.json │ ├── Flow.nuspec │ ├── GitVersion.yml │ └── README.md ├── DataX.Gateway │ ├── DataX.Gateway.Api.Tests │ │ ├── DataX.Gateway.Api.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── Properties │ │ │ └── AssemblyInfo.cs │ │ ├── app.config │ │ └── packages.config │ ├── DataX.Gateway.Api │ │ ├── Api.cs │ │ ├── App.config │ │ ├── Controllers │ │ │ └── GatewayController.cs │ │ ├── DataX.Gateway.Api.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── JsonContentNegotiator.cs │ │ ├── OwinCommunicationListener.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── AssemblyInfo.cs │ │ ├── Startup.cs │ │ ├── Validator.cs │ │ └── packages.config │ ├── DataX.Gateway.Contract │ │ ├── Constants.cs │ │ ├── DataX.Gateway.Contract.csproj │ │ └── FinalPublicKey.snk │ ├── DataX.Gateway.sln │ ├── DataX.Gateway │ │ ├── ApplicationPackageRoot │ │ │ └── ApplicationManifest.xml │ │ ├── ApplicationParameters │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── DataX.Gateway.sfproj │ │ ├── PublishProfiles │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── Scripts │ │ │ └── Deploy-FabricApplication.ps1 │ │ └── packages.config │ ├── DataXGateway.nuspec │ └── GitVersion.yml ├── DataX.Metrics │ ├── DataX.Metrics.Ingestor │ │ ├── Connected Services │ │ │ └── Application Insights │ │ │ │ └── ConnectedService.json │ │ ├── DataX.Metrics.Ingestor.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── Helper │ │ │ ├── IngestorEventProcessor.cs │ │ │ ├── IngestorEventProcessorFactory.cs │ │ │ ├── MetricItem.cs │ │ │ ├── MetricOutput.cs │ │ │ ├── SecretStore.cs │ │ │ └── Utility.cs │ │ ├── Ingestor.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ ├── Properties │ │ │ └── launchSettings.json │ │ ├── ServiceEventSource.cs │ │ ├── Startup.cs │ │ ├── appsettings.Development.json │ │ └── appsettings.json │ ├── DataX.Metrics.Tests │ │ ├── DataX.Metrics.Tests.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ └── MetricsIngestorTests.cs │ ├── DataX.Metrics.sln │ ├── DataX.Metrics │ │ ├── ApplicationPackageRoot │ │ │ └── ApplicationManifest.xml │ │ ├── ApplicationParameters │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── DataX.Metrics.sfproj │ │ ├── PublishProfiles │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── Scripts │ │ │ └── Deploy-FabricApplication.ps1 │ │ └── packages.config │ ├── DataXMetrics.nuspec │ └── GitVersion.yml ├── DataX.ServiceHost │ ├── DataX.ServiceHost.AspNetCore │ │ ├── Authorization │ │ │ ├── DataXAuthConstants.cs │ │ │ ├── DataXAuthorizeAttribute.cs │ │ │ ├── DataXPolicyBuilder.cs │ │ │ ├── Extensions │ │ │ │ └── DataXAuthorizationExtensions.cs │ │ │ ├── Requirements │ │ │ │ ├── DataXReaderRequirement.cs │ │ │ │ └── DataXWriterRequirement.cs │ │ │ └── Roles │ │ │ │ ├── DataXReaderAttribute.cs │ │ │ │ └── DataXWriterAttribute.cs │ │ ├── DataX.ServiceHost.AspNetCore.csproj │ │ ├── Extensions │ │ │ ├── DataXApplicationBuilderExtensions.cs │ │ │ ├── DataXServiceCollectionExtensions.cs │ │ │ └── DataXWebHostBuilderExtensions.cs │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── StartUpBase.cs │ │ └── Startup │ │ │ ├── DataXServiceStartup.cs │ │ │ └── IDataXServiceStartup.cs │ ├── DataX.ServiceHost.ServiceFabric │ │ ├── Authorization │ │ │ └── DataXDefaultGatewayPolicy.cs │ │ ├── DataX.ServiceHost.ServiceFabric.csproj │ │ ├── Extensions │ │ │ └── Configuration │ │ │ │ └── ServiceFabricConfigurationBuilderExtensions.cs │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── ServiceEventSource.cs │ │ ├── ServiceFabricConfigurationSource.cs │ │ └── ServiceFabricUtil.cs │ ├── DataX.ServiceHost.sln │ └── DataX.ServiceHost │ │ ├── Authorization │ │ ├── DataXAuthRequirement.cs │ │ └── Requirements │ │ │ └── DataXOneBoxRequirement.cs │ │ ├── DataX.ServiceHost.csproj │ │ ├── FinalPublicKey.snk │ │ └── HostUtil.cs ├── DataX.SimulatedData │ ├── DataX.SimulatedData.DataGenService │ │ ├── DataGen.cs │ │ ├── DataGenService.cs │ │ ├── DataX.SimulatedData.DataGenService.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── Model │ │ │ ├── DataSchema.cs │ │ │ └── KafkaConnection.cs │ │ ├── PackageRoot │ │ │ ├── Config │ │ │ │ └── Settings.xml │ │ │ └── ServiceManifest.xml │ │ ├── Program.cs │ │ └── ServiceEventSource.cs │ ├── DataX.SimulatedData.DataGenServiceTest │ │ ├── DataGenUnitTests.cs │ │ ├── DataX.SimulatedData.DataGenServiceTest.csproj │ │ ├── FinalPublicKey.snk │ │ ├── testinput1.json │ │ ├── testinput2Array.json │ │ ├── testrandomexpecteddata1.json │ │ ├── testrandomexpecteddata2.json │ │ └── testrulesexpecteddata1.json │ ├── DataX.SimulatedData.sln │ ├── DataX.SimulatedData │ │ ├── ApplicationPackageRoot │ │ │ └── ApplicationManifest.xml │ │ ├── ApplicationParameters │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── DataX.SimulatedData.sfproj │ │ ├── PublishProfiles │ │ │ ├── Cloud.xml │ │ │ ├── Local.1Node.xml │ │ │ └── Local.5Node.xml │ │ ├── Scripts │ │ │ └── Deploy-FabricApplication.ps1 │ │ └── packages.config │ ├── DataX.ruleset │ └── DataXSimulatedData.nuspec ├── DataX.Utilities │ ├── DataX.Utilities.Blob │ │ ├── BlobHelper.cs │ │ ├── BlobStorage.cs │ │ ├── BlobStorageMSI.cs │ │ ├── DataX.Utilities.Blob.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ └── StorageCreator.cs │ ├── DataX.Utilities.CosmosDB │ │ ├── CosmosDB.cs │ │ ├── CosmosDBUtil.cs │ │ ├── DataX.Utilities.CosmosDB.csproj │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Utilities.EventHub │ │ ├── DataX.Utilities.EventHub.csproj │ │ ├── EventHub.cs │ │ ├── EventHubUtil.cs │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Utilities.KeyVault │ │ ├── DataX.Utilities.KeyVault.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── KeyVault.cs │ │ ├── KeyVaultManager.cs │ │ └── KeyVaultUtility.cs │ ├── DataX.Utilities.Telemetry │ │ ├── DataX.Utilities.Telemetry.csproj │ │ ├── FinalPublicKey.snk │ │ ├── OperationParentIdTelemetryInitializer.cs │ │ └── StartUpUtil.cs │ ├── DataX.Utilities.Tests │ │ ├── DataX.Utilities.Tests.csproj │ │ └── UtilityTests.cs │ ├── DataX.Utilities.Web │ │ ├── DataX.Utilities.Web.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ └── RolesCheck.cs │ ├── DataX.Utilities.sln │ ├── DataX.Utility.Blob │ │ ├── BaseTableEntity.cs │ │ ├── BlobUtility.cs │ │ ├── DataX.Utility.Blob.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── IsCriticalException.cs │ │ ├── StorageTableBase.cs │ │ ├── StorageUtils.cs │ │ ├── TableExtensions.cs │ │ └── TaskExtensions.cs │ ├── DataX.Utility.CosmosDB │ │ ├── CosmosDBUtility.cs │ │ ├── DataX.Utility.CosmosDB.csproj │ │ ├── FinalPublicKey.snk │ │ └── GlobalSuppressions.cs │ ├── DataX.Utility.KeyVault │ │ ├── DataX.Utility.KeyVault.csproj │ │ ├── FinalPublicKey.snk │ │ ├── GlobalSuppressions.cs │ │ ├── KeyVaultUtility.cs │ │ └── SecretUriParser.cs │ └── DataX.Utility.Nuget │ │ ├── DataX.Utility.Nuget.csproj │ │ ├── FinalPublicKey.snk │ │ └── VerifySignedFiles.txt ├── DataX.sln ├── JobRunner │ ├── AppConfig.cs │ ├── Connected Services │ │ └── Application Insights │ │ │ └── ConnectedService.json │ ├── JobQueueMessage.cs │ ├── JobRunner.cs │ ├── JobRunner.csproj │ ├── JobRunner.sln │ ├── JobRunnerException.cs │ ├── Jobs │ │ ├── DataXDeployJob.cs │ │ ├── DataXSchemaAndQueryJob.cs │ │ ├── IJob.cs │ │ └── TestJob.cs │ ├── MefContainer.cs │ ├── ObjectModel │ │ ├── ScheduledJobEntity.cs │ │ └── ScheduledJobTable.cs │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── Startup.cs │ ├── StartupHelper.cs │ ├── TableFactory.cs │ └── appsettings.json ├── JobRunnerWebJob │ ├── JobRunnerWebJob.csproj │ ├── Program.cs │ ├── Properties │ │ └── launchSettings.json │ ├── Readme.md │ ├── Settings.job │ ├── appsettings.Development.json │ ├── appsettings.json │ └── run.cmd ├── README.md └── codecoverage.runsettings ├── Tests ├── .gitignore ├── DataXScenarios │ ├── DataXScenarios.sln │ └── DataXScenarios │ │ ├── Context.cs │ │ ├── ContextHelper.cs │ │ ├── DataXScenarios.csproj │ │ ├── InteractiveQueryAndSchemaGenScenarios.cs │ │ ├── SaveAndDeploy.cs │ │ └── ScenarioUri.cs └── ScenarioTester │ ├── .editorconfig │ ├── README.md │ ├── ScenarioTester.nuspec │ ├── ScenarioTester.sln │ ├── ScenarioTester │ ├── DontTimeAttribute.cs │ ├── FinalPublicKey.snk │ ├── Request.cs │ ├── ScenarioContext.cs │ ├── ScenarioDescription.cs │ ├── ScenarioResult.cs │ ├── ScenarioTester.csproj │ ├── StepAttribute.cs │ └── StepResult.cs │ └── ScenarioTesterTests │ ├── ScenarioTesterTests.csproj │ ├── TestDataXScenarios.cs │ └── TestScenarioTester.cs ├── ThirdPartyNotices.txt └── Website ├── .gitignore ├── Packages ├── README.md ├── datax-common │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ │ └── settings.json │ ├── README.md │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── common │ │ │ ├── apiConstants.js │ │ │ └── index.js │ │ ├── components │ │ │ ├── appContent.jsx │ │ │ ├── appLogo.jsx │ │ │ ├── appRoot.jsx │ │ │ ├── centeredMessagePanel.jsx │ │ │ ├── errorMessagePanel.jsx │ │ │ ├── iFramePage.jsx │ │ │ ├── icon.jsx │ │ │ ├── index.js │ │ │ ├── loadingPanel.jsx │ │ │ ├── pageHeader.jsx │ │ │ ├── panel.jsx │ │ │ ├── panelBody.jsx │ │ │ ├── panelHeader.jsx │ │ │ ├── panelHeaderButtons.jsx │ │ │ ├── scrollableContentPane.jsx │ │ │ ├── shell.jsx │ │ │ ├── statementBox.jsx │ │ │ ├── topNav.jsx │ │ │ ├── topNavAppName.jsx │ │ │ ├── topNavButton.jsx │ │ │ ├── topNavIdentity.jsx │ │ │ ├── topNavIdentityMenu.jsx │ │ │ ├── topNavIdentityPhoto.jsx │ │ │ ├── topNavPageLink.jsx │ │ │ ├── topNavSpacer.jsx │ │ │ ├── verticalTabItem.jsx │ │ │ └── verticalTabs.jsx │ │ ├── index.js │ │ ├── modules │ │ │ ├── index.js │ │ │ └── user │ │ │ │ ├── actions.js │ │ │ │ ├── api.js │ │ │ │ ├── reducer.js │ │ │ │ └── selectors.js │ │ ├── styles │ │ │ ├── colors.js │ │ │ ├── iconButtonStyles.js │ │ │ ├── index.js │ │ │ └── styles.css │ │ └── utils │ │ │ ├── helpers.js │ │ │ ├── index.js │ │ │ ├── nodeServiceApi.js │ │ │ ├── serviceApi.js │ │ │ └── utilities.js │ └── webpack.config.js ├── datax-home │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ │ └── settings.json │ ├── README.md │ ├── img │ │ ├── prop-accumulator.svg │ │ ├── prop-aggalert.svg │ │ ├── prop-aggtag.svg │ │ ├── prop-alert.svg │ │ ├── prop-armcustom.svg │ │ ├── prop-debug.svg │ │ ├── prop-debugtelemetry.svg │ │ ├── prop-firstflow.svg │ │ ├── prop-functions.svg │ │ ├── prop-github.svg │ │ ├── prop-livequery.svg │ │ ├── prop-metrics.svg │ │ ├── prop-output-cloud.svg │ │ ├── prop-output-cosmosdb.svg │ │ ├── prop-output-local.svg │ │ ├── prop-race-car-data.svg │ │ ├── prop-reference.svg │ │ ├── prop-role.svg │ │ ├── prop-sample-home-automation.svg │ │ ├── prop-sample-telemetry.svg │ │ ├── prop-scale.svg │ │ ├── prop-schema.svg │ │ ├── prop-spark.svg │ │ ├── prop-sql.svg │ │ ├── prop-tag.svg │ │ ├── prop-wiki.svg │ │ └── prop-window.svg │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── index.js │ │ ├── modules │ │ │ └── home │ │ │ │ ├── components │ │ │ │ ├── card.jsx │ │ │ │ ├── footerItem.jsx │ │ │ │ ├── homePage.jsx │ │ │ │ ├── section.jsx │ │ │ │ ├── welcomeCard.jsx │ │ │ │ └── wideCard.jsx │ │ │ │ ├── composition.js │ │ │ │ └── models.js │ │ └── styles │ │ │ └── styles.css │ └── webpack.config.js ├── datax-jobs │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ │ └── settings.json │ ├── README.md │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── index.js │ │ ├── modules │ │ │ └── jobs │ │ │ │ ├── api.js │ │ │ │ ├── components │ │ │ │ ├── sparkJobs.jsx │ │ │ │ └── sparkJobsList.jsx │ │ │ │ └── constants.js │ │ └── styles │ │ │ └── styles.css │ └── webpack.config.js ├── datax-metrics │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ │ └── settings.json │ ├── README.md │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── index.js │ │ ├── modules │ │ │ └── metrics │ │ │ │ ├── actions.js │ │ │ │ ├── api.js │ │ │ │ ├── apiConstants.js │ │ │ │ ├── components │ │ │ │ ├── d3.colorBox.js │ │ │ │ ├── d3.gauge.js │ │ │ │ ├── metricAllProducts.jsx │ │ │ │ ├── metricDashboard.jsx │ │ │ │ ├── metricExplorer.jsx │ │ │ │ ├── metricWidgetDetailsList.jsx │ │ │ │ ├── metricWidgetGauge.jsx │ │ │ │ ├── metricWidgetGeneric.jsx │ │ │ │ ├── metricWidgetLineChart.jsx │ │ │ │ ├── metricWidgetMultiLineChart.jsx │ │ │ │ ├── metricWidgetNumberBox.jsx │ │ │ │ ├── metricWidgetPercentageBox.jsx │ │ │ │ ├── metricWidgetSimpleBox.jsx │ │ │ │ ├── metricWidgetSimpleTable.jsx │ │ │ │ ├── metricWidgetStackedAreaChart.jsx │ │ │ │ ├── navigationItem.jsx │ │ │ │ └── navigationPanel.jsx │ │ │ │ ├── metrics.aggregator.js │ │ │ │ ├── metrics.buffer.accumulate.js │ │ │ │ ├── metrics.buffer.latest.js │ │ │ │ ├── metrics.datasource.js │ │ │ │ ├── metrics.polling.js │ │ │ │ ├── reducer.js │ │ │ │ ├── selectors.js │ │ │ │ └── serviceGroup.js │ │ └── styles │ │ │ └── styles.css │ └── webpack.config.js ├── datax-pipeline │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ │ └── settings.json │ ├── README.md │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ │ ├── common │ │ │ ├── api.js │ │ │ └── styles.js │ │ ├── index.js │ │ ├── modules │ │ │ ├── flowDefinition │ │ │ │ ├── api.js │ │ │ │ ├── components │ │ │ │ │ ├── flowDefinitionPanel.jsx │ │ │ │ │ ├── function │ │ │ │ │ │ ├── azureFunctionSettings.jsx │ │ │ │ │ │ ├── functionSettingsContent.jsx │ │ │ │ │ │ └── udfSettings.jsx │ │ │ │ │ ├── info │ │ │ │ │ │ └── infoSettingsContent.jsx │ │ │ │ │ ├── input │ │ │ │ │ │ └── inputSettingsContent.jsx │ │ │ │ │ ├── output │ │ │ │ │ │ ├── blobSinkerSettings.jsx │ │ │ │ │ │ ├── cosmosdbSinkerSettings.jsx │ │ │ │ │ │ ├── eventHubSinkerSettings.jsx │ │ │ │ │ │ ├── localSinkerSettings.jsx │ │ │ │ │ │ ├── metricSinkerSettings.jsx │ │ │ │ │ │ ├── outputSettingsContent.jsx │ │ │ │ │ │ └── sqlSinkerSettings.jsx │ │ │ │ │ ├── referenceData │ │ │ │ │ │ ├── csvReferenceDataSettings.jsx │ │ │ │ │ │ └── referenceDataSettingsContent.jsx │ │ │ │ │ ├── rule │ │ │ │ │ │ ├── conditionsPreview.jsx │ │ │ │ │ │ ├── queryBuilder.jsx │ │ │ │ │ │ ├── ruleAggregateColumnSettings.jsx │ │ │ │ │ │ ├── ruleAlertSettings.jsx │ │ │ │ │ │ ├── ruleConditionsSettings.jsx │ │ │ │ │ │ ├── ruleGeneralSettings.jsx │ │ │ │ │ │ ├── rulePivotSettings.jsx │ │ │ │ │ │ ├── rulesSettingsContent.jsx │ │ │ │ │ │ └── tagRuleSettings.jsx │ │ │ │ │ ├── scale │ │ │ │ │ │ └── scaleSettingsContent.jsx │ │ │ │ │ └── schedule │ │ │ │ │ │ ├── oneTimeScheduleSettings.jsx │ │ │ │ │ │ ├── recurringScheduleSettings.jsx │ │ │ │ │ │ └── scheduleSettingsContent.jsx │ │ │ │ ├── flowActions.js │ │ │ │ ├── flowHelpers.js │ │ │ │ ├── flowModels.js │ │ │ │ ├── flowReducer.js │ │ │ │ └── flowSelectors.js │ │ │ └── flowList │ │ │ │ ├── actions.js │ │ │ │ ├── api.js │ │ │ │ ├── components │ │ │ │ └── flowListPanel.jsx │ │ │ │ ├── reducer.js │ │ │ │ └── selectors.js │ │ └── styles │ │ │ └── styles.css │ └── webpack.config.js └── datax-query │ ├── .gitignore │ ├── .prettierrc │ ├── .vscode │ └── settings.json │ ├── README.md │ ├── localdevpatch.bat │ ├── package-lock.json │ ├── package.json │ ├── src │ ├── index.js │ ├── modules │ │ ├── index.js │ │ └── query │ │ │ ├── api.js │ │ │ ├── components │ │ │ ├── index.js │ │ │ ├── monacoeditorcontrol.jsx │ │ │ ├── querySettingsContent.jsx │ │ │ └── sideToolBar.jsx │ │ │ ├── kernelActions.js │ │ │ ├── kernelReducer.js │ │ │ ├── kernelSelectors.js │ │ │ ├── layoutActions.js │ │ │ ├── layoutReducer.js │ │ │ ├── layoutSelectors.js │ │ │ ├── queryActions.js │ │ │ ├── queryModels.js │ │ │ ├── queryReducer.js │ │ │ └── querySelectors.js │ └── styles │ │ └── styles.css │ └── webpack.config.js ├── README.md ├── Website ├── .deployment ├── .gitignore ├── .prettierrc ├── .vscode │ ├── launch.json │ └── settings.json ├── LICENSE ├── README.md ├── appinsights.js ├── auth.js ├── client │ ├── css │ │ └── global.css │ ├── favicon.ico │ ├── index.html │ └── js │ │ └── main.js ├── components │ ├── app.jsx │ ├── layout.jsx │ └── loadingPage.jsx ├── config.js ├── data │ ├── api.js │ └── storeManager.js ├── db │ ├── db.js │ ├── keyvaultClient.js │ └── mongoclient.js ├── deploy │ ├── DataX.Utilities │ │ ├── DataX.Utilities.sln │ │ ├── DataX.Utility.CodeSign │ │ │ ├── DataX.Utility.CodeSign.csproj │ │ │ └── FinalPublicKey.snk │ │ └── VerifySignedFiles.txt │ ├── deploy.cmd │ └── zipsite.js ├── host.js ├── iisnode.yml ├── logger.js ├── metrics │ ├── dataProxy │ │ └── redisProxy.js │ ├── metricService.js │ ├── redisconsole.js │ └── schemaHandlers.js ├── package-lock.json ├── package.json ├── package.loader.js ├── securedSettings.js ├── server.js ├── util │ ├── cache.js │ ├── common.js │ ├── consolelogger.js │ ├── dist.js │ ├── fs.js │ ├── localCache.js │ ├── redisclient.js │ └── transaction.js ├── web.composition.json ├── web.config └── webpack.config.js └── web.nuspec /.github/ISSUE_TEMPLATE/data-accelerator-feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Data Accelerator Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/DataXSparkValidation.yml: -------------------------------------------------------------------------------- 1 | name: DataX.Spark.Validation 2 | 3 | on: 4 | pull_request: 5 | branches: [ "master" ] 6 | 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | 11 | steps: 12 | - uses: actions/checkout@v4 13 | - name: Build datax-core with Maven 14 | run: mvn -e install --file DataProcessing/datax-core/pom.xml 15 | - name: Build datax-keyvault with Maven 16 | run: mvn -e install --file DataProcessing/datax-keyvault/pom.xml 17 | - name: Build datax-utility with Maven 18 | run: mvn -e install --file DataProcessing/datax-utility/pom.xml 19 | - name: Build datax-udf-samples with Maven 20 | run: mvn -e install --file DataProcessing/datax-udf-samples/pom.xml 21 | - name: Build datax-host with Maven 22 | run: mvn -e package --file DataProcessing/datax-host/pom.xml 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # This .gitignore file was automatically created by Microsoft(R) Visual Studio. 3 | ################################################################################ 4 | 5 | /.vs 6 | /v15 7 | npm-debug.log 8 | /DataProcessing/DataX.Utilities/DataX.Utility.CodeSign/obj 9 | /Tests/ScenarioTester/ScenarioTester/bin 10 | /Tests/ScenarioTester/.vs 11 | *.suo 12 | /Tests/ScenarioTester/ScenarioTester/obj 13 | /Tests/ScenarioTester/ScenarioTesterTests/bin 14 | /Tests/ScenarioTester/ScenarioTesterTests/obj 15 | -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "files.watcherExclude": { 3 | "**/target": true 4 | } 5 | } -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. -------------------------------------------------------------------------------- /DataProcessing/.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | */.vs 4 | .idea 5 | */target 6 | */src/main/resources 7 | *.iml 8 | out/ 9 | */obj -------------------------------------------------------------------------------- /DataProcessing/DataX.Utilities/.gitignore: -------------------------------------------------------------------------------- 1 | */obj -------------------------------------------------------------------------------- /DataProcessing/DataX.Utilities/DataX.Utility.CodeSign/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/DataProcessing/DataX.Utilities/DataX.Utility.CodeSign/FinalPublicKey.snk -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/config/DefaultValue.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.config 6 | 7 | import datax.constants.ProductConstant 8 | 9 | object DefaultValue { 10 | def DefaultAppName = ProductConstant.DefaultAppName 11 | } 12 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/config/SparkEnvVariables.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.config 6 | 7 | import org.apache.spark.{SparkEnv, TaskContext} 8 | 9 | object SparkEnvVariables { 10 | def getClientNodeName(appName: String): String = { 11 | appName+"-"+SparkEnv.get.executorId 12 | } 13 | 14 | def getLoggerSuffix(): String = { 15 | val tc = TaskContext.get() 16 | if(tc==null)"" else s"-P${tc.partitionId()}-T${tc.taskAttemptId()}" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/config/UnifiedConfig.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.config 6 | 7 | import org.apache.spark.SparkConf 8 | 9 | case class UnifiedConfig( 10 | sparkConf: SparkConf, 11 | dict: SettingDictionary 12 | ) 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/BlobProperties.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object BlobProperties { 8 | // Define constants for blobs 9 | val BlobHostPath = ".blob.core.windows.net" 10 | val DfsHostPath = ".dfs.core.windows.net" 11 | } 12 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/DatasetName.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object DatasetName { 8 | def DataStreamRaw = s"${NamePrefix.Value}RawInput" 9 | def DataStreamProjection = s"${NamePrefix.Value}ProcessedInput" 10 | def DataStreamProjectionBatch = s"${NamePrefix.Value}ProcessedInput_Batch" 11 | def DataStreamProjectionWithWindow = s"${NamePrefix.Value}ProcessedInput_Window" 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/FeatureName.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object FeatureName { 8 | // Define built-in functions 9 | val FunctionDisableCommonCaching = "disableCommonCaching" 10 | 11 | } 12 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/MetricName.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object MetricName { 8 | val MetricSinkPrefix="Sink_" 9 | } 10 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/NamePrefix.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object NamePrefix { 8 | val DefaultValue = "DataX" 9 | val ConfSetting = "DATAX_NAMEPREFIX" 10 | val Value: String = sys.env.getOrElse(ConfSetting, DefaultValue) 11 | } 12 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/constants/ProcessingPropertyName.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.constants 6 | 7 | object ProcessingPropertyName { 8 | // Define constants for names in the processing Properties column 9 | val BlobPathHint = "Partition" 10 | val BatchTime = "BatchTime" 11 | val BlobTime = "InputTime" 12 | val CPTime = "CPTime" 13 | val CPExecutor = "CPExecutor" 14 | } 15 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/exception/EngineException.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.exception 6 | 7 | case class EngineException(private val message: String = "", private val cause: Throwable = None.orNull) 8 | extends Exception(message, cause) 9 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/extension/PreProjectionProcessor.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.extension 6 | 7 | import datax.config.SettingDictionary 8 | import org.apache.spark.sql.{DataFrame, SparkSession} 9 | 10 | trait PreProjectionProcessor extends Serializable{ 11 | def initialize(spark:SparkSession, dict: SettingDictionary): DataFrame=>DataFrame 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/extension/StringNormalizer.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.extension 6 | 7 | /* 8 | extension to normalize the input string in the streaming pipeline before parsing them as JSON object 9 | */ 10 | trait StringNormalizer { 11 | def normalize(str: String): String 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/host/AppHost.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.host 6 | 7 | import datax.service.{ConfigService, TelemetryService} 8 | import org.apache.spark.SparkConf 9 | import org.apache.spark.sql.SparkSession 10 | 11 | trait AppHost { 12 | def getConfigService(): ConfigService 13 | def getTelemetryService(): TelemetryService 14 | def getSpark(sparkConf: SparkConf): SparkSession 15 | } 16 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/service/ConfigService.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.service 6 | 7 | import datax.config.{SettingDictionary} 8 | 9 | trait ConfigService { 10 | def getActiveDictionary(): SettingDictionary 11 | def setActiveDictionary(conf: SettingDictionary) 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-core/src/main/scala/datax/service/TelemetryService.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.service 6 | 7 | import java.sql.Timestamp 8 | 9 | trait TelemetryService { 10 | def trackEvent(event: String, properties: Map[String, String], measurements: Map[String, Double]) 11 | def trackMetric(event: String, properties: Map[String, String]) 12 | def trackException(e: Exception, properties: Map[String, String], measurements: Map[String, Double]) 13 | def trackBatchEvent(event: String, properties: Map[String, String], measurements: Map[String, Double], batchTime: Timestamp) 14 | def trackBatchMetric(event: String, properties: Map[String, String], batchTime: Timestamp) 15 | } 16 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/app/BatchApp.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.app 6 | 7 | import datax.host.BlobBatchingHost 8 | import datax.processor.CommonProcessorFactory 9 | 10 | object BatchApp { 11 | def main(inputArguments: Array[String]): Unit = { 12 | BlobBatchingHost.runBatchApp( 13 | inputArguments, 14 | config => CommonProcessorFactory.createProcessor(config).asBatchBlobProcessor()) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/app/BlobStreamingApp.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.app 6 | 7 | import com.microsoft.azure.eventhubs.EventData 8 | import datax.host.StreamingHost 9 | import datax.input._ 10 | import datax.processor.CommonProcessorFactory 11 | 12 | object BlobStreamingApp { 13 | def main(inputArguments: Array[String]): Unit = { 14 | new StreamingHost[EventData].runStreamingApp( 15 | inputArguments, 16 | EventHubInputSetting.asInstanceOf[InputSetting[InputConf]], 17 | EventHubStreamingFactory, 18 | config => CommonProcessorFactory.createProcessor(config).asBlobPointerProcessor()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/app/DirectLocalStreamingApp.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.app 6 | 7 | import com.microsoft.azure.eventhubs.EventData 8 | import datax.host.StreamingHost 9 | import datax.input._ 10 | import datax.processor.CommonProcessorFactory 11 | 12 | object DirectLocalStreamingApp { 13 | 14 | def main(inputArguments: Array[String]): Unit = { 15 | new StreamingHost[EventData]().runStreamingApp( 16 | inputArguments, 17 | LocalInputSetting.asInstanceOf[InputSetting[InputConf]], 18 | LocalStreamingFactory, 19 | config => CommonProcessorFactory.createProcessor(config).asDirectLocalProcessor()) 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/app/DirectStreamingApp.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.app 6 | 7 | import com.microsoft.azure.eventhubs.EventData 8 | import datax.host.StreamingHost 9 | import datax.input._ 10 | import datax.processor.CommonProcessorFactory 11 | 12 | object DirectStreamingApp { 13 | def main(inputArguments: Array[String]): Unit = { 14 | new StreamingHost[EventData]().runStreamingApp( 15 | inputArguments, 16 | EventHubInputSetting.asInstanceOf[InputSetting[InputConf]], 17 | EventHubStreamingFactory, 18 | config => CommonProcessorFactory.createProcessor(config).asDirectProcessor()) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/client/cosmosdb/CosmosDBBase.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.client.cosmosdb 6 | 7 | /*** 8 | * Represents the connection info for sinker to access cosmos DB 9 | * @param name name of this cosmosdb connection info mainly for logging purpose 10 | * @param connectionString connection string to the cosmosdb 11 | * @param database name of the cosmos database 12 | * @param collection name of the collection in cosmos db 13 | */ 14 | case class CosmosDBConf(name: String, connectionString: String, database: String, collection: String) 15 | 16 | object CosmosDBBase { 17 | 18 | } 19 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/client/eventhub/EventHubConf.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.client.eventhub 6 | 7 | case class EventHubConf(name: String, connectionString: String) 8 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/client/redis/RedisConnection.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.client.redis 6 | 7 | import io.lettuce.core.api.sync.{RedisSortedSetCommands, RedisStringCommands} 8 | 9 | trait RedisConnection{ 10 | def getStringCommands: RedisStringCommands[String, String] 11 | def getSortedSetCommands: RedisSortedSetCommands[String, String] 12 | def reconnect(): Unit 13 | } 14 | 15 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/data/ProcessResult.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.data 6 | 7 | case class ProcessResult(blobsCount: Long, eventsCount: Long) 8 | 9 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/handler/BuiltInFunctionsHandler.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.handler 6 | 7 | import datax.config.SettingDictionary 8 | import datax.utility.{AzureFunctionCaller, ConcurrentDateFormat} 9 | import org.apache.log4j.LogManager 10 | import org.apache.spark.sql.SparkSession 11 | 12 | object BuiltInFunctionsHandler { 13 | val logger = LogManager.getLogger(this.getClass) 14 | 15 | def initialize(spark: SparkSession, dict: SettingDictionary) = { 16 | spark.udf.register("stringToTimestamp", ConcurrentDateFormat.stringToTimestamp _) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/host/UdfInitializer.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.host 6 | 7 | import datax.config.SettingDictionary 8 | import org.apache.spark.sql.SparkSession 9 | 10 | object UdfInitializer { 11 | def initialize(spark: SparkSession, dict: SettingDictionary) = { 12 | // Register UDF functions 13 | spark.udf.register("filterNull", filterNull _) 14 | } 15 | 16 | def filterNull(elems: Seq[Map[String, String]]) : Seq[Map[String, String]] = { 17 | elems.filter(_!=null) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/input/InputManager.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.input 6 | 7 | import datax.config._ 8 | 9 | 10 | object InputManager { 11 | val NamespacePrefix = SettingNamespace.JobInputPrefix 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/input/InputSetting.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.input 6 | 7 | import datax.config.{SettingDictionary} 8 | 9 | // Interfaces for input config that all input handlers need to implement 10 | trait InputConf { 11 | val connectionString: String 12 | val flushExistingCheckpoints: Option[Boolean] 13 | val repartition: Option[Int] 14 | } 15 | 16 | trait InputSetting[T<:InputConf] { 17 | def getInputConf(dict: SettingDictionary): T 18 | } 19 | 20 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/processor/BatchBlobProcessor.scala: -------------------------------------------------------------------------------- 1 | package datax.processor 2 | 3 | import java.sql.Timestamp 4 | import datax.utility.DateTimeUtil 5 | import org.apache.spark.rdd.RDD 6 | 7 | import scala.concurrent.duration.Duration 8 | 9 | class BatchBlobProcessor(processBatchBlobPaths: (RDD[String], Timestamp, Duration, Timestamp, String, Long) => Map[String, Double]) { 10 | 11 | val process = (rdd: RDD[String], batchTime: Timestamp, batchInterval: Duration, inputPartitionSizeThresholdInBytes: Long) => { 12 | processBatchBlobPaths(rdd, batchTime, batchInterval, batchTime, "", inputPartitionSizeThresholdInBytes) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/processor/EventHubStructuredStreamingProcessor.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.processor 6 | 7 | import org.apache.spark.sql.DataFrame 8 | import org.apache.spark.sql.streaming.StreamingQuery 9 | 10 | class EventHubStructuredStreamingProcessor(processDataFrame: DataFrame=>Map[String, StreamingQuery]) 11 | extends StructuredStreamingProcessor { 12 | override val process: DataFrame => Map[String, StreamingQuery] = processDataFrame 13 | } 14 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/processor/StreamingProcessor.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.processor 6 | 7 | import java.sql.Timestamp 8 | import org.apache.spark.rdd.RDD 9 | 10 | import scala.concurrent.duration.Duration 11 | 12 | // Interface for DStream processor 13 | trait StreamingProcessor[T] { 14 | val process: (RDD[T], Timestamp, Duration) => Map[String, Double] 15 | } 16 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/processor/StructuredStreamingProcessor.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.processor 6 | 7 | import org.apache.spark.sql.DataFrame 8 | import org.apache.spark.sql.streaming.StreamingQuery 9 | 10 | trait StructuredStreamingProcessor { 11 | val process: (DataFrame) => Map[String, StreamingQuery] 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-host/src/main/scala/datax/sink/OutputSetting.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.sink 6 | 7 | object OutputSetting { 8 | 9 | } 10 | -------------------------------------------------------------------------------- /DataProcessing/datax-keyvault/.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | /.vs 4 | /.idea 5 | /target 6 | /src/main/resources 7 | *.iml 8 | -------------------------------------------------------------------------------- /DataProcessing/datax-keyvault/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | 3 | -------------------------------------------------------------------------------- /DataProcessing/datax-udf-samples/src/main/scala/datax/sample/normalizer/RemoveInvalidChars.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.sample.normalizer 6 | 7 | import datax.extension.StringNormalizer 8 | 9 | /* 10 | This normalizer replace invalid json characters from the input string with '#' symbol to avoid failing the JSON parser 11 | */ 12 | class RemoveInvalidChars extends StringNormalizer{ 13 | val invalidControlChars = "[\\x00-\\x08\\x0E-\\x1F]".r 14 | 15 | override def normalize(str: String): String = { 16 | invalidControlChars.replaceAllIn(str, "#") 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /DataProcessing/datax-udf-samples/src/main/scala/datax/sample/udf/UdfHelloWorld.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.sample.udf 6 | 7 | import org.apache.spark.sql.api.java.{UDF1 => jUDF} 8 | 9 | class UdfHelloWorld extends jUDF[String, String]{ 10 | override def call(p1: String): String = { 11 | "hello, " + p1 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/ArgumentsParser.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | object ArgumentsParser { 8 | def getNamedArgs(args: Array[String]): Map[String, String] = { 9 | args.filter(line => line.contains("=")) //take only named arguments 10 | .map(x => (x.substring(0, x.indexOf("=")), x.substring(x.indexOf("=") + 1))) //split into key values 11 | .toMap //convert to a map 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/ConverterUtil.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | import java.util.function.Function 8 | import scala.language.implicitConversions 9 | 10 | object ConverterUtil { 11 | implicit def scalaFunctionToJava[From, To](function: (From) => To): Function[From, To] = { 12 | new Function[From, To] { 13 | override def apply(input: From): To = function(input) 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/DataNormalization.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | object DataNormalization { 8 | def sanitizeColumnName(name: String) = { 9 | val invalidChars = ".- '" 10 | if(invalidChars.find(name.contains(_)).isDefined) s"`$name`" else name 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/DateTimeUtil.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | import java.sql.Timestamp 8 | import java.text.SimpleDateFormat 9 | 10 | object DateTimeUtil { 11 | def getCurrentTime(): Timestamp = { 12 | new Timestamp(new java.util.Date().getTime) 13 | } 14 | 15 | val simpleDateTimeFormat = new SimpleDateFormat("yyyyMMdd-HHmmss") 16 | def formatSimple(t: Timestamp) = { 17 | simpleDateTimeFormat.format(t) 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/FutureUtil.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | import scala.concurrent.{ExecutionContext, Future, Promise} 8 | 9 | object FutureUtil { 10 | def failFast[T](futures: Seq[Future[T]])(implicit ec: ExecutionContext): Future[Seq[T]] = { 11 | val promise = Promise[Seq[T]] 12 | futures.foreach{f => f.onFailure{case ex => promise.failure(ex)}} 13 | val res = Future.sequence(futures) 14 | promise.completeWith(res).future 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/GuidUtil.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | object GuidUtil { 8 | def generateGuid(): String = { 9 | java.util.UUID.randomUUID().toString 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/main/scala/datax/utility/Validation.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility 6 | 7 | import datax.exception.EngineException 8 | 9 | object Validation { 10 | def ensureNotNull(param: Any, name : String): Unit ={ 11 | if(param==null)throw new EngineException(s"$param cannot be null") 12 | } 13 | 14 | def ensureNotEmpty(param: Seq[Any], name: String): Unit = { 15 | if(param==null)throw new EngineException(s"$param cannot be null") 16 | if(param.isEmpty)throw new EngineException(s"$param cannot be empty") 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /DataProcessing/datax-utility/src/test/scala/datax/utility/test/ConcurrentDateFormatTests.scala: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | package datax.utility.test 6 | 7 | import java.sql.Timestamp 8 | 9 | import datax.utility.ConcurrentDateFormat 10 | import org.scalatest.flatspec.AnyFlatSpec 11 | import org.scalatest.matchers.should.Matchers 12 | 13 | class ConcurrentDateFormatTests extends AnyFlatSpec with Matchers{ 14 | "stringToTimestamp" should "parse strings to timestamps correctly" in { 15 | ConcurrentDateFormat.stringToTimestamp("08/10/2018 22:55:3") shouldBe Timestamp.valueOf("2018-08-10 22:55:03") 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /DeploymentCloud/.gitignore: -------------------------------------------------------------------------------- 1 | Deployment.DataX/Outputs/ 2 | Deployment.DataX/Packages/ 3 | Deployment.DataX/Temp/ 4 | Deployment.DataX/adminsteps.parameters.txt 5 | Deployment.DataX/cachedVariables 6 | Deployment.Common/Temp/ 7 | -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/Admin/adminsteps.parameters.txt: -------------------------------------------------------------------------------- 1 | # This is a parameter file for the admin steps script: adminSteps.ps1 2 | 3 | # tenantId 4 | tenantId=$tenantId 5 | 6 | # The application Id of Service AAD app. 7 | serviceAppId=$serviceAppId 8 | 9 | # The application Id of Client AAD app. 10 | clientAppId=$clientAppId 11 | 12 | # ResourceGroupName to generate resources 13 | resourceGroupName=$resourceGroup 14 | 15 | # Role Names 16 | # Writer can manage flows 17 | writerRole=$writerRole -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/CosmosDB/flows.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/CosmosDB/sparkClusters.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "$sparkName", 4 | "connectionString": "keyvault://$servicesKVName/$serviceSecretPrefix-livyconnectionstring-$sparkName" 5 | } 6 | ] -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/CosmosDB/sparkJobs.json: -------------------------------------------------------------------------------- 1 | [] -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/Nuget/NuGet.exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/DeploymentCloud/Deployment.Common/Nuget/NuGet.exe -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/Resources/Parameters/Databricks-Parameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "default_resource_location": { 6 | "value": "$resourceLocation" 7 | }, 8 | "sparkClusterName": { 9 | "value": "$sparkName" 10 | }, 11 | "databricksSku": { 12 | "value": "$databricksSku" 13 | }, 14 | "virtualNetworkName": { 15 | "value": "VNet-$serviceFabricName" 16 | }, 17 | "subnetdb1Name": { 18 | "value": "subnet-db-1" 19 | }, 20 | "subnetdb1Prefix": { 21 | "value": "10.0.1.0/24" 22 | }, 23 | "subnetdb2Name": { 24 | "value": "subnet-db-2" 25 | }, 26 | "subnetdb2Prefix": { 27 | "value": "10.0.2.0/24" 28 | }, 29 | "dbResourceGroupName": { 30 | "value": "$dbResourceGroupName" 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/scripts/msalmsiserverapp.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=hdinsight msalmsiserver app 3 | 4 | [Service] 5 | Type=idle 6 | 7 | ExecStart=/usr/bin/python /usr/hdinsight/msalmsiserver.py & 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Common/scripts/msiserverapp.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=hdinsight msiserver app 3 | 4 | [Service] 5 | Type=idle 6 | 7 | ExecStart=/usr/bin/python /usr/hdinsight/msiserver.py & 8 | 9 | [Install] 10 | WantedBy=multi-user.target -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/Resources/Parameters/IotHub-Parameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "iotHubName": { 6 | "value": "$iotHubName" 7 | }, 8 | "default_resource_location": { 9 | "value": "$resourceLocation" 10 | }, 11 | "sku_name": { 12 | "value": "B1" 13 | }, 14 | "sku_units": { 15 | "value": "1" 16 | }, 17 | "d2c_partitions": { 18 | "value": "4" 19 | }, 20 | "features": { 21 | "value": "None" 22 | }, 23 | "consumerGroupName": { 24 | "value": "iotsample" 25 | }, 26 | "retentionDays": { 27 | "value": "1" 28 | }, 29 | "namespaces_kafkaeventhub_name": { 30 | "value": "$kafkaEventHubNamespaceName" 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/Resources/Parameters/SampleOutputs-Parameter.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", 3 | "contentVersion": "1.0.0.0", 4 | "parameters": { 5 | "sqlcosmosdbname": { 6 | "value": "$sqlDocDBName" 7 | }, 8 | "default_resource_location": { 9 | "value": "$resourceLocation" 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/Samples/usercontent/devices.csv: -------------------------------------------------------------------------------- 1 | deviceId,homeId,deviceName 2 | 4,150,Left Garage Door 3 | 3,32,Front Door bolt 4 | 8,10,Front Door bolt 5 | 3,150,Back Door bolt 6 | 1,150,Furnace in basement 7 | 2,150,Front Window bolt 8 | 8,13,Front Window bolt 9 | 2,25,Front Door bolt 10 | 6,81,Heating Lamp outside 11 | -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/Samples/usercontent/readme.txt: -------------------------------------------------------------------------------- 1 | You can find the query under \flows\ in CosmosDB for the .json file -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/Samples/usercontent/udfsample.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/DeploymentCloud/Deployment.DataX/Samples/usercontent/udfsample.jar -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.DataX/runAdminSteps.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | echo Setting up Admin steps 3 | @set script=adminSteps.ps1 4 | powershell "&{.\adminSteps.ps1 -ParamFile .\adminsteps.parameters.txt; exit $LastExitCode}" 5 | if %errorlevel% neq 0 goto :errorHandler 6 | echo Setting up Admin steps completed 7 | 8 | goto :EOF 9 | 10 | :errorHandler 11 | echo Error occurred at %script% 12 | exit /b %errorlevel% -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mcr.microsoft.com/dotnet/aspnet:3.1 2 | WORKDIR /app/aspnetcore 3 | EXPOSE 5000 4 | ARG servicetarname 5 | ARG servicedllname 6 | ENV servicedllname=$servicedllname 7 | ADD $servicetarname . 8 | ENV ASPNETCORE_URLS="http://*:5000" PATH="$PATH:/app/aspnetcore:/app/aspnetcore/" 9 | COPY "finalrun.sh" "./" 10 | # Adding the execute permission. 11 | RUN chmod +x ./finalrun.sh 12 | ENTRYPOINT ["finalrun.sh"] 13 | -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/Pod-Identity-Service/managed-service-identity-binding.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "aadpodidentity.k8s.io/v1" 2 | kind: AzureIdentityBinding 3 | metadata: 4 | name: managed-service-identity-binding 5 | spec: 6 | AzureIdentity: managed-service-identity 7 | Selector: managed-service-identity -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/Pod-Identity-Service/managed-service-identity.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: "aadpodidentity.k8s.io/v1" 2 | kind: AzureIdentity 3 | metadata: 4 | name: managed-service-identity 5 | spec: 6 | type: 0 7 | ResourceID: /subscriptions/{SubscriptionId}/resourceGroups/{ResourceGroupName}/providers/Microsoft.ManagedIdentity/userAssignedIdentities/{ManagedIdentityName} 8 | ClientID: {ClientId} -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/Pod-Identity-Service/reinstall-pod-identity-service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ./uninstall-pod-identity-service.sh 4 | ./install-pod-identity-service.sh -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/Pod-Identity-Service/uninstall-pod-identity-service.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kubectl delete azureidentity managed-service-identity 4 | kubectl delete azureidentitybinding managed-service-identity-binding 5 | kubectl delete -f ./deployment.yaml -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Kubernetes/finalrun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dotnet "$servicedllname" 3 | -------------------------------------------------------------------------------- /DeploymentCloud/Deployment.Tests/runTests.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | echo Running Tests 3 | powershell "&{.\Tests\ResourceNameTests.ps1}" -------------------------------------------------------------------------------- /DeploymentCloud/README.md: -------------------------------------------------------------------------------- 1 | # Instructions 2 | Please see [these instructions](https://github.com/Microsoft/data-accelerator/wiki/Cloud-deployment) for deploying Data Accelerator to Azure. 3 | -------------------------------------------------------------------------------- /DeploymentLocal/README.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | Please see [these instructions](https://github.com/Microsoft/data-accelerator/wiki/Local-mode-with-Docker) for deploying Data Accelerator locally. 3 | -------------------------------------------------------------------------------- /DeploymentLocal/finalrun.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | dotnet Flow.ManagementService.dll & 3 | npm start 4 | -------------------------------------------------------------------------------- /Docs/Data Accelerator Introductory Whitepaper.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Docs/Data Accelerator Introductory Whitepaper.pdf -------------------------------------------------------------------------------- /Docs/Data Accelerator for Apache Spark.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Docs/Data Accelerator for Apache Spark.pdf -------------------------------------------------------------------------------- /Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/DataXSettings.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | using System.Threading.Tasks; 9 | 10 | namespace DataX.FlowManagement 11 | { 12 | public class DataXSettings 13 | { 14 | public string EnableOneBox { get; set; } 15 | public string LocalRoot { get; set; } 16 | public string SparkHome { get; set; } 17 | public string MetricsHttpEndpoint { get; set; } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Services/AspnetCore/DataX.FlowManagement/DataX.FlowManagement/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | 9 | "DataX": { 10 | "EnableOneBox": "true", 11 | "LocalRoot": "", 12 | "SparkHome": "", 13 | "MetricsHttpEndpoint": "http://localhost:2020/" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | #Building Services 2 | 3 | #Requirements 4 | In order to build the Data Accelerator Services, you will need the following 5 | - Managed Developement with dotnet core 6 | - Service Fabric 7 | 8 | #How to build 9 | - Open the datax.sln in Visual Studio. 10 | - Build 11 | 12 | #How to deploy 13 | - In the datax.sln, right click on each service (Flow, Config, Gateway, Metrics, Simulator) 14 | - Deploy each service from Visual Studio targetting your Service Fabric cluster 15 | 16 | #How to create a PR 17 | - Ensure all tests are passing 18 | - Create a pull request aainst the master branch -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.ConfigurationProviders/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.ConfigurationProviders/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient.Test/DataX.Config.DatabricksClient.Test.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/DataX.Config.DatabricksClient.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | true 6 | FinalPublicKey.snk 7 | true 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | Microsoft400 17 | StrongName 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/DatabricksClientConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Config.DatabricksClient 6 | { 7 | public class DatabricksClientConnectionInfo 8 | { 9 | public string Endpoint { get; set; } 10 | public string DbToken { get; set; } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/DatabricksHttpResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Net; 8 | using System.Text; 9 | 10 | namespace DataX.Config.DatabricksClient 11 | { 12 | public class DatabricksHttpResult 13 | { 14 | public HttpStatusCode StatusCode; 15 | public string Content; 16 | public bool IsSuccess; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/DatabricksJobResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.DatabricksClient 11 | { 12 | public class DatabricksJobResult 13 | { 14 | [JsonProperty("job_id")] 15 | public int JobId { get; set; } 16 | 17 | [JsonProperty("run_id")] 18 | public int RunId { get; set; } 19 | 20 | [JsonProperty("state")] 21 | public Dictionary State { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.DatabricksClient/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/HttpMethod.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.DatabricksClient 10 | { 11 | public enum HttpMethod 12 | { 13 | Get, 14 | Post, 15 | Delete 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/IDatabricksHttpClient.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | 10 | namespace DataX.Config.DatabricksClient 11 | { 12 | public interface IDatabricksHttpClient 13 | { 14 | Task ExecuteHttpRequest(HttpMethod method, Uri uri, string body = ""); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.DatabricksClient/IDatabricksHttpClientFactory.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.DatabricksClient 10 | { 11 | public interface IDatabricksHttpClientFactory 12 | { 13 | IDatabricksHttpClient CreateClientWithBearerToken(string dbToken); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Input.EventHub/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.Input.EventHub/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.KeyVault/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.KeyVault/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient.Test/DataX.Config.LivyClient.Test.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.LivyClient/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/HttpMethod.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.LivyClient 10 | { 11 | public enum HttpMethod 12 | { 13 | Get, 14 | Post, 15 | Delete 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/ILivyHttpClient.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Net.Http; 7 | using System.Threading.Tasks; 8 | 9 | namespace DataX.Config.LivyClient 10 | { 11 | public interface ILivyHttpClient 12 | { 13 | Task ExecuteHttpRequest(HttpMethod method, Uri uri, string body = ""); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/ILivyHttpClientFactory.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.LivyClient 10 | { 11 | public interface ILivyHttpClientFactory 12 | { 13 | ILivyHttpClient CreateClientWithBasicAuth(string userName, string password); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/LivyBatchesResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.LivyClient 11 | { 12 | public class LivyBatchesResult 13 | { 14 | [JsonProperty("from")] 15 | public int From { get; set; } 16 | 17 | [JsonProperty("total")] 18 | public int Total { get; set; } 19 | 20 | [JsonProperty("sessions")] 21 | public LivyBatchResult[] Sessions { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/LivyClientConnectionInfo.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Config.LivyClient 6 | { 7 | public class LivyClientConnectionInfo 8 | { 9 | public string Endpoint { get; set; } 10 | public string UserName { get; set; } 11 | public string Password { get; set; } 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.LivyClient/LivyHttpResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Net; 8 | using System.Text; 9 | 10 | namespace DataX.Config.LivyClient 11 | { 12 | public class LivyHttpResult 13 | { 14 | public HttpStatusCode StatusCode; 15 | public string Content; 16 | public bool IsSuccess; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.PrimaryTests.playlist: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Storage/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.Storage/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test.Utility/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.Test.Utility/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test.Utility/Mock/MockBase.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.Test.Utility.Mock 10 | { 11 | public class MockBase 12 | { 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Extension/CommonDataExtension.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using DataX.Config.ConfigDataModel; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.IO; 9 | using System.Text; 10 | using System.Threading.Tasks; 11 | 12 | namespace DataX.Config.Test.Extension 13 | { 14 | public static class CommonDataExtension 15 | { 16 | public static async Task Add(this ICommonDataManager commonData, string name , string filePath) 17 | { 18 | await commonData.SaveByName(name, JsonConfig.From(await File.ReadAllTextAsync(filePath))); 19 | } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config.Test/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Mock/SparkJobClientFactory.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using DataX.Config.ConfigDataModel; 6 | using DataX.Contract; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Composition; 10 | using System.Text; 11 | using System.Threading.Tasks; 12 | 13 | namespace DataX.Config.Test.Mock 14 | { 15 | [Shared] 16 | [Export(typeof(ISparkJobClientFactory))] 17 | public class SparkJobClientFactory : ISparkJobClientFactory 18 | { 19 | public Task GetClient(string connectionString) 20 | { 21 | throw new NotImplementedException(); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/Flattener/input.json: -------------------------------------------------------------------------------- 1 | { 2 | "firstlevelstring": "firstlevelstringvalue", 3 | "firstlevelobject": { 4 | "secondlevelstring": "secondlevelstringvalue" 5 | }, 6 | "array": [ 7 | { 8 | "namespace": "element1", 9 | "arrayelement": "arrayelement1value" 10 | }, 11 | { 12 | "namespace": "element2", 13 | "arrayelement": "arrayelement2value" 14 | } 15 | ], 16 | "map": { 17 | "mapelement1": { 18 | "mapelement": "element1value" 19 | }, 20 | "mapelement2": { 21 | "mapelement": "element2value" 22 | } 23 | }, 24 | "stringlist": [ "stringlistvalue1", "stringlistvalue2" ], 25 | "mapprops": { 26 | "prop1": "prop1value", 27 | "prop2": "prop2value" 28 | }, 29 | "withdefaultvalue": "defaultvalue", 30 | "withoutdefaultvalue": "non-defaultvalue" 31 | } 32 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/Flattener/output.conf: -------------------------------------------------------------------------------- 1 | some.root.namespace.firstlevelstring=firstlevelstringvalue 2 | some.root.namespace.firstlevelobject.secondlevelstring=secondlevelstringvalue 3 | some.root.namespace.array.element1.arrayelement=arrayelement1value 4 | some.root.namespace.array.element2.arrayelement=arrayelement2value 5 | some.root.namespace.map.mapelement1.mapelement=element1value 6 | some.root.namespace.map.mapelement2.mapelement=element2value 7 | some.root.namespace.stringlist=stringlistvalue1;stringlistvalue2 8 | some.root.namespace.mapprop.prop1=prop1value 9 | some.root.namespace.mapprop.prop2=prop2value 10 | some.root.namespace.withoutdefaultvalue=non-defaultvalue -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/batchSparkJobTemplate.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${sparkJobName}", 3 | "cluster": "test", 4 | "options": { 5 | }, 6 | "batch": null, 7 | "state": "Idle", 8 | "note": null 9 | } -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/jsonTest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "test", 3 | "displayName": "testDisplayName", 4 | "dynamic": { 5 | "foo": "bar", 6 | "bar": [] 7 | }, 8 | "generalObject": { 9 | "foo": "bar" 10 | }, 11 | "map": { 12 | "A.B": "", 13 | "c-d": { 14 | "e": 1 15 | } 16 | } 17 | } -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/projection.txt: -------------------------------------------------------------------------------- 1 | stringToTimestamp(Raw.deviceDetails.eventTime) AS eventTimeStamp 2 | Raw.* -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config.Test/Resource/sparkJobTemplate.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "${jobName}", 3 | "cluster": "test", 4 | "options": { 5 | }, 6 | "batch": null, 7 | "state": "Idle", 8 | "note": null 9 | } -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/Client/ISparkJobClient.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json.Linq; 6 | using DataX.Config.ConfigDataModel; 7 | using DataX.Contract; 8 | using System.Threading.Tasks; 9 | 10 | namespace DataX.Config 11 | { 12 | /// 13 | /// Contract for job management client 14 | /// 15 | public interface ISparkJobClient 16 | { 17 | Task SubmitJob(JToken jobSubmissionData); 18 | Task GetJobs(); 19 | Task StopJob(JToken jobClientData); 20 | Task GetJobInfo(JToken jobClientData); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/Client/ISparkJobClientFactory.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using DataX.Config.ConfigDataModel; 6 | using System.Threading.Tasks; 7 | 8 | namespace DataX.Config 9 | { 10 | public interface ISparkJobClientFactory 11 | { 12 | Task GetClient(string connectionString); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/CommonData/CommonDataConfig.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Text; 10 | 11 | namespace DataX.Config.ConfigDataModel.CommonData 12 | { 13 | public class CommonDataConfig : EntityConfig 14 | { 15 | [JsonProperty("name")] 16 | public string Name { get; set; } 17 | 18 | [JsonProperty("content")] 19 | public JToken Content { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiFunction.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiFunction 10 | { 11 | [JsonProperty("id")] 12 | public string Id { get; set; } 13 | 14 | [JsonProperty("type")] 15 | public string Type { get; set; } 16 | 17 | [JsonProperty("properties")] 18 | public FlowGuiFunctionProperties Properties { get; set; } 19 | 20 | [JsonProperty("typeDisplay")] 21 | public string TypeDisplay { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiInputBatchInput.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiInputBatchInput 10 | { 11 | [JsonProperty("type")] 12 | public string Type { get; set; } 13 | 14 | [JsonProperty("properties")] 15 | public FlowGuiInputBatchInputProperties Properties { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiInputBatchInputProperties.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiInputBatchInputProperties 10 | { 11 | [JsonProperty("connection")] 12 | public string Connection { get; set; } 13 | 14 | [JsonProperty("path")] 15 | public string Path { get; set; } 16 | 17 | [JsonProperty("formatType")] 18 | public string FormatType { get; set; } 19 | 20 | [JsonProperty("compressionType")] 21 | public string CompressionType { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiInputBatchJob.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiInputBatchJob 10 | { 11 | [JsonProperty("id")] 12 | public string Id { get; set; } 13 | 14 | [JsonProperty("type")] 15 | public string Type { get; set; } 16 | 17 | [JsonProperty("properties")] 18 | public FlowGuiInputBatchJobProperties Properties { get; set; } 19 | 20 | [JsonProperty("disabled")] 21 | public bool Disabled { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiOutput.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace DataX.Config.ConfigDataModel 9 | { 10 | public class FlowGuiOutput 11 | { 12 | [JsonProperty("id")] 13 | public string Id { get; set; } 14 | 15 | [JsonProperty("type")] 16 | public string Type { get; set; } 17 | 18 | [JsonProperty("properties")] 19 | public FlowGuiOutputProperties Properties { get; set; } 20 | 21 | [JsonProperty("typeDisplay")] 22 | public string TypeDisplay { get; set; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiOutputTemplate.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Config.ConfigDataModel 6 | { 7 | public class FlowGuiOutputTemplate 8 | { 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiReferenceData.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiReferenceData 10 | { 11 | [JsonProperty("id")] 12 | public string Id { get; set; } 13 | 14 | [JsonProperty("type")] 15 | public string Type { get; set; } 16 | 17 | [JsonProperty("typeDisplay")] 18 | public string TypeDisplay { get; set; } 19 | 20 | [JsonProperty("properties")] 21 | public FlowGuiReferenceDataProperties Properties { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiReferenceDataProperties.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class FlowGuiReferenceDataProperties 10 | { 11 | [JsonProperty("path")] 12 | public string Path { get; set; } 13 | 14 | [JsonProperty("delimiter")] 15 | public string Delimiter { get; set; } 16 | 17 | [JsonProperty("header")] 18 | public bool Header { get; set; } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/FlowGui/FlowGuiRule.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace DataX.Config.ConfigDataModel 9 | { 10 | public class FlowGuiRule 11 | { 12 | [JsonProperty("id")] 13 | public string Id { get; set; } 14 | 15 | [JsonProperty("type")] 16 | public string Type { get; set; } 17 | 18 | [JsonProperty("properties")] 19 | public JObject Properties { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/Metric/MetricSource.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace DataX.Config.ConfigDataModel 9 | { 10 | public class MetricSource 11 | { 12 | [JsonProperty("name")] 13 | public string Name { get; set; } 14 | 15 | [JsonProperty("input")] 16 | public MetricSourceInput Input { get; set; } 17 | 18 | [JsonProperty("output")] 19 | public MetricSourceOutput Output { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/Metric/MetricSourceInput.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using Newtonsoft.Json.Linq; 7 | 8 | namespace DataX.Config.ConfigDataModel 9 | { 10 | public class MetricSourceInput 11 | { 12 | [JsonProperty("type")] 13 | public string Type { get; set; } 14 | 15 | [JsonProperty("pollingInterval", NullValueHandling = NullValueHandling.Ignore)] 16 | public int? PollingInterval { get; set; } 17 | 18 | [JsonProperty("metricKeys")] 19 | public JToken[] DataKeys { get; set; } 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/Metric/MetricSourceInputDataKey.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel 8 | { 9 | public class MetricSourceInputDataKey 10 | { 11 | [JsonProperty("name")] 12 | public string Name { get; set; } 13 | 14 | [JsonProperty("displayName")] 15 | public string DisplayName { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/FlowCosmosDbOutputSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 11 | { 12 | public class FlowCosmosDbOutputSpec 13 | { 14 | [JsonProperty("connectionStringRef")] 15 | public string ConnectionStringRef { get; set; } 16 | 17 | [JsonProperty("database")] 18 | public string Database { get; set; } 19 | 20 | [JsonProperty("collection")] 21 | public string Collection { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/FlowEventHubOutputSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 11 | { 12 | public class FlowEventHubOutputSpec 13 | { 14 | [JsonProperty("connectionStringRef")] 15 | public string ConnectionStringRef { get; set; } 16 | 17 | [JsonProperty("compressionType")] 18 | public string CompressionType { get; set; } 19 | 20 | [JsonProperty("format")] 21 | public string Format { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/FlowHttpOutputSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 11 | { 12 | public class FlowHttpOutputSpec 13 | { 14 | [JsonProperty("endpoint")] 15 | public string Endpoint { get; set; } 16 | 17 | [JsonProperty("filter")] 18 | public string Filter { get; set; } 19 | 20 | [JsonProperty("appendHeaders")] 21 | public IDictionary Headers { get; set; } 22 | } 23 | 24 | } 25 | 26 | 27 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/JarUdfSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System.Collections.Generic; 7 | 8 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 9 | { 10 | public class JarUdfSpec 11 | { 12 | [JsonProperty("name")] 13 | public string Name { get; set; } 14 | 15 | [JsonProperty("path")] 16 | public string Path { get; set; } 17 | 18 | [JsonProperty("class")] 19 | public string Class { get; set; } 20 | 21 | [JsonProperty("libs")] 22 | public IList LibPaths { get; set; } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/StateTableSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | using System; 7 | using System.Collections.Generic; 8 | using System.Text; 9 | 10 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 11 | { 12 | public class StateTableSpec 13 | { 14 | [JsonProperty("name")] 15 | public string Name { get; set; } 16 | 17 | [JsonProperty("schema")] 18 | public string Schema { get; set; } 19 | 20 | [JsonProperty("location")] 21 | public string Location { get; set; } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigDataModel/RuntimeConfig/TimeWindowSpec.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json; 6 | 7 | namespace DataX.Config.ConfigDataModel.RuntimeConfig 8 | { 9 | public class TimeWindowSpec 10 | { 11 | [JsonProperty("name")] 12 | public string Name { get; set; } 13 | 14 | [JsonProperty("windowDuration")] 15 | public string WindowDuration { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/ConfigGeneration/ISequentialProcessor.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Config 6 | { 7 | public interface ISequentialProcessor 8 | { 9 | int GetOrder(); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Config/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/InternalService/ICommonDataManager.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System.Threading.Tasks; 6 | using DataX.Config.ConfigDataModel; 7 | using DataX.Contract; 8 | 9 | namespace DataX.Config 10 | { 11 | public interface ICommonDataManager 12 | { 13 | Task GetByName(string name); 14 | 15 | Task SaveByName(string name, JsonConfig json); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/InternalService/JobState.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Config 6 | { 7 | public enum JobState 8 | { 9 | Idle, 10 | Starting, 11 | Running, 12 | Success, 13 | Error 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/Templating/TokenType.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.Templating 10 | { 11 | public enum TokenType 12 | { 13 | Null, 14 | String, 15 | Object 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/Utility/ResourcePathUtil.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Config.Utility 10 | { 11 | public static class ResourcePathUtil 12 | { 13 | public static string Combine(string folder, string file) 14 | { 15 | if (folder == null) 16 | { 17 | return file; 18 | } 19 | 20 | if (file == null) 21 | { 22 | return folder; 23 | } 24 | 25 | return folder.TrimEnd('/') + '/' + file; 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Config/Utility/VersionGeneration.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | 7 | namespace DataX.Config 8 | { 9 | public static class VersionGeneration 10 | { 11 | private static string _LastGeneratedVersion = null; 12 | 13 | public static string Next() 14 | { 15 | var ver = DateTime.UtcNow.ToString("yyyyMMddHHmmss"); 16 | _LastGeneratedVersion = ver; 17 | return ver; 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Utility.EventHub/EventHubInfo.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Utility.EventHub 10 | { 11 | public class EventHubInfo 12 | { 13 | public string Namespace; 14 | public string Name; 15 | public string Key; 16 | public string Policy; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Utility.EventHub/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Config/DataX.Utility.EventHub/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Utility.Extensions/DataX.Utility.Extensions.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netstandard2.0 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /Services/DataX.Config/DataX.Utility.Extensions/TaskExtensions.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | 7 | namespace DataX.Utility.Extensions 8 | { 9 | public static class TaskExtensions 10 | { 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Services/DataX.Contract/Exception/GeneralException.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | 7 | namespace DataX.Contract.Exception 8 | { 9 | [Serializable] 10 | public class GeneralException : System.Exception 11 | { 12 | public GeneralException() { } 13 | public GeneralException(string message) : base(message) { } 14 | public GeneralException(string message, System.Exception inner) : base(message, inner) { } 15 | protected GeneralException( 16 | System.Runtime.Serialization.SerializationInfo info, 17 | System.Runtime.Serialization.StreamingContext context) : base(info, context) { } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.Contract/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Contract/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Contract/Result/FailedResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Contract 6 | { 7 | public class FailedResult : Result 8 | { 9 | public FailedResult(string msg) 10 | { 11 | this.IsSuccess = false; 12 | this.Message = msg; 13 | } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/DataX.Contract/Result/Result.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.Contract 10 | { 11 | public class Result 12 | { 13 | public bool IsSuccess; 14 | public string Message; 15 | public Dictionary Properties; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.Contract/Result/SuccessResult.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System.Collections.Generic; 6 | 7 | namespace DataX.Contract 8 | { 9 | public class SuccessResult : Result 10 | { 11 | public SuccessResult(string msg = "", Dictionary props = null) 12 | { 13 | this.IsSuccess = true; 14 | this.Message = msg; 15 | this.Properties = props; 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Services/DataX.Contract/Settings/DataXSettingsConstants.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Contract.Settings 6 | { 7 | /// 8 | /// Constants used with 9 | /// 10 | public static class DataXSettingsConstants 11 | { 12 | public const string DataX = nameof(DataX); 13 | 14 | public static readonly string ServiceEnvironment = $"{DataX}:{nameof(ServiceEnvironment)}"; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/CGenCreateMetric.txt: -------------------------------------------------------------------------------- 1 | --DataXQuery-- 2 | T1 = Select 3 | COUNT(deviceDetails.deviceId) AS deviceCount 4 | FROM 5 | DataXProcessedInput 6 | WHERE 7 | deviceDetails.homeId=150 8 | 9 | --DataXQuery-- 10 | DeviceCount = SELECT 11 | DISTINCT DATE_TRUNC('second', 12 | current_timestamp()) AS EventTime, 13 | 'DeviceCount' AS MetricName, 14 | deviceCount AS Metric, 15 | 'P7' AS Product, 16 | '' AS Pivot1 17 | FROM 18 | T1 19 | GROUP BY 20 | EventTime, 21 | MetricName, 22 | Metric, 23 | Product, 24 | Pivot1 -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeAggAlDerTable.txt: -------------------------------------------------------------------------------- 1 | ProcessAggregateAlerts(Foo); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeAggRulDerTable.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessAggregateRules(Foo); 2 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeAggWithDot.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessAggregateRules(DataXProcessedInput); 2 | ProcessAggregateAlerts(DataXProcessedInput); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeCreateMetric.txt: -------------------------------------------------------------------------------- 1 | --DataXQuery-- 2 | T1 = Select COUNT(deviceDetails.deviceId) AS deviceCount 3 | FROM DataXProcessedInput 4 | WHERE deviceDetails.homeId=150; 5 | 6 | 7 | DeviceCount = CreateMetric(T1, deviceCount); 8 | 9 | OUTPUT DeviceCount To Metrics; 10 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeSimAlDerTable.txt: -------------------------------------------------------------------------------- 1 | ProcessAlerts(Foo); 2 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeSimRulDerTable.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessRules(Foo); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeSimRulNonTable.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessRules(Bar); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeSimWithDot.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessRules(DataXProcessedInput); 2 | ProcessAlerts(DataXProcessedInput); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/UserCodeWithTick.txt: -------------------------------------------------------------------------------- 1 | T1 = ProcessRules(DataXProcessedInput); 2 | ProcessAlerts(DataXProcessedInput); 3 | 4 | T2 = ProcessAggregateRules(DataXProcessedInput); 5 | ProcessAggregateAlerts(DataXProcessedInput); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules.Tests/usercode.txt: -------------------------------------------------------------------------------- 1 | --DataXStates-- 2 | CREATE TABLE EventCount (Count: INT); 3 | 4 | T1 = ProcessRules(DataXProcessedInput); 5 | T2 = ProcessAggregateRules(DataXProcessedInput); 6 | 7 | ProcessAlerts(DataXProcessedInput); 8 | ProcessAggregateAlerts(DataXProcessedInput); 9 | 10 | --DataXQuery-- 11 | T3 = SELECT *, 42 AS AnswerToLife2 12 | FROM DataXProcessedInput 13 | TIMEWINDOW('50 seconds'); 14 | 15 | 16 | --DataXQuery-- 17 | Temp = 18 | SELECT COUNT(*) AS Count 19 | FROM DataXProcessedInput 20 | UNION ALL 21 | SELECT * 22 | FROM EventCount; 23 | 24 | --DataXQuery-- 25 | T4 = SELECT * 26 | FROM DataXProcessedInput 27 | TIMEWINDOW('40 seconds'); 28 | 29 | HeaterStateOneIsOn = CreateMetric(DataXProcessedInput, deviceDetails.status); -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/CodeGen.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Flow.CodegenRules 6 | { 7 | public static class CodeGen 8 | { 9 | public static RulesCode GenerateCode(string queryCode, string rulesData, string productName) 10 | { 11 | Engine engine = new Engine(); 12 | 13 | return engine.GenerateCode(queryCode, rulesData, productName); 14 | } 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.CodegenRules/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/OutputTemplates.xsc: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/OutputTemplates.xsd: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/OutputTemplates.xss: -------------------------------------------------------------------------------- 1 |  -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.CodegenRules/Resources/defaultOutputTemplate.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | MAP( 9 | $pivotstemplate 10 | ) AS pivots, 11 | $aggstemplate, 12 | Count AS count, 13 | MAP( 14 | 'ruleId', '$ruleId', 15 | '$tagname', '$tag', 16 | 'description', '$ruleDescription', 17 | 'severity', '$severity' 18 | ) AS result 19 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.Common.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.Common.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.Common/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.Common/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.DeleteHelper/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.DeleteHelper/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.InteractiveQuery.Tests/DataX.Flow.InteractiveQuery.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.InteractiveQuery/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.InteractiveQuery/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.Scheduler/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.Scheduler/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.SchemaInference.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference.Tests/events4.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | { 4 | "Id": "1", 5 | "Name": "Foo", 6 | "Value": { 7 | "Value": "someValue" 8 | } 9 | }, 10 | { 11 | "Id": "2", 12 | "Name": "Bar", 13 | "Value": { 14 | "Value": "someValue" 15 | } 16 | } 17 | ], 18 | [ 19 | { 20 | "Id": "3", 21 | "Name": "Foo", 22 | "Value": { 23 | "Value": "someValue" 24 | } 25 | }, 26 | { 27 | "Id": "4", 28 | "Name": "Bar", 29 | "Age": 20 30 | } 31 | ] 32 | ] -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference.Tests/events5.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "Id": "1", 4 | "Name": "Foo", 5 | "Address": { 6 | "City": "Seattle", 7 | "Street": [1, 2] 8 | } 9 | }, 10 | { 11 | "Id": 2, 12 | "Name": "Bar", 13 | "Age": 20, 14 | "Address": { 15 | "City": [ "Seattle", "Chicago" ] 16 | } 17 | }, 18 | { 19 | "Id": "3", 20 | "Name": ["Baz1", "Baz2"], 21 | "Address": { 22 | "City": "Redmond" 23 | } 24 | }, 25 | { 26 | "Id": "4", 27 | "Name": "FooBaz", 28 | "Age": 45, 29 | "Address": ["pike", "pine"] 30 | } 31 | ] -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference.Tests/events5errors.txt: -------------------------------------------------------------------------------- 1 | Conflict in schema. Key with path '.Id' has different types 2 | Conflict in schema. Key with path '.Address.City' has different types 3 | Conflict in schema. Key with path '.Name' has different types 4 | Conflict in schema. Key with path '.Address' has different types 5 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference/EventsData.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Linq; 8 | using System.Text; 9 | using System.Threading.Tasks; 10 | 11 | namespace DataX.Flow.SchemaInference 12 | { 13 | public class EventsData 14 | { 15 | public EventsData() 16 | { 17 | Events = new List(); 18 | } 19 | 20 | public List Events = null; 21 | public string EventsJson = null; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.SchemaInference/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SchemaInference/IMessageBus.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | using System.Threading.Tasks; 9 | 10 | namespace DataX.Flow.SchemaInference 11 | { 12 | public interface IMessageBus 13 | { 14 | Task GetSampleEvents(int seconds); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SqlParser.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.SqlParser.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SqlParser.Tests/usercode.txt: -------------------------------------------------------------------------------- 1 | --DataXStates-- 2 | CREATE TABLE EventCount (Count: INT); 3 | 4 | T1 = ProcessRules(DataXProcessedInput); 5 | T2 = ProcessAggregateRules(DataXProcessedInput); 6 | 7 | ProcessAlerts(DataXProcessedInput); 8 | ProcessAggregateAlerts(DataXProcessedInput); 9 | 10 | --DataXQuery-- 11 | T3 = SELECT *, 42 AS AnswerToLife2 12 | FROM DataXProcessedInput 13 | TIMEWINDOW('50 seconds'); 14 | 15 | 16 | --DataXQuery-- 17 | Temp = 18 | SELECT COUNT(*) AS Count 19 | FROM DataXProcessedInput 20 | UNION ALL 21 | SELECT * 22 | FROM EventCount; 23 | 24 | --DataXQuery-- 25 | T4 = SELECT * 26 | FROM DataXProcessedInput 27 | TIMEWINDOW('40 seconds'); 28 | 29 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SqlParser/Analyzer.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using Newtonsoft.Json.Linq; 6 | using System.Collections.Generic; 7 | 8 | namespace DataX.Flow.SqlParser 9 | { 10 | public static class Analyzer 11 | { 12 | public static JArray Analyze(string userCode, string codeGen, string inputSchema) 13 | { 14 | SqlParser parser = new SqlParser(); 15 | List td = parser.Analyze(userCode, codeGen, inputSchema); 16 | return JArray.FromObject(td); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow.SqlParser/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Flow.SqlParser/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow/PublishProfiles/Local.1Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow/PublishProfiles/Local.5Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Flow/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Utilities.Composition/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Utilities.Composition/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/DataX.Utility.ServiceCommunication/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/DataX.Utility.ServiceCommunication/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.InteractiveQueryService/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.12.10405.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.InteractiveQueryService/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/Flow.InteractiveQueryService/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.InteractiveQueryService/app.config: -------------------------------------------------------------------------------- 1 |  5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.InteractiveQueryService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | "DataX": { 9 | "ServiceEnvironment": { 10 | "EnableOneBox": false, 11 | "AppInsightsIntrumentationKey": "", 12 | "CosmosDBConfigCollectionName": "", 13 | "CosmosDBConfigConnectionString": "", 14 | "CosmosDBConfigDatabaseName": "", 15 | "TestClientId": "", 16 | "ServiceKeyVaultName": "" 17 | } 18 | }, 19 | "JwtBearerOptions": { 20 | "Audience": "", 21 | "Authority": null // JWT ISS (issuer) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.LiveDataService/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.12.10405.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.LiveDataService/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/Flow.LiveDataService/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.LiveDataService/FlowLiveDataServiceStartup.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace Flow.LiveDataService 6 | { 7 | using Microsoft.AspNetCore.Hosting; 8 | using DataX.ServiceHost.AspNetCore; 9 | using DataX.ServiceHost.AspNetCore.Startup; 10 | using DataX.Contract.Settings; 11 | 12 | /// 13 | /// StartupFilter for Flow.LiveDataService 14 | /// 15 | public class FlowLiveDataServiceStartup : DataXServiceStartup 16 | { 17 | public FlowLiveDataServiceStartup() { } 18 | public FlowLiveDataServiceStartup(DataXSettings settings) 19 | : base(settings) { } 20 | } 21 | } 22 | 23 | 24 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.LiveDataService/app.config: -------------------------------------------------------------------------------- 1 |  5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.LiveDataService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | "DataX": { 9 | "ServiceEnvironment": { 10 | "EnableOneBox": false, 11 | "AppInsightsIntrumentationKey": "", 12 | "CosmosDBConfigCollectionName": "", 13 | "CosmosDBConfigConnectionString": "", 14 | "CosmosDBConfigDatabaseName": "", 15 | "TestClientId": "", 16 | "ServiceKeyVaultName": "" 17 | } 18 | }, 19 | "JwtBearerOptions": { 20 | "Audience": "", 21 | "Authority": null // JWT ISS (issuer) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.ManagementService/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.12.10405.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.ManagementService/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/Flow.ManagementService/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.ManagementService/app.config: -------------------------------------------------------------------------------- 1 |  5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.ManagementService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | "DataX": { 9 | "ServiceEnvironment": { 10 | "EnableOneBox": false, 11 | "AppInsightsIntrumentationKey": "", 12 | "CosmosDBConfigCollectionName": "", 13 | "CosmosDBConfigConnectionString": "", 14 | "CosmosDBConfigDatabaseName": "", 15 | "MefBlobDirectory": null, 16 | "MefContainerName": null, 17 | "MefStorageAccountName": null, 18 | "ServiceKeyVaultName": "", 19 | "TestClientId": "", 20 | "LocalRoot": "", 21 | "SparkHome": "", 22 | "MetricsHttpEndpoint": "http://localhost:2020/" 23 | } 24 | }, 25 | "JwtBearerOptions": { 26 | "Audience": "", 27 | "Authority": null // JWT ISS (issuer) 28 | } 29 | } -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.SchemaInferenceService/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.12.10405.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.SchemaInferenceService/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Flow/Flow.SchemaInferenceService/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.SchemaInferenceService/app.config: -------------------------------------------------------------------------------- 1 |  5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /Services/DataX.Flow/Flow.SchemaInferenceService/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information" 5 | } 6 | }, 7 | "AllowedHosts": "*", 8 | "DataX": { 9 | "ServiceEnvironment": { 10 | "EnableOneBox": false, 11 | "AppInsightsIntrumentationKey": "", 12 | "CosmosDBConfigCollectionName": "", 13 | "CosmosDBConfigConnectionString": "", 14 | "CosmosDBConfigDatabaseName": "", 15 | "ServiceKeyVaultName": "", 16 | "TestClientId": "" 17 | } 18 | }, 19 | "JwtBearerOptions": { 20 | "Audience": "", 21 | "Authority": null // JWT ISS (issuer) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Services/DataX.Flow/GitVersion.yml: -------------------------------------------------------------------------------- 1 | next-version: 2.0 2 | mode: Mainline 3 | branches: {} 4 | ignore: 5 | sha: [] 6 | -------------------------------------------------------------------------------- /Services/DataX.Flow/README.md: -------------------------------------------------------------------------------- 1 | Getting Started 2 | For local development environment using MSI, you have to run the application using a service principal. For details, please refer to this. https://docs.microsoft.com/en-us/azure/key-vault/service-to-service-authentication -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway.Api.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Gateway/DataX.Gateway.Api.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway.Api/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Gateway/DataX.Gateway.Api/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway.Api/Validator.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | 7 | namespace DataX.Gateway.Api 8 | { 9 | internal static class Validator 10 | { 11 | public static void NotNull(object obj) 12 | { 13 | if (obj == null) 14 | { 15 | throw new ArgumentNullException(nameof(obj)); 16 | } 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway.Contract/Constants.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Gateway.Contract 6 | { 7 | /// 8 | /// Constants used by DataX.Gateway 9 | /// 10 | public static class Constants 11 | { 12 | public const string UserNameHeader = "X-DataX-UserName"; 13 | public const string UserIdHeader = "X-DataX-UserId"; 14 | public const string UserGroupsHeader = "X-DataX-UserGroups"; 15 | public const string UserRolesHeader = "X-DataX-UserRoles"; 16 | public const string ReverseProxyPort = "19081"; 17 | public const int DefaultHttpTimeoutSecs = 240; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway.Contract/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Gateway/DataX.Gateway.Contract/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway/PublishProfiles/Local.1Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway/PublishProfiles/Local.5Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Gateway/DataX.Gateway/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Services/DataX.Gateway/GitVersion.yml: -------------------------------------------------------------------------------- 1 | next-version: 2.0 2 | mode: Mainline 3 | branches: {} 4 | ignore: 5 | sha: [] 6 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.14.11009.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Metrics/DataX.Metrics.Ingestor/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/Helper/MetricItem.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Metrics.Ingestor.Helper 6 | { 7 | /// 8 | /// An object for metric input 9 | /// 10 | public class MetricItem 11 | { 12 | // Data EventTime 13 | public string EventTime; 14 | 15 | // TableName 16 | public string MetricName; 17 | 18 | // MetricValue 19 | public string Metric; 20 | 21 | // FlowId 22 | public string Product; 23 | 24 | // Optional. Only the custom metrics need this 25 | public string Pivot; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/Helper/MetricOutput.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | namespace DataX.Metrics.Ingestor.Helper 6 | { 7 | /// 8 | /// An object for metric outputs 9 | /// 10 | public class MetricOutput 11 | { 12 | // RedisKey. Format: : 13 | public string RedisKey; 14 | 15 | // Content. Format: uts:, val:, pivot1: "" 16 | public string Content; 17 | 18 | // Redis score. epoch time (utc time - epochzero) 19 | public double EpochTime; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:65218", 7 | "sslPort": 0 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "DataX.Metrics.Ingestor": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "http://localhost:5000", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Debug", 5 | "System": "Information", 6 | "Microsoft": "Information" 7 | } 8 | } 9 | } -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Ingestor/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Information" 5 | } 6 | }, 7 | "AllowedHosts": "*" 8 | } -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics.Tests/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Metrics/DataX.Metrics.Tests/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics/PublishProfiles/Local.1Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics/PublishProfiles/Local.5Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/DataX.Metrics/packages.config: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Services/DataX.Metrics/GitVersion.yml: -------------------------------------------------------------------------------- 1 | next-version: 2.0 2 | mode: Mainline 3 | branches: {} 4 | ignore: 5 | sha: [] 6 | -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/DataXAuthorizeAttribute.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using DataX.Utilities.Web; 6 | using Microsoft.AspNetCore.Authorization; 7 | using System; 8 | using System.Collections.Generic; 9 | using System.Text; 10 | using System.Threading.Tasks; 11 | 12 | namespace DataX.ServiceHost.AspNetCore.Authorization 13 | { 14 | /// 15 | public abstract class DataXAuthorizeAttribute : AuthorizeAttribute 16 | { 17 | public DataXAuthorizeAttribute() 18 | { 19 | Policy = DataXAuthConstants.PolicyPrefix; 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Requirements/DataXWriterRequirement.cs: -------------------------------------------------------------------------------- 1 | namespace DataX.ServiceHost.AspNetCore.Authorization.Requirements 2 | { 3 | using DataX.ServiceHost.Authorization.Requirements; 4 | using DataX.Contract.Settings; 5 | using DataX.Utilities.Web; 6 | using Microsoft.AspNetCore.Authorization; 7 | 8 | /// 9 | internal class DataXWriterRequirement : DataXOneBoxRequirement 10 | { 11 | public DataXWriterRequirement() { } 12 | 13 | public DataXWriterRequirement(DataXSettings settings) 14 | : base(settings) { } 15 | 16 | /// 17 | protected override bool IsAuthorized(AuthorizationHandlerContext context, DataXSettings settings) 18 | { 19 | return base.IsAuthorized(context, settings) || context.User.IsInRole(RolesCheck.WriterRoleName); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Roles/DataXReaderAttribute.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.ServiceHost.AspNetCore.Authorization.Roles 10 | { 11 | /// 12 | public class DataXReaderAttribute : DataXAuthorizeAttribute 13 | { 14 | public DataXReaderAttribute() 15 | { 16 | Policy = DataXAuthConstants.ReaderPolicyName; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/Authorization/Roles/DataXWriterAttribute.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.ServiceHost.AspNetCore.Authorization.Roles 10 | { 11 | /// 12 | public class DataXWriterAttribute : DataXAuthorizeAttribute 13 | { 14 | public DataXWriterAttribute() 15 | { 16 | Policy = DataXAuthConstants.WriterPolicyName; 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.ServiceHost/DataX.ServiceHost.AspNetCore/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.ServiceHost/DataX.ServiceHost.ServiceFabric/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.ServiceHost/DataX.ServiceHost/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.ServiceHost/DataX.ServiceHost/HostUtil.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.ServiceHost 10 | { 11 | public static class HostUtil 12 | { 13 | /// 14 | /// Returns true if Fabric_ApplicationName environment variable is not null, otherwise false if it is null. 15 | /// 16 | public static bool InServiceFabric => Environment.GetEnvironmentVariable("Fabric_ApplicationName") != null; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenService/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.SimulatedData/DataX.SimulatedData.DataGenService/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenService/Model/KafkaConnection.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | using System; 6 | using System.Collections.Generic; 7 | using System.Text; 8 | 9 | namespace DataX.SimulatedData.DataGenService.Model 10 | { 11 | public class KafkaConnection 12 | { 13 | public string BootstrapServers { get; set; } 14 | public List Topics { get; set; } 15 | public string ConnectionString { get; set; } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenServiceTest/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.SimulatedData/DataX.SimulatedData.DataGenServiceTest/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenServiceTest/testrandomexpecteddata1.json: -------------------------------------------------------------------------------- 1 | { 2 | "msgschema": { 3 | "msgversion": 1 4 | }, 5 | "sensordetails": { 6 | "timestampnano": 19781906739.7071, 7 | "timestamp": "2018-10-23T12:34:36.707747-07:00", 8 | "deviceindex": 7, 9 | "sensor1": 1.05814516314219, 10 | "sensor2": -1.94357779060657, 11 | "sensor3": 0.533396977248321, 12 | "sensor4": 0.446343162304882, 13 | "sensor5": 1.78689793301136, 14 | "sensor6": 32.3179671449205, 15 | "sensor7": 34.458588648801, 16 | "sensor8": 33.5398323957528, 17 | "sensor9": 33.0658565499195, 18 | "sensor10": 0.720475176219118, 19 | "sensor11": 0.398634478169789, 20 | "sensor12": 0.539738239506138, 21 | "sensor13": 7947, 22 | "location": "usa" 23 | } 24 | } -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenServiceTest/testrandomexpecteddata2.json: -------------------------------------------------------------------------------- 1 | { 2 | "msgschema": { "msgversion": 1 }, 3 | "sensordetails": { 4 | "sensor1": [ 1.91276269588282, 1.18322686440462, 0.11629032628438 ], 5 | "sensor2": [ 0.141055523483574, 6.3334924431208, 6.1158579057622, 9.4672448325284, 1.05989048306825 ], 6 | "sensor3": [ 122.929432440051, 76.9916197876407, 53.2928274959758, 108.071276432868, 59.7951717254683, 80.9607359259206, 108.371026352221, 9.15740831250204, 122.274286333599, 67.4668179207793 ] 7 | } 8 | } -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData.DataGenServiceTest/testrulesexpecteddata1.json: -------------------------------------------------------------------------------- 1 | { 2 | "msgschema": { "msgversion": 1 }, 3 | "sensordetails": { 4 | "timestampnano": 19781906739.7071, 5 | "timestamp": "2018-10-24T10:33:06.9337423-07:00", 6 | "deviceindex": 7, 7 | "sensor1": 1.05814516314219, 8 | "sensor2": -1.94357779060657, 9 | "sensor3": 0.533396977248321, 10 | "sensor4": 0.446343162304882, 11 | "sensor5": 1.78689793301136, 12 | "sensor6": 32.3179671449205, 13 | "sensor7": 34.458588648801, 14 | "sensor8": 33.5398323957528, 15 | "sensor9": 33.0658565499195, 16 | "sensor10": 0.720475176219118, 17 | "sensor11": 0.398634478169789, 18 | "sensor12": 0.539738239506138, 19 | "sensor13": 7947, 20 | "location": "usa" 21 | 22 | } 23 | } -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData/PublishProfiles/Cloud.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData/PublishProfiles/Local.1Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.SimulatedData/PublishProfiles/Local.5Node.xml: -------------------------------------------------------------------------------- 1 |  2 | 6 | 7 | 13 | 14 | 15 | -------------------------------------------------------------------------------- /Services/DataX.SimulatedData/DataX.ruleset: -------------------------------------------------------------------------------- 1 |  2 | -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.Blob/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.Blob/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.CosmosDB/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.CosmosDB/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.EventHub/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.EventHub/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.KeyVault/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.KeyVault/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.Telemetry/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.Telemetry/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.Tests/DataX.Utilities.Tests.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp3.1 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utilities.Web/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utilities.Web/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.Blob/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utility.Blob/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.Blob/IsCriticalException.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | using System; 7 | using System.Threading; 8 | 9 | namespace DataX.Utilities 10 | { 11 | public static class ExceptionExtensions 12 | { 13 | /// 14 | /// Returns true if an exception should not be handled by logging code. 15 | /// 16 | public static bool IsCriticalException(this Exception ex) 17 | { 18 | return ex is StackOverflowException || 19 | ex is OutOfMemoryException || 20 | ex is ThreadAbortException || 21 | ex is AccessViolationException; 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.CosmosDB/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utility.CosmosDB/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.KeyVault/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utility.KeyVault/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.Nuget/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Services/DataX.Utilities/DataX.Utility.Nuget/FinalPublicKey.snk -------------------------------------------------------------------------------- /Services/DataX.Utilities/DataX.Utility.Nuget/VerifySignedFiles.txt: -------------------------------------------------------------------------------- 1 | *.dll, some dlls are not supposed to be signed. Ignore them 2 | *.xml, Ignore xml -------------------------------------------------------------------------------- /Services/JobRunner/Connected Services/Application Insights/ConnectedService.json: -------------------------------------------------------------------------------- 1 | { 2 | "ProviderId": "Microsoft.ApplicationInsights.ConnectedService.ConnectedServiceProvider", 3 | "Version": "8.14.20131.1", 4 | "GettingStartedDocument": { 5 | "Uri": "https://go.microsoft.com/fwlink/?LinkID=798432" 6 | } 7 | } -------------------------------------------------------------------------------- /Services/JobRunner/JobQueueMessage.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | namespace JobRunner 7 | { 8 | /// 9 | /// Message object to specify a job to be run asynchronously 10 | /// 11 | internal class JobQueueMessage 12 | { 13 | public string JobKey { get; set; } 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Services/JobRunner/JobRunnerException.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | using System; 7 | 8 | namespace JobRunner 9 | { 10 | /// 11 | /// Custom Exception for JobRunner 12 | /// 13 | public class JobRunnerException : Exception 14 | { 15 | public JobRunnerException(string message) : base(message) 16 | { 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /Services/JobRunner/Jobs/IJob.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | using System.Threading.Tasks; 7 | 8 | namespace JobRunner.Jobs 9 | { 10 | public interface IJob 11 | { 12 | Task RunAsync(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /Services/JobRunner/Jobs/TestJob.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | using System; 7 | using System.Threading.Tasks; 8 | 9 | namespace JobRunner.Jobs 10 | { 11 | /// 12 | /// An example job that shows how a job could be setup 13 | /// 14 | public class TestJob : IJob 15 | { 16 | public TestJob() 17 | { 18 | } 19 | 20 | public async Task RunAsync() 21 | { 22 | Console.WriteLine("TESTJOB"); 23 | await Task.Yield(); 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Services/JobRunner/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "iisSettings": { 3 | "windowsAuthentication": false, 4 | "anonymousAuthentication": true, 5 | "iisExpress": { 6 | "applicationUrl": "http://localhost:19117", 7 | "sslPort": 44362 8 | } 9 | }, 10 | "profiles": { 11 | "IIS Express": { 12 | "commandName": "IISExpress", 13 | "launchBrowser": true, 14 | "environmentVariables": { 15 | "ASPNETCORE_ENVIRONMENT": "Development" 16 | } 17 | }, 18 | "JobRunner": { 19 | "commandName": "Project", 20 | "launchBrowser": true, 21 | "applicationUrl": "https://localhost:5001;http://localhost:5000", 22 | "environmentVariables": { 23 | "ASPNETCORE_ENVIRONMENT": "Development" 24 | } 25 | } 26 | } 27 | } -------------------------------------------------------------------------------- /Services/JobRunner/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "Logging": { 3 | "LogLevel": { 4 | "Default": "Warning" 5 | } 6 | }, 7 | "AllowedHosts": "*" 8 | } 9 | -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/Program.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | namespace JobRunner 7 | { 8 | public class Program 9 | { 10 | public static void Main(string[] args) 11 | { 12 | var config = MefContainer.BuildConfigurationForConsoleApp(); 13 | var mef = MefContainer.CreateWithConfiguration(config); 14 | mef.GetExport().RunForeverAsync().Wait(); 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/Properties/launchSettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "profiles": { 3 | "JobRunnerWebJob": { 4 | "commandName": "Project", 5 | "environmentVariables": { 6 | "ENVIRONMENT": "Development" 7 | } 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/Readme.md: -------------------------------------------------------------------------------- 1 | How to setup your own Job: 2 | 3 | 1. Create a Job class inheriting from `IJob` under the JobRunner project. 4 | 2. You need to schedule the job under `RegisterScheduledJobs` in the JobRunner project. 5 | 6 | Debug locally 7 | 1. Update the ActiveQueue in the appsettings to ensure that the correct queue is accessed in the debugging. 8 | 9 | -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/Settings.job: -------------------------------------------------------------------------------- 1 | { 2 | 3 | // Examples: 4 | 5 | // Runs every minute 6 | // "schedule": "0 * * * * *" 7 | 8 | // Runs every 15 minutes 9 | // "schedule": "0 */15 * * * *" 10 | 11 | // Runs every hour (i.e. whenever the count of minutes is 0) 12 | // "schedule": "0 0 * * * *" 13 | 14 | // Runs every hour from 9 AM to 5 PM 15 | // "schedule": "0 0 9-17 * * *" 16 | 17 | // Runs at 9:30 AM every day 18 | // "schedule": "0 30 9 * * *" 19 | 20 | // Runs at 9:30 AM every week day 21 | // "schedule": "0 30 9 * * 1-5" 22 | } -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/appsettings.Development.json: -------------------------------------------------------------------------------- 1 | { 2 | "JobRunner": { 3 | "StorageConnection": "", 4 | "ServiceBusConnectionString": "", 5 | "PrimaryQueueName": "", 6 | "TestQueueName": "", 7 | "ActiveQueueName": "", 8 | "ServiceUrl": "", 9 | "ServiceKeyVaultName": "", 10 | "AppInsightsIntrumentationKey": "", 11 | "MicrosoftAuthority": "", 12 | "ApplicationIdentifierUri": "", 13 | "ApplicationId": "", 14 | "SecretKey": "", 15 | "BlobConnectionString": "", 16 | "BlobUri": "", 17 | "EvenHubConnectionString": "", 18 | "EventHubName": "", 19 | "IsIotHub": "", 20 | "Seconds": "", 21 | "FlowName": "", 22 | "NormalizationSnippet": "", 23 | "DatabricksToken": "", 24 | "SparkType": "" 25 | } 26 | } -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/appsettings.json: -------------------------------------------------------------------------------- 1 | { 2 | "JobRunner": { 3 | "StorageConnection": "", 4 | "ServiceBusConnectionString": "", 5 | "PrimaryQueueName": "", 6 | "TestQueueName": "", 7 | "ActiveQueueName": "", 8 | "ServiceUrl": "", 9 | "ServiceKeyVaultName": "", 10 | "AppInsightsIntrumentation": "", 11 | "MicrosoftAuthority": "", 12 | "ApplicationIdentifierUri": "", 13 | "ApplicationId": "", 14 | "SecretKey": "", 15 | "BlobConnectionString": "", 16 | "BlobUri": "", 17 | "EvenHubConnectionString": "", 18 | "EventHubName": "", 19 | "IsIotHub": "", 20 | "Seconds": "", 21 | "FlowName": "", 22 | "NormalizationSnippet": "", 23 | "DatabricksToken": "", 24 | "SparkType": "" 25 | } 26 | } -------------------------------------------------------------------------------- /Services/JobRunnerWebJob/run.cmd: -------------------------------------------------------------------------------- 1 | dotnet JobRunnerWebJob.dll -------------------------------------------------------------------------------- /Services/README.md: -------------------------------------------------------------------------------- 1 | Data Accelerator Services 2 | 3 | # Config 4 | Creates the config files and manages jobs going to the cluster 5 | 6 | # Contract 7 | Definition of APIs and Constants across projects 8 | 9 | # Flow 10 | Provides schema and live query services to Flow editing experience 11 | 12 | # Gateway 13 | Manages entry into other services by authenticating and validating roles. 14 | 15 | # Metrics 16 | Pushes metrics data into the Redis cache so it can be consummed by the Web Portal 17 | 18 | # ServiceHost 19 | Layer interacting with the specific host technology 20 | 21 | # SimulatedData 22 | Simulates the data with specified schema and rules 23 | 24 | # Utilities 25 | Common code used across projects 26 | -------------------------------------------------------------------------------- /Tests/DataXScenarios/DataXScenarios/DataXScenarios.csproj: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | netcoreapp2.2 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /Tests/ScenarioTester/ScenarioTester/DontTimeAttribute.cs: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | using System; 7 | 8 | namespace ScenarioTester 9 | { 10 | /// 11 | /// Attribute to identity which steps to skip time reporting on. 12 | /// 13 | internal class DontTimeAttribute : Attribute 14 | { 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /Tests/ScenarioTester/ScenarioTester/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Tests/ScenarioTester/ScenarioTester/FinalPublicKey.snk -------------------------------------------------------------------------------- /Tests/ScenarioTester/ScenarioTester/ScenarioTester.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | ScenarioTester 6 | ScenarioTester 7 | 8 | 9 | 10 | 11 | all 12 | runtime; build; native; contentfiles; analyzers 13 | 14 | 15 | 16 | 17 | 18 | 19 | Microsoft400 20 | StrongName 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /Tests/ScenarioTester/ScenarioTesterTests/ScenarioTesterTests.csproj: -------------------------------------------------------------------------------- 1 |  2 | 3 | 4 | netcoreapp2.2 5 | 6 | false 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /Website/.gitignore: -------------------------------------------------------------------------------- 1 | project-lock.json 2 | /node_modules 3 | /logs 4 | /.vs 5 | /env.bat 6 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/common/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | /** 7 | * Common - Specify here all the common modules you want to expose externally 8 | */ 9 | import { Constants, ApiNames } from './apiConstants'; 10 | 11 | export { Constants, ApiNames }; 12 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/components/shell.jsx: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import React from 'react'; 6 | import Radium from 'radium'; 7 | import PropTypes from 'prop-types'; 8 | 9 | const Shell = Radium( 10 | class Shell extends React.Component { 11 | render() { 12 | return
{this.props.children}
; 13 | } 14 | } 15 | ); 16 | 17 | // Props 18 | Shell.propTypes = { 19 | style: PropTypes.object 20 | }; 21 | 22 | export default Shell; 23 | 24 | // Styles 25 | const rootStyle = { 26 | position: 'absolute', 27 | top: 40, 28 | bottom: 0, 29 | left: 0, 30 | right: 0, 31 | display: 'flex' 32 | }; 33 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/components/topNavSpacer.jsx: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import React from 'react'; 6 | import { Colors } from '../styles'; 7 | 8 | export default class TopNavSpacer extends React.Component { 9 | render() { 10 | return
; 11 | } 12 | } 13 | 14 | // Styles 15 | const style = { 16 | flex: 1, 17 | borderRight: `1px solid ${Colors.white}` 18 | }; 19 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/components/verticalTabItem.jsx: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import React from 'react'; 6 | import { Colors } from '../styles'; 7 | 8 | export default class VerticalTabItem extends React.Component { 9 | render() { 10 | return
{this.props.children}
; 11 | } 12 | } 13 | 14 | // Styles 15 | const rootStyle = { 16 | display: 'flex', 17 | flexDirection: 'column', 18 | flex: 1, 19 | overflowY: 'hidden', 20 | backgroundColor: Colors.neutralLighterAlt 21 | }; 22 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | require('office-ui-fabric-react/dist/css/fabric.min.css'); 7 | import './styles/styles.css'; 8 | 9 | /** 10 | * Export all modules to expose out of this package library 11 | */ 12 | export * from './components'; 13 | export * from './modules'; 14 | export * from './styles'; 15 | export * from './utils'; 16 | export * from './common'; 17 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/modules/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | /** 7 | * Modules - Specify here all the common modules you want to expose externally 8 | */ 9 | // User 10 | import * as UserActions from './user/actions'; 11 | import * as UserSelectors from './user/selectors'; 12 | export { UserActions, UserSelectors }; 13 | export { default as userReducer } from './user/reducer'; 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/modules/user/actions.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import * as Api from './api'; 6 | 7 | export const USER_RECEIVE_USER_INFO = 'USER_RECEIVE_USER_INFO'; 8 | 9 | export function getUserInfo() { 10 | return dispatch => 11 | Api.getUser().then(info => { 12 | dispatch({ 13 | type: USER_RECEIVE_USER_INFO, 14 | payload: { 15 | info: info 16 | } 17 | }); 18 | }); 19 | } 20 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/modules/user/api.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { nodeServiceGetApi } from '../../utils'; 6 | 7 | export function getUser() { 8 | return nodeServiceGetApi('user'); 9 | } 10 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/modules/user/reducer.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import * as Actions from './actions'; 6 | 7 | const INITIAL_USER_INFO_STATE = null; 8 | 9 | export default (state = INITIAL_USER_INFO_STATE, action) => { 10 | switch (action.type) { 11 | case Actions.USER_RECEIVE_USER_INFO: 12 | return Object.freeze(action.payload.info); 13 | 14 | default: 15 | return state; 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/modules/user/selectors.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { createSelector } from 'reselect'; 6 | 7 | export const getUserInfo = state => state.userInfo; 8 | 9 | export const getUserAlias = createSelector( 10 | getUserInfo, 11 | userInfo => (userInfo ? userInfo.id : '') 12 | ); 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/styles/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | /** 6 | * Styles - Specify here all the UX styles you want to expose externally 7 | */ 8 | // Colors 9 | import * as Colors from './colors'; 10 | export { Colors }; 11 | 12 | // Icon Button Styles 13 | import * as IconButtonStyles from './iconButtonStyles'; 14 | export { IconButtonStyles }; 15 | -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/utils/helpers.js: -------------------------------------------------------------------------------- 1 | 2 | // ********************************************************************* 3 | // Copyright (c) Microsoft Corporation. All rights reserved. 4 | // Licensed under the MIT License 5 | // ********************************************************************* 6 | export function isValidNumberAboveZero(value) { 7 | const number = Number(value); 8 | const isNumber = !isNaN(number); 9 | const isValid = isNumber && number > 0 && value[0] !== '0'; 10 | return isValid; 11 | } -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/utils/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | /** 7 | * Utils - Specify here all the utilities you want to expose externally 8 | */ 9 | // API support 10 | export * from './nodeServiceApi'; 11 | export * from './serviceApi'; 12 | 13 | // General common utilities 14 | export * from './utilities'; 15 | 16 | // General common helpers 17 | import * as CommonHelpers from './helpers'; 18 | export { CommonHelpers }; -------------------------------------------------------------------------------- /Website/Packages/datax-common/src/utils/utilities.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | export function getApiErrorMessage(error) { 6 | if (error && error.message) { 7 | return error.message; 8 | } else if (error) { 9 | return error; 10 | } else { 11 | return 'Error occured but service did not return information about the error'; 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-home/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-home/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-home/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-home/localdevpatch.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | REM Local Development Patch 3 | 4 | @echo off 5 | REM Patch this package's dist bits to Website's node_module/this_package_name/dist folder 6 | REM Change destination path as needed if website location is different 7 | 8 | @echo on 9 | REM Patching dist folder to Website 10 | xcopy dist ..\..\Website\node_modules\datax-home\dist /i /s /y 11 | 12 | @echo off 13 | REM Patch all other dist folders that you want to patch here 14 | REM For example, you may want to patch the feature packages that uses this package -------------------------------------------------------------------------------- /Website/Packages/datax-home/src/modules/home/models.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | export const targetTypeEnum = { 6 | cloud: 'cloud', 7 | local: 'local' 8 | }; 9 | 10 | export const targetTypeOptions = [ 11 | { 12 | key: targetTypeEnum.cloud, 13 | text: 'Cloud Tutorials' 14 | }, 15 | { 16 | key: targetTypeEnum.local, 17 | text: 'Local Tutorials' 18 | } 19 | ]; 20 | -------------------------------------------------------------------------------- /Website/Packages/datax-home/src/styles/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | ********************************************************************* 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | Licensed under the MIT License 5 | ********************************************************************* 6 | */ 7 | 8 | /* future CSS definitions goes here */ 9 | -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/localdevpatch.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | REM Local Development Patch 3 | 4 | @echo off 5 | REM Patch this package's dist bits to Website's node_module/this_package_name/dist folder 6 | REM Change destination path as needed if website location is different 7 | 8 | @echo on 9 | REM Patching dist folder to Website 10 | xcopy dist ..\..\Website\node_modules\datax-jobs\dist /i /s /y 11 | 12 | @echo off 13 | REM Patch all other dist folders that you want to patch here 14 | REM For example, you may want to patch the feature packages that uses this package -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/src/modules/jobs/constants.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | export const JobState = { 6 | Idle: 'Idle', 7 | Starting: 'Starting', 8 | Running: 'Running', 9 | Error: 'Error' 10 | }; 11 | 12 | export const Constants = { 13 | // Application name 14 | serviceApplication: 'DataX.Flow', 15 | 16 | // Service application route api 17 | serviceRouteApi: '/api/apiservice', 18 | 19 | // List of supported services 20 | services: { 21 | flow: 'Flow.ManagementService' 22 | } 23 | }; -------------------------------------------------------------------------------- /Website/Packages/datax-jobs/src/styles/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | ********************************************************************* 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | Licensed under the MIT License 5 | ********************************************************************* 6 | */ 7 | 8 | .rotate-icon { 9 | -webkit-animation: rotation 2s infinite linear; 10 | animation: rotation 2s infinite linear; 11 | } 12 | 13 | @-webkit-keyframes rotation { 14 | from { 15 | -webkit-transform: rotate(0deg); 16 | } 17 | to { 18 | -webkit-transform: rotate(359deg); 19 | } 20 | } 21 | 22 | @keyframes rotation { 23 | from { 24 | -webkit-transform: rotate(0deg); 25 | } 26 | to { 27 | -webkit-transform: rotate(359deg); 28 | } 29 | } -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/localdevpatch.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | REM Local Development Patch 3 | 4 | @echo off 5 | REM Patch this package's dist bits to Website's node_module/this_package_name/dist folder 6 | REM Change destination path as needed if website location is different 7 | 8 | @echo on 9 | REM Patching dist folder to Website 10 | xcopy dist ..\..\Website\node_modules\datax-metrics\dist /i /s /y 11 | 12 | @echo off 13 | REM Patch all other dist folders that you want to patch here 14 | REM For example, you may want to patch the feature packages that uses this package -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/src/modules/metrics/apiConstants.js: -------------------------------------------------------------------------------- 1 | export const Constants = { 2 | // Application name 3 | serviceApplication: 'DataX.Flow', 4 | 5 | // Service application route api 6 | serviceRouteApi: '/api/apiservice', 7 | 8 | // List of supported services 9 | services: { 10 | flow: 'Flow.ManagementService' 11 | }, 12 | 13 | batching: 'batching' 14 | }; 15 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/src/modules/metrics/metrics.aggregator.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | const sum = data => data.reduce((s, d) => (isNaN(d) ? s : isNaN(s) ? d : s + d), NaN); 6 | 7 | module.exports = { 8 | sum: sum, 9 | max: data => data.reduce((s, d) => (isNaN(d) ? s : isNaN(s) ? d : s > d ? s : d), NaN), 10 | count: data => data.length, 11 | average: data => { 12 | var valid = data.filter(d => !isNaN(d)); 13 | return valid.length == 0 ? NaN : sum(valid) / valid.length; 14 | }, 15 | first: data => (data.length > 0 ? data[0] : NaN) 16 | }; 17 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/src/modules/metrics/reducer.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import * as Actions from './actions'; 6 | 7 | const INITIAL_METRIC_STATE = { 8 | items: undefined 9 | }; 10 | 11 | export default (state = INITIAL_METRIC_STATE, action) => { 12 | switch (action.type) { 13 | case Actions.METRICS_RECEIVE_PRODUCTS: 14 | return Object.assign({}, state, { items: action.payload }); 15 | 16 | default: 17 | return state; 18 | } 19 | }; 20 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/src/modules/metrics/selectors.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { createSelector } from 'reselect'; 6 | 7 | export const getMetricProducts = state => state.metricProducts; 8 | 9 | export const getMetricProductItems = createSelector( 10 | getMetricProducts, 11 | metricProducts => metricProducts.items 12 | ); 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-metrics/src/styles/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | ********************************************************************* 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | Licensed under the MIT License 5 | ********************************************************************* 6 | */ 7 | 8 | /* future CSS definitions goes here */ 9 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/localdevpatch.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | REM Local Development Patch 3 | 4 | @echo off 5 | REM Patch this package's dist bits to Website's node_module/this_package_name/dist folder 6 | REM Change destination path as needed if website location is different 7 | 8 | @echo on 9 | REM Patching dist folder to Website 10 | xcopy dist ..\..\Website\node_modules\datax-pipeline\dist /i /s /y 11 | 12 | @echo off 13 | REM Patch all other dist folders that you want to patch here 14 | REM For example, you may want to patch the feature packages that uses this package -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/src/common/api.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { nodeServiceGetApi, ApiNames } from 'datax-common'; 6 | 7 | export const functionEnabled = () => nodeServiceGetApi(ApiNames.FunctionEnabled); 8 | export const isDatabricksSparkType = () => nodeServiceGetApi(ApiNames.IsDatabricksSparkType); 9 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/src/common/styles.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { Colors } from 'datax-common'; 6 | 7 | export const panelHeaderStyle = { 8 | color: Colors.neutralSecondary, 9 | backgroundColor: Colors.neutralLight, 10 | borderBottom: `1px solid ${Colors.neutralTertiaryAlt}` 11 | }; 12 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/src/modules/flowList/api.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { serviceGetApi, Constants } from 'datax-common'; 6 | 7 | export const getFlowsList = () => 8 | serviceGetApi(Constants.serviceRouteApi, Constants.serviceApplication, Constants.services.flow, 'flow/getall/min'); 9 | -------------------------------------------------------------------------------- /Website/Packages/datax-pipeline/src/modules/flowList/selectors.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { createSelector } from 'reselect'; 6 | 7 | export const getFlowsList = state => state.flowslist; 8 | 9 | export const getFlowItems = createSelector( 10 | getFlowsList, 11 | flowslist => flowslist.items 12 | ); 13 | 14 | export const getErrorMessage = createSelector( 15 | getFlowsList, 16 | flowslist => flowslist.errorMessage 17 | ); 18 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | dist 3 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/bower_components": true, 11 | "**/dist": true 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/localdevpatch.bat: -------------------------------------------------------------------------------- 1 | @echo on 2 | REM Local Development Patch 3 | 4 | @echo off 5 | REM Patch this package's dist bits to Website's node_module/this_package_name/dist folder 6 | REM Change destination path as needed if website location is different 7 | 8 | @echo on 9 | REM Patching dist folder to pipeline 10 | xcopy dist ..\datax-pipeline\node_modules\datax-query\dist /i /s /y 11 | 12 | @echo off 13 | REM Patch all other dist folders that you want to patch here 14 | REM For example, you may want to patch the feature packages that uses this package -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | 6 | import 'jsoneditor-react/es/editor.min.css'; 7 | import 'monaco-editor/min/vs/editor/editor.main.css'; 8 | import './styles/styles.css'; 9 | /** 10 | * Export all modules to expose out of this package library 11 | */ 12 | export * from './modules'; 13 | export { JsonEditor } from 'jsoneditor-react'; 14 | export { default as MonacoEditorControl} from './modules/query/components/monacoeditorcontrol' -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/modules/query/components/index.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | export { default as QuerySettingsContent } from './querySettingsContent'; 6 | export { default as SideToolBar } from './sideToolBar'; -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/modules/query/kernelSelectors.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { createSelector } from 'reselect'; 6 | 7 | export const getKernelInfo = state => state.queryKernel; 8 | 9 | export const getKernelId = createSelector( 10 | getKernelInfo, 11 | queryKernel => queryKernel.kernelId 12 | ); 13 | 14 | export const getKernelVersion = createSelector( 15 | getKernelInfo, 16 | queryKernel => queryKernel.version 17 | ); 18 | 19 | export const getFetchingKernel = createSelector( 20 | getKernelInfo, 21 | queryKernel => queryKernel.fetchingKernel 22 | ); 23 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/modules/query/layoutActions.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | export const LAYOUT_SET_TESTQUERY_OUTPUT_PANEL_VISIBILITY = 'LAYOUT_SET_TESTQUERY_OUTPUT_PANEL_VISIBILITY'; 6 | 7 | export const onShowTestQueryOutputPanel = isVisible => dispatch => { 8 | return dispatch({ 9 | type: LAYOUT_SET_TESTQUERY_OUTPUT_PANEL_VISIBILITY, 10 | payload: isVisible 11 | }); 12 | }; 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/modules/query/layoutReducer.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import * as Actions from './layoutActions'; 6 | 7 | const INITIAL_LAYOUT_STATE = { 8 | isTestQueryOutputPanelVisible: false 9 | }; 10 | 11 | export default (state = INITIAL_LAYOUT_STATE, action) => { 12 | switch (action.type) { 13 | case Actions.LAYOUT_SET_TESTQUERY_OUTPUT_PANEL_VISIBILITY: 14 | return Object.assign({}, state, { isTestQueryOutputPanelVisible: action.payload }); 15 | 16 | default: 17 | return state; 18 | } 19 | }; 20 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/modules/query/layoutSelectors.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { createSelector } from 'reselect'; 6 | 7 | export const getLayoutSettings = state => state.queryLayoutSettings; 8 | 9 | export const getTestQueryOutputPanelVisibility = createSelector( 10 | getLayoutSettings, 11 | queryLayoutSettings => queryLayoutSettings.isTestQueryOutputPanelVisible 12 | ); 13 | -------------------------------------------------------------------------------- /Website/Packages/datax-query/src/styles/styles.css: -------------------------------------------------------------------------------- 1 | /* 2 | ********************************************************************* 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | Licensed under the MIT License 5 | ********************************************************************* 6 | */ 7 | 8 | /* Query Pane styles */ 9 | .query-pane-button { 10 | height: 27px !important; 11 | padding-left: 5px !important; 12 | padding-right: 5px !important; 13 | min-width: 1px !important; 14 | } 15 | 16 | .ms-IconExample:hover { 17 | cursor: pointer !important; 18 | } 19 | 20 | .query-pane-TextField{ 21 | height: 27px !important; 22 | width: 50px !important; 23 | } 24 | 25 | .query-pane-TextField .ms-TextField-fieldGroup { 26 | height: 27px !important; 27 | } -------------------------------------------------------------------------------- /Website/README.md: -------------------------------------------------------------------------------- 1 | # Website 2 | 3 | This is the DataX Portal 4 | 5 | # Packages 6 | 7 | These are the required components for the DataX Portal. 8 | -------------------------------------------------------------------------------- /Website/Website/.deployment: -------------------------------------------------------------------------------- 1 | [config] 2 | SCM_DO_BUILD_DURING_DEPLOYMENT=true -------------------------------------------------------------------------------- /Website/Website/.prettierrc: -------------------------------------------------------------------------------- 1 | { 2 | "printWidth": 140, 3 | "tabWidth": 4, 4 | "useTabs": false, 5 | "semi": true, 6 | "singleQuote": true, 7 | "jsxSingleQuote": false, 8 | "trailingComma": "none", 9 | "bracketSpacing": true, 10 | "jsxBracketSameLine": false, 11 | "arrowParens": "avoid" 12 | } 13 | -------------------------------------------------------------------------------- /Website/Website/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible Node.js debug attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Launch Web Site", 11 | "program": "${workspaceRoot}\\server.js" 12 | }, 13 | { 14 | "type": "node", 15 | "request": "launch", 16 | "name": "Launch Web Site with Production Dist", 17 | "program": "${workspaceRoot}\\server.js", 18 | "args":[ 19 | "--dist=prod" 20 | ] 21 | } 22 | ] 23 | } -------------------------------------------------------------------------------- /Website/Website/.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "[javascript]": { 3 | "editor.formatOnSave": true 4 | }, 5 | "[javascriptreact]": { 6 | "editor.formatOnSave": true 7 | }, 8 | "search.exclude": { 9 | "**/node_modules": true, 10 | "**/dist": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /Website/Website/client/css/global.css: -------------------------------------------------------------------------------- 1 | /* Global defaults */ 2 | body, html, #app { 3 | padding: 0px; 4 | margin: 0px; 5 | height: 100%; 6 | } 7 | 8 | body { 9 | font-family: Helvetica,-apple-system,BlinkMacSystemFont,Roboto,"Helvetica Neue",Arial,Verdana,sans-serif; 10 | font-size: 14px; 11 | } 12 | 13 | a, a:visited, a:active { 14 | color: inherit; 15 | text-decoration: none; 16 | } 17 | 18 | a:hover { 19 | color: inherit; 20 | text-decoration: underline; 21 | } 22 | 23 | h1 { 24 | font-weight: unset; 25 | } 26 | -------------------------------------------------------------------------------- /Website/Website/client/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Website/Website/client/favicon.ico -------------------------------------------------------------------------------- /Website/Website/client/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | Data Accelerator 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 |
19 | 20 | -------------------------------------------------------------------------------- /Website/Website/client/js/main.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import React from 'react'; 6 | import ReactDOM from 'react-dom'; 7 | import App from 'app'; 8 | import '../index.html'; 9 | import '../favicon.ico'; 10 | import '../css/global.css'; 11 | 12 | window.onload = () => { 13 | ReactDOM.render(, document.getElementById('main')); 14 | }; 15 | -------------------------------------------------------------------------------- /Website/Website/data/api.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | import { nodeServiceGetApi } from 'datax-common'; 6 | 7 | export function getWebComposition() { 8 | return nodeServiceGetApi('web-composition'); 9 | } 10 | -------------------------------------------------------------------------------- /Website/Website/deploy/DataX.Utilities/DataX.Utility.CodeSign/FinalPublicKey.snk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/microsoft/data-accelerator/924ca069622bde18a80e30f776c960a62222b5c2/Website/Website/deploy/DataX.Utilities/DataX.Utility.CodeSign/FinalPublicKey.snk -------------------------------------------------------------------------------- /Website/Website/deploy/DataX.Utilities/VerifySignedFiles.txt: -------------------------------------------------------------------------------- 1 | *.js,ignore js files -------------------------------------------------------------------------------- /Website/Website/deploy/deploy.cmd: -------------------------------------------------------------------------------- 1 | @rem node deploy/zipsite.js 2 | set webappName=%1 3 | set subscriptionName=%2 4 | if [%3] == [] (set resourceGroup=DataX) else (set resourceGroup=%3) 5 | 6 | :deploy 7 | cmd /c az account set -s %subscriptionName% 8 | cmd /c az webapp deployment source config-zip --resource-group %resourceGroup% --name %webappName% --src deploy/deployment.zip 9 | cmd /c az webapp restart --resource-group %resourceGroup% --name %webappName% 10 | 11 | :end 12 | @echo done. -------------------------------------------------------------------------------- /Website/Website/host.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | const express = require('express'); 6 | const http = require('http'); 7 | const app = express(); 8 | const server = http.Server(app); 9 | 10 | const host = { 11 | app: app, 12 | server: server 13 | }; 14 | 15 | module.exports = host; 16 | -------------------------------------------------------------------------------- /Website/Website/iisnode.yml: -------------------------------------------------------------------------------- 1 | loggingEnabled: true 2 | #devErrorsEnabled: true -------------------------------------------------------------------------------- /Website/Website/metrics/redisconsole.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | const config = require('../config'); 6 | const redisServer = config.env.redisData.server; 7 | require('redis-console')(redisServer); 8 | -------------------------------------------------------------------------------- /Website/Website/metrics/schemaHandlers.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | var handlers = []; 6 | 7 | module.exports = { 8 | forSchema: function(schema) { 9 | return handlers.find(function(handler) { 10 | return handler.canHandle(schema); 11 | }); 12 | }, 13 | 14 | register: function(handler) { 15 | handlers.push(handler); 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /Website/Website/util/common.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | module.exports = { 6 | error: function(msg) { 7 | throw new Error(msg); 8 | } 9 | }; 10 | -------------------------------------------------------------------------------- /Website/Website/util/consolelogger.js: -------------------------------------------------------------------------------- 1 | // ********************************************************************* 2 | // Copyright (c) Microsoft Corporation. All rights reserved. 3 | // Licensed under the MIT License 4 | // ********************************************************************* 5 | function format(msg) { 6 | return new Date().toISOString() + '-' + msg; 7 | } 8 | 9 | module.exports = { 10 | info: msg => { 11 | console.info(format(msg)); 12 | }, 13 | error: msg => { 14 | console.error(format(msg)); 15 | } 16 | }; 17 | --------------------------------------------------------------------------------