├── ingest └── src │ ├── test │ ├── resources │ │ ├── empty.csv │ │ ├── mockito-extensions │ │ │ └── org.mockito.plugins.MockMaker │ │ ├── dataset.csv.gz │ │ ├── dataset.json.gz │ │ ├── testdata.json.gz │ │ ├── simplelogger.properties │ │ ├── testdata.csv │ │ ├── dataset.json │ │ ├── testdata.json │ │ ├── dataset_mapping.json │ │ └── dataset.csv │ └── java │ │ └── com │ │ └── microsoft │ │ └── azure │ │ └── kusto │ │ └── ingest │ │ ├── TestDataItem.java │ │ ├── MockTimeProvider.java │ │ ├── CloseableByteArrayInputStream.java │ │ ├── source │ │ ├── SourceInfoTest.java │ │ └── ResultSetSourceInfoTest.java │ │ ├── ManagedStreamingQueuingPolicyTest.java │ │ ├── result │ │ └── IngestionStatusTest.java │ │ ├── IngestClientBaseTest.java │ │ └── TestUtils.java │ └── main │ └── java │ └── com │ └── microsoft │ └── azure │ └── kusto │ └── ingest │ ├── utils │ ├── TimeProvider.java │ ├── RandomProvider.java │ ├── SystemTimeProvider.java │ ├── DefaultRandomProvider.java │ ├── TableWithSas.java │ └── SecurityUtils.java │ ├── source │ ├── CompressionType.java │ ├── SourceInfo.java │ ├── AbstractSourceInfo.java │ ├── FileSourceInfo.java │ ├── ResultSetSourceInfo.java │ ├── StreamSourceInfo.java │ └── BlobSourceInfo.java │ ├── resources │ ├── ResourceWithSas.java │ ├── ContainerWithSas.java │ └── QueueWithSas.java │ ├── Commands.java │ ├── MappingConsts.java │ ├── QueuedIngestClient.java │ ├── exceptions │ ├── IngestionClientException.java │ └── IngestionServiceException.java │ ├── result │ ├── IngestionFailureInfo.java │ ├── IngestionResult.java │ ├── IngestionStatusResult.java │ ├── OperationStatus.java │ ├── TableReportIngestionResult.java │ ├── IngestionStatusInTableDescription.java │ └── ValidationPolicy.java │ ├── ResettableFileInputStream.java │ ├── IngestionResourceManager.java │ ├── TransformationMethod.java │ └── ManagedStreamingQueuingPolicy.java ├── data └── src │ ├── main │ ├── resources │ │ └── app.properties │ └── java │ │ └── com │ │ └── microsoft │ │ └── azure │ │ └── kusto │ │ └── data │ │ ├── KustoCheckedFunction.java │ │ ├── instrumentation │ │ ├── TraceableAttributes.java │ │ ├── FunctionOneException.java │ │ ├── SupplierOneException.java │ │ ├── SupplierTwoExceptions.java │ │ ├── FunctionTwoExceptions.java │ │ └── Tracer.java │ │ ├── exceptions │ │ ├── ParseException.java │ │ ├── JsonPropertyMissingException.java │ │ ├── ThrottleException.java │ │ ├── IODataServiceException.java │ │ ├── KustoClientInvalidConnectionStringException.java │ │ ├── KustoDataExceptionBase.java │ │ ├── WebException.java │ │ ├── DataServiceException.java │ │ ├── DataClientException.java │ │ ├── DataWebException.java │ │ ├── ExceptionUtils.java │ │ ├── OneApiError.java │ │ └── KustoServiceQueryError.java │ │ ├── WellKnownDataSet.java │ │ ├── auth │ │ ├── KeywordData.java │ │ ├── endpoints │ │ │ ├── MatchResult.java │ │ │ ├── MatchRule.java │ │ │ └── WellKnownKustoEndpointsData.java │ │ ├── AccessTokenTokenProvider.java │ │ ├── TokenCredentialProvider.java │ │ ├── AzureCliTokenProvider.java │ │ ├── AsyncCallbackTokenProvider.java │ │ ├── DeviceAuthTokenProvider.java │ │ ├── CallbackTokenProvider.java │ │ ├── ManagedIdentityTokenProvider.java │ │ ├── UserPromptTokenProvider.java │ │ ├── ApplicationKeyTokenProvider.java │ │ ├── SubjectNameIssuerTokenProvider.java │ │ ├── ApplicationCertificateTokenProvider.java │ │ ├── TokenProviderBase.java │ │ ├── HttpClientWrapper.java │ │ ├── CloudDependentTokenProviderBase.java │ │ ├── KnownKeywords.java │ │ ├── KcsbKeywords.java │ │ └── ConfidentialAppTokenProviderBase.java │ │ ├── http │ │ ├── HttpStatus.java │ │ ├── UncloseableStream.java │ │ ├── CloseParentResourcesStream.java │ │ ├── HttpClientFactory.java │ │ └── HttpTracing.java │ │ ├── req │ │ ├── RequestUtils.java │ │ └── KustoRequestContext.java │ │ ├── IngestionSourceStorage.java │ │ ├── format │ │ ├── CslLongFormat.java │ │ ├── CslIntFormat.java │ │ ├── CslRealFormat.java │ │ ├── CslUuidFormat.java │ │ ├── CslBoolFormat.java │ │ ├── CslFormat.java │ │ ├── CslTimespanFormat.java │ │ └── CslStringFormat.java │ │ ├── res │ │ ├── JsonResult.java │ │ └── ResponseState.java │ │ ├── KustoResultColumn.java │ │ ├── CommandType.java │ │ ├── Results.java │ │ ├── Ensure.java │ │ ├── KustoType.java │ │ ├── ExponentialRetry.java │ │ └── KustoResultColumnPopulator.java │ └── test │ ├── resources │ ├── simplelogger.properties │ ├── key.pem │ └── cert.cer │ └── java │ └── com │ └── microsoft │ └── azure │ └── kusto │ └── data │ ├── http │ ├── HttpClientFactoryTest.java │ └── TestHttpResponse.java │ ├── auth │ └── KeyCert.java │ └── KustoDateTimeTest.java ├── .github ├── pull_request_template.md ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ ├── build.yml │ └── codeql-analysis.yml ├── samples ├── src │ └── main │ │ ├── resources │ │ ├── dataset.csv.gz │ │ ├── dataset.jsonz.gz │ │ ├── dataset.json │ │ └── dataset.csv │ │ └── java │ │ ├── InstantSerializerWithMilliSecondPrecision.java │ │ ├── Jmeter │ │ └── jmeter_test_load.properties │ │ ├── Proxy.java │ │ ├── Query.java │ │ ├── TableStatus.java │ │ └── FileIngestion.java └── pom.xml ├── codecov.yml ├── CODE_OF_CONDUCT.md ├── quickstart ├── dataset.json ├── oneclick_instruction_box.md ├── dataset.csv ├── README.md └── kusto_sample_config.json ├── .gitignore ├── UPDATE.md ├── LICENSE └── SECURITY.md /ingest/src/test/resources/empty.csv: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/src/main/resources/app.properties: -------------------------------------------------------------------------------- 1 | version=${project.version} -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ### Added 2 | ### Changed 3 | ### Fixed 4 | -------------------------------------------------------------------------------- /ingest/src/test/resources/mockito-extensions/org.mockito.plugins.MockMaker: -------------------------------------------------------------------------------- 1 | mock-maker-inline 2 | -------------------------------------------------------------------------------- /ingest/src/test/resources/dataset.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-kusto-java/HEAD/ingest/src/test/resources/dataset.csv.gz -------------------------------------------------------------------------------- /ingest/src/test/resources/dataset.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-kusto-java/HEAD/ingest/src/test/resources/dataset.json.gz -------------------------------------------------------------------------------- /ingest/src/test/resources/testdata.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-kusto-java/HEAD/ingest/src/test/resources/testdata.json.gz -------------------------------------------------------------------------------- /samples/src/main/resources/dataset.csv.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-kusto-java/HEAD/samples/src/main/resources/dataset.csv.gz -------------------------------------------------------------------------------- /samples/src/main/resources/dataset.jsonz.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Azure/azure-kusto-java/HEAD/samples/src/main/resources/dataset.jsonz.gz -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/TimeProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.utils; 2 | 3 | public interface TimeProvider { 4 | long currentTimeMillis(); 5 | } 6 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: 4 | default: 5 | informational: true 6 | patch: 7 | default: 8 | informational: true 9 | github_checks: 10 | annotations: false 11 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/RandomProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.utils; 2 | 3 | import java.util.List; 4 | 5 | public interface RandomProvider { 6 | void shuffle(List list); 7 | } 8 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/CompressionType.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | public enum CompressionType { 7 | gz, zip 8 | } 9 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/KustoCheckedFunction.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | @FunctionalInterface 4 | public interface KustoCheckedFunction { 5 | R apply(T t) throws E1, E2; 6 | } 7 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/TraceableAttributes.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | import java.util.Map; 4 | 5 | public interface TraceableAttributes { 6 | Map getTracingAttributes(); 7 | } 8 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/resources/ResourceWithSas.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.resources; 2 | 3 | public interface ResourceWithSas { 4 | String getEndpointWithoutSas(); 5 | 6 | String getAccountName(); 7 | 8 | T getResource(); 9 | } 10 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/SystemTimeProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.utils; 2 | 3 | public class SystemTimeProvider implements TimeProvider { 4 | @Override 5 | public long currentTimeMillis() { 6 | return System.currentTimeMillis(); 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/ParseException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import com.azure.core.exception.AzureException; 4 | 5 | public class ParseException extends AzureException { 6 | 7 | public ParseException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/WellKnownDataSet.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data; 5 | 6 | public enum WellKnownDataSet { 7 | PrimaryResult, 8 | QueryCompletionInformation, 9 | TableOfContents, 10 | QueryProperties 11 | } 12 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/KeywordData.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | public class KeywordData { 4 | public String name; 5 | public String type; 6 | public boolean secret; 7 | public String[] aliases; 8 | 9 | // For Deserialization 10 | public KeywordData() { 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/MatchResult.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth.endpoints; 2 | 3 | public class MatchResult { 4 | Boolean isMatch; 5 | MatchRule matcher; 6 | 7 | public MatchResult(Boolean isMatch, MatchRule matcher) { 8 | this.isMatch = isMatch; 9 | this.matcher = matcher; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/JsonPropertyMissingException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import com.azure.core.exception.AzureException; 4 | 5 | public class JsonPropertyMissingException extends AzureException { 6 | 7 | public JsonPropertyMissingException(String message) { 8 | super(message); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/ThrottleException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | public class ThrottleException extends DataServiceException { 4 | public static final String ERROR_MESSAGE = "Request was throttled, too many requests."; 5 | 6 | public ThrottleException(String ingestionSource) { 7 | super(ingestionSource, ERROR_MESSAGE, false); 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /samples/src/main/java/InstantSerializerWithMilliSecondPrecision.java: -------------------------------------------------------------------------------- 1 | import com.fasterxml.jackson.datatype.jsr310.ser.InstantSerializer; 2 | 3 | import java.time.format.DateTimeFormatterBuilder; 4 | 5 | public class InstantSerializerWithMilliSecondPrecision extends InstantSerializer { 6 | 7 | public InstantSerializerWithMilliSecondPrecision() { 8 | super(InstantSerializer.INSTANCE, false, new DateTimeFormatterBuilder().appendInstant(3).toFormatter()); 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Microsoft Open Source Code of Conduct 2 | 3 | This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). 4 | 5 | Resources: 6 | 7 | - [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/) 8 | - [Microsoft Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) 9 | - Contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with questions or concerns 10 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/Commands.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | class Commands { 7 | public static final String INGESTION_RESOURCES_SHOW_COMMAND = ".show ingestion resources"; 8 | public static final String IDENTITY_GET_COMMAND = ".get kusto identity token"; 9 | public static final String VERSION_SHOW_COMMAND = ".show version"; 10 | } 11 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/SourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import java.util.UUID; 7 | 8 | public interface SourceInfo { 9 | /** 10 | * Checks that this SourceInfo is defined appropriately. 11 | */ 12 | void validate(); 13 | 14 | UUID getSourceId(); 15 | 16 | void setSourceId(UUID sourceId); 17 | } 18 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/http/HttpStatus.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | public class HttpStatus { 4 | public static final int OK = 200; 5 | public static final int FOUND = 302; 6 | public static final int TEMP_REDIRECT = 207; 7 | public static final int UNAUTHORIZED = 401; 8 | public static final int FORBIDDEN = 403; 9 | public static final int NOT_FOUND = 404; 10 | public static final int TOO_MANY_REQS = 429; 11 | } 12 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/FunctionOneException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | /** 4 | * Supplier is a functional interface that can be used to execute data operations. 5 | * @param 6 | * @param 7 | */ 8 | public interface FunctionOneException { 9 | /** 10 | * Execute a data operation. 11 | * @param arg 12 | * @return T 13 | * @throws V 14 | */ 15 | T apply(U arg) throws V; 16 | } 17 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/SupplierOneException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | /** 4 | * Supplier is a functional interface that can be used to execute data operations. 5 | * @param 6 | * @param 7 | */ 8 | @FunctionalInterface 9 | public interface SupplierOneException { 10 | /** 11 | * Execute a data operation. 12 | * 13 | * @return T 14 | * @throws U 15 | */ 16 | T get() throws U; 17 | } 18 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/TestDataItem.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | import java.io.File; 7 | 8 | public class TestDataItem { 9 | public File file; 10 | public IngestionProperties ingestionProperties; 11 | public int rows; 12 | public boolean testOnstreamingIngestion = true; 13 | public boolean testOnManaged = true; 14 | public boolean testReportMethodTable = false; 15 | } 16 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/req/RequestUtils.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.req; 2 | 3 | import com.azure.core.util.Context; 4 | 5 | import java.time.Duration; 6 | 7 | public class RequestUtils { 8 | public static Context contextWithTimeout(Duration timeout) { 9 | // See https://github.com/Azure/azure-sdk-for-java/blob/azure-core-http-netty_1.10.2/sdk/core/azure-core-http-netty/CHANGELOG.md#features-added 10 | return Context.NONE.addData("azure-response-timeout", timeout); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /data/src/test/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | # SLF4J's SimpleLogger configuration file 2 | # Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. 3 | 4 | # Default logging detail level for all instances of SimpleLogger. 5 | # Must be one of ("trace", "debug", "info", "warn", or "error"). 6 | # If not specified, defaults to "info". 7 | org.slf4j.simpleLogger.log.com.microsoft.azure.kusto=debug 8 | #org.slf4j.simpleLogger.showDateTime=true 9 | #org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z -------------------------------------------------------------------------------- /ingest/src/test/resources/simplelogger.properties: -------------------------------------------------------------------------------- 1 | # SLF4J's SimpleLogger configuration file 2 | # Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. 3 | 4 | # Default logging detail level for all instances of SimpleLogger. 5 | # Must be one of ("trace", "debug", "info", "warn", or "error"). 6 | # If not specified, defaults to "info". 7 | org.slf4j.simpleLogger.log.com.microsoft.azure.kusto=debug 8 | #org.slf4j.simpleLogger.showDateTime=true 9 | #org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/SupplierTwoExceptions.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | /** 4 | * Supplier is a functional interface that can be used to execute data operations. 5 | * @param 6 | * @param 7 | * @param 8 | */ 9 | @FunctionalInterface 10 | public interface SupplierTwoExceptions { 11 | /** 12 | * Execute a data operation. 13 | * @return T 14 | * @throws U1 15 | * @throws U2 16 | */ 17 | T get() throws U1, U2; 18 | } 19 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/IODataServiceException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import com.microsoft.azure.kusto.data.Utils; 4 | 5 | import java.io.IOException; 6 | 7 | public class IODataServiceException extends DataServiceException { 8 | public IODataServiceException(String ingestionSource, IOException e, String kind) { 9 | super(ingestionSource, String.format("IOException in %s post request: %s", kind, e.getMessage()), 10 | e, 11 | !Utils.isRetriableIOException(e)); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/IngestionSourceStorage.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | 5 | class IngestionSourceStorage { 6 | public String sourceUri; 7 | 8 | public IngestionSourceStorage(String uri) { 9 | sourceUri = uri; 10 | } 11 | 12 | public String toString() { 13 | try { 14 | return Utils.getObjectMapper().writeValueAsString(this); 15 | } catch (JsonProcessingException e) { 16 | throw new RuntimeException(e); 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/FunctionTwoExceptions.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | /** 4 | * FunctionTwoExceptions is a functional interface that can be used to execute data operations. 5 | * @param 6 | * @param 7 | * @param 8 | */ 9 | public interface FunctionTwoExceptions { 10 | /** 11 | * Execute a data operation. 12 | * @param arg 13 | * @return T 14 | * @throws V1 15 | * @throws V2 16 | */ 17 | T apply(U arg) throws V1, V2; 18 | } 19 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/DefaultRandomProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.utils; 2 | 3 | import java.util.Collections; 4 | import java.util.List; 5 | import java.util.Random; 6 | 7 | public class DefaultRandomProvider implements RandomProvider { 8 | public Random random; 9 | 10 | public DefaultRandomProvider(Random random) { 11 | this.random = random; 12 | } 13 | 14 | public DefaultRandomProvider() { 15 | this.random = new Random(); 16 | } 17 | 18 | @Override 19 | public void shuffle(List list) { 20 | Collections.shuffle(list, random); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/MockTimeProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import com.microsoft.azure.kusto.ingest.utils.TimeProvider; 4 | 5 | public class MockTimeProvider implements TimeProvider { 6 | private long currentTimeMillis; 7 | 8 | public MockTimeProvider(long currentTimeMillis) { 9 | this.currentTimeMillis = currentTimeMillis; 10 | } 11 | 12 | @Override 13 | public long currentTimeMillis() { 14 | return currentTimeMillis; 15 | } 16 | 17 | public void setCurrentTimeMillis(long currentTimeMillis) { 18 | this.currentTimeMillis = currentTimeMillis; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslLongFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | public class CslLongFormat extends CslFormat { 6 | private final Long value; 7 | 8 | public CslLongFormat(long value) { 9 | this.value = value; 10 | } 11 | 12 | @Override 13 | public String getType() { 14 | return "long"; 15 | } 16 | 17 | @Override 18 | public Long getValue() { 19 | return value; 20 | } 21 | 22 | @Override 23 | String getValueAsString() { 24 | Ensure.argIsNotNull(value, "value"); 25 | 26 | return Long.toString(value); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoClientInvalidConnectionStringException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | /** 4 | * Raised when Kusto client is initialized with an invalid endpoint 5 | */ 6 | public class KustoClientInvalidConnectionStringException extends DataClientException { 7 | public KustoClientInvalidConnectionStringException(Exception e) { 8 | super(e); 9 | } 10 | 11 | public KustoClientInvalidConnectionStringException(String msg) { 12 | super(msg); 13 | } 14 | 15 | public KustoClientInvalidConnectionStringException(String clusterURL, String msg, Exception e) { 16 | super(clusterURL, msg, e); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslIntFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | public class CslIntFormat extends CslFormat { 6 | private final Integer value; 7 | 8 | public CslIntFormat(int value) { 9 | this.value = value; 10 | } 11 | 12 | @Override 13 | public String getType() { 14 | return "int"; 15 | } 16 | 17 | @Override 18 | public Integer getValue() { 19 | return value; 20 | } 21 | 22 | @Override 23 | String getValueAsString() { 24 | Ensure.argIsNotNull(value, "value"); 25 | 26 | return Integer.toString(value); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /ingest/src/test/resources/testdata.csv: -------------------------------------------------------------------------------- 1 | demo1, 091231 2 | demo11, 091232 3 | demo12, 091233 4 | demo13, 091234 5 | demo14, 091235 6 | demo15, 091236 7 | demo16, 091237 8 | demo17, 091238 9 | demo18, 091239 10 | demo19, 091230 11 | demo10, 0912311 12 | demo11, 0912322 13 | demo12, 0912333 14 | demo13, 0912344 15 | demo14, 0912355 16 | demo15, 0912366 17 | demo16, 0912377 18 | demo17, 0912388 19 | demo18, 0912399 20 | demo19, 0912300 21 | demo10, 0912113 22 | demo11, 0912223 23 | demo12, 0912333 24 | demo13, 0912443 25 | demo14, 0912553 26 | demo15, 0912663 27 | demo16, 0912773 28 | demo17, 0912883 29 | demo18, 0912399 30 | demo19, 0912003 31 | demo10, 091231 32 | demo11, 091232 33 | demo12, 091233 34 | demo13, 091234 35 | demo14, 091235 -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslRealFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | public class CslRealFormat extends CslFormat { 6 | private final Double value; 7 | 8 | public CslRealFormat(double value) { 9 | this.value = value; 10 | } 11 | 12 | @Override 13 | public String getType() { 14 | return "real"; 15 | } 16 | 17 | @Override 18 | public Double getValue() { 19 | return value; 20 | } 21 | 22 | @Override 23 | String getValueAsString() { 24 | Ensure.argIsNotNull(value, "value"); 25 | 26 | return Double.toString(value); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/res/JsonResult.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.res; 2 | 3 | public class JsonResult { 4 | private String result; 5 | private String endpoint; 6 | 7 | public JsonResult(String result, String endpoint) { 8 | this.result = result; 9 | this.endpoint = endpoint; 10 | } 11 | 12 | public String getResult() { 13 | return result; 14 | } 15 | 16 | public void setResult(String result) { 17 | this.result = result; 18 | } 19 | 20 | public String getEndpoint() { 21 | return endpoint; 22 | } 23 | 24 | public void setEndpoint(String endpoint) { 25 | this.endpoint = endpoint; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /samples/src/main/java/Jmeter/jmeter_test_load.properties: -------------------------------------------------------------------------------- 1 | #Authentication 2 | appKey= 3 | appId= 4 | cluster= 5 | ingestCluster= 6 | tenant= 7 | database= 8 | #Can provide a query without spaces - i.e table|take(100) 9 | queryTable= 10 | 11 | threadCountQuery=400 12 | rampUpSecondsQuery=800 13 | #expect query to run 4s 14 | threadCountQueryStreaming=200 15 | rampUpSecondsQueryStreaming=800 16 | #expect query streaming to run 2 queries in 4s each 17 | ingestThreadCount=80 18 | ingestRampUpSeconds=800 19 | #expect data to move in 20 | queueThreadCount=200 21 | queueRampUpSeconds=800 22 | # only queuing takes 500ms - so each test is running 16 ingestions = 8s 23 | #durationSeconds=600 24 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslUuidFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | import java.util.UUID; 6 | 7 | public class CslUuidFormat extends CslFormat { 8 | private final UUID value; 9 | 10 | public CslUuidFormat(UUID value) { 11 | this.value = value; 12 | } 13 | 14 | @Override 15 | public String getType() { 16 | return "guid"; 17 | } 18 | 19 | @Override 20 | public UUID getValue() { 21 | return value; 22 | } 23 | 24 | @Override 25 | String getValueAsString() { 26 | Ensure.argIsNotNull(value, "value"); 27 | 28 | return value.toString(); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslBoolFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | public class CslBoolFormat extends CslFormat { 6 | private final Boolean value; 7 | 8 | public CslBoolFormat(boolean value) { 9 | this.value = value; 10 | } 11 | 12 | @Override 13 | public String getType() { 14 | return "bool"; 15 | } 16 | 17 | @Override 18 | public Boolean getValue() { 19 | return value; 20 | } 21 | 22 | @Override 23 | String getValueAsString() { 24 | Ensure.argIsNotNull(value, "value"); 25 | 26 | return Boolean.TRUE.equals(value) ? "true" : "false"; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /quickstart/dataset.json: -------------------------------------------------------------------------------- 1 | {"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""} 2 | {"rownumber": 1, "rowguid": "00000001-0000-0000-0001-020304050607", "xdouble": 1.00001, "xfloat": 1.01, "xbool": 1, "xint16": 1, "xint32": 1, "xint64": 1, "xuint8": 1, "xuint16": 1, "xuint32": 1, "xuint64": 1, "xdate": "2015-01-01T01:01:01Z", "xsmalltext": "One", "xtext": "One", "xnumberAsText": "1", "xtime": "00:00:01.0010001", "xtextWithNulls": null, "xdynamicWithNulls": "{\"rowId\":1,\"arr\":[0,1]}"} -------------------------------------------------------------------------------- /ingest/src/test/resources/dataset.json: -------------------------------------------------------------------------------- 1 | {"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""} 2 | {"rownumber": 1, "rowguid": "00000001-0000-0000-0001-020304050607", "xdouble": 1.00001, "xfloat": 1.01, "xbool": 1, "xint16": 1, "xint32": 1, "xint64": 1, "xuint8": 1, "xuint16": 1, "xuint32": 1, "xuint64": 1, "xdate": "2015-01-01T01:01:01Z", "xsmalltext": "One", "xtext": "One", "xnumberAsText": "1", "xtime": "00:00:01.0010001", "xtextWithNulls": null, "xdynamicWithNulls": "{\"rowId\":1,\"arr\":[0,1]}"} -------------------------------------------------------------------------------- /samples/src/main/resources/dataset.json: -------------------------------------------------------------------------------- 1 | {"rownumber": 0, "rowguid": "00000000-0000-0000-0001-020304050607", "xdouble": 0.0, "xfloat": 0.0, "xbool": 0, "xint16": 0, "xint32": 0, "xint64": 0, "xunit8": 0, "xuint16": 0, "xunit32": 0, "xunit64": 0, "xdate": "2014-01-01T01:01:01Z", "xsmalltext": "Zero", "xtext": "Zero", "xnumberAsText": "0", "xtime": "00:00:00", "xtextWithNulls": null, "xdynamicWithNulls": ""} 2 | {"rownumber": 1, "rowguid": "00000001-0000-0000-0001-020304050607", "xdouble": 1.00001, "xfloat": 1.01, "xbool": 1, "xint16": 1, "xint32": 1, "xint64": 1, "xuint8": 1, "xuint16": 1, "xuint32": 1, "xuint64": 1, "xdate": "2015-01-01T01:01:01Z", "xsmalltext": "One", "xtext": "One", "xnumberAsText": "1", "xtime": "00:00:01.0010001", "xtextWithNulls": null, "xdynamicWithNulls": "{\"rowId\":1,\"arr\":[0,1]}"} -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/MappingConsts.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | public enum MappingConsts { 7 | // Json Mapping consts 8 | PATH("Path"), 9 | TRANSFORMATION_METHOD("Transform"), 10 | // csv Mapping consts 11 | ORDINAL("Ordinal"), 12 | CONST_VALUE("ConstValue"), 13 | // Avro Mapping consts 14 | FIELD_NAME("Field"), 15 | COLUMNS("Columns"), 16 | // General Mapping consts 17 | STORAGE_DATA_TYPE("StorageDataType"); 18 | 19 | private String name; 20 | 21 | MappingConsts(String name) { 22 | this.name = name; 23 | } 24 | 25 | String getName() { 26 | return name; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/KustoResultColumn.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data; 5 | 6 | public class KustoResultColumn { 7 | private String columnName; 8 | private String columnType; 9 | private int ordinal; 10 | 11 | public KustoResultColumn(String columnName, String columnType, int ordinal) { 12 | this.columnName = columnName; 13 | this.columnType = columnType; 14 | this.ordinal = ordinal; 15 | } 16 | 17 | public String getColumnName() { 18 | return columnName; 19 | } 20 | 21 | public String getColumnType() { 22 | return columnType; 23 | } 24 | 25 | public int getOrdinal() { 26 | return ordinal; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/AccessTokenTokenProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import java.net.URISyntaxException; 7 | 8 | import org.jetbrains.annotations.NotNull; 9 | import reactor.core.publisher.Mono; 10 | 11 | public class AccessTokenTokenProvider extends TokenProviderBase { 12 | private final String accessToken; 13 | 14 | AccessTokenTokenProvider(@NotNull String clusterUrl, @NotNull String accessToken) throws URISyntaxException { 15 | super(clusterUrl, null); 16 | this.accessToken = accessToken; 17 | } 18 | 19 | @Override 20 | protected Mono acquireAccessTokenImpl() { 21 | return Mono.just(accessToken); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoDataExceptionBase.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import com.azure.core.exception.AzureException; 4 | 5 | public abstract class KustoDataExceptionBase extends AzureException { 6 | private final String ingestionSource; 7 | private final boolean isPermanent; 8 | 9 | protected KustoDataExceptionBase(String ingestionSource, String message, Exception exception, boolean isPermanent) { 10 | super(message, exception); 11 | this.ingestionSource = ingestionSource; 12 | this.isPermanent = isPermanent; 13 | } 14 | 15 | public boolean isPermanent() { 16 | return isPermanent; 17 | } 18 | 19 | public String getIngestionSource() { 20 | return ingestionSource; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/AbstractSourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import com.microsoft.azure.kusto.data.instrumentation.TraceableAttributes; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | import java.util.UUID; 11 | 12 | abstract class AbstractSourceInfo implements SourceInfo, TraceableAttributes { 13 | private UUID sourceId; 14 | 15 | public UUID getSourceId() { 16 | return sourceId; 17 | } 18 | 19 | public void setSourceId(UUID sourceId) { 20 | this.sourceId = sourceId; 21 | } 22 | 23 | @Override 24 | public Map getTracingAttributes() { 25 | return new HashMap<>(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 16 | **Expected behavior** 17 | A clear and concise description of what you expected to happen. 18 | 19 | **Screenshots** 20 | If applicable, add screenshots to help explain your problem. 21 | 22 | **Setup (please complete the following information):** 23 | - JRE Version: [e.g. OpenJDK 8] 24 | - SDK Version: [e.g. 5.0.4] 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Version [e.g. 22] 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/QueuedIngestClient.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | import com.azure.storage.common.policy.RequestRetryOptions; 7 | 8 | public interface QueuedIngestClient extends IngestClient { 9 | /** 10 | * Setter for QueueRequestOptions used by the client on adding ingest message to the Azure queue, read here 11 | * https://docs.microsoft.com/azure/data-explorer/kusto/api/netfx/about-kusto-ingest#ingest-client-flavors 12 | * about Kusto queued ingestion 13 | * @param queueRequestOptions - Options to use when creating QueueClient 14 | */ 15 | void setQueueRequestOptions(RequestRetryOptions queueRequestOptions); 16 | 17 | IngestionResourceManager getResourceManager(); 18 | } 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.nar 17 | *.ear 18 | *.zip 19 | *.tar.gz 20 | *.rar 21 | 22 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 23 | hs_err_pid* 24 | 25 | out/ 26 | 27 | *.logs 28 | 29 | *.iml 30 | .idea/ 31 | .eclipse 32 | # Eclipse Core 33 | .project 34 | # JDT-specific (Eclipse Java Development Tools) 35 | .classpath 36 | 37 | # Maven 38 | target/ 39 | pom.xml.tag 40 | pom.xml.releaseBackup 41 | pom.xml.versionsBackup 42 | pom.xml.next 43 | .flattened-pom.xml 44 | release.properties 45 | dependency-reduced-pom.xml 46 | buildNumber.properties 47 | .mvn/timing.properties 48 | .mvn/wrapper/maven-wrapper.jar 49 | 50 | 51 | -------------------------------------------------------------------------------- /data/src/test/java/com/microsoft/azure/kusto/data/http/HttpClientFactoryTest.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | import com.azure.core.http.HttpClient; 4 | 5 | import org.junit.jupiter.api.Assertions; 6 | import org.junit.jupiter.api.DisplayName; 7 | import org.junit.jupiter.api.Test; 8 | 9 | class HttpClientFactoryTest { 10 | @Test 11 | @DisplayName("test create http client from null properties") 12 | void testNullProperties() { 13 | Assertions.assertDoesNotThrow(() -> HttpClientFactory.create(null)); 14 | } 15 | 16 | @Test 17 | @DisplayName("test create http client from properties") 18 | void testProperties() { 19 | HttpClientProperties properties = HttpClientProperties.builder().build(); 20 | final HttpClient httpClient = HttpClientFactory.create(properties); 21 | Assertions.assertNotNull(httpClient); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/CommandType.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | public enum CommandType { 4 | ADMIN_COMMAND("%s/" + ClientImpl.MGMT_ENDPOINT_VERSION + "/rest/mgmt", "AdminCommand"), 5 | QUERY("%s/" + ClientImpl.QUERY_ENDPOINT_VERSION + "/rest/query", "QueryCommand"), 6 | STREAMING_INGEST("%s/" + ClientImpl.STREAMING_VERSION + "/rest/ingest/%s/%s?streamFormat=%s", "StreamingIngest"); 7 | 8 | private final String endpoint; 9 | private final String activityTypeSuffix; 10 | 11 | CommandType(String endpoint, String activityTypeSuffix) { 12 | this.endpoint = endpoint; 13 | this.activityTypeSuffix = activityTypeSuffix; 14 | } 15 | 16 | public String getEndpoint() { 17 | return endpoint; 18 | } 19 | 20 | public String getActivityTypeSuffix() { 21 | return activityTypeSuffix; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/req/KustoRequestContext.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.req; 2 | 3 | import com.azure.core.http.HttpRequest; 4 | 5 | public class KustoRequestContext { 6 | private KustoRequest sdkRequest; 7 | private HttpRequest httpRequest; 8 | 9 | public KustoRequestContext(KustoRequest sdkRequest, HttpRequest httpRequest) { 10 | this.sdkRequest = sdkRequest; 11 | this.httpRequest = httpRequest; 12 | } 13 | 14 | public KustoRequest getSdkRequest() { 15 | return sdkRequest; 16 | } 17 | 18 | public void setSdkRequest(KustoRequest sdkRequest) { 19 | this.sdkRequest = sdkRequest; 20 | } 21 | 22 | public HttpRequest getHttpRequest() { 23 | return httpRequest; 24 | } 25 | 26 | public void setHttpRequest(HttpRequest httpRequest) { 27 | this.httpRequest = httpRequest; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/CloseableByteArrayInputStream.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.io.IOException; 6 | 7 | // ByteArrayInputStream's close method is a no-op, this class makes reading after close into an error, for test purposes 8 | public class CloseableByteArrayInputStream extends java.io.ByteArrayInputStream { 9 | private boolean closed; 10 | 11 | public CloseableByteArrayInputStream(byte[] buf) { 12 | super(buf); 13 | 14 | closed = false; 15 | } 16 | 17 | @Override 18 | public int read(byte @NotNull [] b) throws IOException { 19 | if (closed) { 20 | throw new IOException("Stream is closed"); 21 | } 22 | return super.read(b); 23 | } 24 | 25 | @Override 26 | public void close() { 27 | closed = true; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/src/test/java/com/microsoft/azure/kusto/data/auth/KeyCert.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import java.security.PrivateKey; 7 | import java.security.cert.X509Certificate; 8 | 9 | public class KeyCert { 10 | private X509Certificate certificate; 11 | private PrivateKey key; 12 | 13 | KeyCert(X509Certificate certificate, PrivateKey key) { 14 | this.certificate = certificate; 15 | this.key = key; 16 | } 17 | 18 | public X509Certificate getCertificate() { 19 | return certificate; 20 | } 21 | 22 | void setCertificate(X509Certificate certificate) { 23 | this.certificate = certificate; 24 | } 25 | 26 | public PrivateKey getKey() { 27 | return key; 28 | } 29 | 30 | void setKey(PrivateKey key) { 31 | this.key = key; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/exceptions/IngestionClientException.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.exceptions; 5 | 6 | import com.azure.core.exception.AzureException; 7 | 8 | public class IngestionClientException extends AzureException { 9 | private String ingestionSource; 10 | 11 | public String getIngestionSource() { 12 | return ingestionSource; 13 | } 14 | 15 | public IngestionClientException(String message) { 16 | super(message); 17 | } 18 | 19 | public IngestionClientException(String message, Throwable throwable) { 20 | super(message, throwable); 21 | } 22 | 23 | public IngestionClientException(String ingestionSource, String message, RuntimeException exception) { 24 | this(message, exception); 25 | this.ingestionSource = ingestionSource; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /quickstart/oneclick_instruction_box.md: -------------------------------------------------------------------------------- 1 | ### Prerequisites 2 | 3 | 1. Set up Java version 8 or higher on your machine. For instructions, consult a Java environment setup tutorial, like [this](https://www.tutorialspoint.com/java/java_environment_setup.htm). 4 | 1. Set up [Apache Maven](https://maven.apache.org/install.html), which is the most popular Java dependency management tool. 5 | 6 | ### Instructions 7 | 8 | 1. Download the **DOWNLOAD_LINK** as a ZIP file. 9 | 1. Extract the app source code. 10 | 1. Open a command line window and navigate to the folder where you extracted the app. 11 | 1. Run `mvn clean install` to compile the source code into a binary. 12 | 1. Run the binary using `java -jar target\kusto-quickstart-[version]-jar-with-dependencies.jar`. 13 | 14 | ### Troubleshooting 15 | 16 | * If you are having trouble running the app from your IDE, first check if the app runs from the command line, then consult the troubleshooting references of your IDE. -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/TokenCredentialProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.credential.TokenCredential; 4 | import com.azure.identity.CredentialBuilderBase; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | import java.net.URISyntaxException; 8 | 9 | public class TokenCredentialProvider extends AzureIdentityTokenProvider { 10 | private final TokenCredential credential; 11 | 12 | TokenCredentialProvider(@NotNull String clusterUrl, @NotNull TokenCredential credential) throws URISyntaxException { 13 | super(clusterUrl, null); 14 | this.credential = credential; 15 | } 16 | 17 | @Override 18 | protected CredentialBuilderBase initBuilder() { 19 | return null; 20 | } 21 | 22 | @Override 23 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 24 | return credential; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/MatchRule.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth.endpoints; 2 | 3 | public class MatchRule { 4 | /// 5 | /// The suffix which the candidate must end with in order to match. 6 | /// 7 | public final String suffix; 8 | 9 | /// 10 | /// Indicates whether the match must be exact (the candidate must 11 | /// not have any prefix) or not. 12 | /// 13 | public final Boolean exact; 14 | 15 | public int getSuffixLength() { 16 | return suffix == null ? 0 : suffix.length(); 17 | } 18 | 19 | public MatchRule(String suffix, Boolean exact) { 20 | this.suffix = suffix; 21 | this.exact = exact; 22 | } 23 | 24 | /// 25 | /// Clones this object. 26 | /// 27 | public MatchRule clone() { 28 | return new MatchRule(suffix, exact); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/exceptions/IngestionServiceException.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.exceptions; 5 | 6 | import com.azure.core.exception.AzureException; 7 | 8 | public class IngestionServiceException extends AzureException { 9 | private String ingestionSource; 10 | 11 | public String getIngestionSource() { 12 | return ingestionSource; 13 | } 14 | 15 | public IngestionServiceException(String message) { 16 | super(message); 17 | } 18 | 19 | public IngestionServiceException(String message, Exception exception) { 20 | super(message, exception); 21 | } 22 | 23 | public IngestionServiceException(String ingestionSource, String message, RuntimeException exception) { 24 | this(message, exception); 25 | this.ingestionSource = ingestionSource; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /UPDATE.md: -------------------------------------------------------------------------------- 1 | - Update `revision` field in the pom files to the new version: 2 | - `pom.xml` in the root directory 3 | - `ingest/pom.xml` 4 | - `data/pom.xml` 5 | - `quickstart/pom.xml` 6 | - `sample/pom.xml` 7 | 8 | - Update `CHANGELOG.md` with the new version and date. Make sure it complies with the [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. 9 | 10 | - Tag the new version in the repository: 11 | ```bash 12 | git tag -a vX.Y.Z -m "Release version X.Y.Z" 13 | ``` 14 | 15 | - Push the tag to the remote repository: 16 | ```bash 17 | git push --tags 18 | ``` 19 | 20 | - The GitHub Actions workflow will automatically create a release on GitHub with the new version and the contents of `CHANGELOG.md`. 21 | 22 | - For now, some of the automations may not work as expected. You might need to upload the files to the storage yourselves , or trigger the release via https://dev.azure.com/azure-sdk/internal/_build?definitionId=1809. 23 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/WebException.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import com.azure.core.exception.AzureException; 4 | import com.azure.core.http.HttpResponse; 5 | import org.jetbrains.annotations.Nullable; 6 | 7 | public class WebException extends AzureException { 8 | @Nullable 9 | protected final HttpResponse httpResponse; 10 | 11 | public WebException(String message, @Nullable HttpResponse httpResponse, Throwable cause) { 12 | super(message, cause); 13 | this.httpResponse = httpResponse; 14 | } 15 | 16 | public @Nullable HttpResponse getHttpResponse() { 17 | return httpResponse; 18 | } 19 | 20 | @Override 21 | public String toString() { 22 | return this.getMessage(); 23 | } 24 | 25 | @Nullable 26 | public Integer getStatusCode() { 27 | return httpResponse != null ? httpResponse.getStatusCode() : null; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataServiceException.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.exceptions; 5 | 6 | import org.jetbrains.annotations.Nullable; 7 | 8 | public class DataServiceException extends KustoDataExceptionBase { 9 | public DataServiceException(String ingestionSource, String message, boolean isPermanent) { 10 | this(ingestionSource, message, null, isPermanent); 11 | } 12 | 13 | public DataServiceException(String ingestionSource, String message, Exception exception, boolean isPermanent) { 14 | super(ingestionSource, message, exception, isPermanent); 15 | } 16 | 17 | @Nullable 18 | public Integer getStatusCode() { 19 | Throwable cause = getCause(); 20 | if (!(cause instanceof WebException)) { 21 | return null; 22 | } 23 | 24 | return ((WebException) cause).getStatusCode(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/source/SourceInfoTest.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import org.junit.jupiter.api.Test; 7 | 8 | import static org.junit.jupiter.api.Assertions.assertThrows; 9 | 10 | class SourceInfoTest { 11 | @Test 12 | void validate_BlankBlobPath_IllegalArgumentException() { 13 | BlobSourceInfo blobSourceInfo = new BlobSourceInfo(""); 14 | assertThrows(IllegalArgumentException.class, blobSourceInfo::validate); 15 | } 16 | 17 | @Test 18 | void validate_BlankFilePath_IllegalArgumentException() { 19 | FileSourceInfo fileSourceInfo = new FileSourceInfo(""); 20 | assertThrows(IllegalArgumentException.class, fileSourceInfo::validate); 21 | } 22 | 23 | @Test 24 | void streamSourceInfoConstructor_StreamIsNull_NullPointerException() { 25 | assertThrows(NullPointerException.class, () -> new StreamSourceInfo(null)); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/AzureCliTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.credential.TokenCredential; 4 | import com.azure.core.http.HttpClient; 5 | import com.azure.identity.AzureCliCredentialBuilder; 6 | import com.azure.identity.CredentialBuilderBase; 7 | import org.jetbrains.annotations.NotNull; 8 | import org.jetbrains.annotations.Nullable; 9 | 10 | import java.net.URISyntaxException; 11 | 12 | public class AzureCliTokenProvider extends AzureIdentityTokenProvider { 13 | public AzureCliTokenProvider(@NotNull String clusterUrl, @Nullable HttpClient httpClient) throws URISyntaxException { 14 | super(clusterUrl, httpClient); 15 | } 16 | 17 | @Override 18 | protected CredentialBuilderBase initBuilder() { 19 | return new AzureCliCredentialBuilder(); 20 | } 21 | 22 | @Override 23 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 24 | return ((AzureCliCredentialBuilder) builder).build(); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/ManagedStreamingQueuingPolicyTest.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import org.junit.jupiter.api.Test; 4 | 5 | import static org.junit.jupiter.api.Assertions.assertEquals; 6 | 7 | class ManagedStreamingQueuingPolicyTest { 8 | 9 | @Test 10 | void shouldUseQueuedIngestion() { 11 | ManagedStreamingQueuingPolicy policy = ManagedStreamingQueuingPolicy.Default; 12 | 13 | // Test with dataSize, rawDataSize, compressed and dataFormat parameters 14 | // Adjust these values according to your needs 15 | long dataSize = 0; 16 | boolean compressed = false; 17 | IngestionProperties.DataFormat dataFormat = IngestionProperties.DataFormat.CSV; 18 | 19 | boolean result = policy.shouldUseQueuedIngestion(dataSize, compressed, dataFormat); 20 | 21 | // Assert the result 22 | // Adjust the expected result according to your needs 23 | boolean expectedResult = false; 24 | assertEquals(expectedResult, result); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/IngestionFailureInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | import java.util.Date; 7 | import java.util.UUID; 8 | 9 | public class IngestionFailureInfo { 10 | public UUID OperationId; 11 | public String Database; 12 | public String Table; 13 | public Date FailedOn; 14 | public UUID IngestionSourceId; 15 | public String IngestionSourcePath; 16 | public String Details; 17 | public FailureStatusValue FailureStatus; 18 | public UUID RootActivityId; 19 | public Boolean OriginatesFromUpdatePolicy; 20 | 21 | public enum FailureStatusValue { 22 | Unknown(0), 23 | Permanent(1), 24 | Transient(2), 25 | Exhausted(3); 26 | 27 | private final int value; 28 | 29 | FailureStatusValue(int v) { 30 | value = v; 31 | } 32 | 33 | public int getValue() { 34 | return value; 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/AsyncCallbackTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.microsoft.azure.kusto.data.exceptions.DataClientException; 4 | import org.jetbrains.annotations.NotNull; 5 | import reactor.core.publisher.Mono; 6 | 7 | import java.net.URISyntaxException; 8 | 9 | public class AsyncCallbackTokenProvider extends TokenProviderBase { 10 | public static final String CALLBACK_TOKEN_PROVIDER = "CallbackTokenProvider"; 11 | private final Mono tokenProvider; 12 | 13 | AsyncCallbackTokenProvider(@NotNull String clusterUrl, @NotNull Mono tokenProvider) throws URISyntaxException { 14 | super(clusterUrl, null); 15 | this.tokenProvider = tokenProvider; 16 | } 17 | 18 | @Override 19 | protected Mono acquireAccessTokenImpl() { 20 | return tokenProvider.onErrorMap(e -> DataClientException.unwrapThrowable(clusterUrl, e)); 21 | } 22 | 23 | @Override 24 | protected String getAuthMethod() { 25 | return CALLBACK_TOKEN_PROVIDER; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/DeviceAuthTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.credential.TokenCredential; 4 | import com.azure.core.http.HttpClient; 5 | import com.azure.identity.CredentialBuilderBase; 6 | import com.azure.identity.DeviceCodeCredentialBuilder; 7 | 8 | import org.jetbrains.annotations.NotNull; 9 | import org.jetbrains.annotations.Nullable; 10 | 11 | import java.net.URISyntaxException; 12 | 13 | public class DeviceAuthTokenProvider extends AzureIdentityTokenProvider { 14 | DeviceAuthTokenProvider(@NotNull String clusterUrl, @Nullable String tenantId, @Nullable HttpClient httpClient) throws URISyntaxException { 15 | super(clusterUrl, null, tenantId, httpClient); 16 | } 17 | 18 | @Override 19 | protected CredentialBuilderBase initBuilder() { 20 | return new DeviceCodeCredentialBuilder(); 21 | } 22 | 23 | @Override 24 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 25 | return ((DeviceCodeCredentialBuilder) builder).build(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /samples/src/main/java/Proxy.java: -------------------------------------------------------------------------------- 1 | import com.azure.core.http.ProxyOptions; 2 | import com.microsoft.azure.kusto.data.Client; 3 | import com.microsoft.azure.kusto.data.ClientFactory; 4 | import com.microsoft.azure.kusto.data.http.HttpClientProperties; 5 | import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; 6 | 7 | import java.net.InetSocketAddress; 8 | 9 | public class Proxy { 10 | public static void main(String[] args) { 11 | 12 | ProxyOptions proxyOptions = new ProxyOptions(ProxyOptions.Type.HTTP, 13 | InetSocketAddress.createUnresolved("host.example.com", 8080)); 14 | 15 | HttpClientProperties providedProperties = HttpClientProperties.builder() 16 | .proxy(proxyOptions) 17 | .build(); 18 | try { 19 | Client client = ClientFactory.createClient(ConnectionStringBuilder.createWithUserPrompt( 20 | "https://ohadev.swedencentral.dev.kusto.windows.net"), providedProperties); 21 | client.executeMgmt(".show version"); 22 | 23 | } catch (Exception e) { 24 | e.printStackTrace(); 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/result/IngestionStatusTest.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.result; 2 | 3 | import com.azure.data.tables.models.TableEntity; 4 | import java.util.UUID; 5 | import java.util.stream.Stream; 6 | import org.junit.jupiter.api.Assertions; 7 | import org.junit.jupiter.params.ParameterizedTest; 8 | import org.junit.jupiter.params.provider.MethodSource; 9 | 10 | class IngestionStatusTest { 11 | static Stream fromIdTestCases() { 12 | UUID uuid = UUID.randomUUID(); 13 | return Stream.of( 14 | //happy path 15 | new Object[]{uuid.toString(), uuid}, 16 | new Object[]{uuid, uuid}, 17 | //edge cases 18 | new Object[]{"", null}, 19 | new Object[]{null, null}, 20 | new Object[]{"not-a-uuid", null} 21 | ); 22 | } 23 | 24 | @ParameterizedTest 25 | @MethodSource("fromIdTestCases") 26 | void testFromId(Object input, UUID expected) { 27 | UUID result = IngestionStatus.fromId(input); 28 | Assertions.assertEquals(expected, result); 29 | } 30 | } -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/IngestionResult.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | import com.azure.data.tables.implementation.models.TableServiceErrorException; 7 | import reactor.core.publisher.Mono; 8 | 9 | import java.io.Serializable; 10 | import java.net.URISyntaxException; 11 | import java.util.List; 12 | 13 | public interface IngestionResult extends Serializable { 14 | 15 | /** 16 | * Retrieves the detailed ingestion status of 17 | * all data ingestion operations into Kusto associated with this com.microsoft.azure.kusto.ingest.IKustoIngestionResult instance. 18 | */ 19 | Mono> getIngestionStatusCollectionAsync() throws URISyntaxException, TableServiceErrorException; 20 | 21 | /** 22 | * Blocking, retrieves the detailed ingestion status of 23 | * all data ingestion operations into Kusto associated with this com.microsoft.azure.kusto.ingest.IKustoIngestionResult instance. 24 | */ 25 | List getIngestionStatusCollection() throws URISyntaxException, TableServiceErrorException; 26 | } 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) Microsoft Corporation. All rights reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE 22 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/IngestionStatusResult.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | import com.azure.data.tables.implementation.models.TableServiceErrorException; 7 | import reactor.core.publisher.Mono; 8 | 9 | import java.net.URISyntaxException; 10 | import java.util.Collections; 11 | import java.util.List; 12 | 13 | public class IngestionStatusResult implements IngestionResult { 14 | 15 | private IngestionStatus ingestionStatus; 16 | 17 | public IngestionStatusResult(IngestionStatus ingestionStatus) { 18 | this.ingestionStatus = ingestionStatus; 19 | } 20 | 21 | @Override 22 | public Mono> getIngestionStatusCollectionAsync() { 23 | if (ingestionStatus != null) { 24 | return Mono.just(Collections.singletonList(ingestionStatus)); 25 | } 26 | 27 | return Mono.empty(); 28 | } 29 | 30 | @Override 31 | public List getIngestionStatusCollection() throws URISyntaxException, TableServiceErrorException { 32 | return getIngestionStatusCollectionAsync().block(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/CallbackTokenProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import com.microsoft.azure.kusto.data.exceptions.DataClientException; 7 | 8 | import org.jetbrains.annotations.NotNull; 9 | import reactor.core.publisher.Mono; 10 | 11 | import java.net.URISyntaxException; 12 | import java.util.concurrent.Callable; 13 | 14 | public class CallbackTokenProvider extends TokenProviderBase { 15 | public static final String CALLBACK_TOKEN_PROVIDER = "CallbackTokenProvider"; 16 | private final @NotNull Callable tokenProvider; 17 | 18 | CallbackTokenProvider(@NotNull String clusterUrl, @NotNull Callable tokenProvider) throws URISyntaxException { 19 | super(clusterUrl, null); 20 | this.tokenProvider = tokenProvider; 21 | } 22 | 23 | @Override 24 | protected Mono acquireAccessTokenImpl() { 25 | return Mono.fromCallable(tokenProvider).onErrorMap(e -> DataClientException.unwrapThrowable(clusterUrl, e)); 26 | } 27 | 28 | @Override 29 | protected String getAuthMethod() { 30 | return CALLBACK_TOKEN_PROVIDER; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/ResettableFileInputStream.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | import java.io.FileInputStream; 7 | import java.io.FilterInputStream; 8 | import java.io.IOException; 9 | import java.nio.channels.FileChannel; 10 | 11 | public class ResettableFileInputStream extends FilterInputStream { 12 | private final FileChannel myFileChannel; 13 | private long mark = -1; 14 | 15 | public ResettableFileInputStream(FileInputStream fis) { 16 | super(fis); 17 | myFileChannel = fis.getChannel(); 18 | mark(0); 19 | } 20 | 21 | @Override 22 | public boolean markSupported() { 23 | return true; 24 | } 25 | 26 | @Override 27 | public synchronized void mark(int ignored) { 28 | try { 29 | mark = myFileChannel.position(); 30 | } catch (IOException ex) { 31 | mark = -1; 32 | } 33 | } 34 | 35 | @Override 36 | public synchronized void reset() throws IOException { 37 | if (mark == -1) { 38 | throw new IOException("not marked"); 39 | } 40 | myFileChannel.position(mark); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/IngestionResourceManager.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import com.microsoft.azure.kusto.ingest.exceptions.IngestionClientException; 4 | import com.microsoft.azure.kusto.ingest.exceptions.IngestionServiceException; 5 | import com.microsoft.azure.kusto.ingest.resources.ContainerWithSas; 6 | import com.microsoft.azure.kusto.ingest.resources.ResourceWithSas; 7 | 8 | import java.util.List; 9 | 10 | public interface IngestionResourceManager { 11 | 12 | /** 13 | * Returns a list of containers with SAS tokens, ranked by their ingestion success rate, and then shuffled. 14 | * You should use this method for each ingestion operation. 15 | * @return List of containers with SAS tokens, ranked by their ingestion success rate, and then shuffled. 16 | */ 17 | List getShuffledContainers() throws IngestionClientException, IngestionServiceException; 18 | 19 | /** 20 | * Report the result of an ingestion operation. 21 | * @param resource The resource that was used to ingest. Can be a container or a queue. 22 | * @param success Whether the ingestion operation was successful. 23 | */ 24 | void reportIngestionResult(ResourceWithSas resource, boolean success); 25 | } 26 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import java.util.regex.Matcher; 4 | import java.util.regex.Pattern; 5 | 6 | public abstract class CslFormat { 7 | public abstract String getType(); 8 | 9 | public abstract Object getValue(); 10 | 11 | // Value must not be null 12 | abstract String getValueAsString(); 13 | 14 | private String getValueOrNullAsString() { 15 | if (getValue() == null) { 16 | return "null"; 17 | } else { 18 | return getValueAsString(); 19 | } 20 | } 21 | 22 | public String toString() { 23 | return getType() + "(" + getValueOrNullAsString() + ")"; 24 | } 25 | 26 | // For example, parses "int(7)" as "7" 27 | // If type wasn't found in valueWithType, returns it unchanged 28 | public static String parseValueFromValueWithType(String valueWithType, String type) { 29 | String valueWithTypeRegex = String.format("%s\\s*\\(\\s*(.*\\S+)\\s*\\)\\s*", type); 30 | Pattern valueWithTypePattern = Pattern.compile(valueWithTypeRegex, Pattern.CASE_INSENSITIVE); 31 | Matcher matcher = valueWithTypePattern.matcher(valueWithType); 32 | return matcher.matches() ? matcher.group(1) : valueWithType; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/ManagedIdentityTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.credential.TokenCredential; 4 | import com.azure.core.http.HttpClient; 5 | import com.azure.identity.CredentialBuilderBase; 6 | import com.azure.identity.ManagedIdentityCredentialBuilder; 7 | 8 | import org.jetbrains.annotations.NotNull; 9 | import org.jetbrains.annotations.Nullable; 10 | 11 | import java.net.URISyntaxException; 12 | 13 | public class ManagedIdentityTokenProvider extends AzureIdentityTokenProvider { 14 | private final String managedIdentityClientId; 15 | 16 | public ManagedIdentityTokenProvider(@NotNull String clusterUrl, @Nullable String managedIdentityClientId, @Nullable HttpClient httpClient) throws URISyntaxException { 17 | super(clusterUrl, httpClient); 18 | this.managedIdentityClientId = managedIdentityClientId; 19 | } 20 | 21 | @Override 22 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 23 | return ((ManagedIdentityCredentialBuilder) builder).build(); 24 | } 25 | 26 | @Override 27 | @NotNull 28 | protected CredentialBuilderBase initBuilder() { 29 | return new ManagedIdentityCredentialBuilder() 30 | .clientId(managedIdentityClientId); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ingest/src/test/resources/testdata.json: -------------------------------------------------------------------------------- 1 | {"Name":"demo1","Code":"091231"} 2 | {"Name":"demo11","Code":"091232"} 3 | {"Name":"demo12","Code":"091233"} 4 | {"Name":"demo13","Code":"091234"} 5 | {"Name":"demo14","Code":"091235"} 6 | {"Name":"demo15","Code":"091236"} 7 | {"Name":"demo16","Code":"091237"} 8 | {"Name":"demo17","Code":"091238"} 9 | {"Name":"demo18","Code":"091239"} 10 | {"Name":"demo19","Code":"091230"} 11 | {"Name":"demo10","Code":"0912311"} 12 | {"Name":"demo11","Code":"0912322"} 13 | {"Name":"demo12","Code":"0912333"} 14 | {"Name":"demo13","Code":"0912344"} 15 | {"Name":"demo14","Code":"0912355"} 16 | {"Name":"demo15","Code":"0912366"} 17 | {"Name":"demo16","Code":"0912377"} 18 | {"Name":"demo17","Code":"0912388"} 19 | {"Name":"demo18","Code":"0912399"} 20 | {"Name":"demo19","Code":"0912300"} 21 | {"Name":"demo10","Code":"0912113"} 22 | {"Name":"demo11","Code":"0912223"} 23 | {"Name":"demo12","Code":"0912333"} 24 | {"Name":"demo13","Code":"0912443"} 25 | {"Name":"demo14","Code":"0912553"} 26 | {"Name":"demo15","Code":"0912663"} 27 | {"Name":"demo16","Code":"0912773"} 28 | {"Name":"demo17","Code":"0912883"} 29 | {"Name":"demo18","Code":"0912399"} 30 | {"Name":"demo19","Code":"0912003"} 31 | {"Name":"demo10","Code":"091231"} 32 | {"Name":"demo11","Code":"091232"} 33 | {"Name":"demo12","Code":"091233"} 34 | {"Name":"demo13","Code":"091234"} 35 | {"Name":"demo14","Code":"091235"} 36 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/res/ResponseState.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.res; 2 | 3 | import com.azure.core.http.HttpResponse; 4 | 5 | /** 6 | * A helper class to store and manage the state during the execution of the 7 | * asynchronous HTTP request to handle streaming output. 8 | */ 9 | public class ResponseState { 10 | 11 | private boolean returnInputStream; 12 | private String errorFromResponse; 13 | private HttpResponse httpResponse; 14 | 15 | public ResponseState() { 16 | this.returnInputStream = false; 17 | this.errorFromResponse = null; 18 | this.httpResponse = null; 19 | } 20 | 21 | public boolean isReturnInputStream() { 22 | return returnInputStream; 23 | } 24 | 25 | public void setReturnInputStream(boolean returnInputStream) { 26 | this.returnInputStream = returnInputStream; 27 | } 28 | 29 | public String getErrorFromResponse() { 30 | return errorFromResponse; 31 | } 32 | 33 | public void setErrorFromResponse(String errorFromResponse) { 34 | this.errorFromResponse = errorFromResponse; 35 | } 36 | 37 | public HttpResponse getHttpResponse() { 38 | return httpResponse; 39 | } 40 | 41 | public void setHttpResponse(HttpResponse httpResponse) { 42 | this.httpResponse = httpResponse; 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/FileSourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import java.util.Map; 7 | import java.util.UUID; 8 | 9 | import static com.microsoft.azure.kusto.data.Ensure.stringIsNotBlank; 10 | 11 | public class FileSourceInfo extends AbstractSourceInfo { 12 | private String filePath; 13 | 14 | public String getFilePath() { 15 | return filePath; 16 | } 17 | 18 | public void setFilePath(String filePath) { 19 | this.filePath = filePath; 20 | } 21 | 22 | public FileSourceInfo(String filePath) { 23 | this.filePath = filePath; 24 | } 25 | 26 | public FileSourceInfo(String filePath, UUID sourceId) { 27 | this(filePath); 28 | this.setSourceId(sourceId); 29 | } 30 | 31 | public void validate() { 32 | stringIsNotBlank(filePath, "filePath"); 33 | } 34 | 35 | public Map getTracingAttributes() { 36 | Map attributes = super.getTracingAttributes(); 37 | attributes.put("resource", filePath); 38 | UUID sourceId = getSourceId(); 39 | if (sourceId != null) { 40 | attributes.put("sourceId", sourceId.toString()); 41 | } 42 | return attributes; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/UserPromptTokenProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import com.azure.core.credential.TokenCredential; 7 | import com.azure.core.http.HttpClient; 8 | import com.azure.identity.CredentialBuilderBase; 9 | import com.azure.identity.InteractiveBrowserCredentialBuilder; 10 | import org.jetbrains.annotations.NotNull; 11 | import org.jetbrains.annotations.Nullable; 12 | 13 | import java.net.URISyntaxException; 14 | 15 | public class UserPromptTokenProvider extends AzureIdentityTokenProvider { 16 | private final String usernameHint; 17 | 18 | UserPromptTokenProvider(@NotNull String clusterUrl, @Nullable String usernameHint, String authorityId, 19 | @Nullable HttpClient httpClient) throws URISyntaxException { 20 | super(clusterUrl, authorityId, null, httpClient); 21 | this.usernameHint = usernameHint; 22 | } 23 | 24 | @Override 25 | protected CredentialBuilderBase initBuilder() { 26 | return new InteractiveBrowserCredentialBuilder() 27 | .loginHint(usernameHint); 28 | } 29 | 30 | @Override 31 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 32 | return ((InteractiveBrowserCredentialBuilder) builder).build(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /ingest/src/test/resources/dataset_mapping.json: -------------------------------------------------------------------------------- 1 | [{"Properties":{"Path":"$.rownumber"},"column":"rownumber","datatype":"int"},{"Properties":{"Path":"$.rowguid"},"column":"rowguid","datatype":"string"},{"Properties":{"Path":"$.xdouble"},"column":"xdouble","datatype":"real"},{"Properties":{"Path":"$.xfloat"},"column":"xfloat","datatype":"real"},{"Properties":{"Path":"$.xbool"},"column":"xbool","datatype":"bool"},{"Properties":{"Path":"$.xint16"},"column":"xint16","datatype":"int"},{"Properties":{"Path":"$.xint32"},"column":"xint32","datatype":"int"},{"Properties":{"Path":"$.xint64"},"column":"xint64","datatype":"long"},{"Properties":{"Path":"$.xuint8"},"column":"xuint8","datatype":"long"},{"Properties":{"Path":"$.xuint16"},"column":"xuint16","datatype":"long"},{"Properties":{"Path":"$.xuint32"},"column":"xuint32","datatype":"long"},{"Properties":{"Path":"$.xuint64"},"column":"xuint64","datatype":"long"},{"Properties":{"Path":"$.xdate"},"column":"xdate","datatype":"datetime"},{"Properties":{"Path":"$.xsmalltext"},"column":"xsmalltext","datatype":"string"},{"Properties":{"Path":"$.xtext"},"column":"xtext","datatype":"string"},{"Properties":{"Path":"$.rowguid"},"column":"xnumberAsText","datatype":"string"},{"Properties":{"Path":"$.xtime"},"column":"xtime","datatype":"timespan"},{"Properties":{"Path":"$.xtextWithNulls"},"column":"xtextWithNulls","datatype":"string"},{"Properties":{"Path":"$.xdynamicWithNulls"},"column":"xdynamicWithNulls","datatype":"dynamic"}] -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/ApplicationKeyTokenProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import com.azure.core.credential.TokenCredential; 7 | import com.azure.core.http.HttpClient; 8 | import com.azure.identity.ClientSecretCredentialBuilder; 9 | import com.azure.identity.CredentialBuilderBase; 10 | 11 | import java.net.URISyntaxException; 12 | 13 | import org.jetbrains.annotations.NotNull; 14 | import org.jetbrains.annotations.Nullable; 15 | 16 | public class ApplicationKeyTokenProvider extends AzureIdentityTokenProvider { 17 | private final String clientSecret; 18 | 19 | public ApplicationKeyTokenProvider(@NotNull String clusterUrl, String clientId, String clientSecret, String tenantId, 20 | @Nullable HttpClient httpClient) throws URISyntaxException { 21 | super(clusterUrl, tenantId, clientId, httpClient); 22 | this.clientSecret = clientSecret; 23 | } 24 | 25 | @Override 26 | protected TokenCredential createTokenCredential(CredentialBuilderBase builder) { 27 | return ((ClientSecretCredentialBuilder) builder) 28 | .clientSecret(clientSecret) 29 | .build(); 30 | } 31 | 32 | @Override 33 | protected CredentialBuilderBase initBuilder() { 34 | return new ClientSecretCredentialBuilder(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataClientException.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.exceptions; 5 | 6 | /* 7 | This class represents an error that happened on the client side and is therefore considered permanent 8 | */ 9 | public class DataClientException extends KustoDataExceptionBase { 10 | public DataClientException(Exception ex) { 11 | this(ex.getMessage()); 12 | } 13 | 14 | public DataClientException(String message) { 15 | this(null, message); 16 | } 17 | 18 | public DataClientException(String ingestionSource, String message) { 19 | this(ingestionSource, message, null); 20 | } 21 | 22 | public DataClientException(String ingestionSource, String message, Exception exception) { 23 | super(ingestionSource, message, exception, true); 24 | } 25 | 26 | public static DataClientException unwrapThrowable(String clusterUrl, Throwable throwable) { 27 | if (throwable instanceof DataClientException) { 28 | return (DataClientException) throwable; 29 | } 30 | if (throwable instanceof Exception) { 31 | Exception ex = (Exception) throwable; 32 | return new DataClientException(clusterUrl, ExceptionUtils.getMessageEx(ex), ex); 33 | } 34 | 35 | return new DataClientException(clusterUrl, throwable.toString(), null); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /quickstart/dataset.csv: -------------------------------------------------------------------------------- 1 | 0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,Zero,0,00:00:00,,null 2 | 1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,One,1,00:01.0,,"{""rowId"": 1, ""arr"": [0,1]}" 3 | 2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,Two,2,-00:00:02.0020002,,"{""rowId"": 2, ""arr"": [0,2]}" 4 | 3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,Three,3,00:03.0,,"{""rowId"": 3, ""arr"": [0,3]}" 5 | 4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,Four,4,-00:00:04.0040004,,"{""rowId"": 4, ""arr"": [0,4]}" 6 | 5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,Five,5,00:05.0,,"{""rowId"": 5, ""arr"": [0,5]}" 7 | 6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,Six,6,-00:00:06.0060006,,"{""rowId"": 6, ""arr"": [0,6]}" 8 | 7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,Seven,7,00:07.0,,"{""rowId"": 7, ""arr"": [0,7]}" 9 | 8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,Eight,8,-00:00:08.0080008,,"{""rowId"": 8, ""arr"": [0,8]}" 10 | 9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,Nine,9,00:09.0,,"{""rowId"": 9, ""arr"": [0,9]}" -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/TransformationMethod.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest; 5 | 6 | /** 7 | * Transformation method can be used over all formats configured with Json mapping type (Json, Orc and Parquet) 8 | * See kusto docs 9 | */ 10 | public enum TransformationMethod { 11 | /** 12 | * Comma-separated value. 13 | */ 14 | None, 15 | 16 | /** 17 | * Property bag array to dictionary. 18 | */ 19 | PropertyBagArrayToDictionary, 20 | 21 | /** 22 | * Source location. 23 | */ 24 | SourceLocation, 25 | 26 | /** 27 | * Source line number. 28 | */ 29 | SourceLineNumber, 30 | 31 | /** 32 | * Get path element. 33 | */ 34 | GetPathElement, 35 | 36 | /** 37 | * Unknown method. 38 | */ 39 | UnknownMethod, 40 | 41 | /** 42 | * Converts UNIX epoch (seconds) to UTC datetime. 43 | */ 44 | DateTimeFromUnixSeconds, 45 | 46 | /** 47 | * Converts UNIX epoch (milliseconds) to UTC datetime. 48 | */ 49 | DateTimeFromUnixMilliseconds, 50 | 51 | /** 52 | * Converts UNIX epoch (microseconds) to UTC datetime. 53 | */ 54 | DateTimeFromUnixMicroseconds, 55 | 56 | /** 57 | * Converts UNIX epoch (nanoseconds) to UTC datetime. 58 | */ 59 | DateTimeFromUnixNanoseconds, 60 | 61 | } 62 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/OperationStatus.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | /// 7 | /// An enum representing the state of a data ingestion operation into Kusto 8 | /// 9 | public enum OperationStatus { 10 | /// 11 | /// Represents a temporary status. 12 | /// Might change during the course of ingestion based on the 13 | /// outcome of the data ingestion operation into Kusto. 14 | /// 15 | Pending, 16 | /// 17 | /// Represents a permanent status. 18 | /// The data has been successfully ingested to Kusto. 19 | /// 20 | Succeeded, 21 | /// 22 | /// Represents a permanent status. 23 | /// The data has not been ingested to Kusto. 24 | /// 25 | Failed, 26 | /// 27 | /// Represents a permanent status. 28 | /// The data has been queued for ingestion. 29 | /// (This does not indicate the ingestion was successful) 30 | /// 31 | Queued, 32 | /// 33 | /// Represents a permanent status. 34 | /// No data was supplied for ingestion. The ingest operation was skipped. 35 | /// 36 | Skipped, 37 | /// 38 | /// Represents a permanent status. 39 | /// Part of the data has been successfully ingested to Kusto while some failed. 40 | /// 41 | PartiallySucceeded 42 | } 43 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/TableWithSas.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.utils; 2 | 3 | import com.azure.core.http.HttpClient; 4 | import com.azure.data.tables.TableAsyncClient; 5 | import com.azure.data.tables.TableClientBuilder; 6 | import com.microsoft.azure.kusto.data.UriUtils; 7 | import reactor.util.annotation.Nullable; 8 | 9 | import java.net.URISyntaxException; 10 | 11 | public class TableWithSas { 12 | private final String uri; 13 | private final TableAsyncClient tableAsyncClient; 14 | 15 | public TableWithSas(String url, @Nullable HttpClient httpClient) throws URISyntaxException { 16 | this.uri = url; 17 | this.tableAsyncClient = createTableClientFromUrl(url, httpClient); 18 | } 19 | 20 | public String getUri() { 21 | return uri; 22 | } 23 | 24 | public TableAsyncClient getTableAsyncClient() { 25 | return tableAsyncClient; 26 | } 27 | 28 | public static TableAsyncClient createTableClientFromUrl(String url, @Nullable HttpClient httpClient) throws URISyntaxException { 29 | String[] parts = UriUtils.getSasAndEndpointFromResourceURL(url); 30 | int tableNameIndex = parts[0].lastIndexOf('/'); 31 | String tableName = parts[0].substring(tableNameIndex + 1); 32 | return new TableClientBuilder() 33 | .endpoint(parts[0].substring(0, tableNameIndex)) 34 | .sasToken(parts[1]) 35 | .tableName(tableName) 36 | .httpClient(httpClient) 37 | .buildAsyncClient(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /ingest/src/test/resources/dataset.csv: -------------------------------------------------------------------------------- 1 | 0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null 2 | 1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,"One",1,00:00:01.0010001,,"{""rowId"": 1, ""arr"": [0,1]}" 3 | 2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,"Two",2,-00:00:02.0020002,,"{""rowId"": 2, ""arr"": [0,2]}" 4 | 3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,"Three",3,00:00:03.0030003,,"{""rowId"": 3, ""arr"": [0,3]}" 5 | 4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,"Four",4,-00:00:04.0040004,,"{""rowId"": 4, ""arr"": [0,4]}" 6 | 5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,"Five",5,00:00:05.0050005,,"{""rowId"": 5, ""arr"": [0,5]}" 7 | 6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,"Six",6,-00:00:06.0060006,,"{""rowId"": 6, ""arr"": [0,6]}" 8 | 7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,"Seven",7,00:00:07.0070007,,"{""rowId"": 7, ""arr"": [0,7]}" 9 | 8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,"Eight",8,-00:00:08.0080008,,"{""rowId"": 8, ""arr"": [0,8]}" 10 | 9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,"Nine",9,00:00:09.0090009,,"{""rowId"": 9, ""arr"": [0,9]}" -------------------------------------------------------------------------------- /samples/src/main/resources/dataset.csv: -------------------------------------------------------------------------------- 1 | 0,00000000-0000-0000-0001-020304050607,0,0,0,0,0,0,0,0,0,0,2014-01-01T01:01:01.0000000Z,Zero,"Zero",0,00:00:00,,null 2 | 1,00000001-0000-0000-0001-020304050607,1.0001,1.01,1,1,1,1,1,1,1,1,2015-01-01T01:01:01.0000000Z,One,"One",1,00:00:01.0010001,,"{""rowId"": 1, ""arr"": [0,1]}" 3 | 2,00000002-0000-0000-0001-020304050607,2.0002,2.02,0,2,2,2,2,2,2,2,2016-01-01T01:01:01.0000000Z,Two,"Two",2,-00:00:02.0020002,,"{""rowId"": 2, ""arr"": [0,2]}" 4 | 3,00000003-0000-0000-0001-020304050607,3.0003,3.03,1,3,3,3,3,3,3,3,2017-01-01T01:01:01.0000000Z,Three,"Three",3,00:00:03.0030003,,"{""rowId"": 3, ""arr"": [0,3]}" 5 | 4,00000004-0000-0000-0001-020304050607,4.0004,4.04,0,4,4,4,4,4,4,4,2018-01-01T01:01:01.0000000Z,Four,"Four",4,-00:00:04.0040004,,"{""rowId"": 4, ""arr"": [0,4]}" 6 | 5,00000005-0000-0000-0001-020304050607,5.0005,5.05,1,5,5,5,5,5,5,5,2019-01-01T01:01:01.0000000Z,Five,"Five",5,00:00:05.0050005,,"{""rowId"": 5, ""arr"": [0,5]}" 7 | 6,00000006-0000-0000-0001-020304050607,6.0006,6.06,0,6,6,6,6,6,6,6,2020-01-01T01:01:01.0000000Z,Six,"Six",6,-00:00:06.0060006,,"{""rowId"": 6, ""arr"": [0,6]}" 8 | 7,00000007-0000-0000-0001-020304050607,7.0007,7.07,1,7,7,7,7,7,7,7,2021-01-01T01:01:01.0000000Z,Seven,"Seven",7,00:00:07.0070007,,"{""rowId"": 7, ""arr"": [0,7]}" 9 | 8,00000008-0000-0000-0001-020304050607,8.0008,8.08,0,8,8,8,8,8,8,8,2022-01-01T01:01:01.0000000Z,Eight,"Eight",8,-00:00:08.0080008,,"{""rowId"": 8, ""arr"": [0,8]}" 10 | 9,00000009-0000-0000-0001-020304050607,9.0009,9.09,1,9,9,9,9,9,9,9,2023-01-01T01:01:01.0000000Z,Nine,"Nine",9,00:00:09.0090009,,"{""rowId"": 9, ""arr"": [0,9]}" -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/Results.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data; 5 | 6 | import java.util.ArrayList; 7 | import java.util.HashMap; 8 | 9 | public class Results { 10 | private HashMap columnNameToIndex; 11 | private HashMap columnNameToType; 12 | private ArrayList> values; 13 | private String exceptionsMessages; 14 | 15 | public Results(HashMap columnNameToIndex, HashMap columnNameToType, 16 | ArrayList> values, String exceptionsMessages) { 17 | this.columnNameToIndex = columnNameToIndex; 18 | this.columnNameToType = columnNameToType; 19 | this.values = values; 20 | this.exceptionsMessages = exceptionsMessages; 21 | } 22 | 23 | public HashMap getColumnNameToIndex() { 24 | return columnNameToIndex; 25 | } 26 | 27 | public HashMap getColumnNameToType() { 28 | return columnNameToType; 29 | } 30 | 31 | public Integer getIndexByColumnName(String columnName) { 32 | return columnNameToIndex.get(columnName); 33 | } 34 | 35 | public String getTypeByColumnName(String columnName) { 36 | return columnNameToType.get(columnName); 37 | } 38 | 39 | public ArrayList> getValues() { 40 | return values; 41 | } 42 | 43 | public String getExceptions() { 44 | return exceptionsMessages; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/DataWebException.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.exceptions; 5 | 6 | import com.azure.core.http.HttpResponse; 7 | import com.fasterxml.jackson.core.JsonProcessingException; 8 | import com.fasterxml.jackson.databind.ObjectMapper; 9 | import com.microsoft.azure.kusto.data.Utils; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.lang.invoke.MethodHandles; 14 | 15 | public class DataWebException extends WebException { 16 | private OneApiError apiError = null; 17 | 18 | private final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); 19 | private ObjectMapper objectMapper = Utils.getObjectMapper(); 20 | 21 | public DataWebException(String message, HttpResponse httpResponse, Throwable cause) { 22 | super(message, httpResponse, cause); 23 | } 24 | 25 | public DataWebException(String message, HttpResponse httpResponse) { 26 | this(message, httpResponse, null); 27 | } 28 | 29 | public DataWebException(String message) { 30 | this(message, null, null); 31 | } 32 | 33 | public OneApiError getApiError() { 34 | if (apiError == null) { 35 | try { 36 | apiError = OneApiError.fromJsonObject(objectMapper.readTree(getMessage()).get("error")); 37 | } catch (JsonProcessingException e) { 38 | log.error(String.format("failed to parse error from message: '%s' ", e.getMessage()), e); 39 | } 40 | } 41 | 42 | return apiError; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/resources/ContainerWithSas.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.resources; 2 | 3 | import com.azure.core.http.HttpClient; 4 | import com.azure.storage.blob.BlobContainerAsyncClient; 5 | import com.azure.storage.blob.BlobContainerClientBuilder; 6 | import com.microsoft.azure.kusto.data.UriUtils; 7 | 8 | import java.net.URISyntaxException; 9 | 10 | public class ContainerWithSas implements ResourceWithSas { 11 | private final String sas; 12 | private final BlobContainerAsyncClient asyncContainer; 13 | 14 | public ContainerWithSas(String url, HttpClient httpClient) throws URISyntaxException { 15 | String[] parts = UriUtils.getSasAndEndpointFromResourceURL(url); 16 | String endpoint = parts[0]; 17 | String sas = parts[1]; 18 | this.sas = '?' + sas; 19 | 20 | this.asyncContainer = new BlobContainerClientBuilder() 21 | .endpoint(endpoint) 22 | .sasToken(sas) 23 | .httpClient(httpClient) 24 | .buildAsyncClient(); 25 | } 26 | 27 | public String getSas() { 28 | return sas; 29 | } 30 | 31 | public BlobContainerAsyncClient getAsyncContainer() { 32 | return asyncContainer; 33 | } 34 | 35 | @Override 36 | public String getEndpointWithoutSas() { 37 | return asyncContainer.getBlobContainerUrl(); 38 | } 39 | 40 | @Override 41 | public String getAccountName() { 42 | return asyncContainer.getAccountName(); 43 | } 44 | 45 | @Override 46 | public BlobContainerAsyncClient getResource() { 47 | return asyncContainer; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/SubjectNameIssuerTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.microsoft.aad.msal4j.ConfidentialClientApplication; 4 | import com.microsoft.aad.msal4j.IClientCertificate; 5 | import com.microsoft.aad.msal4j.IConfidentialClientApplication; 6 | import java.net.MalformedURLException; 7 | import java.net.URISyntaxException; 8 | 9 | import com.azure.core.http.HttpClient; 10 | import org.jetbrains.annotations.NotNull; 11 | import org.jetbrains.annotations.Nullable; 12 | 13 | // Azure identity doesn't provide a solution for all certificate types, so for now we still use MSAL for this. 14 | 15 | public class SubjectNameIssuerTokenProvider extends ConfidentialAppTokenProviderBase { 16 | private final IClientCertificate clientCertificate; 17 | 18 | SubjectNameIssuerTokenProvider(@NotNull String clusterUrl, @NotNull String applicationClientId, @NotNull IClientCertificate clientCertificate, 19 | String authorityId, @Nullable HttpClient httpClient) throws URISyntaxException { 20 | super(clusterUrl, applicationClientId, authorityId, httpClient); 21 | this.clientCertificate = clientCertificate; 22 | } 23 | 24 | @Override 25 | protected IConfidentialClientApplication getClientApplication() throws MalformedURLException { 26 | ConfidentialClientApplication.Builder builder = ConfidentialClientApplication.builder(applicationClientId, clientCertificate) 27 | .authority(aadAuthorityUrl) 28 | .validateAuthority(false) 29 | .sendX5c(true); 30 | if (httpClient != null) { 31 | builder.httpClient(new HttpClientWrapper(httpClient)); 32 | } 33 | return builder 34 | .build(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /data/src/test/resources/key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN PRIVATE KEY----- 2 | MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCxLlqOxVbcrxz8 3 | CiVihZZBEXoopAg9hvbBkza0WM8mqkCvDVWG2K1lu9h/pXe+Yqf/JYZrRzYytNBa 4 | 1Bc1Mykyf2dtkwVxvTuDspTfcB1WXpQ2/e61+U8Wn6ebzqxSZ97xPzRABJdqpHUr 5 | rr08pOJYFipkHLAbuyn/y+XaFXmDNQNXm23RxMw6FAA2gQrJRYL+HkpS9jNJJEgT 6 | 1nQhAYRkL3SG8JPBe22RJrsN1E/IbhVzTO8nPH0KiL0f2i3rFLCLjQ1Jn7JsEBXh 7 | 39ERuleNJGl1OVTZ9NbX7t5cddI3nmjZrq4lvKd86MHIEzbmYCHXUL+DCI5bvpGe 8 | dC4sY0PjAgMBAAECggEBAI7TG+KsxulcK5QsJZi2sLlA+mUaXR4j1nOSVO2qDUpQ 9 | wZWavQ/XSGeSteGno9APvPaa7DE8FT9oq/AbuZ2D4Ti4pMoaUrAt53SxFvrKaIu/ 10 | 7+AocC22i39kMXsSbax72KZF2bulN92aNm2BDUB+dYqmS4Op8HhaNm/hkCZtWydc 11 | 0to0I+mryMejNQAtKQh+BGRkjReKdQd/DzQju4OhFl4esHnnUa/+rKFdp+9XHNL3 12 | bjQbFtbGbeV/ndfPppf60OIv3nb1eAfXU25ol/jR+xwlleAvk8R5RCtbZBBpzz+z 13 | KTjKxUBqIqwDnn0lF+tImiSCc0itXexJN9UpqffzWmkCgYEA6OJitVdJH5drFBG5 14 | qvCHHoo45Unp8x2iWnfjgRJExmHe8F0MdyKzuyfhdH0aH3bVTXIGkL6+55lBv0Rz 15 | 77XKZzpL+ZFXssELelVnHGfgVe4kYDGeIIwoz1u+W9Mnu3mLYwk0MoP1uVRDqtmm 16 | zCcsqIx4/AyWHR6Z0S3p9zh/630CgYEAwsSK+lgC6gyfoAOQk6orlrdpx7wQf2h9 17 | 5dwX7zfJxczUtRqEKWC1ShmOMHVuOFBf2gndd3dxh2jKL5A7wfkFTAzo8CNHACG5 18 | cpQc/ef18putaPIrhdlit3kDRGsvGzU2NCo2pCWgDUXoSzqGOvmUvfAQKkGrA7Dl 19 | Ipq9YY3fSt8CgYEA1iWWbvpcm1g5drL18GJNXZ/swas+HfjaGia0ZTPPWj0VLCHk 20 | HIkkjK0XfG6tXtF/Qz/fGarNEVq0QpVHpt2byWCqDEvCQPWTAGMX9d7vQxVBDdOS 21 | fItl24eZfAanOpWBvwfvn6QIiOsbbjfGUWSUhEjn154bS5ugBeIaiuLBfmECgYB2 22 | j1niZqzveFu69jCOEuqRxniX4z1PsXXh9IaFUjp7M+LPk6Kb5Yi14LSTfwhnM1YF 23 | zjeIP6dWPE3Hgk8+Fq5IyxsJAxB3XhAbVVoInakz09xNpA0mX1weP4XjojN8pnmm 24 | a4PzjAvPyJxNHwUS5VbMAFR2xJ6rkoUI+ZUKy2f/lQKBgA/v8e5S8gpIfgYniBTr 25 | SL1WLjjg8wvbvfZdTlomW/lctuoe5rfycy4pKo5SzSZwWTpVIJjzWDzkIn9TU68q 26 | J7E4grh3GLhd3tWxKU90l1DAx/yq2neo/V69qqhxE58rjjIbyXw6abf/ZjK6SHnn 27 | 01uI2YeUZyPG4h/vGJBWcEyQ 28 | -----END PRIVATE KEY----- 29 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/IngestClientBaseTest.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import org.junit.jupiter.params.ParameterizedTest; 4 | import org.junit.jupiter.params.provider.Arguments; 5 | import org.junit.jupiter.params.provider.MethodSource; 6 | 7 | import java.util.stream.Stream; 8 | 9 | import static org.junit.jupiter.api.Assertions.assertEquals; 10 | 11 | class IngestClientBaseTest { 12 | private static Stream provideStringsForAutoCorrectEndpointTruePass() { 13 | return Stream.of( 14 | Arguments.of("https://testendpoint.dev.kusto.windows.net", "https://ingest-testendpoint.dev.kusto.windows.net"), 15 | Arguments.of("https://shouldwork", "https://ingest-shouldwork"), 16 | Arguments.of("https://192.shouldwork.1.1", "https://ingest-192.shouldwork.1.1"), 17 | Arguments.of("https://2345:shouldwork:0425", "https://ingest-2345:shouldwork:0425"), 18 | Arguments.of("https://376.568.1564.1564", "https://ingest-376.568.1564.1564"), 19 | Arguments.of("https://192.168.1.1", "https://192.168.1.1"), 20 | Arguments.of("https://[2345:0425:2CA1:0000:0000:0567:5673:23b5]", "https://[2345:0425:2CA1:0000:0000:0567:5673:23b5]"), 21 | Arguments.of("https://127.0.0.1", "https://127.0.0.1"), 22 | Arguments.of("https://localhost", "https://localhost"), 23 | Arguments.of("https://onebox.dev.kusto.windows.net", "https://onebox.dev.kusto.windows.net")); 24 | } 25 | 26 | @ParameterizedTest 27 | @MethodSource("provideStringsForAutoCorrectEndpointTruePass") 28 | void autoCorrectEndpoint_True_Pass(String csb, String expected) { 29 | String actual = IngestClientBase.getIngestionEndpoint(csb); 30 | assertEquals(expected, actual); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/resources/QueueWithSas.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.resources; 2 | 3 | import com.azure.core.http.HttpClient; 4 | import com.azure.storage.common.policy.RequestRetryOptions; 5 | import com.azure.storage.queue.QueueAsyncClient; 6 | import com.azure.storage.queue.QueueClientBuilder; 7 | import com.microsoft.azure.kusto.data.UriUtils; 8 | import reactor.util.annotation.Nullable; 9 | 10 | import java.net.URISyntaxException; 11 | 12 | public class QueueWithSas implements ResourceWithSas { 13 | private final String sas; 14 | private final QueueAsyncClient queueAsyncClient; 15 | 16 | public QueueWithSas(String url, HttpClient httpClient, @Nullable RequestRetryOptions retryOptions) throws URISyntaxException { 17 | String[] parts = UriUtils.getSasAndEndpointFromResourceURL(url); 18 | this.sas = '?' + parts[1]; 19 | this.queueAsyncClient = new QueueClientBuilder() 20 | .endpoint(parts[0]) 21 | .sasToken(parts[1]) 22 | .httpClient(httpClient) 23 | .retryOptions(retryOptions) 24 | .buildAsyncClient(); 25 | } 26 | 27 | public String getSas() { 28 | return sas; 29 | } 30 | 31 | public QueueAsyncClient getAsyncQueue() { 32 | return queueAsyncClient; 33 | } 34 | 35 | public String getEndpoint() { 36 | return queueAsyncClient.getQueueUrl() + sas; 37 | } 38 | 39 | @Override 40 | public String getEndpointWithoutSas() { 41 | return queueAsyncClient.getQueueUrl(); 42 | } 43 | 44 | @Override 45 | public String getAccountName() { 46 | return queueAsyncClient.getAccountName(); 47 | } 48 | 49 | @Override 50 | public QueueAsyncClient getResource() { 51 | return queueAsyncClient; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/ApplicationCertificateTokenProvider.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import com.microsoft.aad.msal4j.ConfidentialClientApplication; 7 | import com.microsoft.aad.msal4j.IClientCertificate; 8 | import com.microsoft.aad.msal4j.IConfidentialClientApplication; 9 | import java.net.MalformedURLException; 10 | import java.net.URISyntaxException; 11 | 12 | import com.azure.core.http.HttpClient; 13 | import org.jetbrains.annotations.NotNull; 14 | import org.jetbrains.annotations.Nullable; 15 | 16 | // Azure identity doesn't provide a solution for all certificate types, so for now we still use MSAL for this. 17 | 18 | public class ApplicationCertificateTokenProvider extends ConfidentialAppTokenProviderBase { 19 | private final IClientCertificate clientCertificate; 20 | 21 | ApplicationCertificateTokenProvider(@NotNull String clusterUrl, @NotNull String applicationClientId, @NotNull IClientCertificate clientCertificate, 22 | String authorityId, @Nullable HttpClient httpClient) throws URISyntaxException { 23 | super(clusterUrl, applicationClientId, authorityId, httpClient); 24 | this.clientCertificate = clientCertificate; 25 | } 26 | 27 | @Override 28 | protected IConfidentialClientApplication getClientApplication() throws MalformedURLException { 29 | ConfidentialClientApplication.Builder builder = ConfidentialClientApplication.builder(applicationClientId, clientCertificate) 30 | .authority(aadAuthorityUrl) 31 | .validateAuthority(false); 32 | if (httpClient != null) { 33 | builder.httpClient(new HttpClientWrapper(httpClient)); 34 | } 35 | return clientApplication = builder 36 | .build(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/TokenProviderBase.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.http.HttpClient; 4 | import com.microsoft.azure.kusto.data.UriUtils; 5 | import com.microsoft.azure.kusto.data.instrumentation.MonitoredActivity; 6 | import com.microsoft.azure.kusto.data.instrumentation.TraceableAttributes; 7 | import org.jetbrains.annotations.NotNull; 8 | import org.jetbrains.annotations.Nullable; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import reactor.core.publisher.Mono; 12 | 13 | import java.net.URISyntaxException; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | 17 | public abstract class TokenProviderBase implements TraceableAttributes { 18 | protected final Logger logger = LoggerFactory.getLogger(getClass()); 19 | protected final String clusterUrl; 20 | protected final HttpClient httpClient; 21 | private final String authMethod; 22 | 23 | public TokenProviderBase(@NotNull String clusterUrl, @Nullable HttpClient httpClient) throws URISyntaxException { 24 | this.clusterUrl = UriUtils.setPathForUri(clusterUrl, ""); 25 | this.httpClient = httpClient; 26 | this.authMethod = getClass().getSimpleName(); 27 | } 28 | 29 | public Mono acquireAccessToken() { 30 | return initialize().then(Mono.defer(() -> MonitoredActivity.wrap(this.acquireAccessTokenImpl(), 31 | getAuthMethod().concat(".acquireAccessToken"), getTracingAttributes()))); 32 | } 33 | 34 | Mono initialize() { 35 | return Mono.empty(); 36 | } 37 | 38 | protected abstract Mono acquireAccessTokenImpl(); 39 | 40 | protected String getAuthMethod() { 41 | return authMethod; 42 | } 43 | 44 | @Override 45 | public Map getTracingAttributes() { 46 | return new HashMap<>(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/Ensure.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data; 5 | 6 | import java.io.File; 7 | import java.io.FileNotFoundException; 8 | 9 | 10 | 11 | public class Ensure { 12 | 13 | public static void stringIsNotBlank(String str, String varName) { 14 | if (StringUtils.isBlank(str)) { 15 | throw new IllegalArgumentException(varName + " is blank."); 16 | } 17 | } 18 | 19 | public static void argIsNotNull(Object arg, String varName) { 20 | if (arg == null) { 21 | throw new IllegalArgumentException(varName + " is null."); 22 | } 23 | } 24 | 25 | public static void fileExists(File file, String argFile) throws FileNotFoundException { 26 | argIsNotNull(file, argFile); 27 | 28 | if (!file.exists()) { 29 | throw new FileNotFoundException(argFile); 30 | } 31 | } 32 | 33 | public static void fileExists(String filePath) throws FileNotFoundException { 34 | stringIsNotBlank(filePath, "filePath"); 35 | 36 | File file = new File(filePath); 37 | fileExists(file, filePath); 38 | } 39 | 40 | public static void isTrue(boolean condition, String message) { 41 | if (!condition) { 42 | throw new IllegalArgumentException("Condition evaluated to false: " + message); 43 | } 44 | } 45 | 46 | public static void isFalse(boolean condition, String message) { 47 | if (condition) { 48 | throw new IllegalArgumentException("Condition evaluated to True: " + message); 49 | } 50 | } 51 | 52 | public static void stringIsNotEmpty(String str, String varName) { 53 | if (StringUtils.isEmpty(str)) { 54 | throw new IllegalArgumentException(varName + " is empty."); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/source/ResultSetSourceInfoTest.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import org.junit.jupiter.api.Assertions; 7 | import org.junit.jupiter.api.DisplayName; 8 | import org.junit.jupiter.api.Test; 9 | import org.mockito.Mockito; 10 | 11 | import java.sql.ResultSet; 12 | import java.util.UUID; 13 | 14 | import static org.junit.jupiter.api.Assertions.assertThrows; 15 | 16 | class ResultSetSourceInfoTest { 17 | @Test 18 | @DisplayName("test set/get resultset") 19 | void resultSet_GetSet_Success() { 20 | ResultSet mockResultSet = Mockito.mock(ResultSet.class); 21 | ResultSetSourceInfo resultSetSourceInfo = new ResultSetSourceInfo(mockResultSet); 22 | 23 | // this also tests that the constructor worked as expected 24 | Assertions.assertEquals(mockResultSet, resultSetSourceInfo.getResultSet()); 25 | 26 | // use the setter to replace the resultset 27 | ResultSet mockResultSet1 = Mockito.mock(ResultSet.class); 28 | resultSetSourceInfo.setResultSet(mockResultSet1); 29 | Assertions.assertEquals(mockResultSet1, resultSetSourceInfo.getResultSet()); 30 | } 31 | 32 | @Test 33 | @DisplayName("test new object with null resultset") 34 | void constructor_NullResultSet_Exception() { 35 | assertThrows(NullPointerException.class, () -> new ResultSetSourceInfo(null)); 36 | } 37 | 38 | @Test 39 | @DisplayName("test object toString") 40 | void toString_Success() { 41 | ResultSet mockResultSet = Mockito.mock(ResultSet.class); 42 | UUID uuid = UUID.randomUUID(); 43 | 44 | ResultSetSourceInfo resultSetSourceInfo = new ResultSetSourceInfo(mockResultSet, uuid); 45 | 46 | Assertions.assertTrue(resultSetSourceInfo.toString().contains(uuid.toString())); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | # This workflow will install the dependencies, run tests and lint every push 2 | 3 | name: Build Java 4 | 5 | on: 6 | push: 7 | branches: [ 'master' ] 8 | pull_request: 9 | branches: [ '**' ] 10 | jobs: 11 | build: 12 | runs-on: ubuntu-latest 13 | environment: build 14 | permissions: 15 | checks: write 16 | pull-requests: write 17 | id-token: write 18 | contents: read 19 | strategy: 20 | matrix: 21 | java: ['8', '11','17','21'] 22 | name: Java ${{ matrix.java }} 23 | steps: 24 | - name: Azure login 25 | uses: azure/login@v2 26 | with: 27 | client-id: ${{ secrets.APP_ID }} 28 | tenant-id: ${{ secrets.TENANT_ID }} 29 | subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} 30 | - uses: actions/checkout@v4 31 | - name: Setup java ${{ matrix.java }} 32 | uses: actions/setup-java@v4 33 | with: 34 | distribution: 'adopt' 35 | java-version: ${{ matrix.java }} 36 | cache: 'maven' 37 | - name: Run formatter 38 | run: mvn formatter:validate 39 | continue-on-error: true 40 | - name: Run the Maven verify phase 41 | run: mvn verify 42 | env: 43 | DM_CONNECTION_STRING: ${{ secrets.DM_CONNECTION_STRING }} 44 | ENGINE_CONNECTION_STRING: ${{ secrets.ENGINE_CONNECTION_STRING }} 45 | TEST_DATABASE: ${{ secrets.TEST_DATABASE }} 46 | APP_ID: ${{ secrets.APP_ID }} 47 | TENANT_ID: ${{ secrets.TENANT_ID }} 48 | CI_EXECUTION: 1 49 | - name: Publish Unit Test Results 50 | uses: EnricoMi/publish-unit-test-result-action@v2 51 | if: always() 52 | with: 53 | files: | 54 | data/target/surefire-reports/*.xml 55 | ingest/target/surefire-reports/*.xml 56 | - name: Upload coverage to Codecov 57 | uses: codecov/codecov-action@v4 58 | -------------------------------------------------------------------------------- /data/src/test/resources/cert.cer: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFRDCCAywCCQDi9Sxrynx1JDANBgkqhkiG9w0BAQsFADBkMQswCQYDVQQGEwJJ 3 | TDESMBAGA1UEBwwJSEVyemVsaXlhMQswCQYDVQQKDAJNUzEMMAoGA1UECwwDQ1NF 4 | MQ4wDAYDVQQDDAViYXNpYzEWMBQGCSqGSIb3DQEJARYHYUBhLmNvbTAeFw0xODEw 5 | MjIwODIyNDNaFw0xOTEwMjIwODIyNDNaMGQxCzAJBgNVBAYTAklMMRIwEAYDVQQH 6 | DAlIRXJ6ZWxpeWExCzAJBgNVBAoMAk1TMQwwCgYDVQQLDANDU0UxDjAMBgNVBAMM 7 | BWJhc2ljMRYwFAYJKoZIhvcNAQkBFgdhQGEuY29tMIICIjANBgkqhkiG9w0BAQEF 8 | AAOCAg8AMIICCgKCAgEA72B+pxRQljeu28TwB5QYMMYhqZc1hw8cofI9nhdX7/lS 9 | 3HO1RGQXLxWRM2UNN9jUuC3DL1XTm3uHAq5/jW9gmL67I8tVYRtHiaLWXleJVWVT 10 | L+tDfSAHq+XEAL1St7sgMqy7phK81cN+g83eE3w3jQ5O8ohInDnZTCE6g2MX88KE 11 | f+diIjYSO8ITqNhSrdTF+D/nDjQOMj2TXsBokJHL+CvMVYntyhE9DNCq0VNExuFH 12 | Os1l9w88Xteu6bz3oylB1xGx5z7UCEXb/O+3bo1Cy47e/m4lhlawhtxVDule5eLd 13 | B59T0S5TKoYoTFrEisqrCm8wqlQy5e9aY8Crk7gPhlSoKjbvm5rwqt2tw20lyIpw 14 | Cw3NYsvkdjOTIGfeIskuB2S9u/zAoDSqBY9CruA8GGRk9M+RG2S7WbFSzPujkO+7 15 | HX4At6+mnUDkkzZMK8H59jWWqZr3bdg6L6grodcJjhMD5bv2HHpm8H5mtfGIg9g3 16 | g/hdo7ih02mNHsKL/KFQUQwTOk3ZHfhHZYXLqr7bd+ReH5eB10aPsLJlHRGJbb7k 17 | UawhiDb2Ps8zxLO+ubIPaBrK/emA+C+PhTxajR+BCYB9iT2v1/pilJtCx5YvX2w5 18 | lc1V7ULvAvEVqAClcUetAPYCMpaJCrWLlGTWGPnoyN+lCrMnlDduzapbEy/JPOcC 19 | AwEAATANBgkqhkiG9w0BAQsFAAOCAgEA6Omd7Yf5in1MSV3Chw6qLqS8QZzL5im7 20 | 6acBQDTuuDxYGc3MWrWDsgmePCH7ajxr62G04+ca1U2diUNirnM7lpDhzzUTuLnq 21 | AV5iN/BhiCP56YZoDbkrIKztrvnQiRJFsjKE5o46D7Tj6us+8amnMXOJo+TKRZL5 22 | R7QQKaupJe3vtCEnqrZ8RMyPy3hGTjLq8DAxOZLa9LadcBIZzbLGaGOMoVdh1mlU 23 | c/Z8+zKlzsCcO7HAA0wRr2Yy2ApQOGpi0RkmaqN2FHRXT5b2apJE3He0m1vM5s4v 24 | T/QJE1c/mfSmHwIq3vfcywdXNUzrTaE5pTao/8suRkDZt6gS9/nTnKf9kw8sFLLL 25 | Hj/nmDlGbUXUn9RyxtiCDHanyVBTBXI+cqjNYrDwk3Yny1IVOf6IVe1uNO34v+Py 26 | T7s/RiEMnOpx5JPTjRjbipWtEjivyMwl8BPkcjCGpL2OTReyraF1VABA5iTUK7CR 27 | SUFpuahaXkm9GFAEN2oSRVNYRHV1bwjuwUkpSqJZK7xSrbQTpnHElW69OJO6Zqcs 28 | WPkwNpzXNZr9o6Mj54/PPm/oJWomDZvY7TCvmFd335ziXgyPPBz/RtTC4hxwlHdw 29 | kxVZ8qyiFzwgzPczvcRp9ZfQqGmM/3L0UQo+IJjGd7EE1v4l26ideHDsmlwsGFD1 30 | tIQHGM/Phe4= 31 | -----END CERTIFICATE----- 32 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/endpoints/WellKnownKustoEndpointsData.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth.endpoints; 2 | 3 | import com.fasterxml.jackson.annotation.JsonIgnore; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.microsoft.azure.kusto.data.Utils; 6 | import org.jetbrains.annotations.NotNull; 7 | 8 | import java.io.InputStream; 9 | import java.util.ArrayList; 10 | import java.util.HashMap; 11 | 12 | public class WellKnownKustoEndpointsData { 13 | public static class AllowedEndpoints { 14 | public ArrayList AllowedKustoSuffixes; 15 | public ArrayList AllowedKustoHostnames; 16 | } 17 | 18 | @JsonIgnore 19 | public ArrayList _Comments; 20 | public HashMap AllowedEndpointsByLogin; 21 | private static WellKnownKustoEndpointsData instance = null; 22 | private static final Object object = new Object(); 23 | 24 | public static WellKnownKustoEndpointsData getInstance() { 25 | if (instance != null) 26 | return instance; 27 | synchronized (object) { 28 | if (instance == null) { 29 | instance = readInstance(); 30 | } 31 | } 32 | return instance; 33 | } 34 | 35 | // For Deserialization 36 | public WellKnownKustoEndpointsData() { 37 | } 38 | 39 | @NotNull 40 | private static WellKnownKustoEndpointsData readInstance() { 41 | try { 42 | // Beautiful ! 43 | ObjectMapper objectMapper = Utils.getObjectMapper(); 44 | try (InputStream resourceAsStream = WellKnownKustoEndpointsData.class.getResourceAsStream( 45 | "/WellKnownKustoEndpoints.json")) { 46 | return objectMapper.readValue(resourceAsStream, WellKnownKustoEndpointsData.class); 47 | } 48 | } catch (Exception ex) { 49 | throw new RuntimeException("Failed to read WellKnownKustoEndpoints.json", ex); 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/http/UncloseableStream.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | 8 | /** 9 | * This class exists to handle outside dependencies which close a stream when we don't want to. 10 | * It takes a stream, and simply forwards the call to all of its methods, except for the {@code close()} method - which does nothing. 11 | */ 12 | public class UncloseableStream extends InputStream { 13 | public InputStream getInnerStream() { 14 | return innerStream; 15 | } 16 | 17 | private final InputStream innerStream; 18 | 19 | public UncloseableStream(InputStream innerStream) { 20 | this.innerStream = innerStream; 21 | } 22 | 23 | /** 24 | * Explicitly does nothing, does preserving the inner stream as open 25 | */ 26 | @Override 27 | public void close() { 28 | // Explicitly do nothing 29 | } 30 | 31 | @Override 32 | public int read() throws IOException { 33 | return innerStream.read(); 34 | } 35 | 36 | @Override 37 | public int read(@NotNull byte[] b) throws IOException { 38 | return innerStream.read(b); 39 | } 40 | 41 | @Override 42 | public int read(@NotNull byte[] b, int off, int len) throws IOException { 43 | return innerStream.read(b, off, len); 44 | } 45 | 46 | @Override 47 | public long skip(long n) throws IOException { 48 | return innerStream.skip(n); 49 | } 50 | 51 | @Override 52 | public int available() throws IOException { 53 | return innerStream.available(); 54 | } 55 | 56 | @Override 57 | public void mark(int readlimit) { 58 | innerStream.mark(readlimit); 59 | } 60 | 61 | @Override 62 | public void reset() throws IOException { 63 | innerStream.reset(); 64 | } 65 | 66 | @Override 67 | public boolean markSupported() { 68 | return innerStream.markSupported(); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /samples/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | kusto-samples 7 | 8 | 9 | kusto-client 10 | com.microsoft.azure.kusto 11 | 12 | ${revision} 13 | 14 | 15 | 16 | 17 | 18 | org.apache.maven.plugins 19 | maven-compiler-plugin 20 | ${maven-compiler-plugin.version} 21 | 22 | ${java.version} 23 | ${java.version} 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | com.microsoft.azure.kusto 32 | kusto-data 33 | ${project.parent.version} 34 | 35 | 36 | com.microsoft.azure.kusto 37 | kusto-ingest 38 | ${project.parent.version} 39 | 40 | 41 | com.fasterxml.jackson.core 42 | jackson-databind 43 | ${fasterxml.jackson.core.version} 44 | 45 | 46 | com.fasterxml.jackson.datatype 47 | jackson-datatype-jsr310 48 | ${fasterxml.jackson.core.version} 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/ExceptionUtils.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.exceptions; 2 | 3 | import java.io.IOException; 4 | import java.io.UncheckedIOException; 5 | import java.net.URL; 6 | 7 | import com.microsoft.azure.kusto.data.Utils; 8 | 9 | public class ExceptionUtils { 10 | public static DataServiceException createExceptionOnPost(Exception e, URL url, String kind) { 11 | boolean permanent = false; 12 | boolean isIO = false; 13 | if (e instanceof IOException) { 14 | isIO = true; 15 | } 16 | 17 | if (e instanceof UncheckedIOException) { 18 | e = ((UncheckedIOException) e).getCause(); 19 | isIO = true; 20 | } 21 | 22 | if (isIO) { 23 | return new IODataServiceException(url.toString(), (IOException) e, kind); 24 | 25 | } 26 | 27 | return new DataServiceException(url.toString(), String.format("Exception in %s post request: %s", kind, e.getMessage()), permanent); 28 | } 29 | 30 | // Useful in IOException, where message might not propagate to the base IOException 31 | public static String getMessageEx(Exception e) { 32 | return (e.getMessage() == null && e.getCause() != null) ? e.getCause().getMessage() : e.getMessage(); 33 | } 34 | 35 | public static Exception unwrapCloudInfoException(String clusterUrl, Throwable throwable) { 36 | if (throwable instanceof IOException) { 37 | IOException ex = (IOException) throwable; 38 | if (!Utils.isRetriableIOException(ex)) { 39 | return new DataServiceException(clusterUrl, "IOException when trying to retrieve cluster metadata: " + getMessageEx(ex), 40 | ex, 41 | Utils.isRetriableIOException(ex)); 42 | } 43 | } 44 | 45 | if (throwable instanceof DataServiceException) { 46 | DataServiceException e = (DataServiceException) throwable; 47 | if (e.isPermanent()) { 48 | return e; 49 | } 50 | } 51 | 52 | return new DataClientException(clusterUrl, throwable.toString(), null); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/TableReportIngestionResult.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | import com.azure.data.tables.TableAsyncClient; 7 | import com.azure.data.tables.implementation.models.TableServiceErrorException; 8 | import reactor.core.publisher.Mono; 9 | 10 | import java.net.URISyntaxException; 11 | import java.util.LinkedList; 12 | import java.util.List; 13 | import java.util.stream.Collectors; 14 | 15 | public class TableReportIngestionResult implements IngestionResult { 16 | private final List descriptors; 17 | 18 | public TableReportIngestionResult(List descriptors) { 19 | this.descriptors = descriptors; 20 | } 21 | 22 | @Override 23 | public Mono> getIngestionStatusCollectionAsync() throws TableServiceErrorException { 24 | List> ingestionStatusMonos = descriptors.stream() 25 | .map(descriptor -> { 26 | TableAsyncClient tableAsyncClient = descriptor.getTableAsyncClient(); 27 | if (tableAsyncClient != null) { 28 | return tableAsyncClient.getEntity(descriptor.getPartitionKey(), descriptor.getRowKey()) 29 | .map(IngestionStatus::fromEntity); 30 | } else { 31 | return Mono.empty(); 32 | } 33 | }) 34 | .collect(Collectors.toList()); 35 | 36 | return Mono.zip(ingestionStatusMonos, results -> { 37 | List resultList = new LinkedList<>(); 38 | for (Object result : results) { 39 | resultList.add((IngestionStatus) result); 40 | } 41 | return resultList; 42 | }); 43 | } 44 | 45 | @Override 46 | public List getIngestionStatusCollection() throws URISyntaxException, TableServiceErrorException { 47 | return getIngestionStatusCollectionAsync().block(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/HttpClientWrapper.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.azure.core.http.*; 4 | import com.azure.core.util.Context; 5 | import com.azure.core.util.CoreUtils; 6 | import com.microsoft.aad.msal4j.IHttpClient; 7 | import com.microsoft.aad.msal4j.IHttpResponse; 8 | 9 | import java.util.stream.Collectors; 10 | 11 | /** 12 | * This class wraps an Azure Core HttpClient to be used as an MSAL IHttpClient. 13 | */ 14 | public class HttpClientWrapper implements IHttpClient { 15 | 16 | private final HttpClient httpClient; 17 | 18 | public HttpClientWrapper(HttpClient httpClient) { 19 | this.httpClient = httpClient; 20 | } 21 | 22 | // Implementation of the synchronous HttpClient 23 | @Override 24 | public IHttpResponse send(com.microsoft.aad.msal4j.HttpRequest httpRequest) { 25 | HttpMethod method; 26 | switch (httpRequest.httpMethod()) { 27 | case GET: 28 | method = HttpMethod.GET; 29 | break; 30 | case POST: 31 | method = HttpMethod.POST; 32 | break; 33 | default: 34 | throw new IllegalArgumentException("Unsupported HTTP method: " + httpRequest.httpMethod()); 35 | } 36 | 37 | // Generate an azure core HttpRequest from the existing msal4j HttpRequest 38 | HttpRequest request = new HttpRequest(method, httpRequest.url(), new HttpHeaders(httpRequest.headers())); 39 | if (!CoreUtils.isNullOrEmpty(httpRequest.body())) { 40 | request.setBody(httpRequest.body()); 41 | } 42 | 43 | try (HttpResponse response = httpClient.sendSync(request, Context.NONE)) { 44 | com.microsoft.aad.msal4j.HttpResponse msalResponse = new com.microsoft.aad.msal4j.HttpResponse(); 45 | msalResponse.statusCode(response.getStatusCode()); 46 | msalResponse.body(response.getBodyAsBinaryData().toString()); 47 | msalResponse.addHeaders(response.getHeaders().stream().collect(Collectors.toMap(HttpHeader::getName, 48 | HttpHeader::getValuesList))); 49 | return msalResponse; 50 | } 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /samples/src/main/java/Query.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | import com.microsoft.azure.kusto.data.Client; 5 | import com.microsoft.azure.kusto.data.ClientFactory; 6 | import com.microsoft.azure.kusto.data.ClientRequestProperties; 7 | import com.microsoft.azure.kusto.data.http.HttpClientProperties; 8 | import com.microsoft.azure.kusto.data.KustoOperationResult; 9 | import com.microsoft.azure.kusto.data.KustoResultSetTable; 10 | import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; 11 | 12 | import java.util.List; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | public class Query { 16 | public static void main(String[] args) { 17 | try { 18 | ConnectionStringBuilder csb = ConnectionStringBuilder.createWithAadApplicationCredentials( 19 | System.getProperty("clusterPath"), 20 | System.getProperty("appId"), 21 | System.getProperty("appKey"), 22 | System.getProperty("appTenant")); 23 | 24 | HttpClientProperties properties = HttpClientProperties.builder() 25 | .keepAlive(true) 26 | .maxConnectionsTotal(40) 27 | .build(); 28 | 29 | Client client = ClientFactory.createClient(csb, properties); 30 | 31 | KustoOperationResult results = client.executeQuery(".show version"); 32 | KustoResultSetTable mainTableResult = results.getPrimaryResults(); 33 | System.out.printf("Kusto sent back %s rows.%n", mainTableResult.count()); 34 | 35 | // iterate values 36 | while (mainTableResult.next()) { 37 | List nextValue = mainTableResult.getCurrentRow(); 38 | } 39 | 40 | // in case we want to pass client request properties 41 | ClientRequestProperties clientRequestProperties = new ClientRequestProperties(); 42 | clientRequestProperties.setTimeoutInMilliSec(TimeUnit.MINUTES.toMillis(1)); 43 | 44 | results = client.executeQuery(System.getProperty("dbName"), System.getProperty("query"), clientRequestProperties); 45 | } catch (Exception e) { 46 | e.printStackTrace(); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /quickstart/README.md: -------------------------------------------------------------------------------- 1 | # Quickstart App 2 | 3 | The quickstart application is a **self-contained and runnable** example app that demonstrates authenticating, connecting to, administering, ingesting data into and querying Azure Data Explorer using the azure-kusto-java SDK. 4 | You can use it as a baseline to write your own first kusto client application, altering the code as you go, or copy code sections out of it into your app. 5 | 6 | **Tip:** The app includes comments with tips on recommendations, coding best practices, links to reference materials and recommended TODO changes when adapting the code to your needs. 7 | 8 | 9 | ## Using the App for the first time 10 | 11 | ### Prerequisites 12 | 13 | 1. Set up Java version 8 or higher on your machine. For instructions, consult a Java environment setup tutorial, like [this](https://www.tutorialspoint.com/java/java_environment_setup.htm). 14 | 1. Set up [Apache Maven](https://maven.apache.org/install.html), which is the most popular Java dependency management tool. 15 | 16 | ### Retrieving the app from GitHub 17 | 18 | 1. Download the app files from this GitHub repo. 19 | 1. Modify the `kusto_sample_config.json` file, changing `KustoUri`, `IngestUri` and `DatabaseName` appropriately for your cluster. 20 | 21 | ### Retrieving the app from OneClick 22 | 23 | 1. Open a browser and type your cluster's URL (e.g. https://mycluster.westeurope.kusto.windows.net/). You will be redirected to the _Azure Data Explorer_ Web UI. 24 | 1. On the left menu, select **Data**. 25 | 1. Click **Generate Sample App Code** and then follow the instructions in the wizard. 26 | 1. Download the app as a ZIP file. 27 | 1. Extract the app source code. 28 | **Note**: The configuration parameters defined in the `kusto_sample_config.json` file are preconfigured with the appropriate values for your cluster. Verify that these are correct. 29 | 30 | ### Run the app 31 | 32 | 1. Open a command line window and navigate to the folder where you extracted the app. 33 | 1. Run `mvn clean install` to compile the source code into a binary. 34 | 1. Run the binary using `java -jar target\kusto-quickstart-[version]-jar-with-dependencies.jar`. 35 | 36 | #### Troubleshooting 37 | 38 | * If you are having trouble running the app from your IDE, first check if the app runs from the command line, then consult the troubleshooting references of your IDE. -------------------------------------------------------------------------------- /ingest/src/test/java/com/microsoft/azure/kusto/ingest/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import com.microsoft.azure.kusto.ingest.resources.ContainerWithSas; 4 | import com.microsoft.azure.kusto.ingest.resources.QueueWithSas; 5 | import com.microsoft.azure.kusto.ingest.utils.TableWithSas; 6 | 7 | import java.net.URISyntaxException; 8 | 9 | public class TestUtils { 10 | static QueueWithSas queueWithSasFromQueueName(String queueName) { 11 | try { 12 | return new QueueWithSas(String.format("https://storage.queue.core.windows.net/%s?sas", queueName), null, null); 13 | } catch (URISyntaxException ex) { 14 | return null; 15 | } 16 | } 17 | 18 | static QueueWithSas queueWithSasFromAccountNameAndQueueName(String accountName, String queueName) { 19 | try { 20 | return new QueueWithSas(String.format("https://%s.blob.core.windows.net/%s?sas", accountName, queueName), null, null); 21 | } catch (URISyntaxException ex) { 22 | return null; 23 | } 24 | } 25 | 26 | static ContainerWithSas containerWithSasFromContainerName(String containerName) { 27 | try { 28 | return new ContainerWithSas(String.format("https://storage.blob.core.windows.net/%s?sas", containerName), null); 29 | } catch (URISyntaxException ex) { 30 | return null; 31 | } 32 | } 33 | 34 | static ContainerWithSas containerWithSasFromAccountNameAndContainerName(String accountName, String containerName) { 35 | try { 36 | return new ContainerWithSas(String.format("https://%s.blob.core.windows.net/%s?sas", accountName, containerName), null); 37 | } catch (URISyntaxException ex) { 38 | return null; 39 | } 40 | } 41 | 42 | static String blobWithSasFromAccountNameAndContainerName(String accountName, String containerName, String blobName) { 43 | return String.format("https://%s.blob.core.windows.net/%s/%s?sas", accountName, containerName, blobName); 44 | } 45 | 46 | static TableWithSas tableWithSasFromTableName(String tableName) { 47 | try { 48 | return new TableWithSas(String.format("https://storage.table.core.windows.net/%s?sas", tableName), null); 49 | } catch (URISyntaxException ex) { 50 | return null; 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/KustoType.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | import java.time.Instant; 4 | import java.time.ZonedDateTime; 5 | import java.util.UUID; 6 | import java.util.function.Function; 7 | 8 | /** 9 | * Represents various ADX data types and their bindings to Java classes. 10 | * 11 | * See https://docs.microsoft.com/azure/data-explorer/kusto/query/scalar-data-types/ 12 | * 13 | * @param The column Java data type 14 | */ 15 | public class KustoType { 16 | 17 | public static final KustoType INTEGER = new KustoType<>(Integer.class); 18 | // Longs often return as "Integer", so we need to cast to "number", and then get the long value: 19 | public static final KustoType LONG = new KustoType<>(Long.class, o -> Long.valueOf(Number.class.cast(o).longValue())); 20 | public static final KustoType GUID_UUID = new KustoType<>(UUID.class, o -> UUID.fromString(o.toString())); 21 | public static final KustoType GUID_STRING = new KustoType<>(String.class); 22 | public static final KustoType STRING = new KustoType<>(String.class); 23 | public static final KustoType REAL_DOUBLE = new KustoType<>(Double.class); 24 | public static final KustoType REAL_FLOAT = new KustoType<>(Float.class, o -> Float.valueOf(Double.class.cast(o).floatValue())); 25 | 26 | public static final KustoType DATETIME_ZONED_DATE_TIME = new KustoType<>(ZonedDateTime.class, o -> ZonedDateTime.parse(o.toString())); 27 | public static final KustoType DATETIME_INSTANT = new KustoType<>(Instant.class, o -> Instant.parse(o.toString())); 28 | public static final KustoType DATETIME_LONG = new KustoType<>(Long.class, o -> Instant.parse(o.toString()).toEpochMilli()); 29 | 30 | public static final KustoType OBJECT = new KustoType<>(Object.class); 31 | 32 | final Class clazz; 33 | final Function typer; 34 | 35 | private KustoType(Class clazz) { 36 | this(clazz, clazz::cast); 37 | } 38 | 39 | private KustoType(Class clazz, Function typer) { 40 | this.clazz = clazz; 41 | this.typer = typer; 42 | } 43 | 44 | public C type(Object o) { 45 | return this.typer.apply(o); 46 | } 47 | 48 | public Class getTypeClass() { 49 | return this.clazz; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/ResultSetSourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import java.sql.ResultSet; 7 | import java.util.Map; 8 | import java.util.Objects; 9 | import java.util.UUID; 10 | 11 | /** 12 | * Represents the ResultSet source information used for ingestion. 13 | */ 14 | public class ResultSetSourceInfo extends AbstractSourceInfo { 15 | 16 | private ResultSet resultSet; 17 | 18 | /** 19 | * Creates a ResultSetSourceInfo. 20 | * 21 | * @param resultSet The ResultSet with the data to be ingested. 22 | */ 23 | public ResultSetSourceInfo(ResultSet resultSet) { 24 | setResultSet(resultSet); 25 | } 26 | 27 | /** 28 | * Creates a ResultSetSourceInfo 29 | * 30 | * @param resultSet The ResultSet with the data to be ingested. 31 | * @param sourceId An identifier that could later be used to trace this specific source data. 32 | */ 33 | public ResultSetSourceInfo(ResultSet resultSet, UUID sourceId) { 34 | setResultSet(resultSet); 35 | this.setSourceId(sourceId); 36 | } 37 | 38 | /** 39 | * Gets the ResultSet. 40 | * 41 | * @return The ResultSet in the SourceInfo 42 | */ 43 | public ResultSet getResultSet() { 44 | return resultSet; 45 | } 46 | 47 | /** 48 | * Sets the ResultSet. 49 | * 50 | * @param resultSet The ResultSet with the data to be ingested. 51 | */ 52 | @SuppressWarnings("WeakerAccess") 53 | public void setResultSet(ResultSet resultSet) { 54 | this.resultSet = Objects.requireNonNull(resultSet, "ResultSet cannot be null"); 55 | } 56 | 57 | @Override 58 | public String toString() { 59 | return String.format("ResultSet with SourceId: %s", getSourceId()); 60 | } 61 | 62 | public void validate() { 63 | // nothing to validate as of now. 64 | } 65 | 66 | public Map getTracingAttributes() { 67 | Map attributes = super.getTracingAttributes(); 68 | attributes.put("resource", "resultSet"); 69 | UUID sourceId = getSourceId(); 70 | if (sourceId != null) { 71 | attributes.put("sourceId", sourceId.toString()); 72 | } 73 | return attributes; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/OneApiError.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.exceptions; 5 | 6 | import com.fasterxml.jackson.databind.JsonNode; 7 | import com.fasterxml.jackson.databind.node.ArrayNode; 8 | 9 | import java.util.Spliterator; 10 | import java.util.Spliterators; 11 | import java.util.stream.StreamSupport; 12 | 13 | public class OneApiError { 14 | public OneApiError(String code, String message, String description, String type, JsonNode context, boolean permanent) { 15 | this.code = code; 16 | this.message = message; 17 | this.description = description; 18 | this.type = type; 19 | this.context = context; 20 | this.permanent = permanent; 21 | } 22 | 23 | public static OneApiError[] fromJsonArray(ArrayNode array) { 24 | return StreamSupport.stream(Spliterators.spliteratorUnknownSize(array.elements(), Spliterator.ORDERED), false) 25 | .map(x -> OneApiError.fromJsonObject(x.get("error"))) 26 | .toArray(OneApiError[]::new); 27 | } 28 | 29 | public static OneApiError fromJsonObject(JsonNode jsonObject) { 30 | return new OneApiError( 31 | jsonObject.has("code") ? jsonObject.get("code").asText() : "", 32 | jsonObject.has("message") ? jsonObject.get("message").asText() : "", 33 | jsonObject.has("@message") ? jsonObject.get("@message").asText() : "", 34 | jsonObject.has("@type") ? jsonObject.get("@type").asText() : "", 35 | jsonObject.get("@context"), 36 | jsonObject.has("@permanent") && jsonObject.get("@permanent").asBoolean()); 37 | } 38 | 39 | private final String code; 40 | private final String message; 41 | private final String description; 42 | private final String type; 43 | private final JsonNode context; 44 | private final boolean permanent; 45 | 46 | public String getCode() { 47 | return code; 48 | } 49 | 50 | public String getMessage() { 51 | return message; 52 | } 53 | 54 | public String getDescription() { 55 | return description; 56 | } 57 | 58 | public String getType() { 59 | return type; 60 | } 61 | 62 | public JsonNode getContext() { 63 | return context; 64 | } 65 | 66 | public boolean isPermanent() { 67 | return permanent; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/IngestionStatusInTableDescription.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.result; 5 | 6 | import com.azure.data.tables.TableAsyncClient; 7 | import com.fasterxml.jackson.annotation.JsonIgnore; 8 | import com.fasterxml.jackson.annotation.JsonProperty; 9 | import com.microsoft.azure.kusto.ingest.utils.TableWithSas; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | import reactor.util.annotation.Nullable; 13 | 14 | import java.io.Serializable; 15 | import java.lang.invoke.MethodHandles; 16 | import java.net.URISyntaxException; 17 | 18 | public class IngestionStatusInTableDescription implements Serializable { 19 | private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); 20 | @JsonProperty 21 | private String tableConnectionString; 22 | 23 | @JsonProperty 24 | private String partitionKey; 25 | 26 | @JsonProperty 27 | private String rowKey; 28 | 29 | @JsonIgnore 30 | private transient TableAsyncClient tableAsyncClient; 31 | 32 | public String getTableConnectionString() { 33 | return this.tableConnectionString; 34 | } 35 | 36 | public void setTableConnectionString(String tableConnectionString) { 37 | this.tableConnectionString = tableConnectionString; 38 | } 39 | 40 | public String getPartitionKey() { 41 | return partitionKey; 42 | } 43 | 44 | public void setPartitionKey(String partitionKey) { 45 | this.partitionKey = partitionKey; 46 | } 47 | 48 | public String getRowKey() { 49 | return rowKey; 50 | } 51 | 52 | public void setRowKey(String rowKey) { 53 | this.rowKey = rowKey; 54 | } 55 | 56 | @Nullable 57 | public TableAsyncClient getTableAsyncClient() { 58 | if (tableAsyncClient == null) { 59 | try { 60 | tableAsyncClient = TableWithSas.createTableClientFromUrl(getTableConnectionString(), null); 61 | } catch (URISyntaxException uriSyntaxException) { 62 | log.error("TableConnectionString could not be parsed as URI reference.", uriSyntaxException); 63 | return null; 64 | } 65 | } 66 | 67 | return tableAsyncClient; 68 | } 69 | 70 | public void setAsyncTableClient(TableAsyncClient tableAsyncClient) { 71 | this.tableAsyncClient = tableAsyncClient; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/result/ValidationPolicy.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest.result; 2 | 3 | import com.fasterxml.jackson.annotation.JsonValue; 4 | import java.io.Serializable; 5 | 6 | public class ValidationPolicy implements Serializable { 7 | private ValidationOptions validationOptions = ValidationOptions.DO_NOT_VALIDATE; 8 | private ValidationImplications validationPolicyType = ValidationImplications.VALIDATION_IMPLICATIONS; 9 | 10 | public ValidationPolicy() { 11 | } 12 | 13 | public ValidationPolicy(ValidationOptions validationOptions, ValidationImplications validationPolicyType) { 14 | this.validationOptions = validationOptions; 15 | this.validationPolicyType = validationPolicyType; 16 | } 17 | 18 | public ValidationPolicy(ValidationPolicy other) { 19 | this.validationOptions = other.validationOptions; 20 | this.validationPolicyType = other.validationPolicyType; 21 | } 22 | 23 | public ValidationOptions getValidationOptions() { 24 | return validationOptions; 25 | } 26 | 27 | public void setValidationOptions(ValidationOptions validationOptions) { 28 | this.validationOptions = validationOptions; 29 | } 30 | 31 | public ValidationImplications getValidationPolicyType() { 32 | return validationPolicyType; 33 | } 34 | 35 | public void setValidationPolicyType(ValidationImplications validationPolicyType) { 36 | this.validationPolicyType = validationPolicyType; 37 | } 38 | 39 | public enum ValidationOptions { 40 | DO_NOT_VALIDATE("DoNotValidate"), 41 | VALIDATE_CSV_INPUT_CONSTANT_COLUMNS("ValidateCsvInputConstantColumns"), 42 | VALIDATE_CSV_INPUT_COLUMN_LEVEL_ONLY("ValidateCsvInputColumnLevelOnly"); 43 | 44 | private final String kustoValue; 45 | 46 | ValidationOptions(String kustoValue) { 47 | this.kustoValue = kustoValue; 48 | } 49 | 50 | @JsonValue 51 | public String getKustoValue() { 52 | return kustoValue; 53 | } 54 | } 55 | 56 | public enum ValidationImplications { 57 | FAIL("Fail"), 58 | VALIDATION_IMPLICATIONS("BestEffort"); 59 | 60 | private final String kustoValue; 61 | 62 | ValidationImplications(String kustoValue) { 63 | this.kustoValue = kustoValue; 64 | } 65 | 66 | @JsonValue 67 | public String getKustoValue() { 68 | return kustoValue; 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /data/src/test/java/com/microsoft/azure/kusto/data/http/TestHttpResponse.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | import com.azure.core.http.*; 4 | import reactor.core.publisher.Flux; 5 | import reactor.core.publisher.Mono; 6 | 7 | import java.nio.ByteBuffer; 8 | import java.nio.charset.Charset; 9 | 10 | public class TestHttpResponse extends HttpResponse { 11 | 12 | private int fakeStatusCode; 13 | private final HttpHeaders fakeHeaders = new HttpHeaders(); 14 | 15 | private TestHttpResponse() { 16 | super(null); 17 | } 18 | 19 | protected TestHttpResponse(HttpRequest request) { 20 | super(request); 21 | } 22 | 23 | @Override 24 | public int getStatusCode() { 25 | return fakeStatusCode; 26 | } 27 | 28 | private void setStatusCode(int statusCode) { 29 | this.fakeStatusCode = statusCode; 30 | } 31 | 32 | @Override 33 | public String getHeaderValue(String s) { 34 | return fakeHeaders.getValue(HttpHeaderName.fromString(s)); 35 | } 36 | 37 | @Override 38 | public HttpHeaders getHeaders() { 39 | return fakeHeaders; 40 | } 41 | 42 | private void addHeader(String name, String value) { 43 | fakeHeaders.add(HttpHeaderName.fromString(name), value); 44 | } 45 | 46 | @Override 47 | public Flux getBody() { 48 | return null; 49 | } 50 | 51 | @Override 52 | public Mono getBodyAsByteArray() { 53 | return null; 54 | } 55 | 56 | @Override 57 | public Mono getBodyAsString() { 58 | return null; 59 | } 60 | 61 | @Override 62 | public Mono getBodyAsString(Charset charset) { 63 | return null; 64 | } 65 | 66 | public static TestHttpResponseBuilder newBuilder() { 67 | return new TestHttpResponseBuilder(); 68 | } 69 | 70 | public static class TestHttpResponseBuilder { 71 | 72 | private final TestHttpResponse res = new TestHttpResponse(); 73 | 74 | private TestHttpResponseBuilder() { 75 | } 76 | 77 | public TestHttpResponseBuilder withStatusCode(int statusCode) { 78 | res.setStatusCode(statusCode); 79 | return this; 80 | } 81 | 82 | public TestHttpResponseBuilder addHeader(String name, String value) { 83 | res.addHeader(name, value); 84 | return this; 85 | } 86 | 87 | public TestHttpResponse build() { 88 | return res; 89 | } 90 | 91 | } 92 | 93 | } 94 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/http/CloseParentResourcesStream.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.http; 5 | 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | 9 | import com.azure.core.http.HttpResponse; 10 | import com.microsoft.azure.kusto.data.Utils; 11 | 12 | /** 13 | * This class allows parent resources (HttpClient and HttpResponse) to be closed when the stream is closed. 14 | * This becomes necessary when passing an InputStream externally, as postToStreamingOutput does. The parent resources 15 | * must be closed, but this cannot happen until the InputStream is consumed. 16 | * It takes a stream, and simply forwards the call to all of its methods, except for the {@code close()} method - which does nothing. 17 | */ 18 | public class CloseParentResourcesStream extends InputStream { 19 | private final InputStream innerStream; 20 | private final HttpResponse httpResponse; 21 | 22 | public CloseParentResourcesStream(HttpResponse httpResponse, InputStream inputStream) { 23 | this.innerStream = Utils.resolveInputStream(httpResponse, inputStream); 24 | this.httpResponse = httpResponse; 25 | } 26 | 27 | /** 28 | * Closes all parent resources once the stream is closed 29 | */ 30 | @Override 31 | public void close() throws IOException { 32 | innerStream.close(); 33 | httpResponse.close(); 34 | } 35 | 36 | @Override 37 | public int read() throws IOException { 38 | return innerStream.read(); 39 | } 40 | 41 | @Override 42 | public int read(byte[] b) throws IOException { 43 | return innerStream.read(b); 44 | } 45 | 46 | @Override 47 | public int read(byte[] b, int off, int len) throws IOException { 48 | return innerStream.read(b, off, len); 49 | } 50 | 51 | @Override 52 | public long skip(long n) throws IOException { 53 | return innerStream.skip(n); 54 | } 55 | 56 | @Override 57 | public int available() throws IOException { 58 | return innerStream.available(); 59 | } 60 | 61 | @Override 62 | public synchronized void mark(int readlimit) { 63 | innerStream.mark(readlimit); 64 | } 65 | 66 | @Override 67 | public synchronized void reset() throws IOException { 68 | innerStream.reset(); 69 | } 70 | 71 | @Override 72 | public boolean markSupported() { 73 | return innerStream.markSupported(); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/http/HttpClientFactory.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | import com.azure.core.http.HttpClient; 4 | import com.azure.core.http.HttpHeaderName; 5 | import com.azure.core.util.Header; 6 | import com.azure.core.util.HttpClientOptions; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.time.Duration; 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | 15 | /** 16 | * A static factory for HTTP clients. 17 | */ 18 | public class HttpClientFactory { 19 | 20 | private static final Logger LOGGER = LoggerFactory.getLogger(HttpClientFactory.class); 21 | 22 | /** 23 | * Creates a new HTTP client. 24 | * 25 | * @param properties custom HTTP client properties 26 | * @return a new HTTP client 27 | */ 28 | public static HttpClient create(HttpClientProperties properties) { 29 | LOGGER.info("Creating new HTTP Client"); 30 | HttpClientOptions options = new HttpClientOptions(); 31 | 32 | // If all properties are null, create with default client options 33 | if (properties == null) { 34 | return HttpClient.createDefault(options); 35 | } 36 | 37 | // MS Docs indicate that all setters handle nulls so even if these values are null everything should "just work" 38 | // Note that the first discovered HttpClientProvider class is loaded. HttpClientProviders can be swapped in or out 39 | // by simply 40 | // Docs: https://learn.microsoft.com/en-us/java/api/com.azure.core.util.httpclientoptions?view=azure-java-stable 41 | 42 | options.setMaximumConnectionPoolSize(properties.maxConnectionTotal()); 43 | options.setConnectionIdleTimeout(Duration.ofSeconds(properties.maxIdleTime())); 44 | options.readTimeout(Duration.ofSeconds(properties.readTimeout())); 45 | options.setHttpClientProvider(properties.provider()); 46 | 47 | // Set Keep-Alive headers if they were requested. 48 | // NOTE: Servers are not obligated to honor client requested Keep-Alive values 49 | if (properties.isKeepAlive()) { 50 | Header keepAlive = new Header(HttpHeaderName.CONNECTION.getCaseSensitiveName(), "Keep-Alive"); 51 | // Keep-Alive is Non-standard from the client so core does not have an enum for it 52 | 53 | List
headers = new ArrayList<>(); 54 | headers.add(keepAlive); 55 | 56 | options.setHeaders(headers); 57 | } 58 | 59 | if (properties.getProxy() != null) { 60 | options.setProxyOptions(properties.getProxy()); 61 | } 62 | 63 | return HttpClient.createDefault(options); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/http/HttpTracing.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.http; 2 | 3 | import com.microsoft.azure.kusto.data.ClientDetails; 4 | import com.microsoft.azure.kusto.data.ClientRequestProperties; 5 | import reactor.util.annotation.Nullable; 6 | 7 | public class HttpTracing { 8 | private ClientRequestProperties properties; 9 | private String clientRequestIdPrefix; 10 | private String activityTypeSuffix; 11 | private ClientDetails clientDetails; 12 | 13 | private HttpTracing() { 14 | } 15 | 16 | @Nullable 17 | public ClientRequestProperties getProperties() { 18 | return properties; 19 | } 20 | 21 | public void setProperties(ClientRequestProperties properties) { 22 | this.properties = properties; 23 | } 24 | 25 | public String getClientRequestIdPrefix() { 26 | return clientRequestIdPrefix; 27 | } 28 | 29 | public void setClientRequestIdPrefix(String clientRequestIdPrefix) { 30 | this.clientRequestIdPrefix = clientRequestIdPrefix; 31 | } 32 | 33 | public String getActivityTypeSuffix() { 34 | return activityTypeSuffix; 35 | } 36 | 37 | public void setActivityTypeSuffix(String activityTypeSuffix) { 38 | this.activityTypeSuffix = activityTypeSuffix; 39 | } 40 | 41 | public ClientDetails getClientDetails() { 42 | return clientDetails; 43 | } 44 | 45 | public void setClientDetails(ClientDetails clientDetails) { 46 | this.clientDetails = clientDetails; 47 | } 48 | 49 | public static HttpTracingBuilder newBuilder() { 50 | return new HttpTracingBuilder(); 51 | } 52 | 53 | public static class HttpTracingBuilder { 54 | 55 | private final HttpTracing tracing = new HttpTracing(); 56 | 57 | public HttpTracingBuilder() { 58 | } 59 | 60 | public HttpTracingBuilder withClientDetails(ClientDetails clientDetails) { 61 | tracing.setClientDetails(clientDetails); 62 | return this; 63 | } 64 | 65 | public HttpTracingBuilder withProperties(ClientRequestProperties properties) { 66 | tracing.setProperties(properties); 67 | return this; 68 | } 69 | 70 | public HttpTracingBuilder withRequestPrefix(String requestPrefix) { 71 | tracing.setClientRequestIdPrefix(requestPrefix); 72 | return this; 73 | } 74 | 75 | public HttpTracingBuilder withActivitySuffix(String activitySuffix) { 76 | tracing.setActivityTypeSuffix(activitySuffix); 77 | return this; 78 | } 79 | 80 | public HttpTracing build() { 81 | return tracing; 82 | } 83 | 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/CloudDependentTokenProviderBase.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import java.net.URISyntaxException; 7 | import java.util.HashSet; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import org.jetbrains.annotations.NotNull; 12 | import org.jetbrains.annotations.Nullable; 13 | 14 | import com.azure.core.http.HttpClient; 15 | import com.microsoft.azure.kusto.data.exceptions.DataServiceException; 16 | import com.microsoft.azure.kusto.data.instrumentation.MonitoredActivity; 17 | 18 | import reactor.core.publisher.Mono; 19 | 20 | public abstract class CloudDependentTokenProviderBase extends TokenProviderBase { 21 | private static final String ERROR_INVALID_SERVICE_RESOURCE_URL = "Error determining scope due to invalid Kusto Service Resource URL"; 22 | protected final Set scopes = new HashSet<>(); 23 | private boolean initialized = false; 24 | private CloudInfo cloudInfo; 25 | 26 | CloudDependentTokenProviderBase(@NotNull String clusterUrl, @Nullable HttpClient httpClient) throws URISyntaxException { 27 | super(clusterUrl, httpClient); 28 | } 29 | 30 | @Override 31 | final Mono initialize() { 32 | return Mono.defer(() -> { 33 | if (initialized) { 34 | return Mono.empty(); 35 | } 36 | 37 | // trace retrieveCloudInfo 38 | return MonitoredActivity.wrap( 39 | CloudInfo.retrieveCloudInfoForClusterAsync(clusterUrl, httpClient), 40 | "CloudDependentTokenProviderBase.retrieveCloudInfo", getTracingAttributes()) 41 | .doOnNext(cloudInfoResult -> { 42 | this.cloudInfo = cloudInfoResult; 43 | initializeWithCloudInfo(cloudInfoResult); 44 | this.initialized = true; 45 | }) 46 | .then(); 47 | }); 48 | } 49 | 50 | protected void initializeWithCloudInfo(CloudInfo cloudInfo) { 51 | try { 52 | scopes.add(cloudInfo.determineScope()); 53 | } catch (URISyntaxException e) { 54 | throw new DataServiceException(clusterUrl, ERROR_INVALID_SERVICE_RESOURCE_URL, e, true); 55 | } 56 | } 57 | 58 | @Override 59 | public Map getTracingAttributes() { 60 | Map attributes = super.getTracingAttributes(); 61 | if (cloudInfo != null) { 62 | attributes.putAll(cloudInfo.getTracingAttributes()); 63 | } 64 | attributes.put("http.url", clusterUrl); 65 | return attributes; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/ManagedStreamingQueuingPolicy.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.ingest; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | 5 | interface ManagedStreamingQueuingPolicyPredicator { 6 | boolean shouldUseQueuedIngestion(long dataSize, boolean compressed, IngestionProperties.DataFormat dataFormat); 7 | } 8 | 9 | public class ManagedStreamingQueuingPolicy implements ManagedStreamingQueuingPolicyPredicator { 10 | static final int MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES = 4 * 1024 * 1024; 11 | // Regardless of the format, we don't want to stream more than 10mb 12 | static final int MAX_STREAMING_STREAM_SIZE_BYTES = 10 * 1024 * 1024; 13 | 14 | static final double JSON_UNCOMPRESSED_FACTOR = 1.5d; 15 | static final int NON_BINARY_FACTOR = 2; 16 | static final double BINARY_COMPRESSED_FACTOR = 2d; 17 | static final double BINARY_UNCOMPRESSED_FACTOR = 1.5d; 18 | private final double factor; 19 | 20 | public ManagedStreamingQueuingPolicy(double factor) { 21 | Ensure.isTrue(factor > 0, "ManagedStreamingQueuingPolicy: factor should be greater than 0"); 22 | this.factor = factor; 23 | } 24 | 25 | // Return true if streaming ingestion should not be tried, according to stream size, compression and format 26 | public boolean shouldUseQueuedIngestion(long dataSize, boolean compressed, IngestionProperties.DataFormat dataFormat) { 27 | // In case available() was implemented wrong, do streaming 28 | if (dataSize <= 0) { 29 | return false; 30 | } 31 | 32 | // In any case - don't stream more than 10mb 33 | if (dataSize > factor * MAX_STREAMING_STREAM_SIZE_BYTES) { 34 | return true; 35 | } 36 | 37 | if (!dataFormat.isCompressible()) { 38 | // Binary format 39 | if (compressed) { 40 | return (dataSize * BINARY_COMPRESSED_FACTOR) > factor * MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES; 41 | } 42 | 43 | return (dataSize * BINARY_UNCOMPRESSED_FACTOR) > factor * MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES; 44 | } 45 | 46 | if (compressed) { 47 | // Compressed + non-binary 48 | return (dataSize * NON_BINARY_FACTOR) > factor * MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES; 49 | } 50 | 51 | if (dataFormat.isJsonFormat()) { 52 | // JSON uncompressed format 53 | return (dataSize / JSON_UNCOMPRESSED_FACTOR) > factor * MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES; 54 | } 55 | 56 | // Uncompressed + non-binary 57 | return ((double) dataSize / NON_BINARY_FACTOR) > factor * MAX_STREAMING_UNCOMPRESSED_RAW_SIZE_BYTES; 58 | } 59 | 60 | public static final ManagedStreamingQueuingPolicy Default = new ManagedStreamingQueuingPolicy(1); 61 | } 62 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/StreamSourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import com.microsoft.azure.kusto.data.Ensure; 7 | 8 | import java.io.InputStream; 9 | import java.util.Map; 10 | import java.util.Objects; 11 | import java.util.UUID; 12 | 13 | public class StreamSourceInfo extends AbstractSourceInfo { 14 | 15 | private InputStream stream; 16 | private boolean leaveOpen = false; 17 | private CompressionType compressionType = null; 18 | 19 | public InputStream getStream() { 20 | return stream; 21 | } 22 | 23 | public void setStream(InputStream stream) { 24 | this.stream = Objects.requireNonNull(stream, "stream cannot be null"); 25 | } 26 | 27 | public boolean isLeaveOpen() { 28 | return leaveOpen; 29 | } 30 | 31 | /** 32 | * Weather or not the stream will close after reading from it. 33 | * @param leaveOpen leave the stream open after processing 34 | */ 35 | public void setLeaveOpen(boolean leaveOpen) { 36 | this.leaveOpen = leaveOpen; 37 | } 38 | 39 | public void setCompressionType(CompressionType compressionType) { 40 | this.compressionType = compressionType; 41 | } 42 | 43 | public CompressionType getCompressionType() { 44 | return compressionType; 45 | } 46 | 47 | public StreamSourceInfo(InputStream stream) { 48 | setStream(stream); 49 | } 50 | 51 | public StreamSourceInfo(InputStream stream, boolean leaveOpen) { 52 | this(stream); 53 | setLeaveOpen(leaveOpen); 54 | } 55 | 56 | public StreamSourceInfo(InputStream stream, boolean leaveOpen, UUID sourceId) { 57 | this(stream, leaveOpen); 58 | setSourceId(sourceId); 59 | } 60 | 61 | public StreamSourceInfo(InputStream stream, boolean leaveOpen, UUID sourceId, CompressionType compressionType) { 62 | this(stream, leaveOpen, sourceId); 63 | setCompressionType(compressionType); 64 | } 65 | 66 | public void validate() { 67 | Ensure.argIsNotNull(stream, "stream"); 68 | Ensure.isTrue(compressionType != CompressionType.zip, "streaming ingest is not working with zip compression"); 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return String.format("Stream with SourceId: %s", getSourceId()); 74 | } 75 | 76 | public Map getTracingAttributes() { 77 | Map attributes = super.getTracingAttributes(); 78 | attributes.put("resource", "stream"); 79 | UUID sourceId = getSourceId(); 80 | if (sourceId != null) { 81 | attributes.put("sourceId", sourceId.toString()); 82 | } 83 | return attributes; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/KnownKeywords.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import java.util.Arrays; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | public enum KnownKeywords { 8 | DATA_SOURCE("Data Source", true), 9 | INITIAL_CATALOG("Initial Catalog", true), 10 | FEDERATED_SECURITY("AAD Federated Security", true), 11 | APPLICATION_CLIENT_ID("Application Client Id", true), 12 | APPLICATION_KEY("Application Key", true), 13 | USER_ID("User ID", true), 14 | AUTHORITY_ID("Authority Id", true), 15 | APPLICATION_TOKEN("Application Token", true), 16 | USER_TOKEN("User Token", true), 17 | APPLICATION_CERTIFICATE_X5C("Application Certificate SendX5c", true), 18 | APPLICATION_NAME_FOR_TRACING("Application Name for Tracing", true), 19 | USER_NAME_FOR_TRACING("User Name for Tracing", true), 20 | PASSWORD("Password", false), 21 | APPLICATION_CERTIFICATE_BLOB("Application Certificate Blob", false), // TODO - if we add a way to pass a certificate by a byte[], support this 22 | APPLICATION_CERTIFICATE_THUMBPRINT("Application Certificate Thumbprint", false), 23 | DSTS_FEDERATED_SECURITY("dSTS Federated Security", false), 24 | STREAMING("Streaming", false), 25 | UNCOMPRESSED("Uncompressed", false), 26 | ENFORCE_MFA("EnforceMfa", false), 27 | ACCEPT("Accept", false), 28 | QUERY_CONSISTENCY("Query Consistency", false), 29 | DATA_SOURCE_URI("Data Source Uri", false), 30 | AZURE_REGION("Azure Region", false), 31 | NAMESPACE("Namespace", false), 32 | APPLICATION_CERTIFICATE_ISSUER_DISTINGUISHED_NAME("Application Certificate Issuer Distinguished Name", false), 33 | APPLICATION_CERTIFICATE_SUBJECT_DISTINGUISHED_NAME("Application Certificate Subject Distinguished Name", false); 34 | 35 | private final String canonicalName; 36 | private final boolean isSupported; 37 | private String type; 38 | private boolean isSecret; 39 | 40 | public static final Map knownKeywords = Arrays.stream(KnownKeywords.values()).collect(HashMap::new, 41 | (map, keyword) -> map.put(keyword.canonicalName, keyword), HashMap::putAll); 42 | 43 | KnownKeywords(String canonicalName, boolean isSupported) { 44 | this.canonicalName = canonicalName; 45 | this.isSupported = isSupported; 46 | } 47 | 48 | public String getType() { 49 | return type; 50 | } 51 | 52 | public void setType(String type) { 53 | this.type = type; 54 | } 55 | 56 | public boolean isSecret() { 57 | return isSecret; 58 | } 59 | 60 | public void setSecret(boolean secret) { 61 | isSecret = secret; 62 | } 63 | 64 | public boolean isSupported() { 65 | return isSupported; 66 | } 67 | 68 | public String getCanonicalName() { 69 | return canonicalName; 70 | } 71 | 72 | } 73 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/instrumentation/Tracer.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.instrumentation; 2 | 3 | import com.azure.core.util.Configuration; 4 | import com.azure.core.util.Context; 5 | import com.azure.core.util.tracing.ProcessKind; 6 | 7 | import java.util.Map; 8 | 9 | public class Tracer { 10 | /** 11 | * Private constructor to prevent instantiation as this class provides only static utility methods. 12 | */ 13 | private Tracer() { 14 | } 15 | private static final boolean IS_TRACING_DISABLED = Configuration.getGlobalConfiguration().get(Configuration.PROPERTY_AZURE_TRACING_DISABLED, false); 16 | private static com.azure.core.util.tracing.Tracer tracer; 17 | private static volatile boolean initialized = false; 18 | 19 | public static void initializeTracer(com.azure.core.util.tracing.Tracer tracer) { 20 | if (!Tracer.initialized) { 21 | synchronized (Tracer.class) { 22 | if (!Tracer.initialized) { 23 | Tracer.tracer = IS_TRACING_DISABLED ? null : tracer; 24 | initialized = true; 25 | } 26 | } 27 | } 28 | } 29 | 30 | public static Span startSpan(String spanName) { 31 | return startSpan(spanName, Context.NONE, ProcessKind.PROCESS, null); 32 | } 33 | 34 | public static Span startSpan(String spanName, Map attributes) { 35 | return startSpan(spanName, Context.NONE, ProcessKind.PROCESS, attributes); 36 | } 37 | 38 | public static Span startSpan(String spanName, Context context, ProcessKind kind, Map attributes) { 39 | Context span = tracer == null ? null : tracer.start(spanName, context, kind); 40 | return new Span(span, attributes); 41 | } 42 | 43 | public static class Span implements AutoCloseable { 44 | private final Context span; 45 | private Throwable throwable; 46 | 47 | private Span(Context span, Map attributes) { 48 | this.span = span; 49 | setAttributes(attributes); 50 | } 51 | 52 | @Override 53 | public void close() { 54 | if (tracer != null && span != null) { 55 | String errorCondition = "success"; 56 | if (throwable != null) { 57 | errorCondition = throwable.getLocalizedMessage(); 58 | } 59 | tracer.end(errorCondition, throwable, span); 60 | } 61 | } 62 | 63 | public void addException(Exception e) { 64 | throwable = e; 65 | } 66 | 67 | public void setAttributes(Map attributes) { 68 | if (tracer != null && attributes != null && span != null) { 69 | attributes.forEach((k, v) -> tracer.setAttribute(k, v, span)); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /data/src/test/java/com/microsoft/azure/kusto/data/KustoDateTimeTest.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.fasterxml.jackson.databind.node.ArrayNode; 5 | import org.junit.jupiter.api.Assertions; 6 | import org.junit.jupiter.api.Test; 7 | 8 | import java.time.Instant; 9 | import java.time.LocalDateTime; 10 | import java.time.format.DateTimeFormatter; 11 | import java.time.format.DateTimeFormatterBuilder; 12 | 13 | public class KustoDateTimeTest { 14 | @Test 15 | void kustoResultSet() throws Exception { 16 | ObjectMapper mapper = Utils.getObjectMapper(); 17 | DateTimeFormatter kustoDateTimeFormatter = new DateTimeFormatterBuilder().parseCaseInsensitive() 18 | .append(DateTimeFormatter.ISO_LOCAL_DATE_TIME).appendLiteral('Z').toFormatter(); 19 | ArrayNode rows = mapper.createArrayNode(); 20 | ArrayNode row = mapper.createArrayNode(); 21 | row.add(Instant.parse("2022-05-17T00:00:00Z").toString()); 22 | rows.add(row); 23 | row = mapper.createArrayNode(); 24 | row.add(Instant.parse("2022-05-17T00:00:00.1Z").toString()); 25 | rows.add(row); 26 | row = mapper.createArrayNode(); 27 | row.add(Instant.parse("2022-05-17T00:00:00.12Z").toString()); 28 | rows.add(row); 29 | row = mapper.createArrayNode(); 30 | row.add(Instant.parse("2022-05-17T00:00:00.123Z").toString()); 31 | rows.add(row); 32 | row = mapper.createArrayNode(); 33 | row.add(Instant.parse("2022-05-17T00:00:00.1234Z").toString()); 34 | rows.add(row); 35 | row = mapper.createArrayNode(); 36 | row.add(Instant.parse("2022-05-17T00:00:00.12345Z").toString()); 37 | rows.add(row); 38 | row = mapper.createArrayNode(); 39 | row.add(Instant.parse("2022-05-17T00:00:00.123456Z").toString()); 40 | rows.add(row); 41 | row = mapper.createArrayNode(); 42 | row.add(Instant.parse("2022-05-17T00:00:00.1234567Z").toString()); 43 | rows.add(row); 44 | row = mapper.createArrayNode(); 45 | row.add(Instant.parse("2022-05-17T00:00:00.12345678Z").toString()); 46 | rows.add(row); 47 | row = mapper.createArrayNode(); 48 | row.add(Instant.parse("2022-05-17T00:00:00.123456789Z").toString()); 49 | rows.add(row); 50 | 51 | String columns = "[ { \"ColumnName\": \"a\", \"ColumnType\": \"datetime\" } ]"; 52 | KustoResultSetTable res = new KustoResultSetTable(mapper.createArrayNode().add("{\"TableName\":\"Table_0\"," + 53 | "\"Columns\":" + columns + ",\"Rows\":" + 54 | rows + "}")); 55 | 56 | int rowNum = 0; 57 | while (res.next()) { 58 | Assertions.assertEquals( 59 | LocalDateTime.parse(rows.get(rowNum).get(0).toString(), kustoDateTimeFormatter), 60 | res.getKustoDateTime(0)); 61 | rowNum++; 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | # 7 | # ******** NOTE ******** 8 | # We have attempted to detect the languages in your repository. Please check 9 | # the `language` matrix defined below to confirm you have the correct set of 10 | # supported CodeQL languages. 11 | # 12 | name: "CodeQL" 13 | 14 | on: 15 | push: 16 | branches: [ "master", dev ] 17 | pull_request: 18 | # The branches below must be a subset of the branches above 19 | branches: [ "master" ] 20 | schedule: 21 | - cron: '39 18 * * 2' 22 | 23 | jobs: 24 | analyze: 25 | name: Analyze 26 | runs-on: ubuntu-latest 27 | permissions: 28 | actions: read 29 | contents: read 30 | security-events: write 31 | 32 | strategy: 33 | fail-fast: false 34 | matrix: 35 | language: [ 'java' ] 36 | # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] 37 | # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v3 42 | 43 | # Initializes the CodeQL tools for scanning. 44 | - name: Initialize CodeQL 45 | uses: github/codeql-action/init@v2 46 | with: 47 | languages: ${{ matrix.language }} 48 | # If you wish to specify custom queries, you can do so here or in a config file. 49 | # By default, queries listed here will override any specified in a config file. 50 | # Prefix the list here with "+" to use these queries and those in the config file. 51 | 52 | # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs 53 | # queries: security-extended,security-and-quality 54 | 55 | 56 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 57 | # If this step fails, then you should remove it and run the build manually (see below) 58 | - name: Autobuild 59 | uses: github/codeql-action/autobuild@v2 60 | 61 | # ℹ️ Command-line programs to run using the OS shell. 62 | # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun 63 | 64 | # If the Autobuild fails above, remove it and uncomment the following three lines. 65 | # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. 66 | 67 | # - run: | 68 | # echo "Run, Build Application using script" 69 | # ./location_of_script_within_repo/buildscript.sh 70 | 71 | - name: Perform CodeQL Analysis 72 | uses: github/codeql-action/analyze@v2 73 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslTimespanFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import java.time.Duration; 4 | import java.time.LocalTime; 5 | import java.time.format.DateTimeFormatter; 6 | import java.util.concurrent.TimeUnit; 7 | import java.util.regex.Matcher; 8 | 9 | import com.microsoft.azure.kusto.data.ClientRequestProperties; 10 | import com.microsoft.azure.kusto.data.Ensure; 11 | import com.microsoft.azure.kusto.data.StringUtils; 12 | import com.microsoft.azure.kusto.data.exceptions.ParseException; 13 | 14 | public class CslTimespanFormat extends CslFormat { 15 | public static final String KUSTO_TIMESPAN_PATTERN = "HH:mm:ss.SSSSSSS"; 16 | public static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(KUSTO_TIMESPAN_PATTERN); 17 | 18 | private final Duration value; 19 | 20 | public CslTimespanFormat(Duration value) { 21 | this.value = value; 22 | } 23 | 24 | public CslTimespanFormat(String value) { 25 | if (StringUtils.isBlank(value)) { 26 | this.value = null; 27 | } else { 28 | Matcher matcher = ClientRequestProperties.KUSTO_TIMESPAN_REGEX.matcher(value); 29 | if (!matcher.matches()) { 30 | throw new ParseException(String.format("Failed to parse timeout string as a timespan. Value: %s", value)); 31 | } 32 | 33 | long nanos = 0; 34 | String days = matcher.group(2); 35 | if (days != null && !days.equals("0")) { 36 | nanos = TimeUnit.DAYS.toNanos(Integer.parseInt(days)); 37 | } 38 | 39 | String timespanWithoutDays = ""; 40 | for (int i = 4; i <= 10; i++) { 41 | if (matcher.group(i) != null) { 42 | timespanWithoutDays += matcher.group(i); 43 | } 44 | } 45 | nanos += LocalTime.parse(timespanWithoutDays).toNanoOfDay(); 46 | if ("-".equals(matcher.group(1))) { 47 | this.value = Duration.ofNanos(nanos).negated(); 48 | } else { 49 | this.value = Duration.ofNanos(nanos); 50 | } 51 | } 52 | } 53 | 54 | @Override 55 | public String getType() { 56 | return "time"; 57 | } 58 | 59 | @Override 60 | public Duration getValue() { 61 | return value; 62 | } 63 | 64 | @Override 65 | String getValueAsString() { 66 | Ensure.argIsNotNull(value, "value"); 67 | 68 | String result = ""; 69 | Duration valueWithoutDays = value.isNegative() ? value.negated() : value; 70 | if (valueWithoutDays.toDays() > 0) { 71 | result += valueWithoutDays.toDays() + "."; 72 | valueWithoutDays = valueWithoutDays.minusDays(valueWithoutDays.toDays()); 73 | } 74 | result += LocalTime.MIDNIGHT.plus(valueWithoutDays).format(DATE_TIME_FORMATTER); 75 | if (value.isNegative()) { 76 | result = "-" + result; 77 | } 78 | 79 | return result; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /quickstart/kusto_sample_config.json: -------------------------------------------------------------------------------- 1 | { 2 | "kustoUri": "https://help.kusto.windows.net", 3 | "ingestUri": "https://ingest-help.kusto.windows.net", 4 | "databaseName": "MyDatabase", 5 | "tableName": "SampleTable", 6 | "useExistingTable": true, 7 | "alterTable": true, 8 | "queryData": true, 9 | "ingestData": true, 10 | "authenticationMode": "UserPrompt", 11 | "waitForUser": true, 12 | "ignoreFirstRecord": false, 13 | "waitForIngestSeconds": 20, 14 | "batchingPolicy": "{ 'MaximumBatchingTimeSpan': '00:00:10', 'MaximumNumberOfItems': 500, 'MaximumRawDataSizeMB': 1024 }", 15 | "tableSchema": "(rownumber:int, rowguid:string, xdouble:real, xfloat:real, xbool:bool, xint16:int, xint32:int, xint64:long, xuint8:long, xuint16:long, xuint32:long, xuint64:long, xdate:datetime, xsmalltext:string, xtext:string, xnumberAsText:string, xtime:timespan, xtextWithNulls:string, xdynamicWithNulls:dynamic)", 16 | "data": [ 17 | { 18 | "sourceType": "localFileSource", 19 | "dataSourceUri": "dataset.csv", 20 | "format": "CSV", 21 | "useExistingMapping": true, 22 | "mappingName": "", 23 | "mappingValue": "" 24 | }, 25 | { 26 | "sourceType": "localFileSource", 27 | "dataSourceUri": "dataset.json", 28 | "format": "MULTIJSON", 29 | "useExistingMapping": false, 30 | "mappingName": "SampleTableMapping", 31 | "mappingValue": "[{\"Properties\":{\"Path\":\"$.rownumber\"},\"column\":\"rownumber\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.rowguid\"},\"column\":\"rowguid\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xdouble\"},\"column\":\"xdouble\",\"datatype\":\"real\"}, {\"Properties\":{\"Path\":\"$.xfloat\"},\"column\":\"xfloat\",\"datatype\":\"real\"}, {\"Properties\":{\"Path\":\"$.xbool\"},\"column\":\"xbool\",\"datatype\":\"bool\"}, {\"Properties\":{\"Path\":\"$.xint16\"},\"column\":\"xint16\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.xint32\"},\"column\":\"xint32\",\"datatype\":\"int\"}, {\"Properties\":{\"Path\":\"$.xint64\"},\"column\":\"xint64\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint8\"},\"column\":\"xuint8\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint16\"},\"column\":\"xuint16\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint32\"},\"column\":\"xuint32\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xuint64\"},\"column\":\"xuint64\",\"datatype\":\"long\"}, {\"Properties\":{\"Path\":\"$.xdate\"},\"column\":\"xdate\",\"datatype\":\"datetime\"}, {\"Properties\":{\"Path\":\"$.xsmalltext\"},\"column\":\"xsmalltext\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xtext\"},\"column\":\"xtext\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.rowguid\"},\"column\":\"xnumberAsText\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xtime\"},\"column\":\"xtime\",\"datatype\":\"timespan\"}, {\"Properties\":{\"Path\":\"$.xtextWithNulls\"},\"column\":\"xtextWithNulls\",\"datatype\":\"string\"}, {\"Properties\":{\"Path\":\"$.xdynamicWithNulls\"},\"column\":\"xdynamicWithNulls\",\"datatype\":\"dynamic\"}]" 32 | } 33 | ] 34 | } -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/exceptions/KustoServiceQueryError.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.exceptions; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import com.azure.core.exception.AzureException; 10 | import com.fasterxml.jackson.databind.JsonNode; 11 | import com.fasterxml.jackson.databind.node.ArrayNode; 12 | 13 | /* 14 | This class represents an error that returned from the query result 15 | */ 16 | public class KustoServiceQueryError extends AzureException { 17 | static final String EXCEPTIONS_MESSAGE = "Query execution failed with multiple inner exceptions:\n"; 18 | 19 | private final List exceptions; 20 | 21 | public KustoServiceQueryError(String message, List exceptions) { 22 | super(message); 23 | this.exceptions = exceptions; 24 | } 25 | 26 | public KustoServiceQueryError(String message) { 27 | super(message); 28 | this.exceptions = new ArrayList<>(); 29 | this.exceptions.add(new RuntimeException(message)); 30 | } 31 | 32 | public static KustoServiceQueryError fromOneApiErrorArray(ArrayNode jsonExceptions, boolean isOneApi) { 33 | List exceptions = new ArrayList<>(); 34 | StringBuilder sb = new StringBuilder(); 35 | 36 | if (jsonExceptions == null || jsonExceptions.isEmpty()) { 37 | return new KustoServiceQueryError("No exceptions were returned from the service."); 38 | } 39 | 40 | if (jsonExceptions.size() > 1) { 41 | sb.append(EXCEPTIONS_MESSAGE); 42 | } 43 | 44 | for (int i = 0; i < jsonExceptions.size(); i++) { 45 | JsonNode jsonNode = jsonExceptions.get(i); 46 | String value = jsonNode.isTextual() ? jsonNode.asText() : jsonNode.toString(); 47 | String message = value; 48 | RuntimeException exception; 49 | if (isOneApi) { 50 | DataWebException ex = new DataWebException(value); 51 | OneApiError apiError = ex.getApiError(); 52 | if (apiError != null) { 53 | message = apiError.getCode() + ": " + apiError.getMessage(); 54 | } 55 | exception = ex; 56 | } else { 57 | exception = new RuntimeException(value); 58 | } 59 | exceptions.add(exception); 60 | sb.append(message); 61 | sb.append("\n"); 62 | } 63 | 64 | return new KustoServiceQueryError(sb.toString(), exceptions); 65 | } 66 | 67 | public List getExceptions() { 68 | return exceptions; 69 | } 70 | 71 | @Override 72 | public String toString() { 73 | return exceptions.isEmpty() ? getMessage() : "exceptions\":" + exceptions + "}"; 74 | } 75 | 76 | public boolean isPermanent() { 77 | if (!exceptions.isEmpty() && exceptions.get(0) instanceof DataWebException) { 78 | return ((DataWebException) exceptions.get(0)).getApiError().isPermanent(); 79 | } 80 | 81 | return false; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/source/BlobSourceInfo.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.source; 5 | 6 | import java.io.File; 7 | import java.util.Map; 8 | import java.util.UUID; 9 | 10 | import static com.microsoft.azure.kusto.data.Ensure.stringIsNotBlank; 11 | 12 | public class BlobSourceInfo extends AbstractSourceInfo { 13 | private String blobPath; 14 | 15 | // For internal usage - only when we create the blob 16 | private Long blobExactSize = null; 17 | private CompressionType compressionType; 18 | 19 | public String getBlobPath() { 20 | return blobPath; 21 | } 22 | 23 | public void setBlobPath(String blobPath) { 24 | this.blobPath = blobPath; 25 | } 26 | 27 | public BlobSourceInfo(String blobPath) { 28 | this.blobPath = blobPath; 29 | } 30 | 31 | public BlobSourceInfo(String blobPath, CompressionType compressionType) { 32 | this(blobPath, compressionType, null); 33 | } 34 | 35 | public BlobSourceInfo(String blobPath, CompressionType compressionType, UUID sourceId) { 36 | setBlobPath(blobPath); 37 | setCompressionType(compressionType); 38 | setSourceId(sourceId); 39 | } 40 | 41 | public Long getBlobExactSize() { 42 | return blobExactSize; 43 | } 44 | 45 | public CompressionType getCompressionType() { 46 | return compressionType; 47 | } 48 | 49 | public void setCompressionType(CompressionType compressionType) { 50 | this.compressionType = compressionType; 51 | } 52 | 53 | public void validate() { 54 | stringIsNotBlank(blobPath, "blobPath"); 55 | } 56 | 57 | @Override 58 | public Map getTracingAttributes() { 59 | Map attributes = super.getTracingAttributes(); 60 | attributes.put("resource", blobPath); 61 | UUID sourceId = getSourceId(); 62 | if (sourceId != null) { 63 | attributes.put("sourceId", sourceId.toString()); 64 | } 65 | return attributes; 66 | } 67 | 68 | /* 69 | * For internal usage, adding blobExactSize 70 | */ 71 | public static BlobSourceInfo fromFile(String blobPath, FileSourceInfo fileSourceInfo, CompressionType sourceCompressionType, boolean gotCompressed) { 72 | BlobSourceInfo blobSourceInfo = new BlobSourceInfo(blobPath, gotCompressed ? CompressionType.gz : sourceCompressionType, 73 | fileSourceInfo.getSourceId()); 74 | if (sourceCompressionType == null) { 75 | blobSourceInfo.blobExactSize = new File(fileSourceInfo.getFilePath()).length(); 76 | } 77 | 78 | return blobSourceInfo; 79 | } 80 | 81 | /* 82 | * For internal usage, adding blobExactSize 83 | */ 84 | public static BlobSourceInfo fromStream(String blobPath, Integer size, StreamSourceInfo streamSourceInfo) { 85 | BlobSourceInfo blobSourceInfo = new BlobSourceInfo(blobPath, streamSourceInfo.getCompressionType(), streamSourceInfo.getSourceId()); 86 | blobSourceInfo.blobExactSize = size.longValue(); 87 | return blobSourceInfo; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/ExponentialRetry.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | import java.lang.invoke.MethodHandles; 4 | import java.time.Duration; 5 | import java.util.List; 6 | import java.util.function.Predicate; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.microsoft.azure.kusto.data.exceptions.KustoDataExceptionBase; 12 | 13 | import reactor.util.annotation.Nullable; 14 | import reactor.util.retry.Retry; 15 | 16 | public class ExponentialRetry { 17 | private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); 18 | 19 | private final int maxAttempts; 20 | double sleepBaseSecs; 21 | double maxJitterSecs; 22 | 23 | public ExponentialRetry(int maxAttempts) { 24 | this.maxAttempts = maxAttempts; 25 | this.sleepBaseSecs = 1.0; 26 | this.maxJitterSecs = 1.0; 27 | } 28 | 29 | public ExponentialRetry(int maxAttempts, double sleepBaseSecs, double maxJitterSecs) { 30 | this.maxAttempts = maxAttempts; 31 | this.sleepBaseSecs = sleepBaseSecs; 32 | this.maxJitterSecs = maxJitterSecs; 33 | } 34 | 35 | public ExponentialRetry(ExponentialRetry other) { 36 | this.maxAttempts = other.maxAttempts; 37 | this.sleepBaseSecs = other.sleepBaseSecs; 38 | this.maxJitterSecs = other.maxJitterSecs; 39 | } 40 | 41 | /** 42 | * Creates a retry mechanism with exponential backoff and jitter. 43 | * 44 | * @param retriableErrorClasses A list of error classes that are considered retriable. If null, 45 | * the method does not retry. 46 | * @param filter A filter to use. Default is retrying retriable errors. 47 | * @return A configured {@link Retry} instance 48 | */ 49 | public Retry retry(@Nullable List> retriableErrorClasses, 50 | @Nullable Predicate filter) { 51 | if (retriableErrorClasses != null && filter != null) { 52 | throw new IllegalArgumentException("Cannot specify both retriableErrorClasses and filter"); 53 | } 54 | 55 | Predicate filterToUse = filter == null ? throwable -> shouldRetry(throwable, retriableErrorClasses) : filter; 56 | return Retry.backoff(maxAttempts, Duration.ofSeconds((long) sleepBaseSecs)) 57 | .maxBackoff(Duration.ofSeconds(30)) 58 | .jitter(maxJitterSecs) 59 | .filter(filterToUse) 60 | .doAfterRetry(retrySignal -> { 61 | long currentAttempt = retrySignal.totalRetries() + 1; 62 | log.info("Attempt {} failed.", currentAttempt); 63 | }) 64 | .onRetryExhaustedThrow((spec, signal) -> signal.failure()); 65 | } 66 | 67 | private static boolean shouldRetry(Throwable failure, List> retriableErrorClasses) { 68 | if (failure instanceof KustoDataExceptionBase) { 69 | return !((KustoDataExceptionBase) failure).isPermanent(); 70 | } 71 | 72 | if (retriableErrorClasses != null) { 73 | return retriableErrorClasses.stream() 74 | .anyMatch(errorClass -> errorClass.isInstance(failure)); 75 | } 76 | 77 | return false; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /samples/src/main/java/TableStatus.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule; 6 | import com.microsoft.azure.kusto.data.Utils; 7 | import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; 8 | import com.microsoft.azure.kusto.ingest.IngestClient; 9 | import com.microsoft.azure.kusto.ingest.IngestClientFactory; 10 | import com.microsoft.azure.kusto.ingest.IngestionMapping; 11 | import com.microsoft.azure.kusto.ingest.IngestionProperties; 12 | import com.microsoft.azure.kusto.ingest.result.IngestionResult; 13 | import com.microsoft.azure.kusto.ingest.result.IngestionStatus; 14 | import com.microsoft.azure.kusto.ingest.result.OperationStatus; 15 | import com.microsoft.azure.kusto.ingest.source.FileSourceInfo; 16 | 17 | import java.time.Instant; 18 | import java.util.List; 19 | 20 | import static com.microsoft.azure.kusto.ingest.IngestionProperties.IngestionReportMethod.QUEUE_AND_TABLE; 21 | 22 | public class TableStatus { 23 | public static void main(String[] args) { 24 | try { 25 | Integer timeoutInSec = Integer.getInteger("timeoutInSec"); 26 | 27 | ConnectionStringBuilder csb = ConnectionStringBuilder.createWithAadApplicationCredentials(System.getProperty("clusterPath"), 28 | System.getProperty("appId"), 29 | System.getProperty("appKey"), 30 | System.getProperty("appTenant")); 31 | IngestionResult ingestionResult; 32 | try (IngestClient client = IngestClientFactory.createClient(csb)) { 33 | IngestionProperties ingestionProperties = new IngestionProperties(System.getProperty("dbName"), 34 | System.getProperty("tableName")); 35 | ingestionProperties.setIngestionMapping(System.getProperty("dataMappingName"), IngestionMapping.IngestionMappingKind.JSON); 36 | ingestionProperties.setReportMethod(QUEUE_AND_TABLE); 37 | ingestionProperties.setReportLevel(IngestionProperties.IngestionReportLevel.FAILURES_AND_SUCCESSES); 38 | FileSourceInfo fileSourceInfo = new FileSourceInfo(System.getProperty("filePath")); 39 | ingestionResult = client.ingestFromFile(fileSourceInfo, ingestionProperties); 40 | } 41 | List statuses = ingestionResult.getIngestionStatusCollectionAsync().block(); // TODO: how to handle this 42 | 43 | // step 3: poll on the result. 44 | while (statuses.get(0).status == OperationStatus.Pending && timeoutInSec > 0) { 45 | Thread.sleep(1000); 46 | timeoutInSec -= 1; 47 | statuses = ingestionResult.getIngestionStatusCollectionAsync().block(); 48 | } 49 | 50 | ObjectMapper objectMapper = Utils.getObjectMapper(); 51 | JavaTimeModule module = new JavaTimeModule(); 52 | module.addSerializer(Instant.class, new InstantSerializerWithMilliSecondPrecision()); 53 | objectMapper.registerModule(module); 54 | String resultAsJson = objectMapper.writeValueAsString(statuses.get(0)); 55 | 56 | System.out.println(resultAsJson); 57 | } catch (Exception e) { 58 | e.printStackTrace(); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/KcsbKeywords.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.auth; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.microsoft.azure.kusto.data.Utils; 5 | import org.jetbrains.annotations.NotNull; 6 | 7 | import java.io.InputStream; 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | public class KcsbKeywords { 12 | public KeywordData[] keywords; 13 | public String version; 14 | 15 | private final Map lookup = new HashMap<>(); 16 | 17 | private static KcsbKeywords instance = null; 18 | private static final Object object = new Object(); 19 | 20 | public static KcsbKeywords getInstance() { 21 | if (instance != null) 22 | return instance; 23 | synchronized (object) { 24 | if (instance == null) { 25 | instance = readInstance(); 26 | } 27 | } 28 | return instance; 29 | } 30 | 31 | // For Deserialization 32 | public KcsbKeywords() { 33 | } 34 | 35 | public static String normalize(String keyword) { 36 | return keyword.toLowerCase().replace(" ", ""); 37 | } 38 | 39 | @NotNull 40 | private static KcsbKeywords readInstance() { 41 | try { 42 | ObjectMapper objectMapper = Utils.getObjectMapper(); 43 | try (InputStream resourceAsStream = KcsbKeywords.class.getResourceAsStream( 44 | "/kcsb.json")) { 45 | KcsbKeywords value = objectMapper.readValue(resourceAsStream, KcsbKeywords.class); 46 | 47 | // Validate the keywords 48 | for (KeywordData keywordData : value.keywords) { 49 | KnownKeywords keyword = KnownKeywords.knownKeywords.get(keywordData.name); 50 | if (keyword == null) { 51 | throw new RuntimeException( 52 | "Got unexpected keyword from embedded resource: `" + keywordData.name + "`. This is a bug in the SDK - please report it."); 53 | } 54 | 55 | keyword.setType(keywordData.type); 56 | keyword.setSecret(keywordData.secret); 57 | 58 | value.lookup.put(normalize(keywordData.name), keyword); 59 | 60 | for (String alias : keywordData.aliases) { 61 | if (value.lookup.containsKey(alias)) { 62 | throw new RuntimeException("KCSB keywordMap alias is duplicated: `" + alias + "`"); 63 | } 64 | value.lookup.put(normalize(alias), keyword); 65 | } 66 | } 67 | 68 | return value; 69 | } 70 | } catch (Exception ex) { 71 | throw new RuntimeException("Failed to read kcsb.json", ex); 72 | } 73 | } 74 | 75 | public KnownKeywords get(String keyword) { 76 | KnownKeywords result = lookup.get(normalize(keyword)); 77 | if (result == null) { 78 | throw new IllegalArgumentException("The Connection String keyword `" + keyword + "` is unknown."); 79 | } 80 | 81 | if (!result.isSupported()) { 82 | throw new IllegalArgumentException("The Connection String keyword `" + keyword + "` is not supported by this SDK."); 83 | } 84 | 85 | return result; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/KustoResultColumnPopulator.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data; 2 | 3 | import java.util.function.BiConsumer; 4 | 5 | /** 6 | * Represents a single Kusto result column and its mapping to a java object. Used to extract the column from a {@link KustoResultSetTable} and set the 7 | * value in a pojo. The column is defined by either its name or its ordinal (or both, in which case the ordinal is used). 8 | * 9 | * @param 10 | * the pojo this column needs to set its value to 11 | * @param 12 | * the Java class returned by the ADX column type 13 | * @param 14 | * type of ADX column 15 | */ 16 | class KustoResultColumnPopulator> { 17 | 18 | final static int UNSET_ORDINAL = -1; 19 | 20 | final BiConsumer valueSetter; 21 | final String name; 22 | final T type; 23 | final boolean isNullable; 24 | final int presetOrdinal; 25 | 26 | static > KustoResultColumnPopulator of(String name, int ordinal, T type, boolean isNullable, 27 | BiConsumer valueSetter) { 28 | return new KustoResultColumnPopulator<>(name, ordinal, type, isNullable, valueSetter); 29 | } 30 | 31 | static > KustoResultColumnPopulator of(String name, T type, boolean isNullable, BiConsumer valueSetter) { 32 | return new KustoResultColumnPopulator<>(name, UNSET_ORDINAL, type, isNullable, valueSetter); 33 | } 34 | 35 | static > KustoResultColumnPopulator of(int ordinal, T type, boolean isNullable, BiConsumer valueSetter) { 36 | return new KustoResultColumnPopulator<>(null, ordinal, type, isNullable, valueSetter); 37 | } 38 | 39 | KustoResultColumnPopulator(String name, int ordinal, T type, boolean isNullable, BiConsumer valueSetter) { 40 | this.name = name; 41 | this.presetOrdinal = ordinal; 42 | this.type = type; 43 | this.isNullable = isNullable; 44 | this.valueSetter = valueSetter; 45 | } 46 | 47 | void populateFrom(R objToPopulate, KustoResultSetTable resultSet) { 48 | populateFrom(objToPopulate, resultSet, columnIndexInResultSet(resultSet)); 49 | } 50 | 51 | void populateFrom(R objToPopulate, KustoResultSetTable resultSet, int ordinal) { 52 | Object resultValue = resultSet.getObject(ordinal); 53 | if (resultValue == null) { 54 | if (!this.isNullable) { 55 | throw new NullPointerException(String.format("Column %s (ordinal %d) is not nullable", this.name, ordinal)); 56 | } 57 | this.valueSetter.accept(objToPopulate, null); 58 | return; 59 | } 60 | C typed; 61 | try { 62 | typed = this.type.type(resultValue); 63 | } catch (Exception e) { 64 | throw new IllegalArgumentException(String.format("Column %s (ordinal %d) is of type %s but expected type is %s", this.name, ordinal, 65 | resultValue.getClass().toString(), this.type.clazz.toString()), e); 66 | } 67 | this.valueSetter.accept(objToPopulate, typed); 68 | } 69 | 70 | int columnIndexInResultSet(KustoResultSetTable resultSet) { 71 | return this.presetOrdinal == UNSET_ORDINAL ? resultSet.findColumn(this.name) : this.presetOrdinal; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /ingest/src/main/java/com/microsoft/azure/kusto/ingest/utils/SecurityUtils.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.ingest.utils; 5 | 6 | import java.io.*; 7 | import java.security.GeneralSecurityException; 8 | import java.security.KeyFactory; 9 | import java.security.cert.CertificateException; 10 | import java.security.cert.CertificateFactory; 11 | import java.security.cert.X509Certificate; 12 | import java.security.interfaces.RSAPrivateKey; 13 | import java.security.interfaces.RSAPublicKey; 14 | import java.security.spec.PKCS8EncodedKeySpec; 15 | import java.security.spec.X509EncodedKeySpec; 16 | 17 | public class SecurityUtils { 18 | private SecurityUtils() { 19 | // Hide constructor as this is a Utils class 20 | } 21 | 22 | public static String removeSecretsFromUrl(String url) { 23 | return url.split("[?]", 2)[0].split("[;]", 2)[0]; 24 | } 25 | 26 | public static X509Certificate getPublicCertificate(String fileLoc) throws IOException, CertificateException { 27 | try (FileInputStream is = new FileInputStream(fileLoc)) { 28 | CertificateFactory fact = CertificateFactory.getInstance("X.509"); 29 | return (X509Certificate) fact.generateCertificate(is); 30 | } 31 | } 32 | 33 | public static RSAPrivateKey getPrivateKey(String filename) throws IOException, GeneralSecurityException { 34 | String privateKeyPEM = getKey(filename); 35 | return getPrivateKeyFromString(privateKeyPEM); 36 | } 37 | 38 | public static RSAPublicKey getPublicKey(String filename) throws IOException, GeneralSecurityException { 39 | String publicKeyPEM = getKey(filename); 40 | return getPublicKeyFromString(publicKeyPEM); 41 | } 42 | 43 | private static String getKey(String filename) throws IOException { 44 | // Read key from file 45 | StringBuilder strKeyPEM = new StringBuilder(); 46 | try (BufferedReader br = new BufferedReader(new FileReader(filename))) { 47 | String line; 48 | while ((line = br.readLine()) != null) { 49 | strKeyPEM.append(line).append("\n"); 50 | } 51 | } 52 | return strKeyPEM.toString(); 53 | } 54 | 55 | private static RSAPrivateKey getPrivateKeyFromString(String key) throws GeneralSecurityException { 56 | String keyPEM = key; 57 | keyPEM = keyPEM.replace("-----BEGIN PRIVATE KEY-----\n", ""); 58 | keyPEM = keyPEM.replace("-----END PRIVATE KEY-----", ""); 59 | keyPEM = keyPEM.replace("\n", "").replace("\r", ""); 60 | byte[] decoded = java.util.Base64.getDecoder().decode(keyPEM); 61 | KeyFactory kf = KeyFactory.getInstance("RSA"); 62 | PKCS8EncodedKeySpec keySpec = new PKCS8EncodedKeySpec(decoded); 63 | return (RSAPrivateKey) kf.generatePrivate(keySpec); 64 | } 65 | 66 | private static RSAPublicKey getPublicKeyFromString(String key) throws GeneralSecurityException { 67 | String keyPEM = key; 68 | keyPEM = keyPEM.replace("-----BEGIN PUBLIC KEY-----\n", ""); 69 | keyPEM = keyPEM.replace("-----END PUBLIC KEY-----", ""); 70 | keyPEM = keyPEM.replace("\n", "").replace("\r", ""); 71 | byte[] decoded = java.util.Base64.getDecoder().decode(keyPEM); 72 | KeyFactory kf = KeyFactory.getInstance("RSA"); 73 | X509EncodedKeySpec keySpec = new X509EncodedKeySpec(decoded); 74 | return (RSAPublicKey) kf.generatePublic(keySpec); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/format/CslStringFormat.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.kusto.data.format; 2 | 3 | import com.microsoft.azure.kusto.data.Ensure; 4 | import com.microsoft.azure.kusto.data.StringUtils; 5 | 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | import java.util.stream.Collectors; 9 | import java.util.stream.Stream; 10 | 11 | public class CslStringFormat extends CslFormat { 12 | private static final Set KUSTO_LITERAL_PREFIX = Stream.of("H", "h").collect(Collectors.toCollection(HashSet::new)); 13 | private static final Set KUSTO_MULTILINE_QUOTE_DELIMITERS = Stream.of("```", "~~~").collect(Collectors.toCollection(HashSet::new)); 14 | private static final Set KUSTO_ESCAPE_SEQUENCES = Stream.of("\\\"", "'", "@\\\"", "@'").collect(Collectors.toCollection(HashSet::new)); 15 | 16 | private final String value; 17 | 18 | public CslStringFormat(String value) { 19 | this.value = value; 20 | } 21 | 22 | @Override 23 | public String getType() { 24 | return null; 25 | } 26 | 27 | @Override 28 | public String getValue() { 29 | return value; 30 | } 31 | 32 | @Override 33 | String getValueAsString() { 34 | Ensure.stringIsNotBlank(value, "value"); 35 | 36 | return value; 37 | } 38 | 39 | public static String parseStringLiteral(String value) { 40 | String result = value; 41 | if (KUSTO_LITERAL_PREFIX.contains(result.substring(0, 1))) { 42 | result = result.substring(1); 43 | } 44 | 45 | String multilineString = parseMultilineString(result); 46 | if (StringUtils.isNotBlank(multilineString)) { 47 | return multilineString; 48 | } 49 | 50 | return unescapeString(result); 51 | } 52 | 53 | private static String parseMultilineString(String quotedString) { 54 | for (String quoteDelimiter : KUSTO_MULTILINE_QUOTE_DELIMITERS) { 55 | if (quotedString.startsWith(quoteDelimiter)) { 56 | int twiceQuoteLen = quoteDelimiter.length() * 2; 57 | if (quotedString.length() >= twiceQuoteLen && quotedString.endsWith(quoteDelimiter)) { 58 | return quotedString.substring(quoteDelimiter.length(), quotedString.length() - twiceQuoteLen); 59 | } else { 60 | return quotedString.substring(quoteDelimiter.length()); 61 | } 62 | } 63 | } 64 | 65 | return null; 66 | } 67 | 68 | private static String unescapeString(String escapedString) { 69 | for (String escapeSequence : KUSTO_ESCAPE_SEQUENCES) { 70 | if (escapedString.startsWith(escapeSequence)) { 71 | int escapeSequenceLength = escapeSequence.length() + 1; 72 | if (escapedString.length() >= escapeSequenceLength && escapedString.endsWith(escapeSequence)) { 73 | String unescapedString = escapedString.substring(escapeSequence.length(), escapedString.length() - escapeSequence.length()); 74 | if ("\\\"".equals(escapeSequence) || "'".equals(escapeSequence)) { 75 | return StringUtils.unescapeJava(unescapedString); 76 | } else if (escapeSequence.startsWith("@")) { 77 | String quote = escapeSequence.substring(1); 78 | return escapedString.replaceAll(quote + quote, quote); 79 | } 80 | } 81 | } 82 | } 83 | 84 | return escapedString; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /samples/src/main/java/FileIngestion.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | import com.azure.data.tables.TableAsyncClient; 5 | import com.azure.data.tables.models.TableEntity; 6 | import com.microsoft.azure.kusto.data.auth.ConnectionStringBuilder; 7 | import com.microsoft.azure.kusto.ingest.ColumnMapping; 8 | import com.microsoft.azure.kusto.ingest.IngestClient; 9 | import com.microsoft.azure.kusto.ingest.IngestClientFactory; 10 | import com.microsoft.azure.kusto.ingest.IngestionMapping; 11 | import com.microsoft.azure.kusto.ingest.IngestionProperties; 12 | import com.microsoft.azure.kusto.ingest.result.IngestionResult; 13 | import com.microsoft.azure.kusto.ingest.source.FileSourceInfo; 14 | import com.microsoft.azure.kusto.ingest.source.StreamSourceInfo; 15 | import com.microsoft.azure.kusto.ingest.utils.TableWithSas; 16 | 17 | import java.io.ByteArrayInputStream; 18 | import java.io.ByteArrayOutputStream; 19 | 20 | public class FileIngestion { 21 | public static void main(String[] args) { 22 | try { 23 | // TableAsyncClient tableAsyncClient = 24 | // TableWithSas.createTableClientFromUrl("https://5s8kstrldruthruth01.blob.core.windows.net/20230313-ingestdata-e5c334ee145d4b4-0?sv=2018-03-28&sr=c&sig=QshIuU9ZZ1jvcgcPMnHcr0EvCwO9sxZbvAUaAtI%3D&st=2023-03-13T13%3A16%3A57Z&se=2023-03-17T14%3A16%3A57Z&sp=rw", 25 | // null); 26 | // tableAsyncClient.createEntity(new TableEntity("123", "123")).block(); 27 | 28 | ConnectionStringBuilder csb = ConnectionStringBuilder.createWithUserPrompt("https://ruthruth.eastus.kusto.windows.net"); 29 | try (IngestClient client = IngestClientFactory.createClient(csb)) { 30 | IngestionProperties ingestionProperties = new IngestionProperties("db2", "TestTable"); 31 | ingestionProperties.setReportMethod(IngestionProperties.IngestionReportMethod.TABLE); 32 | ingestionProperties.setReportLevel(IngestionProperties.IngestionReportLevel.FAILURES_AND_SUCCESSES); 33 | FileSourceInfo fileSourceInfo = new FileSourceInfo("C:\\Users\\ohbitton\\OneDrive - Microsoft\\Desktop\\data\\a.csv"); 34 | IngestionResult ingestionResult = client.ingestFromFile(fileSourceInfo, ingestionProperties); 35 | ByteArrayOutputStream st = new ByteArrayOutputStream(); 36 | st.write("asd,2".getBytes()); 37 | ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(st.toByteArray()); 38 | StreamSourceInfo info = new StreamSourceInfo(byteArrayInputStream); 39 | 40 | // Ingest with inline ingestion mapping - less recommended 41 | IngestionProperties ingestionProperties2 = new IngestionProperties(System.getProperty("dbName"), 42 | System.getProperty("tableName")); 43 | ColumnMapping csvColumnMapping = new ColumnMapping("ColA", "string"); 44 | csvColumnMapping.setOrdinal(0); 45 | ColumnMapping csvColumnMapping2 = new ColumnMapping("ColB", "int"); 46 | csvColumnMapping2.setOrdinal(1); 47 | ingestionProperties2.setDataFormat(IngestionProperties.DataFormat.CSV); 48 | ingestionProperties2.setIngestionMapping(new ColumnMapping[] {csvColumnMapping, csvColumnMapping2}, IngestionMapping.IngestionMappingKind.CSV); 49 | 50 | IngestionResult ingestionResult2 = client.ingestFromStream(info, ingestionProperties2); 51 | } 52 | } catch (Exception e) { 53 | e.printStackTrace(); 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /data/src/main/java/com/microsoft/azure/kusto/data/auth/ConfidentialAppTokenProviderBase.java: -------------------------------------------------------------------------------- 1 | // Copyright (c) Microsoft Corporation. 2 | // Licensed under the MIT License. 3 | 4 | package com.microsoft.azure.kusto.data.auth; 5 | 6 | import com.microsoft.aad.msal4j.ClientCredentialParameters; 7 | import com.microsoft.aad.msal4j.IAccount; 8 | import com.microsoft.aad.msal4j.IAuthenticationResult; 9 | import com.microsoft.aad.msal4j.IConfidentialClientApplication; 10 | import com.microsoft.azure.kusto.data.exceptions.DataClientException; 11 | import com.microsoft.azure.kusto.data.exceptions.DataServiceException; 12 | import java.net.MalformedURLException; 13 | import java.net.URISyntaxException; 14 | import java.util.Set; 15 | import java.util.concurrent.CompletableFuture; 16 | import java.util.concurrent.ExecutionException; 17 | import java.util.concurrent.TimeUnit; 18 | import java.util.concurrent.TimeoutException; 19 | 20 | import com.azure.core.http.HttpClient; 21 | import org.jetbrains.annotations.NotNull; 22 | import org.jetbrains.annotations.Nullable; 23 | 24 | public abstract class ConfidentialAppTokenProviderBase extends MsalTokenProviderBase { 25 | final String applicationClientId; 26 | IConfidentialClientApplication clientApplication; 27 | 28 | ConfidentialAppTokenProviderBase(@NotNull String clusterUrl, @NotNull String applicationClientId, String authorityId, 29 | @Nullable HttpClient httpClient) throws URISyntaxException { 30 | super(clusterUrl, authorityId, httpClient); 31 | this.applicationClientId = applicationClientId; 32 | } 33 | 34 | @Override 35 | protected void initializeWithCloudInfo(CloudInfo cloudInfo) throws DataClientException, DataServiceException { 36 | super.initializeWithCloudInfo(cloudInfo); 37 | try { 38 | clientApplication = getClientApplication(); 39 | } catch (MalformedURLException e) { 40 | throw new DataClientException(clusterUrl, ERROR_INVALID_AUTHORITY_URL, e); 41 | } 42 | } 43 | 44 | @Override 45 | protected IAuthenticationResult acquireNewAccessToken() throws DataServiceException { 46 | IAuthenticationResult result; 47 | try { 48 | CompletableFuture future = clientApplication.acquireToken( 49 | ClientCredentialParameters.builder(scopes).build()); 50 | result = future.get(TIMEOUT_MS, TimeUnit.MILLISECONDS); 51 | } catch (ExecutionException | TimeoutException e) { 52 | throw new DataServiceException(clusterUrl, ERROR_ACQUIRING_APPLICATION_ACCESS_TOKEN, e, false); 53 | } catch (InterruptedException e) { 54 | Thread.currentThread().interrupt(); 55 | throw new DataServiceException(clusterUrl, ERROR_ACQUIRING_APPLICATION_ACCESS_TOKEN, e, false); 56 | } 57 | 58 | if (result == null) { 59 | throw new DataServiceException(clusterUrl, "acquireNewAccessToken got 'null' authentication result", false); 60 | } 61 | return result; 62 | } 63 | 64 | @Override 65 | protected IAuthenticationResult acquireAccessTokenSilentlyMsal() 66 | throws MalformedURLException, InterruptedException, ExecutionException, TimeoutException { 67 | CompletableFuture> accounts = clientApplication.getAccounts(); 68 | return clientApplication 69 | .acquireTokenSilently(getSilentParameters(accounts.join())) 70 | .get(TIMEOUT_MS, TimeUnit.MILLISECONDS); 71 | } 72 | 73 | protected abstract IConfidentialClientApplication getClientApplication() throws MalformedURLException; 74 | } 75 | --------------------------------------------------------------------------------