├── src ├── site │ └── markdown │ │ ├── index.md │ │ ├── NextSteps.md │ │ └── Setup-Azure-Active-Directory-Service-Principal.md ├── test │ ├── configfiles │ │ ├── .gitignore │ │ ├── creds.properties │ │ └── config.properties │ ├── java │ │ └── com │ │ │ └── contoso │ │ │ ├── helpers │ │ │ ├── MockHelper.java │ │ │ └── HelperUtils.java │ │ │ ├── acl │ │ │ ├── TestAclSpec.java │ │ │ ├── TestAclEntry.java │ │ │ └── TestAclAction.java │ │ │ ├── unittests │ │ │ └── TestRetryPolicies.java │ │ │ ├── liveservicetests │ │ │ ├── TestPositionedReads.java │ │ │ └── TestCore.java │ │ │ └── mocktests │ │ │ └── TestSdkMock.java │ └── resources │ │ └── log4j2.xml └── main │ ├── resources │ └── adlsdkversion.properties │ └── java │ └── com │ └── microsoft │ └── azure │ └── datalake │ └── store │ ├── retrypolicies │ ├── package-info.java │ ├── RetryPolicy.java │ ├── NoRetryPolicy.java │ ├── NonIdempotentRetryPolicy.java │ ├── ExponentialBackoffPolicy.java │ └── ExponentialBackoffPolicyforMSI.java │ ├── oauth2 │ ├── package-info.java │ ├── AzureADToken.java │ ├── RefreshTokenInfo.java │ ├── UserPasswordTokenProvider.java │ ├── DeviceCodeCallback.java │ ├── ClientCredsTokenProvider.java │ ├── RefreshTokenBasedTokenProvider.java │ ├── DeviceCodeTokenProvider.java │ ├── AccessTokenProvider.java │ ├── MsiTokenProvider.java │ └── DeviceCodeTokenProviderHelper.java │ ├── acl │ ├── AclScope.java │ ├── AclType.java │ ├── package-info.java │ ├── AclStatus.java │ ├── AclAction.java │ └── AclEntry.java │ ├── ReadBufferStatus.java │ ├── DirectoryEntryType.java │ ├── package-info.java │ ├── IfExists.java │ ├── SyncFlag.java │ ├── UserGroupRepresentation.java │ ├── DirectoryEntryListWithContinuationToken.java │ ├── RequestOptions.java │ ├── ExpiryOption.java │ ├── ContentSummary.java │ ├── ReadBuffer.java │ ├── ReadBufferWorker.java │ ├── ADLException.java │ ├── QueryParams.java │ ├── ProcessingQueue.java │ ├── OperationResponse.java │ ├── Utils.java │ ├── LatencyTracker.java │ ├── DirectoryEntry.java │ ├── ContentSummaryProcessor.java │ ├── Operation.java │ ├── SSLSocketFactoryEx.java │ ├── ADLStoreOptions.java │ └── ADLFileOutputStream.java ├── README.md ├── .gitignore ├── LICENSE.txt ├── SECURITY.md ├── CHANGES.md └── pom.xml /src/site/markdown/index.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/site/markdown/NextSteps.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/test/configfiles/.gitignore: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /src/test/configfiles/creds.properties: -------------------------------------------------------------------------------- 1 | 2 | ClientSecret= 3 | -------------------------------------------------------------------------------- /src/site/markdown/Setup-Azure-Active-Directory-Service-Principal.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /src/main/resources/adlsdkversion.properties: -------------------------------------------------------------------------------- 1 | sdkversion=${project.version} 2 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/helpers/MockHelper.java: -------------------------------------------------------------------------------- 1 | package com.contoso.helpers; 2 | 3 | 4 | public class MockHelper { 5 | 6 | 7 | 8 | 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | /** 8 | * All the retry policies. 9 | */ 10 | package com.microsoft.azure.datalake.store.retrypolicies; -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | /** 8 | * Classes to help fetch and manage Azure ActiveDirectory tokens 9 | */ 10 | package com.microsoft.azure.datalake.store.oauth2; -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Azure Datalake Store client for Java 2 | 3 | 4 | - For an introduction to Azure Data Lake, see here: https://azure.microsoft.com/en-us/services/data-lake-store/ 5 | 6 | - For getting started introduction to the SDK, see here: https://github.com/Azure-samples/data-lake-store-java-upload-download-get-started 7 | 8 | - For the SDK Javadoc, see here: http://azure.github.io/azure-data-lake-store-java/javadoc 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/AclScope.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.acl; 8 | 9 | 10 | /** 11 | * The scope of an ACL Entry (access or default). 12 | */ 13 | public enum AclScope { 14 | ACCESS, 15 | DEFAULT 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ReadBufferStatus.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.datalake.store; 2 | 3 | 4 | enum ReadBufferStatus { 5 | NOT_AVAILABLE, // buffers sitting in readaheadqueue have this stats 6 | READING_IN_PROGRESS, // reading is in progress on this buffer. Buffer should be in inProgressList 7 | AVAILABLE, // data is available in buffer. It should be in completedList 8 | READ_FAILED // read completed, but failed. 9 | } -------------------------------------------------------------------------------- /src/test/configfiles/config.properties: -------------------------------------------------------------------------------- 1 | 2 | dirName=${customDir} 3 | 4 | # Info required for acquiring AAD token 5 | StoreAcct= 6 | ClientId= 7 | OAuth2TokenUrl= 8 | ClientAppName= 9 | ClientRedirectUri= 10 | 11 | # Control what unit tests are enabled 12 | CoreTestsEnabled=true 13 | SdkTestsEnabled=true 14 | PositionedReadsTestsEnabled=true 15 | acl.AclActionTestsEnabled=true 16 | acl.AclEntryTestsEnabled=true 17 | acl.AclSpecTestsEnabled=true 18 | LongRunningTestsEnabled=false -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/AclType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.acl; 8 | 9 | 10 | /** 11 | * Type of Acl entry (user, group, other, or mask). 12 | */ 13 | public enum AclType { 14 | USER, 15 | GROUP, 16 | OTHER, 17 | MASK 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/DirectoryEntryType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | /** 11 | * enum to indicate whether a directory entry is a file or a directory. 12 | */ 13 | public enum DirectoryEntryType { 14 | FILE, 15 | DIRECTORY 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | /** 8 | * The main Azure Data Lake Store SDK classes. If you are new to the SDK, start exploring from the 9 | * {@link com.microsoft.azure.datalake.store.ADLStoreClient} class. 10 | * 11 | */ 12 | package com.microsoft.azure.datalake.store; 13 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/package-info.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | /** 8 | * Objects used for ACLs. These objects are used as parameters to the ACL calls in 9 | * {@link com.microsoft.azure.datalake.store.Core} and 10 | * {@link com.microsoft.azure.datalake.store.ADLStoreClient} classes. 11 | * 12 | */ 13 | package com.microsoft.azure.datalake.store.acl; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .classpath 2 | .project 3 | target 4 | node_modules 5 | .idea 6 | *.iml 7 | 8 | *.pydevproject 9 | .metadata 10 | .gradle 11 | bin/** 12 | tmp/** 13 | tmp/**/* 14 | *.tmp 15 | *.bak 16 | *.swp 17 | *~.nib 18 | local.properties 19 | .settings/** 20 | */.settings/** 21 | .loadpath 22 | 23 | # External tool builders 24 | .externalToolBuilders/ 25 | 26 | # Locally stored "Eclipse launch configurations" 27 | *.launch 28 | 29 | # CDT-specific 30 | .cproject 31 | 32 | # PDT-specific 33 | .buildpath 34 | 35 | # TeXlipse plugin 36 | .texlipse 37 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/AzureADToken.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import java.util.Date; 10 | 11 | 12 | /** 13 | * Object represnting the AAD access token to use when making HTTP requests to Azure Data Lake Storage. 14 | */ 15 | public class AzureADToken { 16 | public String accessToken; 17 | public Date expiry; 18 | } -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/IfExists.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | /** 10 | * Enum specifying actions to take if attempting to create a file that already exists. 11 | */ 12 | public enum IfExists { 13 | /** 14 | * Overwrite the file 15 | */ 16 | OVERWRITE, 17 | 18 | /** 19 | * Fail the request 20 | */ 21 | FAIL 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/RefreshTokenInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | 10 | import java.util.Date; 11 | 12 | /** 13 | * Information about the refresh token, and the associated access token 14 | * 15 | */ 16 | public class RefreshTokenInfo { 17 | public String accessToken; 18 | public String refreshToken; 19 | public Date accessTokenExpiry; 20 | } -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/SyncFlag.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | 8 | package com.microsoft.azure.datalake.store; 9 | 10 | /** 11 | * Indicator flags to backend during append. 12 | */ 13 | public enum SyncFlag { 14 | /** 15 | * No update is required, and hold lease. - Performant operation. 16 | */ 17 | DATA, 18 | 19 | /** 20 | * Update metdata of the file after the given data is appended to file. 21 | */ 22 | METADATA, 23 | 24 | /** 25 | * Update metdata of the file after the given data is appended to file. 26 | * And close file handle. Once the file handle is closed, lease on the 27 | * file is released if the stream is opened with leaseid. 28 | */ 29 | CLOSE 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/UserGroupRepresentation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | 8 | package com.microsoft.azure.datalake.store; 9 | 10 | 11 | /** 12 | * Enum specifying how user and group objects should be represented in calls that return user and group ID. 13 | */ 14 | public enum UserGroupRepresentation { 15 | 16 | /** 17 | * Object ID (OID), which is a GUID representing the ID of the user or group. The OID is immutable even if 18 | * the name of the user or group changes. 19 | */ 20 | OID, 21 | 22 | /** 23 | * User Principal Name of the group or user, which is the human-friendly username. Since users and groups are 24 | * stored internally as OID, using the UPN involves an additional lookup into the directory. 25 | */ 26 | UPN 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/DirectoryEntryListWithContinuationToken.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.datalake.store; 2 | 3 | import java.util.List; 4 | 5 | class DirectoryEntryListWithContinuationToken { 6 | private String continuationToken; 7 | private List entries; 8 | 9 | String getContinuationToken() { 10 | return continuationToken; 11 | } 12 | void setContinuationToken(String continuationToken){ 13 | this.continuationToken = continuationToken; 14 | } 15 | 16 | List getEntries() { 17 | return entries; 18 | } 19 | 20 | void setEntries(List entries) { 21 | this.entries = entries; 22 | } 23 | 24 | DirectoryEntryListWithContinuationToken(String continuationToken, List entries){ 25 | this.continuationToken = continuationToken; 26 | this.entries = entries; 27 | } 28 | DirectoryEntryListWithContinuationToken(){ 29 | continuationToken = ""; 30 | entries = null; 31 | } 32 | 33 | 34 | } -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/RequestOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import com.microsoft.azure.datalake.store.retrypolicies.RetryPolicy; 10 | 11 | 12 | /** 13 | * common options to control the behavior of server calls 14 | */ 15 | public class RequestOptions { 16 | /** 17 | * the timeout (in milliseconds) to use for the request. This is used for both 18 | * the readTimeout and the connectTimeout for the request, so 19 | * in effect the actual timout is two times the specified timeout. 20 | * Default is 60,000 (60 seconds). 21 | */ 22 | public int timeout = 60000; 23 | 24 | /** 25 | * the client request ID. the SDK generates a UUID if a request ID is not specified. 26 | */ 27 | public String requestid = null; 28 | 29 | /** 30 | * the {@link RetryPolicy} to use for the request 31 | */ 32 | public RetryPolicy retryPolicy = null; 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ExpiryOption.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | /** 11 | * Enum specifying how to interpret the expiry time specified in setExpiry call. 12 | */ 13 | public enum ExpiryOption { 14 | /** 15 | * No expiry. ExpireTime is ignored. 16 | */ 17 | NeverExpire, 18 | /** 19 | * Interpret as miliseconds from now. ExpireTime is an integer in milliseconds representing the expiration date 20 | * relative to when file expiration is updated 21 | */ 22 | RelativeToNow, 23 | /** 24 | * Interpet as milliseconds from the file's creation date+time. ExpireTime is an integer in milliseconds 25 | * representing the expiration date relative to file creation 26 | */ 27 | RelativeToCreationDate, 28 | /** 29 | * Interpret as date/time. ExpireTime is an integer in milliseconds, as a Unix timestamp relative 30 | * to 1/1/1970 00:00:00 31 | */ 32 | Absolute 33 | } 34 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Azure Data Lake Store Java SDK 2 | Copyright (c) Microsoft Corporation 3 | All rights reserved. 4 | 5 | The MIT License (MIT) 6 | 7 | 8 | Permission is hereby granted, free of charge, to any person obtaining a copy 9 | of this software and associated documentation files (the "Software"), to deal 10 | in the Software without restriction, including without limitation the rights 11 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 12 | copies of the Software, and to permit persons to whom the Software is 13 | furnished to do so, subject to the following conditions: 14 | 15 | The above copyright notice and this permission notice shall be included in all 16 | copies or substantial portions of the Software. 17 | 18 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 21 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 22 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 23 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 24 | SOFTWARE. 25 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ContentSummary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | /** 10 | * structure that contains the return values from getContentSummary call. 11 | */ 12 | public class ContentSummary { 13 | 14 | /** 15 | * length of file 16 | */ 17 | public final long length; 18 | 19 | /** 20 | * number of subdirectories under a directory 21 | */ 22 | public final long directoryCount; 23 | 24 | /** 25 | * number of files under a directory 26 | */ 27 | public final long fileCount; 28 | 29 | /** 30 | * total space consumed by a directory 31 | */ 32 | public final long spaceConsumed; 33 | 34 | public ContentSummary( 35 | long length, 36 | long directoryCount, 37 | long fileCount, 38 | long spaceConsumed 39 | ) { 40 | this.length = length; 41 | this.directoryCount = directoryCount; 42 | this.fileCount = fileCount; 43 | this.spaceConsumed = spaceConsumed; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/AclStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.acl; 8 | 9 | 10 | import java.util.List; 11 | 12 | /** 13 | * Object returned by the {@link com.microsoft.azure.datalake.store.ADLStoreClient#getAclStatus(String)} getAclStatus} 14 | * call, that contains the Acl and Permission information for that file or directory. 15 | * 16 | */ 17 | public class AclStatus { 18 | /** 19 | * {@code List} containing the list of Acl entries for a file 20 | */ 21 | public List aclSpec; 22 | 23 | /** 24 | * String containing the ID of the owner of the file 25 | */ 26 | public String owner; 27 | 28 | /** 29 | * String containing the ID of the group that owns this file 30 | */ 31 | public String group; 32 | 33 | /** 34 | * Unix permissions for the file/directory in Octal form 35 | */ 36 | public String octalPermissions; 37 | 38 | /** 39 | * Sticky bit (only meaningful for a directory) 40 | */ 41 | public boolean stickyBit; 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ReadBuffer.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.datalake.store; 2 | 3 | 4 | import java.util.concurrent.CountDownLatch; 5 | 6 | /** 7 | * This object represents the buffer state as it is going through it's lifecycle. 8 | * The buffer (the byte array) itself is assigned to this object from a free pool, 9 | * so we do not create tons of objects in large object heap. 10 | */ 11 | class ReadBuffer { 12 | ADLFileInputStream file; 13 | long offset; // offset within the file for the buffer 14 | int length; // actual length, set after the buffer is filles 15 | int requestedLength; // requested length of the read 16 | byte[] buffer; // the buffer itself 17 | int bufferindex = -1; // index in the buffers array in Buffer manager 18 | ReadBufferStatus status; // status of the buffer 19 | CountDownLatch latch = null; // signaled when the buffer is done reading, so any client 20 | // waiting on this buffer gets unblocked 21 | 22 | // fields to help with eviction logic 23 | long birthday = 0; // tick at which buffer became available to read 24 | boolean firstByteConsumed = false; 25 | boolean lastByteConsumed = false; 26 | boolean anyByteConsumed = false; 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/UserPasswordTokenProvider.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.datalake.store.oauth2; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * Provides tokens based on username and password 10 | */ 11 | public class UserPasswordTokenProvider extends AccessTokenProvider { 12 | 13 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.UserPasswordTokenProvider"); 14 | private final String clientId, username, password; 15 | 16 | /** 17 | * constructs a token provider based on supplied credentials. 18 | * 19 | * @param username the username 20 | * @param clientId the client ID (GUID) obtained from Azure Active Directory configuration 21 | * @param password the password 22 | */ 23 | public UserPasswordTokenProvider(String clientId, String username, String password) { 24 | this.clientId = clientId; 25 | this.username = username; 26 | this.password = password; 27 | } 28 | 29 | @Override 30 | protected AzureADToken refreshToken() throws IOException { 31 | log.debug("AADToken: refreshing user-password based token"); 32 | return AzureADAuthenticator.getTokenUsingClientCreds(clientId, username, password); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/RetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.retrypolicies; 8 | 9 | /** 10 | * the RetryPolicy controls whether a failed request should be retried, and how long to wait before retrying. 11 | *

12 | * Implementations of this interface implement different retry policies. 13 | *

14 | */ 15 | public interface RetryPolicy { 16 | /** 17 | * boolean indicating whether a failed request should be retried. Implementations can use the 18 | * HTTP response code and any exceptions from the last failure to decide whether to retry. 19 | *

20 | * If the retry policy requires a wait before the next try, then the {@code shouldRetry} method should wait for 21 | * the appropriate time before responding back. i.e., there is not an explicit contract for waits, but it 22 | * is implicit in the time taken by the {@code shouldRetry} method to return. 23 | *

24 | * @param httpResponseCode the HTTP response code received 25 | * @param lastException any exceptions encountered while processing the last request 26 | * @return boolean indicating whether the request should be retried 27 | */ 28 | boolean shouldRetry(int httpResponseCode, Exception lastException); 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/DeviceCodeCallback.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | /** 10 | * 11 | * Shows the login message for device code to user. The default implementation shows on the console. 12 | * Subclasses can override the {@link DeviceCodeCallback#showDeviceCodeMessage(DeviceCodeInfo)} method to 13 | * display the message in a different way, appropriate for the context the program is running in. 14 | * 15 | */ 16 | class DeviceCodeCallback { 17 | 18 | private static DeviceCodeCallback defaultInstance = new DeviceCodeCallback(); 19 | 20 | /** 21 | * Show the message to the user, instructing them to log in using the browser. 22 | * This method displays the message on standard output; subclasses may display 23 | * it differently. 24 | * 25 | * @param dcInfo {@link DeviceCodeInfo} object containing the info to display 26 | */ 27 | public void showDeviceCodeMessage(DeviceCodeInfo dcInfo) { 28 | System.out.println(dcInfo.message); 29 | } 30 | 31 | /** 32 | * Returns an instance of the default {@link DeviceCodeCallback} 33 | * 34 | * @return an instance of the default {@link DeviceCodeCallback} 35 | */ 36 | public static DeviceCodeCallback getDefaultInstance() { 37 | return defaultInstance; 38 | } 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/NoRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.retrypolicies; 8 | 9 | /** 10 | * No retry ever. Always returns false, indicating that erequest should not be retried. 11 | * 12 | * This should be used when retrying is not safe, and user wants at-most-once semantics with the call. This is 13 | * useful for non-idempotent methods, where the error returned by the last call does not conclusively indicate 14 | * success or failure of the call. For example, if an append times out but succeeds on the back-end , then 15 | * retrying it may append the data twice to the file. 16 | * 17 | */ 18 | public class NoRetryPolicy implements RetryPolicy { 19 | 20 | private int retryCount = 0; 21 | private int waitInterval = 100; 22 | 23 | 24 | public boolean shouldRetry(int httpResponseCode, Exception lastException) { 25 | if (httpResponseCode == 401 && retryCount == 0) { 26 | // to mitigate a special problem with intermittent 401's on calls 27 | wait(waitInterval); 28 | retryCount++; 29 | return true; 30 | } 31 | return false; 32 | } 33 | 34 | private void wait(int milliseconds) { 35 | try { 36 | Thread.sleep(milliseconds); 37 | } catch (InterruptedException ex) { 38 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/test/resources/log4j2.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/ClientCredsTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | 14 | /** 15 | * Provides tokens based on client credentials 16 | */ 17 | public class ClientCredsTokenProvider extends AccessTokenProvider { 18 | 19 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.ClientCredsTokenProvider"); 20 | private final String authEndpoint, clientId, clientSecret; 21 | 22 | /** 23 | * constructs a token provider based on supplied credentials. 24 | * 25 | * @param authEndpoint the OAuth 2.0 token endpoint associated with the user's directory 26 | * (obtain from Active Directory configuration) 27 | * @param clientId the client ID (GUID) of the client web app obtained from Azure Active Directory configuration 28 | * @param clientSecret the secret key of the client web app 29 | */ 30 | public ClientCredsTokenProvider(String authEndpoint, String clientId, String clientSecret) { 31 | this.authEndpoint = authEndpoint; 32 | this.clientId = clientId; 33 | this.clientSecret = clientSecret; 34 | } 35 | 36 | @Override 37 | protected AzureADToken refreshToken() throws IOException { 38 | log.debug("AADToken: refreshing client-credential based token"); 39 | return AzureADAuthenticator.getTokenUsingClientCreds(authEndpoint, clientId, clientSecret); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/acl/TestAclSpec.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.acl; 8 | 9 | 10 | import com.contoso.helpers.HelperUtils; 11 | import com.microsoft.azure.datalake.store.acl.AclEntry; 12 | import org.junit.BeforeClass; 13 | import org.junit.Test; 14 | import static org.junit.Assert.*; 15 | 16 | import java.io.IOException; 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | public class TestAclSpec { 21 | final UUID instanceGuid = UUID.randomUUID(); 22 | static Properties prop = null; 23 | static boolean testsEnabled = true; 24 | 25 | @BeforeClass 26 | public static void setup() throws IOException { 27 | prop = HelperUtils.getProperties(); 28 | testsEnabled = Boolean.parseBoolean(prop.getProperty("acl.AclSpecTestsEnabled", "true")); 29 | } 30 | 31 | private void compareToCanonical(String input, String expectedCanonical) { 32 | String actualCanonical = AclEntry.aclListToString(AclEntry.parseAclSpec(input)); //.toString(); 33 | assertTrue("failed " + input + "/" + expectedCanonical + "/" + actualCanonical, 34 | actualCanonical.equals(expectedCanonical)); 35 | } 36 | 37 | @Test 38 | public void parseTests() { 39 | compareToCanonical("user:hello:rwx", "user:hello:rwx"); 40 | compareToCanonical("user::rwx ,default:mask::r-- ,mask: :-wx", "user::rwx,default:mask::r--,mask::-wx"); 41 | compareToCanonical("group:a:rwx, user:b:r--, default:other::--x ", "group:a:rwx,user:b:r--,default:other::--x"); 42 | 43 | compareToCanonical("user:bob:rwx,", "user:bob:rwx"); 44 | compareToCanonical(",user:bob:rwx", "user:bob:rwx"); 45 | compareToCanonical("user:bob:rwx,, ,,group:sales:r--", "user:bob:rwx,group:sales:r--"); 46 | compareToCanonical(" ", ""); 47 | } 48 | 49 | @Test(expected = IllegalArgumentException.class) 50 | public void invalidAclSpec1() { 51 | AclEntry.parseAclSpec("user:bob:rwx,user:hello"); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ReadBufferWorker.java: -------------------------------------------------------------------------------- 1 | package com.microsoft.azure.datalake.store; 2 | 3 | 4 | import com.microsoft.azure.datalake.store.ReadBuffer; 5 | import com.microsoft.azure.datalake.store.ReadBufferManager; 6 | import com.microsoft.azure.datalake.store.ReadBufferStatus; 7 | 8 | import java.util.concurrent.CountDownLatch; 9 | 10 | /** 11 | * Internal use only - do not use. 12 | * The method running in the worker threads. 13 | * 14 | * 15 | */ 16 | class ReadBufferWorker implements Runnable { 17 | 18 | static final CountDownLatch unleashWorkers = new CountDownLatch(1); 19 | private int id; 20 | 21 | ReadBufferWorker(int id) { 22 | this.id = id; 23 | } 24 | 25 | /** 26 | * Waits until a buffer becomes available in ReadAheadQueue. 27 | * Once a buffer becomes available, reads the file specified in it and then posts results back to buffer manager. 28 | * Rinse and repeat. Forever. 29 | */ 30 | public void run() { 31 | try { 32 | unleashWorkers.await(); 33 | } catch (InterruptedException ex) { 34 | Thread.currentThread().interrupt(); 35 | } 36 | ReadBufferManager bufferManager = ReadBufferManager.getBufferManager(); 37 | ReadBuffer buffer; 38 | while (true) { 39 | try { 40 | buffer = bufferManager.getNextBlockToRead(); // blocks, until a buffer is available for this thread 41 | } catch (InterruptedException ex) { 42 | Thread.currentThread().interrupt(); 43 | return; 44 | } 45 | if (buffer != null) { 46 | try { 47 | // do the actual read, from the file. 48 | int bytesRead = buffer.file.readRemote(buffer.offset, buffer.buffer, 0, buffer.requestedLength, true); 49 | bufferManager.doneReading(buffer, ReadBufferStatus.AVAILABLE, bytesRead); // post result back to ReadBufferManager 50 | } catch (Exception ex) { 51 | bufferManager.doneReading(buffer, ReadBufferStatus.READ_FAILED, 0); 52 | } 53 | } 54 | } 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ADLException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | import java.io.IOException; 11 | 12 | /** 13 | * Exception type returned by Azure Data Lake SDK methods. Derives from {@link IOException}. 14 | * Contains a number of additional fields that contain information about 15 | * the success or failure of a server call. 16 | */ 17 | public class ADLException extends IOException { 18 | 19 | /** 20 | * the HTTP response code returned by the server 21 | */ 22 | public int httpResponseCode; 23 | 24 | /** 25 | * The HTTP response message 26 | */ 27 | public String httpResponseMessage; 28 | 29 | /** 30 | * The Server request ID 31 | */ 32 | public String requestId = null; 33 | 34 | /** 35 | * the number of retries attempted before the call failed 36 | */ 37 | public int numRetries; 38 | 39 | /** 40 | * the latency of the call. If retries were made, then this contains the latency of the 41 | * last retry. 42 | */ 43 | public long lastCallLatency = 0; 44 | 45 | /** 46 | * The content length of the response, if the response contained one. 47 | * It can be zero if the response conatined no body, or if the body was sent using 48 | * chunked transfer encoding. 49 | * 50 | */ 51 | public long responseContentLength = 0; 52 | 53 | /** 54 | * The remote exception name returned by the server in an HTTP error message. 55 | */ 56 | public String remoteExceptionName = null; 57 | 58 | /** 59 | * The remote exception message returned by the server in an HTTP error message. 60 | */ 61 | public String remoteExceptionMessage = null; 62 | 63 | /** 64 | * The remote exception's java class name returned by the server in an HTTP error message. 65 | */ 66 | public String remoteExceptionJavaClassName = null; 67 | 68 | public ADLException(String message) { 69 | super(message); 70 | } 71 | 72 | public ADLException(String message, Throwable initCause) { 73 | super(message, initCause); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/QueryParams.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import java.io.UnsupportedEncodingException; 10 | import java.net.URLEncoder; 11 | import java.util.Hashtable; 12 | import java.util.Map; 13 | 14 | 15 | /** 16 | * Internal class for SDK's internal use. DO NOT USE. 17 | */ 18 | public class QueryParams { 19 | 20 | private Hashtable params = new Hashtable(); 21 | Operation op = null; 22 | String apiVersion = null; 23 | String separator = ""; 24 | String serializedString = null; 25 | 26 | public void add(String name, String value) { 27 | params.put(name, value); 28 | serializedString = null; 29 | } 30 | 31 | public void setOp(Operation op) { 32 | this.op = op; 33 | serializedString = null; 34 | } 35 | 36 | public void setApiVersion(String apiVersion) { this.apiVersion = apiVersion; serializedString = null; } 37 | 38 | public String serialize() { 39 | if (serializedString == null) { 40 | StringBuilder sb = new StringBuilder(); 41 | 42 | if (op != null) { 43 | sb.append(separator); 44 | sb.append("op="); 45 | sb.append(op.name); 46 | separator = "&"; 47 | } 48 | 49 | for (String name : params.keySet()) { 50 | try { 51 | sb.append(separator); 52 | sb.append(URLEncoder.encode(name, "UTF-8")); 53 | sb.append('='); 54 | sb.append(URLEncoder.encode(params.get(name), "UTF-8")); 55 | separator = "&"; 56 | } catch (UnsupportedEncodingException ex) { 57 | } 58 | } 59 | 60 | if (apiVersion != null) { 61 | sb.append(separator); 62 | sb.append("api-version="); 63 | sb.append(apiVersion); 64 | separator = "&"; 65 | } 66 | serializedString = sb.toString(); 67 | } 68 | return serializedString; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/NonIdempotentRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.retrypolicies; 8 | 9 | /** 10 | * No retry ever. Always returns false, indicating that erequest should not be retried. 11 | * 12 | * This should be used when retrying is not safe, and user wants at-most-once semantics with the call. This is 13 | * useful for non-idempotent methods, where the error returned by the last call does not conclusively indicate 14 | * success or failure of the call. For example, if an append times out but succeeds on the back-end , then 15 | * retrying it may append the data twice to the file. 16 | * 17 | */ 18 | public class NonIdempotentRetryPolicy implements RetryPolicy { 19 | 20 | private int retryCount401 = 0; 21 | private int waitInterval = 100; 22 | 23 | private int retryCount429 = 0; 24 | private int maxRetries = 4; 25 | private int exponentialRetryInterval = 1000; 26 | private int exponentialFactor = 4; 27 | 28 | 29 | public boolean shouldRetry(int httpResponseCode, Exception lastException) { 30 | if (httpResponseCode == 401 && retryCount401 == 0) { 31 | // this could be because of call delay. Just retry once, in hope of token being renewed by now 32 | wait(waitInterval); 33 | retryCount401++; 34 | return true; 35 | } 36 | 37 | if (httpResponseCode == 429) { 38 | // 429 means that the backend did not change any state. 39 | if (retryCount429 < maxRetries) { 40 | wait(exponentialRetryInterval); 41 | exponentialRetryInterval *= exponentialFactor; 42 | retryCount429++; 43 | return true; 44 | } else { 45 | return false; // max # of retries exhausted 46 | } 47 | } 48 | 49 | return false; 50 | } 51 | 52 | private void wait(int milliseconds) { 53 | try { 54 | Thread.sleep(milliseconds); 55 | } catch (InterruptedException ex) { 56 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 57 | } 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/RefreshTokenBasedTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | 14 | /** 15 | * Provides tokens based on refresh token 16 | */ 17 | public class RefreshTokenBasedTokenProvider extends AccessTokenProvider { 18 | 19 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.RefreshTokenBasedTokenProvider"); 20 | private final String clientId, refreshToken; 21 | 22 | /** 23 | * constructs a token provider based on the refresh token provided 24 | * 25 | * @param refreshToken the refresh token 26 | */ 27 | public RefreshTokenBasedTokenProvider(String refreshToken) { 28 | this.clientId = null; 29 | this.refreshToken = refreshToken; 30 | } 31 | 32 | /** 33 | * constructs a token provider based on the refresh token provided 34 | * 35 | * @param clientId the client ID (GUID) of the client web app obtained from Azure Active Directory configuration 36 | * @param refreshToken the refresh token 37 | */ 38 | public RefreshTokenBasedTokenProvider(String clientId, String refreshToken) { 39 | this.clientId = clientId; 40 | this.refreshToken = refreshToken; 41 | } 42 | 43 | /** 44 | * constructs a token provider based on the refresh token provided 45 | * 46 | * @param clientId the client ID (GUID) of the client web app obtained from Azure Active Directory configuration 47 | * @param refreshToken the refresh token 48 | */ 49 | public RefreshTokenBasedTokenProvider(String clientId, RefreshTokenInfo refreshToken) { 50 | this.clientId = clientId; 51 | this.refreshToken = refreshToken.refreshToken; 52 | if (refreshToken.accessToken != null && 53 | !refreshToken.accessToken.equals("") && 54 | refreshToken.accessTokenExpiry != null) { 55 | this.token = new AzureADToken(); 56 | this.token.accessToken = refreshToken.accessToken; 57 | this.token.expiry = refreshToken.accessTokenExpiry; 58 | } 59 | } 60 | 61 | @Override 62 | protected AzureADToken refreshToken() throws IOException { 63 | log.debug("AADToken: refreshing refresh-token based token"); 64 | return AzureADAuthenticator.getTokenUsingRefreshToken(clientId, refreshToken); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/DeviceCodeTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | 14 | /** 15 | * 16 | * Enables interactive login in non-browser based contexts. Displays a message asking the user to login using 17 | * the browser, using the provided link and code. 18 | * 19 | * 20 | */ 21 | public class DeviceCodeTokenProvider extends AccessTokenProvider { 22 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.DeviceCodeTokenProvider"); 23 | private RefreshTokenBasedTokenProvider tokenProviderInternal = null; 24 | private String refreshTokenString = null; 25 | 26 | /** 27 | * Prompts user to log in and constructs a tokenProvider based on the refresh token obtained from the login. 28 | * 29 | * @param appId the app ID whose name Azure AD will display on the login screen 30 | * @throws IOException in case of errors 31 | */ 32 | 33 | public DeviceCodeTokenProvider(String appId) throws IOException { 34 | this(appId, null); 35 | } 36 | 37 | /** 38 | * Prompts user to log in and constructs a tokenProvider based on the refresh token obtained from the login. 39 | * 40 | * @param appId the app ID whose name Azure AD will display on the login screen. Can be null. 41 | * If not provided, a default appId will be used. 42 | * @param callback callback that can display the message to the user on how to login. Can be null. 43 | * If not provided, the default callback will be used which diplays the message on standard output. 44 | * @throws IOException in case of errors 45 | */ 46 | public DeviceCodeTokenProvider(String appId, DeviceCodeCallback callback) throws IOException { 47 | if (appId == null || appId.trim().length() == 0) throw new IllegalArgumentException("appId is required"); 48 | if (callback == null) callback = DeviceCodeCallback.getDefaultInstance(); 49 | 50 | RefreshTokenInfo token = DeviceCodeTokenProviderHelper.getRefreshToken(appId, callback); 51 | refreshTokenString = token.refreshToken; 52 | tokenProviderInternal = new RefreshTokenBasedTokenProvider(null, token); 53 | } 54 | 55 | @Override 56 | protected AzureADToken refreshToken() throws IOException { 57 | return tokenProviderInternal.refreshToken(); 58 | } 59 | 60 | public String getRefreshToken() { 61 | return refreshTokenString; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Security 4 | 5 | Microsoft takes the security of our software products and services seriously, which includes all source code repositories managed through our GitHub organizations, which include [Microsoft](https://github.com/microsoft), [Azure](https://github.com/Azure), [DotNet](https://github.com/dotnet), [AspNet](https://github.com/aspnet), [Xamarin](https://github.com/xamarin), and [our GitHub organizations](https://opensource.microsoft.com/). 6 | 7 | If you believe you have found a security vulnerability in any Microsoft-owned repository that meets [Microsoft's definition of a security vulnerability](https://aka.ms/opensource/security/definition), please report it to us as described below. 8 | 9 | ## Reporting Security Issues 10 | 11 | **Please do not report security vulnerabilities through public GitHub issues.** 12 | 13 | Instead, please report them to the Microsoft Security Response Center (MSRC) at [https://msrc.microsoft.com/create-report](https://aka.ms/opensource/security/create-report). 14 | 15 | If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). 16 | 17 | You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). 18 | 19 | Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: 20 | 21 | * Type of issue (e.g. buffer overflow, SQL injection, cross-site scripting, etc.) 22 | * Full paths of source file(s) related to the manifestation of the issue 23 | * The location of the affected source code (tag/branch/commit or direct URL) 24 | * Any special configuration required to reproduce the issue 25 | * Step-by-step instructions to reproduce the issue 26 | * Proof-of-concept or exploit code (if possible) 27 | * Impact of the issue, including how an attacker might exploit the issue 28 | 29 | This information will help us triage your report more quickly. 30 | 31 | If you are reporting for a bug bounty, more complete reports can contribute to a higher bounty award. Please visit our [Microsoft Bug Bounty Program](https://aka.ms/opensource/security/bounty) page for more details about our active programs. 32 | 33 | ## Preferred Languages 34 | 35 | We prefer all communications to be in English. 36 | 37 | ## Policy 38 | 39 | Microsoft follows the principle of [Coordinated Vulnerability Disclosure](https://aka.ms/opensource/security/cvd). 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/acl/TestAclEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.acl; 8 | 9 | import com.contoso.helpers.HelperUtils; 10 | import com.microsoft.azure.datalake.store.acl.AclEntry; 11 | import org.junit.BeforeClass; 12 | import org.junit.Test; 13 | import static org.junit.Assert.*; 14 | 15 | import java.io.IOException; 16 | import java.util.Properties; 17 | import java.util.UUID; 18 | 19 | public class TestAclEntry { 20 | final UUID instanceGuid = UUID.randomUUID(); 21 | static Properties prop = null; 22 | static boolean testsEnabled = true; 23 | 24 | @BeforeClass 25 | public static void setup() throws IOException { 26 | prop = HelperUtils.getProperties(); 27 | testsEnabled = Boolean.parseBoolean(prop.getProperty("acl.AclEntryTestsEnabled", "true")); 28 | } 29 | 30 | @Test 31 | public void parseTests() { 32 | compareToCanonical("user:hello:rwx", "user:hello:rwx"); 33 | compareToCanonical("user::rwx ", "user::rwx"); 34 | compareToCanonical("group:AA1-hdhg-hngDjdfh-23928:rwx", "group:AA1-hdhg-hngDjdfh-23928:rwx"); 35 | compareToCanonical("group::rwx ", "group::rwx"); 36 | compareToCanonical("mask:: RwX", "mask::rwx"); 37 | 38 | compareToCanonical("default:user:hello:rwx", "default:user:hello:rwx"); 39 | compareToCanonical("default:user ::--- ", "default:user::---"); 40 | compareToCanonical("default: group: AA1-hdhg-hngDjdfh-23928:rwx", "default:group:AA1-hdhg-hngDjdfh-23928:rwx"); 41 | compareToCanonical("default:group :: R-X", "default:group::r-x"); 42 | compareToCanonical("default:mask:: RwX", "default:mask::rwx"); 43 | } 44 | 45 | private void compareToCanonical(String input, String expectedCanonical) { 46 | String actualCanonical = AclEntry.parseAclEntry(input).toString(); 47 | assertTrue("failed " + input + "/" + expectedCanonical + "/" + actualCanonical, 48 | actualCanonical.equals(expectedCanonical)); 49 | } 50 | 51 | @Test(expected = IllegalArgumentException.class) 52 | public void invalidAclEntry1() { 53 | AclEntry.parseAclEntry("user:hello", false); 54 | } 55 | 56 | @Test(expected = IllegalArgumentException.class) 57 | public void invalidAclEntry2() { 58 | AclEntry.parseAclEntry("user:hello:rwx:h"); 59 | } 60 | 61 | @Test(expected = IllegalArgumentException.class) 62 | public void invalidAclEntry3() { 63 | AclEntry.parseAclEntry("user:hello:rwwx"); 64 | } 65 | 66 | @Test(expected = IllegalArgumentException.class) 67 | public void invalidAclEntry4() { 68 | AclEntry.parseAclEntry("default:mask:hello:rwx"); 69 | } 70 | 71 | @Test(expected = IllegalArgumentException.class) 72 | public void invalidAclEntry5() { 73 | AclEntry.parseAclEntry("default::hello:rwx"); 74 | } 75 | 76 | @Test(expected = IllegalArgumentException.class) 77 | public void invalidAclEntry6() { 78 | AclEntry.parseAclEntry(":user:hello:rwx"); 79 | } 80 | 81 | @Test(expected = IllegalArgumentException.class) 82 | public void invalidAclEntry7() { 83 | AclEntry.parseAclEntry("other:hello:rwx"); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/AccessTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | import java.util.Date; 14 | 15 | /** 16 | * Returns an Azure Active Directory token when requested. The provider can cache the token if it has already 17 | * retrieved one. If it does, then the provider is responsible for checking expiry and refreshing as needed. 18 | *

19 | * In other words, this is is a token cache that fetches tokens when requested, if the cached token has expired. 20 | *

21 | */ 22 | public abstract class AccessTokenProvider { 23 | 24 | protected AzureADToken token; 25 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.AccessTokenProvider"); 26 | 27 | /** 28 | * returns the {@link AzureADToken} cached (or retrieved) by this instance. 29 | * 30 | * @return {@link AzureADToken} containing the access token 31 | * @throws IOException if there is an error fetching the token 32 | */ 33 | public synchronized AzureADToken getToken() throws IOException { 34 | if (isTokenAboutToExpire()) { 35 | log.debug("AAD Token is missing or expired: Calling refresh-token from abstract base class"); 36 | token = refreshToken(); 37 | } 38 | return token; 39 | } 40 | 41 | /** 42 | * the method to fetch the access token. Derived classes should override this method to 43 | * actually get the token from Azure Active Directory. 44 | *

45 | * This method will be called initially, and then once when the token is about to expire. 46 | *

47 | * 48 | * 49 | * @return {@link AzureADToken} containing the access token 50 | * @throws IOException if there is an error fetching the token 51 | */ 52 | protected abstract AzureADToken refreshToken() throws IOException; 53 | 54 | /** 55 | * Checks if the token is about to expire in the next 5 minutes. The 5 minute allowance is to 56 | * allow for clock skew and also to allow for token to be refreshed in that much time. 57 | * 58 | * 59 | * @return true if the token is expiring in next 5 minutes 60 | */ 61 | protected boolean isTokenAboutToExpire() { 62 | if (token==null) { 63 | log.debug("AADToken: no token. Returning expiring=true"); 64 | return true; // no token should have same response as expired token 65 | } 66 | if (token.expiry == null) { 67 | log.debug("AADToken: no token expiry set. Returning expiring=true"); 68 | return true; // if don't know expiry then assume expired (should not happen with a correctly implemented token) 69 | } 70 | long offset = FIVE_MINUTES; 71 | long approximatelyNow = System.currentTimeMillis() + offset; // allow x minutes for clock skew, depends on type of provider 72 | boolean expiring = (token.expiry.getTime() < approximatelyNow); 73 | if (expiring) { 74 | log.debug("AADToken: token expiring: " + token.expiry.toString() + " : " + offset + " milliseconds window: " + new Date(approximatelyNow).toString()); 75 | } 76 | 77 | return expiring; 78 | } 79 | private static final long FIVE_MINUTES = 300 * 1000; // 5 minutes in milliseconds 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/AclAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.acl; 8 | 9 | /** 10 | * Specifies the possible combinations of actions allowed in an ACL. 11 | * 12 | */ 13 | public enum AclAction { 14 | NONE ("---"), 15 | EXECUTE ("--x"), 16 | WRITE ("-w-"), 17 | WRITE_EXECUTE ("-wx"), 18 | READ ("r--"), 19 | READ_EXECUTE ("r-x"), 20 | READ_WRITE ("rw-"), 21 | ALL ("rwx"); 22 | 23 | private final String rwx; 24 | private static final AclAction[] values = AclAction.values(); 25 | 26 | AclAction(String rwx) { 27 | this.rwx = rwx; 28 | } 29 | 30 | /** 31 | * returns the Unix rwx string representation of the {@code AclAction} 32 | * @return string representation of the {@code AclAction} 33 | */ 34 | public String toString() { 35 | return this.rwx; 36 | } 37 | 38 | /** 39 | * static method that returns the Unix rwx string representation of the supplied {@code AclAction} 40 | * @param action the {@code AclAction} enum value to convert to string 41 | * 42 | * @return string representation of the {@code AclAction} 43 | */ 44 | public static String toString(AclAction action) { 45 | return action.rwx; 46 | } 47 | 48 | /** 49 | * Returns an {@code AclAction} enum value represented by the supplied Unix rwx permission string 50 | * @param rwx the string containing the unix permission in rwx form 51 | * 52 | * @return The {@code AclAction} enum value corresponding to the string 53 | */ 54 | public static AclAction fromRwx(String rwx) { 55 | if (rwx==null) throw new IllegalArgumentException("access specifier is null"); 56 | rwx = rwx.trim().toLowerCase(); 57 | for (AclAction a: values) { 58 | if (a.rwx.equals(rwx)) { return a; } 59 | } 60 | throw new IllegalArgumentException(rwx + " is not a valid access specifier"); 61 | } 62 | 63 | /** 64 | * Checks to see if the supplied string is a valid unix rwx permission string 65 | * @param input the string to check 66 | * 67 | * @return true if the string is a valid rwx permission string, false otherwise 68 | */ 69 | public static boolean isValidRwx(String input) { 70 | try { 71 | fromRwx(input); 72 | return true; 73 | } catch (IllegalArgumentException ex) { 74 | return false; 75 | } 76 | } 77 | 78 | /** 79 | * Returns an {@code AclAction} enum value represented by the supplied Octal digit 80 | * @param perm the octal digit representing the permission 81 | * 82 | * @return The {@code AclAction} enum value corresponding to the octal digit 83 | */ 84 | public static AclAction fromOctal(int perm) { 85 | if (perm <0 || perm>7) throw new IllegalArgumentException(perm + " is not a valid access specifier"); 86 | return values[perm]; 87 | } 88 | 89 | /** 90 | * returns the octal representation of the {@code AclAction} 91 | * @return octal representation of the {@code AclAction} 92 | */ 93 | public int toOctal() { 94 | return this.ordinal(); 95 | } 96 | 97 | /** 98 | * static method that returns the octal representation of the supplied {@code AclAction} 99 | * @param action the {@code AclAction} enum value to convert to octal 100 | * 101 | * @return octal representation of the {@code AclAction} 102 | */ 103 | public static int toOctal(AclAction action) { 104 | return action.ordinal(); 105 | } 106 | 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/MsiTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | import java.io.IOException; 13 | 14 | /** 15 | * Provides tokens based on Azure VM's Managed Service Identity 16 | */ 17 | public class MsiTokenProvider extends AccessTokenProvider { 18 | 19 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.MsiTokenProvider"); 20 | private final int localPort = -1; 21 | private final String tenantGuid; 22 | private final String clientId; 23 | private long tokenFetchTime =-1; 24 | /** 25 | * Constructs a token provider that fetches tokens from the MSI token-service running on an Azure IaaS VM. This 26 | * only works on an Azure VM with the MSI extansion enabled. 27 | */ 28 | public MsiTokenProvider() { 29 | this(null, null); 30 | } 31 | 32 | /** 33 | * Constructs a token provider that fetches tokens from the MSI token-service running on an Azure IaaS VM. This 34 | * only works on an Azure VM with the MSI extansion enabled. 35 | * 36 | * @deprecated localPort is not relevant anymore in the new MSI mechanism 37 | * 38 | * @param localPort port on localhost for the MSI token service. (the port that was set in the deployment template). 39 | * If 0 or negative number is specified, then assume default port number of 50342. 40 | */ 41 | @Deprecated 42 | public MsiTokenProvider(int localPort) { 43 | this(null, null); 44 | } 45 | 46 | /** 47 | * Constructs a token provider that fetches tokens from the MSI token-service running on an Azure IaaS VM. This 48 | * only works on an Azure VM with the MSI extansion enabled. 49 | * 50 | * @deprecated localPort is not relevant anymore in the new MSI mechanism 51 | * 52 | * @param localPort port on localhost for the MSI token service. (the port that was set in the deployment template). 53 | * If 0 or negative number is specified, then assume default port number of 50342. 54 | * @param tenantGuid (optional) AAD Tenant ID {@code guid}. Can be {@code null}. 55 | */ 56 | @Deprecated 57 | public MsiTokenProvider(int localPort, String tenantGuid) { 58 | this(tenantGuid, null); 59 | } 60 | 61 | public MsiTokenProvider(String tenantGuid, String clientId) { 62 | this.tenantGuid = tenantGuid; 63 | this.clientId = clientId; 64 | } 65 | 66 | /** 67 | * Checks if the token is about to expire as per base expiry logic. Otherwise try to expire every 1 hour 68 | * 69 | * 70 | * @return true if the token is expiring in next 5 minutes 71 | */ 72 | @Override 73 | protected boolean isTokenAboutToExpire() { 74 | if( super.isTokenAboutToExpire()){ 75 | return true; 76 | } 77 | if (tokenFetchTime == -1){ 78 | return true; 79 | } 80 | long offset = ONE_HOUR; 81 | if ((tokenFetchTime +offset) < System.currentTimeMillis()) { 82 | log.debug("MSIToken: token renewing : " + offset + " milliseconds window"); 83 | return true; 84 | } 85 | 86 | return false; 87 | } 88 | 89 | @Override 90 | protected AzureADToken refreshToken() throws IOException { 91 | log.debug("AADToken: refreshing token from MSI with expiry"); 92 | AzureADToken newToken = AzureADAuthenticator.getTokenFromMsi(tenantGuid, clientId, false); 93 | tokenFetchTime=System.currentTimeMillis(); 94 | return newToken; 95 | } 96 | private static final long ONE_HOUR = 3600 * 1000; // 5 minutes in milliseconds 97 | } -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ProcessingQueue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | import java.util.LinkedList; 11 | import java.util.Queue; 12 | 13 | /* 14 | ProcessingQueue keeps track of directories queued to process. 15 | 16 | Desired behavior: 17 | 1. Caller can enqueue directories to process that will be picked up by an open thread (add() method) 18 | 2. Caller can dequeue directories when it is ready to process (poll() method) 19 | 3. If the queue is empty, then caller blocks until an item becomes available in the queue (behavior of poll()) 20 | These three properties above can provide required behavior during runtime. However, we need something 21 | more to detect completion - queue being empty is not good enough, since queue can also be empty 22 | during run when all directories queued have been picked up by some thread or another, and they may queue 23 | more as they find out about more items. So the termination condition is really that "queue is empty *and* 24 | no threads are processing any items". So we have to also keep track of items being processed. So additional 25 | requirements: 26 | 4. Caller should indicate when it is done processing an item it popped from the queue (unregister() method) 27 | 5. Caller should indicate when it has started processing an item. (that is implicit here, since 28 | dequeueing an item automatically implies start of processing) 29 | 6. Caller processing the last item initiates completion, as per termination condition above (done in unregister()) 30 | 7. poll() can return null if processing is complete, but not otherwise 31 | 32 | 33 | Notes: 34 | The processing threads spend their time doing I/O most of the time. If there isnt enough work, they 35 | spend time blocked on the semaphore in poll(). 36 | 37 | At the end, all but one threads block on poll(), only the last thread out detects completion - i.e., the thread 38 | that was processing the last item. It is this thread's job to now wake up all the other threads from their 39 | blocking state so they can complete. 40 | */ 41 | 42 | 43 | /** 44 | * Internal class, used to coordinate among the multiple threads doing recursive directory traversal 45 | * for getContentSummary 46 | * 47 | * @param The type of items in the queue 48 | */ 49 | class ProcessingQueue { 50 | private Queue internalQueue = new LinkedList<>(); 51 | private int processorCount = 0; 52 | 53 | public synchronized void add(T item) { 54 | if (item == null) throw new IllegalArgumentException("Cannot put null into queue"); 55 | internalQueue.add(item); 56 | this.notifyAll(); 57 | } 58 | 59 | public synchronized T poll() { 60 | try { 61 | while (isQueueEmpty() && !done()) 62 | this.wait(); 63 | if (!isQueueEmpty()) { 64 | processorCount++; // current thread is now processing the item we pop 65 | return internalQueue.poll(); 66 | } 67 | if (done()) { 68 | return null; 69 | } 70 | } catch (InterruptedException ex) { 71 | Thread.currentThread().interrupt(); 72 | } 73 | return null; // just to keep the compiler happy - it couldn't infer that all code-paths are covered above. 74 | } 75 | 76 | public synchronized void unregister() { 77 | processorCount--; 78 | if (processorCount < 0) { 79 | throw new IllegalStateException("too many unregister()'s. processorCount is now " + processorCount); 80 | } 81 | if (done()) this.notifyAll(); 82 | } 83 | 84 | private boolean done() { 85 | return (processorCount == 0 && isQueueEmpty()); 86 | } 87 | 88 | private boolean isQueueEmpty() { 89 | return (internalQueue.peek() == null); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/helpers/HelperUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.helpers; 8 | 9 | 10 | import com.microsoft.azure.datalake.store.*; 11 | import com.microsoft.azure.datalake.store.retrypolicies.ExponentialBackoffPolicy; 12 | 13 | import java.io.*; 14 | import java.security.SecureRandom; 15 | import java.util.Properties; 16 | 17 | public class HelperUtils { 18 | 19 | private static Properties prop = null; 20 | public static Properties getProperties() throws IOException { 21 | if (prop==null) { 22 | Properties defaultProps = new Properties(); 23 | defaultProps.load(new FileInputStream("target/test-classes/config.properties")); 24 | prop = new Properties(defaultProps); 25 | prop.load(new FileInputStream("target/test-classes/creds.properties")); 26 | } 27 | return prop; 28 | } 29 | 30 | private static byte[] buf4mb = null; 31 | public static byte[] getRandom4mbBuffer() { 32 | if (buf4mb == null) { 33 | SecureRandom prng = new SecureRandom(); 34 | buf4mb = new byte[4 * 1024 * 1024]; 35 | prng.nextBytes(buf4mb); 36 | } 37 | return buf4mb; 38 | } 39 | 40 | public static void getRandomBytes(byte[] buf) { 41 | SecureRandom prng = new SecureRandom(); 42 | prng.nextBytes(buf); 43 | } 44 | 45 | public static byte[] getRandomBuffer(int len) { 46 | SecureRandom prng = new SecureRandom(); 47 | byte[] b = new byte[len]; 48 | prng.nextBytes(b); 49 | return b; 50 | } 51 | 52 | public static void createEmptyFile(ADLStoreClient client, String filename) throws IOException { 53 | RequestOptions opts = new RequestOptions(); 54 | opts.retryPolicy = new ExponentialBackoffPolicy(); 55 | OperationResponse resp = new OperationResponse(); 56 | Core.create(filename, true, null, null, 0, 0, null, 57 | null, true, SyncFlag.CLOSE, client, opts, resp); 58 | if (!resp.successful) { 59 | throw client.getExceptionFromResponse(resp, "Error creating file " + filename); 60 | } 61 | } 62 | 63 | public static void createRandomContentFile(ADLStoreClient client, String filename, int contentLength) throws IOException { 64 | OutputStream ostr = client.createFile(filename, IfExists.OVERWRITE); 65 | ostr.write(getRandomBuffer(contentLength)); 66 | ostr.close(); 67 | } 68 | 69 | public static void create256BFile(ADLStoreClient client, String filename) throws IOException { 70 | createRandomContentFile(client, filename, 256); 71 | } 72 | 73 | public static byte[] getSampleText1() { 74 | ByteArrayOutputStream b = new ByteArrayOutputStream(1024); 75 | PrintStream out = new PrintStream(b); 76 | try { 77 | for (int i = 1; i <= 10; i++) { 78 | out.println("This is line #" + i); 79 | out.format("This is the same line (%d), but using formatted output. %n", i); 80 | } 81 | out.close(); 82 | } catch (Exception ex) { 83 | System.out.println(ex.getMessage()); 84 | } 85 | return b.toByteArray(); 86 | // length of returned array is 742 bytes 87 | } 88 | 89 | public static byte[] getSampleText2() { 90 | ByteArrayOutputStream s = new ByteArrayOutputStream(); 91 | PrintStream out = new PrintStream(s); 92 | out.println("This is a line"); 93 | out.println("This is another line"); 94 | out.println("This is yet another line"); 95 | out.println("This is yet yet another line"); 96 | out.println("This is yet yet yet another line"); 97 | out.println("... and so on, ad infinitum"); 98 | out.println(); 99 | out.close(); 100 | byte[] buf = s.toByteArray(); 101 | return buf; 102 | // length of returned array is 159 bytes 103 | } 104 | 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/ExponentialBackoffPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.retrypolicies; 8 | 9 | /** 10 | * implements different retry decisions based on the error. 11 | * 12 | *
    13 | *
  • For nonretryable errors (3xx, most 4xx, and some 5xx return codes), do no retry.
    • 14 | *
  • For throttling error, do a retry with exponential backoff
    • 15 | *
  • for all other errors, do a retry with linear backoff
    • 16 | *
17 | */ 18 | public class ExponentialBackoffPolicy implements RetryPolicy { 19 | 20 | private int retryCount = 0; 21 | private int maxRetries = 4; 22 | private int exponentialRetryInterval = 1000; 23 | private int exponentialFactor = 4; 24 | private long lastAttemptStartTime = System.nanoTime(); 25 | 26 | public ExponentialBackoffPolicy() { 27 | } 28 | 29 | /** 30 | * @param maxRetries maximum number of retries 31 | * @param linearRetryInterval interval to use for linear retries (in milliseconds). 32 | * Deprecated, not used in the retry policy. 33 | * @param exponentialRetryInterval (starting) interval to use for exponential backoff retries (in milliseconds) 34 | */ 35 | public ExponentialBackoffPolicy(int maxRetries, @Deprecated int linearRetryInterval, int exponentialRetryInterval) { 36 | this.maxRetries = maxRetries; 37 | this.exponentialRetryInterval = exponentialRetryInterval; 38 | } 39 | 40 | public ExponentialBackoffPolicy(int maxRetries, @Deprecated int linearRetryInterval, int exponentialRetryInterval, int exponentialFactor) { 41 | this.maxRetries = maxRetries; 42 | this.exponentialRetryInterval = exponentialRetryInterval; 43 | this.exponentialFactor = exponentialFactor; 44 | } 45 | 46 | public boolean shouldRetry(int httpResponseCode, Exception lastException) { 47 | 48 | // Non-retryable error 49 | if ( (httpResponseCode >= 300 && httpResponseCode < 500 // 3xx and 4xx, except specific ones below 50 | && httpResponseCode != 408 51 | && httpResponseCode != 429 52 | && httpResponseCode != 401) 53 | || (httpResponseCode == 501) // Not Implemented 54 | || (httpResponseCode == 505) // Version Not Supported 55 | ) { 56 | return false; 57 | } 58 | 59 | // Retryable error, retry with exponential backoff 60 | if ( lastException!=null || httpResponseCode >=500 // exception or 5xx, + specific ones below 61 | || httpResponseCode == 408 62 | || httpResponseCode == 429 63 | || httpResponseCode == 401) { 64 | if (retryCount < maxRetries) { 65 | int timeSpent = (int)((System.nanoTime() - lastAttemptStartTime) / 1000000); 66 | wait(exponentialRetryInterval - timeSpent); 67 | exponentialRetryInterval *= exponentialFactor; 68 | retryCount++; 69 | lastAttemptStartTime = System.nanoTime(); 70 | return true; 71 | } else { 72 | return false; // max # of retries exhausted 73 | } 74 | } 75 | 76 | // these are not errors - this method should never have been called with this 77 | if (httpResponseCode >= 100 && httpResponseCode <300) 78 | { 79 | return false; 80 | } 81 | 82 | // Dont know what happened - we should never get here 83 | return false; 84 | } 85 | 86 | private void wait(int milliseconds) { 87 | if (milliseconds <= 0) { 88 | return; 89 | } 90 | 91 | try { 92 | Thread.sleep(milliseconds); 93 | } catch (InterruptedException ex) { 94 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/acl/TestAclAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.acl; 8 | 9 | import com.contoso.helpers.HelperUtils; 10 | 11 | import com.microsoft.azure.datalake.store.acl.AclAction; 12 | import org.junit.BeforeClass; 13 | import org.junit.Test; 14 | import static org.junit.Assert.*; 15 | 16 | import java.io.IOException; 17 | import java.util.Properties; 18 | import java.util.UUID; 19 | 20 | 21 | public class TestAclAction { 22 | final UUID instanceGuid = UUID.randomUUID(); 23 | static Properties prop = null; 24 | static boolean testsEnabled = true; 25 | 26 | @BeforeClass 27 | public static void setup() throws IOException { 28 | prop = HelperUtils.getProperties(); 29 | testsEnabled = Boolean.parseBoolean(prop.getProperty("acl.AclActionTestsEnabled", "true")); 30 | } 31 | 32 | @Test 33 | public void testAclActionRwx() { 34 | assertTrue(AclAction.fromRwx("---") == AclAction.NONE); 35 | assertTrue(AclAction.fromRwx("--x") == AclAction.EXECUTE); 36 | assertTrue(AclAction.fromRwx("-w-") == AclAction.WRITE); 37 | assertTrue(AclAction.fromRwx("-wx") == AclAction.WRITE_EXECUTE); 38 | assertTrue(AclAction.fromRwx("r--") == AclAction.READ); 39 | assertTrue(AclAction.fromRwx("r-x") == AclAction.READ_EXECUTE); 40 | assertTrue(AclAction.fromRwx("rw-") == AclAction.READ_WRITE); 41 | assertTrue(AclAction.fromRwx("rwx") == AclAction.ALL); 42 | 43 | assertTrue(AclAction.fromRwx("R-x") == AclAction.READ_EXECUTE); 44 | assertTrue(AclAction.fromRwx("r-x ") == AclAction.READ_EXECUTE); 45 | assertTrue(AclAction.fromRwx(" RWx") == AclAction.ALL); 46 | } 47 | 48 | @Test(expected = IllegalArgumentException.class) 49 | public void testAclActionRwxMalformed1() { 50 | AclAction.fromRwx("rws"); 51 | } 52 | 53 | @Test(expected = IllegalArgumentException.class) 54 | public void testAclActionRwxMalformed2() { 55 | AclAction.fromRwx("r wx"); 56 | } 57 | 58 | @Test(expected = IllegalArgumentException.class) 59 | public void testAclActionRwxMalformed3() { 60 | AclAction.fromRwx("rrwx"); 61 | } 62 | 63 | @Test(expected = IllegalArgumentException.class) 64 | public void testAclActionRwxMalformed4() { 65 | AclAction.fromRwx(""); 66 | } 67 | 68 | @Test(expected = IllegalArgumentException.class) 69 | public void testAclActionRwxMalformed5() { 70 | AclAction.fromRwx(null); 71 | } 72 | 73 | @Test 74 | public void testAclActionOctal() { 75 | assertTrue(AclAction.fromOctal(0) == AclAction.NONE); 76 | assertTrue(AclAction.fromOctal(1) == AclAction.EXECUTE); 77 | assertTrue(AclAction.fromOctal(2) == AclAction.WRITE); 78 | assertTrue(AclAction.fromOctal(3) == AclAction.WRITE_EXECUTE); 79 | assertTrue(AclAction.fromOctal(4) == AclAction.READ); 80 | assertTrue(AclAction.fromOctal(5) == AclAction.READ_EXECUTE); 81 | assertTrue(AclAction.fromOctal(6) == AclAction.READ_WRITE); 82 | assertTrue(AclAction.fromOctal(7) == AclAction.ALL); 83 | } 84 | 85 | @Test(expected = IllegalArgumentException.class) 86 | public void testAclActionOctalMalformed1() { 87 | AclAction.fromOctal(-1); 88 | } 89 | 90 | @Test(expected = IllegalArgumentException.class) 91 | public void testAclActionOctalMalformed2() { 92 | AclAction.fromOctal(8); 93 | } 94 | 95 | @Test 96 | public void testSerialization() { 97 | assertTrue(AclAction.NONE.toString().equals("---")); 98 | assertTrue(AclAction.EXECUTE.toString().equals("--x")); 99 | assertTrue(AclAction.WRITE.toString().equals("-w-")); 100 | assertTrue(AclAction.WRITE_EXECUTE.toString().equals("-wx")); 101 | assertTrue(AclAction.READ.toString().equals("r--")); 102 | assertTrue(AclAction.READ_EXECUTE.toString().equals("r-x")); 103 | assertTrue(AclAction.READ_WRITE.toString().equals("rw-")); 104 | assertTrue(AclAction.ALL.toString().equals("rwx")); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/retrypolicies/ExponentialBackoffPolicyforMSI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.retrypolicies; 8 | 9 | /** 10 | * implements different retry decisions based on the error. 11 | * 12 | *
    13 | *
  • For nonretryable errors (3xx, most 4xx, and some 5xx return codes), do no retry.
    • 14 | *
  • For throttling error, do a retry with exponential backoff
    • 15 | *
  • for all other errors, do a retry with linear backoff
    • 16 | *
17 | */ 18 | public class ExponentialBackoffPolicyforMSI implements RetryPolicy { 19 | 20 | private int retryCount = 0; 21 | private int maxRetries = 4; 22 | private int exponentialRetryInterval = 1000; 23 | private int exponentialFactor = 4; 24 | private long lastAttemptStartTime = System.nanoTime(); 25 | 26 | public ExponentialBackoffPolicyforMSI() { 27 | } 28 | 29 | /** 30 | * Implements Exponential backoff policy, with error condition checks specific to MSI 31 | * @param maxRetries maximum number of retries 32 | * @param exponentialRetryInterval (starting) interval to use for exponential backoff retries (in milliseconds) 33 | * @param exponentialFactor factor to multiply the retry interval by 34 | */ 35 | public ExponentialBackoffPolicyforMSI(int maxRetries, int exponentialRetryInterval, int exponentialFactor) { 36 | this.maxRetries = maxRetries; 37 | this.exponentialRetryInterval = exponentialRetryInterval; 38 | this.exponentialFactor = exponentialFactor; 39 | } 40 | 41 | 42 | public boolean shouldRetry(int httpResponseCode, Exception lastException) { 43 | 44 | // Non-retryable error 45 | if ( (httpResponseCode >= 300 && httpResponseCode < 500 // 3xx and 4xx, except specific ones below 46 | && httpResponseCode != 404 // 404 is required for MSI; for some error conditions 47 | // they return a 404 that goes away on retry 48 | && httpResponseCode != 408 49 | && httpResponseCode != 429 50 | && httpResponseCode != 401) 51 | || (httpResponseCode == 501) // Not Implemented 52 | || (httpResponseCode == 505) // Version Not Supported 53 | ) { 54 | return false; 55 | } 56 | 57 | // Retryable error, retry with exponential backoff 58 | if ( lastException!=null || httpResponseCode >=500 // exception or 5xx, + specific ones below 59 | || httpResponseCode == 404 // see comment above for 404 60 | || httpResponseCode == 408 61 | || httpResponseCode == 429 62 | || httpResponseCode == 401) { 63 | if (retryCount < maxRetries) { 64 | int timeSpent = (int)((System.nanoTime() - lastAttemptStartTime) / 1000000); 65 | wait(exponentialRetryInterval - timeSpent); 66 | exponentialRetryInterval *= exponentialFactor; 67 | retryCount++; 68 | lastAttemptStartTime = System.nanoTime(); 69 | return true; 70 | } else { 71 | return false; // max # of retries exhausted 72 | } 73 | } 74 | 75 | // these are not errors - this method should never have been called with this 76 | if (httpResponseCode >= 100 && httpResponseCode <300) 77 | { 78 | return false; 79 | } 80 | 81 | // Dont know what happened - we should never get here 82 | return false; 83 | } 84 | 85 | private void wait(int milliseconds) { 86 | if (milliseconds <= 0) { 87 | return; 88 | } 89 | 90 | try { 91 | Thread.sleep(milliseconds); 92 | } catch (InterruptedException ex) { 93 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/OperationResponse.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import java.io.InputStream; 10 | 11 | /** 12 | * information about a response from a server call. 13 | * 14 | * This class is a container for all the information from making a server call. 15 | * 16 | * 17 | */ 18 | public class OperationResponse { 19 | /** 20 | * whether the request was successful. Callers should always check for success before using any return value from 21 | * any of the calls. 22 | */ 23 | public boolean successful = true; 24 | 25 | 26 | /** 27 | * The WebHDFS opCode of the remote operation 28 | */ 29 | public String opCode = null; 30 | 31 | /** 32 | * the HTTP response code from the call 33 | */ 34 | public int httpResponseCode; 35 | 36 | /** 37 | * the message that came with the HTTP response 38 | */ 39 | public String httpResponseMessage; 40 | 41 | /** 42 | * for methods that return data from server, this field contains the 43 | * {@link com.microsoft.azure.datalake.store.ADLFileInputStream ADLFileInputStream}. {@code null} for methods that 44 | * return no data in the HTTP body. 45 | * 46 | */ 47 | public InputStream responseStream = null; 48 | 49 | /** 50 | * the server request ID. 51 | */ 52 | public String requestId = null; 53 | 54 | /** 55 | * the number of retries attempted before returning from the call 56 | */ 57 | public int numRetries; 58 | 59 | /** 60 | * the latency of the last try, in milliseconds 61 | */ 62 | public long lastCallLatency = 0; 63 | 64 | 65 | /** 66 | * time taken to get the token for this request, in nanoseconds. Should mostly be small. 67 | */ 68 | public long tokenAcquisitionLatency = 0; 69 | 70 | /** 71 | * Content-Length of the returned HTTP body (if return was not chunked). Callers should look at both this and 72 | * {@link #responseChunked} values to determine whether any data was returned by server. 73 | */ 74 | public long responseContentLength = 0; 75 | 76 | 77 | /** 78 | * indicates whether HTTP body used chunked for {@code Transfer-Encoding} of the response 79 | */ 80 | public boolean responseChunked = false; 81 | 82 | /** 83 | * the exception name as reported by the server, if the call failed on server 84 | */ 85 | public String remoteExceptionName = null; 86 | 87 | /** 88 | * the exception message as reported by the server, if the call failed on server 89 | */ 90 | public String remoteExceptionMessage = null; 91 | 92 | /** 93 | * the exception's Java Class Name as reported by the server, if the call failed on server 94 | * This is there for WebHDFS compatibility. 95 | */ 96 | public String remoteExceptionJavaClassName = null; 97 | 98 | /** 99 | * exceptions encountered when processing the request or response 100 | */ 101 | public Exception ex = null; 102 | 103 | /** 104 | * error message, used for errors that originate within the SDK 105 | */ 106 | public String message; 107 | 108 | /** 109 | * Comma-separated list of exceptions encountered but not thrown by this call. This may happen because of retries. 110 | */ 111 | public String exceptionHistory = null; 112 | 113 | /** 114 | * Reset response object to initial state 115 | */ 116 | public void reset() { 117 | this.successful = true; 118 | this.opCode = null; 119 | this.httpResponseCode = 0; 120 | this.httpResponseMessage = null; 121 | this.responseStream = null; 122 | this.requestId = null; 123 | this.numRetries = 0; 124 | this.lastCallLatency = 0; 125 | this.responseContentLength = 0; 126 | this.responseChunked = false; 127 | this.remoteExceptionName = null; 128 | this.remoteExceptionMessage = null; 129 | this.remoteExceptionJavaClassName = null; 130 | this.ex = null; 131 | this.message = null; 132 | // do not reset exceptionHistory 133 | } 134 | } 135 | 136 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/Utils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import com.microsoft.azure.datalake.store.retrypolicies.ExponentialBackoffPolicy; 10 | import com.microsoft.azure.datalake.store.retrypolicies.NonIdempotentRetryPolicy; 11 | 12 | import java.io.*; 13 | 14 | /** 15 | * Utility methods to enable one-liners for simple functionality. 16 | * 17 | * The methods are all based on calls to the SDK methods, these are 18 | * just convenience methods for common tasks. 19 | */ 20 | public class Utils { 21 | 22 | private ADLStoreClient client; 23 | 24 | Utils(ADLStoreClient client) { 25 | this.client = client; 26 | } 27 | 28 | /** 29 | * Uploads the contents of a local file to an Azure Data Lake file. 30 | * 31 | * @param filename path of file to upload to 32 | * @param localFilename path to local file 33 | * @param mode {@link IfExists} {@code enum} specifying whether to overwite or throw 34 | * an exception if the file already exists 35 | * @throws IOException thrown on error 36 | */ 37 | public void upload(String filename, String localFilename, IfExists mode) throws IOException { 38 | if (localFilename == null || localFilename.trim().equals("")) 39 | throw new IllegalArgumentException("localFilename cannot be null"); 40 | 41 | try (FileInputStream in = new FileInputStream(localFilename)){ 42 | upload(filename, in, mode); 43 | } 44 | } 45 | 46 | /** 47 | * Uploads an {@link InputStream} to an Azure Data Lake file. 48 | * 49 | * @param filename path of file to upload to 50 | * @param in {@link InputStream} whose contents will be uploaded 51 | * @param mode {@link IfExists} {@code enum} specifying whether to overwite or throw 52 | * an exception if the file already exists 53 | * @throws IOException thrown on error 54 | */ 55 | public void upload(String filename, InputStream in, IfExists mode) throws IOException { 56 | if (filename == null || filename.trim().equals("")) 57 | throw new IllegalArgumentException("filename cannot be null"); 58 | if (in == null) throw new IllegalArgumentException("InputStream cannot be null"); 59 | 60 | try (ADLFileOutputStream out = client.createFile(filename, mode)) { 61 | int bufSize = 4 * 1000 * 1000; 62 | out.setBufferSize(bufSize); 63 | byte[] buffer = new byte[bufSize]; 64 | int n; 65 | 66 | while ((n = in.read(buffer)) != -1) { 67 | out.write(buffer, 0, n); 68 | } 69 | } 70 | in.close(); 71 | } 72 | 73 | /** 74 | * Uploads the contents of byte array to an Azure Data Lake file. 75 | * 76 | * @param filename path of file to upload to 77 | * @param contents byte array containing the bytes to upload 78 | * @param mode {@link IfExists} {@code enum} specifying whether to overwite or throw 79 | * an exception if the file already exists 80 | * @throws IOException thrown on error 81 | */ 82 | public void upload(String filename, byte[] contents, IfExists mode) throws IOException { 83 | if (filename == null || filename.trim().equals("")) 84 | throw new IllegalArgumentException("filename cannot be null"); 85 | 86 | if (contents.length <= 4 * 1024 * 1024) { // if less than 4MB, then do a direct CREATE in a single operation 87 | boolean overwrite = (mode==IfExists.OVERWRITE); 88 | RequestOptions opts = new RequestOptions(); 89 | opts.retryPolicy = overwrite ? client.makeExponentialBackoffPolicy() : new NonIdempotentRetryPolicy(); 90 | OperationResponse resp = new OperationResponse(); 91 | Core.create(filename, overwrite, null, contents, 0, contents.length, null, null, true, SyncFlag.CLOSE, client, opts, resp); 92 | if (!resp.successful) { 93 | throw client.getExceptionFromResponse(resp, "Error creating file " + filename); 94 | } 95 | } else { 96 | try (ByteArrayInputStream bis = new ByteArrayInputStream(contents)){ 97 | upload(filename, bis, mode); 98 | } 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/LatencyTracker.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import java.util.concurrent.ArrayBlockingQueue; 10 | 11 | 12 | /** 13 | * {@code LatencyTracker} keeps track of client-preceived request latencies, to be reported on the next REST request. 14 | * Every request adds its result (success/failure and latency) to LatencyTracker. When a request is made, 15 | * the SDK checks LatencyTracker to see if there are any latency stats to be reported. If so, the stats are added 16 | * as an HTTP header ({@code x-ms-adl-client-latency}) on the next request. 17 | *

18 | * To disable this reporting, user can call {@link #disable()}. 19 | *

20 | * Contents of data reported back: 21 | *
    22 | *
  • Client Request ID of last request
    • 23 | *
  • latency in milliseconds
    • 24 | *
  • error code (if request failed)
    • 25 | *
  • Operation
    • 26 | *
  • Request+response body Size
    • 27 | *
  • number for AzureDataLakeStoreClient that made this call
    • 28 | *
29 | * 30 | */ 31 | public class LatencyTracker { 32 | /* 33 | Schema: 34 | Single entry, comma separated: 35 | 1. Client Request ID 36 | 2. latency in milliseconds 37 | 3. error code (if request failed) 38 | 4. Operation 39 | 5. Request+response body Size (if available, zero otherwise) 40 | 6. Instance of ADLStoreClient (a unique number per instance in this VM) 41 | 42 | Multiple entries can be on a single request. Entries will be separated by semicolons 43 | Limit max entries on a single request to three, to limit increase in HTTP request size. 44 | */ 45 | 46 | private static final ArrayBlockingQueue Q = new ArrayBlockingQueue(256); 47 | private static final int MAXPERLINE = 3; 48 | private static volatile boolean disabled = false; 49 | 50 | private LatencyTracker() {} // Prevent instantiation - static methods only 51 | 52 | /** 53 | * Disable reporting of client-perceived latency stats to the server. 54 | *

55 | * This is a static method that disables all future reporting from this JVM instance. 56 | *

57 | * 58 | */ 59 | public static synchronized void disable() { 60 | // using synchronized causes update to disabled to be published, so other threads will see updated value. 61 | // Deadlocks: 62 | // Since this is the only method that acquires lock on the class object, deadlock with Q's lock is not an 63 | // issue (i.e., lock order is same throughout the class: LatencyTracker.class, then Q) 64 | disabled = true; 65 | Q.clear(); 66 | // The clear does not guarantee that Q will be empty afterwards - e.g., if another thread was in the middle 67 | // of add. However, the clear is not critical. Also, disable is one-way, so a little bit of crud leftover 68 | // doesnt matter. If in the future we offer re-enable, then the enable would have to clear the Q, to prevent 69 | // very old entries from being sent. 70 | } 71 | 72 | static void addLatency(String clientRequestId, int retryNum, long latency, String operation, long size, long clientId) { 73 | if (disabled) return; 74 | String line = String.format("%s.%d,%d,,%s,%d,%d", clientRequestId, retryNum, latency, operation, size, clientId); 75 | Q.offer(line); // non-blocking append. If queue is full then silently discard 76 | } 77 | 78 | static void addError(String clientRequestId, int retryNum, long latency, String error, String operation, long size, long clientId) { 79 | if (disabled) return; 80 | String line = String.format("%s.%d,%d,%s,%s,%d,%d", clientRequestId, retryNum, latency, error, operation, size, clientId); 81 | Q.offer(line); // non-blocking append. If queue is full then silently discard 82 | } 83 | 84 | static String get() { 85 | if (disabled) return null; 86 | int count = 0; 87 | String entry = null; 88 | String separator = ""; 89 | StringBuilder line = new StringBuilder(MAXPERLINE * 2); 90 | 91 | do { 92 | entry = Q.poll(); 93 | if (entry != null) 94 | { 95 | line.append(separator); 96 | line.append(entry); 97 | separator = ";"; 98 | count++; 99 | } 100 | } while (entry != null && count <= MAXPERLINE); 101 | return (count > 0) ? line.toString() : null; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/DirectoryEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import java.util.Date; 10 | 11 | /** 12 | * filesystem metadata of a directory enrty (a file or a directory) in ADL. 13 | */ 14 | public class DirectoryEntry { 15 | 16 | /** 17 | * the filename (minus the path) of the direcotry entry 18 | */ 19 | public final String name; 20 | 21 | /** 22 | * the full path of the directory enrty. 23 | */ 24 | public final String fullName; 25 | 26 | /** 27 | * the length of a file. zero for directories. 28 | */ 29 | public final long length; 30 | 31 | /** 32 | * the ID of the group that owns this file/directory. 33 | */ 34 | public final String group; 35 | 36 | /** 37 | * the ID of the user that owns this file/directory. 38 | */ 39 | public final String user; 40 | 41 | /** 42 | * the timestamp of the last time the file was accessed 43 | */ 44 | public final Date lastAccessTime; 45 | 46 | /** 47 | * the timestamp of the last time the file was modified 48 | */ 49 | public final Date lastModifiedTime; 50 | 51 | /** 52 | * {@link DirectoryEntryType} enum indicating whether the object is a file or a directory 53 | */ 54 | public final DirectoryEntryType type; 55 | 56 | /** 57 | * Block size reported by server. 58 | * This is present for compatibility with WebHDFS - in the case of Azure Data Lake store this is always 256MB. 59 | * 60 | */ 61 | public final long blocksize; 62 | 63 | /** 64 | * Replication Factor reported by server. 65 | * This is present for compatibility with WebHDFS - in the case of Azure Data Lake store this is always 66 | * reported as 1 - the Azure Data Lake store does appropriate replication on the server side to ensure 67 | * durability. 68 | * 69 | */ 70 | public final int replicationFactor; 71 | 72 | /** 73 | * boolean indicating whether file has ACLs set on it. 74 | */ 75 | public final boolean aclBit; 76 | 77 | /** 78 | * Date+time at which the file expires, as UTC time. It is null if the file has no expiry. It is null for 79 | * directories. 80 | */ 81 | public final Date expiryTime; 82 | 83 | 84 | /** 85 | * the unix-style permission string for this file or directory 86 | */ 87 | public final String permission; 88 | 89 | final String fileContextId ; 90 | 91 | public DirectoryEntry(String name, 92 | String fullName, 93 | long length, 94 | String group, 95 | String user, 96 | Date lastAccessTime, 97 | Date lastModifiedTime, 98 | DirectoryEntryType type, 99 | long blocksize, 100 | int replicationFactor, 101 | String permission, 102 | boolean aclBit, 103 | Date expiryTime) { 104 | 105 | this(name, fullName, length, group, user, lastAccessTime, lastModifiedTime, type, blocksize, replicationFactor, permission, aclBit, expiryTime, null); 106 | 107 | } 108 | DirectoryEntry(String name, 109 | String fullName, 110 | long length, 111 | String group, 112 | String user, 113 | Date lastAccessTime, 114 | Date lastModifiedTime, 115 | DirectoryEntryType type, 116 | long blocksize, 117 | int replicationFactor, 118 | String permission, 119 | boolean aclBit, 120 | Date expiryTime, 121 | String contextId) { 122 | this.name = name; 123 | this.fullName = fullName; 124 | this.length = length; 125 | this.group = group; 126 | this.user = user; 127 | this.lastAccessTime = new Date(lastAccessTime.getTime()); // creating a new instance, since the passed-in Date is mutable 128 | this.lastModifiedTime = new Date(lastModifiedTime.getTime()); // basically this is copying the date from passed in date 129 | this.type = type; 130 | this.permission = permission; 131 | this.blocksize = blocksize; 132 | this.replicationFactor = replicationFactor; 133 | this.aclBit = aclBit; 134 | this.expiryTime = expiryTime; 135 | this.fileContextId = contextId; 136 | 137 | } 138 | } 139 | 140 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ContentSummaryProcessor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | import com.microsoft.azure.datalake.store.retrypolicies.ExponentialBackoffPolicy; 11 | 12 | import java.io.IOException; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.concurrent.atomic.AtomicLong; 16 | 17 | /** 18 | * Internal class, used to do client-side enumeration of directories to produce result for 19 | * getContentSummary(). 20 | * 21 | */ 22 | class ContentSummaryProcessor { 23 | 24 | private AtomicLong fileCount = new AtomicLong(0); 25 | private AtomicLong directoryCount = new AtomicLong(0); 26 | private AtomicLong totalBytes = new AtomicLong(0); 27 | private ProcessingQueue queue = new ProcessingQueue<>(); 28 | private ADLStoreClient client; 29 | 30 | private static final int NUM_THREADS = 16; 31 | private static final int ENUMERATION_PAGESIZE = 16000; 32 | 33 | // this is a one-shot class, really like a functor. Do not reuse to make multiple calls. 34 | public ContentSummary getContentSummary(ADLStoreClient client, String dirname) throws IOException { 35 | this.client = client; 36 | DirectoryEntry de = client.getDirectoryEntry(dirname); 37 | if (de.type == DirectoryEntryType.FILE) { 38 | processFile(de); 39 | } else { 40 | queue.add(de); 41 | processDirectory(de); 42 | 43 | // Start threads in the processing thread-pool 44 | Thread[] threads = new Thread[NUM_THREADS]; 45 | for (int i = 0; i < NUM_THREADS; i++) { 46 | threads[i] = new Thread(new ThreadProcessor()); 47 | threads[i].start(); 48 | } 49 | 50 | // wait for all threads to get done 51 | for (Thread t : threads) { 52 | try { 53 | t.join(); 54 | } catch (InterruptedException ex) { 55 | Thread.currentThread().interrupt(); 56 | } 57 | } 58 | } 59 | return new ContentSummary(totalBytes.get(), directoryCount.get(), fileCount.get(), totalBytes.get()); 60 | } 61 | 62 | private class ThreadProcessor implements Runnable { 63 | 64 | public void run() { 65 | try { 66 | DirectoryEntry de; 67 | while ((de = queue.poll()) != null) { 68 | try { 69 | if (de.type == DirectoryEntryType.DIRECTORY) { // we should only ever get directories here 70 | processDirectoryTree(de.fullName); 71 | } 72 | } catch (ADLException ex) { 73 | if (ex.httpResponseCode == 404) { 74 | // swallow - the file or directory got deleted after we enumerated it 75 | } else { 76 | throw ex; 77 | } 78 | } finally { 79 | queue.unregister(); 80 | } 81 | } 82 | } catch (IOException ex) { 83 | throw new RuntimeException("IOException processing Directory tree", ex); 84 | } 85 | } 86 | } 87 | 88 | private void processDirectoryTree(String directoryName) throws IOException { 89 | int pagesize = ENUMERATION_PAGESIZE; 90 | ArrayList list; 91 | String startAfter = null; 92 | String continuationToken; 93 | do { 94 | DirectoryEntryListWithContinuationToken directoryEntryListWithContinuationToken = client.enumerateDirectoryInternal(directoryName, pagesize, startAfter,null,null); 95 | continuationToken = directoryEntryListWithContinuationToken.getContinuationToken(); 96 | list = (ArrayList) directoryEntryListWithContinuationToken.getEntries(); 97 | if (list == null || list.size() == 0) break; 98 | for (DirectoryEntry de : list) { 99 | if (de.type == DirectoryEntryType.DIRECTORY) { 100 | queue.add(de); 101 | processDirectory(de); 102 | } 103 | if (de.type == DirectoryEntryType.FILE) { 104 | processFile(de); 105 | } 106 | startAfter = de.name; 107 | } 108 | } while (continuationToken!=""); 109 | } 110 | 111 | private void processDirectory(DirectoryEntry de) { 112 | directoryCount.incrementAndGet(); 113 | } 114 | 115 | private void processFile(DirectoryEntry de) { 116 | fileCount.incrementAndGet(); 117 | totalBytes.addAndGet(de.length); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/Operation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | 10 | /** 11 | * The WebHDFS methods, and their associated properties (e.g., what HTTP method to use, etc.) 12 | */ 13 | enum Operation { 14 | OPEN ("OPEN", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 15 | GETFILESTATUS ("GETFILESTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 16 | MSGETFILESTATUS ("MSGETFILESTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 17 | LISTSTATUS ("LISTSTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 18 | MSLISTSTATUS ("MSLISTSTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 19 | GETCONTENTSUMMARY ("GETCONTENTSUMMARY", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 20 | GETFILECHECKSUM ("GETFILECHECKSUM", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 21 | GETACLSTATUS ("GETACLSTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 22 | MSGETACLSTATUS ("MSGETACLSTATUS", "GET", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 23 | CHECKACCESS ("CHECKACCESS", "GET", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 24 | CREATE ("CREATE", "PUT", C.requiresBodyTrue, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 25 | MKDIRS ("MKDIRS", "PUT", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 26 | RENAME ("RENAME", "PUT", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 27 | SETOWNER ("SETOWNER", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 28 | SETPERMISSION ("SETPERMISSION", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 29 | SETTIMES ("SETTIMES", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 30 | MODIFYACLENTRIES ("MODIFYACLENTRIES", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 31 | REMOVEACLENTRIES ("REMOVEACLENTRIES", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 32 | REMOVEDEFAULTACL ("REMOVEDEFAULTACL", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 33 | REMOVEACL ("REMOVEACL", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 34 | SETACL ("SETACL", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 35 | CREATENONRECURSIVE ("CREATENONRECURSIVE", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 36 | APPEND ("APPEND", "POST", C.requiresBodyTrue, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 37 | CONCAT ("CONCAT", "POST", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfs), 38 | MSCONCAT ("MSCONCAT", "POST", C.requiresBodyTrue, C.returnsBodyFalse, C.enforceMimeTypeJsonTrue, C.webHdfs), 39 | DELETE ("DELETE", "DELETE", C.requiresBodyFalse, C.returnsBodyTrue, C.enforceMimeTypeJsonFalse, C.webHdfs), 40 | CONCURRENTAPPEND ("CONCURRENTAPPEND", "POST", C.requiresBodyTrue, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfsExt), 41 | SETEXPIRY ("SETEXPIRY", "PUT", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfsExt), 42 | GETFILEINFO ("GETFILEINFO", "GET", C.requiresBodyFalse, C.returnsBodyFalse, C.enforceMimeTypeJsonFalse, C.webHdfsExt); 43 | 44 | String name; 45 | String method; 46 | boolean requiresBody; 47 | boolean returnsBody; 48 | boolean enforceMimeTypeJson; 49 | String namespace; 50 | 51 | Operation(String name, String method, boolean requiresBody, boolean returnsBody, boolean enforceMimeTypeJson, String namespace) { 52 | this.name = name; 53 | this.method = method; 54 | this.requiresBody = requiresBody; 55 | this.returnsBody = returnsBody; 56 | this.namespace = namespace; 57 | this.enforceMimeTypeJson = enforceMimeTypeJson; 58 | } 59 | 60 | private static class C { 61 | static final boolean requiresBodyTrue = true; 62 | static final boolean requiresBodyFalse = false; 63 | static final boolean returnsBodyTrue = true; 64 | static final boolean returnsBodyFalse = false; 65 | static final boolean enforceMimeTypeJsonTrue = true; 66 | static final boolean enforceMimeTypeJsonFalse = false; 67 | static final String webHdfs = "/webhdfs/v1"; 68 | static final String webHdfsExt = "/WebHdfsExt"; 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/unittests/TestRetryPolicies.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.unittests; 8 | 9 | import com.contoso.helpers.HelperUtils; 10 | import com.microsoft.azure.datalake.store.retrypolicies.ExponentialBackoffPolicy; 11 | import org.junit.Assume; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | 15 | import java.io.IOException; 16 | import java.util.Properties; 17 | 18 | import static org.junit.Assert.*; 19 | 20 | public class TestRetryPolicies { 21 | 22 | private boolean testsEnabled = true; 23 | private boolean longRunningEnabled = true; 24 | 25 | private class Timer 26 | { 27 | private long startTime = System.nanoTime(); 28 | 29 | int getElapsedMilliseconds() 30 | { 31 | return (int)((System.nanoTime() - startTime) / 1000000); 32 | } 33 | } 34 | 35 | 36 | @Before 37 | public void setup() throws IOException { 38 | Properties prop = HelperUtils.getProperties(); 39 | 40 | testsEnabled = Boolean.parseBoolean(prop.getProperty("MockTestsEnabled", "true")); 41 | longRunningEnabled = Boolean.parseBoolean(prop.getProperty("LongRunningTestsEnabled", "true")); 42 | } 43 | 44 | @Test 45 | public void testExponentialRetryRunOutOfRetries() { 46 | Assume.assumeTrue(testsEnabled); 47 | Assume.assumeTrue(longRunningEnabled); 48 | System.out.println("Running testExponentialRetryRunOutOfRetries"); 49 | 50 | 51 | ExponentialBackoffPolicy retryPolicy = new ExponentialBackoffPolicy(); 52 | 53 | { 54 | Timer timer = new Timer(); 55 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 56 | assertEquals(1000, timer.getElapsedMilliseconds(), 500); 57 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 58 | } 59 | 60 | { 61 | Timer timer = new Timer(); 62 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 63 | assertEquals(4000, timer.getElapsedMilliseconds(), 500); 64 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 65 | } 66 | 67 | { 68 | Timer timer = new Timer(); 69 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 70 | assertEquals(16000, timer.getElapsedMilliseconds(), 500); 71 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 72 | } 73 | 74 | { 75 | Timer timer = new Timer(); 76 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 77 | assertEquals(64000, timer.getElapsedMilliseconds(), 500); 78 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 79 | } 80 | 81 | { 82 | Timer timer = new Timer(); 83 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 84 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 85 | assertTrue("shouldRetry was incorrect", shouldRetry == false); 86 | } 87 | } 88 | 89 | @Test 90 | public void testExponentialRetryDoesNotWaitUnnecessarily() { 91 | Assume.assumeTrue(testsEnabled); 92 | Assume.assumeTrue(longRunningEnabled); 93 | System.out.println("Running testExponentialRetryDoesNotWaitUnnecessarily"); 94 | 95 | ExponentialBackoffPolicy retryPolicy = new ExponentialBackoffPolicy(); 96 | 97 | { 98 | wait(1000); 99 | Timer timer = new Timer(); 100 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 101 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 102 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 103 | } 104 | 105 | { 106 | wait(4000); 107 | Timer timer = new Timer(); 108 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 109 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 110 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 111 | } 112 | 113 | { 114 | wait(16000); 115 | Timer timer = new Timer(); 116 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 117 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 118 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 119 | } 120 | 121 | { 122 | wait(64000); 123 | Timer timer = new Timer(); 124 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 125 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 126 | assertTrue("shouldRetry was incorrect", shouldRetry == true); 127 | } 128 | 129 | { 130 | Timer timer = new Timer(); 131 | boolean shouldRetry = retryPolicy.shouldRetry(503, null); 132 | assertEquals(0, timer.getElapsedMilliseconds(), 500); 133 | assertTrue("shouldRetry was incorrect", shouldRetry == false); 134 | } 135 | } 136 | 137 | private void wait(int milliseconds) { 138 | if (milliseconds <= 0) { 139 | return; 140 | } 141 | 142 | try { 143 | Thread.sleep(milliseconds); 144 | } catch (InterruptedException ex) { 145 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 146 | } 147 | } 148 | } -------------------------------------------------------------------------------- /src/test/java/com/contoso/liveservicetests/TestPositionedReads.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.liveservicetests; 8 | 9 | import com.contoso.helpers.HelperUtils; 10 | import com.microsoft.azure.datalake.store.*; 11 | 12 | import com.microsoft.azure.datalake.store.oauth2.AzureADAuthenticator; 13 | import com.microsoft.azure.datalake.store.oauth2.AzureADToken; 14 | import org.junit.AfterClass; 15 | import org.junit.Assume; 16 | import org.junit.BeforeClass; 17 | import org.junit.Test; 18 | import static org.junit.Assert.*; 19 | 20 | import java.io.IOException; 21 | import java.io.OutputStream; 22 | import java.util.*; 23 | 24 | public class TestPositionedReads { 25 | private final UUID instanceGuid = UUID.randomUUID(); 26 | 27 | private static String directory = null; 28 | private static ADLStoreClient client = null; 29 | private static boolean testsEnabled = true; 30 | 31 | @BeforeClass 32 | public static void setup() throws IOException { 33 | Properties prop; 34 | AzureADToken aadToken; 35 | 36 | prop = HelperUtils.getProperties(); 37 | aadToken = AzureADAuthenticator.getTokenUsingClientCreds(prop.getProperty("OAuth2TokenUrl"), 38 | prop.getProperty("ClientId"), 39 | prop.getProperty("ClientSecret") ); 40 | UUID guid = UUID.randomUUID(); 41 | directory = "/" + prop.getProperty("dirName", "unitTests") + "/" + UUID.randomUUID(); 42 | String account = prop.getProperty("StoreAcct") + ".azuredatalakestore.net"; 43 | client = ADLStoreClient.createClient(account, aadToken); 44 | testsEnabled = Boolean.parseBoolean(prop.getProperty("PositionedReadsTestsEnabled", "true")); 45 | } 46 | 47 | @AfterClass 48 | public static void teardown() throws IOException { 49 | client.deleteRecursive(directory); 50 | } 51 | 52 | @Test 53 | public void smallFileSeek() throws IOException { 54 | Assume.assumeTrue(testsEnabled); 55 | String filename = directory + "/" + "PositionedReads.smallFileSeek.dat"; 56 | System.out.println("Running smallFileSeek"); 57 | 58 | int fileLength = 1024; 59 | OutputStream stream = client.createFile(filename, IfExists.OVERWRITE); 60 | byte[] content = HelperUtils.getRandomBuffer(fileLength); 61 | stream.write(content); 62 | stream.close(); 63 | 64 | ADLFileInputStream instream = client.getReadStream(filename); 65 | assertTrue("File length should be as expected", instream.length() == fileLength); 66 | assertTrue("File position initially should be 0", instream.getPos() == 0); 67 | instream.seek(instream.length() - 2); 68 | assertTrue("Premature EOF at (-2)", instream.read() != -1); 69 | assertTrue("Premature EOF at (-1)", instream.read() != -1); 70 | assertTrue("read() should return -1 at EOF", instream.read() == -1); 71 | } 72 | 73 | @Test 74 | public void seekAndCheck() throws IOException { 75 | Assume.assumeTrue(testsEnabled); 76 | String filename = directory + "/" + "PositionedReads.seekAndCheck.txt"; 77 | System.out.println("Running seekAndCheck"); 78 | 79 | OutputStream stream = client.createFile(filename, IfExists.OVERWRITE); 80 | byte[] content = HelperUtils.getSampleText1(); 81 | stream.write(content); 82 | stream.close(); 83 | 84 | ADLFileInputStream in = client.getReadStream(filename); 85 | in.setBufferSize(20); 86 | assertTrue("should be able to seek past buffer in the beginning",checkByteAt(21, in, content)); 87 | assertTrue("should be able to seek to beginning from anywhere",checkByteAt(0, in, content)); 88 | assertTrue("should be able to seek past buffer",checkByteAt(60, in, content)); 89 | assertTrue("re-seeking to same location should work",checkByteAt(60, in, content)); 90 | assertTrue("seeking within buffer should work",checkByteAt(61, in, content)); 91 | assertTrue("seeking to within buffer should work",checkByteAt(75, in, content)); 92 | assertTrue("back-and-forth within buffer should work",checkByteAt(74, in, content)); 93 | assertTrue("more back-and-forth within buffer should work",checkByteAt(62, in, content)); 94 | assertTrue("seeking backwards should work",checkByteAt(21, in, content)); 95 | assertTrue("seeking forwards should work",checkByteAt(45, in, content)); 96 | assertTrue("seeking forward with many buffer's gap should work",checkByteAt(80, in, content)); 97 | assertTrue("seeking backwards with many buffer's gap should work",checkByteAt(23, in, content)); 98 | assertTrue("seeking backwards to one byte before buffer should work",checkByteAt(22, in, content)); 99 | assertTrue("more seeks - just for good measure",checkByteAt(99, in, content)); 100 | assertTrue("even more seeks - just for good measure",checkByteAt(11, in, content)); 101 | assertTrue("seeks to early offset",checkByteAt(3, in, content)); 102 | assertTrue("seek back to zero after many seeks",checkByteAt(0, in, content)); 103 | } 104 | 105 | private static boolean checkByteAt(long pos, ADLFileInputStream in, byte[] content) throws IOException { 106 | in.seek(pos); 107 | int expected = content[(int)pos]; 108 | int actual = in.read(); 109 | if (actual != expected) { 110 | System.out.format("Mismatch at %d: expected %c, found %c\n", pos, expected, actual ); 111 | return false; 112 | } else { 113 | return true; 114 | } 115 | } 116 | 117 | @Test 118 | public void resizeBuffer() throws IOException { 119 | Assume.assumeTrue(false); 120 | String filename = directory + "/" + "PositionedReads.resizeBuffer.txt"; 121 | 122 | OutputStream stream = client.createFile(filename, IfExists.OVERWRITE); 123 | byte[] content = HelperUtils.getSampleText1(); 124 | stream.write(content); 125 | stream.close(); 126 | 127 | ADLFileInputStream instr = client.getReadStream(filename); 128 | byte[] b1 = new byte[200]; 129 | instr.read(b1); 130 | 131 | instr.setBufferSize(87); 132 | instr.read(b1); 133 | 134 | 135 | 136 | assertTrue("This test needs to be finished", false); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changes to the SDK 2 | ### Version 2.3.10 3 | 1. Update log4j to mitigate CVE-2021-44228. Also update junit. 4 | 2. ERRATA: This impacts only tests in this repository. SLF4J(See http://slf4j.org/log4shell.html) interface is used for logging and if log4j is available, it can be used depending on customer configuration. The version depends on customer application built using the sdk. Previous versions of sdk(<2.3.10) are not impacted by log4j CVE 5 | 6 | ### Version 2.3.9 7 | 1. Fix the setters in adlstoreoptions to return instance of adlstoreoptions 8 | 2. Fetch MSI token directly from AAD if first fetch from MSI cache is the same expiring token. 9 | 3. Fix getcontentsummary to do enumerates using continuation token 10 | 11 | ### Version 2.3.8 12 | 1. Add logging for token acquisition. 13 | 2. Allow user configured retry count, interval, and exponential factor for ExponentialBackoffPolicy 14 | 15 | ### Version 2.3.7 16 | 1. Implement create with overwrite using conditional delete. Enable this based on config. 17 | 18 | ### Version 2.3.6 19 | 1. Added configuration option to choose SSL channel mode 20 | (AdlStoreOptions.setSSLChannelMode(String mode) with possible mode values 21 | being - (a) OpenSSL (b) Default_JSSE (c) Default 22 | [(c) is the default choice if the config is not present or is invalid. 23 | When set to (c), connection will be created in OpenSSL mode and will 24 | default to Default_JSSE on any failure.] 25 | 26 | ### Version 2.3.5 27 | 1. Updated wildfly openssl version and removed shading of the package 28 | 2. Fix bug in ordering in json parsing for liststatus response 29 | 30 | ### Version 2.3.4 31 | 1. Updated enumerateDirectory to use continuationToken 32 | 2. Updated api-version to 2018-09-01 33 | 3. Update version of jackson.core and maven-javadoc 34 | 4. separate out append and closehandle for SyncFlag close to avoid race condition on retries 35 | 36 | ### Version 2.3.3 37 | 1. Source files list will go in json format for concat operation to handle special characters in source filenames 38 | 2. Prevent FileAlreadyExists exception for create with overwrite 39 | 3. Update com.fasterxml.jackson.core:jackson-core to 2.7.9 to avoid security vulnerability 40 | 4. Disable wildfly logs to the console. 41 | 42 | ### Version 2.3.2 43 | 1. Add special handling for 404 errors when requesting tokens from MSI 44 | 2. Fix liststatus response parsing when filestatus object contains array in one field. 45 | 3. Use wildfly openssl native binding with Java. This is a workaround to https://bugs.openjdk.java.net/browse/JDK-8046943 issue. 2X performance boost over HTTPS. 46 | 47 | ### Version 2.3.1 48 | 1. Made the default queue depth to zero (basically disabling read-ahead by default). Set readahead queue depth 49 | using ADLStoreOptions.setReadAheadQueueDepth() to enable read-ahead 50 | 51 | ### Version 2.3.0-preview2 52 | 1. Made timeouts more aggressive, and made the ADLStoreClient's default timeout configurable (ADLStoreOptions.setDefaultTimeout) 53 | 2. Do automatic retry for the case where a read requests succeeds, but then reading the content stream a fails because of network issue 54 | 55 | ### Version 2.3.0-preview1 56 | 1. ADLInputStream can now optionally do read-ahead (on by default, with queue depth=4). Added 57 | configuration option to ADLStoreOptions to set the queue depth for read-ahead. 58 | 2. Changed REST API version to 2017-08-01 (required for read-aheads) 59 | 3. Internal bug fixes 60 | 61 | 62 | ### Version 2.2.8 63 | 1. Increase default timeout to 60 seconds 64 | 65 | ### Version 2.2.7 66 | 1. Bugfix to IMDS-based MSI 67 | 2. More text added to AAD token acquisition exception message 68 | 69 | ### Version 2.2.6 70 | 1. Changed implementation of the MSI token provider to use the new IMDS REST endpoint required by AAD 71 | 2. Made AAD token acquisition more robust - added timeout, retry, disable keep-alive, improved exception text 72 | 73 | ### Version 2.2.5 74 | 1. Made HTTP 429 error retry-able on non-idempotent calls, since HTTP429 does not make a state change on the server 75 | 76 | ### Version 2.2.4 77 | 1. Made timeouts more aggressive, and made the ADLStoreClient's default timeout configurable (`ADLStoreOptions.setDefaultTimeout`) 78 | 2. Do automatic retry for the case where a read requests succeeds, but then reading the content stream a fails because of network issue 79 | 80 | ### Version 2.2.3 81 | 1. Made port number and tenant Guid optional for MsiTokenProvider. 82 | 83 | ### Version 2.2.2 84 | 1. (internal-only fix, no user impact) Change MSI token acquisition call to add HTTP header (Metadata: true) 85 | 86 | ### Version 2.2.1 87 | 1. Added support for DeviceCode auth flow 88 | 2. Added support for acquiring token using Azure VM's MSI service 89 | 3. Switched all internal TokenProviders to use https://datalake.azure.net/ as "resource" in AAD Tokens instead of 90 | https://management.core.windows.net/ 91 | 4. Misc robustness fixes in ADLStoreClient.GetContentSummary 92 | 93 | ### Version 2.1.5 94 | 1. Fixed bug in ADLFileOutputStream, where calling close() right after calling flush() would not release the lease 95 | on the file, locking the file out for 10 mins 96 | 2. Added server trace ID to exception messages, so failures are easier to troubleshoot for customer-support calls 97 | 3. Changed exception handling for token fetching; exceptions from the token fetching process were previously getting 98 | replaced with a generic (unhelpful) error message 99 | 100 | ### Version 2.1.4 101 | 1. Fixed bug in Core.listStatus() for expiryTime parsing 102 | 103 | ### Version 2.1.2 104 | 1. Changed implementation of ADLStoreClient.getContentSummary to do the directory enumeration on client side rather than 105 | server side. This makes the call more performant and reliable. 106 | 2. Removed short-circuit check in Core.concurrentAppend, which bypassed sending append to server for a 0-length append. 107 | 108 | ### Version 2.1.1 109 | 1. Added setExpiry method 110 | 2. Core.concat has an additional parameter called deleteSourceDirectory, to address specific needs for tool-writers 111 | 3. enumerateDirectory and getDirectoryEntry (liststatus and getfilestatus in Core) now return two additional 112 | fields: aclBit and expiryTime 113 | 4. enumerateDirectory, getDirectoryEntry and getAclStatus now take additional parameter for 114 | UserGroupRepresentation (OID or UPN) 115 | 5. enumerateDirectory now does paging on the client side, to avoid timeouts on lerge directories (no change 116 | to API interface) 117 | 118 | ### Version 2.0.11 119 | - Initial Release 120 | 121 | 122 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/SSLSocketFactoryEx.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.wildfly.openssl.OpenSSLProvider; 12 | import org.wildfly.openssl.SSL; 13 | 14 | import javax.net.ssl.*; 15 | import java.io.IOException; 16 | import java.net.InetAddress; 17 | import java.net.Socket; 18 | import java.net.SocketException; 19 | import java.security.KeyManagementException; 20 | import java.security.NoSuchAlgorithmException; 21 | import java.security.SecureRandom; 22 | import java.util.ArrayList; 23 | import java.util.logging.Level; 24 | 25 | 26 | /** 27 | * Extension to use native OpenSSL library instead of JSE for better performance. 28 | */ 29 | public class SSLSocketFactoryEx extends SSLSocketFactory { 30 | 31 | public enum SSLChannelMode { 32 | OpenSSL, 33 | /** 34 | * Ordered, preferred OpenSSL, if failed to load then fall back to 35 | * Default_JSE 36 | */ 37 | Default, 38 | Default_JSE 39 | } 40 | 41 | private static SSLSocketFactoryEx instance = null; 42 | private static Object lock = new Object(); 43 | private static final Logger log = LoggerFactory 44 | .getLogger("com.microsoft.azure.datalake.store.SSLSocketFactoryEx"); 45 | private String userAgent; 46 | private SSLContext m_ctx; 47 | private String[] m_ciphers; 48 | private SSLChannelMode channelMode; 49 | 50 | public static SSLSocketFactoryEx getDefaultFactory(SSLChannelMode sslChannelMode) throws IOException { 51 | if (instance == null) { 52 | synchronized (lock) { 53 | if (instance == null) { 54 | instance = new SSLSocketFactoryEx(sslChannelMode); 55 | log.debug("SSLSocketFactoryEx created in " + sslChannelMode.name() + " mode"); 56 | } 57 | } 58 | } 59 | return instance; 60 | } 61 | 62 | static { 63 | OpenSSLProvider.register(); 64 | } 65 | 66 | public SSLSocketFactoryEx(SSLChannelMode channelMode) throws IOException { 67 | this.channelMode = channelMode; 68 | try { 69 | initSSLSocketFactoryEx(null, null, null); 70 | } catch (NoSuchAlgorithmException e) { 71 | throw new IOException(e); 72 | } catch (KeyManagementException e) { 73 | throw new IOException(e); 74 | } 75 | 76 | userAgent = m_ctx.getProvider().getName() + "-" + m_ctx.getProvider().getVersion(); 77 | } 78 | 79 | public String getUserAgent() { 80 | return userAgent; 81 | } 82 | 83 | public String[] getDefaultCipherSuites() { 84 | return m_ciphers; 85 | } 86 | 87 | public String[] getSupportedCipherSuites() { 88 | return m_ciphers; 89 | } 90 | 91 | public Socket createSocket() throws IOException { 92 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 93 | SSLSocket ss = (SSLSocket) factory.createSocket(); 94 | configureSocket(ss); 95 | return ss; 96 | } 97 | 98 | @Override 99 | public Socket createSocket(Socket s, String host, int port, boolean autoClose) 100 | throws IOException { 101 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 102 | SSLSocket ss = (SSLSocket) factory.createSocket(s, host, port, autoClose); 103 | 104 | configureSocket(ss); 105 | return ss; 106 | } 107 | 108 | @Override 109 | public Socket createSocket(InetAddress address, int port, 110 | InetAddress localAddress, int localPort) throws IOException { 111 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 112 | SSLSocket ss = (SSLSocket) factory 113 | .createSocket(address, port, localAddress, localPort); 114 | 115 | configureSocket(ss); 116 | return ss; 117 | } 118 | 119 | @Override 120 | public Socket createSocket(String host, int port, InetAddress localHost, 121 | int localPort) throws IOException { 122 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 123 | SSLSocket ss = (SSLSocket) factory 124 | .createSocket(host, port, localHost, localPort); 125 | 126 | configureSocket(ss); 127 | 128 | return ss; 129 | } 130 | 131 | @Override 132 | public Socket createSocket(InetAddress host, int port) throws IOException { 133 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 134 | SSLSocket ss = (SSLSocket) factory.createSocket(host, port); 135 | 136 | configureSocket(ss); 137 | 138 | return ss; 139 | } 140 | 141 | @Override 142 | public Socket createSocket(String host, int port) throws IOException { 143 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 144 | SSLSocket ss = (SSLSocket) factory.createSocket(host, port); 145 | 146 | configureSocket(ss); 147 | 148 | return ss; 149 | } 150 | 151 | private void configureSocket(SSLSocket ss) throws SocketException { 152 | ss.setEnabledCipherSuites(m_ciphers); 153 | } 154 | 155 | private void initSSLSocketFactoryEx(KeyManager[] km, TrustManager[] tm, 156 | SecureRandom random) 157 | throws NoSuchAlgorithmException, KeyManagementException, IOException { 158 | 159 | switch (channelMode) { 160 | case Default: 161 | try { 162 | java.util.logging.Logger.getLogger(SSL.class.getName()).setLevel(Level.WARNING); 163 | m_ctx = SSLContext.getInstance("openssl.TLS"); 164 | m_ctx.init(km, tm, random); 165 | channelMode = SSLChannelMode.OpenSSL; 166 | } catch (NoSuchAlgorithmException e) { 167 | log.info("Failed to load OpenSSL library. Fallback to default JSE. ", e); 168 | m_ctx = SSLContext.getDefault(); 169 | channelMode = SSLChannelMode.Default_JSE; 170 | } 171 | break; 172 | 173 | case Default_JSE: 174 | m_ctx = SSLContext.getDefault(); 175 | break; 176 | 177 | case OpenSSL: 178 | m_ctx = SSLContext.getInstance("openssl.TLS"); 179 | m_ctx.init(km, tm, random); 180 | break; 181 | } 182 | 183 | // Get list of supported cipher suits from the SSL factory. 184 | SSLSocketFactory factory = m_ctx.getSocketFactory(); 185 | String[] defaultCiphers = factory.getSupportedCipherSuites(); 186 | String version = System.getProperty("java.version"); 187 | 188 | m_ciphers = (channelMode == SSLChannelMode.Default_JSE && version.startsWith("1.8")) ? 189 | alterCipherList(defaultCiphers) : defaultCiphers; 190 | } 191 | 192 | private String[] alterCipherList(String[] defaultCiphers) { 193 | 194 | ArrayList preferredSuits = new ArrayList<>(); 195 | 196 | // Remove GCM mode based ciphers from the supported list. 197 | for (int i = 0; i < defaultCiphers.length; i++) { 198 | if (defaultCiphers[i].contains("_GCM_")) { 199 | log.info("Removed Cipher - " + defaultCiphers[i]); 200 | } else { 201 | preferredSuits.add(defaultCiphers[i]); 202 | } 203 | } 204 | 205 | m_ciphers = preferredSuits.toArray(new String[0]); 206 | return m_ciphers; 207 | } 208 | } 209 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | com.microsoft.azure 6 | azure-data-lake-store-sdk 7 | 2.3.10 8 | jar 9 | Azure Data Lake Store - Java client SDK 10 | https://azure.microsoft.com/en-us/services/data-lake-store 11 | Java SDK for Microsoft Azure Data Lake Store 12 | 13 | 14 | Microsoft Corporation 15 | http://www.microsoft.com 16 | 17 | 18 | 19 | 20 | Microsoft 21 | Microsoft Corporation 22 | Microsoft Corporation 23 | http://www.microsoft.com 24 | 25 | 26 | 27 | 28 | 29 | MIT License 30 | https://github.com/Azure/azure-data-lake-store-java/blob/master/LICENSE.txt 31 | 32 | 33 | 34 | 35 | scm:git:https://github.com/Azure/azure-data-lake-store-java.git 36 | https://github.com/Azure/azure-data-lake-store-java 37 | 38 | 39 | 40 | UTF-8 41 | 42 | 43 | 44 | 45 | 46 | true 47 | ${basedir}/src/main/resources 48 | 49 | 50 | 51 | 52 | 53 | src/test/configfiles 54 | true 55 | 56 | **/*.properties 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | org.apache.maven.plugins 65 | maven-compiler-plugin 66 | 3.5.1 67 | 68 | 1.8 69 | 1.8 70 | 71 | 72 | 73 | org.apache.maven.plugins 74 | maven-install-plugin 75 | 2.5.2 76 | 77 | true 78 | 79 | 80 | 81 | org.apache.maven.plugins 82 | maven-jar-plugin 83 | 2.5 84 | 85 | 86 | 87 | true 88 | true 89 | 90 | 91 | Microsoft Corporation 92 | 93 | 94 | 95 | 96 | 97 | org.apache.maven.plugins 98 | maven-source-plugin 99 | 3.0.0 100 | 101 | 102 | attach-sources 103 | 104 | jar 105 | 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-javadoc-plugin 112 | 3.0.1 113 | 114 | true 115 | 116 | 117 | 118 | attach-javadocs 119 | 120 | jar 121 | 122 | 123 | 124 | 125 | 126 | org.apache.maven.plugins 127 | maven-surefire-plugin 128 | 3.0.0-M7 129 | 130 | true 131 | true 132 | 133 | 134 | 135 | org.apache.maven.plugins 136 | maven-resources-plugin 137 | 2.6 138 | 139 | 140 | copy-resources 141 | process-test-resources 142 | 143 | copy-resources 144 | 145 | 146 | ${basedir}/target 147 | 148 | 149 | src/test/configfiles 150 | false 151 | 152 | .gitignore 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | com.fasterxml.jackson.core 166 | jackson-core 167 | 2.8.6 168 | 169 | 170 | org.slf4j 171 | slf4j-api 172 | 1.7.21 173 | 174 | 175 | org.wildfly.openssl 176 | wildfly-openssl 177 | 1.0.7.Final 178 | 179 | 180 | 181 | 182 | junit 183 | junit 184 | 4.13.1 185 | test 186 | 187 | 188 | com.squareup.okhttp3 189 | mockwebserver 190 | 3.2.0 191 | test 192 | 193 | 194 | org.apache.logging.log4j 195 | log4j-slf4j-impl 196 | 2.15.0 197 | test 198 | 199 | 200 | org.apache.logging.log4j 201 | log4j-api 202 | 2.17.0 203 | test 204 | 205 | 206 | org.apache.logging.log4j 207 | log4j-core 208 | 2.17.1 209 | test 210 | 211 | 212 | 213 | 214 | 215 | adlsRelease 216 | 217 | 218 | 219 | org.apache.maven.plugins 220 | maven-gpg-plugin 221 | 1.6 222 | 223 | 224 | sign-artifacts 225 | verify 226 | 227 | sign 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | ossrh 237 | https://oss.sonatype.org/content/repositories/snapshots 238 | 239 | 240 | ossrh 241 | https://oss.sonatype.org/service/local/staging/deploy/maven2/ 242 | 243 | 244 | 245 | 246 | 247 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ADLStoreOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import com.microsoft.azure.datalake.store.SSLSocketFactoryEx.SSLChannelMode; 10 | 11 | /** 12 | * Options to configure the behavior of {@link ADLStoreClient} 13 | */ 14 | public class ADLStoreOptions { 15 | 16 | private String userAgentSuffix = null; 17 | private boolean insecureTransport = false; 18 | private boolean enableRemoteExceptions = false; 19 | private String pathPrefix = null; 20 | private int readAheadQueueDepth = -1; // no preference set by caller, use default in ADLFileInputStream 21 | private int defaultTimeout = -1; 22 | private boolean alterCipherSuits = true; 23 | private SSLChannelMode sslChannelMode = SSLChannelMode.Default; 24 | 25 | private int maxRetries = DEFAULT_MAX_RETRIES; 26 | private int exponentialRetryInterval = DEFAULT_EXPONENTIAL_RETRY_INTERVAL; 27 | private int exponentialFactor = DEFAULT_EXPONENTIAL_FACTOR; 28 | 29 | static final int DEFAULT_MAX_RETRIES = 4; 30 | static final int DEFAULT_EXPONENTIAL_RETRY_INTERVAL = 1000; 31 | static final int DEFAULT_EXPONENTIAL_FACTOR = 4; 32 | private boolean enableConditionalCreate = false; 33 | 34 | public ADLStoreOptions() { 35 | } 36 | 37 | /** 38 | * sets the user agent suffix to be added to the User-Agent header in all HTTP requests made to the server. 39 | * This suffix is appended to the end of the User-Agent string constructed by the SDK. 40 | * 41 | * @param userAgentSuffix the suffix 42 | * @return {@code this} 43 | */ 44 | public ADLStoreOptions setUserAgentSuffix(String userAgentSuffix) { 45 | this.userAgentSuffix = userAgentSuffix; 46 | return this; 47 | } 48 | 49 | /** 50 | * gets the user agent suffix configured in this object. 51 | * @return the user agent suffix configured in this object 52 | */ 53 | String getUserAgentSuffix() { 54 | return this.userAgentSuffix; 55 | } 56 | 57 | /** 58 | * Use http as transport for back-end calls, instead of https. This is to allow unit 59 | * testing using mock or fake web servers. 60 | *

61 | * Warning: Do not use this for talking to real Azure Data Lake service, 62 | * since https is the only supported protocol on the server. 63 | *

64 | * @return {@code this} 65 | */ 66 | public ADLStoreOptions setInsecureTransport() { 67 | this.insecureTransport = true; 68 | return this; 69 | } 70 | 71 | /** 72 | * gets the transport security configured in this object. 73 | * @return the transport security configured in this object 74 | */ 75 | boolean isUsingInsecureTransport() { 76 | return this.insecureTransport; 77 | } 78 | 79 | /** 80 | * Throw server-returned exception name instead of ADLExcetption. 81 | *

82 | * ADLStoreClient methods throw {@link ADLException} on failure. {@link ADLException} 83 | * contains additional fields that have details on the error that occurred, like the HTTP 84 | * response code and the server request ID, etc. 85 | *

86 | * However, in some cases, server returns an exception name in it's HTTP error stream. 87 | * Calling this method causes the ADLStoreClient methods to throw the exception name 88 | * returned by the server rather than {@link ADLException}. 89 | *

90 | * In general, this is not recommended, since the name of the remote exception can also 91 | * be retrieved from {@link ADLException}. This method exists to enable usage within 92 | * Hadoop as a file system. 93 | *

94 | * @return {@code this} 95 | */ 96 | public ADLStoreOptions enableThrowingRemoteExceptions() { 97 | this.enableRemoteExceptions = true; 98 | return this; 99 | } 100 | 101 | /** 102 | * gets the exception behavior configured in this object. 103 | * @return the exception behavior configured in this object 104 | */ 105 | boolean isThrowingRemoteExceptionsEnabled() { 106 | return this.enableRemoteExceptions; 107 | } 108 | 109 | /** 110 | * Set a prefix that will be prepended to all file paths from this client. This allows the 111 | * client to be scoped to a subset of the directory Azure Data Lake Store tree. 112 | * 113 | * @param prefix {@code String} containing the prefix to be prepended 114 | * @return {@code this} 115 | */ 116 | public ADLStoreOptions setFilePathPrefix(String prefix) { 117 | this.pathPrefix = prefix; 118 | return this; 119 | } 120 | 121 | /** 122 | * gets the file path prefix configured in this object 123 | * @return the file path prefix configured in this object 124 | */ 125 | String getFilePathPrefix() { 126 | return this.pathPrefix; 127 | } 128 | 129 | 130 | /** 131 | * Sets the default Queue depth to be used for read-aheads in {@link ADLFileInputStream}. 132 | * 133 | * @param queueDepth the desired queue depth, set to 0 to disable read-ahead 134 | * @return {@code this} 135 | */ 136 | public ADLStoreOptions setReadAheadQueueDepth(int queueDepth) { 137 | if (queueDepth < 0) throw new IllegalArgumentException("Queue depth has to be 0 or more"); 138 | this.readAheadQueueDepth = queueDepth; 139 | return this; 140 | } 141 | 142 | 143 | /** 144 | * Gets the default Queue depth used for read-aheads in {@link ADLFileInputStream} 145 | * @return the queue depth 146 | */ 147 | int getReadAheadQueueDepth() { 148 | return this.readAheadQueueDepth; 149 | } 150 | 151 | 152 | /** 153 | * sets the default timeout for calls make by methods in ADLStoreClient objects 154 | * @param defaultTimeoutInMillis default timeout, in Milliseconds 155 | * @return {@code this} 156 | */ 157 | public ADLStoreOptions setDefaultTimeout(int defaultTimeoutInMillis) { 158 | this.defaultTimeout = defaultTimeoutInMillis; 159 | return this; 160 | } 161 | 162 | /** 163 | * gets the default timeout for calls make by methods in ADLStoreClient objects 164 | * @return default timeout, in Milliseconds 165 | */ 166 | int getDefaultTimeout() { 167 | return this.defaultTimeout; 168 | } 169 | 170 | /** 171 | * Java 1.8 version with GCM cipher suite does not use hardware acceleration. 172 | * If set to true then SDK would try to optimize cipher suite selection. 173 | * If set to false then SDK would use default cipher suite. 174 | * 175 | * @param alterCipherSuits true if the cipher suite alteration is required. 176 | */ 177 | public ADLStoreOptions alterCipherSuits(boolean alterCipherSuits) { 178 | this.alterCipherSuits = alterCipherSuits; 179 | return this; 180 | } 181 | 182 | boolean shouldAlterCipherSuits() { 183 | return this.alterCipherSuits; 184 | } 185 | 186 | /** 187 | * Set SSL Channel mode 188 | * @param sslChannelMode SSL Channel mdoe to set 189 | * @return 190 | */ 191 | public ADLStoreOptions setSSLChannelMode(String sslChannelMode) { 192 | SSLChannelMode[] sslChannelModes = SSLChannelMode.values(); 193 | for(SSLChannelMode mode : sslChannelModes) 194 | { 195 | if (sslChannelMode.equalsIgnoreCase(mode.name())) 196 | { 197 | this.sslChannelMode = mode; 198 | return this; 199 | } 200 | } 201 | 202 | this.sslChannelMode = SSLChannelMode.Default; 203 | return this; 204 | } 205 | 206 | SSLChannelMode getSSLChannelMode() { 207 | return this.sslChannelMode; 208 | } 209 | 210 | int getMaxRetries() { 211 | return maxRetries; 212 | } 213 | 214 | /** 215 | * sets the number of retries for exponential retry policy used by methods in ADLStoreClient objects 216 | * @param maxRetries number of retries for exponential retry policy 217 | * 218 | */ 219 | public ADLStoreOptions setMaxRetries(int maxRetries) { 220 | this.maxRetries = maxRetries; 221 | return this; 222 | } 223 | 224 | int getExponentialRetryInterval() { 225 | return exponentialRetryInterval; 226 | } 227 | 228 | /** 229 | * sets the retry interval for exponential retry policy used by methods in ADLStoreClient objects 230 | * @param exponentialRetryInterval retry interval for exponential retry policy in milliseconds 231 | * 232 | */ 233 | public ADLStoreOptions setExponentialRetryInterval(int exponentialRetryInterval) { 234 | this.exponentialRetryInterval = exponentialRetryInterval; 235 | return this; 236 | } 237 | 238 | int getExponentialFactor() { 239 | return exponentialFactor; 240 | } 241 | 242 | /** 243 | * sets the factor of backoff for exponential retry policy used by methods in ADLStoreClient objects 244 | * @param exponentialFactor retry backoff factor for exponential retry policy 245 | * 246 | */ 247 | public ADLStoreOptions setExponentialFactor(int exponentialFactor) { 248 | this.exponentialFactor = exponentialFactor; 249 | return this; 250 | } 251 | 252 | /** 253 | * Config to enable create only if conditional delete has passed 254 | * @param enableConditionalCreate boolean value to set 255 | * @return 256 | */ 257 | public ADLStoreOptions setEnableConditionalCreate(boolean enableConditionalCreate) { 258 | this.enableConditionalCreate = enableConditionalCreate; 259 | return this; 260 | } 261 | 262 | boolean shouldEnableConditionalCreate(){ 263 | return this.enableConditionalCreate; 264 | } 265 | 266 | } 267 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/oauth2/DeviceCodeTokenProviderHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.oauth2; 8 | 9 | import com.fasterxml.jackson.core.JsonFactory; 10 | import com.fasterxml.jackson.core.JsonParser; 11 | import com.fasterxml.jackson.core.JsonToken; 12 | import com.microsoft.azure.datalake.store.QueryParams; 13 | import org.slf4j.Logger; 14 | import org.slf4j.LoggerFactory; 15 | 16 | import java.io.IOException; 17 | import java.io.InputStream; 18 | import java.net.HttpURLConnection; 19 | import java.net.URL; 20 | import java.util.Date; 21 | 22 | /** 23 | * Internal use. Contains device code info. 24 | */ 25 | class DeviceCodeInfo { 26 | String usercode; 27 | String verificationUrl; 28 | String message; 29 | String devicecode; 30 | int pollingInterval; 31 | Date expiry; 32 | String clientId; 33 | } 34 | 35 | 36 | /** 37 | * Internal use. Methods to obtain the refresh token using interactive login (using device code AAD flow). 38 | * 39 | */ 40 | class DeviceCodeTokenProviderHelper { 41 | 42 | /* 43 | AAD .net sample: https://azure.microsoft.com/en-us/resources/samples/active-directory-dotnet-deviceprofile/ 44 | How this stuff works: https://developers.google.com/identity/protocols/OAuth2ForDevices?hl=en 45 | */ 46 | 47 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.oauth2.DeviceCodeTokenProvider"); 48 | private static final String defaultAppId = "c8964590-6116-42e6-8a29-ec0865dff3d5"; 49 | private static final String resource = AzureADAuthenticator.resource; 50 | private static final String deviceCodeUrl = "https://login.microsoftonline.com/common/oauth2/devicecode"; 51 | private static final String tokenUrl = "https://login.microsoftonline.com/common/oauth2/token"; 52 | 53 | public static RefreshTokenInfo getRefreshToken(String appId, DeviceCodeCallback callback) throws IOException { 54 | if (appId == null) appId = defaultAppId; 55 | if (callback == null) callback = DeviceCodeCallback.getDefaultInstance(); 56 | 57 | DeviceCodeInfo dcInfo = getDeviceCodeInfo(appId); 58 | log.debug("AADToken: obtained device code, prompting user to login through browser"); 59 | callback.showDeviceCodeMessage(dcInfo); 60 | RefreshTokenInfo token = getTokenFromDeviceCode(dcInfo); 61 | log.debug("AADToken: obtained refresh token from device-code based user login"); 62 | return token; 63 | } 64 | 65 | 66 | private static DeviceCodeInfo getDeviceCodeInfo(String appId) throws IOException { 67 | 68 | QueryParams qp = new QueryParams(); 69 | qp.add("resource", resource); 70 | qp.add("client_id", appId); 71 | String queryString = qp.serialize(); 72 | 73 | URL url = new URL(deviceCodeUrl + "?" + queryString); 74 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 75 | conn.setRequestMethod("GET"); 76 | 77 | DeviceCodeInfo dcInfo = new DeviceCodeInfo(); 78 | dcInfo.clientId = appId; 79 | int httpResponseCode = conn.getResponseCode(); 80 | if (httpResponseCode == 200) { 81 | InputStream httpResponseStream = conn.getInputStream(); 82 | try { 83 | int expiryPeriod = 0; 84 | 85 | JsonFactory jf = new JsonFactory(); 86 | JsonParser jp = jf.createParser(httpResponseStream); 87 | String fieldName, fieldValue; 88 | jp.nextToken(); 89 | while (jp.hasCurrentToken()) { 90 | if (jp.getCurrentToken() == JsonToken.FIELD_NAME) { 91 | fieldName = jp.getCurrentName(); 92 | jp.nextToken(); // field value 93 | fieldValue = jp.getText(); 94 | 95 | if (fieldName.equals("user_code")) dcInfo.usercode = fieldValue; 96 | if (fieldName.equals("device_code")) dcInfo.devicecode = fieldValue; 97 | if (fieldName.equals("verification_url")) dcInfo.verificationUrl = fieldValue; 98 | if (fieldName.equals("message")) dcInfo.message = fieldValue; 99 | if (fieldName.equals("expires_in")) expiryPeriod = Integer.parseInt(fieldValue); 100 | if (fieldName.equals("interval")) dcInfo.pollingInterval = Integer.parseInt(fieldValue); 101 | } 102 | jp.nextToken(); 103 | } 104 | jp.close(); 105 | long expiry = System.currentTimeMillis(); 106 | expiry = expiry + expiryPeriod * 1000L; // convert expiryPeriod to milliseconds and add 107 | dcInfo.expiry = new Date(expiry); 108 | } finally { 109 | httpResponseStream.close(); 110 | } 111 | } else { 112 | String message = "Failed to get device code from AzureAD. Http response: " + httpResponseCode + " " + conn.getResponseMessage(); 113 | log.debug(message); 114 | throw new IOException(message); 115 | } 116 | log.debug("Obtained device code from AAD: " + dcInfo.usercode); 117 | return dcInfo; 118 | } 119 | 120 | private static RefreshTokenInfo getTokenFromDeviceCode(final DeviceCodeInfo dcInfo) throws IOException { 121 | RefreshTokenInfo refreshToken = null; 122 | int sleepDuration = (dcInfo.pollingInterval + 1) * 1000; 123 | while (dcInfo.expiry.getTime() > (new Date()).getTime() && refreshToken == null) { 124 | try { 125 | Thread.sleep(sleepDuration); 126 | refreshToken = getTokenInternal(dcInfo.devicecode, dcInfo.clientId); 127 | } catch (InterruptedException ex) { 128 | Thread.currentThread().interrupt(); // http://www.ibm.com/developerworks/library/j-jtp05236/ 129 | } catch (Exception ex) { 130 | log.debug("Exception getting token from device code " + ex.toString()); 131 | throw ex; 132 | } 133 | } 134 | return refreshToken; 135 | } 136 | 137 | private static RefreshTokenInfo getTokenInternal(final String deviceCode, final String clientId) throws IOException { 138 | QueryParams qp = new QueryParams(); 139 | qp.add("resource", resource); 140 | qp.add("client_id", clientId); 141 | qp.add("grant_type", "device_code"); 142 | qp.add("code", deviceCode); 143 | String bodyString = qp.serialize(); 144 | 145 | URL url = new URL(tokenUrl); 146 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 147 | conn.setRequestMethod("POST"); 148 | conn.setDoOutput(true); 149 | conn.getOutputStream().write(bodyString.getBytes("UTF-8")); 150 | 151 | RefreshTokenInfo token = new RefreshTokenInfo(); 152 | String tokentype = null; 153 | String scope = null; 154 | int httpResponseCode = conn.getResponseCode(); 155 | if (httpResponseCode == 200) { 156 | InputStream httpResponseStream = conn.getInputStream(); 157 | try { 158 | int expiryPeriod = 0; 159 | JsonFactory jf = new JsonFactory(); 160 | JsonParser jp = jf.createParser(httpResponseStream); 161 | String fieldName, fieldValue; 162 | jp.nextToken(); 163 | while (jp.hasCurrentToken()) { 164 | if (jp.getCurrentToken() == JsonToken.FIELD_NAME) { 165 | fieldName = jp.getCurrentName(); 166 | jp.nextToken(); // field value 167 | fieldValue = jp.getText(); 168 | 169 | if (fieldName.equals("token_type")) tokentype = fieldValue; 170 | if (fieldName.equals("scope")) scope = fieldValue; 171 | if (fieldName.equals("expires_in")) expiryPeriod = Integer.parseInt(fieldValue); 172 | if (fieldName.equals("access_token")) token.accessToken = fieldValue; 173 | if (fieldName.equals("refresh_token")) token.refreshToken = fieldValue; 174 | } 175 | jp.nextToken(); 176 | } 177 | jp.close(); 178 | 179 | if (!"Bearer".equals(tokentype) || !"user_impersonation".equals(scope) ) { 180 | throw new IOException("not sure what kind of token we got"); 181 | } 182 | 183 | long expiry = System.currentTimeMillis(); 184 | expiry = expiry + expiryPeriod * 1000L; // convert expiryPeriod to milliseconds and add 185 | token.accessTokenExpiry = new Date(expiry); 186 | return token; 187 | } catch (Exception ex) { 188 | log.debug("Exception retrieving token from AAD response" + ex.toString()); 189 | throw ex; 190 | } finally { 191 | httpResponseStream.close(); 192 | } 193 | } else if (httpResponseCode == 400) { 194 | InputStream httpResponseStream = conn.getErrorStream(); 195 | try { 196 | String error = null; 197 | 198 | JsonFactory jf = new JsonFactory(); 199 | JsonParser jp = jf.createParser(httpResponseStream); 200 | String fieldName, fieldValue; 201 | jp.nextToken(); 202 | while (jp.hasCurrentToken()) { 203 | if (jp.getCurrentToken() == JsonToken.FIELD_NAME) { 204 | fieldName = jp.getCurrentName(); 205 | jp.nextToken(); // field value 206 | fieldValue = jp.getText(); 207 | 208 | if (fieldName.equals("error")) error = fieldValue; 209 | } 210 | jp.nextToken(); 211 | } 212 | jp.close(); 213 | 214 | if (!"authorization_pending".equals(error)) { 215 | String message = "Failed to acquire token from AzureAD. Http response: " + httpResponseCode + " Error: " + error; 216 | log.debug(message); 217 | throw new IOException(message); 218 | } else { 219 | log.debug("polled AAD for token, got authorization_pending (still waiting for user to complete login)"); 220 | } 221 | } finally { 222 | httpResponseStream.close(); 223 | } 224 | } else { 225 | String message = "Failed to acquire token from AzureAD. Http response: " + httpResponseCode + " " + conn.getResponseMessage(); 226 | log.debug(message); 227 | throw new IOException(message); 228 | } 229 | return null; 230 | } 231 | 232 | private static void printDeviceCodeInfo(DeviceCodeInfo dcInfo) { 233 | System.out.println("UserCode: " + dcInfo.usercode); 234 | System.out.println("VerificationUrl: " + dcInfo.verificationUrl); 235 | System.out.println("Polling Interval: " + dcInfo.devicecode); 236 | System.out.println("Expires: " + dcInfo.expiry); 237 | System.out.println("Message: " + dcInfo.message); 238 | System.out.println("Devicecode: " + dcInfo.devicecode); 239 | System.out.println(); 240 | } 241 | } 242 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/acl/AclEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store.acl; 8 | 9 | import java.util.LinkedList; 10 | import java.util.List; 11 | 12 | /** 13 | * Contains one ACL entry. An ACL entry consists of a scope (access or default), 14 | * the type of the ACL (user, group, other or mask), the name of the user or group 15 | * associated with this ACL (can be blank to specify the default permissions for 16 | * users and groups, and must be blank for mask entries), and the action permitted 17 | * by this ACL entry. 18 | *

19 | * An ACL for an object consists of a {@code List} of Acl entries. 20 | *

21 | *

22 | * This class also provides a number of convenience methods for converting ACL entries 23 | * and ACLs to and back from strings. 24 | *

25 | */ 26 | public class AclEntry { 27 | /** 28 | * {@link AclScope} specifying scope of the Acl entry (access or default) 29 | */ 30 | public AclScope scope; 31 | 32 | /** 33 | * {@link AclType} specifying the type of the Acl entry (user, group, other or mask) 34 | */ 35 | public AclType type; 36 | 37 | /** 38 | * String specifying the name of the user or group associated with this Acl entry. Can be 39 | * blank to specify the default permissions for users and groups, and must be blank 40 | * for mask entries. 41 | */ 42 | public String name; 43 | 44 | 45 | /** 46 | * {@link AclAction} enum specifying the action permitted by this Acl entry. Has 47 | * convenience methods to create value from unix-style permission string (e.g., 48 | * {@code AclAction.fromRwx("rw-")}), or from unix Octal permission (e.g., 49 | * {@code AclAction.fromOctal(6)}). 50 | */ 51 | public AclAction action; 52 | 53 | public AclEntry() { 54 | 55 | } 56 | 57 | /** 58 | * creates and Acl Entry from the supplied scope, type, name and action 59 | * @param scope {@link AclScope} specifying scope of the Acl entry (access or default) 60 | * @param type {@link AclType} specifying the type of the Acl entry (user, group, other or mask) 61 | * @param name String specifying the name of the user or group associated with this Acl entry 62 | * @param action {@link AclAction} specifying the action permitted by this Acl entry 63 | */ 64 | public AclEntry(AclScope scope, AclType type, String name, AclAction action) { 65 | if (scope == null) throw new IllegalArgumentException("AclScope is null"); 66 | if (type == null ) throw new IllegalArgumentException("AclType is null"); 67 | if (type == AclType.MASK && name != null && !name.trim().equals("")) 68 | throw new IllegalArgumentException("mask should not have user/group name"); 69 | if (type == AclType.OTHER && name != null && !name.trim().equals("")) 70 | throw new IllegalArgumentException("ACL entry type 'other' should not have user/group name"); 71 | 72 | this.scope = scope; 73 | this.type = type; 74 | this.name = name; 75 | this.action = action; 76 | } 77 | 78 | /** 79 | * Parses an Acl entry from its posix string form. For example: {@code "default:user:bob:r-x"} 80 | * @param entryString Acl entry string to parse 81 | * 82 | * @return {@link AclEntry} parsed from the string 83 | * 84 | * @throws IllegalArgumentException if the string is invalid 85 | */ 86 | public static AclEntry parseAclEntry(String entryString) throws IllegalArgumentException { 87 | return parseAclEntry(entryString, false); 88 | } 89 | 90 | /** 91 | * Parses a single Acl entry from its posix string form. For example: {@code "default:user:bob:r-x"}. 92 | *

93 | * If the Acl string will be used to remove an existing acl from a file or folder, then the permission 94 | * level does not need to be specified. Passing false to the {@code removeAcl} argument tells the parser 95 | * to accept such strings. 96 | *

97 | * @param entryString Acl entry string to parse 98 | * @param removeAcl boolean specifying whether to parse this string like a remove Acl (that is, with 99 | * permission optional) 100 | * 101 | * @return {@link AclEntry} parsed from the string 102 | * 103 | * @throws IllegalArgumentException if the string is invalid 104 | */ 105 | public static AclEntry parseAclEntry(String entryString, boolean removeAcl) throws IllegalArgumentException { 106 | if (entryString == null || entryString.equals("")) return null; 107 | AclEntry aclEntry = new AclEntry(); 108 | String aclString = entryString.trim(); 109 | 110 | // check if this is default ACL 111 | int fColonPos = aclString.indexOf(":"); 112 | String firstToken = aclString.substring(0, fColonPos).toLowerCase().trim(); 113 | if (firstToken.equals("default")) { 114 | aclEntry.scope = com.microsoft.azure.datalake.store.acl.AclScope.DEFAULT; 115 | aclString = aclString.substring(fColonPos+1); 116 | } else { 117 | aclEntry.scope = com.microsoft.azure.datalake.store.acl.AclScope.ACCESS; 118 | } 119 | 120 | // remaining string should have 3 entries (or 2 for removeacl) 121 | String[] parts = aclString.split(":", 5); // without the 5, java discards trailing empty strings 122 | if (parts.length <2 || parts.length >3) throw new IllegalArgumentException("invalid aclEntryString " + entryString); 123 | if (parts.length == 2 && !removeAcl) throw new IllegalArgumentException("invalid aclEntryString " + entryString); 124 | 125 | // entry type (user/group/other/mask) 126 | try { 127 | aclEntry.type = AclType.valueOf(parts[0].toUpperCase().trim()); 128 | } catch (IllegalArgumentException ex) { 129 | throw new IllegalArgumentException("Invalid ACL AclType in " + entryString); 130 | } catch (NullPointerException ex) { 131 | throw new IllegalArgumentException("ACL Entry AclType missing in " + entryString); 132 | } 133 | 134 | // user/group name 135 | aclEntry.name = parts[1].trim(); 136 | if (aclEntry.type == AclType.MASK && !aclEntry.name.equals("")) 137 | throw new IllegalArgumentException("mask entry cannot contain user/group name: " + entryString); 138 | if (aclEntry.type == AclType.OTHER && !aclEntry.name.equals("")) 139 | throw new IllegalArgumentException("entry of type 'other' should not contain user/group name: " + entryString); 140 | 141 | // permission (rwx) 142 | if (!removeAcl) { 143 | try { 144 | aclEntry.action = AclAction.fromRwx(parts[2]); 145 | } catch (IllegalArgumentException ex) { 146 | throw new IllegalArgumentException("Invalid ACL action in " + entryString); 147 | } catch (NullPointerException ex) { 148 | throw new IllegalArgumentException("ACL action missing in " + entryString); 149 | } 150 | } 151 | return aclEntry; 152 | } 153 | 154 | /** 155 | * Returns the posix string form of this acl entry. For example: {@code "default:user:bob:r-x"}. 156 | 157 | * @return the posix string form of this acl entry 158 | */ 159 | public String toString() { 160 | return this.toString(false); 161 | } 162 | 163 | /** 164 | * Returns the posix string form of this acl entry. For example: {@code "default:user:bob:r-x"}. 165 | *

166 | * If the Acl string will be used to remove an existing acl from a file or folder, then the permission 167 | * level does not need to be specified. Passing true to the {@code removeAcl} argument omits the permission 168 | * level in the output string. 169 | *

170 | * 171 | * @param removeAcl passing true will result in an acl entry string with no permission specified 172 | * 173 | * @return the string form of the {@code AclEntry} 174 | */ 175 | public String toString(boolean removeAcl) { 176 | StringBuilder str = new StringBuilder(); 177 | if (this.scope == null) throw new IllegalArgumentException("Acl Entry has no scope"); 178 | if (this.type == null) throw new IllegalArgumentException("Acl Entry has no type"); 179 | 180 | if (this.scope == com.microsoft.azure.datalake.store.acl.AclScope.DEFAULT) str.append("default:"); 181 | 182 | str.append(this.type.toString().toLowerCase()); 183 | str.append(":"); 184 | 185 | if (this.name!=null) str.append(this.name); 186 | 187 | if (this.action != null && !removeAcl) { 188 | str.append(":"); 189 | str.append(this.action.toString()); 190 | } 191 | return str.toString(); 192 | } 193 | 194 | /** 195 | * parses a posix acl spec string into a {@link List} of {@code AclEntry}s. 196 | * @param aclString the acl spec string tp parse 197 | * 198 | * @return {@link List}{@code } represented by the acl spec string 199 | * 200 | */ 201 | public static List parseAclSpec(String aclString) throws IllegalArgumentException { 202 | if (aclString == null || aclString.trim().equals("")) return new LinkedList(); 203 | 204 | aclString = aclString.trim(); 205 | String car, // the first entry 206 | cdr; // the rest of the list after first entry 207 | int commaPos = aclString.indexOf(","); 208 | if (commaPos < 0) { 209 | car = aclString; 210 | cdr = null; 211 | } else { 212 | car = aclString.substring(0, commaPos).trim(); 213 | cdr = aclString.substring(commaPos+1); 214 | } 215 | LinkedList aclSpec = (LinkedList) parseAclSpec(cdr); 216 | if (!car.equals("")) { 217 | aclSpec.addFirst(parseAclEntry(car)); 218 | } 219 | return aclSpec; 220 | } 221 | 222 | /** 223 | * converts a {@link List}{@code } to its posix aclspec string form 224 | * @param list {@link List}{@code } to covert to string 225 | * 226 | * @return posix acl spec string 227 | */ 228 | public static String aclListToString(List list) { 229 | return aclListToString(list, false); 230 | } 231 | 232 | /** 233 | * converts a {@link List}{@code } to its posix aclspec string form. 234 | *

235 | * If the aclspec string will be used to remove an existing acl from a file or folder, then the permission 236 | * level does not need to be specified. Passing true to the {@code removeAcl} argument omits the permission 237 | * level in the output string. 238 | *

239 | * 240 | * @param list {@link List}{@code } to covert to string 241 | * @param removeAcl removeAcl passing true will result in an aclspec with no permission specified 242 | * 243 | * @return posix acl spec string 244 | */ 245 | public static String aclListToString(List list, boolean removeAcl) { 246 | if (list == null || list.size() == 0) return ""; 247 | String separator = ""; 248 | StringBuilder output = new StringBuilder(); 249 | 250 | for (AclEntry entry : list) { 251 | output.append(separator); 252 | output.append(entry.toString(removeAcl)); 253 | separator = ","; 254 | } 255 | return output.toString(); 256 | } 257 | } 258 | -------------------------------------------------------------------------------- /src/main/java/com/microsoft/azure/datalake/store/ADLFileOutputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.microsoft.azure.datalake.store; 8 | 9 | import com.microsoft.azure.datalake.store.retrypolicies.ExponentialBackoffPolicy; 10 | import org.slf4j.Logger; 11 | import org.slf4j.LoggerFactory; 12 | 13 | import java.io.IOException; 14 | import java.io.OutputStream; 15 | import java.util.UUID; 16 | 17 | /** 18 | * {@code ADLFileOutputStream} is used to add data to an Azure Data Lake File. 19 | * It is a buffering stream that accumulates user writes, and then writes to the server 20 | * in chunks. Default chunk size is 4MB. 21 | * 22 | *

Thread-safety: Note that methods in this class are NOT thread-safe.

23 | * 24 | */ 25 | public class ADLFileOutputStream extends OutputStream { 26 | 27 | private static final Logger log = LoggerFactory.getLogger("com.microsoft.azure.datalake.store.ADLFileOutputStream"); 28 | 29 | private final String filename; 30 | private final ADLStoreClient client; 31 | private final boolean isCreate; 32 | private final String leaseId; 33 | 34 | private int blocksize = 4 * 1024 *1024; // default buffer size of 4MB 35 | private byte[] buffer = null; // will be initialized at first use 36 | 37 | private int cursor = 0; 38 | private long remoteCursor = 0; 39 | private boolean streamClosed = false; 40 | private boolean lastFlushUpdatedMetadata = false; 41 | 42 | // package-private constructor - use Factory Method in AzureDataLakeStoreClient 43 | ADLFileOutputStream(String filename, 44 | ADLStoreClient client, 45 | boolean isCreate, 46 | String leaseId) throws IOException { 47 | this.filename = filename; 48 | this.client = client; 49 | this.isCreate = isCreate; 50 | this.leaseId = (leaseId == null)? UUID.randomUUID().toString() : leaseId; 51 | 52 | if (!isCreate) initializeAppendStream(); 53 | 54 | if (log.isTraceEnabled()) { 55 | log.trace("ADLFIleOutputStream created for client {} for file {}, create={}", client.getClientId(), filename, isCreate); 56 | } 57 | } 58 | 59 | private void initializeAppendStream() throws IOException { 60 | boolean append0succeeded = doZeroLengthAppend(-1); // do 0-length append with sync flag to update length 61 | if (!append0succeeded) { 62 | throw new IOException("Error doing 0-length append for append stream for file " + filename); 63 | } 64 | DirectoryEntry dirent = client.getDirectoryEntry(filename); 65 | if (dirent != null) { 66 | remoteCursor = dirent.length; 67 | } else { 68 | throw new IOException("Failure getting directoryEntry during append stream creation for file " + filename); 69 | } 70 | } 71 | 72 | @Override 73 | public void write(int b) throws IOException { 74 | byte buf[] = new byte[1]; 75 | buf[0] = (byte) b; 76 | write(buf, 0, 1); 77 | } 78 | 79 | @Override 80 | public void write(byte[] b) throws IOException { 81 | if (b == null) return; 82 | write(b, 0, b.length); 83 | } 84 | 85 | @Override 86 | public void write(byte[] b, int off, int len) throws IOException { 87 | if (streamClosed) throw new IOException("attempting to write to a closed stream;"); 88 | if (b == null) { 89 | throw new NullPointerException(); 90 | } 91 | if ((off < 0) || (off > b.length) || (len < 0) || 92 | ((off + len) > b.length) || ((off + len) < 0)) { 93 | throw new IndexOutOfBoundsException(); 94 | } 95 | if (len == 0) { 96 | return; 97 | } 98 | 99 | if (off > b.length || len > (b.length - off)) throw new IllegalArgumentException("array offset and length are > array size"); 100 | 101 | if (log.isTraceEnabled()) { 102 | log.trace("Stream write of size {} for client {} for file {}", len, client.getClientId(), filename); 103 | } 104 | 105 | if (buffer == null) buffer = new byte[blocksize]; 106 | 107 | // if len > 4MB, then we force-break the write into 4MB chunks 108 | while (len > blocksize) { 109 | flush(SyncFlag.DATA); // flush first, because we want to preserve record 110 | // boundary of last append 111 | addToBuffer(b, off, blocksize); 112 | off += blocksize; 113 | len -= blocksize; 114 | } 115 | // now len == the remaining length 116 | 117 | //if adding this to buffer would overflow buffer, then flush buffer first 118 | if (len > buffer.length - cursor) { 119 | flush(SyncFlag.DATA); 120 | } 121 | // now we know b will fit in remaining buffer, so just add it in 122 | addToBuffer(b, off, len); 123 | } 124 | 125 | 126 | 127 | private void addToBuffer(byte[] b, int off, int len) { 128 | if (len > buffer.length - cursor) { // if requesting to copy more than remaining space in buffer 129 | throw new IllegalArgumentException("invalid buffer copy requested in addToBuffer"); 130 | } 131 | System.arraycopy(b, off, buffer, cursor, len); 132 | cursor += len; 133 | } 134 | 135 | @Override 136 | public void flush() throws IOException { 137 | flush(SyncFlag.METADATA); 138 | } 139 | 140 | private void flush(SyncFlag syncFlag) throws IOException { 141 | if (log.isTraceEnabled()) { 142 | log.trace("flush() with data size {} at offset {} for client {} for file {}", cursor, remoteCursor, client.getClientId(), filename); 143 | } 144 | // Ignoring this, because HBase actually calls flush after close() 145 | if (streamClosed) return; 146 | if (cursor == 0 && (syncFlag==SyncFlag.DATA)) return; // nothing to flush 147 | if (cursor == 0 && lastFlushUpdatedMetadata && syncFlag == SyncFlag.METADATA) return; // do not send a 148 | // flush if the last flush updated metadata and there is no data 149 | if (cursor == 0 && syncFlag == SyncFlag.CLOSE) // For no data and syncflag.close just do close and return 150 | { 151 | closehandle(); 152 | return; 153 | } 154 | if (buffer == null) buffer = new byte[blocksize]; 155 | SyncFlag origsyncFlag = syncFlag; 156 | if(syncFlag==SyncFlag.CLOSE) // syncflag close- we will do a close separately 157 | { 158 | syncFlag = SyncFlag.DATA; 159 | origsyncFlag=SyncFlag.CLOSE; 160 | } 161 | RequestOptions opts = new RequestOptions(); 162 | opts.retryPolicy = client.makeExponentialBackoffPolicy(); 163 | opts.timeout = client.timeout + (1000 + (buffer.length / 1000 / 1000)); // 1 second grace per MB to upload 164 | OperationResponse resp = new OperationResponse(); 165 | Core.append(filename, remoteCursor, buffer, 0, cursor, leaseId, 166 | leaseId, syncFlag, client, opts, resp); 167 | if (!resp.successful) { 168 | if (resp.numRetries > 0 && resp.httpResponseCode == 400 && "BadOffsetException".equals(resp.remoteExceptionName)) { 169 | // if this was a retry and we get bad offset, then this might be because we got a transient 170 | // failure on first try, but request succeeded on back-end. In that case, the retry would fail 171 | // with bad offset. To detect that, we check if there was a retry done, and if the current error we 172 | // have is bad offset. 173 | // If so, do a zero-length append at the current expected Offset, and if that succeeds, 174 | // then the file length must be good - swallow the error. If this append fails, then the last append 175 | // did not succeed and we have some other offset on server - bubble up the error. 176 | long expectedRemoteLength = remoteCursor + cursor; 177 | boolean append0Succeeded = doZeroLengthAppend(expectedRemoteLength); 178 | if (append0Succeeded) { 179 | log.debug("zero-length append succeeded at expected offset (" + expectedRemoteLength + "), " + 180 | " ignoring BadOffsetException for session: " + leaseId + ", file: " + filename); 181 | remoteCursor += cursor; 182 | cursor = 0; 183 | lastFlushUpdatedMetadata = false; 184 | if(origsyncFlag == SyncFlag.CLOSE){ 185 | closehandle(); 186 | } 187 | return; 188 | } else { 189 | log.debug("Append failed at expected offset(" + expectedRemoteLength + 190 | "). Bubbling exception up for session: " + leaseId + ", file: " + filename); 191 | } 192 | } 193 | throw client.getExceptionFromResponse(resp, "Error appending to file " + filename); 194 | } 195 | remoteCursor += cursor; 196 | cursor = 0; 197 | lastFlushUpdatedMetadata = (syncFlag != SyncFlag.DATA); 198 | if(origsyncFlag == SyncFlag.CLOSE){ 199 | closehandle(); 200 | } 201 | } 202 | 203 | private void closehandle() throws IOException{ 204 | RequestOptions opts = new RequestOptions(); 205 | opts.retryPolicy = client.makeExponentialBackoffPolicy(); 206 | opts.timeout = client.timeout; 207 | OperationResponse resp = new OperationResponse(); 208 | Core.append(filename, remoteCursor, null, 0, 0, leaseId, 209 | leaseId, SyncFlag.CLOSE, client, opts, resp); 210 | if (!resp.successful) { 211 | // Ignore lease conflict problem for retries and also bad offset. 212 | if (!(resp.numRetries > 0 && resp.httpResponseCode == 400 && (resp.remoteExceptionMessage.contains("83090a16") || "BadOffsetException".equals(resp.remoteExceptionName)))) { 213 | throw client.getExceptionFromResponse(resp, "Error closing file " + filename); 214 | } 215 | } 216 | } 217 | private boolean doZeroLengthAppend(long offset) throws IOException { 218 | RequestOptions opts = new RequestOptions(); 219 | opts.retryPolicy = client.makeExponentialBackoffPolicy(); 220 | OperationResponse resp = new OperationResponse(); 221 | Core.append(filename, offset, null, 0, 0, leaseId, leaseId, SyncFlag.METADATA, 222 | client, opts, resp); 223 | return resp.successful; 224 | } 225 | 226 | /** 227 | * Sets the size of the internal write buffer (default is 4MB). 228 | * 229 | * @param newSize requested size of buffer 230 | * @throws IOException throws {@link ADLException} if there is an error 231 | */ 232 | public void setBufferSize(int newSize) throws IOException { 233 | if (newSize <=0) throw new IllegalArgumentException("Buffer size cannot be zero or less: " + newSize); 234 | if (newSize == blocksize) return; // nothing to do 235 | 236 | if (cursor != 0) { // if there's data in the buffer then flush it first 237 | flush(SyncFlag.DATA); 238 | } 239 | blocksize = newSize; 240 | buffer = null; 241 | } 242 | 243 | @Override 244 | public void close() throws IOException { 245 | if(streamClosed) return; // Return silently upon multiple closes 246 | flush(SyncFlag.CLOSE); 247 | streamClosed = true; 248 | buffer = null; // release byte buffer so it can be GC'ed even if app continues to hold reference to stream 249 | if (log.isTraceEnabled()) { 250 | log.trace("Stream closed for client {} for file {}", client.getClientId(), filename); 251 | } 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/mocktests/TestSdkMock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.mocktests; 8 | 9 | 10 | import com.contoso.helpers.HelperUtils; 11 | import com.microsoft.azure.datalake.store.*; 12 | import okhttp3.mockwebserver.MockResponse; 13 | import okhttp3.mockwebserver.MockWebServer; 14 | import okhttp3.mockwebserver.QueueDispatcher; 15 | import org.junit.After; 16 | import org.junit.Assume; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import static org.junit.Assert.*; 20 | 21 | import java.util.HashSet; 22 | import java.util.List; 23 | import java.io.IOException; 24 | import java.util.Properties; 25 | 26 | public class TestSdkMock { 27 | private String directory = "/mockTestDirectory"; 28 | private ADLStoreClient client = null; 29 | private boolean testsEnabled = true; 30 | private boolean longRunningEnabled = true; 31 | MockWebServer server = null; 32 | 33 | @Before 34 | public void setup() throws IOException { 35 | Properties prop = HelperUtils.getProperties(); 36 | 37 | server = new MockWebServer(); 38 | QueueDispatcher dispatcher = new QueueDispatcher(); 39 | dispatcher.setFailFast(new MockResponse().setResponseCode(400)); 40 | server.setDispatcher(dispatcher); 41 | server.start(); 42 | String accountFQDN = server.getHostName() + ":" + server.getPort(); 43 | String dummyToken = "testDummyAadToken"; 44 | 45 | client = ADLStoreClient.createClient(accountFQDN, dummyToken); 46 | client.setOptions(new ADLStoreOptions().setInsecureTransport()); 47 | 48 | testsEnabled = Boolean.parseBoolean(prop.getProperty("MockTestsEnabled", "true")); 49 | longRunningEnabled = Boolean.parseBoolean(prop.getProperty("LongRunningTestsEnabled", "true")); 50 | } 51 | 52 | @After 53 | public void teardown() throws IOException { 54 | server.shutdown(); 55 | } 56 | 57 | @Test 58 | public void testExponentialRetryTiming() throws IOException { 59 | Assume.assumeTrue(testsEnabled); 60 | Assume.assumeTrue(longRunningEnabled); 61 | String filename = directory + "/" + "Mock.testExponentialRetryTiming"; 62 | 63 | MockResponse gfsResponse = (new MockResponse()).setResponseCode(503); 64 | server.enqueue(gfsResponse); // original failure 65 | server.enqueue(gfsResponse); // first retry 66 | server.enqueue(gfsResponse); // second retry 67 | server.enqueue(gfsResponse); // third retry 68 | server.enqueue(gfsResponse); // fourth retry 69 | 70 | long start = System.currentTimeMillis(); 71 | try { 72 | client.getDirectoryEntry(filename); 73 | fail("should have received 503 exception"); 74 | } catch (ADLException ex) { 75 | assertTrue("error should be 503", ex.httpResponseCode == 503); 76 | assertTrue("retries should be 4", ex.numRetries == 4); 77 | } 78 | long end = System.currentTimeMillis(); 79 | 80 | long duration = end - start; 81 | long expectedDuration = 85 * 1000; 82 | long expectedDurationMin = expectedDuration - 1000; 83 | long expectedDurationMax = expectedDuration + 1000; 84 | 85 | assertTrue("Total time was more than max duration expected", expectedDuration < expectedDurationMax); 86 | assertTrue("Total time was less than min duration expected", expectedDuration > expectedDurationMin); 87 | } 88 | 89 | @Test 90 | public void test500Then200Pattern() throws IOException { 91 | Assume.assumeTrue(testsEnabled); 92 | String filename = directory + "/" + "Mock.test500Then200Pattern"; 93 | 94 | server.enqueue(new MockResponse().setResponseCode(200)); 95 | ADLFileOutputStream os = client.createFile(filename, IfExists.OVERWRITE); 96 | String s = "Test string with data\n"; 97 | byte[] data = s.getBytes(); 98 | 99 | server.enqueue(new MockResponse().setResponseCode(200)); // an append that succeeds 100 | os.write(data); 101 | os.flush(); 102 | 103 | server.enqueue(new MockResponse().setResponseCode(500)); // second append that should fail 104 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed on retry 105 | try { 106 | os.write(data); 107 | os.flush(); 108 | } catch (IOException ex) { 109 | fail("should not get here - request should succeed on retry"); 110 | } 111 | 112 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed for the 0-length append 113 | os.close(); 114 | } 115 | 116 | @Test 117 | public void test500Then200PatternWithAtomicCreate() throws IOException { 118 | Assume.assumeTrue(testsEnabled); 119 | String filename = directory + "/" + "Mock.test500Then200Pattern"; 120 | ADLStoreOptions o = new ADLStoreOptions(); 121 | o.setEnableConditionalCreate(true); 122 | client.setOptions(o); 123 | server.enqueue(new MockResponse().setResponseCode(404).setBody("{\"RemoteException\":{\"exception\":\"FileNotFoundException\",\"message\":\"File/Folder does not exist: /Test [874d24ca-dcb3-4b3c-9027-5337e9fdbeee][2019-06-20T12:16:48.2506507-07:00]\",\"javaClassName\":\"java.io.FileNotFoundException\"}}")); // the first empty CREATE request 124 | server.enqueue(new MockResponse().setResponseCode(200)); 125 | ADLFileOutputStream os = client.createFile(filename, IfExists.OVERWRITE); 126 | String s = "Test string with data\n"; 127 | byte[] data = s.getBytes(); 128 | 129 | server.enqueue(new MockResponse().setResponseCode(200)); // an append that succeeds 130 | os.write(data); 131 | os.flush(); 132 | 133 | server.enqueue(new MockResponse().setResponseCode(500)); // second append that should fail 134 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed on retry 135 | try { 136 | os.write(data); 137 | os.flush(); 138 | } catch (IOException ex) { 139 | fail("should not get here - request should succeed on retry"); 140 | } 141 | 142 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed for the 0-length append 143 | os.close(); 144 | o.setEnableConditionalCreate(false); 145 | client.setOptions(o); 146 | } 147 | 148 | 149 | @Test 150 | public void testConnectionReset() throws IOException { 151 | Assume.assumeTrue(false); // test half-done 152 | String filename = directory + "/" + "Mock.testConnectionReset"; 153 | 154 | server.enqueue(new MockResponse().setResponseCode(200)); // the first empty CREATE request 155 | ADLFileOutputStream os = client.createFile(filename, IfExists.OVERWRITE); 156 | String s = "Test string with data\n"; 157 | byte[] data = s.getBytes(); 158 | 159 | server.enqueue(new MockResponse().setResponseCode(200)); // an append that succeeds 160 | os.write(data); 161 | os.flush(); 162 | 163 | server.enqueue(new MockResponse().setResponseCode(500)); // second append that should fail 164 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed on retry 165 | //new MockResponse().setSocketPolicy(SocketPolicy.DISCONNECT_AFTER_REQUEST); 166 | 167 | try { 168 | os.write(data); 169 | os.flush(); 170 | } catch (IOException ex) { 171 | fail("should not get here - request should succeed on retry"); 172 | } 173 | 174 | server.enqueue(new MockResponse().setResponseCode(200)); // succeed for the 0-length append 175 | os.close(); 176 | } 177 | 178 | @Test 179 | public void testListStatusWithArrayInResponse() throws IOException { 180 | String liststatusResponse = "{\"FileStatuses\":{\"continuationToken\":\"\",\"FileStatus\":[{\"length\":0,\"pathSuffix\":\"Test01\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test02\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[\"Share\",\"PartOfShare\"]}]}}"; 181 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse)); 182 | List entries=client.enumerateDirectory("/TestShare"); 183 | HashSet hset = new HashSet(); 184 | for (DirectoryEntry entry : entries) { 185 | hset.add(entry.fullName); 186 | } 187 | assertTrue(hset.size() == 2); 188 | assertTrue(hset.contains("/TestShare/Test01")); 189 | assertTrue(hset.contains("/TestShare/Test02")); 190 | } 191 | @Test 192 | public void testListStatusWithMultipleArrayInResponse() throws IOException { 193 | String liststatusResponse = "{\"FileStatuses\":{\"continuationToken\":\"\",\"FileStatus\":[{\"length\":0,\"pathSuffix\":\"Test01\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test02\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[[\"Share\",\"Share1\"],[\"PartOfShare\"]]}]}}"; 194 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse)); 195 | List entries=client.enumerateDirectory("/TestShare"); 196 | HashSet hset = new HashSet(); 197 | for (DirectoryEntry entry : entries) { 198 | hset.add(entry.fullName); 199 | } 200 | assertTrue(hset.size() == 2); 201 | assertTrue(hset.contains("/TestShare/Test01")); 202 | assertTrue(hset.contains("/TestShare/Test02")); 203 | } 204 | 205 | @Test 206 | public void testlistStatusWithTokenAtEnd() throws IOException { 207 | String liststatusResponse1 = "{\"FileStatuses\":" + 208 | "{" + 209 | "\"FileStatus\":" + 210 | "[" + 211 | "{\"length\":0,\"pathSuffix\":\"Test01\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test02\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[[\"Share\",\"Share1\"],[\"PartOfShare\"]]" + "}" + 212 | "]" + 213 | "," + 214 | "\"continuationToken\":\"hasToken\"" + 215 | "}" + 216 | "}"; 217 | String liststatusResponse2 = "{\"FileStatuses\":" + 218 | "{" + 219 | "\"FileStatus\":" + 220 | "[" + 221 | "{\"length\":0,\"pathSuffix\":\"Test03\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test04\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[[\"Share\",\"Share1\"],[\"PartOfShare\"]]" + "}" + 222 | "]" + 223 | "," + 224 | "\"continuationToken\":\"\"" + 225 | "}" + 226 | "}"; 227 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse1)); 228 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse2)); 229 | 230 | List entries = client.enumerateDirectory("/TestShare"); 231 | HashSet hset = new HashSet(); 232 | 233 | for (DirectoryEntry entry : entries) { 234 | hset.add(entry.fullName); 235 | } 236 | 237 | assertTrue(hset.size() == 4); 238 | assertTrue(hset.contains("/TestShare/Test01")); 239 | assertTrue(hset.contains("/TestShare/Test02")); 240 | assertTrue(hset.contains("/TestShare/Test03")); 241 | assertTrue(hset.contains("/TestShare/Test04")); 242 | } 243 | 244 | @Test 245 | public void testlistStatusWithTokenAtStart() throws IOException { 246 | String liststatusResponse1 = "{\"FileStatuses\":" + 247 | "{" + 248 | "\"continuationToken\":\"hasToken\"" + 249 | "," + 250 | "\"FileStatus\":" + 251 | "[" + 252 | "{\"length\":0,\"pathSuffix\":\"Test01\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test02\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[[\"Share\",\"Share1\"],[\"PartOfShare\"]]" + "}" + 253 | "]" + 254 | "}" + 255 | "}"; 256 | String liststatusResponse2 = "{\"FileStatuses\":" + 257 | "{" + 258 | "\"continuationToken\":\"\"" + 259 | "," + 260 | "\"FileStatus\":" + 261 | "[" + 262 | "{\"length\":0,\"pathSuffix\":\"Test03\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1528320290048,\"modificationTime\":1528320362596,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner1\",\"group\":\"ownergroup1\",\"aclBit\":true},{\"length\":0,\"pathSuffix\":\"Test04\",\"type\":\"DIRECTORY\",\"blockSize\":0,\"accessTime\":1531515372559,\"modificationTime\":1531523888360,\"replication\":0,\"permission\":\"770\",\"owner\":\"owner2\",\"group\":\"ownergroup2\",\"aclBit\":true,\"attributes\":[[\"Share\",\"Share1\"],[\"PartOfShare\"]]" + "}" + 263 | "]" + 264 | "}" + 265 | "}"; 266 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse1)); 267 | server.enqueue(new MockResponse().setResponseCode(200).setBody(liststatusResponse2)); 268 | 269 | List entries = client.enumerateDirectory("/TestShare"); 270 | HashSet hset = new HashSet(); 271 | 272 | for (DirectoryEntry entry : entries) { 273 | hset.add(entry.fullName); 274 | } 275 | 276 | assertTrue(hset.size() == 4); 277 | assertTrue(hset.contains("/TestShare/Test01")); 278 | assertTrue(hset.contains("/TestShare/Test02")); 279 | assertTrue(hset.contains("/TestShare/Test03")); 280 | assertTrue(hset.contains("/TestShare/Test04")); 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /src/test/java/com/contoso/liveservicetests/TestCore.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) Microsoft Corporation. All rights reserved. 3 | * Licensed under the MIT License. 4 | * See License.txt in the project root for license information. 5 | */ 6 | 7 | package com.contoso.liveservicetests; 8 | 9 | import com.contoso.helpers.HelperUtils; 10 | import com.microsoft.azure.datalake.store.*; 11 | import com.microsoft.azure.datalake.store.oauth2.AzureADAuthenticator; 12 | import com.microsoft.azure.datalake.store.oauth2.AzureADToken; 13 | 14 | import com.microsoft.azure.datalake.store.retrypolicies.NonIdempotentRetryPolicy; 15 | import org.junit.AfterClass; 16 | import org.junit.Assume; 17 | import org.junit.BeforeClass; 18 | import org.junit.Test; 19 | 20 | import static org.junit.Assert.*; 21 | 22 | import java.io.ByteArrayOutputStream; 23 | import java.io.IOException; 24 | import java.io.InputStream; 25 | import java.util.Arrays; 26 | import java.util.Properties; 27 | import java.util.UUID; 28 | 29 | 30 | 31 | public class TestCore { 32 | 33 | final UUID instanceGuid = UUID.randomUUID(); 34 | static Properties prop = null; 35 | static AzureADToken aadToken = null; 36 | static String directory = null; 37 | static ADLStoreClient client = null; 38 | static boolean testsEnabled = true; 39 | 40 | 41 | @BeforeClass 42 | public static void setup() throws IOException { 43 | prop = HelperUtils.getProperties(); 44 | aadToken = AzureADAuthenticator.getTokenUsingClientCreds(prop.getProperty("OAuth2TokenUrl"), 45 | prop.getProperty("ClientId"), 46 | prop.getProperty("ClientSecret") ); 47 | UUID guid = UUID.randomUUID(); 48 | directory = "/" + prop.getProperty("dirName") + "/" + UUID.randomUUID(); 49 | String account = prop.getProperty("StoreAcct") + ".azuredatalakestore.net"; 50 | client = ADLStoreClient.createClient(account, aadToken); 51 | testsEnabled = Boolean.parseBoolean(prop.getProperty("CoreTestsEnabled", "true")); 52 | } 53 | 54 | @AfterClass 55 | public static void teardown() throws IOException { 56 | client.deleteRecursive(directory); 57 | } 58 | 59 | /* 60 | Create tests 61 | */ 62 | 63 | @Test 64 | public void createSmallFileWithOverWrite() throws IOException { 65 | Assume.assumeTrue(testsEnabled); 66 | String filename = directory + "/" + "Core.CreateSmallFileWithOverWrite.txt"; 67 | System.out.println("Running createSmallFileWithOverWrite"); 68 | 69 | byte [] contents = HelperUtils.getSampleText1(); 70 | putFileContents(filename, contents, true); 71 | 72 | byte[] b = getFileContents(filename, contents.length * 2); 73 | assertTrue("file contents should match", Arrays.equals(b, contents)); 74 | assertTrue("file length should match what was written", b.length == contents.length); 75 | } 76 | 77 | @Test 78 | public void createSmallFileWithNoOverwrite() throws IOException { 79 | Assume.assumeTrue(testsEnabled); 80 | String filename = directory + "/" + "Core.CreateSmallFileWithNoOverwrite.txt"; 81 | System.out.println("Running createSmallFileWithNoOverwrite"); 82 | 83 | byte [] contents = HelperUtils.getSampleText1(); 84 | putFileContents(filename, contents, false); 85 | 86 | byte[] b = getFileContents(filename, contents.length * 2); 87 | assertTrue("file length should match what was written", b.length == contents.length); 88 | assertTrue("file contents should match", Arrays.equals(b, contents)); 89 | } 90 | 91 | 92 | @Test 93 | public void createEmptyFileWithConcurrentAppend() throws IOException { 94 | Assume.assumeTrue(false); // pending change to server behavior 95 | String filename = directory + "/" + "Core.createEmptyFileWithConcurrentAppend.txt"; 96 | System.out.println("Running createEmptyFileWithConcurrentAppend"); 97 | 98 | byte [] contents = null; 99 | RequestOptions opts = new RequestOptions(); 100 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 101 | OperationResponse resp = new OperationResponse(); 102 | Core.concurrentAppend(filename, contents, 0, 0, true, 103 | client, opts, resp); 104 | if (!resp.successful) { 105 | throw client.getExceptionFromResponse(resp, "Error in ConcurrentAppend with null content " + filename); 106 | } 107 | 108 | DirectoryEntry de = dir(filename); 109 | assertTrue("File type should be FILE", de.type == DirectoryEntryType.FILE); 110 | assertTrue("File length in DirectoryEntry should be 0 for null-content file", de.length == 0); 111 | 112 | byte[] b = getFileContents(filename, contents.length * 2); 113 | assertTrue("file length should be 0 for null content", b.length == 0); 114 | } 115 | 116 | 117 | @Test 118 | public void create0LengthFileWithConcurrentAppend() throws IOException { 119 | Assume.assumeTrue(false); // pending change to server behavior 120 | String filename = directory + "/" + "Core.create0LengthFileWithConcurrentAppend.txt"; 121 | System.out.println("Running create0LengthFileWithConcurrentAppend"); 122 | 123 | byte [] contents = new byte[0]; 124 | RequestOptions opts = new RequestOptions(); 125 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 126 | OperationResponse resp = new OperationResponse(); 127 | Core.concurrentAppend(filename, contents, 0, 0, true, 128 | client, opts, resp); 129 | if (!resp.successful) { 130 | throw client.getExceptionFromResponse(resp, "Error in ConcurrentAppend with 0-len content " + filename); 131 | } 132 | 133 | DirectoryEntry de = dir(filename); 134 | assertTrue("File type should be FILE for 0-len file", de.type == DirectoryEntryType.FILE); 135 | assertTrue("File length in DirectoryEntry should be 0 for 0-len file", de.length == 0); 136 | 137 | byte[] b = getFileContents(filename, contents.length * 2); 138 | assertTrue("file length should be 0 for 0-len file", b.length == 0); 139 | } 140 | 141 | 142 | @Test 143 | public void createSmallFileWithConcurrentAppend() throws IOException { 144 | Assume.assumeTrue(testsEnabled); 145 | String filename = directory + "/" + "Core.createSmallFileWithConcurrentAppend.txt"; 146 | System.out.println("Running createSmallFileWithConcurrentAppend"); 147 | 148 | byte [] contents = HelperUtils.getSampleText1(); 149 | RequestOptions opts = new RequestOptions(); 150 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 151 | OperationResponse resp = new OperationResponse(); 152 | Core.concurrentAppend(filename, contents, 0, contents.length, true, 153 | client, opts, resp); 154 | if (!resp.successful) { 155 | throw client.getExceptionFromResponse(resp, "Error in ConcurrentAppend with small content " + filename); 156 | } 157 | 158 | getFileContents(filename, contents.length * 2); // read, to force metadata sync 159 | 160 | DirectoryEntry de = dir(filename); 161 | assertTrue("File type should be FILE", de.type == DirectoryEntryType.FILE); 162 | assertTrue("File length in DirectoryEntry should match (" + de.length + "!=" + contents.length + ")", 163 | de.length == contents.length); 164 | 165 | byte[] b = getFileContents(filename, contents.length * 2); 166 | assertTrue("file length should match after ConcurrentAppend", b.length == contents.length); 167 | assertTrue("file contents should match after ConcurrentAppend", Arrays.equals(contents, b)); 168 | } 169 | 170 | @Test(expected = ADLException.class) 171 | public void concurrentAppendWithoutAutocreate() throws IOException { 172 | Assume.assumeTrue(testsEnabled); 173 | String filename = directory + "/" + "Core.concurrentAppendWithoutAutocreate.txt"; 174 | System.out.println("Running concurrentAppendWithoutAutocreate"); 175 | 176 | byte [] contents = HelperUtils.getSampleText1(); 177 | RequestOptions opts = new RequestOptions(); 178 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 179 | OperationResponse resp = new OperationResponse(); 180 | Core.concurrentAppend(filename, contents, 0, contents.length, false, 181 | client, opts, resp); 182 | if (!resp.successful) { 183 | throw client.getExceptionFromResponse(resp, "(expected) Exception from concurrentAppend " + filename); 184 | } 185 | // should throw exception 186 | } 187 | 188 | 189 | @Test 190 | public void concurrentAppendToExistingFile() throws IOException { 191 | Assume.assumeTrue(testsEnabled); 192 | String filename = directory + "/" + "Core.concurrentAppendToExistingFile.txt"; 193 | System.out.println("Running concurrentAppendToExistingFile"); 194 | 195 | ByteArrayOutputStream bos = new ByteArrayOutputStream(16 * 1024); 196 | 197 | byte [] contents = HelperUtils.getSampleText1(); 198 | RequestOptions opts = new RequestOptions(); 199 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 200 | OperationResponse resp = new OperationResponse(); 201 | Core.concurrentAppend(filename, contents, 0, contents.length, true, 202 | client, opts, resp); 203 | if (!resp.successful) { 204 | throw client.getExceptionFromResponse(resp, "Exception from concurrentAppend " + filename); 205 | } 206 | bos.write(contents); 207 | 208 | contents = HelperUtils.getSampleText2(); 209 | opts = new RequestOptions(); 210 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 211 | resp = new OperationResponse(); 212 | Core.concurrentAppend(filename, contents, 0, contents.length, false, 213 | client, opts, resp); 214 | if (!resp.successful) { 215 | throw client.getExceptionFromResponse(resp, "Exception from concurrentAppend " + filename); 216 | } 217 | bos.write(contents); 218 | 219 | bos.close(); 220 | byte[] b1 = bos.toByteArray(); 221 | 222 | getFileContents(filename, contents.length * 2); // read, to force metadata sync 223 | 224 | DirectoryEntry de = dir(filename); 225 | assertTrue("File type should be FILE", de.type == DirectoryEntryType.FILE); 226 | assertTrue("File length in DirectoryEntry should match", de.length == b1.length); 227 | 228 | byte[] b = getFileContents(filename, b1.length * 2); 229 | assertTrue("file length should match after ConcurrentAppend", b.length == b1.length); 230 | assertTrue("file contents should match after ConcurrentAppend", Arrays.equals(b1, b)); 231 | } 232 | 233 | 234 | 235 | @Test 236 | public void create4MBFile() throws IOException { 237 | Assume.assumeTrue(false); // subsumed by TestFileSdk tests 238 | String filename = directory + "/" + "Core.Create4MBFile.txt"; 239 | System.out.println("Running create4MBFile"); 240 | 241 | byte [] contents = HelperUtils.getRandomBuffer(4 * 1024 * 1024); 242 | putFileContents(filename, contents, true); 243 | 244 | byte[] b = getFileContents(filename, contents.length * 2); 245 | assertTrue("file length should match what was written", b.length == contents.length); 246 | assertTrue("file contents should match", Arrays.equals(b, contents)); 247 | } 248 | 249 | @Test 250 | public void create5MBFile() throws IOException { 251 | Assume.assumeTrue(false); // subsumed by TestFileSdk tests 252 | String filename = directory + "/" + "Core.Create5MBFile.txt"; 253 | System.out.println("Running create5MBFile"); 254 | 255 | byte [] contents = HelperUtils.getRandomBuffer(11 * 1024 * 1024); 256 | putFileContents(filename, contents, true); 257 | 258 | byte[] b = getFileContents(filename, contents.length * 2); 259 | assertTrue("file length should match what was written", b.length == contents.length); 260 | assertTrue("file contents should match", Arrays.equals(b, contents)); 261 | } 262 | 263 | @Test 264 | public void createOverwriteFile() throws IOException { 265 | Assume.assumeTrue(testsEnabled); 266 | String filename = directory + "/" + "Core.CreateOverWriteFile.txt"; 267 | System.out.println("Running createOverwriteFile"); 268 | 269 | byte[] contents = HelperUtils.getSampleText1(); 270 | putFileContents(filename, contents, true); 271 | 272 | contents = HelperUtils.getSampleText2(); 273 | putFileContents(filename, contents, true); 274 | 275 | byte[] b = getFileContents(filename, contents.length * 2); 276 | assertTrue("file length should match what was written", b.length == contents.length); 277 | assertTrue("file contents should match", Arrays.equals(b, contents)); 278 | } 279 | 280 | @Test(expected = ADLException.class) 281 | public void createNoOverwriteFile() throws IOException { 282 | Assume.assumeTrue(testsEnabled); 283 | String filename = directory + "/" + "Core.CreateNoOverWriteFile.txt"; 284 | System.out.println("Running createNoOverwriteFile"); 285 | 286 | byte[] contents = HelperUtils.getSampleText1(); 287 | putFileContents(filename, contents, true); 288 | 289 | // attempt to overwrite the same file, but with overwrite flag as false. Should fail. 290 | contents = HelperUtils.getSampleText2(); 291 | putFileContents(filename, contents, false); 292 | } 293 | 294 | 295 | private void putFileContents(String filename, byte[] b, boolean overwrite) throws IOException { 296 | RequestOptions opts = new RequestOptions(); 297 | OperationResponse resp = new OperationResponse(); 298 | Core.create(filename, overwrite, null, b, 0, b.length, null, null, 299 | true, SyncFlag.CLOSE, client, opts, resp); 300 | if (!resp.successful) throw client.getExceptionFromResponse(resp, "Error creating file " + filename); 301 | } 302 | 303 | private byte[] getFileContents(String filename, int maxLength) throws IOException { 304 | byte[] b = new byte[maxLength]; 305 | int count = 0; 306 | boolean eof = false; 307 | 308 | while (!eof && count= b.length) break; 321 | } 322 | in.close(); 323 | } 324 | byte[] b2 = Arrays.copyOfRange(b, 0, count); 325 | return b2; 326 | } 327 | 328 | private DirectoryEntry dir(String filename) throws IOException { 329 | RequestOptions opts = new RequestOptions(); 330 | opts.retryPolicy = new NonIdempotentRetryPolicy(); 331 | OperationResponse resp = new OperationResponse(); 332 | DirectoryEntry de = Core.getFileStatus(filename, UserGroupRepresentation.OID, client, opts, resp); 333 | if (!resp.successful) { 334 | throw client.getExceptionFromResponse(resp, "Error in ConcurrentAppend with null content " + filename); 335 | } 336 | return de; 337 | } 338 | 339 | } 340 | --------------------------------------------------------------------------------