├── .gitignore ├── .mvn ├── jvm.config └── wrapper │ └── maven-wrapper.properties ├── CONTRIBUTING.md ├── LICENSE ├── README-template.md ├── README.md ├── cloudbuild ├── Dockerfile ├── cloudbuild.yaml └── presubmit.sh ├── codecov.yml ├── coverage └── pom.xml ├── gcs ├── CHANGES.md ├── CONFIGURATION.md ├── INSTALL.md ├── README-template.md ├── README.md ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── google │ │ │ └── cloud │ │ │ └── hadoop │ │ │ └── fs │ │ │ └── gcs │ │ │ ├── AbstractGhfsStatisticsSource.java │ │ │ ├── DelegationTokenStatistics.java │ │ │ ├── FsBenchmark.java │ │ │ ├── GhfsGlobalStorageStatistics.java │ │ │ ├── GhfsInputStreamStatistics.java │ │ │ ├── GhfsInstrumentation.java │ │ │ ├── GhfsOutputStreamStatistics.java │ │ │ ├── GhfsStatistic.java │ │ │ ├── GhfsStatisticInterface.java │ │ │ ├── GhfsStorageStatistics.java │ │ │ ├── GhfsStreamStats.java │ │ │ ├── GhfsThreadLocalStatistics.java │ │ │ ├── GoogleCloudStorageEventSubscriber.java │ │ │ ├── GoogleHadoopFS.java │ │ │ ├── GoogleHadoopFSInputStream.java │ │ │ ├── GoogleHadoopFileStatus.java │ │ │ ├── GoogleHadoopFileSystem.java │ │ │ ├── GoogleHadoopFileSystemConfiguration.java │ │ │ ├── GoogleHadoopOutputStream.java │ │ │ ├── HadoopConfigurationProperty.java │ │ │ ├── HadoopCredentialsConfiguration.java │ │ │ ├── InMemoryGlobberFileSystem.java │ │ │ ├── VectoredIOImpl.java │ │ │ ├── VectoredReadOptions.java │ │ │ └── auth │ │ │ ├── AbstractDelegationTokenBinding.java │ │ │ ├── DelegationTokenIOException.java │ │ │ ├── GcsDelegationTokens.java │ │ │ └── GcsDtFetcher.java │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ ├── org.apache.hadoop.fs.FileSystem │ │ │ └── org.apache.hadoop.security.token.DtFetcher │ │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── fs │ │ └── gcs │ │ └── gcs.properties │ └── test │ ├── java │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── fs │ │ └── gcs │ │ ├── GhfsThreadLocalStatisticsTest.java │ │ ├── GoogleCloudStorageStatisticsTest.java │ │ ├── GoogleHadoopFSInputStreamIntegrationTest.java │ │ ├── GoogleHadoopFSIntegrationTest.java │ │ ├── GoogleHadoopFileSystemConfigurationTest.java │ │ ├── GoogleHadoopFileSystemDelegationTokensTest.java │ │ ├── GoogleHadoopFileSystemHTTPClientIntegrationTest.java │ │ ├── GoogleHadoopFileSystemIntegrationHelper.java │ │ ├── GoogleHadoopFileSystemIntegrationTest.java │ │ ├── GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java │ │ ├── GoogleHadoopFileSystemNewIntegrationTest.java │ │ ├── GoogleHadoopFileSystemTest.java │ │ ├── GoogleHadoopFileSystemTestBase.java │ │ ├── GoogleHadoopFileSystemTestHelper.java │ │ ├── GoogleHadoopFileSystemXAttrsIntegrationTest.java │ │ ├── GoogleHadoopOutputStreamIntegrationTest.java │ │ ├── GoogleHadoopOutputStreamTest.java │ │ ├── HadoopFileSystemIntegrationHelper.java │ │ ├── HadoopFileSystemIntegrationTest.java │ │ ├── HadoopFileSystemTestBase.java │ │ ├── HdfsBehavior.java │ │ ├── InMemoryGlobberFileSystemTest.java │ │ ├── InMemoryGoogleHadoopFileSystem.java │ │ ├── TestUtils.java │ │ ├── VectoredIOImplTest.java │ │ ├── WebHdfsIntegrationTest.java │ │ ├── auth │ │ ├── TestDelegationTokenBindingImpl.java │ │ └── TestTokenIdentifierImpl.java │ │ ├── contract │ │ ├── AbstractGoogleContractConcatTest.java │ │ ├── GoogleContract.java │ │ ├── InMemoryGoogleContract.java │ │ ├── TestGoogleContractAppend.java │ │ ├── TestGoogleContractConcat.java │ │ ├── TestGoogleContractCreate.java │ │ ├── TestGoogleContractDelete.java │ │ ├── TestGoogleContractGetFileStatus.java │ │ ├── TestGoogleContractMkdir.java │ │ ├── TestGoogleContractOpen.java │ │ ├── TestGoogleContractRename.java │ │ ├── TestGoogleContractRootDirectory.java │ │ ├── TestGoogleContractSeek.java │ │ ├── TestInMemoryGoogleContractAppend.java │ │ ├── TestInMemoryGoogleContractConcat.java │ │ ├── TestInMemoryGoogleContractCreate.java │ │ ├── TestInMemoryGoogleContractDelete.java │ │ ├── TestInMemoryGoogleContractMkdir.java │ │ ├── TestInMemoryGoogleContractOpen.java │ │ ├── TestInMemoryGoogleContractRename.java │ │ ├── TestInMemoryGoogleContractRootDirectory.java │ │ └── TestInMemoryGoogleContractSeek.java │ │ └── hcfs │ │ ├── GoogleHadoopFSMainOperationsTest.java │ │ └── GoogleHadoopFileSystemContractTest.java │ └── resources │ ├── META-INF │ └── services │ │ └── org.apache.hadoop.security.token.TokenIdentifier │ ├── contract │ └── gs.xml │ └── core-site.xml ├── gcsio ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── gcsio │ │ ├── BatchExecutor.java │ │ ├── BatchHelper.java │ │ ├── CloudMonitoringMetricsRecorder.java │ │ ├── CreateBucketOptions.java │ │ ├── CreateFileOptions.java │ │ ├── CreateObjectOptions.java │ │ ├── DeleteFolderOperation.java │ │ ├── EventLoggingHttpRequestInitializer.java │ │ ├── FileInfo.java │ │ ├── FolderInfo.java │ │ ├── ForwardingGoogleCloudStorage.java │ │ ├── GoogleCloudStorage.java │ │ ├── GoogleCloudStorageClientGrpcDownscopingInterceptor.java │ │ ├── GoogleCloudStorageClientGrpcStatisticsInterceptor.java │ │ ├── GoogleCloudStorageClientGrpcTracingInterceptor.java │ │ ├── GoogleCloudStorageClientImpl.java │ │ ├── GoogleCloudStorageClientReadChannel.java │ │ ├── GoogleCloudStorageClientWriteChannel.java │ │ ├── GoogleCloudStorageExceptions.java │ │ ├── GoogleCloudStorageFileSystem.java │ │ ├── GoogleCloudStorageFileSystemImpl.java │ │ ├── GoogleCloudStorageFileSystemOptions.java │ │ ├── GoogleCloudStorageImpl.java │ │ ├── GoogleCloudStorageItemInfo.java │ │ ├── GoogleCloudStorageOptions.java │ │ ├── GoogleCloudStorageReadChannel.java │ │ ├── GoogleCloudStorageReadOptions.java │ │ ├── GoogleCloudStorageStatistics.java │ │ ├── GoogleCloudStorageStrings.java │ │ ├── GoogleCloudStorageTracingFields.java │ │ ├── GoogleCloudStorageWriteChannel.java │ │ ├── GrpcStreamType.java │ │ ├── ListFileOptions.java │ │ ├── ListFolderOptions.java │ │ ├── ListObjectOptions.java │ │ ├── MetricsRecorder.java │ │ ├── ObjectWriteConditions.java │ │ ├── PerformanceCachingGoogleCloudStorage.java │ │ ├── PerformanceCachingGoogleCloudStorageOptions.java │ │ ├── PrefixMappedItemCache.java │ │ ├── StatisticTypeEnum.java │ │ ├── StatisticsTrackingHttpRequestInitializer.java │ │ ├── StorageRequestFactory.java │ │ ├── StorageRequestToAccessBoundaryConverter.java │ │ ├── StorageResourceId.java │ │ ├── StringPaths.java │ │ ├── UpdatableItemInfo.java │ │ ├── UriPaths.java │ │ ├── VerificationAttributes.java │ │ └── testing │ │ ├── GcsItemInfoTestBuilder.java │ │ ├── InMemoryBucketEntry.java │ │ ├── InMemoryGoogleCloudStorage.java │ │ ├── InMemoryObjectEntry.java │ │ ├── InMemoryObjectReadChannel.java │ │ └── TestConfiguration.java │ └── test │ ├── java │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── gcsio │ │ ├── AssertingLogHandler.java │ │ ├── BatchExecutorTest.java │ │ ├── BatchHelperTest.java │ │ ├── CreateFileOptionsTest.java │ │ ├── CreateObjectOptionsTest.java │ │ ├── DeleteFolderOperationTest.java │ │ ├── DeletionBehavior.java │ │ ├── EventLoggingHttpRequestInitializerTest.java │ │ ├── FakeReadChannel.java │ │ ├── FakeServer.java │ │ ├── FakeWriteChannel.java │ │ ├── FileInfoTest.java │ │ ├── FolderInfoTest.java │ │ ├── ForwardingGoogleCloudStorageTest.java │ │ ├── GoogleCloudStorageClientGrpcDownscopingInterceptorTest.java │ │ ├── GoogleCloudStorageClientReadChannelTest.java │ │ ├── GoogleCloudStorageClientTest.java │ │ ├── GoogleCloudStorageClientWriteChannelTest.java │ │ ├── GoogleCloudStorageExceptionsTest.java │ │ ├── GoogleCloudStorageFileSystemHTTPClientTest.java │ │ ├── GoogleCloudStorageFileSystemHttpClientNewIntegrationTest.java │ │ ├── GoogleCloudStorageFileSystemIntegrationHelper.java │ │ ├── GoogleCloudStorageFileSystemIntegrationTest.java │ │ ├── GoogleCloudStorageFileSystemJavaStorageClientTest.java │ │ ├── GoogleCloudStorageFileSystemJavaStorageNewIntegrationTest.java │ │ ├── GoogleCloudStorageFileSystemNewIntegrationTestBase.java │ │ ├── GoogleCloudStorageFileSystemOptionsTest.java │ │ ├── GoogleCloudStorageFileSystemPerformanceCacheIntegrationTest.java │ │ ├── GoogleCloudStorageFileSystemTestBase.java │ │ ├── GoogleCloudStorageHttpClientNewIntegrationTest.java │ │ ├── GoogleCloudStorageImplCreateTest.java │ │ ├── GoogleCloudStorageIntegrationHelper.java │ │ ├── GoogleCloudStorageItemInfoTest.java │ │ ├── GoogleCloudStorageJavaStorageNewIntegrationTest.java │ │ ├── GoogleCloudStorageMockitoTest.java │ │ ├── GoogleCloudStorageNewIntegrationTestBase.java │ │ ├── GoogleCloudStorageOptionsTest.java │ │ ├── GoogleCloudStorageReadChannelTest.java │ │ ├── GoogleCloudStorageReadOptionsTest.java │ │ ├── GoogleCloudStorageStringsTest.java │ │ ├── GoogleCloudStorageTest.java │ │ ├── GoogleCloudStorageTestUtils.java │ │ ├── GoogleCloudStorageWriteChannelTest.java │ │ ├── MethodOutcome.java │ │ ├── MkdirsBehavior.java │ │ ├── MockGoogleCloudStorageImplFactory.java │ │ ├── MockStorage.java │ │ ├── PerformanceCachingGoogleCloudStorageIntegrationTest.java │ │ ├── PerformanceCachingGoogleCloudStorageTest.java │ │ ├── PrefixMappedItemCacheTest.java │ │ ├── RenameBehavior.java │ │ ├── StorageRequestFactoryTest.java │ │ ├── StorageRequestToAccessBoundaryConverterTest.java │ │ ├── TestServerHeaderInterceptor.java │ │ ├── TrackingGrpcRequestInterceptor.java │ │ ├── TrackingHttpRequestInitializer.java │ │ ├── UriEncodingPathCodecTest.java │ │ └── integration │ │ ├── CsekEncryptionIntegrationTest.java │ │ ├── GoogleCloudStorageClientImplIntegrationTest.java │ │ ├── GoogleCloudStorageClientInterceptorIntegrationTest.java │ │ ├── GoogleCloudStorageImplTest.java │ │ ├── GoogleCloudStorageIntegrationTest.java │ │ ├── GoogleCloudStorageTest.java │ │ └── GoogleCloudStorageTestHelper.java │ └── resources │ └── log4j.properties ├── mvnw ├── mvnw.cmd ├── pom.xml ├── tools └── run_integration_tests.sh ├── util-hadoop ├── pom.xml └── src │ ├── main │ └── java │ │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── util │ │ ├── AccessTokenProvider.java │ │ ├── HadoopConfigurationProperty.java │ │ ├── HadoopCredentialsConfiguration.java │ │ └── testing │ │ ├── CredentialsConfigurationUtil.java │ │ ├── HadoopConfigurationUtils.java │ │ └── TestingAccessTokenProvider.java │ └── test │ ├── java │ └── com │ │ └── google │ │ └── cloud │ │ └── hadoop │ │ └── util │ │ ├── HadoopConfigurationPropertyTest.java │ │ └── HadoopCredentialsConfigurationTest.java │ └── resources │ ├── test-credentials.json │ └── test-wip-config.json └── util ├── pom.xml └── src ├── main └── java │ └── com │ └── google │ └── cloud │ └── hadoop │ └── util │ ├── AbstractGoogleAsyncWriteChannel.java │ ├── AccessBoundary.java │ ├── ApiErrorExtractor.java │ ├── AsyncWriteChannelOptions.java │ ├── ChainingHttpRequestInitializer.java │ ├── CheckedFunction.java │ ├── ClientRequestHelper.java │ ├── ErrorTypeExtractor.java │ ├── GcsJsonApiEvent.java │ ├── GcsRequestExecutionEvent.java │ ├── GoogleCloudStorageEventBus.java │ ├── GrpcErrorTypeExtractor.java │ ├── HttpTransportFactory.java │ ├── IGcsJsonApiEvent.java │ ├── ITraceFactory.java │ ├── ITraceOperation.java │ ├── IoExceptionHelper.java │ ├── LazyExecutorService.java │ ├── LoggingMediaHttpUploaderProgressListener.java │ ├── PropertyUtil.java │ ├── RedactedString.java │ ├── RequestTracker.java │ ├── RequesterPaysOptions.java │ ├── ResilientOperation.java │ ├── RetryBoundedBackOff.java │ ├── RetryDeterminer.java │ ├── RetryHttpInitializer.java │ ├── RetryHttpInitializerOptions.java │ ├── ThreadTrace.java │ ├── TraceFactory.java │ ├── TraceOperation.java │ ├── interceptors │ └── InvocationIdInterceptor.java │ └── testing │ └── MockHttpTransportHelper.java └── test ├── java └── com │ └── google │ └── cloud │ └── hadoop │ └── util │ ├── AccessBoundaryTest.java │ ├── ApiErrorExtractorTest.java │ ├── GcsJsonApiEventTest.java │ ├── GrpcErrorTypeExtractorTest.java │ ├── HttpTransportFactoryTest.java │ ├── LazyExecutorServiceTest.java │ ├── LoggingMediaHttpUploaderProgressListenerTest.java │ ├── PropertyUtilTest.java │ ├── RequestTrackerTest.java │ ├── ResilientOperationTest.java │ ├── RetryBoundedBackOffTest.java │ ├── RetryDeterminerTest.java │ ├── RetryHttpInitializerTest.java │ ├── TestRequestTracker.java │ ├── ThreadTraceTest.java │ ├── TraceFactoryTest.java │ └── testing │ ├── FakeCredentials.java │ ├── ThrowingInputStream.java │ └── logging │ ├── AssertingHandler.java │ └── ExpectedLogMessages.java └── resources ├── com └── google │ └── cloud │ └── hadoop │ └── util │ └── test.properties └── log4j.properties /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | bin/ 3 | 4 | # Ignore IntelliJ files. 5 | .idea/ 6 | *.iml 7 | *.ipr 8 | *.iws 9 | 10 | # MacOS folder files 11 | .DS_Store 12 | 13 | # Ignore Maven wrapper jar 14 | .mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/jvm.config: -------------------------------------------------------------------------------- 1 | --add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED 2 | --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED 3 | --add-exports jdk.compiler/com.sun.tools.javac.main=ALL-UNNAMED 4 | --add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED 5 | --add-exports jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED 6 | --add-exports jdk.compiler/com.sun.tools.javac.processing=ALL-UNNAMED 7 | --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED 8 | --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED 9 | --add-opens jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED 10 | --add-opens jdk.compiler/com.sun.tools.javac.comp=ALL-UNNAMED 11 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, 12 | # software distributed under the License is distributed on an 13 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 14 | # KIND, either express or implied. See the License for the 15 | # specific language governing permissions and limitations 16 | # under the License. 17 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.5/apache-maven-3.9.5-bin.zip 18 | wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.1/maven-wrapper-3.1.1.jar 19 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Apache Hadoop Connectors 2 | 3 | [![GitHub release](https://img.shields.io/github/release/GoogleCloudDataproc/hadoop-connectors.svg)](https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/latest) 4 | [![GitHub release date](https://img.shields.io/github/release-date/GoogleCloudDataproc/hadoop-connectors.svg)](https://github.com/GoogleCloudDataproc/hadoop-connectors/releases/latest) 5 | [![codecov](https://codecov.io/gh/GoogleCloudDataproc/hadoop-connectors/branch/master/graph/badge.svg)](https://codecov.io/gh/GoogleCloudDataproc/hadoop-connectors) 6 | 7 | Libraries and tools for interoperability between Apache Hadoop related 8 | open-source software and Google Cloud Platform. 9 | 10 | ## Google Cloud Storage connector for Apache Hadoop (HCFS) 11 | 12 | [![Maven Central](https://img.shields.io/maven-central/v/com.google.cloud.bigdataoss/gcs-connector/hadoop1.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:com.google.cloud.bigdataoss%20AND%20a:gcs-connector%20AND%20v:hadoop1-*) 13 | [![Maven Central](https://img.shields.io/maven-central/v/com.google.cloud.bigdataoss/gcs-connector/hadoop2.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:com.google.cloud.bigdataoss%20AND%20a:gcs-connector%20AND%20v:hadoop2-*) 14 | [![Maven Central](https://img.shields.io/maven-central/v/com.google.cloud.bigdataoss/gcs-connector.svg?label=Maven%20Central)](https://search.maven.org/search?q=g:%22com.google.cloud.bigdataoss%22%20AND%20a:%22gcs-connector%22) 15 | 16 | The Google Cloud Storage connector for Hadoop enables running MapReduce jobs 17 | directly on data in Google Cloud Storage by implementing the Hadoop FileSystem 18 | interface. For details, see [the README](gcs/README.md). 19 | 20 | ## Building the Cloud Storage connector 21 | 22 | > Note that build requires Java 11+ and fails with older Java versions. 23 | 24 | To build the connector for specific Hadoop version, run the following commands 25 | from the main directory: 26 | 27 | ```bash 28 | ./mvnw clean package 29 | ``` 30 | 31 | In order to verify test coverage for specific Hadoop version, run the following 32 | commands from the main directory: 33 | 34 | ```bash 35 | ./mvnw -P coverage clean verify 36 | ``` 37 | 38 | The Cloud Storage connector JAR can be found in `gcs/target/` directory. 39 | 40 | ## Adding the Cloud Storage connector to your build 41 | 42 | Maven group ID is `com.google.cloud.bigdataoss` and artifact ID for Cloud 43 | Storage connector is `gcs-connector`. 44 | 45 | To add a dependency on Cloud Storage connector using Maven, use the following: 46 | 47 | ```xml 48 | 49 | com.google.cloud.bigdataoss 50 | gcs-connector 51 | 3.0.0 52 | 53 | ``` 54 | 55 | ## Resources 56 | 57 | On **Stack Overflow**, use the tag 58 | [`google-cloud-dataproc`](https://stackoverflow.com/tags/google-cloud-dataproc) 59 | for questions about the connectors in this repository. This tag receives 60 | responses from the Stack Overflow community and Google engineers, who monitor 61 | the tag and offer unofficial support. 62 | -------------------------------------------------------------------------------- /cloudbuild/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This Dockerfile creates an image for running presubmit tests. 16 | FROM openjdk:11-jdk 17 | 18 | # Copy everything into the container to allow concurrent build execution 19 | COPY . /hadoop-connectors 20 | -------------------------------------------------------------------------------- /cloudbuild/cloudbuild.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | steps: 16 | # 1. Create a Docker image containing hadoop-connectors repo 17 | - name: 'gcr.io/cloud-builders/docker' 18 | id: 'docker-build' 19 | args: ['build', '--tag=gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit', '-f', 'cloudbuild/Dockerfile', '.'] 20 | 21 | # 2. Run unit tests 22 | - name: 'gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit' 23 | id: 'unit-tests' 24 | waitFor: ['docker-build'] 25 | entrypoint: 'bash' 26 | args: ['/hadoop-connectors/cloudbuild/presubmit.sh', 'unittest'] 27 | env: 28 | - 'CODECOV_TOKEN=$_CODECOV_TOKEN' 29 | - 'VCS_BRANCH_NAME=$BRANCH_NAME' 30 | - 'VCS_COMMIT_ID=$COMMIT_SHA' 31 | - 'VCS_TAG=$TAG_NAME' 32 | - 'CI_BUILD_ID=$BUILD_ID' 33 | 34 | # 3. Run integration tests concurrently with unit tests 35 | - name: 'gcr.io/$PROJECT_ID/dataproc-hadoop-connectors-presubmit' 36 | id: 'integration-tests' 37 | waitFor: ['docker-build'] 38 | entrypoint: 'bash' 39 | args: ['/hadoop-connectors/cloudbuild/presubmit.sh', 'integrationtest'] 40 | env: 41 | - 'GCS_TEST_PROJECT_ID=$PROJECT_ID' 42 | - 'CODECOV_TOKEN=$_CODECOV_TOKEN' 43 | - 'VCS_BRANCH_NAME=$BRANCH_NAME' 44 | - 'VCS_COMMIT_ID=$COMMIT_SHA' 45 | - 'VCS_TAG=$TAG_NAME' 46 | - 'CI_BUILD_ID=$BUILD_ID' 47 | - 'GCS_TEST_DIRECT_PATH_PREFERRED=false' 48 | 49 | # Tests take on average 25 minutes to run 50 | timeout: 2400s 51 | 52 | options: 53 | machineType: 'N1_HIGHCPU_32' 54 | 55 | -------------------------------------------------------------------------------- /cloudbuild/presubmit.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -euxo pipefail 18 | 19 | readonly TEST_TYPE="${1:-unittest}" 20 | 21 | cd /hadoop-connectors 22 | 23 | # Run unit or integration tests and generate test coverage report 24 | if [[ $TEST_TYPE == unittest ]]; then 25 | ./mvnw -B -e -Pcoverage clean verify 26 | else 27 | ./mvnw -B -e -Pintegration-test -Pcoverage clean verify 28 | fi 29 | 30 | # Upload test coverage report to Codecov 31 | bash <(curl -s https://codecov.io/bash) -K -F "${TEST_TYPE}" 32 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | ignore: 16 | - ".*/generated-sources/.*" 17 | -------------------------------------------------------------------------------- /coverage/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 4.0.0 20 | 21 | 22 | com.google.cloud.bigdataoss 23 | bigdataoss-parent 24 | 3.0.0-SNAPSHOT 25 | 26 | 27 | coverage 28 | 3.0.0-SNAPSHOT 29 | pom 30 | 31 | 32 | 33 | coverage 34 | 35 | false 36 | 37 | 38 | 39 | 40 | org.jacoco 41 | jacoco-maven-plugin 42 | 43 | 44 | verify 45 | 46 | report-aggregate 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | com.google.cloud.bigdataoss 59 | gcs-connector 60 | ${bigdataoss.version} 61 | 62 | 63 | com.google.cloud.bigdataoss 64 | gcsio 65 | ${bigdataoss.version} 66 | 67 | 68 | com.google.cloud.bigdataoss 69 | util 70 | ${bigdataoss.version} 71 | 72 | 73 | com.google.cloud.bigdataoss 74 | util-hadoop 75 | ${bigdataoss.version} 76 | 77 | 78 | 79 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/DelegationTokenStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | /** 20 | * Statistics updated by a {@link com.google.cloud.hadoop.fs.gcs.auth.GcsDelegationTokens} during 21 | * its use. 22 | */ 23 | public interface DelegationTokenStatistics extends GhfsStatisticInterface { 24 | 25 | /** A token has been issued. */ 26 | void tokenIssued(); 27 | } 28 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsInputStreamStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | /** 20 | * Statistics updated by a {@link GoogleHadoopFSInputStream} during its use. It also contains 21 | * getters for tests. 22 | */ 23 | interface GhfsInputStreamStatistics extends AutoCloseable, GhfsStatisticInterface { 24 | 25 | /** 26 | * Seek backwards, incrementing the seek and backward seek counters. 27 | * 28 | * @param negativeOffset how far was the seek? This is expected to be negative. 29 | */ 30 | void seekBackwards(long negativeOffset); 31 | 32 | /** 33 | * Record a forward seek, adding a seek operation, a forward seek operation. 34 | * 35 | * @param skipped bytes moved forward in stream If the seek was implemented by a close + reopen, 36 | * set this to zero. 37 | */ 38 | void seekForwards(long skipped); 39 | 40 | /** An ignored stream read exception was received. */ 41 | void readException(); 42 | 43 | /** 44 | * Increment the bytes read counter by the number of bytes; no-op if the argument is negative. 45 | * 46 | * @param bytes number of bytes read 47 | */ 48 | void bytesRead(long bytes); 49 | 50 | /** 51 | * A read operation has completed. 52 | * 53 | * @param requested number of requested bytes 54 | * @param actual the actual number of bytes 55 | */ 56 | void readOperationCompleted(int requested, int actual); 57 | 58 | @Override 59 | void close(); 60 | } 61 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsOutputStreamStatistics.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import java.io.Closeable; 20 | 21 | /** 22 | * Statistics updated by a {@link GoogleHadoopOutputStream} during its use. It also contains getters 23 | * for tests. 24 | */ 25 | interface GhfsOutputStreamStatistics extends Closeable, GhfsStatisticInterface { 26 | /** 27 | * Record bytes written. 28 | * 29 | * @param count number of bytes 30 | */ 31 | void writeBytes(long count); 32 | } 33 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStatisticInterface.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import org.apache.hadoop.fs.statistics.DurationTrackerFactory; 20 | import org.apache.hadoop.fs.statistics.IOStatisticsSource; 21 | 22 | /** 23 | * Base Interface for GoogleHadoopFileSystem Statistics, as a way of managing Input Stream 24 | * Statistics and Output Stream Statistics. 25 | */ 26 | public interface GhfsStatisticInterface extends IOStatisticsSource, DurationTrackerFactory {} 27 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GhfsStreamStats.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import javax.annotation.Nonnull; 20 | 21 | class GhfsStreamStats { 22 | private final GhfsGlobalStorageStatistics storageStatistics; 23 | private final GhfsStatistic durationStat; 24 | private final Object context; 25 | private long maxLatencyNs; 26 | private int operationCount; 27 | private long minLatencyNs; 28 | private long totalNs; 29 | 30 | GhfsStreamStats( 31 | @Nonnull GhfsGlobalStorageStatistics storageStatistics, 32 | GhfsStatistic durationStat, 33 | Object context) { 34 | this.storageStatistics = storageStatistics; 35 | this.durationStat = durationStat; 36 | this.context = context; 37 | } 38 | 39 | void close() { 40 | if (operationCount == 0) { 41 | return; 42 | } 43 | 44 | storageStatistics.updateStats( 45 | durationStat, 46 | toMillis(minLatencyNs), 47 | toMillis(maxLatencyNs), 48 | toMillis(totalNs), 49 | operationCount, 50 | context); 51 | this.totalNs = 0; 52 | this.operationCount = 0; 53 | } 54 | 55 | void updateWriteStreamStats(int len, long start) { 56 | updateStats(start); 57 | storageStatistics.streamWriteBytes(len); 58 | } 59 | 60 | void updateReadStreamSeekStats(long start) { 61 | updateStats(start); 62 | } 63 | 64 | void updateReadStreamStats(int len, long start) { 65 | updateStats(start); 66 | storageStatistics.streamReadBytes(len); 67 | } 68 | 69 | void updateVectoredReadStreamStats(long start) { 70 | updateStats(start); 71 | } 72 | 73 | private static long toMillis(long nano) { 74 | return nano / 1000_000; 75 | } 76 | 77 | private void updateStats(long start) { 78 | long latency = System.nanoTime() - start; 79 | this.maxLatencyNs = Math.max(latency, this.maxLatencyNs); 80 | if (operationCount == 0) { 81 | this.minLatencyNs = latency; 82 | } else { 83 | this.minLatencyNs = Math.min(latency, this.minLatencyNs); 84 | } 85 | 86 | this.totalNs += latency; 87 | this.operationCount++; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import com.google.cloud.hadoop.gcsio.FileInfo; 20 | import org.apache.hadoop.fs.FileStatus; 21 | import org.apache.hadoop.fs.Path; 22 | import org.apache.hadoop.fs.permission.FsPermission; 23 | 24 | /** 25 | * {@link FileStatus} for a {@link GoogleHadoopFileSystem} operations. 26 | * 27 | *

{@link FileStatus} that has access {@link FileInfo} that has additional GCS metadata. 28 | */ 29 | class GoogleHadoopFileStatus extends FileStatus { 30 | 31 | private final FileInfo fileInfo; 32 | 33 | /** Constructs an instance GoogleHadoopFileStatus */ 34 | GoogleHadoopFileStatus( 35 | FileInfo fileInfo, 36 | Path filePath, 37 | int blockReplication, 38 | long defaultBlockSize, 39 | FsPermission reportedPermissions, 40 | String userName) { 41 | super( 42 | fileInfo.getSize(), 43 | fileInfo.isDirectory(), 44 | blockReplication, 45 | defaultBlockSize, 46 | /* modificationTime= */ fileInfo.getModificationTime(), 47 | /* accessTime= */ fileInfo.getModificationTime(), 48 | reportedPermissions, 49 | /* owner= */ userName, 50 | /* group= */ userName, 51 | filePath); 52 | this.fileInfo = fileInfo; 53 | } 54 | 55 | /** Returns GCS ItemInfo for the corresponding GoogleHadoopFileStatus instance */ 56 | FileInfo getFileInfo() { 57 | return fileInfo; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/HadoopConfigurationProperty.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | /** Hadoop configuration property for Google Cloud Storage Connector */ 20 | public class HadoopConfigurationProperty 21 | extends com.google.cloud.hadoop.util.HadoopConfigurationProperty { 22 | 23 | public HadoopConfigurationProperty(String key) { 24 | super(key); 25 | } 26 | 27 | public HadoopConfigurationProperty(String key, T defaultValue, String... deprecatedKeys) { 28 | super(key, defaultValue, deprecatedKeys); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/HadoopCredentialsConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | /** {@code HadoopCredentialsConfiguration} for Google Cloud Storage Connector */ 20 | public class HadoopCredentialsConfiguration 21 | extends com.google.cloud.hadoop.util.HadoopCredentialsConfiguration {} 22 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/VectoredReadOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import com.google.auto.value.AutoValue; 20 | 21 | @AutoValue 22 | public abstract class VectoredReadOptions { 23 | 24 | public static VectoredReadOptions DEFAULT = builder().build(); 25 | 26 | public static VectoredReadOptions.Builder builder() { 27 | return new AutoValue_VectoredReadOptions.Builder() 28 | .setMinSeekVectoredReadSize(4 * 1024) // 4KB 29 | .setMergeRangeMaxSize(8 * 1024 * 1024) // 8MB 30 | .setReadThreads(16); 31 | } 32 | 33 | public abstract int getMinSeekVectoredReadSize(); 34 | 35 | public abstract int getMergeRangeMaxSize(); 36 | 37 | public abstract int getReadThreads(); 38 | 39 | public abstract VectoredReadOptions.Builder toBuilder(); 40 | 41 | @AutoValue.Builder 42 | public abstract static class Builder { 43 | 44 | public abstract VectoredReadOptions build(); 45 | 46 | public abstract Builder setMinSeekVectoredReadSize(int minSeekVectoredReadSize); 47 | 48 | public abstract Builder setMergeRangeMaxSize(int mergedRangeMazSize); 49 | 50 | public abstract Builder setReadThreads(int readThreads); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/auth/DelegationTokenIOException.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.auth; 18 | 19 | import java.io.IOException; 20 | import org.apache.hadoop.io.Text; 21 | import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; 22 | 23 | /** 24 | * General IOException for Delegation Token issues. Includes recommended error strings, which can be 25 | * used in tests when looking for specific errors. 26 | */ 27 | public class DelegationTokenIOException extends IOException { 28 | 29 | /** 30 | * Version number for serialization. See more info at: 31 | * https://docs.oracle.com/en/java/javase/11/docs/api/java.base/java/io/Serializable.html 32 | */ 33 | private static final long serialVersionUID = 5431764092856006083L; 34 | 35 | public static DelegationTokenIOException wrongTokenType( 36 | Class expectedClass, 37 | DelegationTokenIdentifier identifier) { 38 | return new DelegationTokenIOException( 39 | String.format( 40 | "Delegation token type is incorrect;" 41 | + " expected a token identifier of type %s but got %s and kind %s", 42 | expectedClass, identifier.getClass(), identifier.getKind())); 43 | } 44 | 45 | public static DelegationTokenIOException tokenMismatch( 46 | Text service, Text expectedKind, Text actualKind) { 47 | return new DelegationTokenIOException( 48 | String.format( 49 | "Token mismatch: expected token for %s of type %s but got a token of type %s", 50 | service, expectedKind, actualKind)); 51 | } 52 | 53 | public DelegationTokenIOException(String message) { 54 | super(message); 55 | } 56 | 57 | public DelegationTokenIOException(String message, Throwable cause) { 58 | super(message, cause); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /gcs/src/main/java/com/google/cloud/hadoop/fs/gcs/auth/GcsDtFetcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.auth; 18 | 19 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem.SCHEME; 20 | 21 | import java.net.URI; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.FileSystem; 24 | import org.apache.hadoop.io.Text; 25 | import org.apache.hadoop.security.Credentials; 26 | import org.apache.hadoop.security.UserGroupInformation; 27 | import org.apache.hadoop.security.token.DtFetcher; 28 | import org.apache.hadoop.security.token.Token; 29 | 30 | /** 31 | * A DT fetcher for GCS. This is a copy-and-paste of {@code org.apache.hadoop.hdfs.HdfsDtFetcher}. 32 | */ 33 | public class GcsDtFetcher implements DtFetcher { 34 | 35 | /** Returns the service name for GCS, which is also a valid URL prefix. */ 36 | @Override 37 | public Text getServiceName() { 38 | return new Text(SCHEME); 39 | } 40 | 41 | @Override 42 | public boolean isTokenRequired() { 43 | return UserGroupInformation.isSecurityEnabled(); 44 | } 45 | 46 | /** 47 | * Returns Token object via FileSystem, null if bad argument. 48 | * 49 | * @param conf - a Configuration object used with FileSystem.get() 50 | * @param creds - a Credentials object to which token(s) will be added 51 | * @param renewer - the renewer to send with the token request 52 | * @param url - the URL to which the request is sent 53 | * @return a Token, or null if fetch fails. 54 | */ 55 | @Override 56 | public Token addDelegationTokens( 57 | Configuration conf, Credentials creds, String renewer, String url) throws Exception { 58 | if (!url.startsWith(SCHEME)) { 59 | url = SCHEME + "://" + url; 60 | } 61 | FileSystem fs = FileSystem.get(URI.create(url), conf); 62 | Token token = fs.getDelegationToken(renewer); 63 | if (token == null) { 64 | throw new DelegationTokenIOException("Filesystem not generating Delegation Tokens: " + url); 65 | } 66 | creds.addToken(token.getService(), token); 67 | return token; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /gcs/src/main/resources/META-INF/services/org.apache.hadoop.fs.FileSystem: -------------------------------------------------------------------------------- 1 | # Copyright 2014 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem 16 | -------------------------------------------------------------------------------- /gcs/src/main/resources/META-INF/services/org.apache.hadoop.security.token.DtFetcher: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | com.google.cloud.hadoop.fs.gcs.auth.GcsDtFetcher 16 | -------------------------------------------------------------------------------- /gcs/src/main/resources/com/google/cloud/hadoop/fs/gcs/gcs.properties: -------------------------------------------------------------------------------- 1 | gcs.connector.version=${project.version} -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemHTTPClientIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; 20 | import org.junit.Before; 21 | import org.junit.runner.RunWith; 22 | import org.junit.runners.JUnit4; 23 | 24 | @RunWith(JUnit4.class) 25 | public class GoogleHadoopFileSystemHTTPClientIntegrationTest 26 | extends GoogleHadoopFileSystemIntegrationTest { 27 | @Before 28 | public void before() throws Exception { 29 | storageClientType = ClientType.HTTP_API_CLIENT; 30 | super.before(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemIntegrationHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import static com.google.cloud.hadoop.gcsio.testing.TestConfiguration.GCS_TEST_PROJECT_ID; 20 | import static com.google.common.base.Preconditions.checkNotNull; 21 | 22 | import com.google.cloud.hadoop.gcsio.testing.TestConfiguration; 23 | import com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType; 24 | import java.net.URI; 25 | import org.apache.hadoop.conf.Configuration; 26 | 27 | public final class GoogleHadoopFileSystemIntegrationHelper { 28 | 29 | private static final String ENV_VAR_MSG_FMT = "Environment variable %s should be set"; 30 | 31 | public static GoogleHadoopFileSystem createGhfs(URI path, Configuration config) throws Exception { 32 | GoogleHadoopFileSystem ghfs = new GoogleHadoopFileSystem(); 33 | ghfs.initialize(path, config); 34 | return ghfs; 35 | } 36 | 37 | /** 38 | * Helper to load all the GHFS-specific config values from environment variables, such as those 39 | * needed for setting up the credentials of a real GoogleCloudStorage. 40 | */ 41 | public static Configuration getTestConfig() { 42 | Configuration config = new Configuration(); 43 | config.setBoolean("fs.gs.implicit.dir.repair.enable", true); 44 | // Allow buckets to be deleted in test cleanup: 45 | config.setBoolean("fs.gs.bucket.delete.enable", true); 46 | 47 | // Configure test authentication 48 | TestConfiguration testConf = TestConfiguration.getInstance(); 49 | String projectId = checkNotNull(testConf.getProjectId(), ENV_VAR_MSG_FMT, GCS_TEST_PROJECT_ID); 50 | config.set("fs.gs.project.id", projectId); 51 | if (testConf.getServiceAccountJsonKeyFile() != null) { 52 | config.setEnum("fs.gs.auth.type", AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE); 53 | config.set( 54 | "fs.gs.auth.service.account.json.keyfile", testConf.getServiceAccountJsonKeyFile()); 55 | } 56 | 57 | config.setBoolean("fs.gs.grpc.directpath.enable", testConf.isDirectPathPreferred()); 58 | 59 | return config; 60 | } 61 | 62 | private GoogleHadoopFileSystemIntegrationHelper() {} 63 | } 64 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystemJavaStorageClientIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.google.cloud.hadoop.fs.gcs; 2 | /* 3 | * Copyright 2023 Google LLC 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; 19 | import org.junit.Before; 20 | import org.junit.Ignore; 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.junit.runners.JUnit4; 24 | 25 | // TODO: Ignored test for gRPC-directpath 26 | // https://github.com/GoogleCloudDataproc/hadoop-connectors/issues/998 27 | @RunWith(JUnit4.class) 28 | public class GoogleHadoopFileSystemJavaStorageClientIntegrationTest 29 | extends GoogleHadoopFileSystemIntegrationTest { 30 | 31 | @Before 32 | public void before() throws Exception { 33 | storageClientType = ClientType.STORAGE_CLIENT; 34 | super.before(); 35 | } 36 | 37 | @Ignore 38 | @Test 39 | public void testImpersonationGroupNameIdentifierUsed() {} 40 | 41 | @Ignore 42 | @Test 43 | public void testImpersonationServiceAccountAndUserAndGroupNameIdentifierUsed() {} 44 | 45 | @Ignore 46 | @Test 47 | public void testImpersonationServiceAccountUsed() {} 48 | 49 | @Ignore 50 | @Test 51 | public void testImpersonationUserAndGroupNameIdentifiersUsed() {} 52 | 53 | @Ignore 54 | @Test 55 | public void testImpersonationUserNameIdentifierUsed() {} 56 | 57 | @Ignore 58 | @Test 59 | public void unauthenticatedAccessToPublicBuckets_fsGsProperties() {} 60 | 61 | @Ignore 62 | @Test 63 | public void unauthenticatedAccessToPublicBuckets_googleCloudProperties() {} 64 | 65 | @Ignore 66 | @Test 67 | public void testRenameWithMoveOperation() {} 68 | 69 | @Ignore 70 | @Test 71 | public void testGcsJsonAPIMetrics() { 72 | // TODO: Update this will once gRPC API metrics are added 73 | } 74 | 75 | @Ignore 76 | @Test 77 | public void testGetFileStatusWithHint() throws Exception { 78 | // TODO: Update this will once gRPC API metrics are added 79 | } 80 | 81 | @Ignore 82 | @Test 83 | public void testGcsThreadLocalMetrics() { 84 | // TODO: Update this will once gRPC API metrics are added 85 | } 86 | 87 | @Ignore 88 | @Test 89 | public void multiThreadTest() { 90 | // TODO: Update this will once gRPC API metrics are added 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/InMemoryGlobberFileSystemTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import com.google.common.collect.ImmutableSet; 23 | import java.io.FileNotFoundException; 24 | import org.apache.hadoop.conf.Configuration; 25 | import org.apache.hadoop.fs.FileSystem; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.JUnit4; 29 | 30 | /** Unit tests for {@link InMemoryGlobberFileSystem} class. */ 31 | @RunWith(JUnit4.class) 32 | public class InMemoryGlobberFileSystemTest { 33 | 34 | @Test 35 | public void testGetFileStatus_shouldReturnExceptionForNotExistingFile() throws Exception { 36 | GoogleHadoopFileSystem ghfs = new InMemoryGoogleHadoopFileSystem(); 37 | 38 | FileSystem helperFileSystem = 39 | InMemoryGlobberFileSystem.createInstance( 40 | new Configuration(), ghfs.getWorkingDirectory(), ImmutableSet.of()); 41 | 42 | FileNotFoundException e = 43 | assertThrows( 44 | FileNotFoundException.class, 45 | () -> helperFileSystem.getFileStatus(ghfs.getWorkingDirectory())); 46 | 47 | assertThat(e) 48 | .hasMessageThat() 49 | .startsWith( 50 | String.format( 51 | "Path '%s' (qualified: '%s') does not exist.", 52 | ghfs.getWorkingDirectory(), ghfs.getWorkingDirectory())); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/TestUtils.java: -------------------------------------------------------------------------------- 1 | package com.google.cloud.hadoop.fs.gcs; 2 | 3 | /* 4 | * Copyright 2023 Google Inc. All Rights Reserved. 5 | * 6 | * Licensed under the Apache License, Version 2.0 (the "License"); 7 | * you may not use this file except in compliance with the License. 8 | * You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | 21 | import org.apache.hadoop.fs.GlobalStorageStatistics; 22 | import org.apache.hadoop.fs.statistics.IOStatistics; 23 | import org.apache.hadoop.fs.statistics.StoreStatisticNames; 24 | 25 | class TestUtils { 26 | 27 | static void verifyDurationMetric(IOStatistics ioStatistics, String symbol, int expected) { 28 | assertThat(ioStatistics.counters().get(symbol)).isEqualTo(expected); 29 | String minKey = String.format("%s%s", symbol, StoreStatisticNames.SUFFIX_MIN); 30 | String meanKey = String.format("%s%s", symbol, StoreStatisticNames.SUFFIX_MEAN); 31 | String maxKey = String.format("%s%s", symbol, StoreStatisticNames.SUFFIX_MAX); 32 | 33 | Long minValue = ioStatistics.minimums().get(minKey); 34 | Long maxValue = ioStatistics.maximums().get(maxKey); 35 | long meanValue = Double.valueOf(ioStatistics.meanStatistics().get(meanKey).mean()).longValue(); 36 | 37 | assertThat(minValue).isLessThan(maxValue + 1); 38 | assertThat(minValue).isLessThan(meanValue + 1); 39 | assertThat(meanValue).isLessThan(maxValue + 1); 40 | } 41 | 42 | static void verifyDurationMetric( 43 | GhfsGlobalStorageStatistics stats, String statistic, int expected) { 44 | String symbol = statistic; 45 | long minValue = stats.getMin(symbol); 46 | long maxValue = stats.getMax(symbol); 47 | long meanValue = Double.valueOf(stats.getMean(symbol)).longValue(); 48 | 49 | assertThat(stats.getLong(symbol)).isEqualTo(expected); 50 | assertThat(minValue).isLessThan(maxValue + 1); 51 | assertThat(minValue).isLessThan(meanValue + 1); 52 | assertThat(meanValue).isLessThan(maxValue + 1); 53 | } 54 | 55 | static GhfsGlobalStorageStatistics getStorageStatistics() { 56 | GhfsGlobalStorageStatistics stats = 57 | (GhfsGlobalStorageStatistics) 58 | GlobalStorageStatistics.INSTANCE.get(GhfsGlobalStorageStatistics.NAME); 59 | stats.reset(); 60 | return stats; 61 | } 62 | 63 | static void verifyCounter( 64 | GhfsGlobalStorageStatistics stats, GhfsStatistic statName, int expected) { 65 | assertThat(stats.getLong(statName.getSymbol())).isEqualTo(expected); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/auth/TestTokenIdentifierImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.auth; 18 | 19 | import org.apache.hadoop.io.Text; 20 | import org.apache.hadoop.security.token.delegation.web.DelegationTokenIdentifier; 21 | 22 | /** A test delegation token identifier implementation */ 23 | public class TestTokenIdentifierImpl extends DelegationTokenIdentifier { 24 | 25 | public static final Text KIND = new Text("GCPDelegationToken/Test"); 26 | 27 | public TestTokenIdentifierImpl() { 28 | super(KIND); 29 | } 30 | 31 | public TestTokenIdentifierImpl(Text owner, Text renewer, Text realUser, Text service) { 32 | super(KIND, owner, renewer, realUser); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/AbstractGoogleContractConcatTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import static org.apache.hadoop.fs.contract.ContractTestUtils.assertFileHasLength; 20 | import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile; 21 | import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset; 22 | 23 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorage; 24 | import org.apache.hadoop.fs.Path; 25 | import org.apache.hadoop.fs.contract.AbstractContractConcatTest; 26 | import org.apache.hadoop.fs.contract.ContractTestUtils; 27 | import org.junit.Test; 28 | 29 | public abstract class AbstractGoogleContractConcatTest extends AbstractContractConcatTest { 30 | 31 | @Test 32 | public void testConcatMultiple() throws Throwable { 33 | int numFiles = GoogleCloudStorage.MAX_COMPOSE_OBJECTS * 3 / 2; 34 | Path testPath = path("test"); 35 | 36 | byte[][] blocks = new byte[numFiles][0]; 37 | Path[] srcs = new Path[numFiles]; 38 | for (int i = 0; i < numFiles; i++) { 39 | Path srcFile = new Path(testPath, "" + i); 40 | blocks[i] = dataset(TEST_FILE_LEN, i, 255); 41 | createFile(getFileSystem(), srcFile, true, blocks[i]); 42 | srcs[i] = srcFile; 43 | } 44 | Path target = new Path(testPath, "target"); 45 | 46 | createFile(getFileSystem(), target, false, new byte[0]); 47 | getFileSystem().concat(target, srcs); 48 | assertFileHasLength(getFileSystem(), target, TEST_FILE_LEN * numFiles); 49 | ContractTestUtils.validateFileContent( 50 | ContractTestUtils.readDataset(getFileSystem(), target, TEST_FILE_LEN * numFiles), blocks); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/GoogleContract.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 20 | import com.google.cloud.hadoop.gcsio.testing.TestConfiguration; 21 | import com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType; 22 | import java.io.IOException; 23 | import org.apache.hadoop.conf.Configuration; 24 | import org.apache.hadoop.fs.FileSystem; 25 | import org.apache.hadoop.fs.Path; 26 | import org.apache.hadoop.fs.contract.AbstractBondedFSContract; 27 | 28 | /** Contract of GoogleHadoopFileSystem via scheme "gs". */ 29 | public class GoogleContract extends AbstractBondedFSContract { 30 | 31 | public static final String TEST_BUCKET_NAME_PREFIX = "ghfs-contract-test"; 32 | 33 | private static final String CONTRACT_XML = "contract/gs.xml"; 34 | 35 | public GoogleContract(Configuration conf, TestBucketHelper bucketHelper) { 36 | super(conf); 37 | addConfResource(CONTRACT_XML); 38 | conf.set("fs.contract.test.fs.gs", "gs://" + bucketHelper.getUniqueBucketPrefix()); 39 | 40 | TestConfiguration testConf = TestConfiguration.getInstance(); 41 | if (testConf.getProjectId() != null) { 42 | conf.set("fs.gs.project.id", testConf.getProjectId()); 43 | } 44 | if (testConf.getServiceAccountJsonKeyFile() != null) { 45 | conf.setEnum("fs.gs.auth.type", AuthenticationType.SERVICE_ACCOUNT_JSON_KEYFILE); 46 | conf.set("fs.gs.auth.service.account.json.keyfile", testConf.getServiceAccountJsonKeyFile()); 47 | } 48 | } 49 | 50 | @Override 51 | public void init() throws IOException { 52 | super.init(); 53 | FileSystem testFs = getTestFileSystem(); 54 | testFs.mkdirs(new Path(testFs.getUri())); 55 | } 56 | 57 | @Override 58 | public String getScheme() { 59 | return "gs"; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/InMemoryGoogleContract.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemTestHelper; 20 | import java.io.IOException; 21 | import org.apache.hadoop.conf.Configuration; 22 | import org.apache.hadoop.fs.FileSystem; 23 | import org.apache.hadoop.fs.Path; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | 26 | /** 27 | * Contract of GoogleHadoopFileSystem via scheme "gs" based on an in-memory fake of the GCS service. 28 | */ 29 | public class InMemoryGoogleContract extends AbstractFSContract { 30 | public static final String CONTRACT_XML = "contract/gs.xml"; 31 | 32 | // This bucket comes from our helper method which produces an in-memory fake for the test to use. 33 | public static final String FAKE_TEST_PATH = 34 | GoogleHadoopFileSystemTestHelper.IN_MEMORY_TEST_BUCKET + "/contract-test/"; 35 | 36 | private FileSystem fs; 37 | 38 | public InMemoryGoogleContract(Configuration conf) { 39 | super(conf); 40 | // Always enable these in-memory unittest versions of the contract tests. 41 | conf.set("fs.contract.test.fs.gs", FAKE_TEST_PATH); 42 | addConfResource(CONTRACT_XML); 43 | } 44 | 45 | @Override 46 | public void init() throws IOException { 47 | super.init(); 48 | fs = GoogleHadoopFileSystemTestHelper.createInMemoryGoogleHadoopFileSystem(); 49 | } 50 | 51 | @Override 52 | public FileSystem getTestFileSystem() throws IOException { 53 | return fs; 54 | } 55 | 56 | @Override 57 | public String getScheme() { 58 | return "gs"; 59 | } 60 | 61 | @Override 62 | public Path getTestPath() { 63 | return new Path(FAKE_TEST_PATH); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractAppend.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractAppendTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.apache.hadoop.fs.contract.ContractTestUtils; 26 | import org.junit.AfterClass; 27 | import org.junit.Before; 28 | import org.junit.runner.RunWith; 29 | import org.junit.runners.JUnit4; 30 | 31 | @RunWith(JUnit4.class) 32 | public class TestGoogleContractAppend extends AbstractContractAppendTest { 33 | 34 | private static final TestBucketHelper TEST_BUCKET_HELPER = 35 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 36 | 37 | private static final AtomicReference fs = new AtomicReference<>(); 38 | 39 | @Before 40 | public void before() { 41 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 42 | } 43 | 44 | @AfterClass 45 | public static void cleanup() throws Exception { 46 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 47 | } 48 | 49 | @Override 50 | protected AbstractFSContract createContract(Configuration conf) { 51 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 52 | } 53 | 54 | @Override 55 | public void testRenameFileBeingAppended() throws Throwable { 56 | ContractTestUtils.skip("blobstores can not rename file that being appended"); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractConcat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractFSContract; 24 | import org.junit.AfterClass; 25 | import org.junit.Before; 26 | 27 | /** GCS contract tests covering file concat. */ 28 | public class TestGoogleContractConcat extends AbstractGoogleContractConcatTest { 29 | 30 | private static final TestBucketHelper TEST_BUCKET_HELPER = 31 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 32 | 33 | private static final AtomicReference fs = new AtomicReference<>(); 34 | 35 | @Before 36 | public void before() { 37 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 38 | } 39 | 40 | @AfterClass 41 | public static void cleanup() throws Exception { 42 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 43 | } 44 | 45 | @Override 46 | protected AbstractFSContract createContract(Configuration conf) { 47 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractCreate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractCreateTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.apache.hadoop.fs.contract.ContractTestUtils; 26 | import org.junit.AfterClass; 27 | import org.junit.Before; 28 | import org.junit.runner.RunWith; 29 | import org.junit.runners.JUnit4; 30 | 31 | @RunWith(JUnit4.class) 32 | public class TestGoogleContractCreate extends AbstractContractCreateTest { 33 | 34 | private static final TestBucketHelper TEST_BUCKET_HELPER = 35 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 36 | 37 | private static final AtomicReference fs = new AtomicReference<>(); 38 | 39 | @Before 40 | public void before() { 41 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 42 | } 43 | 44 | @AfterClass 45 | public static void cleanup() throws Exception { 46 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 47 | } 48 | 49 | @Override 50 | protected AbstractFSContract createContract(Configuration conf) { 51 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 52 | } 53 | 54 | @Override 55 | public void testOverwriteEmptyDirectory() throws Throwable { 56 | ContractTestUtils.skip("blobstores can't distinguish empty directories from files"); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractDelete.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | 28 | /** GCS contract tests covering file delete. */ 29 | public class TestGoogleContractDelete extends AbstractContractDeleteTest { 30 | 31 | private static final TestBucketHelper TEST_BUCKET_HELPER = 32 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 33 | 34 | private static final AtomicReference fs = new AtomicReference<>(); 35 | 36 | @Before 37 | public void before() { 38 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 39 | } 40 | 41 | @AfterClass 42 | public static void cleanup() throws Exception { 43 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 44 | } 45 | 46 | @Override 47 | protected AbstractFSContract createContract(Configuration conf) { 48 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractGetFileStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.JUnit4; 29 | 30 | @RunWith(JUnit4.class) 31 | public class TestGoogleContractGetFileStatus extends AbstractContractGetFileStatusTest { 32 | 33 | private static final TestBucketHelper TEST_BUCKET_HELPER = 34 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 35 | 36 | private static final AtomicReference fs = new AtomicReference<>(); 37 | 38 | @Before 39 | public void before() { 40 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 41 | } 42 | 43 | @AfterClass 44 | public static void cleanup() throws Exception { 45 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 46 | } 47 | 48 | @Override 49 | protected AbstractFSContract createContract(Configuration conf) { 50 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractMkdir.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | 28 | /** GCS contract tests covering file mkdir. */ 29 | public class TestGoogleContractMkdir extends AbstractContractMkdirTest { 30 | 31 | private static final TestBucketHelper TEST_BUCKET_HELPER = 32 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 33 | 34 | private static final AtomicReference fs = new AtomicReference<>(); 35 | 36 | @Before 37 | public void before() { 38 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 39 | } 40 | 41 | @AfterClass 42 | public static void cleanup() throws Exception { 43 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 44 | } 45 | 46 | @Override 47 | protected AbstractFSContract createContract(Configuration conf) { 48 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractOpen.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractOpenTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | 28 | /** GCS contract tests covering file open. */ 29 | public class TestGoogleContractOpen extends AbstractContractOpenTest { 30 | 31 | private static final TestBucketHelper TEST_BUCKET_HELPER = 32 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 33 | 34 | private static final AtomicReference fs = new AtomicReference<>(); 35 | 36 | @Before 37 | public void before() { 38 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 39 | } 40 | 41 | @AfterClass 42 | public static void cleanup() throws Exception { 43 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 44 | } 45 | 46 | @Override 47 | protected AbstractFSContract createContract(Configuration conf) { 48 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractRename.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractRenameTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | 28 | /** GCS contract tests covering file rename. */ 29 | public class TestGoogleContractRename extends AbstractContractRenameTest { 30 | 31 | private static final TestBucketHelper TEST_BUCKET_HELPER = 32 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 33 | 34 | private static final AtomicReference fs = new AtomicReference<>(); 35 | 36 | @Before 37 | public void before() { 38 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 39 | } 40 | 41 | @AfterClass 42 | public static void cleanup() throws Exception { 43 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 44 | } 45 | 46 | @Override 47 | protected AbstractFSContract createContract(Configuration conf) { 48 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractRootDirectory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.JUnit4; 29 | 30 | /** GCS contract tests covering file root directory. */ 31 | @RunWith(JUnit4.class) 32 | public class TestGoogleContractRootDirectory extends AbstractContractRootDirectoryTest { 33 | 34 | private static final TestBucketHelper TEST_BUCKET_HELPER = 35 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 36 | 37 | private static final AtomicReference fs = new AtomicReference<>(); 38 | 39 | @Before 40 | public void before() { 41 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 42 | } 43 | 44 | @AfterClass 45 | public static void cleanup() throws Exception { 46 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 47 | } 48 | 49 | @Override 50 | protected AbstractFSContract createContract(Configuration conf) { 51 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestGoogleContractSeek.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; 20 | import com.google.cloud.hadoop.gcsio.integration.GoogleCloudStorageTestHelper.TestBucketHelper; 21 | import java.util.concurrent.atomic.AtomicReference; 22 | import org.apache.hadoop.conf.Configuration; 23 | import org.apache.hadoop.fs.contract.AbstractContractSeekTest; 24 | import org.apache.hadoop.fs.contract.AbstractFSContract; 25 | import org.junit.AfterClass; 26 | import org.junit.Before; 27 | 28 | /** GCS contract tests covering file seek. */ 29 | public class TestGoogleContractSeek extends AbstractContractSeekTest { 30 | 31 | private static final TestBucketHelper TEST_BUCKET_HELPER = 32 | new TestBucketHelper(GoogleContract.TEST_BUCKET_NAME_PREFIX); 33 | 34 | private static final AtomicReference fs = new AtomicReference<>(); 35 | 36 | @Before 37 | public void before() { 38 | fs.compareAndSet(null, (GoogleHadoopFileSystem) getFileSystem()); 39 | } 40 | 41 | @AfterClass 42 | public static void cleanup() throws Exception { 43 | TEST_BUCKET_HELPER.cleanup(fs.get().getGcsFs().getGcs()); 44 | } 45 | 46 | @Override 47 | protected AbstractFSContract createContract(Configuration conf) { 48 | return new GoogleContract(conf, TEST_BUCKET_HELPER); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractAppend.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractAppendTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | import org.apache.hadoop.fs.contract.ContractTestUtils; 23 | import org.junit.runner.RunWith; 24 | import org.junit.runners.JUnit4; 25 | 26 | @RunWith(JUnit4.class) 27 | public class TestInMemoryGoogleContractAppend extends AbstractContractAppendTest { 28 | 29 | @Override 30 | protected AbstractFSContract createContract(Configuration conf) { 31 | return new InMemoryGoogleContract(conf); 32 | } 33 | 34 | @Override 35 | public void testRenameFileBeingAppended() throws Throwable { 36 | ContractTestUtils.skip("blobstores can not rename file that being appended"); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractConcat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractFSContract; 21 | import org.junit.runner.RunWith; 22 | import org.junit.runners.JUnit4; 23 | 24 | @RunWith(JUnit4.class) 25 | public class TestInMemoryGoogleContractConcat extends AbstractGoogleContractConcatTest { 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractCreate.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractCreateTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | import org.apache.hadoop.fs.contract.ContractTestUtils; 23 | import org.junit.runner.RunWith; 24 | import org.junit.runners.JUnit4; 25 | 26 | @RunWith(JUnit4.class) 27 | public class TestInMemoryGoogleContractCreate extends AbstractContractCreateTest { 28 | 29 | @Override 30 | protected AbstractFSContract createContract(Configuration conf) { 31 | return new InMemoryGoogleContract(conf); 32 | } 33 | 34 | @Override 35 | public void testOverwriteEmptyDirectory() throws Throwable { 36 | ContractTestUtils.skip("blobstores can't distinguish empty directories from files"); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractDelete.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file delete using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractDelete extends AbstractContractDeleteTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractMkdir.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file mkdir using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractMkdir extends AbstractContractMkdirTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractOpen.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractOpenTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file open using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractOpen extends AbstractContractOpenTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractRename.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractRenameTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file rename using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractRename extends AbstractContractRenameTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractRootDirectory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file root directory using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractRootDirectory extends AbstractContractRootDirectoryTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/contract/TestInMemoryGoogleContractSeek.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.contract; 18 | 19 | import org.apache.hadoop.conf.Configuration; 20 | import org.apache.hadoop.fs.contract.AbstractContractSeekTest; 21 | import org.apache.hadoop.fs.contract.AbstractFSContract; 22 | 23 | /** GCS contract tests covering file seek using in-memory fakes. */ 24 | public class TestInMemoryGoogleContractSeek extends AbstractContractSeekTest { 25 | 26 | @Override 27 | protected AbstractFSContract createContract(Configuration conf) { 28 | return new InMemoryGoogleContract(conf); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /gcs/src/test/java/com/google/cloud/hadoop/fs/gcs/hcfs/GoogleHadoopFileSystemContractTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.fs.gcs.hcfs; 18 | 19 | import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystemTestHelper; 20 | import java.io.IOException; 21 | import org.apache.hadoop.fs.FileSystemContractBaseTest; 22 | import org.apache.hadoop.fs.Path; 23 | import org.junit.Before; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.JUnit4; 26 | 27 | /** 28 | * Runs the Hadoop tests in FileSystemContractBaseTest over the GoogleHadoopFileSystem. Tests that 29 | * the GoogleHadoopFileSystem obeys the file system contract specified for Hadoop. 30 | */ 31 | @RunWith(JUnit4.class) 32 | public class GoogleHadoopFileSystemContractTest extends FileSystemContractBaseTest { 33 | 34 | @Before 35 | @SuppressWarnings("MissingOverride") 36 | public void setUp() throws IOException { 37 | fs = GoogleHadoopFileSystemTestHelper.createInMemoryGoogleHadoopFileSystem(); 38 | } 39 | 40 | /** 41 | * Gets the default working directory of GHBRFS used by testWorkingDirectory(). 42 | * 43 | *

Note that the real default working directory of GHBRFS is gs:/. That value causes 44 | * testWorkingDirectory to fail because it is an incorrectly written test (it assumes that default 45 | * working directory is always a non-root directory). Therefore we use a non-root directory below. 46 | */ 47 | @Override 48 | protected String getDefaultWorkingDirectory() { 49 | return fs.getUri().resolve("test-work-dir").toString(); 50 | } 51 | 52 | /** Tests get/setWorkingDirectory(). */ 53 | @Override 54 | public void testWorkingDirectory() throws Exception { 55 | // Set the pseudo default working directory before the test begins. 56 | fs.setWorkingDirectory(new Path(getDefaultWorkingDirectory())); 57 | super.testWorkingDirectory(); 58 | } 59 | 60 | // Ignore: FS-level permissions are not supported in GCS 61 | @Override 62 | public void testMkdirsWithUmask() {} 63 | } 64 | -------------------------------------------------------------------------------- /gcs/src/test/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | com.google.cloud.hadoop.fs.gcs.auth.TestTokenIdentifierImpl 16 | -------------------------------------------------------------------------------- /gcs/src/test/resources/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 20 | 21 | 22 | hadoop.tmp.dir 23 | target/build/test 24 | A base for other temporary directories. 25 | true 26 | 27 | 28 | 29 | hadoop.security.authentication 30 | simple 31 | 32 | 33 | fs.gs.impl 34 | com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem 35 | 36 | 37 | fs.AbstractFileSystem.gs.impl 38 | com.google.cloud.hadoop.fs.gcs.GoogleHadoopFS 39 | 40 | 41 | 51 | 53 | 54 | 55 | 56 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/CreateBucketOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.auto.value.AutoValue; 20 | import java.time.Duration; 21 | import javax.annotation.Nullable; 22 | 23 | /** Options that can be specified when creating a bucket in the {@code GoogleCloudStorage}. */ 24 | @AutoValue 25 | public abstract class CreateBucketOptions { 26 | 27 | /** Create bucket with all default settings. */ 28 | public static final CreateBucketOptions DEFAULT = builder().build(); 29 | 30 | public static Builder builder() { 31 | return new AutoValue_CreateBucketOptions.Builder().setHierarchicalNamespaceEnabled(false); 32 | } 33 | 34 | public abstract Builder toBuilder(); 35 | 36 | /** Returns the bucket location. */ 37 | @Nullable 38 | public abstract String getLocation(); 39 | 40 | /** Returns the bucket storage class. */ 41 | @Nullable 42 | public abstract String getStorageClass(); 43 | 44 | public abstract boolean getHierarchicalNamespaceEnabled(); 45 | 46 | /** Returns the bucket retention period. */ 47 | @Nullable 48 | public abstract Duration getTtl(); 49 | 50 | /** Builder for {@link CreateBucketOptions} */ 51 | @AutoValue.Builder 52 | public abstract static class Builder { 53 | public abstract Builder setHierarchicalNamespaceEnabled(boolean enabled); 54 | 55 | public abstract Builder setLocation(String location); 56 | 57 | public abstract Builder setStorageClass(String storageClass); 58 | 59 | public abstract Builder setTtl(Duration ttl); 60 | 61 | public abstract CreateBucketOptions build(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageClientGrpcStatisticsInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google Inc. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.cloud.hadoop.util.GcsRequestExecutionEvent; 20 | import com.google.cloud.hadoop.util.GoogleCloudStorageEventBus; 21 | import com.google.common.annotations.VisibleForTesting; 22 | import io.grpc.CallOptions; 23 | import io.grpc.Channel; 24 | import io.grpc.ClientCall; 25 | import io.grpc.ClientInterceptor; 26 | import io.grpc.ForwardingClientCall.SimpleForwardingClientCall; 27 | import io.grpc.ForwardingClientCallListener.SimpleForwardingClientCallListener; 28 | import io.grpc.Metadata; 29 | import io.grpc.MethodDescriptor; 30 | import io.grpc.Status; 31 | 32 | /** This is a gRPC interceptor to capture the statistics related to calls made to gcs backend. */ 33 | @VisibleForTesting 34 | public class GoogleCloudStorageClientGrpcStatisticsInterceptor implements ClientInterceptor { 35 | 36 | @Override 37 | public ClientCall interceptCall( 38 | MethodDescriptor method, CallOptions callOptions, Channel next) { 39 | return new SimpleForwardingClientCall(next.newCall(method, callOptions)) { 40 | @Override 41 | public void start(Listener responseListener, Metadata headers) { 42 | try { 43 | GoogleCloudStorageEventBus.onGcsRequest(new GcsRequestExecutionEvent()); 44 | } finally { 45 | super.start( 46 | new SimpleForwardingClientCallListener(responseListener) { 47 | @Override 48 | public void onClose(Status status, Metadata trailers) { 49 | try { 50 | GoogleCloudStorageEventBus.onGrpcStatus(status); 51 | } finally { 52 | super.onClose(status, trailers); 53 | } 54 | } 55 | }, 56 | headers); 57 | } 58 | } 59 | }; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageTracingFields.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.common.annotations.VisibleForTesting; 20 | 21 | @VisibleForTesting 22 | public enum GoogleCloudStorageTracingFields { 23 | 24 | // GRPC specific fields 25 | RPC_METHOD("rpcMethod"), 26 | IDEMPOTENCY_TOKEN("idempotency-token"), 27 | REQUEST_COUNTER("requestCounter"), 28 | RESPONSE_COUNTER("responseCounter"), 29 | 30 | // Common fields 31 | RESOURCE("resource"), 32 | UPLOAD_ID("uploadId"), 33 | WRITE_OFFSET("writeOffset"), 34 | 35 | FINALIZE_WRITE("finalizeWrite"), 36 | CONTENT_LENGTH("contentLength"), 37 | PERSISTED_SIZE("persistedSize"), 38 | READ_OFFSET("readOffset"), 39 | READ_LIMIT("readLimit"), 40 | REQUEST_START_OFFSET("requestStartOffset"), 41 | BYTES_READ("bytesRead"), 42 | STREAM_OPERATION("streamOperation"), 43 | STATUS("status"); 44 | 45 | public final String name; 46 | 47 | GoogleCloudStorageTracingFields(String name) { 48 | this.name = name; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/GrpcStreamType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.common.annotations.VisibleForTesting; 20 | import java.util.Arrays; 21 | import java.util.Map; 22 | import java.util.stream.Collectors; 23 | 24 | @VisibleForTesting 25 | public enum GrpcStreamType { 26 | START_RESUMABLE_WRITE("StartResumableWrite"), 27 | WRITE_OBJECT("WriteObject"), 28 | READ_OBJECT("ReadObject"), 29 | OTHER("Other"); 30 | 31 | public final String name; 32 | 33 | GrpcStreamType(String name) { 34 | this.name = name; 35 | } 36 | 37 | private static final Map names = 38 | Arrays.stream(GrpcStreamType.values()) 39 | .collect(Collectors.toMap(x -> x.name.toUpperCase(), x -> x)); 40 | 41 | public static GrpcStreamType getTypeFromName(String name) { 42 | return names.getOrDefault(name.toUpperCase(), GrpcStreamType.OTHER); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/ListFileOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.OBJECT_FIELDS; 20 | 21 | import com.google.auto.value.AutoValue; 22 | import javax.annotation.Nullable; 23 | 24 | /** Options that can be specified when listing files in the {@link GoogleCloudStorageFileSystem}. */ 25 | @AutoValue 26 | public abstract class ListFileOptions { 27 | 28 | /** List all files in the directory. */ 29 | public static final ListFileOptions DEFAULT = builder().build(); 30 | 31 | public static Builder builder() { 32 | return new AutoValue_ListFileOptions.Builder().setFields(OBJECT_FIELDS); 33 | } 34 | 35 | public abstract Builder toBuilder(); 36 | 37 | /** 38 | * Comma separated list of object fields to include in the list response. 39 | * 40 | *

See 42 | * object resource for reference. 43 | */ 44 | @Nullable 45 | public abstract String getFields(); 46 | 47 | /** Builder for {@link ListFileOptions} */ 48 | @AutoValue.Builder 49 | public abstract static class Builder { 50 | public abstract Builder setFields(String delimiter); 51 | 52 | public abstract ListFileOptions build(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/ListFolderOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except 5 | * in compliance with the License. You may obtain a copy of the License at 6 | * 7 | * https://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software distributed under the 10 | * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either 11 | * express or implied. See the License for the specific language governing permissions and 12 | * limitations under the License. 13 | */ 14 | 15 | package com.google.cloud.hadoop.gcsio; 16 | 17 | import com.google.auto.value.AutoValue; 18 | 19 | /** 20 | * Options that can be specified when listing Folders in the {@link GoogleCloudStorage} and is valid 21 | * for only HN enabled bucket 22 | */ 23 | @AutoValue 24 | public abstract class ListFolderOptions { 25 | 26 | /** List all folders in the directory. */ 27 | public static final ListFolderOptions DEFAULT = builder().build(); 28 | 29 | public static Builder builder() { 30 | return new AutoValue_ListFolderOptions.Builder().setPageSize(5000); 31 | } 32 | 33 | public abstract Builder toBuilder(); 34 | 35 | /** Maximum folder resources in a single page */ 36 | public abstract int getPageSize(); 37 | 38 | /** Builder for {@link ListObjectOptions} */ 39 | @AutoValue.Builder 40 | public abstract static class Builder { 41 | 42 | public abstract Builder setPageSize(int pageSize); 43 | 44 | public abstract ListFolderOptions build(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/ListObjectOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorage.MAX_RESULTS_UNLIMITED; 20 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorage.PATH_DELIMITER; 21 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl.OBJECT_FIELDS; 22 | 23 | import com.google.auto.value.AutoValue; 24 | import javax.annotation.Nullable; 25 | 26 | /** Options that can be specified when listing objects in the {@link GoogleCloudStorage}. */ 27 | @AutoValue 28 | public abstract class ListObjectOptions { 29 | 30 | /** List all objects in the directory. */ 31 | public static final ListObjectOptions DEFAULT = builder().build(); 32 | 33 | /** List all objects with the prefix. */ 34 | public static final ListObjectOptions DEFAULT_FLAT_LIST = builder().setDelimiter(null).build(); 35 | 36 | public static Builder builder() { 37 | return new AutoValue_ListObjectOptions.Builder() 38 | .setDelimiter(PATH_DELIMITER) 39 | .setIncludePrefix(false) 40 | .setMaxResults(MAX_RESULTS_UNLIMITED) 41 | .setFields(OBJECT_FIELDS); 42 | } 43 | 44 | public abstract Builder toBuilder(); 45 | 46 | /** Delimiter to use (typically {@code /}), otherwise {@code null}. */ 47 | @Nullable 48 | public abstract String getDelimiter(); 49 | 50 | /** Whether to include prefix object in the result. */ 51 | public abstract boolean isIncludePrefix(); 52 | 53 | /** Maximum number of results to return, unlimited if negative or zero. */ 54 | public abstract long getMaxResults(); 55 | 56 | /** 57 | * Comma separated list of object fields to include in the list response. 58 | * 59 | *

See 61 | * object resource for reference. 62 | */ 63 | @Nullable 64 | public abstract String getFields(); 65 | 66 | /** Builder for {@link ListObjectOptions} */ 67 | @AutoValue.Builder 68 | public abstract static class Builder { 69 | public abstract Builder setDelimiter(String delimiter); 70 | 71 | public abstract Builder setIncludePrefix(boolean includePrefix); 72 | 73 | public abstract Builder setMaxResults(long maxResults); 74 | 75 | public abstract Builder setFields(String fields); 76 | 77 | public abstract ListObjectOptions build(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/MetricsRecorder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import io.opencensus.stats.Measure.MeasureLong; 20 | import io.opencensus.tags.TagKey; 21 | 22 | /** Interface for exposing utility methods to publish metrics via open-census api-spec. */ 23 | interface MetricsRecorder { 24 | 25 | /** 26 | * Publishes metric for specified open-census measurement 27 | * 28 | * @param key represents filterable attribute 29 | * @param value for the TagKey 30 | * @param ml - Measurement to be recorded 31 | * @param n - Value of the measurement recorded 32 | */ 33 | void recordTaggedStat(TagKey key, String value, MeasureLong ml, Long n); 34 | 35 | /** 36 | * Publishes metric for specified open-census measurement 37 | * 38 | * @param keys - List of TagKeys attributes used for filtration 39 | * @param values - corresponding values for list of TagKeys 40 | * @param ml - Measurement to be recorded 41 | * @param n - Value of the measurement recorded 42 | */ 43 | void recordLong(TagKey[] keys, String[] values, MeasureLong ml, Long n); 44 | } 45 | 46 | /** No-Op metrics publisher */ 47 | class NoOpMetricsRecorder implements MetricsRecorder { 48 | 49 | @Override 50 | public void recordTaggedStat(TagKey key, String value, MeasureLong ml, Long n) { 51 | // No-op 52 | } 53 | 54 | @Override 55 | public void recordLong(TagKey[] keys, String[] values, MeasureLong ml, Long n) { 56 | // No-op 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/ObjectWriteConditions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.api.services.storage.Storage; 20 | import com.google.auto.value.AutoValue; 21 | import javax.annotation.Nullable; 22 | 23 | /** 24 | * Conditions on which a object write should be allowed to continue. Corresponds to setting {@code 25 | * IfGenerationMatch} and {@code IfMetaGenerationMatch} in API requests. 26 | */ 27 | @AutoValue 28 | public abstract class ObjectWriteConditions { 29 | 30 | /** No conditions for completing the write. */ 31 | public static final ObjectWriteConditions NONE = builder().build(); 32 | 33 | public static Builder builder() { 34 | return new AutoValue_ObjectWriteConditions.Builder() 35 | .setMetaGenerationMatch(null) 36 | .setContentGenerationMatch(null); 37 | } 38 | 39 | public abstract Builder toBuilder(); 40 | 41 | @Nullable 42 | public abstract Long getMetaGenerationMatch(); 43 | 44 | public boolean hasMetaGenerationMatch() { 45 | return getMetaGenerationMatch() != null; 46 | } 47 | 48 | @Nullable 49 | public abstract Long getContentGenerationMatch(); 50 | 51 | public boolean hasContentGenerationMatch() { 52 | return getContentGenerationMatch() != null; 53 | } 54 | 55 | /** Apply the conditions represented by this object to an Insert operation. */ 56 | public void apply(Storage.Objects.Insert objectToInsert) { 57 | if (hasContentGenerationMatch()) { 58 | objectToInsert.setIfGenerationMatch(getContentGenerationMatch()); 59 | } 60 | 61 | if (hasMetaGenerationMatch()) { 62 | objectToInsert.setIfMetagenerationMatch(getMetaGenerationMatch()); 63 | } 64 | } 65 | 66 | @AutoValue.Builder 67 | public abstract static class Builder { 68 | public abstract Builder setMetaGenerationMatch(Long metaGenerationMatch); 69 | 70 | public abstract Builder setContentGenerationMatch(Long contentGenerationMatch); 71 | 72 | public abstract ObjectWriteConditions build(); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/PerformanceCachingGoogleCloudStorageOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.auto.value.AutoValue; 20 | import java.time.Duration; 21 | 22 | /** Configurable options for {@link PerformanceCachingGoogleCloudStorage}. */ 23 | @AutoValue 24 | public abstract class PerformanceCachingGoogleCloudStorageOptions { 25 | 26 | public static final PerformanceCachingGoogleCloudStorageOptions DEFAULT = builder().build(); 27 | 28 | public static Builder builder() { 29 | return new AutoValue_PerformanceCachingGoogleCloudStorageOptions.Builder() 30 | .setMaxEntryAge(Duration.ofSeconds(5)); 31 | } 32 | 33 | public abstract Builder toBuilder(); 34 | 35 | /** Gets the max age of an item in cache in milliseconds. */ 36 | public abstract Duration getMaxEntryAge(); 37 | 38 | /** Builder class for PerformanceCachingGoogleCloudStorageOptions. */ 39 | @AutoValue.Builder 40 | public abstract static class Builder { 41 | 42 | /** Sets the max age of an item in cache in milliseconds. */ 43 | public abstract Builder setMaxEntryAge(Duration maxEntryAge); 44 | 45 | public abstract PerformanceCachingGoogleCloudStorageOptions build(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticTypeEnum.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | /** Enum of statistic types. */ 20 | public enum StatisticTypeEnum { 21 | 22 | /** Counter. Counts the number of occurrences of each operation */ 23 | TYPE_COUNTER, 24 | 25 | /** Duration. Duration for the execution of opertaion */ 26 | TYPE_DURATION, 27 | 28 | /** Gauge. */ 29 | TYPE_GAUGE, 30 | 31 | /* Duration. Stores everything stored by TYPE_DURATION and total time taken. 32 | * This is to avoid storing and computing total duration of an operation repeatedly. Instread this can be done at the time of querying metric by multipying mean and count 33 | * */ 34 | TYPE_DURATION_TOTAL 35 | } 36 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/StatisticsTrackingHttpRequestInitializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.common.collect.ImmutableMap.toImmutableMap; 20 | 21 | import com.google.api.client.http.HttpRequest; 22 | import com.google.api.client.http.HttpRequestInitializer; 23 | import com.google.common.base.Ascii; 24 | import com.google.common.collect.ImmutableMap; 25 | import java.io.IOException; 26 | import java.util.Map; 27 | import java.util.concurrent.ConcurrentHashMap; 28 | import java.util.concurrent.ConcurrentMap; 29 | import java.util.concurrent.atomic.AtomicLong; 30 | 31 | /** Tracks and updates the statistics related to HTTP requests */ 32 | class StatisticsTrackingHttpRequestInitializer implements HttpRequestInitializer { 33 | 34 | private final ConcurrentMap statistics = new ConcurrentHashMap<>(); 35 | 36 | @Override 37 | public void initialize(HttpRequest request) throws IOException { 38 | request.setResponseInterceptor( 39 | httpResponse -> { 40 | String requestMethod = Ascii.toLowerCase(httpResponse.getRequest().getRequestMethod()); 41 | String statKey = String.format("http_%s_%d", requestMethod, httpResponse.getStatusCode()); 42 | statistics.computeIfAbsent(statKey, k -> new AtomicLong()).incrementAndGet(); 43 | }); 44 | } 45 | 46 | public ImmutableMap getStatistics() { 47 | return statistics.entrySet().stream() 48 | .collect(toImmutableMap(Map.Entry::getKey, e -> e.getValue().get())); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/UpdatableItemInfo.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import java.util.Map; 20 | 21 | /** Item information that can be updated without re-writing an entire object. */ 22 | public class UpdatableItemInfo { 23 | 24 | private final StorageResourceId storageResourceId; 25 | private final Map metadata; 26 | 27 | /** 28 | * @param storageResourceId The StorageResourceId for which these records pertain 29 | * @param metadata The object metadata for this object. 30 | */ 31 | public UpdatableItemInfo(StorageResourceId storageResourceId, Map metadata) { 32 | this.storageResourceId = storageResourceId; 33 | this.metadata = metadata; 34 | } 35 | 36 | public StorageResourceId getStorageResourceId() { 37 | return storageResourceId; 38 | } 39 | 40 | public Map getMetadata() { 41 | return metadata; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/VerificationAttributes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2016 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import java.util.Arrays; 20 | import javax.annotation.Nullable; 21 | 22 | /** GCS provided validation attributes for a single object. */ 23 | public class VerificationAttributes { 24 | private final byte[] md5hash; 25 | private final byte[] crc32c; 26 | 27 | public VerificationAttributes(@Nullable byte[] md5hash, @Nullable byte[] crc32c) { 28 | this.md5hash = md5hash; 29 | this.crc32c = crc32c; 30 | } 31 | 32 | /** MD5 hash of an object, if available. */ 33 | @Nullable 34 | public byte[] getMd5hash() { 35 | return md5hash; 36 | } 37 | 38 | /** CRC32c checksum of an object, if available. */ 39 | @Nullable 40 | public byte[] getCrc32c() { 41 | return crc32c; 42 | } 43 | 44 | @Override 45 | public boolean equals(Object o) { 46 | if (this == o) { 47 | return true; 48 | } 49 | if (!(o instanceof VerificationAttributes)) { 50 | return false; 51 | } 52 | VerificationAttributes that = (VerificationAttributes) o; 53 | return Arrays.equals(md5hash, that.md5hash) && Arrays.equals(crc32c, that.crc32c); 54 | } 55 | 56 | @Override 57 | public int hashCode() { 58 | int result = Arrays.hashCode(md5hash); 59 | result = 31 * result + Arrays.hashCode(crc32c); 60 | return result; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /gcsio/src/main/java/com/google/cloud/hadoop/gcsio/testing/TestConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio.testing; 18 | 19 | /** Access to test configurations values. */ 20 | public abstract class TestConfiguration { 21 | public static final String GCS_TEST_PROJECT_ID = "GCS_TEST_PROJECT_ID"; 22 | public static final String GCS_TEST_JSON_KEYFILE = "GCS_TEST_JSON_KEYFILE"; 23 | 24 | public static final String GCS_TEST_DIRECT_PATH_PREFERRED = "GCS_TEST_DIRECT_PATH_PREFERRED"; 25 | 26 | /** Environment-based test configuration. */ 27 | public static class EnvironmentBasedTestConfiguration extends TestConfiguration { 28 | @Override 29 | public String getProjectId() { 30 | return System.getenv(GCS_TEST_PROJECT_ID); 31 | } 32 | 33 | @Override 34 | public String getServiceAccountJsonKeyFile() { 35 | return System.getenv(GCS_TEST_JSON_KEYFILE); 36 | } 37 | 38 | @Override 39 | public boolean isDirectPathPreferred() { 40 | String envVar = System.getenv(GCS_TEST_DIRECT_PATH_PREFERRED); 41 | // if env variable is not configured default behaviour is to attempt directPath 42 | if (envVar == null) { 43 | return true; 44 | } 45 | return Boolean.parseBoolean(envVar); 46 | } 47 | } 48 | 49 | public static TestConfiguration getInstance() { 50 | return LazyHolder.INSTANCE; 51 | } 52 | 53 | private static class LazyHolder { 54 | private static final TestConfiguration INSTANCE = new EnvironmentBasedTestConfiguration(); 55 | } 56 | 57 | public abstract String getProjectId(); 58 | 59 | public abstract String getServiceAccountJsonKeyFile(); 60 | 61 | public abstract boolean isDirectPathPreferred(); 62 | } 63 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/BatchExecutorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import com.google.common.util.concurrent.FutureCallback; 23 | import java.io.IOException; 24 | import org.junit.Before; 25 | import org.junit.Test; 26 | import org.junit.runner.RunWith; 27 | import org.junit.runners.JUnit4; 28 | 29 | /** Unit tests for {@link BatchExecutor}. */ 30 | @RunWith(JUnit4.class) 31 | public class BatchExecutorTest { 32 | 33 | private BatchExecutor batchExecutor; 34 | 35 | @Before 36 | public void setUp() { 37 | batchExecutor = new BatchExecutor(10); 38 | } 39 | 40 | @Test 41 | public void queue_succeeds() { 42 | batchExecutor.queue(() -> true, /* callback*/ assertCallBack()); 43 | } 44 | 45 | @Test 46 | public void queue_throwsException_afterShutdownCalled() throws IOException { 47 | batchExecutor.shutdown(); 48 | 49 | IllegalStateException e = 50 | assertThrows(IllegalStateException.class, () -> batchExecutor.queue(() -> null, null)); 51 | 52 | assertThat(e) 53 | .hasMessageThat() 54 | .startsWith("requestExecutor should not be terminated to queue request"); 55 | } 56 | 57 | private FutureCallback assertCallBack() { 58 | return new FutureCallback<>() { 59 | @Override 60 | public void onSuccess(Boolean result) { 61 | assertThat(result).isEqualTo(true); 62 | } 63 | 64 | @Override 65 | public void onFailure(Throwable throwable) {} 66 | }; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/CreateFileOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static java.nio.charset.StandardCharsets.UTF_8; 21 | import static org.junit.Assert.assertThrows; 22 | 23 | import com.google.common.collect.ImmutableMap; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.JUnit4; 27 | 28 | @RunWith(JUnit4.class) 29 | public class CreateFileOptionsTest { 30 | @Test 31 | public void validOptions() { 32 | CreateFileOptions.builder() 33 | .setAttributes(ImmutableMap.of("Innocuous", "text".getBytes(UTF_8))) 34 | .setContentType("text") 35 | .setWriteMode(CreateFileOptions.WriteMode.OVERWRITE) 36 | .setOverwriteGenerationId(0) 37 | .build(); 38 | } 39 | 40 | @Test 41 | public void invalidOptions_contentType_shouldNotBeSetViaAttributes() { 42 | IllegalArgumentException thrown = 43 | assertThrows( 44 | IllegalArgumentException.class, 45 | () -> 46 | CreateFileOptions.builder() 47 | .setAttributes(ImmutableMap.of("Content-Type", "text".getBytes(UTF_8))) 48 | .setContentType("text") 49 | .build()); 50 | 51 | assertThat(thrown) 52 | .hasMessageThat() 53 | .isEqualTo("The Content-Type attribute must be set via the contentType option"); 54 | } 55 | 56 | @Test 57 | public void invalidOptions_createNew_overwriteGenerationShouldNotBeSet() { 58 | IllegalArgumentException thrown = 59 | assertThrows( 60 | IllegalArgumentException.class, 61 | () -> 62 | CreateFileOptions.builder() 63 | .setWriteMode(CreateFileOptions.WriteMode.CREATE_NEW) 64 | .setOverwriteGenerationId(0) 65 | .build()); 66 | 67 | assertThat(thrown) 68 | .hasMessageThat() 69 | .isEqualTo("overwriteGenerationId is set to 0 but it can be set only in OVERWRITE mode"); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/CreateObjectOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2015 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static java.nio.charset.StandardCharsets.UTF_8; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import com.google.common.collect.ImmutableMap; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.JUnit4; 26 | 27 | @RunWith(JUnit4.class) 28 | public class CreateObjectOptionsTest { 29 | 30 | @Test 31 | public void build_checksContentTypeMetadata() { 32 | CreateObjectOptions.builder() 33 | .setMetadata(ImmutableMap.of("Innocuous-Type", "".getBytes(UTF_8))) 34 | .build(); 35 | 36 | assertThrows( 37 | IllegalArgumentException.class, 38 | () -> 39 | CreateObjectOptions.builder() 40 | .setMetadata(ImmutableMap.of("Content-Type", "".getBytes(UTF_8))) 41 | .build()); 42 | } 43 | 44 | @Test 45 | public void build_checksContentEncodingMetadata() { 46 | CreateObjectOptions.builder() 47 | .setMetadata(ImmutableMap.of("Innocuous-Encoding", "".getBytes(UTF_8))) 48 | .build(); 49 | 50 | assertThrows( 51 | IllegalArgumentException.class, 52 | () -> 53 | CreateObjectOptions.builder() 54 | .setMetadata(ImmutableMap.of("Content-Encoding", "".getBytes(UTF_8))) 55 | .build()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/DeletionBehavior.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | /** 20 | * DeletionBehavior distills out the fine-grained expected behavior of a FileSystem in various 21 | * edge/error cases; implementations associated with each possible FileSystem thus encapsulate the 22 | * minor differences in behavior in delete() calls. 23 | */ 24 | public interface DeletionBehavior { 25 | /** Returns the MethodOutcome of trying non-recursive deletes for non-empty directories. */ 26 | MethodOutcome nonEmptyDeleteOutcome(); 27 | 28 | /** Returns the MethodOutcome of trying to delete a non-existent filename. */ 29 | MethodOutcome nonExistentDeleteOutcome(); 30 | } 31 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/FakeServer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.cloud.NoCredentials; 20 | import com.google.cloud.storage.GrpcStorageOptions; 21 | import com.google.cloud.storage.StorageOptions; 22 | import com.google.storage.v2.StorageGrpc; 23 | import io.grpc.Server; 24 | import io.grpc.netty.shaded.io.grpc.netty.NettyServerBuilder; 25 | import java.io.IOException; 26 | import java.net.InetSocketAddress; 27 | import java.util.concurrent.TimeUnit; 28 | 29 | final class FakeServer implements AutoCloseable { 30 | 31 | private final Server server; 32 | private final GrpcStorageOptions grpcStorageOptions; 33 | 34 | FakeServer(Server server, GrpcStorageOptions grpcStorageOptions) { 35 | this.server = server; 36 | this.grpcStorageOptions = grpcStorageOptions; 37 | } 38 | 39 | public GrpcStorageOptions getGrpcStorageOptions() { 40 | return grpcStorageOptions; 41 | } 42 | 43 | @Override 44 | public void close() throws InterruptedException { 45 | server.shutdownNow().awaitTermination(10, TimeUnit.SECONDS); 46 | } 47 | 48 | public static FakeServer of(StorageGrpc.StorageImplBase service) throws IOException { 49 | InetSocketAddress address = new InetSocketAddress("localhost", 0); 50 | Server server = NettyServerBuilder.forAddress(address).addService(service).build(); 51 | server.start(); 52 | String endpoint = String.format("%s:%d", address.getHostString(), server.getPort()); 53 | GrpcStorageOptions grpcStorageOptions = 54 | StorageOptions.grpc() 55 | .setHost("http://" + endpoint) 56 | .setProjectId("test-proj") 57 | .setCredentials(NoCredentials.getInstance()) 58 | .build(); 59 | return new FakeServer(server, grpcStorageOptions); 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/FakeWriteChannel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.cloud.RestorableState; 20 | import com.google.cloud.WriteChannel; 21 | import java.io.IOException; 22 | import java.nio.ByteBuffer; 23 | 24 | /** FakeWriterChannel which writes only half the passed in byteBuffer capacity at a time. */ 25 | public class FakeWriteChannel implements WriteChannel { 26 | 27 | private boolean open = true; 28 | 29 | private final boolean throwExceptionOnWrite; 30 | 31 | public FakeWriteChannel() { 32 | this.throwExceptionOnWrite = false; 33 | } 34 | 35 | public FakeWriteChannel(boolean throwExceptionOnWrite) { 36 | this.throwExceptionOnWrite = throwExceptionOnWrite; 37 | } 38 | 39 | @Override 40 | public void setChunkSize(int i) {} 41 | 42 | @Override 43 | public RestorableState capture() { 44 | return null; 45 | } 46 | 47 | @Override 48 | public int write(ByteBuffer src) throws IOException { 49 | if (throwExceptionOnWrite) { 50 | throw new IOException("Intentionally triggered"); 51 | } 52 | int bytesWritten = 0; 53 | // always writes half or less from the provided byte buffer capacity 54 | int capacity = src.capacity(); 55 | if ((src.limit() - src.position()) <= capacity / 2) { 56 | bytesWritten = src.limit(); 57 | src.position(src.limit()); 58 | } else { 59 | bytesWritten = capacity / 2; 60 | src.position(src.position() + capacity / 2); 61 | } 62 | return bytesWritten; 63 | } 64 | 65 | @Override 66 | public boolean isOpen() { 67 | return open; 68 | } 69 | 70 | @Override 71 | public void close() throws IOException { 72 | open = false; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemHTTPClientTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; 20 | import org.junit.Before; 21 | import org.junit.runner.RunWith; 22 | import org.junit.runners.JUnit4; 23 | 24 | @RunWith(JUnit4.class) 25 | public class GoogleCloudStorageFileSystemHTTPClientTest 26 | extends GoogleCloudStorageFileSystemTestBase { 27 | 28 | @Before 29 | public void before() throws Exception { 30 | storageClientType = ClientType.HTTP_API_CLIENT; 31 | super.before(); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageFileSystemJavaStorageClientTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystemOptions.ClientType; 20 | import java.io.IOException; 21 | import org.junit.Before; 22 | import org.junit.Ignore; 23 | import org.junit.Test; 24 | import org.junit.runner.RunWith; 25 | import org.junit.runners.JUnit4; 26 | 27 | @RunWith(JUnit4.class) 28 | public class GoogleCloudStorageFileSystemJavaStorageClientTest 29 | extends GoogleCloudStorageFileSystemTestBase { 30 | 31 | @Before 32 | public void before() throws Exception { 33 | storageClientType = ClientType.STORAGE_CLIENT; 34 | super.before(); 35 | } 36 | 37 | @Override 38 | @Ignore("DirectPath is not supported with null credentials") 39 | @Test 40 | public void testConstructor() throws IOException {} 41 | 42 | @Override 43 | @Ignore("DirectPath is not supported with null credentials") 44 | @Test 45 | public void testClientType() {} 46 | } 47 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageReadOptionsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.junit.runners.JUnit4; 25 | 26 | @RunWith(JUnit4.class) 27 | public class GoogleCloudStorageReadOptionsTest { 28 | 29 | @Test 30 | public void build_throwsException_whenInplaceSeekLimitLowerThanZero() { 31 | long inplaceSeekLimit = -123; 32 | GoogleCloudStorageReadOptions.Builder builder = 33 | GoogleCloudStorageReadOptions.builder().setInplaceSeekLimit(inplaceSeekLimit); 34 | 35 | IllegalStateException e = assertThrows(IllegalStateException.class, builder::build); 36 | 37 | assertThat(e) 38 | .hasMessageThat() 39 | .isEqualTo("inplaceSeekLimit must be non-negative! Got " + inplaceSeekLimit); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/GoogleCloudStorageWriteChannelTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2019 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.BUCKET_NAME; 20 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.HTTP_TRANSPORT; 21 | import static com.google.cloud.hadoop.gcsio.GoogleCloudStorageTestUtils.OBJECT_NAME; 22 | import static com.google.common.truth.Truth.assertThat; 23 | import static com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService; 24 | 25 | import com.google.api.client.http.InputStreamContent; 26 | import com.google.api.client.json.gson.GsonFactory; 27 | import com.google.api.services.storage.Storage; 28 | import com.google.cloud.hadoop.util.AsyncWriteChannelOptions; 29 | import com.google.cloud.hadoop.util.ClientRequestHelper; 30 | import java.io.ByteArrayInputStream; 31 | import java.io.IOException; 32 | import org.junit.Test; 33 | import org.junit.runner.RunWith; 34 | import org.junit.runners.JUnit4; 35 | 36 | /** Unit tests for {@link GoogleCloudStorageWriteChannel} class. */ 37 | @RunWith(JUnit4.class) 38 | public class GoogleCloudStorageWriteChannelTest { 39 | 40 | @Test 41 | public void createRequest_shouldSetKmsKeyName() throws IOException { 42 | String kmsKeyName = "testKmsKey"; 43 | 44 | GoogleCloudStorageWriteChannel writeChannel = 45 | new GoogleCloudStorageWriteChannel( 46 | new Storage(HTTP_TRANSPORT, GsonFactory.getDefaultInstance(), r -> {}), 47 | new ClientRequestHelper<>(), 48 | newDirectExecutorService(), 49 | AsyncWriteChannelOptions.DEFAULT, 50 | new StorageResourceId(BUCKET_NAME, OBJECT_NAME), 51 | CreateObjectOptions.DEFAULT_NO_OVERWRITE.toBuilder().setKmsKeyName(kmsKeyName).build(), 52 | ObjectWriteConditions.NONE); 53 | 54 | Storage.Objects.Insert request = 55 | writeChannel.createRequest( 56 | new InputStreamContent("plain/text", new ByteArrayInputStream(new byte[0]))); 57 | 58 | assertThat(request.getKmsKeyName()).isEqualTo(kmsKeyName); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/MethodOutcome.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | /** 20 | * MethodOutcome encapsulates the possible outcomes of calling a method which returns a boolean 21 | * value; either "return true", "return false", or "throw exception"; the latter case will also have 22 | * an associated expected Exception class. 23 | */ 24 | public class MethodOutcome { 25 | /** 26 | * The possible high-level categories of outcomes which are mutually exclusive; this way, there is 27 | * no need to worry about whether some boolean is necessarily "true" or "false" in the case of an 28 | * exception being thrown, since there will be no return value in such a case. 29 | */ 30 | public enum Type { 31 | RETURNS_TRUE, 32 | RETURNS_FALSE, 33 | THROWS_EXCEPTION 34 | } 35 | 36 | private final Type expectedOutcome; 37 | private final Class expectedExceptionClass; 38 | 39 | public MethodOutcome(Type expectedOutcome) { 40 | this.expectedOutcome = expectedOutcome; 41 | this.expectedExceptionClass = null; 42 | } 43 | 44 | public MethodOutcome(Type expectedOutcome, Class expectedExceptionClass) { 45 | this.expectedOutcome = expectedOutcome; 46 | this.expectedExceptionClass = expectedExceptionClass; 47 | } 48 | 49 | @Override 50 | public String toString() { 51 | switch (expectedOutcome) { 52 | case RETURNS_TRUE: 53 | return "Returns true"; 54 | case RETURNS_FALSE: 55 | return "Returns false"; 56 | case THROWS_EXCEPTION: 57 | return "Throws " + expectedExceptionClass; 58 | default: 59 | return "Unknown MethodOutcome.Type: " + expectedOutcome; 60 | } 61 | } 62 | 63 | /** Accessor for expected Type enum. */ 64 | public Type getType() { 65 | return expectedOutcome; 66 | } 67 | 68 | /** Accessor for expected Exception class. */ 69 | public Class getExceptionClass() { 70 | return expectedExceptionClass; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/MkdirsBehavior.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | /** 20 | * MkdirsBehavior distills out the fine-grained expected behavior of a FileSystem in various 21 | * edge/error cases; implementations associated with each possible FileSystem thus encapsulate the 22 | * minor differences in behavior in mkdirs() calls. 23 | */ 24 | public interface MkdirsBehavior { 25 | /** Returns the MethodOutcome of trying to mkdirs root. */ 26 | MethodOutcome mkdirsRootOutcome(); 27 | 28 | /** Returns the MethodOutcome of trying mkdirs if a file of the same name already exists. */ 29 | MethodOutcome fileAlreadyExistsOutcome(); 30 | } 31 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/RenameBehavior.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | /** 20 | * RenameBehavior distills out the fine-grained expected behavior of a FileSystem in various 21 | * edge/error cases; implementations associated with each possible FileSystem thus encapsulate the 22 | * minor differences in behavior in rename() calls. 23 | */ 24 | public interface RenameBehavior { 25 | /** Returns the MethodOutcome of trying to rename an existing file into the root directory. */ 26 | MethodOutcome renameFileIntoRootOutcome(); 27 | 28 | /** Returns the MethodOutcome of trying to rename the root. */ 29 | MethodOutcome renameRootOutcome(); 30 | 31 | /** Returns the MethodOutcome when the src doesn't exist. */ 32 | MethodOutcome nonExistentSourceOutcome(); 33 | 34 | /** Returns the MethodOutcome when dst already exists, is a file, and src is also a file. */ 35 | MethodOutcome destinationFileExistsSrcIsFileOutcome(); 36 | 37 | /** Returns the MethodOutcome when dst already exists, is a file, and src is a directory. */ 38 | MethodOutcome destinationFileExistsSrcIsDirectoryOutcome(); 39 | 40 | /** Returns the MethodOutcome when a parent of file dst doesn't exist. */ 41 | MethodOutcome nonExistentDestinationFileParentOutcome(); 42 | 43 | /** Returns the MethodOutcome when a parent of directory dst doesn't exist. */ 44 | MethodOutcome nonExistentDestinationDirectoryParentOutcome(); 45 | } 46 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/StorageRequestFactoryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import static org.junit.Assert.assertThrows; 20 | 21 | import com.google.api.client.http.HttpTransport; 22 | import com.google.api.client.http.javanet.NetHttpTransport; 23 | import com.google.api.client.json.gson.GsonFactory; 24 | import com.google.api.services.storage.Storage; 25 | import org.junit.Before; 26 | import org.junit.Test; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.JUnit4; 29 | 30 | /** Tests storage requests creation and runtime behavior. */ 31 | @RunWith(JUnit4.class) 32 | public class StorageRequestFactoryTest { 33 | 34 | public static final HttpTransport HTTP_TRANSPORT = new NetHttpTransport(); 35 | 36 | private Storage storage; 37 | 38 | @Before 39 | public void setUp() { 40 | storage = 41 | new Storage( 42 | HTTP_TRANSPORT, GsonFactory.getDefaultInstance(), /* httpRequestInitializer= */ null); 43 | } 44 | 45 | @Test 46 | public void throwExceptionWhenCallExecute() { 47 | String BUCKET = "TEST_BUCKET"; 48 | String OBJECT = "TEST_OBJECT"; 49 | assertThrows( 50 | StorageRequestFactory.WrongRequestTypeException.class, 51 | new StorageRequestFactory(storage).objectsGetData(BUCKET, OBJECT)::execute); 52 | } 53 | 54 | @Test 55 | public void throwExceptionWhenCallExecuteMedia() { 56 | String BUCKET = "TEST_BUCKET"; 57 | String OBJECT = "TEST_OBJECT"; 58 | assertThrows( 59 | StorageRequestFactory.WrongRequestTypeException.class, 60 | new StorageRequestFactory(storage).objectsGetMetadata(BUCKET, OBJECT)::executeMedia); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/TestServerHeaderInterceptor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio; 18 | 19 | import io.grpc.Metadata; 20 | import io.grpc.ServerCall; 21 | import io.grpc.ServerCallHandler; 22 | import io.grpc.ServerInterceptor; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | 26 | // Header interceptor to capture the header information received at the server side 27 | class TestServerHeaderInterceptor implements ServerInterceptor { 28 | private final List allMeta = new ArrayList<>(); 29 | 30 | @Override 31 | public ServerCall.Listener interceptCall( 32 | ServerCall serverCall, 33 | Metadata metadata, 34 | ServerCallHandler serverCallHandler) { 35 | this.allMeta.add(metadata); 36 | 37 | return serverCallHandler.startCall(serverCall, metadata); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /gcsio/src/test/java/com/google/cloud/hadoop/gcsio/integration/GoogleCloudStorageIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.gcsio.integration; 18 | 19 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorage; 20 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl; 21 | import com.google.cloud.hadoop.gcsio.GoogleCloudStorageOptions; 22 | import com.google.cloud.hadoop.gcsio.PerformanceCachingGoogleCloudStorage; 23 | import com.google.cloud.hadoop.gcsio.PerformanceCachingGoogleCloudStorageOptions; 24 | import java.io.IOException; 25 | import java.util.Arrays; 26 | import java.util.Collection; 27 | import org.junit.runner.RunWith; 28 | import org.junit.runners.Parameterized; 29 | 30 | @RunWith(Parameterized.class) 31 | public class GoogleCloudStorageIntegrationTest extends GoogleCloudStorageTest { 32 | 33 | @Parameterized.Parameters 34 | public static Collection getConstructorArguments() throws IOException { 35 | return Arrays.asList( 36 | new Object[] {getGoogleCloudStorage()}, 37 | new Object[] {GoogleCloudStorageTestHelper.createGcsClientImpl()}, 38 | new Object[] {getPerformanceCachingGoogleCloudStorage()}); 39 | } 40 | 41 | private static GoogleCloudStorage getPerformanceCachingGoogleCloudStorage() throws IOException { 42 | return new PerformanceCachingGoogleCloudStorage( 43 | getGoogleCloudStorage(GoogleCloudStorageTestHelper.getStandardOptionBuilder()), 44 | PerformanceCachingGoogleCloudStorageOptions.DEFAULT); 45 | } 46 | 47 | private static GoogleCloudStorage getGoogleCloudStorage() throws IOException { 48 | return getGoogleCloudStorage(GoogleCloudStorageTestHelper.getStandardOptionBuilder()); 49 | } 50 | 51 | private static GoogleCloudStorage getGoogleCloudStorage( 52 | GoogleCloudStorageOptions.Builder optionsBuilder) throws IOException { 53 | return GoogleCloudStorageImpl.builder() 54 | .setOptions(optionsBuilder.build()) 55 | .setCredentials(GoogleCloudStorageTestHelper.getCredentials()) 56 | .build(); 57 | } 58 | 59 | public GoogleCloudStorageIntegrationTest(GoogleCloudStorage gcs) { 60 | super(gcs); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /gcsio/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,stdout 2 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 3 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.stdout.layout.ConversionPattern=%d{yyMMdd HH:mm:ss.SSS}:%1p [%c] %m%n 5 | -------------------------------------------------------------------------------- /tools/run_integration_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2013 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # tools/run_integration_tests.sh [optional_maven_parameters, [...]] 18 | 19 | set -Eeuo pipefail 20 | 21 | export GCS_TEST_PROJECT_ID=$1 22 | GCS_TEST_JSON_KEYFILE=$2 23 | 24 | print_usage() { 25 | echo -en "Usage:\n $0 [optional_maven_parameters, [...]]" 26 | } 27 | 28 | check_required_param() { 29 | local -r value=$1 30 | local -r error_msg=$2 31 | if [[ -z ${value} ]]; then 32 | echo "${error_msg}" 33 | print_usage 34 | exit 1 35 | fi 36 | } 37 | 38 | check_required_params() { 39 | check_required_param "${GCS_TEST_PROJECT_ID}" "Project ID parameter is required." 40 | check_required_param "${GCS_TEST_JSON_KEYFILE}" "Keyfile parameter is required." 41 | 42 | if [[ ! -f ${GCS_TEST_JSON_KEYFILE} ]]; then 43 | echo "Can't find keyfile: ${GCS_TEST_JSON_KEYFILE}" 44 | print_usage 45 | exit 1 46 | fi 47 | } 48 | 49 | check_required_params 50 | 51 | # When tests run, they run in the root of the module directory. Anything 52 | # relative to our current directory won't work properly 53 | if [[ $(uname -s) == Darwin ]]; then 54 | # On MacOS `readlink` does not support `-f` parameter 55 | # so we need manually resolve absolute path and symlink after that 56 | abs_path() { 57 | echo "$(cd $(dirname "$1"); pwd)/$(basename "$1")" 58 | } 59 | GCS_TEST_JSON_KEYFILE=$(abs_path "${GCS_TEST_JSON_KEYFILE}") 60 | while [[ -L ${GCS_TEST_JSON_KEYFILE} ]]; do 61 | GCS_TEST_JSON_KEYFILE=$(readlink -n "${GCS_TEST_JSON_KEYFILE}") 62 | done 63 | else 64 | GCS_TEST_JSON_KEYFILE=$(readlink -n -f "${GCS_TEST_JSON_KEYFILE}") 65 | fi 66 | if [[ ! -f ${GCS_TEST_JSON_KEYFILE} ]]; then 67 | echo "Resolved keyfile does not exist: ${GCS_TEST_JSON_KEYFILE}" 68 | exit 1 69 | fi 70 | 71 | export GCS_TEST_JSON_KEYFILE 72 | export HDFS_ROOT=file:///tmp 73 | export RUN_INTEGRATION_TESTS=true 74 | 75 | ./mvnw -B -e -T1C -Pintegration-test clean verify "${@:3}" 76 | -------------------------------------------------------------------------------- /util-hadoop/src/main/java/com/google/cloud/hadoop/util/AccessTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import java.io.IOException; 20 | import java.time.Instant; 21 | import java.util.List; 22 | import org.apache.hadoop.conf.Configurable; 23 | 24 | /** A provider to provide access token, and upon access token expiration, the utility to refresh. */ 25 | public interface AccessTokenProvider extends Configurable { 26 | 27 | /** Supported access token types. */ 28 | enum AccessTokenType { 29 | /** The same access token generated for all the GCS requests. */ 30 | GENERIC, 31 | 32 | /** A downscoped access token generated for each request. */ 33 | DOWNSCOPED 34 | } 35 | 36 | /** An access token and its expiration time. */ 37 | class AccessToken { 38 | 39 | private final String token; 40 | private final Instant expirationTime; 41 | 42 | public AccessToken(String token, Instant expirationTime) { 43 | this.token = token; 44 | this.expirationTime = expirationTime; 45 | } 46 | 47 | /** 48 | * @return the Access Token string. 49 | */ 50 | public String getToken() { 51 | return token; 52 | } 53 | 54 | /** 55 | * @return the Time when the token will expire, expressed in milliseconds. 56 | */ 57 | public Instant getExpirationTime() { 58 | return expirationTime; 59 | } 60 | } 61 | 62 | /** 63 | * @return an access token type. 64 | */ 65 | default AccessTokenType getAccessTokenType() { 66 | return AccessTokenType.GENERIC; 67 | } 68 | 69 | /** 70 | * @return an access token. 71 | */ 72 | AccessToken getAccessToken(); 73 | 74 | /** 75 | * @param accessBoundaries access boundaries used to generate a downscoped access token. 76 | * @return an access token. 77 | */ 78 | default AccessToken getAccessToken(List accessBoundaries) { 79 | throw new UnsupportedOperationException("Downscoped access tokens are not supported"); 80 | } 81 | 82 | /** 83 | * Force this provider to refresh its access token. 84 | * 85 | * @throws IOException when refresh fails. 86 | */ 87 | void refresh() throws IOException; 88 | } 89 | -------------------------------------------------------------------------------- /util-hadoop/src/main/java/com/google/cloud/hadoop/util/testing/CredentialsConfigurationUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util.testing; 18 | 19 | import static com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AUTHENTICATION_TYPE_SUFFIX; 20 | import static com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.BASE_KEY_PREFIX; 21 | 22 | import com.google.cloud.hadoop.util.HadoopCredentialsConfiguration.AuthenticationType; 23 | import org.apache.hadoop.conf.Configuration; 24 | 25 | /** Utility methods for creating Configuration objects for use in testing. */ 26 | public class CredentialsConfigurationUtil { 27 | 28 | public static void addTestConfigurationSettings(Configuration configuration) { 29 | configuration.setEnum( 30 | BASE_KEY_PREFIX + AUTHENTICATION_TYPE_SUFFIX.getKey(), AuthenticationType.UNAUTHENTICATED); 31 | } 32 | 33 | public static Configuration getTestConfiguration() { 34 | Configuration configuration = new Configuration(); 35 | addTestConfigurationSettings(configuration); 36 | return configuration; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /util-hadoop/src/main/java/com/google/cloud/hadoop/util/testing/HadoopConfigurationUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util.testing; 18 | 19 | import com.google.cloud.hadoop.util.HadoopConfigurationProperty; 20 | import java.lang.reflect.Field; 21 | import java.util.Arrays; 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | /** Utils for Hadoop configuration testing */ 26 | public class HadoopConfigurationUtils { 27 | 28 | public static Map getDefaultProperties(Class configurationClass) { 29 | return Arrays.stream(configurationClass.getDeclaredFields()) 30 | .filter(f -> HadoopConfigurationProperty.class.isAssignableFrom(f.getType())) 31 | .map(HadoopConfigurationUtils::getDefaultProperty) 32 | .collect( 33 | HashMap::new, 34 | (map, p) -> { 35 | for (String deprecatedKey : p.getDeprecatedKeys()) { 36 | map.put(deprecatedKey, p.getDefault()); 37 | } 38 | map.put(p.getKey(), p.getDefault()); 39 | }, 40 | HashMap::putAll); 41 | } 42 | 43 | private static HadoopConfigurationProperty getDefaultProperty(Field field) { 44 | try { 45 | return (HadoopConfigurationProperty) field.get(null); 46 | } catch (IllegalAccessException e) { 47 | throw new RuntimeException( 48 | String.format("Failed to get '%s' field value", field.getName()), e); 49 | } 50 | } 51 | 52 | private HadoopConfigurationUtils() {} 53 | } 54 | -------------------------------------------------------------------------------- /util-hadoop/src/main/java/com/google/cloud/hadoop/util/testing/TestingAccessTokenProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util.testing; 18 | 19 | import com.google.cloud.hadoop.util.AccessTokenProvider; 20 | import java.time.Instant; 21 | import org.apache.hadoop.conf.Configuration; 22 | 23 | /** A mock implementation of the {@link AccessTokenProvider} interface to be used in tests. */ 24 | public final class TestingAccessTokenProvider implements AccessTokenProvider { 25 | 26 | public static final String FAKE_ACCESS_TOKEN = "invalid-access-token"; 27 | public static final Instant EXPIRATION_TIME = Instant.now().plusSeconds(600); 28 | 29 | private Configuration config; 30 | 31 | @Override 32 | public AccessToken getAccessToken() { 33 | return new AccessToken(FAKE_ACCESS_TOKEN, EXPIRATION_TIME); 34 | } 35 | 36 | @Override 37 | public void refresh() {} 38 | 39 | @Override 40 | public void setConf(Configuration config) { 41 | this.config = config; 42 | } 43 | 44 | @Override 45 | public Configuration getConf() { 46 | return config; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util-hadoop/src/test/resources/test-credentials.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "service_account", 3 | "project_id": "test-project", 4 | "private_key_id": "test-key-id", 5 | "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCA/Yvv/PRpgXY4\n38cAyI+HLuTI7fI6YLd9bnqS0ivuIQof4+ymr8RBXvPwkSFubKoOIrYxu2DS9PH2\n3z84y1vvv5RnCM0I/V6JJ5sR41iWsT8pAws9ZonbXuc0DElkzJiFjc+Eo4EYB8eQ\nD27CZT6r/ikuglvYzRk3jVqs5dOwU/XXYDTWsQJmxZWQjsBIpbRO6U//XczX88A1\nAP3LzDeZX5JGI+2TRdzbI+wrHm3zl0XMBI0o/Q985M7nctBR8dauRLeYLpKHIda/\nFWCuPu201+U+v9OGNxcUtTDGBJ5aDVT/0XLcpD4kaIWBVyp9kh9XSh/WNlP747Zh\nFIVan5PDAgMBAAECggEAdAAQLbWFM7CaWdFczKaQ9NhGJTBfawJ5YRLOpmTd3tvV\nmBJISyq8SrGSnOBXeU+9qtKqmJTTIUPoqcPgvjmbp0LZucMfaJs6vgthGDgk/ffO\nPhqfrAsfpKl3MMZv5ndcHpcGbVkBClRMspA+YhHcx53c+ReAc7sWl2/GgMB6nBHH\nLazWXu2oeXiRVyHvN9kbBpHpaYU32mpxRYLMpebcXxjZOOVwLMPqwp4PIrNmgPxz\n0HeyqMyga97rpJm47I9J6vnw1s9hHBiBngyauR9oO8Akr+klLfh6wmEIimNPdz+1\noYWUl2zUbOUcJ8Jf1f84AM29/6nw+7pyMLR/saR/eQKBgQDg+H3WbBZpQQXWlgqO\ngT8PHRFfkRBjk6pBke+jv3j1RbXxZRxtF4/XWgkg47QJt1q70oavA3jPhlVF2Dsu\nd14v4Nav7bQ0gycwkLQeeHCQpa2HX9f9VoYr4fLE5bU/PgSf6vJzejVNBmcsej2T\nOmzLL0tn2sU6+6WMKdK5S1J5FQKBgQCSyBSGP5oXthmy6c4m2e8Mb2+zd11iqoEV\n8ERDPJtRjkQK84wx4nENG5CKwWmc9zU1MjVdDU3T4gtVxaS6i8urT1tYMGZKl9io\nPw73It8nhxvOEhDFQ1a5gObwRbVLlAInQcdjt8DXa+oa1hoDkr4QQJb1iKzFmJ6a\nMrXp1JDfdwKBgQCSaMWkdUWEjF+bTpxiL7T1RgqXclYDKAemkq4tmB/FJ/AHwLDm\nQm5HTX4hXG8/34b6M0GZxp2qGReH0/h3/hFbAW1Wo1Ei17P6hEHkjqdbS6eKMZLk\neZnqFysifwgct2TVGlddVcw4a5FbvdYy6dQBy668UQEIqNsNQ+6yzlVHHQKBgDy/\nul7lq+zsXS6DPeMuh5Vp+Z0ZZfkUAGEIE/iP+OiELP0rV3a99XQrWGNVMVTZq5fc\ns+FTREbRY0eYiVj7ysuYCIIzZeKxfeYEUzlNoXZGoOdk6rP57sbP5sTC1Qn2s/1z\naAaJYrLF4xpASnv4G9m2IcR1UKxnQh0np8qjQ2J5AoGBALtVc9rK59VN1YmS/IsY\nEiEVJtfdSfZdYevTTM8kAw1ZGq8TaaO4bg0/cDP9fZ+XRDWxO+iqEkBZ51utI6Vl\n19RYCWmECcomWb3tVCh1LhsAHN3RUVwFnqi/bYkZ1VZ2c0sTnvyXucG2hvsPP1pK\nMT3oYkMbkP/T3hg80HDA3ssF\n-----END PRIVATE KEY-----\n", 6 | "client_email": "test-email@gserviceaccount.com", 7 | "client_id": "test-client-id", 8 | "auth_uri": "https://accounts.google.com/o/oauth2/auth", 9 | "token_uri": "https://accounts.google.com/o/oauth2/token", 10 | "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", 11 | "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test-email%40gserviceaccount.com" 12 | } -------------------------------------------------------------------------------- /util-hadoop/src/test/resources/test-wip-config.json: -------------------------------------------------------------------------------- 1 | { 2 | "type": "external_account", 3 | "audience": "//iam.googleapis.com/projects/test/locations/global/workloadIdentityPools/test-pool/providers/tester", 4 | "subject_token_type": "urn:ietf:params:oauth:token-type:jwt", 5 | "service_account_impersonation_url": "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/tester@test.com:generateAccessToken", 6 | "token_url": "https://sts.googleapis.com/v1/token", 7 | "credential_source": { 8 | "url": "http://localhost:5000/token", 9 | "format": { 10 | "type": "json", 11 | "subject_token_field_name": "id_token" 12 | } 13 | } 14 | } -------------------------------------------------------------------------------- /util/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 19 | 4.0.0 20 | 21 | 22 | com.google.cloud.bigdataoss 23 | bigdataoss-parent 24 | 3.0.0-SNAPSHOT 25 | ../pom.xml 26 | 27 | 28 | util 29 | 3.0.0-SNAPSHOT 30 | 31 | 32 | 33 | com.google.http-client 34 | google-http-client-gson 35 | 36 | 37 | com.google.api-client 38 | google-api-client-gson 39 | 40 | 41 | com.google.apis 42 | google-api-services-iamcredentials 43 | 44 | 45 | com.google.apis 46 | google-api-services-storage 47 | 48 | 49 | com.google.guava 50 | guava 51 | 52 | 53 | com.google.auto.value 54 | auto-value-annotations 55 | 56 | 57 | com.google.auth 58 | google-auth-library-oauth2-http 59 | 60 | 61 | com.google.oauth-client 62 | google-oauth-client 63 | 64 | 65 | com.google.flogger 66 | google-extensions 67 | 68 | 69 | com.google.flogger 70 | flogger-system-backend 71 | 72 | 73 | 74 | 75 | junit 76 | junit 77 | 78 | 79 | org.mockito 80 | mockito-core 81 | 82 | 83 | com.google.truth 84 | truth 85 | 86 | 87 | io.grpc 88 | grpc-api 89 | 90 | 91 | 92 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/AccessBoundary.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.auto.value.AutoValue; 20 | 21 | /** An access boundary used to generate a downscoped access token. */ 22 | @AutoValue 23 | public abstract class AccessBoundary { 24 | public static AccessBoundary create(String bucketName, String objectName, Action action) { 25 | return new AutoValue_AccessBoundary(bucketName, objectName, action); 26 | } 27 | 28 | public abstract String bucketName(); 29 | 30 | public abstract String objectName(); 31 | 32 | public abstract Action action(); 33 | 34 | public enum Action { 35 | CREATE_BUCKETS, 36 | DELETE_BUCKETS, 37 | DELETE_OBJECTS, 38 | // EDIT_OBJECTS implies READ_OBJECTS and WRITE_OBJECTS 39 | EDIT_OBJECTS, 40 | GET_BUCKETS, 41 | GET_METADATA_OBJECTS, 42 | LIST_BUCKETS, 43 | LIST_OBJECTS, 44 | READ_OBJECTS, 45 | UNSPECIFIED_ACTION, 46 | WRITE_OBJECTS, 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/ClientRequestHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2013 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.api.client.googleapis.services.AbstractGoogleClientRequest; 20 | import com.google.api.client.http.HttpHeaders; 21 | 22 | /** 23 | * ClientRequestHelper provides wrapper methods around final methods of AbstractGoogleClientRequest 24 | * to allow overriding them if necessary. Typically should be used for testing purposes only. 25 | */ 26 | public class ClientRequestHelper { 27 | /** Wraps AbstractGoogleClientRequest.getRequestHeaders(). */ 28 | public HttpHeaders getRequestHeaders(AbstractGoogleClientRequest clientRequest) { 29 | return clientRequest.getRequestHeaders(); 30 | } 31 | 32 | /** Wraps AbstractGoogleClientRequest.getMediaHttpUploader(). */ 33 | public void setChunkSize(AbstractGoogleClientRequest clientRequest, int chunkSize) { 34 | clientRequest.getMediaHttpUploader().setChunkSize(chunkSize); 35 | } 36 | 37 | /** 38 | * Configures the {@code clientRequest} to enable/disable direct (single-request) uploads 39 | * according to {@code enable}. 40 | */ 41 | public void setDirectUploadEnabled(AbstractGoogleClientRequest clientRequest, boolean enable) { 42 | clientRequest.getMediaHttpUploader().setDirectUploadEnabled(enable); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/ErrorTypeExtractor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | /** Translates exceptions from API calls into ErrorType */ 20 | public interface ErrorTypeExtractor { 21 | 22 | enum ErrorType { 23 | NOT_FOUND, 24 | OUT_OF_RANGE, 25 | ALREADY_EXISTS, 26 | FAILED_PRECONDITION, 27 | INTERNAL, 28 | RESOURCE_EXHAUSTED, 29 | UNAVAILABLE, 30 | UNKNOWN 31 | } 32 | 33 | ErrorType getErrorType(Exception exception); 34 | 35 | /** Determines if the given exception indicates that bucket already exists. */ 36 | boolean bucketAlreadyExists(Exception e); 37 | } 38 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/GcsRequestExecutionEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.common.annotations.VisibleForTesting; 20 | 21 | /** This an Event which is published in EvenBus queue whenever a gcs request is created/executed. */ 22 | @VisibleForTesting 23 | public class GcsRequestExecutionEvent {} 24 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/GoogleCloudStorageEventBus.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.common.eventbus.EventBus; 20 | import io.grpc.Status; 21 | import java.io.IOException; 22 | 23 | /** Event Bus class */ 24 | public class GoogleCloudStorageEventBus { 25 | 26 | /** Hold the instance of the event bus here */ 27 | private static EventBus eventBus = new EventBus(); 28 | 29 | private static IOException exception = new IOException(); 30 | 31 | /** 32 | * Method to register an obj to event bus 33 | * 34 | * @param obj to register to event bus 35 | */ 36 | public static void register(Object obj) { 37 | eventBus.register(obj); 38 | } 39 | 40 | /** 41 | * Method to unregister an obj to event bus 42 | * 43 | * @param obj to unregister from event bus 44 | * @throws IllegalArgumentException if the object was not previously registered. 45 | */ 46 | public static void unregister(Object obj) { 47 | eventBus.unregister(obj); 48 | } 49 | 50 | /** 51 | * Posting Gcs request execution event i.e. request to gcs is being initiated. 52 | * 53 | * @param event dummy event to map to request execution type. 54 | */ 55 | public static void onGcsRequest(GcsRequestExecutionEvent event) { 56 | eventBus.post(event); 57 | } 58 | 59 | /** 60 | * Posting Exception to invoke corresponding Subscriber method. Passing a dummy exception as 61 | * EventBus has @ElementTypesAreNonnullByDefault annotation. 62 | */ 63 | public static void postOnException() { 64 | eventBus.post(exception); 65 | } 66 | 67 | /** 68 | * Posting grpc Status to invoke the corresponding Subscriber method. 69 | * 70 | * @param status status object of grpc response 71 | */ 72 | public static void onGrpcStatus(Status status) { 73 | eventBus.post(status); 74 | } 75 | 76 | public static void postGcsJsonApiEvent(IGcsJsonApiEvent gcsJsonApiEvent) { 77 | eventBus.post(gcsJsonApiEvent); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/IGcsJsonApiEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | public interface IGcsJsonApiEvent { 20 | GcsJsonApiEvent.EventType getEventType(); 21 | 22 | Object getContext(); 23 | 24 | String getMethod(); 25 | 26 | Object getProperty(String key); 27 | } 28 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/ITraceFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | public interface ITraceFactory { 20 | ITraceOperation createRootWithLogging(String name, Object context); 21 | } 22 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/ITraceOperation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | public interface ITraceOperation extends AutoCloseable { 20 | @Override 21 | void close(); 22 | 23 | ThreadTrace getTrace(); 24 | 25 | void annotate(String resultSize, int size); 26 | } 27 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/PropertyUtil.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.common.flogger.GoogleLogger; 20 | import java.io.IOException; 21 | import java.io.InputStream; 22 | import java.util.Properties; 23 | 24 | /** Helpers for interacting with properties files */ 25 | public class PropertyUtil { 26 | private static final GoogleLogger logger = GoogleLogger.forEnclosingClass(); 27 | 28 | /** 29 | * Get the value of a property or a default value if there's an error retrieving the property key. 30 | * 31 | * @param referenceClass The class which should be used to find the property file. The property 32 | * file is expected to be packaged in the same directory as this class. 33 | * @param propertyFile The name of the property file to be read. 34 | * @param key The property key to find in the property file. 35 | * @param defaultValue The value to return if no property with the given key is found or if the 36 | * property file cannot be found. 37 | * @return The value specified in the property file or defaultValue if an error occurs or if the 38 | * key could not be found 39 | */ 40 | public static String getPropertyOrDefault( 41 | Class referenceClass, String propertyFile, String key, String defaultValue) { 42 | try (InputStream stream = referenceClass.getResourceAsStream(propertyFile)) { 43 | if (stream == null) { 44 | logger.atSevere().log("Could not load properties file '%s'", propertyFile); 45 | return defaultValue; 46 | } 47 | Properties properties = new Properties(); 48 | properties.load(stream); 49 | String value = properties.getProperty(key); 50 | if (value == null) { 51 | logger.atSevere().log("Key %s not found in properties file %s.", key, propertyFile); 52 | return defaultValue; 53 | } 54 | return value; 55 | } catch (IOException e) { 56 | logger.atSevere().withCause(e).log( 57 | "Error while trying to get property value for key %s", key); 58 | return defaultValue; 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/RedactedString.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.common.base.Strings.isNullOrEmpty; 20 | 21 | import com.google.auto.value.AutoValue; 22 | import javax.annotation.Nullable; 23 | 24 | /** 25 | * Holder class for string values that should not be logged and displayed when {@code toString} 26 | * method called. For example, it should be used for credentials. 27 | */ 28 | @AutoValue 29 | public abstract class RedactedString { 30 | 31 | @Nullable 32 | public static RedactedString create(@Nullable String value) { 33 | return isNullOrEmpty(value) ? null : new AutoValue_RedactedString(value); 34 | } 35 | 36 | public abstract String value(); 37 | 38 | @Override 39 | public final String toString() { 40 | return ""; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/RequesterPaysOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.auto.value.AutoValue; 20 | import com.google.common.collect.ImmutableSet; 21 | import java.util.Collection; 22 | import javax.annotation.Nullable; 23 | 24 | /** Options for the GCS Requester Pays feature. */ 25 | @AutoValue 26 | public abstract class RequesterPaysOptions { 27 | 28 | /** Operational modes of GCS Requester Pays feature. */ 29 | public enum RequesterPaysMode { 30 | AUTO, 31 | CUSTOM, 32 | DISABLED, 33 | ENABLED 34 | } 35 | 36 | public static final RequesterPaysOptions DEFAULT = builder().build(); 37 | 38 | public static Builder builder() { 39 | return new AutoValue_RequesterPaysOptions.Builder() 40 | .setBuckets(ImmutableSet.of()) 41 | .setMode(RequesterPaysMode.DISABLED); 42 | } 43 | 44 | public abstract Builder toBuilder(); 45 | 46 | public abstract RequesterPaysMode getMode(); 47 | 48 | @Nullable 49 | public abstract String getProjectId(); 50 | 51 | public abstract ImmutableSet getBuckets(); 52 | 53 | /** Builder for {@link RequesterPaysOptions} */ 54 | @AutoValue.Builder 55 | public abstract static class Builder { 56 | public abstract Builder setMode(RequesterPaysMode mode); 57 | 58 | public abstract Builder setProjectId(@Nullable String projectId); 59 | 60 | public abstract Builder setBuckets(Collection value); 61 | 62 | public abstract Builder setBuckets(String... value); 63 | 64 | public abstract RequesterPaysOptions build(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/RetryBoundedBackOff.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.api.client.util.Preconditions.checkArgument; 20 | 21 | import com.google.api.client.util.BackOff; 22 | import java.io.IOException; 23 | import javax.annotation.concurrent.NotThreadSafe; 24 | 25 | /** 26 | * An implementation of {@link BackOff} that limits the number of calls on another {@link BackOff}. 27 | * This class will call the same methods of another BackOff until the maximum number of retries are 28 | * reached, and then it will return {@code BackOff.STOP}. 29 | */ 30 | @NotThreadSafe 31 | public class RetryBoundedBackOff implements BackOff { 32 | private int retriesAttempted = 0; 33 | private final int maxRetries; 34 | private final BackOff backoff; 35 | 36 | /** 37 | * RetryBoundedBackOff takes a {@link BackOff} and limits the retries. 38 | * 39 | * @param maxRetries Number of retries to attempt. Must be greater or equal to 0. 40 | * @param backoff The underlying {@link BackOff} to use. 41 | */ 42 | public RetryBoundedBackOff(BackOff backoff, int maxRetries) { 43 | checkArgument(maxRetries >= 0, "Maximum number of retries must not be less than 0."); 44 | this.backoff = backoff; 45 | this.maxRetries = maxRetries; 46 | } 47 | 48 | @Override 49 | public void reset() throws IOException { 50 | backoff.reset(); 51 | retriesAttempted = 0; 52 | } 53 | 54 | @Override 55 | public long nextBackOffMillis() throws IOException { 56 | if (retriesAttempted >= maxRetries) { 57 | return BackOff.STOP; 58 | } 59 | long next = backoff.nextBackOffMillis(); 60 | if (next == BackOff.STOP) { 61 | return BackOff.STOP; 62 | } 63 | retriesAttempted++; 64 | return next; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/RetryDeterminer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import com.google.api.client.http.HttpResponseException; 20 | import java.io.IOException; 21 | 22 | /** 23 | * This abstract class is designed to tell if an exception is transient and should result in a retry 24 | * or not, and should result in a returned exception to the caller. Meant to be used with a {@link 25 | * ResilientOperation}. 26 | * 27 | * @param The type of exception you are checking and could possibly return. 28 | */ 29 | @FunctionalInterface 30 | public interface RetryDeterminer { 31 | 32 | /** Retries exception when either {@code SOCKET_ERRORS} or {@code SERVER_ERRORS} would retry. */ 33 | RetryDeterminer DEFAULT = 34 | new RetryDeterminer<>() { 35 | @Override 36 | public boolean shouldRetry(Exception e) { 37 | return e instanceof IOException 38 | && (SOCKET_ERRORS.shouldRetry((IOException) e) 39 | || SERVER_ERRORS.shouldRetry((IOException) e)); 40 | } 41 | }; 42 | 43 | /** Always retries. */ 44 | RetryDeterminer ALL_ERRORS = e -> true; 45 | 46 | /** Socket errors retry determiner retries on socket exceptions. */ 47 | RetryDeterminer SOCKET_ERRORS = IoExceptionHelper::isSocketError; 48 | 49 | /** Server errors RetryDeterminer decides to retry on HttpResponseExceptions that return a 500. */ 50 | RetryDeterminer SERVER_ERRORS = 51 | e -> 52 | e instanceof HttpResponseException 53 | // TODO: Find out what we should do for 500 codes that are not always transient. 54 | && ((HttpResponseException) e).getStatusCode() / 100 == 5; 55 | 56 | /** A rate limited determiner that uses a default {@link ApiErrorExtractor}. */ 57 | RetryDeterminer RATE_LIMIT_ERRORS = ApiErrorExtractor.INSTANCE::rateLimited; 58 | 59 | /** 60 | * Determines if we should attempt a retry depending on the caught exception. 61 | * 62 | *

To indicate that no retry should be made, return false. If no retry, the exception should be 63 | * returned to the user. 64 | * 65 | * @param e Exception of type X that can be examined to determine if a retry is possible. 66 | * @return true if should retry, false otherwise 67 | */ 68 | boolean shouldRetry(X e); 69 | } 70 | -------------------------------------------------------------------------------- /util/src/main/java/com/google/cloud/hadoop/util/TraceFactory.java: -------------------------------------------------------------------------------- 1 | package com.google.cloud.hadoop.util; 2 | /* 3 | * Copyright 2023 Google Inc. 4 | * 5 | * Licensed under the Apache License, Version 2.0 (the "License"); 6 | * you may not use this file except in compliance with the License. 7 | * You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | */ 17 | 18 | public class TraceFactory { 19 | private TraceFactory() {} 20 | 21 | private static final ITraceFactory NOOP = new NoopTraceFactory(); 22 | 23 | public static ITraceFactory get(boolean isEnabled) { 24 | if (isEnabled) { 25 | return new EnabledTraceFactory(); 26 | } 27 | 28 | return TraceFactory.NOOP; 29 | } 30 | 31 | private static class EnabledTraceFactory implements ITraceFactory { 32 | @Override 33 | public ITraceOperation createRootWithLogging(String name, Object context) { 34 | return TraceOperation.createRootWithLogging(name, context); 35 | } 36 | } 37 | 38 | private static class NoopTraceFactory implements ITraceFactory { 39 | @Override 40 | public ITraceOperation createRootWithLogging(String name, Object context) { 41 | return TraceOperation.NOOP; 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/AccessBoundaryTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | 21 | import com.google.cloud.hadoop.util.AccessBoundary.Action; 22 | import org.junit.Test; 23 | import org.junit.runner.RunWith; 24 | import org.junit.runners.JUnit4; 25 | 26 | @RunWith(JUnit4.class) 27 | public class AccessBoundaryTest { 28 | 29 | /** 30 | * This test is to confirm the enum names don't change announced. The Action enum is a downstream 31 | * contract that shouldn't change current values to avoid breaking changes. 32 | */ 33 | @Test 34 | public void Action_enumNames() { 35 | assertThat(Action.LIST_OBJECTS.name()).isEqualTo("LIST_OBJECTS"); 36 | assertThat(Action.READ_OBJECTS.name()).isEqualTo("READ_OBJECTS"); 37 | assertThat(Action.WRITE_OBJECTS.name()).isEqualTo("WRITE_OBJECTS"); 38 | assertThat(Action.EDIT_OBJECTS.name()).isEqualTo("EDIT_OBJECTS"); 39 | assertThat(Action.GET_BUCKETS.name()).isEqualTo("GET_BUCKETS"); 40 | assertThat(Action.CREATE_BUCKETS.name()).isEqualTo("CREATE_BUCKETS"); 41 | assertThat(Action.DELETE_BUCKETS.name()).isEqualTo("DELETE_BUCKETS"); 42 | assertThat(Action.LIST_BUCKETS.name()).isEqualTo("LIST_BUCKETS"); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/GcsJsonApiEventTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2024 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.cloud.hadoop.util.testing.MockHttpTransportHelper.emptyResponse; 20 | import static com.google.cloud.hadoop.util.testing.MockHttpTransportHelper.mockTransport; 21 | import static com.google.common.truth.Truth.assertThat; 22 | 23 | import com.google.api.client.http.GenericUrl; 24 | import com.google.api.client.http.HttpResponse; 25 | import java.io.IOException; 26 | import java.util.concurrent.ThreadLocalRandom; 27 | import org.junit.Test; 28 | import org.junit.runner.RunWith; 29 | import org.junit.runners.JUnit4; 30 | 31 | @RunWith(JUnit4.class) 32 | public class GcsJsonApiEventTest { 33 | private static final String URL = "http://fake-url.com"; 34 | 35 | @Test 36 | public void testGetResponseEvent() throws IOException { 37 | int duration = Math.abs(ThreadLocalRandom.current().nextInt(1000000)); 38 | GcsJsonApiEvent event = GcsJsonApiEvent.getResponseEvent(getResponse(), duration, URL); 39 | 40 | assertThat(event.getContext().toString()).isEqualTo(URL); 41 | assertThat(event.getMethod()).isEqualTo("GET"); 42 | assertThat(event.getEventType()).isEqualTo(GcsJsonApiEvent.EventType.RESPONSE); 43 | assertThat(event.getProperty(GcsJsonApiEvent.DURATION)).isEqualTo(duration); 44 | } 45 | 46 | @Test 47 | public void testGetExceptionEvent() throws IOException { 48 | GcsJsonApiEvent event = GcsJsonApiEvent.getExceptionEvent(getResponse().getRequest(), URL); 49 | 50 | assertThat(event.getContext().toString()).isEqualTo(URL); 51 | assertThat(event.getMethod()).isEqualTo("GET"); 52 | assertThat(event.getEventType()).isEqualTo(GcsJsonApiEvent.EventType.EXCEPTION); 53 | } 54 | 55 | private HttpResponse getResponse() throws IOException { 56 | return mockTransport(emptyResponse(200)) 57 | .createRequestFactory() 58 | .buildGetRequest(new GenericUrl(URL)) 59 | .execute(); 60 | } 61 | 62 | @Test 63 | public void testGetNonExistingProperty() throws IOException { 64 | GcsJsonApiEvent event = GcsJsonApiEvent.getResponseEvent(getResponse(), 1, URL); 65 | assertThat(event.getProperty("foo")).isNull(); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/GrpcErrorTypeExtractorTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | 21 | import com.google.cloud.hadoop.util.ErrorTypeExtractor.ErrorType; 22 | import io.grpc.Status; 23 | import io.grpc.StatusRuntimeException; 24 | import org.junit.Test; 25 | 26 | public class GrpcErrorTypeExtractorTest { 27 | 28 | private static final GrpcErrorTypeExtractor typeExtractor = GrpcErrorTypeExtractor.INSTANCE; 29 | 30 | @Test 31 | public void testNotFound() { 32 | Exception ex = new StatusRuntimeException(Status.NOT_FOUND); 33 | assertThat(typeExtractor.getErrorType(ex)).isEqualTo(ErrorType.NOT_FOUND); 34 | } 35 | 36 | @Test 37 | public void testOutOfRange() { 38 | Exception ex = new StatusRuntimeException(Status.OUT_OF_RANGE); 39 | assertThat(typeExtractor.getErrorType(ex)).isEqualTo(ErrorType.OUT_OF_RANGE); 40 | } 41 | 42 | @Test 43 | public void testBucketAlreadyExistsFailedPreconditionException() { 44 | Exception ex = 45 | new Exception( 46 | new StatusRuntimeException( 47 | Status.FAILED_PRECONDITION.withDescription( 48 | "Your previous request to create the named bucket succeeded and you already own it."))); 49 | assertThat(typeExtractor.bucketAlreadyExists(ex)).isEqualTo(true); 50 | } 51 | 52 | @Test 53 | public void testBucketAlreadyExists() { 54 | Exception ex = new Exception(new StatusRuntimeException(Status.ALREADY_EXISTS)); 55 | assertThat(typeExtractor.bucketAlreadyExists(ex)).isEqualTo(true); 56 | } 57 | 58 | @Test 59 | public void testBucketAlreadyExistsInvalidException() { 60 | Exception ex = new StatusRuntimeException(Status.ABORTED); 61 | assertThat(typeExtractor.bucketAlreadyExists(ex)).isEqualTo(false); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/PropertyUtilTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | 21 | import org.junit.Test; 22 | import org.junit.runner.RunWith; 23 | import org.junit.runners.JUnit4; 24 | 25 | @RunWith(JUnit4.class) 26 | public class PropertyUtilTest { 27 | @Test 28 | public void defaultValueIsReturnedWhenNoFile() { 29 | assertThat( 30 | PropertyUtil.getPropertyOrDefault(getClass(), "NonExistentFile", "testKey", "NotFound")) 31 | .isEqualTo("NotFound"); 32 | } 33 | 34 | @Test 35 | public void defaultValueIsReturnedWhenKeyNotFound() { 36 | assertThat( 37 | PropertyUtil.getPropertyOrDefault( 38 | getClass(), "test.properties", "testKey2", "NotFound")) 39 | .isEqualTo("NotFound"); 40 | } 41 | 42 | @Test 43 | public void valueIsReturnedForFoundKeyAndFile() { 44 | assertThat( 45 | PropertyUtil.getPropertyOrDefault(getClass(), "test.properties", "testKey", "NotFound")) 46 | .isEqualTo("testValue"); 47 | } 48 | 49 | @Test 50 | public void valueWithWhitespaceIsReadProperly() { 51 | assertThat( 52 | PropertyUtil.getPropertyOrDefault( 53 | getClass(), "test.properties", "whitespaceKey", "NotFound")) 54 | .isEqualTo("test value with whitespace"); 55 | } 56 | 57 | @Test 58 | public void valueWithEscapedCharactersIsReadUnescaped() { 59 | assertThat( 60 | PropertyUtil.getPropertyOrDefault( 61 | getClass(), "test.properties", "escapedValueKey", "NotFound")) 62 | .isEqualTo("http://www.example.com"); 63 | } 64 | 65 | @Test 66 | public void keysAfterCommentsAreFound() { 67 | assertThat( 68 | PropertyUtil.getPropertyOrDefault( 69 | getClass(), "test.properties", "postCommentKey", "NotFound")) 70 | .isEqualTo("postCommentValue"); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/RetryBoundedBackOffTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2014 Google Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util; 18 | 19 | import static com.google.common.truth.Truth.assertThat; 20 | import static org.junit.Assert.assertThrows; 21 | 22 | import com.google.api.client.util.BackOff; 23 | import java.io.IOException; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.JUnit4; 27 | 28 | /** Unit tests for {@link RetryBoundedBackOff}. */ 29 | @RunWith(JUnit4.class) 30 | public class RetryBoundedBackOffTest { 31 | @Test 32 | public void testValidCallHasNoRetries() throws Exception { 33 | IllegalArgumentException thrown = 34 | assertThrows( 35 | IllegalArgumentException.class, () -> new RetryBoundedBackOff(new BackOffTester(), -7)); 36 | assertThat(thrown) 37 | .hasMessageThat() 38 | .contains("Maximum number of retries must not be less than 0."); 39 | } 40 | 41 | @Test 42 | public void stopsAfterAttempts() throws Exception { 43 | BackOff backoff = new RetryBoundedBackOff(new BackOffTester(), 5); 44 | for (int i = 0; i < 5; i++) { 45 | assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis()); 46 | } 47 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 48 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 49 | } 50 | 51 | @Test 52 | public void resetsCorrectly() throws Exception { 53 | BackOff backoff = new RetryBoundedBackOff(new BackOffTester(), 5); 54 | for (int i = 0; i < 5; i++) { 55 | assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis()); 56 | } 57 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 58 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 59 | backoff.reset(); 60 | for (int i = 0; i < 3; i++) { 61 | assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis()); 62 | } 63 | backoff.reset(); 64 | for (int i = 0; i < 5; i++) { 65 | assertThat(i + 1).isEqualTo(backoff.nextBackOffMillis()); 66 | } 67 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 68 | assertThat(backoff.nextBackOffMillis()).isEqualTo(BackOff.STOP); 69 | } 70 | 71 | /** A simple {@link BackOff} to help with testing. */ 72 | public static class BackOffTester implements BackOff { 73 | public int counter = 0; 74 | 75 | @Override 76 | public void reset() { 77 | counter = 0; 78 | } 79 | 80 | @Override 81 | public long nextBackOffMillis() throws IOException { 82 | return ++counter; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/testing/FakeCredentials.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util.testing; 18 | 19 | import com.google.auth.Credentials; 20 | import com.google.common.collect.ImmutableList; 21 | import com.google.common.collect.ImmutableMap; 22 | import java.net.URI; 23 | import java.util.List; 24 | import java.util.Map; 25 | 26 | /** Fake {@link Credentials} to use in unit tests */ 27 | public class FakeCredentials extends Credentials { 28 | 29 | private final String authHeaderValue; 30 | 31 | public FakeCredentials() { 32 | this(/* authHeaderValue= */ null); 33 | } 34 | 35 | public FakeCredentials(String authHeaderValue) { 36 | this.authHeaderValue = authHeaderValue; 37 | } 38 | 39 | @Override 40 | public String getAuthenticationType() { 41 | return "test-auth"; 42 | } 43 | 44 | @Override 45 | public Map> getRequestMetadata(URI uri) { 46 | return authHeaderValue == null 47 | ? ImmutableMap.of() 48 | : ImmutableMap.of("Authorization", ImmutableList.of(authHeaderValue)); 49 | } 50 | 51 | @Override 52 | public boolean hasRequestMetadata() { 53 | return true; 54 | } 55 | 56 | @Override 57 | public boolean hasRequestMetadataOnly() { 58 | return true; 59 | } 60 | 61 | @Override 62 | public void refresh() { 63 | throw new UnsupportedOperationException(); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /util/src/test/java/com/google/cloud/hadoop/util/testing/ThrowingInputStream.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.hadoop.util.testing; 18 | 19 | import static org.junit.Assert.fail; 20 | 21 | import java.io.IOException; 22 | import java.io.InputStream; 23 | 24 | /** 25 | * Test implementation of the {@link InputStream} that throws exception in {@code #read()} and/or 26 | * {@code close()} methods 27 | */ 28 | public class ThrowingInputStream extends InputStream { 29 | 30 | private final Throwable readException; 31 | private final Throwable closeException; 32 | 33 | public ThrowingInputStream(Throwable readException) { 34 | this(readException, /* closeException= */ null); 35 | } 36 | 37 | public ThrowingInputStream(Throwable readException, Throwable closeException) { 38 | this.readException = readException; 39 | this.closeException = closeException; 40 | } 41 | 42 | @Override 43 | public int available() { 44 | return 1; 45 | } 46 | 47 | @Override 48 | public int read() { 49 | throwUnchecked(readException); 50 | fail("Exception should have been thrown"); 51 | return -1; 52 | } 53 | 54 | @Override 55 | public void close() throws IOException { 56 | if (closeException != null) { 57 | throwUnchecked(closeException); 58 | } 59 | super.close(); 60 | } 61 | 62 | public static void throwUnchecked(Throwable e) { 63 | throwAny(e); 64 | fail(String.format("Exception '%s' should have been thrown", e)); 65 | } 66 | 67 | @SuppressWarnings("unchecked") 68 | private static void throwAny(Throwable e) throws E { 69 | throw (E) e; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /util/src/test/resources/com/google/cloud/hadoop/util/test.properties: -------------------------------------------------------------------------------- 1 | testKey=testValue 2 | whitespaceKey=test value with whitespace 3 | escapedValueKey=http\://www.example.com 4 | ! This line is a comment 5 | # So is this 6 | postCommentKey=postCommentValue -------------------------------------------------------------------------------- /util/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger=INFO,stdout 2 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 3 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 4 | log4j.appender.stdout.layout.ConversionPattern=%d{yyMMdd HH:mm:ss.SSS}:%1p [%c] %m%n 5 | --------------------------------------------------------------------------------