├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.yml │ └── feature-request.yml ├── dependabot.yml ├── pull_request_template.md └── workflows │ ├── build-upload.yml │ ├── gradle-integration-test.yml │ ├── gradle-reference-test.yml │ ├── gradle.yml │ └── release.yml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── NOTICE ├── README.md ├── THIRD-PARTY-NOTICES ├── buildSrc ├── build.gradle.kts ├── settings.gradle.kts └── src │ └── main │ └── kotlin │ ├── buildlogic.java-common-conventions.gradle.kts │ └── buildlogic.java-library-conventions.gradle.kts ├── common ├── build.gradle.kts ├── spotbugs-exclude.xml └── src │ ├── main │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ ├── common │ │ ├── ConnectorConfiguration.java │ │ ├── Metrics.java │ │ ├── ObjectRange.java │ │ ├── Preconditions.java │ │ └── telemetry │ │ │ ├── Attribute.java │ │ │ ├── Clock.java │ │ │ ├── CommonAttributes.java │ │ │ ├── ConfigurableTelemetry.java │ │ │ ├── DefaultElapsedClock.java │ │ │ ├── DefaultEpochClock.java │ │ │ ├── DefaultTelemetry.java │ │ │ ├── DefaultTelemetryFormat.java │ │ │ ├── EpochFormatter.java │ │ │ ├── GroupTelemetryReporter.java │ │ │ ├── JSONTelemetryFormat.java │ │ │ ├── LoggingTelemetryReporter.java │ │ │ ├── Metric.java │ │ │ ├── MetricMeasurement.java │ │ │ ├── MetricMeasurementKind.java │ │ │ ├── NoOpTelemetryReporter.java │ │ │ ├── Operation.java │ │ │ ├── OperationContext.java │ │ │ ├── OperationMeasurement.java │ │ │ ├── OperationSupplier.java │ │ │ ├── PrintStreamTelemetryReporter.java │ │ │ ├── Telemetry.java │ │ │ ├── TelemetryAction.java │ │ │ ├── TelemetryConfiguration.java │ │ │ ├── TelemetryDatapoint.java │ │ │ ├── TelemetryDatapointAggregator.java │ │ │ ├── TelemetryDatapointMeasurement.java │ │ │ ├── TelemetryFormat.java │ │ │ ├── TelemetryLevel.java │ │ │ ├── TelemetryReporter.java │ │ │ └── TelemetrySupplier.java │ │ ├── request │ │ ├── GetRequest.java │ │ ├── HeadRequest.java │ │ ├── ObjectClient.java │ │ ├── ObjectContent.java │ │ ├── ObjectMetadata.java │ │ ├── Range.java │ │ ├── ReadMode.java │ │ ├── Referrer.java │ │ └── StreamAuditContext.java │ │ └── util │ │ ├── BlockKey.java │ │ ├── InputPolicy.java │ │ ├── LogHelper.java │ │ ├── MetricComputationUtils.java │ │ ├── MetricKey.java │ │ ├── ObjectKey.java │ │ ├── OpenStreamInformation.java │ │ └── S3URI.java │ └── test │ └── java │ └── software │ └── amazon │ └── s3 │ └── analyticsaccelerator │ ├── CustomAssertions.java │ ├── SpotBugsLambdaWorkaround.java │ ├── common │ ├── ConnectorConfigurationTest.java │ ├── MetricKeyTest.java │ ├── ObjectRangeTest.java │ ├── PreconditionsTest.java │ └── telemetry │ │ ├── AttributeTest.java │ │ ├── CollectingTelemetryReporter.java │ │ ├── ConfigurableTelemetryTest.java │ │ ├── DefaultElapsedClockTest.java │ │ ├── DefaultEpochClockTest.java │ │ ├── DefaultTelemetryFormatTest.java │ │ ├── DefaultTelemetryTest.java │ │ ├── EpochFormatterTest.java │ │ ├── GroupTelemetryReporterTest.java │ │ ├── JSONTelemetryFormatTest.java │ │ ├── LoggingTelemetryReporterTest.java │ │ ├── MetricMeasurementTest.java │ │ ├── MetricTest.java │ │ ├── NoOpTelemetryReporterTest.java │ │ ├── OperationContextTest.java │ │ ├── OperationMeasurementTest.java │ │ ├── OperationTest.java │ │ ├── PrintStreamTelemetryReporterTest.java │ │ ├── TelemetryConfigurationTest.java │ │ ├── TelemetryDatapointAggregatorTest.java │ │ ├── TelemetryLevelTest.java │ │ ├── TelemetryTest.java │ │ └── TickingClock.java │ ├── request │ ├── ObjectMetadataTest.java │ ├── RangeTest.java │ └── ReferrerTest.java │ └── util │ ├── LogHelperTest.java │ ├── MetricComputationUtilsTest.java │ ├── ObjectKeyTest.java │ ├── OpenStreamInformationTest.java │ └── S3URITest.java ├── config └── checkstyle │ ├── checkstyle.xml │ └── suppression.xml ├── doc ├── CODE_OF_CONDUCT.md ├── CONFIGURATION.md ├── CONTRIBUTING.md └── DEVELOPMENT.md ├── gradle ├── libs.versions.toml ├── licenseHeader └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── input-stream ├── .jqwik-database ├── build.gradle.kts ├── spotbugs-exclude.xml └── src │ ├── integrationTest │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ └── access │ │ ├── AALPrefetchingIntegrationTest.java │ │ ├── ChecksumAssertions.java │ │ ├── ConcurrencyCorrectnessTest.java │ │ ├── ExceptionCorrectnessTest.java │ │ ├── GrayFailureTest.java │ │ ├── IntegrationTestBase.java │ │ ├── ReadCorrectnessTest.java │ │ ├── ReadVectoredTest.java │ │ └── SequentialFileTypeIntegrationTest.java │ ├── jmh │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ └── benchmarks │ │ ├── AALBenchmark.java │ │ ├── BenchmarkBase.java │ │ ├── ComparisonBenchmarkBase.java │ │ ├── S3ClientAndStreamKind.java │ │ ├── StreamPatternComparisonBenchmark.java │ │ ├── StreamPatternComparisonBenchmarks.java │ │ └── data │ │ └── generation │ │ ├── BenchmarkDataGeneratorDriver.java │ │ ├── BenchmarkObjectGenerator.java │ │ └── RandomSequentialObjectGenerator.java │ ├── main │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ ├── RandomAccessReadable.java │ │ ├── S3SeekableInputStream.java │ │ ├── S3SeekableInputStreamConfiguration.java │ │ ├── S3SeekableInputStreamFactory.java │ │ ├── SeekableInputStream.java │ │ ├── io │ │ ├── logical │ │ │ ├── LogicalIO.java │ │ │ ├── LogicalIOConfiguration.java │ │ │ ├── impl │ │ │ │ ├── DefaultLogicalIOImpl.java │ │ │ │ ├── ParquetColumnPrefetchStore.java │ │ │ │ ├── ParquetLogicalIOImpl.java │ │ │ │ ├── ParquetPrefetcher.java │ │ │ │ ├── SequentialLogicalIOImpl.java │ │ │ │ └── SequentialPrefetcher.java │ │ │ └── parquet │ │ │ │ ├── ColumnMappers.java │ │ │ │ ├── ColumnMetadata.java │ │ │ │ ├── FileTail.java │ │ │ │ ├── FooterPrefetchSize.java │ │ │ │ ├── ParquetMetadataParsingTask.java │ │ │ │ ├── ParquetParser.java │ │ │ │ ├── ParquetPredictivePrefetchingTask.java │ │ │ │ ├── ParquetPrefetchRemainingColumnTask.java │ │ │ │ ├── ParquetPrefetchTailTask.java │ │ │ │ ├── ParquetReadTailTask.java │ │ │ │ └── ParquetUtils.java │ │ └── physical │ │ │ ├── PhysicalIO.java │ │ │ ├── PhysicalIOConfiguration.java │ │ │ ├── data │ │ │ ├── Blob.java │ │ │ ├── BlobStore.java │ │ │ ├── BlobStoreIndexCache.java │ │ │ ├── Block.java │ │ │ ├── BlockManager.java │ │ │ ├── BlockStore.java │ │ │ ├── IOPlanner.java │ │ │ ├── MetadataStore.java │ │ │ └── RangeOptimiser.java │ │ │ ├── impl │ │ │ └── PhysicalIOImpl.java │ │ │ ├── plan │ │ │ ├── IOPlan.java │ │ │ ├── IOPlanExecution.java │ │ │ └── IOPlanState.java │ │ │ └── prefetcher │ │ │ ├── SequentialPatternDetector.java │ │ │ └── SequentialReadProgression.java │ │ └── util │ │ ├── AnalyticsAcceleratorUtils.java │ │ ├── Constants.java │ │ ├── ObjectFormat.java │ │ ├── ObjectFormatSelector.java │ │ ├── PrefetchMode.java │ │ ├── StreamAttributes.java │ │ ├── StreamUtils.java │ │ └── VectoredReadUtils.java │ ├── referenceTest │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ ├── arbitraries │ │ └── StreamArbitraries.java │ │ ├── model │ │ └── InMemorySeekableStream.java │ │ ├── property │ │ ├── InMemoryS3SeekableInputStream.java │ │ └── SeekableStreamPropertiesTest.java │ │ └── reference │ │ └── S3MockVsInMemoryReferenceTest.java │ ├── test │ ├── java │ │ └── software │ │ │ └── amazon │ │ │ └── s3 │ │ │ └── analyticsaccelerator │ │ │ ├── S3SeekableInputStreamConfigurationTest.java │ │ │ ├── S3SeekableInputStreamFactoryTest.java │ │ │ ├── S3SeekableInputStreamTest.java │ │ │ ├── S3SeekableInputStreamTestBase.java │ │ │ ├── SpotBugsLambdaWorkaround.java │ │ │ ├── TestTelemetry.java │ │ │ ├── io │ │ │ ├── logical │ │ │ │ ├── LogicalIOConfigurationTest.java │ │ │ │ ├── impl │ │ │ │ │ ├── DefaultLogicalIOImplTest.java │ │ │ │ │ ├── ParquetColumnPrefetchStoreTest.java │ │ │ │ │ ├── ParquetLogicalIOImplTest.java │ │ │ │ │ ├── ParquetPrefetcherTest.java │ │ │ │ │ ├── SequentialLogicalIOImplTest.java │ │ │ │ │ └── SequentialPrefetcherTest.java │ │ │ │ └── parquet │ │ │ │ │ ├── IOPlanMatcher.java │ │ │ │ │ ├── ParquetMetadataParsingTaskTest.java │ │ │ │ │ ├── ParquetParserTest.java │ │ │ │ │ ├── ParquetPredictivePrefetchingTaskTest.java │ │ │ │ │ ├── ParquetPrefetchRemainingColumnTaskTest.java │ │ │ │ │ ├── ParquetPrefetchTailTaskTest.java │ │ │ │ │ ├── ParquetReadTailTaskTest.java │ │ │ │ │ └── ParquetUtilsTest.java │ │ │ └── physical │ │ │ │ ├── PhysicalIOConfigurationTest.java │ │ │ │ ├── data │ │ │ │ ├── BlobStoreIndexCacheTest.java │ │ │ │ ├── BlobStoreTest.java │ │ │ │ ├── BlobTest.java │ │ │ │ ├── BlockManagerTest.java │ │ │ │ ├── BlockStoreTest.java │ │ │ │ ├── BlockTest.java │ │ │ │ ├── IOPlannerTest.java │ │ │ │ ├── MetadataStoreTest.java │ │ │ │ └── RangeOptimiserTest.java │ │ │ │ ├── impl │ │ │ │ └── PhysicalIOImplTest.java │ │ │ │ └── prefetcher │ │ │ │ ├── SequentialPatternDetectorTest.java │ │ │ │ └── SequentialReadProgressionTest.java │ │ │ ├── plan │ │ │ └── IOPlanTest.java │ │ │ └── util │ │ │ ├── AnalyticsAcceleratorUtilsTest.java │ │ │ ├── FakeObjectClient.java │ │ │ ├── FakeStuckObjectClient.java │ │ │ ├── ObjectFormatSelectorTest.java │ │ │ ├── PrefetchModeTest.java │ │ │ ├── StreamUtilsTest.java │ │ │ └── VectoredReadUtilsTest.java │ └── resources │ │ ├── call_center.parquet │ │ ├── call_center_file_metadata.ser │ │ ├── multi_row_group.parquet │ │ ├── multi_row_group.ser │ │ ├── nested_data.parquet │ │ ├── nested_data_metadata.ser │ │ ├── nested_data_mrg.parquet │ │ └── nested_data_mrg_metadata.ser │ └── testFixtures │ └── java │ └── software │ └── amazon │ └── s3 │ └── analyticsaccelerator │ └── access │ ├── AALInputStreamConfigurationKind.java │ ├── ExecutionBase.java │ ├── FaultyS3AsyncClient.java │ ├── S3AALClientStreamReader.java │ ├── S3AsyncClientFactory.java │ ├── S3AsyncClientFactoryConfiguration.java │ ├── S3AsyncClientStreamReader.java │ ├── S3ClientKind.java │ ├── S3ExecutionConfiguration.java │ ├── S3ExecutionContext.java │ ├── S3InputStreamKind.java │ ├── S3Object.java │ ├── S3ObjectKind.java │ ├── S3StreamReaderBase.java │ ├── SizeConstants.java │ ├── StreamRead.java │ ├── StreamReadPattern.java │ ├── StreamReadPatternFactory.java │ └── StreamReadPatternKind.java ├── lombok.config ├── object-client ├── build.gradle.kts ├── spotbugs-exclude.xml └── src │ ├── main │ └── java │ │ └── software │ │ └── amazon │ │ └── s3 │ │ └── analyticsaccelerator │ │ ├── ObjectClientConfiguration.java │ │ ├── ObjectClientTelemetry.java │ │ ├── S3SdkObjectClient.java │ │ ├── exceptions │ │ └── ExceptionHandler.java │ │ └── request │ │ ├── Constants.java │ │ └── UserAgent.java │ └── test │ └── java │ └── software │ └── amazon │ └── s3 │ └── analyticsaccelerator │ ├── ObjectClientConfigurationTest.java │ ├── S3SdkObjectClientTest.java │ ├── SpotBugsLambdaWorkaround.java │ ├── exceptions │ └── ExceptionHandlerTest.java │ └── request │ └── UserAgentTest.java └── settings.gradle.kts /.github/ISSUE_TEMPLATE/bug-report.yml: -------------------------------------------------------------------------------- 1 | name: Bug report 2 | description: File a bug report 3 | labels: ["bug"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thank you for taking the time to submit a bug report! 9 | - type: markdown 10 | attributes: 11 | value: | 12 | **Note: Security issues should not be reported here.** 13 | Please follow the [security policy for this repository](../../CONTRIBUTING.md#security-issue-notifications)). 14 | - type: input 15 | id: dats3-version 16 | attributes: 17 | label: Analytics Accelerator Library for Amazon S3 Version 18 | description: | 19 | Which version of Analytics Accelerator Library for Amazon S3 are you using? 20 | If you are building from source or a fork, please state that. 21 | placeholder: x.y 22 | validations: 23 | required: true 24 | - type: input 25 | id: region 26 | attributes: 27 | label: AWS Region 28 | description: Which AWS region did you experience the bug in? 29 | placeholder: us-east-1 30 | validations: 31 | required: false 32 | - type: textarea 33 | id: environment 34 | attributes: 35 | label: Describe the running environment 36 | description: | 37 | What else can you share about the environment you are running the project in? 38 | For example, was this using Amazon EC2? Which type/OS version/architecture? 39 | placeholder: Running in EC2 on Amazon Linux 2. 40 | validations: 41 | required: true 42 | - type: textarea 43 | id: behavior 44 | attributes: 45 | label: What happened? 46 | description: Please also tell us what you expected to happen. 47 | placeholder: A description of the issue. 48 | validations: 49 | required: true 50 | - type: textarea 51 | id: logs 52 | attributes: 53 | label: Relevant log output 54 | description: Please copy and paste any relevant log output. This will be automatically formatted into code, so no need for backticks. 55 | render: shell 56 | validations: 57 | required: false 58 | - type: checkboxes 59 | id: terms 60 | attributes: 61 | label: Code of Conduct 62 | description: By submitting this issue, you agree to follow our [Code of Conduct](../../CODE_OF_CONDUCT.md) 63 | options: 64 | - label: I agree to follow this project's Code of Conduct 65 | required: true -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.yml: -------------------------------------------------------------------------------- 1 | name: Feature request 2 | description: File a feature request. This might be a feature you'd like to see added, or one you'd like to contribute. 3 | labels: ["enhancement"] 4 | body: 5 | - type: markdown 6 | attributes: 7 | value: | 8 | Thanks for taking the time to share ideas for new features. 9 | - type: textarea 10 | id: feature-desc 11 | attributes: 12 | label: Tell us more about this new feature. 13 | placeholder: I would like to see... 14 | validations: 15 | required: true 16 | - type: checkboxes 17 | id: terms 18 | attributes: 19 | label: Code of Conduct 20 | description: By submitting this issue, you agree to follow our [Code of Conduct](../../CODE_OF_CONDUCT.md) 21 | options: 22 | - label: I agree to follow this project's Code of Conduct 23 | required: true -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | day: "sunday" 8 | - package-ecosystem: "gradle" 9 | directory: "/" 10 | schedule: 11 | interval: "weekly" 12 | day: "sunday" -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description of change 2 | 3 | 4 | 5 | 6 | #### Relevant issues 7 | 8 | 9 | 10 | #### Does this contribution introduce any breaking changes to the existing APIs or behaviors? 11 | 12 | 13 | #### Does this contribution introduce any new public APIs or behaviors? 14 | 15 | 16 | #### How was the contribution tested? 17 | 18 | 19 | #### Does this contribution need a changelog entry? 20 | - [ ] I have updated the CHANGELOG or README if appropriate 21 | 22 | --- 23 | 24 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and I agree to the terms of the [Developer Certificate of Origin (DCO)](https://developercertificate.org/). -------------------------------------------------------------------------------- /.github/workflows/gradle-integration-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time 6 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle 7 | 8 | name: Build and run integration tests 9 | 10 | on: 11 | push: 12 | branches: 13 | - main 14 | - cicd-builds 15 | pull_request_target: 16 | branches: [ "main" ] 17 | 18 | env: 19 | S3_TEST_REGION : ${{ vars.S3_TEST_REGION }} 20 | S3_TEST_BUCKET : ${{ vars.S3_TEST_BUCKET }} 21 | S3_TEST_PREFIX : ${{ vars.S3_TEST_PREFIX }} 22 | ROLE_TO_ASSUME: ${{ secrets.S3_TEST_ASSUME_ROLE_ARN }} 23 | 24 | jobs: 25 | build: 26 | name: Integration Tests 27 | runs-on: codebuild-s3-analytics-accelerator-eu-west-1-${{ github.run_id }}-${{ github.run_attempt }} 28 | environment: integration-tests 29 | permissions: 30 | contents: read 31 | id-token: write 32 | 33 | steps: 34 | - uses: actions/checkout@v4 35 | with: 36 | ref: ${{ github.event.pull_request.head.sha }} 37 | - name: Set up JDK 8 38 | uses: actions/setup-java@v4 39 | with: 40 | java-version: '8' 41 | distribution: 'temurin' 42 | 43 | - name: Setup Gradle 44 | uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 45 | 46 | - name: Configure aws credentials 47 | uses: aws-actions/configure-aws-credentials@v4.0.2 48 | with: 49 | role-to-assume: ${{ env.ROLE_TO_ASSUME }} 50 | role-session-name: GitHub_to_AWS_via_FederatedOIDC 51 | aws-region: ${{ env.S3_TEST_REGION }} 52 | 53 | - name: Build with Gradle Wrapper 54 | run: ./gradlew integrationTest 55 | 56 | 57 | -------------------------------------------------------------------------------- /.github/workflows/gradle-reference-test.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time 6 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle 7 | 8 | name: Build and run reference tests 9 | 10 | on: 11 | push: 12 | branches: [ "main" ] 13 | pull_request: 14 | branches: [ "main" ] 15 | 16 | jobs: 17 | build: 18 | 19 | runs-on: ubuntu-latest 20 | permissions: 21 | contents: read 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | - name: Set up JDK 8 26 | uses: actions/setup-java@v4 27 | with: 28 | java-version: '8' 29 | distribution: 'temurin' 30 | 31 | - name: Setup Gradle 32 | uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 33 | 34 | - name: Build with Gradle Wrapper 35 | run: ./gradlew referenceTest 36 | 37 | 38 | -------------------------------------------------------------------------------- /.github/workflows/gradle.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # This workflow will build a Java project with Gradle and cache/restore any dependencies to improve the workflow execution time 6 | # For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-java-with-gradle 7 | 8 | name: Build and run unit tests 9 | 10 | on: 11 | push: 12 | branches: [ "main" ] 13 | pull_request: 14 | branches: [ "main" ] 15 | 16 | jobs: 17 | build: 18 | 19 | runs-on: ubuntu-latest 20 | permissions: 21 | contents: read 22 | 23 | steps: 24 | - uses: actions/checkout@v4 25 | - name: Set up JDK 8 26 | uses: actions/setup-java@v4 27 | with: 28 | java-version: '8' 29 | distribution: 'temurin' 30 | 31 | - name: Setup Gradle 32 | uses: gradle/actions/setup-gradle@af1da67850ed9a4cedd57bfd976089dd991e2582 # v4.0.0 33 | 34 | - name: Build with Gradle Wrapper 35 | run: ./gradlew build 36 | 37 | 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .gradle/ 2 | .idea/ 3 | /bin 4 | /build 5 | /buildSrc/build/ 6 | /wrapper/ 7 | 8 | # Ignore Gradle project-specific cache directory 9 | .gradle 10 | 11 | # Ignore Gradle build output directory 12 | bin 13 | build 14 | 15 | # Ignore junk and IDE files 16 | .DS_Store 17 | .gitattributes 18 | *.iml 19 | .vscode/ 20 | 21 | . 22 | 23 | gradle.properties -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | -------------------------------------------------------------------------------- /buildSrc/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | // Support convention plugins written in Kotlin. Convention plugins are build scripts in 'src/main' that automatically become available as plugins in the main build. 7 | `kotlin-dsl` 8 | } 9 | 10 | repositories { 11 | // Use the plugin portal to apply community plugins in convention plugins. 12 | gradlePluginPortal() 13 | } 14 | 15 | dependencies { 16 | implementation(libs.spotless) 17 | implementation(libs.spotbugs) 18 | implementation(libs.test.logger) 19 | } 20 | -------------------------------------------------------------------------------- /buildSrc/settings.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * This settings file is used to specify which projects to include in your build-logic build. 5 | */ 6 | 7 | dependencyResolutionManagement { 8 | // Reuse version catalog from the main build. 9 | versionCatalogs { 10 | create("libs", { from(files("../gradle/libs.versions.toml")) }) 11 | } 12 | } 13 | 14 | rootProject.name = "buildSrc" 15 | -------------------------------------------------------------------------------- /buildSrc/src/main/kotlin/buildlogic.java-library-conventions.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | // Apply the common convention plugin for shared build configuration between library and application projects. 7 | id("buildlogic.java-common-conventions") 8 | 9 | // Apply the java-library plugin for API and implementation separation. 10 | `java-library` 11 | } 12 | -------------------------------------------------------------------------------- /common/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | id("buildlogic.java-library-conventions") 7 | id("io.freefair.lombok") version "8.10.2" 8 | `maven-publish` 9 | } 10 | 11 | dependencies { 12 | implementation(libs.slf4j.api) 13 | 14 | testImplementation(libs.junit.jupiter) 15 | testImplementation(libs.mockito.core) 16 | testImplementation(libs.mockito.inline) 17 | testImplementation(libs.mockito.junit.jupiter) 18 | 19 | testRuntimeOnly(libs.junit.jupiter.launcher) 20 | } 21 | 22 | tasks.withType().configureEach { 23 | } -------------------------------------------------------------------------------- /common/spotbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/ObjectRange.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common; 17 | 18 | import java.nio.ByteBuffer; 19 | import java.util.concurrent.CompletableFuture; 20 | import lombok.Value; 21 | 22 | /** 23 | * This class is a container for information required when doing asynchronous reads. It allows for a 24 | * future to be passed in, which is completed when the data requested has been read into the 25 | * ByteBuffer. 26 | */ 27 | @Value 28 | public class ObjectRange { 29 | CompletableFuture byteBuffer; 30 | long offset; 31 | int length; 32 | 33 | /** 34 | * Constructor to create object range. 35 | * 36 | * @param byteBuffer future to be completed when read is completed. 37 | * @param offset position to start the read from 38 | * @param length length of the read 39 | */ 40 | public ObjectRange(CompletableFuture byteBuffer, long offset, int length) { 41 | Preconditions.checkNotNull(byteBuffer); 42 | Preconditions.checkArgument(offset >= 0); 43 | Preconditions.checkArgument(length >= 0); 44 | 45 | this.byteBuffer = byteBuffer; 46 | this.offset = offset; 47 | this.length = length; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/Attribute.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import lombok.NonNull; 19 | import lombok.ToString; 20 | import lombok.Value; 21 | 22 | /** 23 | * Telemetry attribute. An attribute is key/value pair associated with a telemetry item, such as 24 | * operation. 25 | */ 26 | @Value(staticConstructor = "of") 27 | @ToString(includeFieldNames = false) 28 | public class Attribute { 29 | /** Attribute name. */ 30 | @NonNull String name; 31 | 32 | /** Attribute value. */ 33 | @NonNull Object value; 34 | } 35 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/Clock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** 19 | * Represents a clock that can be used to measure time. The assumption is that the time grows 20 | * monotonically, in nanoseconds. 21 | */ 22 | interface Clock { 23 | /** 24 | * Returns current timestamp in nanoseconds. 25 | * 26 | * @return current timestamp in nanoseconds. 27 | */ 28 | long getCurrentTimeNanos(); 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/CommonAttributes.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** Common telemetry attributes. */ 22 | @Getter 23 | @AllArgsConstructor 24 | public enum CommonAttributes { 25 | THREAD_ID("thread_id"); 26 | private final String name; 27 | 28 | /** 29 | * Creates a and {@link Attribute} for a {@link CommonAttributes#THREAD_ID}. 30 | * 31 | * @param thread the {@link CommonAttributes#THREAD_ID} to create the attribute from. 32 | * @return The new instance of the {@link Attribute}. 33 | */ 34 | public static Attribute threadId(Thread thread) { 35 | return Attribute.of(CommonAttributes.THREAD_ID.getName(), thread.getId()); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/DefaultElapsedClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** 19 | * Default clock used in all non-test contexts to measure elapsed time. Uses `{@link 20 | * System#nanoTime()}. 21 | */ 22 | final class DefaultElapsedClock implements Clock { 23 | /** 24 | * Returns current timestamp in nanoseconds. 25 | * 26 | * @return current timestamp in nanoseconds. 27 | */ 28 | @Override 29 | public long getCurrentTimeNanos() { 30 | return System.nanoTime(); 31 | } 32 | 33 | /** Default clock. */ 34 | public static final Clock DEFAULT = new DefaultElapsedClock(); 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/DefaultEpochClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** 19 | * Default clock used in all non-test contexts to measure wall clock time. Uses `{@link 20 | * System#currentTimeMillis()}. 21 | */ 22 | final class DefaultEpochClock implements Clock { 23 | /** 24 | * Returns current timestamp in nanoseconds. 25 | * 26 | * @return current timestamp in nanoseconds. 27 | */ 28 | @Override 29 | public long getCurrentTimeNanos() { 30 | return System.currentTimeMillis() * 1_000_000; 31 | } 32 | 33 | /** Default clock. */ 34 | public static final Clock DEFAULT = new DefaultEpochClock(); 35 | } 36 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/EpochFormatter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import java.text.SimpleDateFormat; 19 | import java.util.Locale; 20 | import java.util.TimeZone; 21 | import java.util.concurrent.TimeUnit; 22 | import lombok.Getter; 23 | import lombok.NonNull; 24 | 25 | /** Formatter used to output dates and times */ 26 | public final class EpochFormatter { 27 | private final @Getter @NonNull String pattern; 28 | private final @Getter @NonNull Locale locale; 29 | private final @Getter @NonNull TimeZone timeZone; 30 | private final @NonNull ThreadLocal dateFormat; 31 | 32 | /** Default pattern */ 33 | public static final String DEFAULT_PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; 34 | /** Default locale. */ 35 | public static final Locale DEFAULT_LOCALE = Locale.getDefault(); 36 | /** Default time zone. */ 37 | public static final TimeZone DEFAULT_TIMEZONE = TimeZone.getDefault(); 38 | /** Default {@link EpochFormatter}. */ 39 | public static final EpochFormatter DEFAULT = new EpochFormatter(); 40 | 41 | /** 42 | * Creates a new instance of {@link EpochFormatter}. 43 | * 44 | * @param pattern format pattern for epochs. 45 | * @param timeZone time zone. 46 | * @param locale locale. 47 | */ 48 | public EpochFormatter( 49 | @NonNull String pattern, @NonNull TimeZone timeZone, @NonNull Locale locale) { 50 | this.pattern = pattern; 51 | this.timeZone = timeZone; 52 | this.locale = locale; 53 | this.dateFormat = 54 | ThreadLocal.withInitial( 55 | () -> { 56 | SimpleDateFormat simpleDateFormat = new SimpleDateFormat(pattern, locale); 57 | simpleDateFormat.setTimeZone(timeZone); 58 | return simpleDateFormat; 59 | }); 60 | } 61 | 62 | /** Creates the {@link EpochFormatter} with sensible default. */ 63 | public EpochFormatter() { 64 | this(DEFAULT_PATTERN, DEFAULT_TIMEZONE, DEFAULT_LOCALE); 65 | } 66 | 67 | /** 68 | * Formats the epoch timestamp measured in milliseconds. 69 | * 70 | * @param epochMillis epoch. 71 | * @return formatted epoch 72 | */ 73 | public String formatMillis(long epochMillis) { 74 | return dateFormat.get().format(epochMillis); 75 | } 76 | 77 | /** 78 | * Formats the epoch timestamp measured in nanoseconds. This loses nano precision. 79 | * 80 | * @param epochNanos epoch. 81 | * @return formatted epoch 82 | */ 83 | public String formatNanos(long epochNanos) { 84 | return dateFormat.get().format(TimeUnit.NANOSECONDS.toMillis(epochNanos)); 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/GroupTelemetryReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import java.util.Collection; 19 | import java.util.Collections; 20 | import lombok.Getter; 21 | import lombok.NonNull; 22 | 23 | /** A {@link TelemetryReporter that reports telemetry to a group of independent reporterts} */ 24 | @Getter 25 | class GroupTelemetryReporter implements TelemetryReporter { 26 | private final @NonNull Collection reporters; 27 | 28 | /** 29 | * Creates a new instance of {@link GroupTelemetryReporter}. 30 | * 31 | * @param reporters a group of reporters to fan out telemetry events to. 32 | */ 33 | public GroupTelemetryReporter(@NonNull Collection reporters) { 34 | this.reporters = Collections.unmodifiableCollection(reporters); 35 | } 36 | 37 | /** 38 | * Reports the start of an operation 39 | * 40 | * @param epochTimestampNanos wall clock time for the operation start 41 | * @param operation and instance of {@link Operation} to start 42 | */ 43 | @Override 44 | public void reportStart(long epochTimestampNanos, Operation operation) { 45 | for (TelemetryReporter reporter : reporters) { 46 | reporter.reportStart(epochTimestampNanos, operation); 47 | } 48 | } 49 | 50 | /** 51 | * Outputs the current contents of {@link OperationMeasurement} to each of the supplied reporters. 52 | * 53 | * @param datapointMeasurement operation execution. 54 | */ 55 | @Override 56 | public void reportComplete(TelemetryDatapointMeasurement datapointMeasurement) { 57 | for (TelemetryReporter reporter : reporters) { 58 | reporter.reportComplete(datapointMeasurement); 59 | } 60 | } 61 | 62 | /** Flushes any intermediate state of the reporters */ 63 | @Override 64 | public void flush() { 65 | for (TelemetryReporter reporter : reporters) { 66 | reporter.flush(); 67 | } 68 | } 69 | 70 | /** Closes the underlying reporters */ 71 | @Override 72 | public void close() { 73 | for (TelemetryReporter reporter : reporters) { 74 | reporter.close(); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/Metric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import java.util.Map; 19 | import lombok.EqualsAndHashCode; 20 | import lombok.Getter; 21 | 22 | /** 23 | * Telemetry metric. A metric is a template for a specific measurement and is defined by a name and 24 | * a set of attributes It doesn't carry the actual value - that is defined by a specific instance of 25 | * a metric, represented by {@link MetricMeasurement} 26 | */ 27 | // Implementation note: the builder is implemented by hand, as opposed to via Lombok to create more 28 | // streamlined syntax for attribute specification 29 | @Getter 30 | @EqualsAndHashCode(callSuper = true) 31 | public class Metric extends TelemetryDatapoint { 32 | /** 33 | * Creates a new instance of {@link Metric}. 34 | * 35 | * @param name operation name. 36 | * @param attributes operation attributes. 37 | */ 38 | private Metric(String name, Map attributes) { 39 | super(name, attributes); 40 | } 41 | 42 | /** 43 | * Creates a builder for {@link Metric} 44 | * 45 | * @return a new instance of {@link MetricBuilder} 46 | */ 47 | public static MetricBuilder builder() { 48 | return new MetricBuilder(); 49 | } 50 | 51 | /** Builder for {@link MetricBuilder} */ 52 | public static class MetricBuilder extends TelemetryDatapointBuilder { 53 | 54 | /** 55 | * Builds the {@link Metric} 56 | * 57 | * @return a new instance of {@link Metric} 58 | */ 59 | @Override 60 | protected Metric buildCore() { 61 | return new Metric(this.getName(), this.getAttributes()); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/MetricMeasurementKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** This represents the metric measurement kind */ 22 | @Getter 23 | @AllArgsConstructor 24 | public enum MetricMeasurementKind { 25 | RAW(1), // raw data point, reported directly to telemetry 26 | AGGREGATE(2); // aggregate value, reported as a result of aggregation 27 | private final int value; 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/NoOpTelemetryReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** a {@link TelemetryReporter} that does nothing. */ 19 | class NoOpTelemetryReporter implements TelemetryReporter { 20 | /** 21 | * Reports the start of an operation 22 | * 23 | * @param epochTimestampNanos wall clock time for the operation start 24 | * @param operation and instance of {@link Operation} to start 25 | */ 26 | @Override 27 | public void reportStart(long epochTimestampNanos, Operation operation) {} 28 | 29 | /** 30 | * Reports the completion of an operation 31 | * 32 | * @param datapointMeasurement an instance of {@link OperationMeasurement}. 33 | */ 34 | @Override 35 | public void reportComplete(TelemetryDatapointMeasurement datapointMeasurement) {} 36 | 37 | /** Flushes any intermediate state of the reporters In this case, this is a no-op */ 38 | @Override 39 | public void flush() {} 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/OperationSupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** Syntactic sugar over {@link TelemetrySupplier} */ 19 | @FunctionalInterface 20 | public interface OperationSupplier extends TelemetrySupplier {} 21 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TelemetryAction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** A functional interface that mimics {@link Runnable}, but allows exceptions to be thrown. */ 19 | @FunctionalInterface 20 | public interface TelemetryAction { 21 | /** 22 | * Functional representation of the code that takes no parameters and returns no value. The code 23 | * is allowed to throw any exception. 24 | * 25 | * @throws Exception on error condition. 26 | */ 27 | void apply() throws Throwable; 28 | } 29 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TelemetryLevel.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** This defines the level of reporting for telemetry operations */ 22 | @Getter 23 | @AllArgsConstructor 24 | public enum TelemetryLevel { 25 | CRITICAL(2), 26 | STANDARD(1), 27 | VERBOSE(0); 28 | private final int value; 29 | } 30 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TelemetryReporter.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import java.io.Closeable; 19 | 20 | /** Interface that represents a telemetry reporter. */ 21 | public interface TelemetryReporter extends Closeable { 22 | /** 23 | * Reports the start of an operation 24 | * 25 | * @param epochTimestampNanos wall clock time for the operation start 26 | * @param operation and instance of {@link Operation} to start 27 | */ 28 | void reportStart(long epochTimestampNanos, Operation operation); 29 | 30 | /** 31 | * Reports the completion of an operation 32 | * 33 | * @param datapointMeasurement an instance of {@link TelemetryDatapointMeasurement}. 34 | */ 35 | void reportComplete(TelemetryDatapointMeasurement datapointMeasurement); 36 | 37 | /** Flushes any intermediate state of the reporters */ 38 | void flush(); 39 | 40 | /** 41 | * Default implementation of {@link AutoCloseable#close()}, that calls {@link 42 | * TelemetryReporter#flush()} 43 | */ 44 | default void close() { 45 | this.flush(); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TelemetrySupplier.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | /** 19 | * A function that mimics {@link java.util.function.Supplier}, but allows exceptions to be thrown 20 | * 21 | * @param 22 | */ 23 | @FunctionalInterface 24 | public interface TelemetrySupplier { 25 | /** 26 | * Functional representation of the code that takes no parameters and returns a value of type 27 | * {@link T}. The code is allowed to throw any exception. 28 | * 29 | * @return a value of type {@link T}. 30 | * @throws Throwable on error condition. 31 | */ 32 | T apply() throws Throwable; 33 | } 34 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/GetRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 19 | import lombok.Builder; 20 | import lombok.NonNull; 21 | import lombok.Value; 22 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 23 | 24 | /** 25 | * Object representing arguments to a GetObject call. This class helps us abstract away from S3 SDK 26 | * constructs. 27 | */ 28 | @Value 29 | @Builder 30 | @SuppressFBWarnings(value = "NM_CONFUSING", justification = "Sharing Getter names is not confusing") 31 | public class GetRequest { 32 | @NonNull S3URI s3Uri; 33 | @NonNull Range range; 34 | @NonNull Referrer referrer; 35 | @NonNull String etag; 36 | } 37 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/HeadRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import lombok.Builder; 19 | import lombok.Data; 20 | import lombok.NonNull; 21 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 22 | 23 | /** 24 | * Object representing arguments to a HeadObject call. This class helps us abstract away from S3 SDK 25 | * constructs. 26 | */ 27 | @Data 28 | @Builder 29 | public class HeadRequest { 30 | @NonNull S3URI s3Uri; 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/ObjectClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import java.io.Closeable; 19 | import java.util.concurrent.CompletableFuture; 20 | import software.amazon.s3.analyticsaccelerator.util.OpenStreamInformation; 21 | 22 | /** Represents APIs of an Amazon S3 compatible object store */ 23 | public interface ObjectClient extends Closeable { 24 | 25 | /** 26 | * Make a headObject request to the object store. 27 | * 28 | * @param headRequest The HEAD request to be sent 29 | * @param openStreamInformation contains stream information 30 | * @return an instance of {@link CompletableFuture} of type {@link ObjectMetadata} 31 | */ 32 | CompletableFuture headObject( 33 | HeadRequest headRequest, OpenStreamInformation openStreamInformation); 34 | 35 | /** 36 | * Make a getObject request to the object store. 37 | * 38 | * @param getRequest The GET request to be sent 39 | * @param openStreamInformation contains stream information 40 | * @return an instance of {@link CompletableFuture} of type {@link ObjectContent} 41 | */ 42 | CompletableFuture getObject( 43 | GetRequest getRequest, OpenStreamInformation openStreamInformation); 44 | } 45 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/ObjectContent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import java.io.InputStream; 19 | import lombok.Builder; 20 | import lombok.Data; 21 | 22 | /** Wrapper class around GetObjectResponse abstracting away from S3-specific details */ 23 | @Data 24 | @Builder 25 | public class ObjectContent { 26 | InputStream stream; 27 | } 28 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/ObjectMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import lombok.Builder; 19 | import lombok.Data; 20 | import lombok.NonNull; 21 | import software.amazon.s3.analyticsaccelerator.common.Preconditions; 22 | 23 | /** Wrapper class around HeadObjectResponse abstracting away from S3-specific details */ 24 | @Data 25 | @Builder 26 | public class ObjectMetadata { 27 | /** 28 | * The length of the object content in bytes. This value must be non-negative. 29 | * 30 | * @param contentLength the content length to set 31 | * @return the builder instance 32 | * @throws IllegalArgumentException if contentLength is negative 33 | */ 34 | long contentLength; 35 | 36 | /** 37 | * The entity tag of the object. 38 | * 39 | * @param etag the etag to set 40 | * @return the builder instance 41 | * @throws NullPointerException if etag is null 42 | */ 43 | @NonNull String etag; 44 | 45 | @Builder 46 | private ObjectMetadata(long contentLength, @NonNull String etag) { 47 | Preconditions.checkArgument(contentLength >= 0, "content length must be non-negative"); 48 | 49 | this.contentLength = contentLength; 50 | this.etag = etag; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/Range.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import lombok.Getter; 19 | import lombok.Value; 20 | import software.amazon.s3.analyticsaccelerator.common.Preconditions; 21 | 22 | /** 23 | * Object representing a byte range. This class helps us abstract away from S3 SDK constructs and 24 | * helps with testing. For example, it gets rid of the need of having to generate and parse strings 25 | * like "bytes=0-555" -- this is SDK detail we should not care about in layers above Object Client. 26 | */ 27 | @Value 28 | public class Range { 29 | @Getter long start; 30 | @Getter long end; 31 | 32 | private static final String TO_HTTP_STRING_FORMAT = "bytes=%d-%d"; 33 | private static final String TO_STRING_FORMAT = "%d-%d"; 34 | 35 | /** 36 | * Construct a range. At least one of the start and end of range should be present. 37 | * 38 | * @param start the start of the range, possibly empty 39 | * @param end the end of the range, possibly empty 40 | */ 41 | public Range(long start, long end) { 42 | Preconditions.checkArgument(start >= 0, "`start` must not be negative"); 43 | Preconditions.checkArgument(end >= 0, "`end` must not be negative"); 44 | Preconditions.checkArgument(start <= end, "`start` must be less than equal to `end`"); 45 | 46 | this.start = start; 47 | this.end = end; 48 | } 49 | 50 | /** 51 | * Does this range contain the position? 52 | * 53 | * @param pos the position 54 | * @return true if the byte at the position is contained by this range 55 | */ 56 | public boolean contains(long pos) { 57 | return start <= pos && pos <= end; 58 | } 59 | 60 | /** 61 | * Return the length of the range. 62 | * 63 | * @return the length of the range in bytes 64 | */ 65 | public int getLength() { 66 | return (int) (this.end - this.start + 1); 67 | } 68 | 69 | /** 70 | * Returns the textual representation of {@link Range}. 71 | * 72 | * @return the textual representation of {@link Range}. 73 | */ 74 | @Override 75 | public String toString() { 76 | return String.format(TO_STRING_FORMAT, start, end); 77 | } 78 | 79 | /** 80 | * Returns the string representation of {@link Range} used in S3 requests, as defined by the Http 81 | * RFC. 82 | * 83 | * @return the HTTP RFC compatible representation of {@link Range}. 84 | */ 85 | public String toHttpString() { 86 | return String.format(TO_HTTP_STRING_FORMAT, start, end); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/ReadMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | /** 19 | * Enum to help with the annotation of reads. We mark reads SYNC when they were triggered by a 20 | * synchronous read or ASYNC when they were to do logical or physical prefetching. 21 | */ 22 | public enum ReadMode { 23 | SYNC, 24 | ASYNC, 25 | SMALL_OBJECT_PREFETCH; 26 | } 27 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/Referrer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import lombok.Value; 19 | 20 | /** Represents the referrer header to be passed in when making a request. */ 21 | @Value 22 | public class Referrer { 23 | private final String range; 24 | private final ReadMode readMode; 25 | 26 | /** 27 | * Construct a referrer object. 28 | * 29 | * @param range range of data requested 30 | * @param readMode is this a sync or async read? 31 | */ 32 | public Referrer(String range, ReadMode readMode) { 33 | this.range = range; 34 | this.readMode = readMode; 35 | } 36 | 37 | @Override 38 | public String toString() { 39 | StringBuilder referrer = new StringBuilder(); 40 | referrer.append(range); 41 | referrer.append(",readMode=").append(readMode); 42 | return referrer.toString(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/request/StreamAuditContext.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import lombok.AccessLevel; 19 | import lombok.Builder; 20 | import lombok.Getter; 21 | 22 | /** 23 | * This is a class to contain any stream specific information, such as what is the higher level 24 | * operation that this stream belongs to? Useful for the integration with S3A, it allows S3A to pass 25 | * in the spanId and operation name with which a stream is opened. This is used by S3A's audit 26 | * logging, which is of the form "3eb3c657-3e59-4f20-b484-e215c90c49f2-00000011 Executing op_open 27 | * with {action_http_head_request....", where 3eb3c657-3e59-4f20-b484-e215c90c49f2-00000011 is the 28 | * span_id for the open() operation, and ` op_open` is the operation name. See ... 30 | * for more details. 31 | */ 32 | @Builder(access = AccessLevel.PUBLIC) 33 | @Getter 34 | public class StreamAuditContext { 35 | private final String spanId; 36 | private final String operationName; 37 | } 38 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/BlockKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Builder; 20 | import lombok.Getter; 21 | import lombok.NonNull; 22 | import software.amazon.s3.analyticsaccelerator.request.Range; 23 | 24 | /** Container used to represent an S3 object for a specific version/etag */ 25 | @Getter 26 | @AllArgsConstructor 27 | @Builder 28 | public class BlockKey { 29 | @NonNull ObjectKey objectKey; 30 | @NonNull Range range; 31 | } 32 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/InputPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | /** 19 | * Input policy to be used when reading an object. Useful for integrating applications to pass down 20 | * a policy they would like AAL to use, rather than us guessing based on object key extension. 21 | * 22 | *

A sequential policy means that the object will be read sequentially, regardless of the format. 23 | * Useful for sequential data types such as CSV, or for applications that read columnar formats 24 | * sequentially. 25 | * 26 | *

When no input policy is supplied, AAL will infer what optimisations to use based on the object 27 | * key extension. 28 | */ 29 | public enum InputPolicy { 30 | None, 31 | Sequential 32 | } 33 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/LogHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import java.util.Optional; 19 | import org.slf4j.Logger; 20 | import org.slf4j.event.Level; 21 | 22 | /** A helper class for logging related methods. */ 23 | public class LogHelper { 24 | 25 | /** 26 | * Logs at the specified level. 27 | * 28 | * @param logger logger to use 29 | * @param level level to log at all 30 | * @param message message to log 31 | * @param error optional error to log 32 | */ 33 | public static void logAtLevel( 34 | Logger logger, Level level, String message, Optional error) { 35 | switch (level) { 36 | case ERROR: 37 | if (error.isPresent()) { 38 | logger.error(message, error.get()); 39 | } else { 40 | logger.error(message); 41 | } 42 | break; 43 | case WARN: 44 | logger.warn(message); 45 | break; 46 | case INFO: 47 | logger.info(message); 48 | break; 49 | case DEBUG: 50 | logger.debug(message); 51 | break; 52 | case TRACE: 53 | logger.trace(message); 54 | break; 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/MetricComputationUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | /** Utility class for computing various metrics related to cache performance. */ 19 | public final class MetricComputationUtils { 20 | 21 | /** Private constructor to prevent instantiation of utility class. */ 22 | private MetricComputationUtils() { 23 | throw new AssertionError("No instances of MetricComputationUtils should be created"); 24 | } 25 | 26 | /** 27 | * Computes the cache hit rate as a percentage based on the number of cache hits and misses. The 28 | * hit rate is calculated as (cache hits / (cache hits + cache misses)) * 100. If there are no 29 | * hits and misses (total is zero), returns 0. 30 | * 31 | * @param cacheHits The number of cache hits 32 | * @param cacheMiss The number of cache misses 33 | * @return The cache hit rate as a percentage (0-100) 34 | */ 35 | public static double computeCacheHitRate(long cacheHits, long cacheMiss) { 36 | long hitsAndMiss = cacheHits + cacheMiss; 37 | double hitRate = hitsAndMiss == 0 ? 0 : (double) cacheHits / hitsAndMiss; 38 | return hitRate * 100; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/MetricKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | /** 19 | * Enum representing different types of metrics that can be tracked in the system. Each metric is 20 | * identified by a unique name and can be used to track various performance and operational aspects. 21 | */ 22 | public enum MetricKey { 23 | /** 24 | * Tracks the memory usage in bytes. Used to monitor the amount of memory being consumed by the 25 | * blobstore. 26 | */ 27 | MEMORY_USAGE("MemoryUsage"), 28 | 29 | /** 30 | * Tracks the number of successful cache hits. Incremented when requested block is found in the 31 | * cache. 32 | */ 33 | CACHE_HIT("CacheHit"), 34 | 35 | /** 36 | * Tracks the number of cache misses. Incremented when requested block is not found in the cache 37 | */ 38 | CACHE_MISS("CacheMiss"); 39 | 40 | /** The string name representation of the metric. */ 41 | private final String name; 42 | 43 | /** 44 | * Constructs a new MetricKey with the specified name. 45 | * 46 | * @param name The string identifier for the metric 47 | */ 48 | MetricKey(String name) { 49 | this.name = name; 50 | } 51 | 52 | /** 53 | * Returns the string name of the metric. 54 | * 55 | * @return The name of the metric 56 | */ 57 | public String getName() { 58 | return name; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/ObjectKey.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Builder; 20 | import lombok.Getter; 21 | import lombok.NonNull; 22 | 23 | /** Container used to represent an S3 object for a specific version/etag */ 24 | @Getter 25 | @AllArgsConstructor 26 | @Builder 27 | public class ObjectKey { 28 | @NonNull S3URI s3URI; 29 | @NonNull String etag; 30 | 31 | @Override 32 | public final boolean equals(Object o) { 33 | if (this == o) return true; 34 | if (o == null || getClass() != o.getClass()) return false; 35 | 36 | ObjectKey objectKey = (ObjectKey) o; 37 | return s3URI.equals(objectKey.s3URI) && etag.equals(objectKey.etag); 38 | } 39 | 40 | @Override 41 | public int hashCode() { 42 | return s3URI.hashCode() + etag.hashCode(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/OpenStreamInformation.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import lombok.AccessLevel; 19 | import lombok.Builder; 20 | import lombok.Getter; 21 | import software.amazon.s3.analyticsaccelerator.request.ObjectMetadata; 22 | import software.amazon.s3.analyticsaccelerator.request.StreamAuditContext; 23 | 24 | /** 25 | * Open stream information, useful for allowing the stream opening application to pass down known 26 | * information and callbacks when opening the stream. @Value annotation makes this class immutable 27 | * and automatically generates: - All-args constructor - Getters for all fields - 28 | * equals/hashCode/toString methods 29 | * 30 | *

Available getters: - getStreamContext(): Returns the stream context - getObjectMetadata(): 31 | * Returns the object metadata - getInputPolicy(): Returns the input policy 32 | * 33 | *

Builder usage: OpenStreamInformation info = OpenStreamInformation.builder() 34 | * .streamContext(context) .objectMetadata(metadata) .inputPolicy(policy) .build(); 35 | * 36 | *

Or use the default instance: {@code OpenStreamInformation.DEFAULT} 37 | */ 38 | @Builder(access = AccessLevel.PUBLIC) 39 | @Getter 40 | public class OpenStreamInformation { 41 | private final StreamAuditContext streamAuditContext; 42 | private final ObjectMetadata objectMetadata; 43 | private final InputPolicy inputPolicy; 44 | 45 | /** Default set of settings for {@link OpenStreamInformation} */ 46 | public static final OpenStreamInformation DEFAULT = OpenStreamInformation.builder().build(); 47 | } 48 | -------------------------------------------------------------------------------- /common/src/main/java/software/amazon/s3/analyticsaccelerator/util/S3URI.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import java.net.URI; 19 | import lombok.NonNull; 20 | import lombok.Value; 21 | 22 | /** Container for representing an 's3://' or 's3a://'-style S3 location. */ 23 | @Value(staticConstructor = "of") 24 | public class S3URI { 25 | @NonNull String bucket; 26 | @NonNull String key; 27 | 28 | private static String URI_FORMAT_STRING = "%s://%s/%s"; 29 | private static String URI_SCHEME_DEFAULT = "s3"; 30 | 31 | /** 32 | * Creates the {@link URI} corresponding to this {@link S3URI}. 33 | * 34 | * @return the newly created {@link S3URI} 35 | */ 36 | public URI toURI() { 37 | return toURI(URI_SCHEME_DEFAULT); 38 | } 39 | 40 | /** 41 | * Creates the {@link URI} corresponding to this {@link S3URI}. 42 | * 43 | * @param scheme URI scheme to use 44 | * @return the newly created {@link S3URI} 45 | */ 46 | public URI toURI(String scheme) { 47 | return URI.create(this.toString(scheme)); 48 | } 49 | 50 | /** 51 | * Creates the string representation of the {@link S3URI}. 52 | * 53 | * @return the string representation of the {@link URI} 54 | */ 55 | @Override 56 | public String toString() { 57 | return toString(URI_SCHEME_DEFAULT); 58 | } 59 | 60 | /** 61 | * Creates the string representation of the {@link S3URI}. 62 | * 63 | * @param scheme URI scheme to use 64 | * @return the string representation of the {@link URI} 65 | */ 66 | public String toString(@NonNull String scheme) { 67 | return String.format(URI_FORMAT_STRING, scheme, this.getBucket(), this.getKey()); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/CustomAssertions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertTrue; 19 | 20 | import java.util.Arrays; 21 | 22 | /** Helper class for custom assertions. */ 23 | public final class CustomAssertions { 24 | 25 | /** 26 | * Asserts that a string contains other substrings and displays an easy to interpret message when 27 | * it does not. 28 | * 29 | * @param content the String to search in 30 | * @param substr the substrings to search for 31 | */ 32 | public static void assertContains(String content, String... substr) { 33 | Arrays.stream(substr) 34 | .sequential() 35 | .forEach( 36 | str -> { 37 | assertTrue( 38 | content.contains(str), 39 | String.format("Expected '%s' to contain '%s'", content, str)); 40 | }); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/SpotBugsLambdaWorkaround.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertInstanceOf; 19 | import static org.junit.jupiter.api.Assertions.fail; 20 | 21 | import java.io.Closeable; 22 | import lombok.SneakyThrows; 23 | import org.junit.jupiter.api.function.ThrowingSupplier; 24 | 25 | /** 26 | * This class is here to help work around certain SpotBugs quirks. Specifically, SpotBugs don't 27 | * allow to apply suppressions to Lambdas 28 | */ 29 | public final class SpotBugsLambdaWorkaround { 30 | /** 31 | * In situations where a method or constructor return a class that implements Closeable, Spotbugs 32 | * wants to see it closed. When testing bounds and expecting exceptions, this is unnecessary, and 33 | * trying to work around this in tests results in a lot of boilerplate. This method absorbs all 34 | * the boilerplate and acts as `assertThrows`. 35 | * 36 | * @param expectedType exception type expected 37 | * @param executable code that returns something Closeable, and expected to throw 38 | * @param exception type 39 | * @param return type 40 | */ 41 | @SneakyThrows 42 | @SuppressWarnings("try") 43 | public static void assertThrowsClosableResult( 44 | Class expectedType, ThrowingSupplier executable) { 45 | try (C result = executable.get()) { 46 | } catch (Throwable throwable) { 47 | assertInstanceOf(expectedType, throwable); 48 | return; 49 | } 50 | fail(String.format("Exception of type '%s' was expected. Nothing was thrown", expectedType)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/MetricKeyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import org.junit.jupiter.api.Test; 21 | import software.amazon.s3.analyticsaccelerator.util.MetricKey; 22 | 23 | public class MetricKeyTest { 24 | 25 | @Test 26 | public void testMetricKeyNames() { 27 | assertEquals("MemoryUsage", MetricKey.MEMORY_USAGE.getName()); 28 | assertEquals("CacheHit", MetricKey.CACHE_HIT.getName()); 29 | assertEquals("CacheMiss", MetricKey.CACHE_MISS.getName()); 30 | } 31 | 32 | @Test 33 | public void testEnumValues() { 34 | MetricKey[] values = MetricKey.values(); 35 | assertEquals(3, values.length); 36 | assertEquals(MetricKey.MEMORY_USAGE, values[0]); 37 | assertEquals(MetricKey.CACHE_HIT, values[1]); 38 | assertEquals(MetricKey.CACHE_MISS, values[2]); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/ObjectRangeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertNotNull; 19 | import static org.junit.jupiter.api.Assertions.assertThrows; 20 | 21 | import java.util.concurrent.CompletableFuture; 22 | import org.junit.jupiter.api.Test; 23 | 24 | public class ObjectRangeTest { 25 | 26 | @Test 27 | void testDefaultConstructor() { 28 | ObjectRange objectRange = new ObjectRange(new CompletableFuture<>(), 0, 5); 29 | assertNotNull(objectRange); 30 | } 31 | 32 | @Test 33 | void testPreconditions() { 34 | assertThrows(NullPointerException.class, () -> new ObjectRange(null, 0, 5)); 35 | assertThrows( 36 | IllegalArgumentException.class, () -> new ObjectRange(new CompletableFuture<>(), -1, 5)); 37 | assertThrows( 38 | IllegalArgumentException.class, () -> new ObjectRange(new CompletableFuture<>(), 0, -10)); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/AttributeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 21 | import org.junit.jupiter.api.Test; 22 | 23 | @SuppressFBWarnings( 24 | value = "NP_NONNULL_PARAM_VIOLATION", 25 | justification = "We mean to pass nulls to checks") 26 | public class AttributeTest { 27 | @Test 28 | void testCreateAttribute() { 29 | Attribute attribute = Attribute.of("Foo", 42); 30 | assertEquals(attribute.getName(), "Foo"); 31 | assertEquals(attribute.getValue(), 42); 32 | } 33 | 34 | @Test 35 | void testCreateAttributeNullNameThrows() { 36 | assertThrows(NullPointerException.class, () -> Attribute.of(null, 42)); 37 | } 38 | 39 | @Test 40 | void testCreateAttributeNullValueThrows() { 41 | assertThrows(NullPointerException.class, () -> Attribute.of("Foo", null)); 42 | } 43 | 44 | @Test 45 | void testCreateAttributeNullNameAndValueThrows() { 46 | assertThrows(NullPointerException.class, () -> Attribute.of(null, null)); 47 | } 48 | 49 | @Test 50 | void testEquality() { 51 | assertEquals(Attribute.of("Foo", 42), Attribute.of("Foo", 42)); 52 | assertNotEquals(Attribute.of("Foo", 42), Attribute.of("Foo", 43)); 53 | assertNotEquals(Attribute.of("Bar", 42), Attribute.of("Foo", 42)); 54 | assertNotEquals(Attribute.of("Foo", 42), null); 55 | } 56 | 57 | @Test 58 | void testHashCode() { 59 | assertEquals(Attribute.of("Foo", 42).hashCode(), Attribute.of("Foo", 42).hashCode()); 60 | assertNotEquals(Attribute.of("Foo", 42).hashCode(), Attribute.of("Foo", 43).hashCode()); 61 | assertNotEquals(Attribute.of("Bar", 42).hashCode(), Attribute.of("Foo", 42).hashCode()); 62 | } 63 | 64 | @Test 65 | void testToString() { 66 | assertEquals("Attribute(Foo, 42)", Attribute.of("Foo", 42).toString()); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/DefaultElapsedClockTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | public class DefaultElapsedClockTest { 23 | @Test 24 | void testClockTicksForward() { 25 | Clock clock = new DefaultElapsedClock(); 26 | long timeStamp1 = clock.getCurrentTimeNanos(); 27 | long timeStamp2 = clock.getCurrentTimeNanos(); 28 | assertTrue(timeStamp1 <= timeStamp2); 29 | } 30 | 31 | @Test 32 | void testClockTicksForwardForDefault() { 33 | Clock clock = DefaultElapsedClock.DEFAULT; 34 | long timeStamp1 = clock.getCurrentTimeNanos(); 35 | long timeStamp2 = clock.getCurrentTimeNanos(); 36 | assertTrue(timeStamp1 <= timeStamp2); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/DefaultEpochClockTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import java.util.concurrent.TimeUnit; 21 | import org.junit.jupiter.api.Test; 22 | 23 | public class DefaultEpochClockTest { 24 | @Test 25 | void testClockIsAlignedWithWallTime() { 26 | Clock clock = new DefaultEpochClock(); 27 | long before = System.currentTimeMillis(); 28 | long now = TimeUnit.NANOSECONDS.toMillis(clock.getCurrentTimeNanos()); 29 | long after = System.currentTimeMillis(); 30 | assertTrue(before <= now); 31 | assertTrue(now <= after); 32 | } 33 | 34 | @Test 35 | void testClockIsAlignedWithWallTimeForDefault() { 36 | Clock clock = DefaultEpochClock.DEFAULT; 37 | long before = System.currentTimeMillis(); 38 | long now = TimeUnit.NANOSECONDS.toMillis(clock.getCurrentTimeNanos()); 39 | long after = System.currentTimeMillis(); 40 | assertTrue(before <= now); 41 | assertTrue(now <= after); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/NoOpTelemetryReporterTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import org.junit.jupiter.api.Test; 19 | 20 | public class NoOpTelemetryReporterTest { 21 | private static final long TEST_EPOCH_NANOS = 1722944779101123456L; 22 | 23 | @Test 24 | void testCreateAndReportComplete() { 25 | Operation operation = Operation.builder().name("foo").attribute("A", 42).build(); 26 | OperationMeasurement operationMeasurement = 27 | OperationMeasurement.builder() 28 | .operation(operation) 29 | .level(TelemetryLevel.STANDARD) 30 | .epochTimestampNanos(TEST_EPOCH_NANOS) 31 | .elapsedStartTimeNanos(10) 32 | .elapsedCompleteTimeNanos(5000000) 33 | .build(); 34 | 35 | try (TelemetryReporter reporter = new NoOpTelemetryReporter()) { 36 | reporter.reportComplete(operationMeasurement); 37 | } 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TelemetryLevelTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | public class TelemetryLevelTest { 23 | @Test 24 | void testGetPriority() { 25 | assertEquals(0, TelemetryLevel.VERBOSE.getValue()); 26 | assertEquals(1, TelemetryLevel.STANDARD.getValue()); 27 | assertEquals(2, TelemetryLevel.CRITICAL.getValue()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/common/telemetry/TickingClock.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.common.telemetry; 17 | 18 | import java.util.concurrent.atomic.AtomicLong; 19 | 20 | /** This is a logical clock, that progresses only ahead and only does so when told. */ 21 | class TickingClock implements Clock { 22 | private final AtomicLong ticks = new AtomicLong(); 23 | 24 | /** 25 | * Creates the clock with the given tick count 26 | * 27 | * @param ticks - initial tick count. 28 | */ 29 | public TickingClock(long ticks) { 30 | this.ticks.set(ticks); 31 | } 32 | 33 | /** 34 | * Moves the clock forward 35 | * 36 | * @param ticks - tick count. 37 | * @return total ticks 38 | */ 39 | public long tick(long ticks) { 40 | return this.ticks.addAndGet(ticks); 41 | } 42 | 43 | /** 44 | * Current ticks. 45 | * 46 | * @return current ticks. 47 | */ 48 | @Override 49 | public long getCurrentTimeNanos() { 50 | return ticks.get(); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/request/RangeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static org.junit.jupiter.api.Assertions.assertThrows; 20 | 21 | import java.util.stream.Stream; 22 | import org.junit.jupiter.api.Test; 23 | import org.junit.jupiter.params.ParameterizedTest; 24 | import org.junit.jupiter.params.provider.Arguments; 25 | import org.junit.jupiter.params.provider.MethodSource; 26 | 27 | public class RangeTest { 28 | 29 | @ParameterizedTest 30 | @MethodSource("invalidRanges") 31 | void testInvalidRangesThrow(long start, long end) { 32 | assertThrows(IllegalArgumentException.class, () -> new Range(start, end)); 33 | } 34 | 35 | @ParameterizedTest 36 | @MethodSource("validStringRanges") 37 | void testToString(long start, long end, String expected) { 38 | assertEquals(expected, new Range(start, end).toString()); 39 | } 40 | 41 | @ParameterizedTest 42 | @MethodSource("validHttpStringRanges") 43 | void testToHttpString(long start, long end, String expected) { 44 | assertEquals(expected, new Range(start, end).toHttpString()); 45 | } 46 | 47 | @Test 48 | void testSize() { 49 | assertEquals(1, new Range(0, 0).getLength()); 50 | assertEquals(100, new Range(0, 99).getLength()); 51 | } 52 | 53 | static Stream validStringRanges() { 54 | return Stream.of( 55 | Arguments.of(1, 5, "1-5"), 56 | Arguments.of(0, 0, "0-0"), 57 | Arguments.of(100, Long.MAX_VALUE, "100-" + Long.MAX_VALUE)); 58 | } 59 | 60 | static Stream validHttpStringRanges() { 61 | return Stream.of( 62 | Arguments.of(1, 5, "bytes=1-5"), 63 | Arguments.of(0, 0, "bytes=0-0"), 64 | Arguments.of(100, Long.MAX_VALUE, "bytes=100-" + Long.MAX_VALUE)); 65 | } 66 | 67 | static Stream invalidRanges() { 68 | return Stream.of( 69 | Arguments.of(7, 5), Arguments.of(-100, 5), Arguments.of(1, -100), Arguments.of(-1, 1)); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/request/ReferrerTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static org.junit.jupiter.api.Assertions.assertNotNull; 20 | 21 | import org.junit.jupiter.api.Test; 22 | 23 | public class ReferrerTest { 24 | 25 | @Test 26 | void testConstructor() { 27 | Referrer referrer = new Referrer(null, ReadMode.SYNC); 28 | assertNotNull(referrer); 29 | } 30 | 31 | @Test 32 | void testReferrerToString() { 33 | Referrer referrer = new Referrer("bytes=11083511-19472118", ReadMode.ASYNC); 34 | assertEquals(referrer.toString(), "bytes=11083511-19472118,readMode=ASYNC"); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/util/MetricComputationUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | public class MetricComputationUtilsTest { 23 | 24 | private static final double DELTA = 0.0001; // Delta for comparing double values 25 | 26 | @Test 27 | public void testComputeCacheHitRate_BothZero() { 28 | double result = MetricComputationUtils.computeCacheHitRate(0, 0); 29 | assertEquals(0.0, result, DELTA); 30 | } 31 | 32 | @Test 33 | public void testComputeCacheHitRate_AllHits() { 34 | double result = MetricComputationUtils.computeCacheHitRate(100, 0); 35 | assertEquals(100.0, result, DELTA); 36 | } 37 | 38 | @Test 39 | public void testComputeCacheHitRate_AllMisses() { 40 | double result = MetricComputationUtils.computeCacheHitRate(0, 100); 41 | assertEquals(0.0, result, DELTA); 42 | } 43 | 44 | @Test 45 | public void testComputeCacheHitRate_MixedHitsAndMisses() { 46 | double result = MetricComputationUtils.computeCacheHitRate(75, 25); 47 | assertEquals(75.0, result, DELTA); 48 | } 49 | 50 | @Test 51 | public void testComputeCacheHitRate_LargeNumbers() { 52 | double result = MetricComputationUtils.computeCacheHitRate(1000000, 1000000); 53 | assertEquals(50.0, result, DELTA); 54 | } 55 | 56 | @Test 57 | public void testComputeCacheHitRate_SingleHit() { 58 | double result = MetricComputationUtils.computeCacheHitRate(1, 0); 59 | assertEquals(100.0, result, DELTA); 60 | } 61 | 62 | @Test 63 | public void testComputeCacheHitRate_SingleMiss() { 64 | double result = MetricComputationUtils.computeCacheHitRate(0, 1); 65 | assertEquals(0.0, result, DELTA); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /common/src/test/java/software/amazon/s3/analyticsaccelerator/util/S3URITest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static org.junit.jupiter.api.Assertions.assertThrows; 20 | 21 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 22 | import java.net.URI; 23 | import org.junit.jupiter.api.Test; 24 | 25 | @SuppressFBWarnings( 26 | value = "NP_NONNULL_PARAM_VIOLATION", 27 | justification = "We mean to pass nulls to checks") 28 | public class S3URITest { 29 | @Test 30 | void testCreateValidatesArguments() { 31 | assertThrows(NullPointerException.class, () -> S3URI.of(null, "key")); 32 | assertThrows(NullPointerException.class, () -> S3URI.of("bucket", null)); 33 | } 34 | 35 | @Test 36 | void testCreate() { 37 | S3URI s3URI = S3URI.of("bucket", "key"); 38 | 39 | assertEquals("bucket", s3URI.getBucket()); 40 | assertEquals("key", s3URI.getKey()); 41 | } 42 | 43 | @Test 44 | void testToString() { 45 | S3URI s3URI = S3URI.of("bucket", "key"); 46 | assertEquals("s3://bucket/key", s3URI.toString()); 47 | assertEquals("s3a://bucket/key", s3URI.toString("s3a")); 48 | assertThrows(NullPointerException.class, () -> s3URI.toString(null)); 49 | } 50 | 51 | @Test 52 | void testToUri() { 53 | S3URI s3URI = S3URI.of("bucket", "key"); 54 | assertEquals(URI.create("s3://bucket/key"), s3URI.toURI()); 55 | assertEquals(URI.create("s3a://bucket/key"), s3URI.toURI("s3a")); 56 | assertThrows(NullPointerException.class, () -> s3URI.toURI(null)); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /config/checkstyle/checkstyle.xml: -------------------------------------------------------------------------------- 1 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /config/checkstyle/suppression.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /doc/CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /gradle/libs.versions.toml: -------------------------------------------------------------------------------- 1 | [versions] 2 | junit = "5.10.1" 3 | mockito = "4.11.0" 4 | spotless = "6.13.0" 5 | spotbugs = "6.0.20" 6 | s3 = "2.25.31" 7 | s3mock = "3.11.0" 8 | testcontainers = "1.19.7" 9 | crt = "0.29.10" 10 | lombok = "1.18.32" 11 | parquetFormat = "2.10.0" 12 | jqwik = "1.9.1" 13 | jqwik-testcontainers = "0.5.2" 14 | testcontainers-testcontainers = "1.20.2" 15 | test-logger="4.0.0" 16 | slf4j="2.0.16" 17 | caffeine = "2.9.3" 18 | 19 | 20 | [libraries] 21 | # S3-related dependencies 22 | s3 = { module = "software.amazon.awssdk:s3", version.ref = "s3" } 23 | s3-transfer-manager = { module = "software.amazon.awssdk:s3-transfer-manager", version.ref = "s3" } 24 | sdk-url-connection-client = { module = "software.amazon.awssdk:url-connection-client", version.ref = "s3" } 25 | sdk-bom = { group = "software.amazon.awssdk", name = "bom", version.ref = "s3" } 26 | crt = { module = "software.amazon.awssdk.crt:aws-crt", version.ref = "crt" } 27 | netty-nio-client = { module = "software.amazon.awssdk:netty-nio-client", version.ref = "s3" } 28 | parquet-format = { module = "org.apache.parquet:parquet-format", version.ref = "parquetFormat" } 29 | slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j"} 30 | caffeine = { module = "com.github.ben-manes.caffeine:caffeine", version.ref = "caffeine" } 31 | 32 | # Code maintenance and best practices 33 | lombok = { module = "org.projectlombok:lombok", version.ref = "lombok" } 34 | spotless = { module = "com.diffplug.spotless:spotless-plugin-gradle", version.ref = "spotless" } 35 | spotbugs = { module = "com.github.spotbugs.snom:spotbugs-gradle-plugin", version.ref = "spotbugs" } 36 | test-logger = { module = "com.adarshr:gradle-test-logger-plugin", version.ref = "test-logger" } 37 | 38 | 39 | # Unit tests 40 | junit-jupiter = { module = "org.junit.jupiter:junit-jupiter", version.ref = "junit" } 41 | junit-jupiter-launcher = { module = "org.junit.platform:junit-platform-launcher" } 42 | mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } 43 | mockito-inline = { module = "org.mockito:mockito-inline", version.ref = "mockito" } 44 | mockito-junit-jupiter = { module = "org.mockito:mockito-junit-jupiter", version.ref = "mockito" } 45 | 46 | # Reference tests 47 | jqwik = { module = "net.jqwik:jqwik", version.ref = "jqwik" } 48 | jqwik-testcontainers = { module = "net.jqwik:jqwik-testcontainers", version.ref = "jqwik-testcontainers" } 49 | s3mock-testcontainers = { module = "com.adobe.testing:s3mock-testcontainers", version.ref = "s3mock" } 50 | testcontainers = { module = "org.testcontainers:testcontainers", version.ref = "testcontainers-testcontainers" } 51 | testcontainers-junit-jupiter = { module = "org.testcontainers:junit-jupiter", version.ref = "testcontainers" } -------------------------------------------------------------------------------- /gradle/licenseHeader: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-bin.zip 4 | networkTimeout=10000 5 | validateDistributionUrl=true 6 | zipStoreBase=GRADLE_USER_HOME 7 | zipStorePath=wrapper/dists 8 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 1>&2 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 48 | echo. 1>&2 49 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 50 | echo location of your Java installation. 1>&2 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 1>&2 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 62 | echo. 1>&2 63 | echo Please set the JAVA_HOME variable in your environment to match the 1>&2 64 | echo location of your Java installation. 1>&2 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /input-stream/.jqwik-database: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/.jqwik-database -------------------------------------------------------------------------------- /input-stream/spotbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /input-stream/src/integrationTest/java/software/amazon/s3/analyticsaccelerator/access/AALPrefetchingIntegrationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import static software.amazon.s3.analyticsaccelerator.access.S3Object.smallBinaryObjects; 19 | 20 | import java.io.IOException; 21 | import java.util.stream.Stream; 22 | import org.junit.jupiter.params.ParameterizedTest; 23 | import org.junit.jupiter.params.provider.Arguments; 24 | import org.junit.jupiter.params.provider.MethodSource; 25 | 26 | public class AALPrefetchingIntegrationTest extends IntegrationTestBase { 27 | 28 | @ParameterizedTest 29 | @MethodSource("prefetchingTest") 30 | void testSmallObjectPrefetchedData( 31 | S3ClientKind s3ClientKind, 32 | S3Object s3Object, 33 | StreamReadPatternKind streamReadPattern, 34 | AALInputStreamConfigurationKind configuration) 35 | throws IOException { 36 | testSmallObjectPrefetching(s3ClientKind, s3Object, streamReadPattern, configuration); 37 | } 38 | 39 | static Stream prefetchingTest() { 40 | return argumentsFor( 41 | getS3ClientKinds(), 42 | smallBinaryObjects(), 43 | parquetPatterns(), 44 | getS3SeekableInputStreamConfigurations()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /input-stream/src/integrationTest/java/software/amazon/s3/analyticsaccelerator/access/ChecksumAssertions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import software.amazon.awssdk.core.checksums.SdkChecksum; 21 | 22 | public class ChecksumAssertions { 23 | /** 24 | * Checksum assertion 25 | * 26 | * @param expected expected checksum 27 | * @param actual actual checksum 28 | */ 29 | public static void assertChecksums(SdkChecksum expected, SdkChecksum actual) { 30 | assertEquals( 31 | expected.getValue(), 32 | actual.getValue(), 33 | "Checksums of read data do not match! Possible correctness issue"); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /input-stream/src/integrationTest/java/software/amazon/s3/analyticsaccelerator/access/ExceptionCorrectnessTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertThrows; 19 | 20 | import java.io.FileNotFoundException; 21 | import java.io.IOException; 22 | import java.util.stream.Stream; 23 | import org.junit.jupiter.params.ParameterizedTest; 24 | import org.junit.jupiter.params.provider.Arguments; 25 | import org.junit.jupiter.params.provider.MethodSource; 26 | import software.amazon.awssdk.services.s3.S3AsyncClient; 27 | import software.amazon.s3.analyticsaccelerator.S3SdkObjectClient; 28 | import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamConfiguration; 29 | import software.amazon.s3.analyticsaccelerator.S3SeekableInputStreamFactory; 30 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 31 | 32 | public class ExceptionCorrectnessTest extends IntegrationTestBase { 33 | 34 | private static final String NON_EXISTENT_OBJECT = "non-existent.bin"; 35 | 36 | @ParameterizedTest 37 | @MethodSource("provideTestParameters") 38 | void testNonExistentObjectThrowsRightException(S3ClientKind clientKind) throws IOException { 39 | final S3ExecutionConfiguration config = this.getS3ExecutionContext().getConfiguration(); 40 | final S3URI nonExistentURI = 41 | S3URI.of(config.getBucket(), config.getPrefix() + NON_EXISTENT_OBJECT); 42 | S3AsyncClient s3AsyncClient = clientKind.getS3Client(getS3ExecutionContext()); 43 | S3SeekableInputStreamFactory factory = createInputStreamFactory(s3AsyncClient); 44 | assertThrows(FileNotFoundException.class, () -> factory.createStream(nonExistentURI)); 45 | } 46 | 47 | private static Stream provideTestParameters() { 48 | return getS3ClientKinds().stream().map(Arguments::of); 49 | } 50 | 51 | private static S3SeekableInputStreamFactory createInputStreamFactory( 52 | S3AsyncClient s3AsyncClient) { 53 | return new S3SeekableInputStreamFactory( 54 | new S3SdkObjectClient(s3AsyncClient), S3SeekableInputStreamConfiguration.DEFAULT); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /input-stream/src/integrationTest/java/software/amazon/s3/analyticsaccelerator/access/ReadVectoredTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import java.io.IOException; 19 | import java.nio.ByteBuffer; 20 | import java.util.ArrayList; 21 | import java.util.List; 22 | import java.util.stream.Stream; 23 | import org.junit.jupiter.params.ParameterizedTest; 24 | import org.junit.jupiter.params.provider.Arguments; 25 | import org.junit.jupiter.params.provider.MethodSource; 26 | 27 | public class ReadVectoredTest extends IntegrationTestBase { 28 | @ParameterizedTest 29 | @MethodSource("vectoredReads") 30 | void testVectoredReads( 31 | S3ClientKind s3ClientKind, 32 | S3Object s3Object, 33 | StreamReadPatternKind streamReadPattern, 34 | AALInputStreamConfigurationKind configuration) 35 | throws IOException { 36 | // Run with non-direct buffers 37 | testReadVectored( 38 | s3ClientKind, s3Object, streamReadPattern, configuration, ByteBuffer::allocate); 39 | // Run with direct buffers 40 | testReadVectored( 41 | s3ClientKind, s3Object, streamReadPattern, configuration, ByteBuffer::allocateDirect); 42 | } 43 | 44 | static Stream vectoredReads() { 45 | List readVectoredObjects = new ArrayList<>(); 46 | readVectoredObjects.add(S3Object.RANDOM_1GB); 47 | readVectoredObjects.add(S3Object.CSV_20MB); 48 | 49 | return argumentsFor( 50 | getS3ClientKinds(), 51 | readVectoredObjects, 52 | sequentialPatterns(), 53 | getS3SeekableInputStreamConfigurations()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/AALBenchmark.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks; 17 | 18 | import java.io.IOException; 19 | import org.openjdk.jmh.annotations.*; 20 | import software.amazon.s3.analyticsaccelerator.access.S3ClientKind; 21 | import software.amazon.s3.analyticsaccelerator.access.S3Object; 22 | import software.amazon.s3.analyticsaccelerator.access.StreamReadPatternKind; 23 | 24 | /** 25 | * Benchmarks that measure performance of DAT via CRT by replaying all patterns against multiple 26 | * object sizes 27 | */ 28 | public class AALBenchmark extends BenchmarkBase { 29 | // NOTE: all params here must come after "object", so they should start with any letter after "o". 30 | @Param public StreamReadPatternKind pattern; 31 | 32 | /** 33 | * Runs the benchmark for a given pattern and object 34 | * 35 | * @throws IOException IO error, if encountered 36 | */ 37 | @Override 38 | protected void executeBenchmark() throws IOException { 39 | executeReadPatternOnDAT(); 40 | } 41 | 42 | /** 43 | * S3 Client Kind 44 | * 45 | * @return {@link S3ClientKind} 46 | */ 47 | @Override 48 | protected S3ClientKind getClientKind() { 49 | // Always use CRT client 50 | return S3ClientKind.SDK_V2_CRT_ASYNC; 51 | } 52 | 53 | @Override 54 | protected S3Object getObject() { 55 | return object; 56 | } 57 | 58 | @Override 59 | protected StreamReadPatternKind getReadPatternKind() { 60 | return pattern; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/ComparisonBenchmarkBase.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks; 17 | 18 | import java.io.IOException; 19 | import software.amazon.s3.analyticsaccelerator.access.S3InputStreamKind; 20 | 21 | /** Base class for benchmarks that compare performance of different clients */ 22 | public abstract class ComparisonBenchmarkBase extends BenchmarkBase { 23 | /** 24 | * Benchmarks can override this to return the {@link S3InputStreamKind} 25 | * 26 | * @return {@link S3InputStreamKind} 27 | */ 28 | protected abstract S3InputStreamKind getS3InputStreamKind(); 29 | 30 | /** 31 | * Runs the benchmark for a given pattern and object 32 | * 33 | * @throws IOException IO error, if encountered 34 | */ 35 | @Override 36 | protected void executeBenchmark() throws IOException { 37 | switch (this.getS3InputStreamKind()) { 38 | case S3_DAT_GET: 39 | this.executeReadPatternOnDAT(); 40 | break; 41 | case S3_SDK_GET: 42 | this.executeReadPatternDirectly(); 43 | break; 44 | default: 45 | throw new IllegalArgumentException( 46 | "Unsupported S3 input kind: " + this.getS3InputStreamKind()); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/S3ClientAndStreamKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | import software.amazon.s3.analyticsaccelerator.access.S3ClientKind; 21 | import software.amazon.s3.analyticsaccelerator.access.S3InputStreamKind; 22 | 23 | /** Client and stream kind combined. This simplifies benchmarking parameterization and reports */ 24 | @AllArgsConstructor 25 | @Getter 26 | public enum S3ClientAndStreamKind { 27 | SDK_ASYNC_JAVA(S3ClientKind.SDK_V2_JAVA_ASYNC, S3InputStreamKind.S3_SDK_GET), 28 | SDK_DAT_JAVA(S3ClientKind.SDK_V2_JAVA_ASYNC, S3InputStreamKind.S3_DAT_GET), 29 | SDK_ASYNC_CRT(S3ClientKind.SDK_V2_CRT_ASYNC, S3InputStreamKind.S3_SDK_GET), 30 | SDK_DAT_CRT(S3ClientKind.SDK_V2_CRT_ASYNC, S3InputStreamKind.S3_DAT_GET); 31 | 32 | private final S3ClientKind clientKind; 33 | private final S3InputStreamKind inputStreamKind; 34 | } 35 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/StreamPatternComparisonBenchmark.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks; 17 | 18 | import lombok.NonNull; 19 | import org.openjdk.jmh.annotations.Param; 20 | import software.amazon.s3.analyticsaccelerator.access.S3ClientKind; 21 | import software.amazon.s3.analyticsaccelerator.access.S3InputStreamKind; 22 | import software.amazon.s3.analyticsaccelerator.access.S3Object; 23 | import software.amazon.s3.analyticsaccelerator.access.StreamReadPatternKind; 24 | 25 | /** 26 | * This benchmarks fixes the ReadPattern, and then, for each object size, replays the pattern on 27 | * "raw" clients as well as "DAT" streams that sit on the clients The main point here is to 28 | * determine which combo is the fastest for each pattern. 29 | */ 30 | public abstract class StreamPatternComparisonBenchmark extends ComparisonBenchmarkBase { 31 | // NOTE: all params here must come after "object", so they should start with any letter after "o". 32 | @Param public S3ClientAndStreamKind s3ClientAndStreamKind; 33 | 34 | @NonNull private final StreamReadPatternKind patternKind; 35 | 36 | /** 37 | * Creates a new instance of {@link StreamPatternComparisonBenchmark} 38 | * 39 | * @param patternKind pattern kind 40 | */ 41 | protected StreamPatternComparisonBenchmark(@NonNull StreamReadPatternKind patternKind) { 42 | this.patternKind = patternKind; 43 | } 44 | 45 | @Override 46 | protected S3InputStreamKind getS3InputStreamKind() { 47 | return this.s3ClientAndStreamKind.getInputStreamKind(); 48 | } 49 | 50 | @Override 51 | protected S3Object getObject() { 52 | return this.object; 53 | } 54 | 55 | @Override 56 | protected StreamReadPatternKind getReadPatternKind() { 57 | return this.patternKind; 58 | } 59 | 60 | @Override 61 | protected S3ClientKind getClientKind() { 62 | return this.s3ClientAndStreamKind.getClientKind(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/StreamPatternComparisonBenchmarks.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks; 17 | 18 | import software.amazon.s3.analyticsaccelerator.access.StreamReadPatternKind; 19 | 20 | /** 21 | * Container for the benchmarks based on {@link StreamPatternComparisonBenchmark} * We create 22 | * separate classes for each type of read pattern so that we get separate reports for each type of 23 | * pattern 24 | */ 25 | public class StreamPatternComparisonBenchmarks { 26 | /** SEQUENTIAL pattern */ 27 | public static class ComparisonSequential extends StreamPatternComparisonBenchmark { 28 | /** Constructor */ 29 | public ComparisonSequential() { 30 | super(StreamReadPatternKind.SEQUENTIAL); 31 | } 32 | } 33 | 34 | /** SKIPPING_FORWARD pattern */ 35 | public static class ComparisonSkippingForward extends StreamPatternComparisonBenchmark { 36 | /** Constructor */ 37 | public ComparisonSkippingForward() { 38 | super(StreamReadPatternKind.SKIPPING_FORWARD); 39 | } 40 | } 41 | 42 | /** SKIPPING_BACKWARD pattern */ 43 | public static class ComparisonSkippingBackward extends StreamPatternComparisonBenchmark { 44 | /** Constructor */ 45 | public ComparisonSkippingBackward() { 46 | super(StreamReadPatternKind.SKIPPING_BACKWARD); 47 | } 48 | } 49 | 50 | /** QUASI_PARQUET_ROW_GROUP pattern */ 51 | public static class ComparisonQuasiParquetRowGroup extends StreamPatternComparisonBenchmark { 52 | /** Constructor */ 53 | public ComparisonQuasiParquetRowGroup() { 54 | super(StreamReadPatternKind.QUASI_PARQUET_ROW_GROUP); 55 | } 56 | } 57 | 58 | /** QUASI_PARQUET_COLUMN_CHUNK */ 59 | public static class ComparisonQuasiParquetColumnChunk extends StreamPatternComparisonBenchmark { 60 | /** Constructor */ 61 | public ComparisonQuasiParquetColumnChunk() { 62 | super(StreamReadPatternKind.QUASI_PARQUET_COLUMN_CHUNK); 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /input-stream/src/jmh/java/software/amazon/s3/analyticsaccelerator/benchmarks/data/generation/BenchmarkObjectGenerator.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.benchmarks.data.generation; 17 | 18 | import lombok.Getter; 19 | import lombok.NonNull; 20 | import lombok.RequiredArgsConstructor; 21 | import software.amazon.s3.analyticsaccelerator.access.S3ExecutionContext; 22 | import software.amazon.s3.analyticsaccelerator.access.S3ObjectKind; 23 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 24 | 25 | /** Base class for all generators */ 26 | @Getter 27 | @RequiredArgsConstructor 28 | public abstract class BenchmarkObjectGenerator { 29 | @NonNull private final S3ExecutionContext context; 30 | @NonNull private final S3ObjectKind kind; 31 | 32 | /** 33 | * Generate data 34 | * 35 | * @param s3URI S3 URI to generate data into 36 | * @param size object size 37 | */ 38 | public abstract void generate(S3URI s3URI, long size); 39 | } 40 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/LogicalIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical; 17 | 18 | import software.amazon.s3.analyticsaccelerator.RandomAccessReadable; 19 | 20 | /** 21 | * Interface responsible for implementing "logical" reads. Logical reads are not concerned with the 22 | * "how" of reading data but only with "what" data to read. 23 | * 24 | *

On a high level the logical IO layer sits above a physical IO layer and "observes" what data 25 | * is being read by the user calling the stream. Based on this history and other hints (such as key 26 | * name, metadata information) the logical IO layer can formulate "what" data should be read. The 27 | * logical layer should be able to create an IOPlan based on this and use the physical layer to 28 | * execute this asynchronously. 29 | * 30 | *

For now, this interface is a marker interface but should become more soon. 31 | */ 32 | public interface LogicalIO extends RandomAccessReadable {} 33 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/impl/SequentialLogicalIOImpl.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.impl; 17 | 18 | import java.io.IOException; 19 | import lombok.NonNull; 20 | import software.amazon.s3.analyticsaccelerator.common.telemetry.Telemetry; 21 | import software.amazon.s3.analyticsaccelerator.io.logical.LogicalIOConfiguration; 22 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIO; 23 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 24 | 25 | /** 26 | * A sequential read-aware implementation of the LogicalIO layer. This implementation is capable of 27 | * configurable prefetching based on the provided LogicalIOConfiguration. It specifically utilizes 28 | * the 'sparkPartitionSize' configuration to determine the size of data to prefetch, optimizing for 29 | * Spark's data processing patterns. 30 | */ 31 | public class SequentialLogicalIOImpl extends DefaultLogicalIOImpl { 32 | private final SequentialPrefetcher sequentialPrefetcher; 33 | 34 | /** 35 | * Constructs an instance of SequentialLogicalIOImpl. 36 | * 37 | * @param s3URI the S3 URI of the object fetched 38 | * @param physicalIO underlying physical IO that knows how to fetch bytes 39 | * @param telemetry an instance of {@link Telemetry} to use 40 | * @param logicalIOConfiguration configuration for this logical IO implementation 41 | */ 42 | public SequentialLogicalIOImpl( 43 | @NonNull S3URI s3URI, 44 | @NonNull PhysicalIO physicalIO, 45 | @NonNull Telemetry telemetry, 46 | @NonNull LogicalIOConfiguration logicalIOConfiguration) { 47 | super(s3URI, physicalIO, telemetry); 48 | this.sequentialPrefetcher = 49 | new SequentialPrefetcher(s3URI, physicalIO, telemetry, logicalIOConfiguration); 50 | } 51 | 52 | /** 53 | * Reads data into the provided buffer 54 | * 55 | * @param buf buffer to read data into 56 | * @param off start position in buffer at which data is written 57 | * @param len length of data to be read 58 | * @param position the position to begin reading from 59 | * @return an unsigned int representing the byte that was read 60 | * @throws IOException IO error, if incurred. 61 | */ 62 | @Override 63 | public int read(byte[] buf, int off, int len, long position) throws IOException { 64 | sequentialPrefetcher.prefetch(position); 65 | return super.read(buf, off, len, position); 66 | } 67 | 68 | @Override 69 | public void close() throws IOException { 70 | super.closeWithEviction(true); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/parquet/ColumnMappers.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.parquet; 17 | 18 | import java.util.List; 19 | import java.util.Map; 20 | import lombok.Value; 21 | 22 | /** Mappings of parquet column file offset index to column name and vice versa. */ 23 | @Value 24 | public class ColumnMappers { 25 | Map offsetIndexToColumnMap; 26 | Map> columnNameToColumnMap; 27 | } 28 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/parquet/ColumnMetadata.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.parquet; 17 | 18 | import lombok.Data; 19 | 20 | /** Container for storing necessary parquet column information. */ 21 | @Data 22 | public class ColumnMetadata { 23 | private final int rowGroupIndex; 24 | private final String columnName; 25 | private final long dataPageOffset; 26 | private final long dictionaryOffset; 27 | private final long startPos; 28 | private final long compressedSize; 29 | private final int schemaHash; 30 | } 31 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/parquet/FileTail.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.parquet; 17 | 18 | import java.nio.ByteBuffer; 19 | import lombok.Data; 20 | 21 | /** Container for tail of a parquet file. */ 22 | @Data 23 | public class FileTail { 24 | private final ByteBuffer fileTail; 25 | private final int fileTailLength; 26 | } 27 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/logical/parquet/FooterPrefetchSize.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.parquet; 17 | 18 | import lombok.Value; 19 | 20 | /** Stores sizes to prefetch for file metadata and page index structures. */ 21 | @Value 22 | public class FooterPrefetchSize { 23 | long fileMetadataPrefetchSize; 24 | long pageIndexPrefetchSize; 25 | 26 | /** 27 | * Default constructor to store footer prefetch sizes 28 | * 29 | * @param fileMetadataPrefetchSize size of file metadata to prefetch from file tail 30 | * @param pageIndexPrefetchSize size of page index to prefetch from file tail 31 | */ 32 | public FooterPrefetchSize(long fileMetadataPrefetchSize, long pageIndexPrefetchSize) { 33 | this.fileMetadataPrefetchSize = fileMetadataPrefetchSize; 34 | this.pageIndexPrefetchSize = pageIndexPrefetchSize; 35 | } 36 | 37 | /** 38 | * Gets total size of data to be read from the tail of the file 39 | * 40 | * @return size of file tail read 41 | */ 42 | public long getSize() { 43 | return fileMetadataPrefetchSize + pageIndexPrefetchSize; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/PhysicalIO.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical; 17 | 18 | import java.io.IOException; 19 | import java.nio.ByteBuffer; 20 | import java.util.List; 21 | import java.util.function.IntFunction; 22 | import software.amazon.s3.analyticsaccelerator.RandomAccessReadable; 23 | import software.amazon.s3.analyticsaccelerator.common.ObjectRange; 24 | import software.amazon.s3.analyticsaccelerator.io.physical.plan.IOPlan; 25 | import software.amazon.s3.analyticsaccelerator.io.physical.plan.IOPlanExecution; 26 | 27 | /** An interface defining how a logical IO layer gets hooked into Physical IO. */ 28 | public interface PhysicalIO extends RandomAccessReadable { 29 | 30 | /** 31 | * Async method capable of executing a logical IO plan. 32 | * 33 | * @param ioPlan the plan to execute asynchronously 34 | * @return an IOPlanExecution object tracking the execution of the submitted plan 35 | */ 36 | IOPlanExecution execute(IOPlan ioPlan) throws IOException; 37 | 38 | /** 39 | * Fetches the list of provided ranges in parallel. Byte buffers are created using the allocate 40 | * method, and may be direct or non-direct depending on the implementation of the allocate method. 41 | * When a provided range has been fully read, the associated future for it is completed. 42 | * 43 | * @param objectRanges Ranges to be fetched in parallel 44 | * @param allocate the function to allocate ByteBuffer 45 | * @throws IOException on any IO failure 46 | */ 47 | void readVectored(List objectRanges, IntFunction allocate) 48 | throws IOException; 49 | 50 | /** 51 | * Closes the PhysicalIO and optionally evicts associated data. 52 | * 53 | * @param shouldEvict whether associated data should be evicted 54 | * @throws IOException if an I/O error occurs 55 | */ 56 | void close(boolean shouldEvict) throws IOException; 57 | } 58 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/data/RangeOptimiser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.data; 17 | 18 | import java.util.LinkedList; 19 | import java.util.List; 20 | import lombok.Value; 21 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIOConfiguration; 22 | import software.amazon.s3.analyticsaccelerator.request.Range; 23 | 24 | /** 25 | * RangeSplitter is responsible for splitting up big ranges into smaller reads. The need for such 26 | * functionality arises from sequential prefetching. When we decide that, e.g., the next 128MB chunk 27 | * of an object is needed with high confidence, then we should not fetch this in a single request. 28 | * 29 | *

This class is capable of implementing heuristics on how to fetch ranges of different sizes 30 | * optimally. 31 | */ 32 | @Value 33 | public class RangeOptimiser { 34 | PhysicalIOConfiguration configuration; 35 | 36 | /** 37 | * Given a list of ranges, return a potentially new set of ranges which is more optimal to fetch 38 | * (i.e., split up huge ranges based on a heuristic). 39 | * 40 | * @param ranges a list of ranges 41 | * @return a potentially different list of ranges with big ranges split up 42 | */ 43 | public List splitRanges(List ranges) { 44 | List splits = new LinkedList<>(); 45 | for (Range range : ranges) { 46 | if (range.getLength() > configuration.getMaxRangeSizeBytes()) { 47 | splitRange(range.getStart(), range.getEnd()).forEach(splits::add); 48 | } else { 49 | splits.add(range); 50 | } 51 | } 52 | 53 | return splits; 54 | } 55 | 56 | private List splitRange(long start, long end) { 57 | long nextRangeStart = start; 58 | List generatedRanges = new LinkedList<>(); 59 | 60 | while (nextRangeStart < end) { 61 | long rangeEnd = Math.min(nextRangeStart + configuration.getPartSizeBytes() - 1, end); 62 | generatedRanges.add(new Range(nextRangeStart, rangeEnd)); 63 | nextRangeStart = rangeEnd + 1; 64 | } 65 | 66 | return generatedRanges; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/plan/IOPlan.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.plan; 17 | 18 | import java.util.ArrayList; 19 | import java.util.Collection; 20 | import java.util.Collections; 21 | import java.util.stream.Collectors; 22 | import lombok.Getter; 23 | import lombok.NonNull; 24 | import software.amazon.s3.analyticsaccelerator.request.Range; 25 | 26 | /** A logical IO plan */ 27 | @Getter 28 | public class IOPlan { 29 | private final ArrayList prefetchRanges; 30 | public static final IOPlan EMPTY_PLAN = new IOPlan(Collections.emptyList()); 31 | 32 | /** 33 | * Creates a new instance of {@link IOPlan} 34 | * 35 | * @param prefetchRange single prefetch range 36 | */ 37 | public IOPlan(@NonNull Range prefetchRange) { 38 | this.prefetchRanges = new ArrayList<>(1); 39 | this.prefetchRanges.add(prefetchRange); 40 | } 41 | /** 42 | * Creates a new instance of {@link IOPlan} 43 | * 44 | * @param prefetchRanges prefetch ranges 45 | */ 46 | public IOPlan(@NonNull Collection prefetchRanges) { 47 | this.prefetchRanges = new ArrayList<>(prefetchRanges); 48 | } 49 | 50 | /** 51 | * Returns a string representation of the object. 52 | * 53 | * @return a string representation of the object. 54 | */ 55 | @Override 56 | public String toString() { 57 | return "[" 58 | + this.prefetchRanges.stream().map(Range::toString).collect(Collectors.joining(",")) 59 | + "]"; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/plan/IOPlanExecution.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.plan; 17 | 18 | import lombok.Builder; 19 | import lombok.Data; 20 | import lombok.NonNull; 21 | 22 | /** 23 | * Object representing the state of an IO Plan execution. Effectively, this will be a handle on an 24 | * IOPlan execution and PhysicalIO should be able to communicate with LogicalIO through this. 25 | */ 26 | @Builder 27 | @Data 28 | public class IOPlanExecution { 29 | 30 | /** State of the IO Plan execution */ 31 | @NonNull private final IOPlanState state; 32 | } 33 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/plan/IOPlanState.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.plan; 17 | 18 | /** Enum representing a state of an IOPlan that was submitted from LogicalIO. */ 19 | public enum IOPlanState { 20 | 21 | /** IOPlan was successfully submitted and is executed by the physical IO */ 22 | SUBMITTED, 23 | 24 | /** No IOPlan was submitted to PhysicalIO */ 25 | SKIPPED, 26 | 27 | /** Failed to submit IOPlan to PhysicalIO */ 28 | FAILED 29 | } 30 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/prefetcher/SequentialPatternDetector.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.prefetcher; 17 | 18 | import lombok.RequiredArgsConstructor; 19 | import software.amazon.s3.analyticsaccelerator.common.Preconditions; 20 | import software.amazon.s3.analyticsaccelerator.io.physical.data.BlockStore; 21 | 22 | /** 23 | * Class capable of detecting sequential read patterns. 24 | * 25 | *

The SequentialPatternDetector depends on the BlockStore which it is capable of 'inspecting'. 26 | * For now, pattern detection is very simple: if the previous byte is already stored by a block, 27 | * then we conclude that the byte is requested by a sequential read. In the future we should 28 | * probably extend this to work with some kind of tolerance or radius (i.e., look back more than a 29 | * single byte). 30 | * 31 | *

The other responsibility of this class is to compute what the 'generation' of a position is in 32 | * the BlockStore. 33 | */ 34 | @RequiredArgsConstructor 35 | public class SequentialPatternDetector { 36 | 37 | private final BlockStore blockStore; 38 | 39 | /** 40 | * Given that the byte at 'pos' will be read next, is this read part of a sequential read? 41 | * 42 | * @param pos the position of a byte that is requested 43 | * @return returns true if this read is part of a sequential read pattern 44 | */ 45 | public boolean isSequentialRead(long pos) { 46 | Preconditions.checkArgument(pos >= 0, "`pos` must be non-negative"); 47 | 48 | if (pos == 0) { 49 | return false; 50 | } 51 | 52 | return blockStore.getBlock(pos - 1).isPresent(); 53 | } 54 | 55 | /** 56 | * Given that the byte at 'pos' will be read next, what generation does it belong to? 57 | * 58 | * @param pos the position of a byte that is requested 59 | * @return returns the generation of the byte 60 | */ 61 | public long getGeneration(long pos) { 62 | Preconditions.checkArgument(pos >= 0, "`pos` must be non-negative"); 63 | 64 | if (isSequentialRead(pos)) { 65 | return blockStore.getBlock(pos - 1).get().getGeneration() + 1; 66 | } 67 | 68 | return 0; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/io/physical/prefetcher/SequentialReadProgression.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.prefetcher; 17 | 18 | import static software.amazon.s3.analyticsaccelerator.util.Constants.ONE_MB; 19 | 20 | import lombok.AllArgsConstructor; 21 | import software.amazon.s3.analyticsaccelerator.common.Preconditions; 22 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIOConfiguration; 23 | 24 | /** 25 | * Class that implements a mathematical function telling us the size of blocks we should prefetch in 26 | * a sequential read. 27 | */ 28 | @AllArgsConstructor 29 | public class SequentialReadProgression { 30 | 31 | private final PhysicalIOConfiguration configuration; 32 | 33 | /** 34 | * Given a generation, returns the size of a sequential prefetch block for that generation. This 35 | * function is effectively a geometric series today but can be fine-tuned later. 36 | * 37 | * @param generation zero-indexed integer representing the generation of a read 38 | * @return a block size in bytes 39 | */ 40 | public long getSizeForGeneration(long generation) { 41 | Preconditions.checkArgument(0 <= generation, "`generation` must be non-negative"); 42 | 43 | // 2, 8, 32, 64 44 | return 2 45 | * ONE_MB 46 | * (long) 47 | Math.pow( 48 | configuration.getSequentialPrefetchBase(), 49 | Math.floor(configuration.getSequentialPrefetchSpeed() * generation)); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/util/AnalyticsAcceleratorUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIOConfiguration; 19 | 20 | /** 21 | * Utility class for object size-related operations and determinations. Provides methods to classify 22 | * objects based on their size and configuration settings. 23 | */ 24 | public class AnalyticsAcceleratorUtils { 25 | /** 26 | * Determines if an object should be treated as a small object based on the given configuration. 27 | * 28 | * @param configuration The PhysicalIOConfiguration containing small object settings 29 | * @param contentLength The length of the object in bytes 30 | * @return true if the object should be treated as a small object 31 | */ 32 | public static boolean isSmallObject(PhysicalIOConfiguration configuration, long contentLength) { 33 | return configuration.isSmallObjectsPrefetchingEnabled() 34 | && contentLength <= configuration.getSmallObjectSizeThreshold(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/util/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | /** S3SeekableInputStream constants. */ 19 | public final class Constants { 20 | /** Prevent instantiation, this is meant to be a facade */ 21 | private Constants() {} 22 | 23 | public static final int ONE_KB = 1024; 24 | public static final int ONE_MB = 1024 * 1024; 25 | public static final long ONE_GB = 1024 * 1024 * 1024; 26 | public static final int PARQUET_MAGIC_STR_LENGTH = 4; 27 | public static final int PARQUET_FOOTER_LENGTH_SIZE = 4; 28 | public static final long DEFAULT_MIN_ADJACENT_COLUMN_LENGTH = 500 * ONE_KB; 29 | } 30 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/util/ObjectFormat.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | /** Enum for file formats. */ 19 | public enum ObjectFormat { 20 | PARQUET, 21 | SEQUENTIAL, 22 | DEFAULT 23 | } 24 | -------------------------------------------------------------------------------- /input-stream/src/main/java/software/amazon/s3/analyticsaccelerator/util/PrefetchMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | /** Class defining prefetch modes. */ 22 | public enum PrefetchMode { 23 | OFF("off"), 24 | ALL("all"), 25 | ROW_GROUP("row_group"), 26 | COLUMN_BOUND("column_bound"); 27 | 28 | private final String name; 29 | 30 | private static final Logger LOG = LoggerFactory.getLogger(PrefetchMode.class); 31 | 32 | PrefetchMode(String name) { 33 | this.name = name; 34 | } 35 | 36 | /** 37 | * Converts user supplied configuration to enum. Defaults to ROW_GROUP is user input is not 38 | * recognised. 39 | * 40 | * @param prefetchMode user supplier prefetch mode 41 | * @return PrefetchMode enum to use 42 | */ 43 | public static PrefetchMode fromString(String prefetchMode) { 44 | for (PrefetchMode value : values()) { 45 | if (value.name.equalsIgnoreCase(prefetchMode)) { 46 | return value; 47 | } 48 | } 49 | LOG.debug("Unknown prefetch mode {}, using default row_group mode.", prefetchMode); 50 | 51 | return ROW_GROUP; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /input-stream/src/referenceTest/java/software/amazon/s3/analyticsaccelerator/arbitraries/StreamArbitraries.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.arbitraries; 17 | 18 | import static software.amazon.s3.analyticsaccelerator.util.Constants.ONE_MB; 19 | 20 | import net.jqwik.api.Arbitraries; 21 | import net.jqwik.api.Arbitrary; 22 | import net.jqwik.api.Provide; 23 | 24 | public class StreamArbitraries { 25 | 26 | private static final int MAX_STREAM_LENGTH_UNDER_TEST = 16 * ONE_MB; 27 | 28 | @Provide 29 | Arbitrary streamSizes() { 30 | return Arbitraries.integers().between(0, MAX_STREAM_LENGTH_UNDER_TEST); 31 | } 32 | 33 | @Provide 34 | Arbitrary positiveStreamSizes() { 35 | return Arbitraries.integers().between(1, MAX_STREAM_LENGTH_UNDER_TEST); 36 | } 37 | 38 | @Provide 39 | Arbitrary sizeBiggerThanOne() { 40 | return Arbitraries.integers().between(2, MAX_STREAM_LENGTH_UNDER_TEST); 41 | } 42 | 43 | @Provide 44 | Arbitrary validPositions() { 45 | return Arbitraries.integers().between(0, MAX_STREAM_LENGTH_UNDER_TEST); 46 | } 47 | 48 | @Provide 49 | Arbitrary invalidPositions() { 50 | return Arbitraries.integers().lessOrEqual(-1); 51 | } 52 | 53 | @Provide 54 | Arbitrary bufferSizes() { 55 | return Arbitraries.integers().between(1, 16 * 1024 * 1024); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/SpotBugsLambdaWorkaround.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import java.io.Closeable; 21 | import lombok.SneakyThrows; 22 | import org.junit.jupiter.api.function.ThrowingSupplier; 23 | 24 | /** 25 | * This class is here to help work around certain SpotBugs quirks. Specifically, SpotBugs don't 26 | * allow to apply suppressions to Lambdas 27 | */ 28 | public final class SpotBugsLambdaWorkaround { 29 | /** 30 | * In situations where a method or constructor return a class that implements Closeable, Spotbugs 31 | * wants to see it closed. When testing bounds and expecting exceptions, this is unnecessary, and 32 | * trying to work around this in tests results in a lot of boilerplate. This method absorbs all 33 | * the boilerplate and acts as `assertThrows`. 34 | * 35 | * @param expectedType exception type expected 36 | * @param executable code that returns something Closeable, and expected to throw 37 | * @param exception type 38 | * @param return type 39 | */ 40 | @SneakyThrows 41 | @SuppressWarnings("try") 42 | public static void assertThrowsClosableResult( 43 | Class expectedType, ThrowingSupplier executable) { 44 | try (C result = executable.get()) { 45 | } catch (Throwable throwable) { 46 | assertInstanceOf(expectedType, throwable); 47 | return; 48 | } 49 | fail(String.format("Exception of type '%s' was expected. Nothing was thrown", expectedType)); 50 | } 51 | 52 | /** 53 | * In situations where a read method of a class extending InputStream, spotbugs wants to assert 54 | * return value is checked (@link https://spotbugs.readthedocs.io/en/stable/bugDescriptions.html). 55 | * 56 | * @param expectedType exception type expected 57 | * @param executable code that returns something, and expected to throw 58 | * @param expected expected return value, though will never be executed as executable is throwing 59 | * @param exception type 60 | * @param return type 61 | */ 62 | public static void assertReadResult( 63 | Class expectedType, ThrowingSupplier executable, R expected) { 64 | try { 65 | R result = executable.get(); 66 | assertEquals(expected, result); 67 | } catch (Throwable throwable) { 68 | assertInstanceOf(expectedType, throwable); 69 | return; 70 | } 71 | fail(String.format("Exception of type '%s' was expected. Nothing was thrown", expectedType)); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/TestTelemetry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import software.amazon.s3.analyticsaccelerator.common.telemetry.Telemetry; 19 | import software.amazon.s3.analyticsaccelerator.common.telemetry.TelemetryConfiguration; 20 | import software.amazon.s3.analyticsaccelerator.common.telemetry.TelemetryLevel; 21 | 22 | public final class TestTelemetry { 23 | private TestTelemetry() {} 24 | 25 | /** 26 | * Default telemetry to use for tests - combination of NoOp telemetry with VERBOSE level provides 27 | * the best code coverage 28 | */ 29 | public static final Telemetry DEFAULT = 30 | Telemetry.createTelemetry( 31 | TelemetryConfiguration.builder() 32 | .loggingEnabled(false) 33 | .stdOutEnabled(false) 34 | .level(TelemetryLevel.VERBOSE.toString()) 35 | .build()); 36 | } 37 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/io/logical/parquet/IOPlanMatcher.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.logical.parquet; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertArrayEquals; 19 | 20 | import java.util.List; 21 | import org.mockito.ArgumentMatcher; 22 | import software.amazon.s3.analyticsaccelerator.io.physical.plan.IOPlan; 23 | import software.amazon.s3.analyticsaccelerator.request.Range; 24 | 25 | class IOPlanMatcher implements ArgumentMatcher { 26 | private final List expectedRanges; 27 | 28 | public IOPlanMatcher(List expectedRanges) { 29 | this.expectedRanges = expectedRanges; 30 | } 31 | 32 | @Override 33 | public boolean matches(IOPlan argument) { 34 | assertArrayEquals(argument.getPrefetchRanges().toArray(), expectedRanges.toArray()); 35 | return true; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/io/physical/data/RangeOptimiserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.data; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static software.amazon.s3.analyticsaccelerator.util.Constants.ONE_MB; 20 | 21 | import java.util.LinkedList; 22 | import java.util.List; 23 | import org.junit.jupiter.api.Test; 24 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIOConfiguration; 25 | import software.amazon.s3.analyticsaccelerator.request.Range; 26 | 27 | public class RangeOptimiserTest { 28 | 29 | @Test 30 | public void test__splitRanges__smallRangesCauseNoSplit() { 31 | // Given: small ranges 32 | RangeOptimiser rangeOptimiser = new RangeOptimiser(PhysicalIOConfiguration.DEFAULT); 33 | List ranges = new LinkedList<>(); 34 | ranges.add(new Range(0, 100)); 35 | ranges.add(new Range(200, 300)); 36 | ranges.add(new Range(300, 400)); 37 | ranges.add(new Range(400, 500)); 38 | 39 | // When: splitRanges is called 40 | List splitRanges = rangeOptimiser.splitRanges(ranges); 41 | 42 | // Then: nothing happens 43 | assertEquals(ranges, splitRanges); 44 | } 45 | 46 | @Test 47 | public void test__splitRanges__bigRangesResultInSplits() { 48 | // Given: a 16MB range 49 | RangeOptimiser rangeOptimiser = new RangeOptimiser(PhysicalIOConfiguration.DEFAULT); 50 | List ranges = new LinkedList<>(); 51 | ranges.add(new Range(0, 16 * ONE_MB - 1)); 52 | 53 | // When: splitRanges is called 54 | List splitRanges = rangeOptimiser.splitRanges(ranges); 55 | 56 | // Then: 16MB range is split into 4x4MB ranges 57 | List expected = new LinkedList<>(); 58 | expected.add(new Range(0, 8 * ONE_MB - 1)); 59 | expected.add(new Range(8 * ONE_MB, 16 * ONE_MB - 1)); 60 | assertEquals(expected, splitRanges); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/io/physical/prefetcher/SequentialReadProgressionTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.io.physical.prefetcher; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | import static software.amazon.s3.analyticsaccelerator.util.Constants.ONE_MB; 20 | 21 | import org.junit.jupiter.api.Test; 22 | import software.amazon.s3.analyticsaccelerator.io.physical.PhysicalIOConfiguration; 23 | 24 | public class SequentialReadProgressionTest { 25 | 26 | @Test 27 | public void test__sequentialReadProgression__asExpected() { 28 | // Given: a SequentialReadProgression 29 | SequentialReadProgression sequentialReadProgression = 30 | new SequentialReadProgression(PhysicalIOConfiguration.DEFAULT); 31 | 32 | // When & Then: size is requested for a generation --> size is correct 33 | assertEquals(2 * ONE_MB, sequentialReadProgression.getSizeForGeneration(0)); 34 | assertEquals(4 * ONE_MB, sequentialReadProgression.getSizeForGeneration(1)); 35 | assertEquals(8 * ONE_MB, sequentialReadProgression.getSizeForGeneration(2)); 36 | assertEquals(16 * ONE_MB, sequentialReadProgression.getSizeForGeneration(3)); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/plan/IOPlanTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.plan; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import edu.umd.cs.findbugs.annotations.SuppressFBWarnings; 21 | import java.util.ArrayList; 22 | import java.util.Collection; 23 | import org.junit.jupiter.api.Test; 24 | import software.amazon.s3.analyticsaccelerator.io.physical.plan.IOPlan; 25 | import software.amazon.s3.analyticsaccelerator.request.Range; 26 | 27 | @SuppressFBWarnings( 28 | value = "NP_NONNULL_PARAM_VIOLATION", 29 | justification = "We mean to pass nulls to checks") 30 | public class IOPlanTest { 31 | @Test 32 | void testRangeListConstructor() { 33 | ArrayList ranges = new ArrayList<>(); 34 | ranges.add(new Range(1, 2)); 35 | ranges.add(new Range(10, 20)); 36 | IOPlan ioPlan = new IOPlan(ranges); 37 | assertArrayEquals(ioPlan.getPrefetchRanges().toArray(), ranges.toArray()); 38 | } 39 | 40 | @Test 41 | void testRangeConstructor() { 42 | IOPlan ioPlan = new IOPlan(new Range(1, 2)); 43 | assertArrayEquals(ioPlan.getPrefetchRanges().toArray(), new Range[] {new Range(1, 2)}); 44 | } 45 | 46 | @Test 47 | void testConstructorThrowOnNulls() { 48 | assertThrows(NullPointerException.class, () -> new IOPlan((Collection) null)); 49 | assertThrows(NullPointerException.class, () -> new IOPlan((Range) null)); 50 | } 51 | 52 | @Test 53 | void testRangeListToString() { 54 | ArrayList ranges = new ArrayList<>(); 55 | ranges.add(new Range(1, 2)); 56 | ranges.add(new Range(10, 20)); 57 | IOPlan ioPlan = new IOPlan(ranges); 58 | assertEquals("[1-2,10-20]", ioPlan.toString()); 59 | } 60 | 61 | @Test 62 | void testRangeToString() { 63 | IOPlan ioPlan = new IOPlan(new Range(1, 2)); 64 | assertEquals("[1-2]", ioPlan.toString()); 65 | } 66 | 67 | @Test 68 | void testEmptyPlanToString() { 69 | assertEquals("[]", IOPlan.EMPTY_PLAN.toString()); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/util/FakeStuckObjectClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import java.util.concurrent.CompletableFuture; 19 | import java.util.concurrent.TimeoutException; 20 | import software.amazon.s3.analyticsaccelerator.request.GetRequest; 21 | import software.amazon.s3.analyticsaccelerator.request.ObjectContent; 22 | 23 | public class FakeStuckObjectClient extends FakeObjectClient { 24 | 25 | /** 26 | * Instantiate a fake Object Client backed by some string as data. 27 | * 28 | * @param data the data making up the object 29 | */ 30 | public FakeStuckObjectClient(String data) { 31 | super(data); 32 | } 33 | 34 | @Override 35 | public CompletableFuture getObject( 36 | GetRequest getRequest, OpenStreamInformation openStreamInformation) { 37 | CompletableFuture failedFuture = new CompletableFuture<>(); 38 | failedFuture.completeExceptionally(new TimeoutException("Request timed out")); 39 | return failedFuture; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /input-stream/src/test/java/software/amazon/s3/analyticsaccelerator/util/PrefetchModeTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.util; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertEquals; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | public class PrefetchModeTest { 23 | 24 | @Test 25 | public void testPrefetchModeFromString() { 26 | assertEquals(PrefetchMode.ALL, PrefetchMode.fromString("aLL")); 27 | assertEquals(PrefetchMode.OFF, PrefetchMode.fromString("OFF")); 28 | assertEquals(PrefetchMode.COLUMN_BOUND, PrefetchMode.fromString("column_bound")); 29 | assertEquals(PrefetchMode.ROW_GROUP, PrefetchMode.fromString("row_group")); 30 | 31 | // defaults to ROW_GROUP mode 32 | assertEquals(PrefetchMode.ROW_GROUP, PrefetchMode.fromString("xyz")); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /input-stream/src/test/resources/call_center.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/call_center.parquet -------------------------------------------------------------------------------- /input-stream/src/test/resources/call_center_file_metadata.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/call_center_file_metadata.ser -------------------------------------------------------------------------------- /input-stream/src/test/resources/multi_row_group.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/multi_row_group.parquet -------------------------------------------------------------------------------- /input-stream/src/test/resources/multi_row_group.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/multi_row_group.ser -------------------------------------------------------------------------------- /input-stream/src/test/resources/nested_data.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/nested_data.parquet -------------------------------------------------------------------------------- /input-stream/src/test/resources/nested_data_metadata.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/nested_data_metadata.ser -------------------------------------------------------------------------------- /input-stream/src/test/resources/nested_data_mrg.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/nested_data_mrg.parquet -------------------------------------------------------------------------------- /input-stream/src/test/resources/nested_data_mrg_metadata.ser: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/analytics-accelerator-s3/a4ca8c82d1f8be40b4fd35e96b9b1291b8fe092a/input-stream/src/test/resources/nested_data_mrg_metadata.ser -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/FaultyS3AsyncClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import java.util.Collections; 19 | import java.util.HashSet; 20 | import java.util.Set; 21 | import java.util.concurrent.CompletableFuture; 22 | import software.amazon.awssdk.core.async.AsyncResponseTransformer; 23 | import software.amazon.awssdk.services.s3.DelegatingS3AsyncClient; 24 | import software.amazon.awssdk.services.s3.S3AsyncClient; 25 | import software.amazon.awssdk.services.s3.model.GetObjectRequest; 26 | import software.amazon.awssdk.services.s3.model.GetObjectResponse; 27 | 28 | /** 29 | * A faulty S3 Client implementation that injects failure to S3 calls. Current version only adds 30 | * infinite waiting to Get calls. We should extend this in the future to except a FaultKind or a 31 | * FaultConfiguration. 32 | */ 33 | public class FaultyS3AsyncClient extends DelegatingS3AsyncClient { 34 | 35 | Set stuckObjects; 36 | 37 | /** 38 | * Create an instance of a S3 client, for interaction with Amazon S3 compatible object stores. 39 | * This takes ownership of the passed client and will close it on its own close(). 40 | * 41 | * @param delegate Underlying client to be used for making requests to S3. 42 | */ 43 | public FaultyS3AsyncClient(S3AsyncClient delegate) { 44 | super(delegate); 45 | this.stuckObjects = Collections.synchronizedSet(new HashSet<>()); 46 | } 47 | 48 | /** 49 | * A faulty override of getObject. First GET call to an object will wait forever. All requests to 50 | * the same object will be delegated to regular S3AsyncClient. 51 | * 52 | * @param request request 53 | * @param asyncResponseTransformer response transformer 54 | * @return an instance of {@link S3AsyncClient} 55 | */ 56 | @Override 57 | public CompletableFuture getObject( 58 | GetObjectRequest request, 59 | AsyncResponseTransformer asyncResponseTransformer) { 60 | if (!stuckObjects.contains(request.key())) { 61 | stuckObjects.add(request.key()); 62 | return CompletableFuture.supplyAsync( 63 | () -> { 64 | try { 65 | while (true) { 66 | Thread.sleep(10000); 67 | } 68 | } catch (InterruptedException e) { 69 | e.printStackTrace(); 70 | } 71 | return null; 72 | }); 73 | } else { 74 | return super.getObject(request, asyncResponseTransformer); 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/S3AsyncClientFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import lombok.NonNull; 19 | import software.amazon.awssdk.services.s3.S3AsyncClient; 20 | 21 | /** Small factory that creates the Async client */ 22 | public class S3AsyncClientFactory { 23 | /** 24 | * Builds a regular async Java client 25 | * 26 | * @param configuration configuration 27 | * @return an instance of {@link S3AsyncClient} 28 | */ 29 | public static S3AsyncClient createS3AsyncClient( 30 | @NonNull S3AsyncClientFactoryConfiguration configuration) { 31 | return S3AsyncClient.builder().region(configuration.getRegion()).build(); 32 | } 33 | 34 | /** 35 | * Builds a regular async Java client 36 | * 37 | * @param configuration configuration 38 | * @return an instance of {@link S3AsyncClient} 39 | */ 40 | public static S3AsyncClient createS3CrtAsyncClient( 41 | @NonNull S3AsyncClientFactoryConfiguration configuration) { 42 | return S3AsyncClient.crtBuilder() 43 | .region(configuration.getRegion()) 44 | .minimumPartSizeInBytes(configuration.getCrtPartSizeInBytes()) 45 | .maxNativeMemoryLimitInBytes(configuration.getCrtNativeMemoryLimitInBytes()) 46 | .maxConcurrency(configuration.getCrtMaxConcurrency()) 47 | .checksumValidationEnabled(configuration.isCrtChecksumValidationEnabled()) 48 | .targetThroughputInGbps((double) configuration.getCrtTargetThroughputGbps()) 49 | .build(); 50 | } 51 | 52 | /** 53 | * Builds a faulty async Java client 54 | * 55 | * @param configuration configuration 56 | * @return an instance of {@link S3AsyncClient} 57 | */ 58 | public static S3AsyncClient createFaultyS3Client( 59 | @NonNull S3AsyncClientFactoryConfiguration configuration) { 60 | S3AsyncClient delegate = createS3AsyncClient(configuration); 61 | return new FaultyS3AsyncClient(delegate); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/S3ClientKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import java.util.Arrays; 19 | import java.util.List; 20 | import lombok.AllArgsConstructor; 21 | import lombok.Getter; 22 | import lombok.NonNull; 23 | import software.amazon.awssdk.services.s3.S3AsyncClient; 24 | 25 | /** Kind of S3 Client used */ 26 | @AllArgsConstructor 27 | @Getter 28 | public enum S3ClientKind { 29 | SDK_V2_JAVA_ASYNC("ASYNC_JAVA"), 30 | SDK_V2_CRT_ASYNC("ASYNC_CRT"), 31 | FAULTY_S3_CLIENT("FAULTY_S3"); 32 | 33 | private final String value; 34 | /** 35 | * Creates the S3 client based on the context and client kind. This is used by benchmarks, and 36 | * allows us easily run the same benchmarks against different clients and configurations 37 | * 38 | * @param s3ExecutionContext benchmark context 39 | * @return an instance of {@link S3AsyncClient} 40 | */ 41 | public S3AsyncClient getS3Client(@NonNull S3ExecutionContext s3ExecutionContext) { 42 | switch (this) { 43 | case SDK_V2_JAVA_ASYNC: 44 | return s3ExecutionContext.getS3Client(); 45 | case SDK_V2_CRT_ASYNC: 46 | return s3ExecutionContext.getS3CrtClient(); 47 | case FAULTY_S3_CLIENT: 48 | return s3ExecutionContext.getFaultyS3Client(); 49 | default: 50 | throw new IllegalArgumentException("Unsupported client kind: " + this); 51 | } 52 | } 53 | 54 | /** 55 | * Trusted S3 Clients 56 | * 57 | * @return small objects 58 | */ 59 | public static List trustedClients() { 60 | return Arrays.asList(SDK_V2_JAVA_ASYNC, SDK_V2_CRT_ASYNC); 61 | } 62 | 63 | /** 64 | * Faulty S3 Clients 65 | * 66 | * @return small objects 67 | */ 68 | public static List faultyClients() { 69 | return Arrays.asList(FAULTY_S3_CLIENT); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/S3ExecutionConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import static software.amazon.s3.analyticsaccelerator.access.SizeConstants.ONE_MB_IN_BYTES; 19 | 20 | import lombok.Builder; 21 | import lombok.NonNull; 22 | import lombok.Value; 23 | import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration; 24 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 25 | 26 | /** Configuration for benchmarks */ 27 | @Value 28 | @Builder 29 | public class S3ExecutionConfiguration { 30 | public static final String BUCKET_KEY = "S3_TEST_BUCKET"; 31 | public static final String PREFIX_KEY = "S3_TEST_PREFIX"; 32 | public static final String READ_BUFFER_SIZE_MB_KEY = "S3_TEST_READ_BUFFER_SIZE_MB"; 33 | public static final int DEFAULT_READ_BUFFER_SIZE_MB_KEY = 8; 34 | 35 | @NonNull String bucket; 36 | @NonNull String prefix; 37 | int bufferSizeMb; 38 | @NonNull S3AsyncClientFactoryConfiguration clientFactoryConfiguration; 39 | 40 | /** 41 | * Creates the {@link S3ExecutionConfiguration} from the supplied configuration 42 | * 43 | * @param configuration an instance of configuration 44 | * @return anew instance of {@link S3AsyncClientFactoryConfiguration} 45 | */ 46 | public static S3ExecutionConfiguration fromConfiguration(ConnectorConfiguration configuration) { 47 | return S3ExecutionConfiguration.builder() 48 | .bucket(configuration.getRequiredString(BUCKET_KEY)) 49 | .prefix(configuration.getRequiredString(PREFIX_KEY)) 50 | .bufferSizeMb( 51 | configuration.getInt(READ_BUFFER_SIZE_MB_KEY, DEFAULT_READ_BUFFER_SIZE_MB_KEY)) 52 | .clientFactoryConfiguration( 53 | S3AsyncClientFactoryConfiguration.fromConfiguration(configuration)) 54 | .build(); 55 | } 56 | 57 | /** 58 | * Returns the base Uri 59 | * 60 | * @return base Uri 61 | */ 62 | public S3URI getBaseUri() { 63 | return S3URI.of(this.getBucket(), this.getPrefix()); 64 | } 65 | 66 | /** 67 | * Returns the buffer size in bytes 68 | * 69 | * @return buffer size in bytes 70 | */ 71 | public int getBufferSizeBytes() { 72 | return ONE_MB_IN_BYTES * this.getBufferSizeMb(); 73 | } 74 | 75 | /** 76 | * Creates the {@link S3ExecutionConfiguration} from the environment 77 | * 78 | * @return anew instance of {@link S3ExecutionConfiguration} 79 | */ 80 | public static S3ExecutionConfiguration fromEnvironment() { 81 | return fromConfiguration(new ConnectorConfiguration(System.getenv())); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/S3InputStreamKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** Kind of the stream we read from S3 */ 22 | @AllArgsConstructor 23 | @Getter 24 | public enum S3InputStreamKind { 25 | // SDK backed 26 | S3_SDK_GET("SDK"), 27 | // Ours 28 | S3_DAT_GET("DAT"); 29 | private final String value; 30 | } 31 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/S3ObjectKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** S3 Object kind */ 22 | @AllArgsConstructor 23 | @Getter 24 | public enum S3ObjectKind { 25 | RANDOM_SEQUENTIAL("sequential"), 26 | RANDOM_PARQUET("parquet"); 27 | 28 | private final String value; 29 | } 30 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/SizeConstants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | /** Sizing constants * */ 19 | class SizeConstants { 20 | public static final int ONE_KB_IN_BYTES = 1024; 21 | public static final int ONE_MB_IN_BYTES = 1024 * ONE_KB_IN_BYTES; 22 | public static final long ONE_GB_IN_BYTES = 1024 * ONE_MB_IN_BYTES; 23 | } 24 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/StreamRead.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import lombok.Builder; 19 | import lombok.Value; 20 | 21 | /** Object representing a read. Has a start and a length. */ 22 | @Value 23 | @Builder 24 | public class StreamRead { 25 | long start; 26 | long length; 27 | } 28 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/StreamReadPattern.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import java.util.List; 19 | import lombok.Builder; 20 | import lombok.NonNull; 21 | import lombok.Singular; 22 | import lombok.Value; 23 | 24 | /** Represents a read pattern - a sequence of reads */ 25 | @Value 26 | @Builder 27 | public class StreamReadPattern { 28 | @Singular @NonNull List streamReads; 29 | } 30 | -------------------------------------------------------------------------------- /input-stream/src/testFixtures/java/software/amazon/s3/analyticsaccelerator/access/StreamReadPatternKind.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.access; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | 21 | /** Enum that describes different supported stream seeks */ 22 | @AllArgsConstructor 23 | @Getter 24 | public enum StreamReadPatternKind { 25 | SEQUENTIAL("SEQUENTIAL"), 26 | SKIPPING_FORWARD("SKIPPING_FORWARD"), 27 | SKIPPING_BACKWARD("SKIPPING_BACKWARD"), 28 | QUASI_PARQUET_ROW_GROUP("QUASI_PARQUET_ROW_GROUP"), 29 | QUASI_PARQUET_COLUMN_CHUNK("QUASI_PARQUET_COLUMN_CHUNK"); 30 | private final String value; 31 | 32 | /** 33 | * Gets the stream read pattern based on the value of this enum 34 | * 35 | * @param s3Object {@link S3Object} to read 36 | * @return the stream read pattern 37 | */ 38 | public StreamReadPattern getStreamReadPattern(S3Object s3Object) { 39 | switch (this) { 40 | case SEQUENTIAL: 41 | return StreamReadPatternFactory.getSequentialReadPattern(s3Object); 42 | case SKIPPING_FORWARD: 43 | return StreamReadPatternFactory.getForwardSeekReadPattern(s3Object); 44 | case SKIPPING_BACKWARD: 45 | return StreamReadPatternFactory.getBackwardSeekReadPattern(s3Object); 46 | case QUASI_PARQUET_ROW_GROUP: 47 | return StreamReadPatternFactory.getQuasiParquetRowGroupPattern(s3Object); 48 | case QUASI_PARQUET_COLUMN_CHUNK: 49 | return StreamReadPatternFactory.getQuasiParquetColumnChunkPattern(s3Object); 50 | default: 51 | throw new IllegalArgumentException("Unknown stream read pattern: " + this); 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /lombok.config: -------------------------------------------------------------------------------- 1 | lombok.addLombokGeneratedAnnotation = true -------------------------------------------------------------------------------- /object-client/build.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | */ 4 | 5 | plugins { 6 | id("buildlogic.java-library-conventions") 7 | id("io.freefair.lombok") version "8.10.2" 8 | `maven-publish` 9 | } 10 | 11 | dependencies { 12 | implementation(project(":common")) 13 | implementation(platform(libs.sdk.bom)) 14 | implementation(libs.s3); 15 | implementation(libs.crt); 16 | 17 | testImplementation(libs.junit.jupiter) 18 | testImplementation(libs.mockito.core) 19 | testImplementation(libs.mockito.junit.jupiter) 20 | testRuntimeOnly(libs.junit.jupiter.launcher) 21 | } 22 | 23 | tasks.test { 24 | environment("AWS_REGION", "eu-west-1") 25 | } -------------------------------------------------------------------------------- /object-client/spotbugs-exclude.xml: -------------------------------------------------------------------------------- 1 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /object-client/src/main/java/software/amazon/s3/analyticsaccelerator/ObjectClientConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import lombok.Builder; 19 | import lombok.EqualsAndHashCode; 20 | import lombok.Getter; 21 | import lombok.NonNull; 22 | import software.amazon.s3.analyticsaccelerator.common.ConnectorConfiguration; 23 | import software.amazon.s3.analyticsaccelerator.common.telemetry.TelemetryConfiguration; 24 | import software.amazon.s3.analyticsaccelerator.request.ObjectClient; 25 | 26 | /** Configuration for {@link ObjectClient} */ 27 | @Getter 28 | @Builder 29 | @EqualsAndHashCode 30 | public class ObjectClientConfiguration { 31 | public static final String DEFAULT_USER_AGENT_PREFIX = null; 32 | public static final String USER_AGENT_PREFIX_KEY = "useragentprefix"; 33 | public static final String TELEMETRY_PREFIX = "telemetry"; 34 | 35 | /** User Agent Prefix. {@link ObjectClientConfiguration#DEFAULT_USER_AGENT_PREFIX} by default. */ 36 | @Builder.Default private String userAgentPrefix = DEFAULT_USER_AGENT_PREFIX; 37 | 38 | /** Telemetry configuration */ 39 | @Builder.Default @NonNull private TelemetryConfiguration telemetryConfiguration = TelemetryConfiguration.DEFAULT; 40 | 41 | public static final ObjectClientConfiguration DEFAULT = 42 | ObjectClientConfiguration.builder().build(); 43 | 44 | /** 45 | * Constructs {@link ObjectClientConfiguration} from {@link ConnectorConfiguration} object. 46 | * 47 | * @param configuration Configuration object to generate ObjectClientConfiguration from 48 | * @return ObjectClientConfiguration 49 | */ 50 | public static ObjectClientConfiguration fromConfiguration(ConnectorConfiguration configuration) { 51 | return ObjectClientConfiguration.builder() 52 | .userAgentPrefix(configuration.getString(USER_AGENT_PREFIX_KEY, DEFAULT_USER_AGENT_PREFIX)) 53 | .telemetryConfiguration( 54 | TelemetryConfiguration.fromConfiguration(configuration.map(TELEMETRY_PREFIX))) 55 | .build(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /object-client/src/main/java/software/amazon/s3/analyticsaccelerator/ObjectClientTelemetry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import lombok.AllArgsConstructor; 19 | import lombok.Getter; 20 | import software.amazon.s3.analyticsaccelerator.common.telemetry.Attribute; 21 | import software.amazon.s3.analyticsaccelerator.request.Range; 22 | import software.amazon.s3.analyticsaccelerator.util.S3URI; 23 | 24 | /** Helper class to streamline Telemetry calls. */ 25 | @Getter 26 | @AllArgsConstructor 27 | enum ObjectClientTelemetry { 28 | URI("uri"), 29 | RANGE("range"), 30 | RANGE_LENGTH("range.length"); 31 | private final String name; 32 | 33 | public static final String OPERATION_GET = "s3.client.get"; 34 | public static final String OPERATION_HEAD = "s3.client.head"; 35 | 36 | /** 37 | * Creates an {@link Attribute} for a {@link S3URI}. 38 | * 39 | * @param s3URI the {@link S3URI} to create the attribute from. 40 | * @return The new instance of the {@link Attribute}. 41 | */ 42 | public static Attribute uri(S3URI s3URI) { 43 | return Attribute.of(ObjectClientTelemetry.URI.getName(), s3URI.toString()); 44 | } 45 | 46 | /** 47 | * Creates an {@link Attribute} for the length of a range. 48 | * 49 | * @param range the range 50 | * @return The new instance of the {@link Attribute}. 51 | */ 52 | public static Attribute range(Range range) { 53 | return Attribute.of(ObjectClientTelemetry.RANGE.getName(), range.toString()); 54 | } 55 | 56 | /** 57 | * Creates an {@link Attribute} for the length of a range. 58 | * 59 | * @param range the length of the range to measure 60 | * @return The new instance of the {@link Attribute}. 61 | */ 62 | public static Attribute rangeLength(Range range) { 63 | return Attribute.of( 64 | ObjectClientTelemetry.RANGE_LENGTH.getName(), Long.toString(range.getLength())); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /object-client/src/main/java/software/amazon/s3/analyticsaccelerator/request/Constants.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import software.amazon.awssdk.core.interceptor.ExecutionAttribute; 19 | 20 | /** Class for request related constants. */ 21 | public final class Constants { 22 | private Constants() {} 23 | 24 | public static final String HEADER_USER_AGENT = "User-Agent"; 25 | public static final String HEADER_REFERER = "Referer"; 26 | 27 | // These execution attributes are specific to the hadoop S3A integration, and are required for 28 | // S3A's auditing feature. These execution attributes can be set per request, which are then 29 | // picked up 30 | // by execution interceptors in S3A. See S3A's LoggingAuditor for usage. 31 | public static final ExecutionAttribute SPAN_ID = new ExecutionAttribute<>("span"); 32 | public static final ExecutionAttribute OPERATION_NAME = 33 | new ExecutionAttribute<>("operation"); 34 | } 35 | -------------------------------------------------------------------------------- /object-client/src/main/java/software/amazon/s3/analyticsaccelerator/request/UserAgent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import java.util.Objects; 19 | import lombok.Getter; 20 | 21 | /** User-Agent to be used by ObjectClients */ 22 | @Getter 23 | public final class UserAgent { 24 | // Hard-coded user-agent string 25 | private static final String UA_STRING = "s3analyticsaccelerator"; 26 | 27 | private static final String VERSION_INFO = "1.1.0"; 28 | /** 29 | * Disallowed characters in the user agent token: @see RFC 7230 31 | */ 32 | private static final String UA_DENYLIST_REGEX = "[() ,/:;<=>?@\\[\\]{}\\\\]"; 33 | 34 | private String userAgent = UA_STRING + "/" + VERSION_INFO; 35 | 36 | /** 37 | * Prepend hard-coded user-agent string with input string provided. 38 | * 39 | * @param userAgentPrefix to prepend the default user-agent string 40 | */ 41 | public void prepend(String userAgentPrefix) { 42 | if (Objects.nonNull(userAgentPrefix)) 43 | this.userAgent = sanitizeInput(userAgentPrefix) + " " + this.userAgent; 44 | } 45 | 46 | private static String sanitizeInput(String input) { 47 | return input == null ? "unknown" : input.replaceAll(UA_DENYLIST_REGEX, "_"); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /object-client/src/test/java/software/amazon/s3/analyticsaccelerator/SpotBugsLambdaWorkaround.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator; 17 | 18 | import static org.junit.jupiter.api.Assertions.assertInstanceOf; 19 | import static org.junit.jupiter.api.Assertions.fail; 20 | 21 | import java.io.Closeable; 22 | import lombok.SneakyThrows; 23 | import org.junit.jupiter.api.function.ThrowingSupplier; 24 | 25 | /** 26 | * This class is here to help work around certain SpotBugs quirks. Specifically, SpotBugs don't 27 | * allow to apply suppressions to Lambdas 28 | */ 29 | public final class SpotBugsLambdaWorkaround { 30 | /** 31 | * In situations where a method or constructor return a class that implements Closeable, Spotbugs 32 | * wants to see it closed. When testing bounds and expecting exceptions, this is unnecessary, and 33 | * trying to work around this in tests results in a lot of boilerplate. This method absorbs all 34 | * the boilerplate and acts as `assertThrows`. 35 | * 36 | * @param expectedType exception type expected 37 | * @param executable code that returns something Closeable, and expected to throw 38 | * @param exception type 39 | * @param return type 40 | */ 41 | @SneakyThrows 42 | @SuppressWarnings("try") 43 | public static void assertThrowsClosableResult( 44 | Class expectedType, ThrowingSupplier executable) { 45 | try (C result = executable.get()) { 46 | } catch (Throwable throwable) { 47 | assertInstanceOf(expectedType, throwable); 48 | return; 49 | } 50 | fail(String.format("Exception of type '%s' was expected. Nothing was thrown", expectedType)); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /object-client/src/test/java/software/amazon/s3/analyticsaccelerator/request/UserAgentTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"). 5 | * You may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package software.amazon.s3.analyticsaccelerator.request; 17 | 18 | import static org.junit.jupiter.api.Assertions.*; 19 | 20 | import org.junit.jupiter.api.Test; 21 | 22 | public class UserAgentTest { 23 | 24 | @Test 25 | void testDefaultUserAgent() { 26 | UserAgent agent = new UserAgent(); 27 | assertNotNull(agent.getUserAgent()); 28 | } 29 | 30 | @Test 31 | void testPrependUserAgent() { 32 | UserAgent agent = new UserAgent(); 33 | agent.prepend("unit_test"); 34 | assertTrue(agent.getUserAgent().startsWith("unit_test")); 35 | } 36 | 37 | @Test 38 | void testNullPrependIsNoop() { 39 | UserAgent agent = new UserAgent(); 40 | String pre = agent.getUserAgent(); 41 | agent.prepend(null); 42 | String pos = agent.getUserAgent(); 43 | assertEquals(pre, pos); 44 | } 45 | 46 | @Test 47 | void testInvalidInputIsSanitised() { 48 | UserAgent agent = new UserAgent(); 49 | agent.prepend("unit/test"); 50 | assertTrue(agent.getUserAgent().startsWith("unit_test")); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /settings.gradle.kts: -------------------------------------------------------------------------------- 1 | /* 2 | * This file was generated by the Gradle 'init' task. 3 | * 4 | * The settings file is used to specify which projects to include in your build. 5 | * For more detailed information on multi-project builds, please refer to https://docs.gradle.org/8.7/userguide/multi_project_builds.html in the Gradle documentation. 6 | */ 7 | 8 | plugins { 9 | // Apply the foojay-resolver plugin to allow automatic download of JDKs 10 | id("org.gradle.toolchains.foojay-resolver-convention") version "0.8.0" 11 | } 12 | 13 | rootProject.name = "analytics-accelerator-s3" 14 | include("common", "object-client", "input-stream") 15 | --------------------------------------------------------------------------------