├── src
    ├── test
    │   ├── resources
    │   │   ├── test.txt
    │   │   └── test-exact-buffer.txt
    │   └── java
    │   │   └── edu
    │   │       └── colorado
    │   │           └── cires
    │   │               └── cmg
    │   │                   └── s3out
    │   │                       ├── DefaultContentTypeResolverTest.java
    │   │                       ├── AwsS3ClientMultipartUploadTest.java
    │   │                       ├── S3OutputStreamTest.java
    │   │                       └── ObjectMetadataTest.java
    └── main
    │   └── java
    │       └── edu
    │           └── colorado
    │               └── cires
    │                   └── cmg
    │                       └── s3out
    │                           ├── NoContentTypeResolver.java
    │                           ├── ObjectMetadataCustomizer.java
    │                           ├── ContentTypeResolver.java
    │                           ├── S3ClientMultipartUpload.java
    │                           ├── MultipartUploadRequest.java
    │                           ├── UploadPartParams.java
    │                           ├── DefaultContentTypeResolver.java
    │                           ├── AwsS3ClientMultipartUpload.java
    │                           ├── FileMockS3ClientMultipartUpload.java
    │                           ├── S3OutputStream.java
    │                           └── ObjectMetadata.java
├── settings.xml
├── spotbugs-exclude.xml
├── site-resources
    └── index.html
├── LICENSE
├── .gitignore
├── .github
    └── workflows
    │   ├── branch-release.yml
    │   ├── tag-release.yml
    │   └── ci.yaml
├── owasp-dep-check-suppression.xml
├── README.md
└── pom.xml


/src/test/resources/test.txt:
--------------------------------------------------------------------------------
1 | 12345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890


--------------------------------------------------------------------------------
/src/test/resources/test-exact-buffer.txt:
--------------------------------------------------------------------------------
1 | 1234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/NoContentTypeResolver.java:
--------------------------------------------------------------------------------
 1 | package edu.colorado.cires.cmg.s3out;
 2 | 
 3 | import java.util.Optional;
 4 | 
 5 | /**
 6 |  * A {@link ContentTypeResolver} that never resolves the MIME type.
 7 |  */
 8 | public class NoContentTypeResolver implements ContentTypeResolver{
 9 | 
10 |   @Override
11 |   public Optional<String> resolveContentType(String key) {
12 |     return Optional.empty();
13 |   }
14 | }
15 | 


--------------------------------------------------------------------------------
/settings.xml:
--------------------------------------------------------------------------------
 1 | <settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 2 |   xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
 3 |   <servers>
 4 |     <server>
 5 |       <id>central</id>
 6 |       <username>${env.MAVEN_USERNAME}</username>
 7 |       <password>${env.MAVEN_PASSWORD}</password>
 8 |     </server>
 9 |   </servers>
10 | </settings>


--------------------------------------------------------------------------------
/spotbugs-exclude.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <FindBugsFilter
 3 |   xmlns="https://github.com/spotbugs/filter/3.0.0"
 4 |   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 5 |   xsi:schemaLocation="https://github.com/spotbugs/filter/3.0.0 https://raw.githubusercontent.com/spotbugs/spotbugs/3.1.0/spotbugs/etc/findbugsfilter.xsd">
 6 |   <Match>
 7 |     <Bug pattern="EI_EXPOSE_REP" />
 8 |   </Match>
 9 |   <Match>
10 |     <Bug pattern="EI_EXPOSE_REP2" />
11 |   </Match>
12 | </FindBugsFilter>


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/ObjectMetadataCustomizer.java:
--------------------------------------------------------------------------------
 1 | package edu.colorado.cires.cmg.s3out;
 2 | 
 3 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest.Builder;
 4 | 
 5 | /**
 6 |  * Allows for customization of the S3 object metadata when uploading
 7 |  */
 8 | public interface ObjectMetadataCustomizer {
 9 | 
10 |   /**
11 |    * Applies any customizations to the upload request.
12 |    * @param builder the {@link Builder}
13 |    */
14 |   void apply(Builder builder);
15 | }
16 | 


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/ContentTypeResolver.java:
--------------------------------------------------------------------------------
 1 | package edu.colorado.cires.cmg.s3out;
 2 | 
 3 | import java.util.Optional;
 4 | 
 5 | /**
 6 |  * Resolves the MIME type for a file to be uploaded.
 7 |  */
 8 | public interface ContentTypeResolver {
 9 | 
10 |   /**
11 |    * Resolves the MIME type for a file to be uploaded.
12 |    *
13 |    * @param key the S3 key for the uploaded file
14 |    * @return a populated {@link Optional} with the MIME type or empty if not defined
15 |    */
16 |   Optional<String> resolveContentType(String key);
17 | 
18 | }
19 | 


--------------------------------------------------------------------------------
/site-resources/index.html:
--------------------------------------------------------------------------------
 1 | <html>
 2 |   <body>
 3 |     <script>
 4 |       (async () => {
 5 |         const response = await fetch('https://api.github.com/repos/ci-cmg/project-documentation/contents/@git.remote.origin.url.suffix@');
 6 |         const data = await response.json();
 7 |         let htmlString = '<ul>';
 8 |         for (let file of data) {
 9 |           htmlString += `<li><a href="${file.name}">${file.name}</a></li>`;
10 |         }
11 |         htmlString += '</ul>';
12 |         document.getElementsByTagName('body')[0].innerHTML = htmlString;
13 |       })()
14 |     </script>
15 |   <body>
16 | </html>
17 | 


--------------------------------------------------------------------------------
/src/test/java/edu/colorado/cires/cmg/s3out/DefaultContentTypeResolverTest.java:
--------------------------------------------------------------------------------
 1 | package edu.colorado.cires.cmg.s3out;
 2 | 
 3 | import static org.junit.jupiter.api.Assertions.*;
 4 | 
 5 | import org.junit.jupiter.api.Test;
 6 | 
 7 | public class DefaultContentTypeResolverTest {
 8 | 
 9 |   @Test
10 |   public void testMapped() throws Exception {
11 |     DefaultContentTypeResolver resolver = new DefaultContentTypeResolver();
12 |     assertEquals("image/png", resolver.resolveContentType("foo/bar/cats.png").get());
13 |   }
14 | 
15 |   @Test
16 |   public void testNotFound() throws Exception {
17 |     DefaultContentTypeResolver resolver = new DefaultContentTypeResolver();
18 |     assertFalse(resolver.resolveContentType("foo/bar/cats.dog").isPresent());
19 |   }
20 | 
21 |   @Test
22 |   public void testNoExt() throws Exception {
23 |     DefaultContentTypeResolver resolver = new DefaultContentTypeResolver();
24 |     assertFalse(resolver.resolveContentType("foo/bar/cats").isPresent());
25 |   }
26 | }


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2022 CIRES, University of Colorado
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
 1 | ### Java template
 2 | # Compiled class file
 3 | *.class
 4 | 
 5 | # Log file
 6 | *.log
 7 | 
 8 | # BlueJ files
 9 | *.ctxt
10 | 
11 | # Mobile Tools for Java (J2ME)
12 | .mtj.tmp/
13 | 
14 | # Package Files #
15 | *.jar
16 | *.war
17 | *.nar
18 | *.ear
19 | *.zip
20 | *.tar.gz
21 | *.rar
22 | 
23 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml
24 | hs_err_pid*
25 | 
26 | ### Maven template
27 | target/
28 | pom.xml.tag
29 | pom.xml.releaseBackup
30 | pom.xml.versionsBackup
31 | pom.xml.next
32 | release.properties
33 | dependency-reduced-pom.xml
34 | buildNumber.properties
35 | .mvn/timing.properties
36 | # https://github.com/takari/maven-wrapper#usage-without-binary-jar
37 | .mvn/wrapper/maven-wrapper.jar
38 | 
39 | ### macOS template
40 | # General
41 | .DS_Store
42 | .AppleDouble
43 | .LSOverride
44 | 
45 | # Icon must end with two \r
46 | Icon
47 | 
48 | # Thumbnails
49 | ._*
50 | 
51 | # Files that might appear in the root of a volume
52 | .DocumentRevisions-V100
53 | .fseventsd
54 | .Spotlight-V100
55 | .TemporaryItems
56 | .Trashes
57 | .VolumeIcon.icns
58 | .com.apple.timemachine.donotpresent
59 | 
60 | # Directories potentially created on remote AFP share
61 | .AppleDB
62 | .AppleDesktop
63 | Network Trash Folder
64 | Temporary Items
65 | .apdisk
66 | 
67 | .idea
68 | *.iml
69 | 
70 | 


--------------------------------------------------------------------------------
/.github/workflows/branch-release.yml:
--------------------------------------------------------------------------------
 1 | name: maven branch release
 2 | 
 3 | on:
 4 |   workflow_dispatch:
 5 | 
 6 | jobs:
 7 |   build:
 8 |     runs-on: ubuntu-latest
 9 |     steps:
10 |       - name: checkout
11 |         uses: actions/checkout@v4
12 |       - name: Set up JDK 11
13 |         uses: actions/setup-java@v4
14 |         with:
15 |           java-version: '11'
16 |           distribution: 'temurin'
17 |           cache: maven
18 |       - name: set up git
19 |         run: git config --global user.email bloop@bloop.org && git config --global user.name 'Bloopy McBloopFace'
20 |       - name: build with maven
21 |         run: |
22 |           mvn -B -s settings.xml \
23 |           build-helper:parse-version \
24 |           -Dgit.password=${{ secrets.RELEASE_PAT }} \
25 |           -Dgit.username=${{ secrets.RELEASE_USERNAME }} \
26 |           -DbranchName='${parsedVersion.majorVersion}.${parsedVersion.minorVersion}' \
27 |           -DdevelopmentVersion='${parsedVersion.majorVersion}.${parsedVersion.nextMinorVersion}.0-SNAPSHOT' \
28 |           release:branch
29 |         env:
30 |           NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
31 |           SIGN_KEY_PASS: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
32 |           SIGN_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
33 |           MAVEN_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
34 |           MAVEN_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}


--------------------------------------------------------------------------------
/.github/workflows/tag-release.yml:
--------------------------------------------------------------------------------
 1 | name: maven tag release
 2 | 
 3 | on:
 4 |   workflow_dispatch:
 5 | 
 6 | jobs:
 7 |   build:
 8 |     runs-on: ubuntu-latest
 9 |     steps:
10 |       - name: checkout
11 |         uses: actions/checkout@v4
12 |       - name: Set up JDK 11
13 |         uses: actions/setup-java@v4
14 |         with:
15 |           java-version: '11'
16 |           distribution: 'temurin'
17 |           cache: maven
18 |       - name: set up git
19 |         run: git config --global user.email bloop@bloop.org && git config --global user.name 'Bloopy McBloopFace'
20 |       - name: build with maven
21 |         run: |
22 |           mvn -B -s settings.xml \
23 |           build-helper:parse-version \
24 |           -Dgit.password=${{ secrets.RELEASE_PAT }} \
25 |           -Dgit.username=${{ secrets.RELEASE_USERNAME }} \
26 |           -Dresume=false -Dtag='v${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}' \
27 |           -DreleaseVersion='${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.incrementalVersion}' \
28 |           -DdevelopmentVersion='${parsedVersion.majorVersion}.${parsedVersion.minorVersion}.${parsedVersion.nextIncrementalVersion}-SNAPSHOT' \
29 |           release:prepare release:perform
30 |         env:
31 |           NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
32 |           SIGN_KEY_PASS: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
33 |           SIGN_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
34 |           MAVEN_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
35 |           MAVEN_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}


--------------------------------------------------------------------------------
/.github/workflows/ci.yaml:
--------------------------------------------------------------------------------
 1 | name: maven build
 2 | 
 3 | on:
 4 |   push:
 5 |     branches: [ "**" ]
 6 |   workflow_dispatch:
 7 | 
 8 | jobs:
 9 |   build:
10 |     runs-on: ubuntu-latest
11 |     steps:
12 |       - name: checkout
13 |         uses: actions/checkout@v4
14 |       - name: Set up JDK 11
15 |         uses: actions/setup-java@v4
16 |         with:
17 |           java-version: '11'
18 |           distribution: 'temurin'
19 |           cache: maven
20 |       - name: build with maven
21 |         run: mvn -B -s settings.xml clean deploy -Pdep-check
22 |         env:
23 |           NVD_API_KEY: ${{ secrets.NVD_API_KEY }}
24 |           SIGN_KEY_PASS: ${{ secrets.MAVEN_GPG_PASSPHRASE }}
25 |           SIGN_KEY: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }}
26 |           MAVEN_USERNAME: ${{ secrets.MAVEN_CENTRAL_USERNAME }}
27 |           MAVEN_PASSWORD: ${{ secrets.MAVEN_CENTRAL_PASSWORD }}
28 |       - name: notify dependencies
29 |         run: |
30 |           set -ex
31 |           mvn dependency:list | grep 'SNAPSHOT:compile' > dependencies.txt || true
32 |           mvn -q -Dexec.executable='echo' -Dexec.args='${project.groupId}:${project.artifactId}:${project.packaging}:${project.version}' exec:exec | grep :jar: > artifacts.txt
33 |           echo "{\"event_type\":\"update-dependencies\",\"client_payload\":{\"project\":\"${{ github.event.repository.name }}:${{ github.head_ref || github.ref_name }}\",\"artifacts\":\"$( cat artifacts.txt | awk -v ORS='\\n' '1' )\",\"dependencies\":\"$( cat dependencies.txt | awk -v ORS='\\n' '1' )\"}}" > dependency-action.json
34 |           curl -n "https://api.github.com/repos/CI-CMG/maven-dependency-build/dispatches" \
35 |           --header 'Accept: application/vnd.github+json' \
36 |           --header "Authorization: token ${{ secrets.RELEASE_PAT }}" \
37 |           --data @dependency-action.json


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/S3ClientMultipartUpload.java:
--------------------------------------------------------------------------------
 1 | package edu.colorado.cires.cmg.s3out;
 2 | 
 3 | import java.nio.ByteBuffer;
 4 | import java.util.Collection;
 5 | import software.amazon.awssdk.services.s3.S3Client;
 6 | import software.amazon.awssdk.services.s3.model.CompletedPart;
 7 | 
 8 | /**
 9 |  * Acts as a wrapper around a {@link S3Client}, which allows for implementations that could
10 |  * allow for testing or custom behavior.
11 |  */
12 | public interface S3ClientMultipartUpload {
13 | 
14 |   /**
15 |    * Creates a default S3ClientMultipartUpload that should work for most scenarios.
16 |    *
17 |    * @param s3 the {@link S3Client} to access a S3 bucket
18 |    * @return a default implementation of S3ClientMultipartUpload
19 |    */
20 |   static S3ClientMultipartUpload createDefault(S3Client s3) {
21 |     return AwsS3ClientMultipartUpload.builder().s3(s3).build();
22 |   }
23 | 
24 |   /**
25 |    * Initiates a multipart upload to a S3 bucket.
26 |    *
27 |    * @param bucket the bucket name
28 |    * @param key the key where a file will be uploaded to in the bucket
29 |    * @return a upload ID for the pending upload
30 |    * @deprecated Replaced by {@link #createMultipartUpload(MultipartUploadRequest)}
31 |    */
32 |   @Deprecated
33 |   String createMultipartUpload(String bucket, String key);
34 | 
35 |   /**
36 |    * Initiates a multipart upload to a S3 bucket.
37 |    *
38 |    * @param multipartUploadRequest details for the upload: bucket, key, metadata, etc.
39 |    * @return a upload ID for the pending upload
40 |    */
41 |   default String createMultipartUpload(MultipartUploadRequest multipartUploadRequest) {
42 |     return createMultipartUpload(multipartUploadRequest.getBucket(), multipartUploadRequest.getKey());
43 |   }
44 | 
45 |   /**
46 |    * Uploads a part of a multipart upload.
47 |    *
48 |    * @param bucket the bucket name
49 |    * @param key the key where a file will be uploaded to in the bucket
50 |    * @param uploadId the upload ID for the initiated upload
51 |    * @param partNumber the incrementing number for this part in the upload
52 |    * @param buffer a {@link ByteBuffer} containing the data to be uploaded in this part
53 |    * @return a {@link CompletedPart} response object from the completed part upload
54 |    * @deprecated Replaced by {@link #uploadPart(UploadPartParams)}
55 |    */
56 |   CompletedPart uploadPart(String bucket, String key, String uploadId, int partNumber, ByteBuffer buffer);
57 | 
58 |   /**
59 |    * Uploads a part of a multipart upload.
60 |    *
61 |    * @param uploadPartParams details for the upload: bucket, key, metadata, etc.
62 |    * @return a {@link CompletedPart} response object from the completed part upload
63 |    */
64 |   default CompletedPart uploadPart(UploadPartParams uploadPartParams) {
65 |     return uploadPart(uploadPartParams.getBucket(), uploadPartParams.getKey(), uploadPartParams.getUploadId(), uploadPartParams.getPartNumber(), uploadPartParams.getBuffer());
66 |   }
67 | 
68 |   /**
69 |    * Triggers completion of the multipart upload.
70 |    *
71 |    * @param bucket the bucket name
72 |    * @param key the key where a file will be uploaded to in the bucket
73 |    * @param uploadId the upload ID for the initiated upload
74 |    * @param completedParts a collection of {@link CompletedPart} for all the parts uploaded
75 |    */
76 |   void completeMultipartUpload(String bucket, String key, String uploadId, Collection<CompletedPart> completedParts);
77 | 
78 |   /**
79 |    * Signals an abortion of a multipart upload.
80 |    *
81 |    * @param bucket the bucket name
82 |    * @param key the key where a file will be uploaded to in the bucket
83 |    * @param uploadId the upload ID for the initiated upload
84 |    */
85 |   void abortMultipartUpload(String bucket, String key, String uploadId);
86 | }
87 | 


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/MultipartUploadRequest.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.util.Objects;
  4 | import java.util.Optional;
  5 | 
  6 | public class MultipartUploadRequest {
  7 | 
  8 |   /**
  9 |    * Creates a new builder for a MultipartUploadRequest.
 10 |    *
 11 |    * @return a new builder for a MultipartUploadRequest
 12 |    */
 13 |   public static Builder builder() {
 14 |     return new Builder();
 15 |   }
 16 | 
 17 |   private final String bucket;
 18 |   private final String key;
 19 |   private final String checksumAlgorithm;
 20 |   private final ObjectMetadataCustomizer objectMetadata;
 21 | 
 22 |   private MultipartUploadRequest(String bucket, String key, String checksumAlgorithm, ObjectMetadataCustomizer objectMetadata) {
 23 |     this.bucket = bucket;
 24 |     this.key = key;
 25 |     this.checksumAlgorithm = checksumAlgorithm;
 26 |     this.objectMetadata = objectMetadata;
 27 |   }
 28 | 
 29 |   /**
 30 |    * Returns the bucket name.
 31 |    * Never null.
 32 |    * @return the bucket name
 33 |    */
 34 |   public String getBucket() {
 35 |     return bucket;
 36 |   }
 37 | 
 38 |   /**
 39 |    * Returns the bucket key
 40 |    * Never null.
 41 |    * @return the bucket key
 42 |    */
 43 |   public String getKey() {
 44 |     return key;
 45 |   }
 46 | 
 47 |   /**
 48 |    * Returns the checksum algorithm.
 49 |    * Never null.
 50 |    * @return the checksum algorithm
 51 |    */
 52 |   public String getChecksumAlgorithm() {
 53 |     return checksumAlgorithm;
 54 |   }
 55 | 
 56 |   /**
 57 |    * Returns and {@link Optional} containing a {@link ObjectMetadataCustomizer} if available or an empty {@link Optional} otherwise.
 58 |    * @return an {@link Optional}lly wrapped {@link ObjectMetadataCustomizer}
 59 |    */
 60 |   public Optional<ObjectMetadataCustomizer> getObjectMetadata() {
 61 |     return Optional.ofNullable(objectMetadata);
 62 |   }
 63 | 
 64 | 
 65 |   /**
 66 |    * Builds a {@link MultipartUploadRequest}.
 67 |    */
 68 |   public static class Builder {
 69 | 
 70 |     private String bucket;
 71 |     private String key;
 72 |     private String checksumAlgorithm;
 73 |     private ObjectMetadataCustomizer objectMetadata;
 74 | 
 75 |     private Builder() {
 76 | 
 77 |     }
 78 | 
 79 |     /**
 80 |      * Sets the bucket name for the {@link MultipartUploadRequest}. Required.
 81 |      *
 82 |      * @param bucket the bucket name
 83 |      * @return this Builder
 84 |      */
 85 |     public Builder bucket(String bucket) {
 86 |       this.bucket = bucket;
 87 |       return this;
 88 |     }
 89 | 
 90 |     /**
 91 |      * Sets the key where the file will be uploaded to in the bucket. Required.
 92 |      *
 93 |      * @param key the key where the file will be uploaded to in the bucket
 94 |      * @return this Builder
 95 |      */
 96 |     public Builder key(String key) {
 97 |       this.key = key;
 98 |       return this;
 99 |     }
100 | 
101 |     /**
102 |      * Sets the checksum algorithm for the {@link MultipartUploadRequest}. Required.
103 |      *
104 |      * @param checksumAlgorithm the checksum algorithm
105 |      * @return this Builder
106 |      */
107 |     public Builder checksumAlgorithm(String checksumAlgorithm) {
108 |       this.checksumAlgorithm = checksumAlgorithm;
109 |       return this;
110 |     }
111 | 
112 |     /**
113 |      * Sets the metadata that will be applied to a file.
114 |      *
115 |      * @param objectMetadata the file metadata
116 |      * @return this Builder
117 |      */
118 |     public Builder objectMetadata(ObjectMetadataCustomizer objectMetadata) {
119 |       this.objectMetadata = objectMetadata;
120 |       return this;
121 |     }
122 | 
123 |     /**
124 |      * Builds a new {@link MultipartUploadRequest}
125 |      *
126 |      * @return a new {@link MultipartUploadRequest}
127 |      */
128 |     public MultipartUploadRequest build() {
129 |       return new MultipartUploadRequest(
130 |           Objects.requireNonNull(bucket, "bucket is required"),
131 |           Objects.requireNonNull(key, "key is required"),
132 |           checksumAlgorithm,
133 |           objectMetadata);
134 |     }
135 |   }
136 | }
137 | 


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/UploadPartParams.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.nio.ByteBuffer;
  4 | import java.util.Objects;
  5 | 
  6 | public class UploadPartParams {
  7 | 
  8 |   /**
  9 |    * Creates a new builder for a UploadPartParams.
 10 |    *
 11 |    * @return a new builder for a UploadPartParams
 12 |    */
 13 |   public static Builder builder() {
 14 |     return new Builder();
 15 |   }
 16 | 
 17 |   private final String bucket;
 18 |   private final String key;
 19 |   private final String uploadId;
 20 |   private final int partNumber;
 21 |   private final ByteBuffer buffer;
 22 |   private final String checksumAlgorithm;
 23 | 
 24 |   private UploadPartParams(String bucket, String key, String uploadId, int partNumber, ByteBuffer buffer, String checksumAlgorithm) {
 25 |     this.bucket = bucket;
 26 |     this.key = key;
 27 |     this.uploadId = uploadId;
 28 |     this.partNumber = partNumber;
 29 |     this.buffer = buffer;
 30 |     this.checksumAlgorithm = checksumAlgorithm;
 31 |   }
 32 | 
 33 |   /**
 34 |    * Returns the bucket name.
 35 |    * Never null.
 36 |    *
 37 |    * @return the bucket name
 38 |    */
 39 |   public String getBucket() {
 40 |     return bucket;
 41 |   }
 42 | 
 43 |   /**
 44 |    * Returns the bucket key
 45 |    * Never null.
 46 |    *
 47 |    * @return the bucket key
 48 |    */
 49 |   public String getKey() {
 50 |     return key;
 51 |   }
 52 | 
 53 |   /**
 54 |    * Returns the upload id
 55 |    * Never null.
 56 |    *
 57 |    * @return the upload id
 58 |    */
 59 |   public String getUploadId() {
 60 |     return uploadId;
 61 |   }
 62 | 
 63 |   /**
 64 |    * Returns the part number
 65 |    * Never null.
 66 |    *
 67 |    * @return the part number
 68 |    */
 69 |   public int getPartNumber() {
 70 |     return partNumber;
 71 |   }
 72 | 
 73 |   /**
 74 |    * Returns the buffer
 75 |    * Never null.
 76 |    *
 77 |    * @return the buffer
 78 |    */
 79 |   public ByteBuffer getBuffer() {
 80 |     return buffer;
 81 |   }
 82 | 
 83 |   /**
 84 |    * Returns the checksumAlgorithm;
 85 |    *
 86 |    * @return the checksumAlgorithm
 87 |    */
 88 |   public String getChecksumAlgorithm() {
 89 |     return checksumAlgorithm;
 90 |   }
 91 | 
 92 | 
 93 |   /**
 94 |    * Builds a {@link UploadPartParams}.
 95 |    */
 96 |   public static class Builder {
 97 | 
 98 |     private String bucket;
 99 |     private String key;
100 |     private String uploadId;
101 |     private int partNumber;
102 |     private ByteBuffer buffer;
103 |     private String checksumAlgorithm;
104 | 
105 |     private Builder() {
106 | 
107 |     }
108 | 
109 |     /**
110 |      * Sets the bucket name for the {@link UploadPartParams}. Required.
111 |      *
112 |      * @param bucket the bucket name
113 |      * @return this Builder
114 |      */
115 |     public Builder bucket(String bucket) {
116 |       this.bucket = bucket;
117 |       return this;
118 |     }
119 | 
120 |     /**
121 |      * Sets the key where the file will be uploaded to in the bucket. Required.
122 |      *
123 |      * @param key the key where the file will be uploaded to in the bucket
124 |      * @return this Builder
125 |      */
126 |     public Builder key(String key) {
127 |       this.key = key;
128 |       return this;
129 |     }
130 | 
131 |     /**
132 |      * Sets the upload id. Required.
133 |      *
134 |      * @param uploadId the upload id
135 |      * @return this Builder
136 |      */
137 |     public Builder uploadId(String uploadId) {
138 |       this.uploadId = uploadId;
139 |       return this;
140 |     }
141 | 
142 |     /**
143 |      * Sets the part number. Required.
144 |      *
145 |      * @param partNumber the part number
146 |      * @return this Builder
147 |      */
148 |     public Builder partNumber(int partNumber) {
149 |       this.partNumber = partNumber;
150 |       return this;
151 |     }
152 | 
153 |     /**
154 |      * Sets the buffer. Required.
155 |      *
156 |      * @param buffer the buffer
157 |      * @return this Builder
158 |      */
159 |     public Builder buffer(ByteBuffer buffer) {
160 |       this.buffer = buffer;
161 |       return this;
162 |     }
163 | 
164 |     /**
165 |      * Sets the checksumAlgorithm that will be applied.
166 |      *
167 |      * @param checksumAlgorithm the file metadata
168 |      * @return this Builder
169 |      */
170 |     public Builder checksumAlgorithm(String checksumAlgorithm) {
171 |       this.checksumAlgorithm = checksumAlgorithm;
172 |       return this;
173 |     }
174 | 
175 |     /**
176 |      * Builds a new {@link UploadPartParams}
177 |      *
178 |      * @return a new {@link UploadPartParams}
179 |      */
180 |     public UploadPartParams build() {
181 |       return new UploadPartParams(
182 |               Objects.requireNonNull(bucket, "bucket is required"),
183 |               Objects.requireNonNull(key, "key is required"),
184 |               Objects.requireNonNull(uploadId, "uploadId is required"),
185 |               Objects.requireNonNull(partNumber, "partNumber is required"),
186 |               Objects.requireNonNull(buffer, "buffer is required"),
187 |               checksumAlgorithm);
188 |     }
189 |   }
190 | }


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/DefaultContentTypeResolver.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.net.URLConnection;
  4 | import java.util.Collections;
  5 | import java.util.HashMap;
  6 | import java.util.Locale;
  7 | import java.util.Map;
  8 | import java.util.Optional;
  9 | 
 10 | /**
 11 |  * A {@link ContentTypeResolver} that uses common file extensions to determine the MIME type.
 12 |  */
 13 | public class DefaultContentTypeResolver implements ContentTypeResolver {
 14 | 
 15 |   private final static Map<String, String> TYPES;
 16 | 
 17 |   static {
 18 |     Map<String, String> types = new HashMap<>();
 19 |     types.put("aac", "audio/aac");
 20 |     types.put("abw", "application/x-abiword");
 21 |     types.put("arc", "application/x-freearc");
 22 |     types.put("avi", "video/x-msvideo");
 23 |     types.put("azw", "application/vnd.amazon.ebook");
 24 |     types.put("bin", "application/octet-stream");
 25 |     types.put("bmp", "image/bmp");
 26 |     types.put("bz", "application/x-bzip");
 27 |     types.put("bz2", "application/x-bzip2");
 28 |     types.put("csh", "application/x-csh");
 29 |     types.put("css", "text/css");
 30 |     types.put("csv", "text/csv");
 31 |     types.put("doc", "application/msword");
 32 |     types.put("docx", "application/vnd.openxmlformats-officedocument.wordprocessingml.document");
 33 |     types.put("eot", "application/vnd.ms-fontobject");
 34 |     types.put("epub", "application/epub+zip");
 35 |     types.put("gz", "application/gzip");
 36 |     types.put("gif", "image/gif");
 37 |     types.put("htm", "text/html");
 38 |     types.put("html", "text/html");
 39 |     types.put("ico", "image/vnd.microsoft.icon");
 40 |     types.put("ics", "text/calendar");
 41 |     types.put("jar", "application/java-archive");
 42 |     types.put("jpeg", "image/jpeg");
 43 |     types.put("jpg", "image/jpeg");
 44 |     types.put("js", "text/javascript");
 45 |     types.put("json", "application/json");
 46 |     types.put("jsonld", "application/ld+json");
 47 |     types.put("mid", "audio/midi");
 48 |     types.put("midi", "audio/midi");
 49 |     types.put("mjs", "text/javascript");
 50 |     types.put("mp3", "audio/mpeg");
 51 |     types.put("mpeg", "video/mpeg");
 52 |     types.put("mpkg", "application/vnd.apple.installer+xml");
 53 |     types.put("odp", "application/vnd.oasis.opendocument.presentation");
 54 |     types.put("ods", "application/vnd.oasis.opendocument.spreadsheet");
 55 |     types.put("odt", "application/vnd.oasis.opendocument.text");
 56 |     types.put("oga", "audio/ogg");
 57 |     types.put("ogv", "video/ogg");
 58 |     types.put("ogx", "application/ogg");
 59 |     types.put("opus", "audio/opus");
 60 |     types.put("otf", "font/otf");
 61 |     types.put("png", "image/png");
 62 |     types.put("pdf", "application/pdf");
 63 |     types.put("php", "application/php");
 64 |     types.put("ppt", "application/vnd.ms-powerpoint");
 65 |     types.put("pptx", "application/vnd.openxmlformats-officedocument.presentationml.presentation");
 66 |     types.put("rar", "application/vnd.rar");
 67 |     types.put("rtf", "application/rtf");
 68 |     types.put("sh", "application/x-sh");
 69 |     types.put("svg", "image/svg+xml");
 70 |     types.put("swf", "application/x-shockwave-flash");
 71 |     types.put("tar", "application/x-tar");
 72 |     types.put("tif", "image/tiff");
 73 |     types.put("tiff", "image/tiff");
 74 |     types.put("ts", "video/mp2t");
 75 |     types.put("ttf", "font/ttf");
 76 |     types.put("txt", "text/plain");
 77 |     types.put("vsd", "application/vnd.visio");
 78 |     types.put("wav", "audio/wav");
 79 |     types.put("weba", "audio/webm");
 80 |     types.put("webm", "video/webm");
 81 |     types.put("webp", "image/webp");
 82 |     types.put("woff", "font/woff");
 83 |     types.put("woff2", "font/woff2");
 84 |     types.put("xhtml", "application/xhtml+xml");
 85 |     types.put("xls", "application/vnd.ms-excel");
 86 |     types.put("xlsx", "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet");
 87 |     types.put("xml", "text/xml");
 88 |     types.put("xul", "application/vnd.mozilla.xul+xml");
 89 |     types.put("zip", "application/zip");
 90 |     types.put("7z", "application/x-7z-compressed");
 91 |     types.put("yml", "application/x-yaml");
 92 |     types.put("yaml", "application/x-yaml");
 93 |     TYPES = Collections.unmodifiableMap(types);
 94 |   }
 95 | 
 96 | 
 97 |   @Override
 98 |   public Optional<String> resolveContentType(String key) {
 99 |     String fileName = getFileName(key);
100 |     String ext = getKeyExtension(fileName);
101 |     if(ext == null) {
102 |       return Optional.empty();
103 |     }
104 |     String type = TYPES.get(ext);
105 |     if(type == null) {
106 |       type = URLConnection.guessContentTypeFromName(fileName);
107 |     }
108 |     return Optional.ofNullable(type);
109 |   }
110 | 
111 |   private static String getFileName(String key) {
112 |     String[] parts = key.split("/");
113 |     return parts[parts.length - 1];
114 |   }
115 | 
116 |   private static String getKeyExtension(String key) {
117 |     int i = key.lastIndexOf('.');
118 |     String ext = null;
119 |     if (i > 0 && i < key.length() - 1) {
120 |       ext = key.substring(i + 1).toLowerCase(Locale.ENGLISH);
121 |     }
122 |     return ext;
123 |   }
124 | 
125 | }
126 | 


--------------------------------------------------------------------------------
/owasp-dep-check-suppression.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <suppressions xmlns="https://jeremylong.github.io/DependencyCheck/dependency-suppression.1.3.xsd">
  3 |   <suppress>
  4 |     <notes><![CDATA[
  5 |    file name: netty-tcnative-classes-2.0.46.Final.jar
  6 |    ]]></notes>
  7 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
  8 |     <cve>CVE-2014-3488</cve>
  9 |   </suppress>
 10 |   <suppress>
 11 |     <notes><![CDATA[
 12 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 13 |    ]]></notes>
 14 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 15 |     <cve>CVE-2015-2156</cve>
 16 |   </suppress>
 17 |   <suppress>
 18 |     <notes><![CDATA[
 19 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 20 |    ]]></notes>
 21 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 22 |     <cve>CVE-2019-16869</cve>
 23 |   </suppress>
 24 |   <suppress>
 25 |     <notes><![CDATA[
 26 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 27 |    ]]></notes>
 28 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 29 |     <cve>CVE-2019-20444</cve>
 30 |   </suppress>
 31 |   <suppress>
 32 |     <notes><![CDATA[
 33 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 34 |    ]]></notes>
 35 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 36 |     <cve>CVE-2019-20445</cve>
 37 |   </suppress>
 38 |   <suppress>
 39 |     <notes><![CDATA[
 40 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 41 |    ]]></notes>
 42 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 43 |     <cve>CVE-2021-21290</cve>
 44 |   </suppress>
 45 |   <suppress>
 46 |     <notes><![CDATA[
 47 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 48 |    ]]></notes>
 49 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 50 |     <cve>CVE-2021-21295</cve>
 51 |   </suppress>
 52 |   <suppress>
 53 |     <notes><![CDATA[
 54 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 55 |    ]]></notes>
 56 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 57 |     <cve>CVE-2021-21409</cve>
 58 |   </suppress>
 59 |   <suppress>
 60 |     <notes><![CDATA[
 61 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 62 |    ]]></notes>
 63 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 64 |     <cve>CVE-2021-37136</cve>
 65 |   </suppress>
 66 |   <suppress>
 67 |     <notes><![CDATA[
 68 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 69 |    ]]></notes>
 70 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 71 |     <cve>CVE-2021-37137</cve>
 72 |   </suppress>
 73 |   <suppress>
 74 |     <notes><![CDATA[
 75 |    file name: netty-tcnative-classes-2.0.46.Final.jar
 76 |    ]]></notes>
 77 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-tcnative\-classes@.*$</packageUrl>
 78 |     <cve>CVE-2021-43797</cve>
 79 |   </suppress>
 80 |   <suppress>
 81 |     <notes><![CDATA[
 82 |    file name: commons-codec-1.11.jar
 83 |    ]]></notes>
 84 |     <packageUrl regex="true">^pkg:maven/commons\-codec/commons\-codec@.*$</packageUrl>
 85 |     <cve>CVE-2021-37533</cve>
 86 |   </suppress>
 87 |   <suppress>
 88 |     <notes><![CDATA[
 89 |    file name: commons-logging-1.2.jar
 90 |    ]]></notes>
 91 |     <packageUrl regex="true">^pkg:maven/commons\-logging/commons\-logging@.*$</packageUrl>
 92 |     <cve>CVE-2021-37533</cve>
 93 |   </suppress>
 94 |   <suppress>
 95 |     <notes><![CDATA[
 96 |    file name: utils-2.19.13.jar
 97 |    ]]></notes>
 98 |     <packageUrl regex="true">^pkg:maven/software\.amazon\.awssdk/utils@.*$</packageUrl>
 99 |     <cve>CVE-2021-4277</cve>
100 |   </suppress>
101 |   <suppress>
102 |     <notes><![CDATA[
103 |    file name: jackson-core-2.13.5.jar
104 |    reason: not using hutool
105 |    ]]></notes>
106 |     <packageUrl regex="true">^pkg:maven/com\.fasterxml\.jackson\.core/jackson\-core@.*$</packageUrl>
107 |     <cve>CVE-2022-45688</cve>
108 |   </suppress>
109 |   <suppress>
110 |     <notes><![CDATA[
111 |    file name: jackson-core-2.13.5.jar
112 |    reason: not using json-java
113 |    ]]></notes>
114 |     <packageUrl regex="true">^pkg:maven/com\.fasterxml\.jackson\.core/jackson\-core@.*$</packageUrl>
115 |     <cve>CVE-2023-5072</cve>
116 |   </suppress>
117 |   <suppress>
118 |     <notes><![CDATA[
119 |    file name: netty-handler-4.1.100.Final.jar
120 |    reason: not connecting to variable host
121 |    ]]></notes>
122 |     <packageUrl regex="true">^pkg:maven/io\.netty/netty\-handler@.*$</packageUrl>
123 |     <vulnerabilityName>CVE-2023-4586</vulnerabilityName>
124 |   </suppress>
125 |   <suppress>
126 |     <notes><![CDATA[
127 |    file name: json-utils-2.21.37.jar
128 |    reason: not using fredsmithutils
129 |    ]]></notes>
130 |     <packageUrl regex="true">^pkg:maven/software\.amazon\.awssdk/json\-utils@.*$</packageUrl>
131 |     <cve>CVE-2021-4277</cve>
132 |   </suppress>
133 |   <suppress>
134 |     <notes><![CDATA[
135 |    file name: json-utils-2.21.37.jar
136 |    reason: hutool
137 |    ]]></notes>
138 |     <packageUrl regex="true">^pkg:maven/software\.amazon\.awssdk/json\-utils@.*$</packageUrl>
139 |     <cve>CVE-2022-45688</cve>
140 |   </suppress>
141 |   <suppress>
142 |     <notes><![CDATA[
143 |    file name: json-utils-2.21.37.jar
144 |    reason: not using json-java
145 |    ]]></notes>
146 |     <packageUrl regex="true">^pkg:maven/software\.amazon\.awssdk/json\-utils@.*$</packageUrl>
147 |     <cve>CVE-2023-5072</cve>
148 |   </suppress>
149 | </suppressions>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # aws-s3-outputstream
  2 | 
  3 | The aws-s3-outputstream project allows for multipart uploads to an AWS S3 bucket through a java.io.OutputStream.
  4 | 
  5 | Additional project information, javadocs, and test coverage is located at https://ci-cmg.github.io/project-documentation/aws-s3-outputstream/
  6 | 
  7 | ## Adding To Your Project
  8 | 
  9 | Add the following dependency to your Maven pom.xml
 10 | 
 11 | ```xml
 12 |     <dependency>
 13 |       <groupId>io.github.ci-cmg</groupId>
 14 |       <artifactId>aws-s3-outputstream</artifactId>
 15 |       <version>1.1.0</version>
 16 |     </dependency>
 17 | ```
 18 | 
 19 | ## Usage
 20 | 
 21 | The minimal way to create a S3OutputStream is as follows:
 22 | ```java
 23 | MultipartUploadRequest request = MultipartUploadRequest.builder().bucket(bucketName).key(key).build();
 24 | OutputStream out = S3OutputStream.builder().s3(s3).uploadRequest(request).build();
 25 | ```
 26 | 
 27 | Where in the above example s3 is an instance of S3ClientMultipartUpload (described below), bucketName is the name of
 28 | a S3 bucket, and key is the key that will be uploaded to in the bucket.
 29 | 
 30 | Here is how to create a S3OutputStream with all the available options:
 31 | ```java
 32 | OutputStream out = S3OutputStream.builder()
 33 |     .s3(s3)
 34 |     .uploadRequest(MultipartUploadRequest.builder().bucket(bucketName).key(key).build())
 35 |     .partSizeMib(partSizeMib)
 36 |     .uploadQueueSize(queueSize)
 37 |     .autoComplete(true)
 38 |     .build();
 39 | ```
 40 | 
 41 | ### S3ClientMultipartUpload
 42 | s3 is an instance of S3ClientMultipartUpload.  The S3ClientMultipartUpload is a wrapper
 43 | around the S3Client from the AWS SDK v2.  This allows for calls to the S3Client to 
 44 | be mocked for testing.  Two implementations are provided:
 45 | 
 46 | 1. AwsS3ClientMultipartUpload - This uses the S3Client to make calls using the AWS SDK.
 47 | 2. FileMockS3ClientMultipartUpload - This reads and writes from the local file system. This should only be used for testing.
 48 | 
 49 | An instance of AwsS3ClientMultipartUpload can be created as follows:
 50 | ```java
 51 | S3ClientMultipartUpload s3 = AwsS3ClientMultipartUpload.builder()
 52 |     .s3(s3Client)
 53 |     .contentTypeResolver(contentTypeResolver)
 54 |     .build();
 55 | ```
 56 | 
 57 | The contentTypeResolver resolves the MIME type for files uploaded to S3. A default
 58 | implementation is provided if not specified in the AwsS3ClientMultipartUpload builder.
 59 | An instance of NoContentTypeResolver can be provided if MIME types should not be used.
 60 | 
 61 | ### Object Metadata
 62 | Object metadata can be supplied in the MultipartUploadRequest object via the objectMetadata() method in
 63 | the builder.  This accepts an implementation of ObjectMetadataCustomizer.  This library provides
 64 | one implementation, ObjectMetadata, which allows the user to set most of the values allowed in the AWS
 65 | SDK through a builder.  However, if new metadata types are made available, a custom implementation of the
 66 | interface has access to the SDK builder and can set additional values.
 67 | 
 68 | ### Upload Performance
 69 | A S3OutputStream uploads a file in parts. partSizeMib represents the size of the parts to 
 70 | upload in MiB.  This value must be at least 5, which is the default.
 71 | 
 72 | A S3OutputStream uses a queue to allow multipart uploads to S3 to happen while additional
 73 | buffers are being filled concurrently. The uploadQueueSize defines the number of parts
 74 | to be queued before blocking population of additional parts.  The default value is 1.
 75 | Specifying a higher value may improve upload speed at the expense of more heap usage.
 76 | Using a value higher than one should be tested to see if any performance gains are achieved
 77 | for your situation.
 78 | 
 79 | ### Auto Completion
 80 | When a multipart file upload is completed, AWS S3 must be notified. Autocompletion is a
 81 | convenience feature that allows a S3OutputStream to work like a normal java.io.OutputStream.  The
 82 | main use case for this is where your code generates a S3OutputStream that must be 
 83 | passed to another library as a java.io.OutputStream that you do not control that is responsible for closing 
 84 | it. With autocompletion enabled, the AWS completion notification will always happen
 85 | when the OutputStream is closed. This is fine, unless an exception occurs and close()
 86 | is called in a finally block or try-with-resources (as should always be done).  In this
 87 | scenario, the upload will be completed even if there was an error, rather than aborting
 88 | the upload.  To ensure compatibility with java.io.OutputStream, autocompletion is enabled
 89 | by default.
 90 | 
 91 | If you have control over the code closing the S3OutputStream it is best to disable autocompletion 
 92 | as follows:
 93 | ```java
 94 |     try (
 95 |         InputStream inputStream = Files.newInputStream(source);
 96 |         S3OutputStream s3OutputStream = S3OutputStream.builder()
 97 |             .s3(s3)
 98 |             .uploadRequest(MultipartUploadRequest.builder().bucket(bucketName).key(key).build())
 99 |             .autoComplete(false)
100 |             .build();
101 |     ) {
102 |       IOUtils.copy(inputStream, outputStream);
103 |       s3OutputStream.done();
104 |     }
105 | ```
106 | 
107 | Note the call to s3OutputStream.done(). This should be called after all data has been uploaded, before
108 | calling close or the end of the try-with-resources block. This signals that the upload was successful
109 | and when the S3OutputStream is closed, the completion signal will be sent. If done() is not called
110 | before close() and error was assumed to have occurred and an abort signal will be sent in close() 
111 | instead.
112 | 
113 | 
114 | 
115 | 
116 | 
117 | 
118 | 
119 | 


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/AwsS3ClientMultipartUpload.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.nio.ByteBuffer;
  4 | import java.util.Collection;
  5 | import software.amazon.awssdk.core.sync.RequestBody;
  6 | import software.amazon.awssdk.services.s3.S3Client;
  7 | import software.amazon.awssdk.services.s3.model.AbortMultipartUploadRequest;
  8 | import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
  9 | import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
 10 | import software.amazon.awssdk.services.s3.model.CompletedPart;
 11 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 12 | import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 13 | import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 14 | 
 15 | /**
 16 |  * A {@link S3ClientMultipartUpload} that uses a {@link S3Client} to make calls to the AWS S3 SDK.
 17 |  */
 18 | public class AwsS3ClientMultipartUpload implements S3ClientMultipartUpload {
 19 | 
 20 |   /**
 21 |    * Creates a new {@link Builder} to build a S3ClientMultipartUpload
 22 |    *
 23 |    * @return a new Builder
 24 |    */
 25 |   public static Builder builder() {
 26 |     return new Builder();
 27 |   }
 28 | 
 29 |   /**
 30 |    * Builds a {@link AwsS3ClientMultipartUpload}
 31 |    */
 32 |   public static class Builder {
 33 |     private S3Client s3;
 34 |     private ContentTypeResolver contentTypeResolver = new DefaultContentTypeResolver();
 35 | 
 36 |     private Builder() {
 37 | 
 38 |     }
 39 | 
 40 |     /**
 41 |      * Sets the {@link S3Client}.
 42 |      * Required.
 43 |      *
 44 |      * @param s3 the {@link S3Client}
 45 |      * @return this Builder
 46 |      */
 47 |     public Builder s3(S3Client s3) {
 48 |       this.s3 = s3;
 49 |       return this;
 50 |     }
 51 | 
 52 |     /**
 53 |      * Sets the {@link ContentTypeResolver}
 54 |      * Default: {@link DefaultContentTypeResolver}
 55 |      *
 56 |      * @param contentTypeResolver the {@link ContentTypeResolver}
 57 |      * @return this Builder
 58 |      */
 59 |     public Builder contentTypeResolver(ContentTypeResolver contentTypeResolver) {
 60 |       this.contentTypeResolver = contentTypeResolver;
 61 |       return this;
 62 |     }
 63 | 
 64 |     /**
 65 |      * Builds a new {@link AwsS3ClientMultipartUpload}
 66 |      *
 67 |      * @return a new {@link AwsS3ClientMultipartUpload}
 68 |      */
 69 |     public AwsS3ClientMultipartUpload build() {
 70 |       return new AwsS3ClientMultipartUpload(s3, contentTypeResolver);
 71 |     }
 72 |   }
 73 | 
 74 |   private final S3Client s3;
 75 |   private final ContentTypeResolver contentTypeResolver;
 76 | 
 77 |   private AwsS3ClientMultipartUpload(S3Client s3, ContentTypeResolver contentTypeResolver) {
 78 |     this.s3 = s3;
 79 |     this.contentTypeResolver = contentTypeResolver;
 80 |   }
 81 | 
 82 |   @Override
 83 |   public String createMultipartUpload(String bucket, String key) {
 84 |     return createMultipartUpload(MultipartUploadRequest.builder().bucket(bucket).key(key).build());
 85 |   }
 86 | 
 87 |   @Override
 88 |   public String createMultipartUpload(MultipartUploadRequest multipartUploadRequest) {
 89 | 
 90 |     CreateMultipartUploadRequest.Builder builder = CreateMultipartUploadRequest.builder()
 91 |             .bucket(multipartUploadRequest.getBucket())
 92 |             .key(multipartUploadRequest.getKey())
 93 |             .checksumAlgorithm(multipartUploadRequest.getChecksumAlgorithm());
 94 | 
 95 |     contentTypeResolver.resolveContentType(multipartUploadRequest.getKey()).ifPresent(builder::contentType);
 96 | 
 97 |     multipartUploadRequest.getObjectMetadata().ifPresent(objectMetadata -> objectMetadata.apply(builder));
 98 | 
 99 |     return s3.createMultipartUpload(builder.build()).uploadId();
100 |   }
101 | 
102 |   @Override
103 |   public CompletedPart uploadPart(String bucket, String key, String uploadId, int partNumber, ByteBuffer buffer) {
104 |     return uploadPart(UploadPartParams.builder()
105 |             .bucket(bucket)
106 |             .key(key)
107 |             .uploadId(uploadId)
108 |             .partNumber(partNumber)
109 |             .buffer(buffer)
110 |             .build());
111 |   }
112 | 
113 |   @Override
114 |   public CompletedPart uploadPart(UploadPartParams uploadPartParams) {
115 |     UploadPartRequest.Builder builder = UploadPartRequest.builder()
116 |             .bucket(uploadPartParams.getBucket())
117 |             .key(uploadPartParams.getKey())
118 |             .checksumAlgorithm(uploadPartParams.getChecksumAlgorithm())
119 |             .uploadId(uploadPartParams.getUploadId())
120 |             .partNumber(uploadPartParams.getPartNumber());
121 | 
122 |     UploadPartResponse response = s3.uploadPart(builder.build(), RequestBody.fromRemainingByteBuffer(uploadPartParams.getBuffer()));
123 | 
124 |     return CompletedPart.builder()
125 |             .partNumber(uploadPartParams.getPartNumber())
126 |             .checksumCRC32(response.checksumCRC32())
127 |             .checksumCRC32C(response.checksumCRC32C())
128 |             .checksumSHA1(response.checksumSHA1())
129 |             .checksumSHA256(response.checksumSHA256())
130 |             .eTag(response.eTag())
131 |             .build();
132 |   }
133 | 
134 |   @Override
135 |   public void completeMultipartUpload(String bucket, String key, String uploadId, Collection<CompletedPart> completedParts) {
136 |     CompletedMultipartUpload completedMultipartUpload = CompletedMultipartUpload.builder()
137 |         .parts(completedParts)
138 |         .build();
139 | 
140 |     CompleteMultipartUploadRequest completeMultipartUploadRequest =
141 |         CompleteMultipartUploadRequest.builder()
142 |             .bucket(bucket)
143 |             .key(key)
144 |             .uploadId(uploadId)
145 |             .multipartUpload(completedMultipartUpload)
146 |             .build();
147 | 
148 |     s3.completeMultipartUpload(completeMultipartUploadRequest);
149 |   }
150 | 
151 |   @Override
152 |   public void abortMultipartUpload(String bucket, String key, String uploadId) {
153 |     s3.abortMultipartUpload(AbortMultipartUploadRequest.builder()
154 |         .bucket(bucket)
155 |         .key(key)
156 |         .uploadId(uploadId)
157 |         .build());
158 |   }
159 | }
160 | 


--------------------------------------------------------------------------------
/src/test/java/edu/colorado/cires/cmg/s3out/AwsS3ClientMultipartUploadTest.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | 
  4 | import static org.junit.jupiter.api.Assertions.assertEquals;
  5 | import static org.mockito.ArgumentMatchers.any;
  6 | import static org.mockito.Mockito.mock;
  7 | import static org.mockito.Mockito.verify;
  8 | import static org.mockito.Mockito.when;
  9 | 
 10 | import java.nio.ByteBuffer;
 11 | import java.util.Optional;
 12 | import org.junit.jupiter.api.Test;
 13 | import org.mockito.ArgumentCaptor;
 14 | import software.amazon.awssdk.core.sync.RequestBody;
 15 | import software.amazon.awssdk.services.s3.S3Client;
 16 | import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
 17 | import software.amazon.awssdk.services.s3.model.CompletedPart;
 18 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 19 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
 20 | import software.amazon.awssdk.services.s3.model.UploadPartRequest;
 21 | import software.amazon.awssdk.services.s3.model.UploadPartResponse;
 22 | 
 23 | public class AwsS3ClientMultipartUploadTest {
 24 | 
 25 |   @Test
 26 |   public void testObjectMetadata() throws Exception {
 27 |     S3Client s3Client = mock(S3Client.class);
 28 |     ContentTypeResolver contentTypeResolver = mock(ContentTypeResolver.class);
 29 |     CreateMultipartUploadResponse createMultipartUploadResponse = CreateMultipartUploadResponse.builder().uploadId("myUploadId").build();
 30 | 
 31 |     when(contentTypeResolver.resolveContentType(any())).thenReturn(Optional.of("defaultContentType"));
 32 |     when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn(createMultipartUploadResponse);
 33 | 
 34 |     AwsS3ClientMultipartUpload awsS3ClientMultipartUpload = AwsS3ClientMultipartUpload.builder()
 35 |         .s3(s3Client)
 36 |         .contentTypeResolver(contentTypeResolver)
 37 |         .build();
 38 |     String result = awsS3ClientMultipartUpload.createMultipartUpload(
 39 |         MultipartUploadRequest.builder()
 40 |             .bucket("myBucket")
 41 |             .key("myKey")
 42 |             .objectMetadata(ObjectMetadata.builder()
 43 |                 .acl("myAcl")
 44 |                 .contentType("myContentType")
 45 |                 .build())
 46 |             .build());
 47 | 
 48 |     assertEquals("myUploadId", result);
 49 | 
 50 |     ArgumentCaptor<CreateMultipartUploadRequest> argument = ArgumentCaptor.forClass(CreateMultipartUploadRequest.class);
 51 |     verify(s3Client).createMultipartUpload(argument.capture());
 52 |     assertEquals("myAcl", argument.getValue().aclAsString());
 53 |     assertEquals("myContentType", argument.getValue().contentType());
 54 | 
 55 |   }
 56 | 
 57 |   @Test
 58 |   public void testObjectMetadataDefaultContentType() throws Exception {
 59 |     S3Client s3Client = mock(S3Client.class);
 60 |     ContentTypeResolver contentTypeResolver = mock(ContentTypeResolver.class);
 61 |     CreateMultipartUploadResponse createMultipartUploadResponse = CreateMultipartUploadResponse.builder().uploadId("myUploadId").build();
 62 | 
 63 |     when(contentTypeResolver.resolveContentType(any())).thenReturn(Optional.of("defaultContentType"));
 64 |     when(s3Client.createMultipartUpload(any(CreateMultipartUploadRequest.class))).thenReturn(createMultipartUploadResponse);
 65 | 
 66 |     AwsS3ClientMultipartUpload awsS3ClientMultipartUpload = AwsS3ClientMultipartUpload.builder()
 67 |         .s3(s3Client)
 68 |         .contentTypeResolver(contentTypeResolver)
 69 |         .build();
 70 |     String result = awsS3ClientMultipartUpload.createMultipartUpload(
 71 |         MultipartUploadRequest.builder()
 72 |             .bucket("myBucket")
 73 |             .key("myKey")
 74 |             .objectMetadata(ObjectMetadata.builder()
 75 |                 .acl("myAcl")
 76 |                 .build())
 77 |             .build());
 78 | 
 79 |     assertEquals("myUploadId", result);
 80 | 
 81 |     ArgumentCaptor<CreateMultipartUploadRequest> argument = ArgumentCaptor.forClass(CreateMultipartUploadRequest.class);
 82 |     verify(s3Client).createMultipartUpload(argument.capture());
 83 |     assertEquals("myAcl", argument.getValue().aclAsString());
 84 |     assertEquals("defaultContentType", argument.getValue().contentType());
 85 | 
 86 |   }
 87 | 
 88 |   @Test
 89 |   public void testUploadPartParams() throws Exception {
 90 |     S3Client s3Client = mock(S3Client.class);
 91 |     ContentTypeResolver contentTypeResolver = mock(ContentTypeResolver.class);
 92 |     UploadPartResponse uploadPartResponse = UploadPartResponse.builder()
 93 |         .checksumCRC32("myChecksumCRC32")
 94 |         .checksumCRC32C("myChecksumCRC32C")
 95 |         .checksumSHA1("myChecksumSHA1")
 96 |         .checksumSHA256("myChecksumSHA256")
 97 |         .eTag("myEtag")
 98 |         .build();
 99 | 
100 |     when(s3Client.uploadPart(any(UploadPartRequest.class), any(RequestBody.class))).thenReturn(uploadPartResponse);
101 | 
102 |     AwsS3ClientMultipartUpload awsS3ClientMultipartUpload = AwsS3ClientMultipartUpload.builder()
103 |         .s3(s3Client)
104 |         .contentTypeResolver(contentTypeResolver)
105 |         .build();
106 | 
107 |     UploadPartParams params = UploadPartParams.builder()
108 |         .bucket("myBucket")
109 |         .key("myKey")
110 |         .uploadId("myUploadId")
111 |         .partNumber(5)
112 |         .buffer(ByteBuffer.allocate(256))
113 |         .checksumAlgorithm("SHA1")
114 |         .build();
115 | 
116 |     CompletedPart completedPart = awsS3ClientMultipartUpload.uploadPart(params);
117 | 
118 |     ArgumentCaptor<UploadPartRequest> uploadPartRequestArg = ArgumentCaptor.forClass(UploadPartRequest.class);
119 |     ArgumentCaptor<RequestBody> requestBodyArg = ArgumentCaptor.forClass(RequestBody.class);
120 |     verify(s3Client).uploadPart(uploadPartRequestArg.capture(), requestBodyArg.capture());
121 |     UploadPartRequest uploadPartRequest = uploadPartRequestArg.getValue();
122 |     RequestBody requestBody = requestBodyArg.getValue();
123 | 
124 |     assertEquals("myBucket", uploadPartRequest.bucket());
125 |     assertEquals("myKey", uploadPartRequest.key());
126 |     assertEquals(ChecksumAlgorithm.SHA1, uploadPartRequest.checksumAlgorithm());
127 |     assertEquals("myUploadId", uploadPartRequest.uploadId());
128 |     assertEquals(5, uploadPartRequest.partNumber());
129 | 
130 |     assertEquals(256, requestBody.contentLength());
131 | 
132 |     assertEquals(5, completedPart.partNumber());
133 |     assertEquals("myChecksumCRC32", completedPart.checksumCRC32());
134 |     assertEquals("myChecksumCRC32C", completedPart.checksumCRC32C());
135 |     assertEquals("myChecksumSHA1", completedPart.checksumSHA1());
136 |     assertEquals("myChecksumSHA256", completedPart.checksumSHA256());
137 |     assertEquals("myEtag", completedPart.eTag());
138 |   }
139 | }


--------------------------------------------------------------------------------
/src/test/java/edu/colorado/cires/cmg/s3out/S3OutputStreamTest.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import static org.junit.jupiter.api.Assertions.*;
  4 | 
  5 | import java.io.InputStream;
  6 | import java.io.OutputStream;
  7 | import java.nio.charset.StandardCharsets;
  8 | import java.nio.file.Files;
  9 | import java.nio.file.Path;
 10 | import java.nio.file.Paths;
 11 | import org.apache.commons.io.FileUtils;
 12 | import org.apache.commons.io.IOUtils;
 13 | import org.junit.jupiter.api.BeforeEach;
 14 | import org.junit.jupiter.api.Test;
 15 | import org.junit.jupiter.params.ParameterizedTest;
 16 | import org.junit.jupiter.params.provider.CsvSource;
 17 | 
 18 | public class S3OutputStreamTest {
 19 | 
 20 |   private static final Path MOCK_BUCKETS_DIR = Paths.get("target/mock-buckets");
 21 |   private static final String BUCKET = "my-test-bucket";
 22 |   private static final Path BUCKET_DIR = MOCK_BUCKETS_DIR.resolve(BUCKET);
 23 | 
 24 |   @BeforeEach
 25 |   public void setup() throws Exception {
 26 |     FileUtils.deleteQuietly(BUCKET_DIR.toFile());
 27 |     Files.createDirectories(BUCKET_DIR);
 28 |   }
 29 | 
 30 |   @ParameterizedTest
 31 |   @CsvSource({
 32 |       "test.txt,100,1,src/test/resources/test.txt,100",
 33 |       "test.txt,100,1,src/test/resources/test.txt,1",
 34 |       "foo/bar/test.txt,100,1,src/test/resources/test-exact-buffer.txt,100",
 35 |       "test.txt,49,1,src/test/resources/test.txt,25",
 36 |       "test.txt,1,3,src/test/resources/test.txt,2",
 37 |    })
 38 |   public void testAutoComplete(String key, int maxBufferSize, int queueSize, String source, int copyBufferSize) throws Exception {
 39 |     boolean autoComplete = true;
 40 |     Path sourcePath = Paths.get(source);
 41 | 
 42 |     FileMockS3ClientMultipartUpload s3 = FileMockS3ClientMultipartUpload.builder().mockBucketDir(MOCK_BUCKETS_DIR).build();
 43 | 
 44 |     try (
 45 |         InputStream inputStream = Files.newInputStream(sourcePath);
 46 |         OutputStream outputStream = new S3OutputStream(s3, MultipartUploadRequest.builder().bucket(BUCKET).key(key).build(), maxBufferSize, autoComplete, queueSize);
 47 |     ) {
 48 |       if (copyBufferSize == 1){
 49 |         long count;
 50 |         int n;
 51 |         byte[] buffer = new byte[1];
 52 |         for(count = 0L; -1 != (n = inputStream.read(buffer)); count += n) {
 53 |           outputStream.write(buffer[0]);
 54 |         }
 55 |       } else {
 56 |         IOUtils.copy(inputStream, outputStream, copyBufferSize);
 57 |       }
 58 |     }
 59 | 
 60 |     String expected = new String(Files.readAllBytes(sourcePath), StandardCharsets.UTF_8);
 61 |     String actual = new String(Files.readAllBytes(BUCKET_DIR.resolve(key)), StandardCharsets.UTF_8);
 62 | 
 63 |     assertEquals(expected, actual);
 64 |     assertEquals(0, s3.getUploadStateMap().size());
 65 |   }
 66 | 
 67 |   @ParameterizedTest
 68 |   @CsvSource({
 69 |       "test.txt,1,src/test/resources/test.txt,100",
 70 |       "test.txt,1,src/test/resources/test.txt,1",
 71 |       "foo/bar/test.txt,1,src/test/resources/test-exact-buffer.txt,100",
 72 |       "test.txt,1,src/test/resources/test.txt,25",
 73 |       "test.txt,3,src/test/resources/test.txt,2",
 74 |   })
 75 |   public void testBuilder(String key, int queueSize, String source, int copyBufferSize) throws Exception {
 76 |     Path sourcePath = Paths.get(source);
 77 | 
 78 |     FileMockS3ClientMultipartUpload s3 = FileMockS3ClientMultipartUpload.builder().mockBucketDir(MOCK_BUCKETS_DIR).build();
 79 | 
 80 |     try (
 81 |         InputStream inputStream = Files.newInputStream(sourcePath);
 82 |         OutputStream outputStream = S3OutputStream.builder().s3(s3).bucket(BUCKET).key(key).autoComplete(true).partSizeMib(5).uploadQueueSize(queueSize).build();
 83 |     ) {
 84 |         IOUtils.copy(inputStream, outputStream, copyBufferSize);
 85 |     }
 86 | 
 87 |     String expected = new String(Files.readAllBytes(sourcePath), StandardCharsets.UTF_8);
 88 |     String actual = new String(Files.readAllBytes(BUCKET_DIR.resolve(key)), StandardCharsets.UTF_8);
 89 | 
 90 |     assertEquals(expected, actual);
 91 |     assertEquals(0, s3.getUploadStateMap().size());
 92 |   }
 93 | 
 94 |   @Test
 95 |   public void testDone() throws Exception {
 96 |     String key = "test.txt";
 97 |     int maxBufferSize = 100;
 98 |     boolean autoComplete = false;
 99 |     int queueSize = 1;
100 |     Path source = Paths.get("src/test/resources/test.txt");
101 | 
102 |     FileMockS3ClientMultipartUpload s3 = FileMockS3ClientMultipartUpload.builder().mockBucketDir(MOCK_BUCKETS_DIR).build();
103 | 
104 |     try (
105 |         InputStream inputStream = Files.newInputStream(source);
106 |         S3OutputStream outputStream = new S3OutputStream(s3, MultipartUploadRequest.builder().bucket(BUCKET).key(key).build(), maxBufferSize, autoComplete, queueSize);
107 |     ) {
108 |       IOUtils.copy(inputStream, outputStream);
109 |       outputStream.done();
110 |     }
111 | 
112 |     String expected = new String(Files.readAllBytes(source), StandardCharsets.UTF_8);
113 |     String actual = new String(Files.readAllBytes(BUCKET_DIR.resolve(key)), StandardCharsets.UTF_8);
114 | 
115 |     assertEquals(expected, actual);
116 |     assertEquals(0, s3.getUploadStateMap().size());
117 |   }
118 | 
119 |   @Test
120 |   public void testDoneAbort() throws Exception {
121 |     String key = "test.txt";
122 |     int maxBufferSize = 100;
123 |     boolean autoComplete = false;
124 |     int queueSize = 1;
125 |     Path source = Paths.get("src/test/resources/test.txt");
126 | 
127 |     FileMockS3ClientMultipartUpload s3 = FileMockS3ClientMultipartUpload.builder().mockBucketDir(MOCK_BUCKETS_DIR).build();
128 | 
129 |     RuntimeException thrown = assertThrows(RuntimeException.class, () -> {
130 |       try (
131 |           InputStream inputStream = Files.newInputStream(source);
132 |           S3OutputStream outputStream = new S3OutputStream(s3, MultipartUploadRequest.builder().bucket(BUCKET).key(key).build(), maxBufferSize, autoComplete, queueSize);
133 |       ) {
134 |         IOUtils.copy(inputStream, outputStream);
135 |         throw new RuntimeException("test error");
136 |       }
137 |     });
138 | 
139 |     assertEquals("test error", thrown.getMessage());
140 | 
141 |     assertEquals(0, s3.getUploadStateMap().size());
142 | 
143 |   }
144 | 
145 |   @Test
146 |   public void testRepeatedClose() throws Exception {
147 |     String key = "test.txt";
148 |     int maxBufferSize = 100;
149 |     boolean autoComplete = false;
150 |     int queueSize = 1;
151 |     Path source = Paths.get("src/test/resources/test.txt");
152 | 
153 |     FileMockS3ClientMultipartUpload s3 = FileMockS3ClientMultipartUpload.builder().mockBucketDir(MOCK_BUCKETS_DIR).build();
154 | 
155 |     try (
156 |         InputStream inputStream = Files.newInputStream(source);
157 |         S3OutputStream outputStream = new S3OutputStream(s3, MultipartUploadRequest.builder().bucket(BUCKET).key(key).build(), maxBufferSize, autoComplete, queueSize);
158 |     ) {
159 |       IOUtils.copy(inputStream, outputStream);
160 |       outputStream.done();
161 |       outputStream.close();
162 |     }
163 | 
164 |     String expected = new String(Files.readAllBytes(source), StandardCharsets.UTF_8);
165 |     String actual = new String(Files.readAllBytes(BUCKET_DIR.resolve(key)), StandardCharsets.UTF_8);
166 | 
167 |     assertEquals(expected, actual);
168 |     assertEquals(0, s3.getUploadStateMap().size());
169 |   }
170 | 
171 | }


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/FileMockS3ClientMultipartUpload.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.io.BufferedOutputStream;
  4 | import java.io.IOException;
  5 | import java.io.OutputStream;
  6 | import java.nio.ByteBuffer;
  7 | import java.nio.file.Files;
  8 | import java.nio.file.Path;
  9 | import java.util.ArrayList;
 10 | import java.util.Collection;
 11 | import java.util.Collections;
 12 | import java.util.HashMap;
 13 | import java.util.List;
 14 | import java.util.Map;
 15 | import java.util.Objects;
 16 | import java.util.UUID;
 17 | import software.amazon.awssdk.services.s3.model.CompletedPart;
 18 | import software.amazon.awssdk.utils.BinaryUtils;
 19 | 
 20 | /**
 21 |  * A mock implementation of a {@link S3ClientMultipartUpload} that uses the local filesystem.
 22 |  * ONLY to be used for testing.
 23 |  */
 24 | public class FileMockS3ClientMultipartUpload implements S3ClientMultipartUpload {
 25 | 
 26 |   /**
 27 |    * A Builder that builds a {@link FileMockS3ClientMultipartUpload}
 28 |    *
 29 |    * @return the Builder
 30 |    */
 31 |   public static Builder builder() {
 32 |     return new Builder();
 33 |   }
 34 | 
 35 |   /**
 36 |    * A Builder that builds a {@link FileMockS3ClientMultipartUpload}
 37 |    */
 38 |   public static class Builder {
 39 |     private Path mockBucketDir;
 40 | 
 41 |     private Builder() {
 42 | 
 43 |     }
 44 | 
 45 |     /**
 46 |      * Sets a directory that will contain directories representing buckets for testing.
 47 |      *
 48 |      * @param mockBucketDir a directory that will contain directories representing buckets for testing
 49 |      * @return this Builder
 50 |      */
 51 |     public Builder mockBucketDir(Path mockBucketDir) {
 52 |       this.mockBucketDir = mockBucketDir;
 53 |       return this;
 54 |     }
 55 | 
 56 |     /**
 57 |      * Builds a new {@link FileMockS3ClientMultipartUpload}
 58 |      *
 59 |      * @return a new {@link FileMockS3ClientMultipartUpload}
 60 |      */
 61 |     public FileMockS3ClientMultipartUpload build() {
 62 |       return new FileMockS3ClientMultipartUpload(mockBucketDir);
 63 |     }
 64 |   }
 65 | 
 66 |   private final Map<String, MultipartUploadState> uploadStateMap = Collections.synchronizedMap(new HashMap<>());
 67 |   private final Path mockBucketDir;
 68 | 
 69 |   private FileMockS3ClientMultipartUpload(Path mockBucketDir) {
 70 |     this.mockBucketDir = Objects.requireNonNull(mockBucketDir);
 71 |   }
 72 | 
 73 |   @Override
 74 |   public String createMultipartUpload(String bucket, String key) {
 75 |     return createMultipartUpload(MultipartUploadRequest.builder().bucket(bucket).key(key).build());
 76 |   }
 77 | 
 78 |   @Override
 79 |   public String createMultipartUpload(MultipartUploadRequest multipartUploadRequest) {
 80 |     MultipartUploadState multipartUploadState = new MultipartUploadState(
 81 |         multipartUploadRequest.getBucket(),
 82 |         multipartUploadRequest.getKey());
 83 |     uploadStateMap.put(multipartUploadState.getId(), multipartUploadState);
 84 |     return multipartUploadState.getId();
 85 |   }
 86 | 
 87 |   @Override
 88 |   public CompletedPart uploadPart(String bucket, String key, String uploadId, int partNumber, ByteBuffer buffer) {
 89 |     return uploadPart(UploadPartParams.builder()
 90 |             .bucket(bucket)
 91 |             .key(key)
 92 |             .uploadId(uploadId)
 93 |             .partNumber(partNumber)
 94 |             .buffer(buffer)
 95 |             .build());
 96 |   }
 97 | 
 98 |   @Override
 99 |   public CompletedPart uploadPart(UploadPartParams uploadPartParams) {
100 |     MultipartUploadState multipartUploadState = uploadStateMap.get(uploadPartParams.getUploadId());
101 |     if (!multipartUploadState.getBucket().equals(uploadPartParams.getBucket())) {
102 |       throw new IllegalStateException("Incorrect bucket: " + uploadPartParams.getBucket() + " : " + multipartUploadState.getBucket());
103 |     }
104 |     if (!multipartUploadState.getKey().equals(uploadPartParams.getKey())) {
105 |       throw new IllegalStateException("Incorrect key: " + uploadPartParams.getKey() + " : " + multipartUploadState.getKey());
106 |     }
107 |     if (uploadPartParams.getPartNumber() != multipartUploadState.getParts().size() + 1) {
108 |       throw new IllegalStateException("Incorrect part number: " + uploadPartParams.getPartNumber() + " : " + (multipartUploadState.getParts().size() + 1));
109 |     }
110 |     multipartUploadState.getParts().add(uploadPartParams.getBuffer());
111 |     return CompletedPart.builder().build();
112 |   }
113 | 
114 |   @Override
115 |   public void completeMultipartUpload(String bucket, String key, String uploadId, Collection<CompletedPart> completedParts) {
116 |     MultipartUploadState multipartUploadState = uploadStateMap.remove(uploadId);
117 |     if (!multipartUploadState.getBucket().equals(bucket)) {
118 |       throw new IllegalStateException("Incorrect bucket: " + bucket + " : " + multipartUploadState.getBucket());
119 |     }
120 |     if (!multipartUploadState.getKey().equals(key)) {
121 |       throw new IllegalStateException("Incorrect key: " + key + " : " + multipartUploadState.getKey());
122 |     }
123 |     Path path = mockBucketDir.resolve(bucket).resolve(key);
124 |     Path parent = path.getParent();
125 |     if (parent != null) {
126 |       try {
127 |         Files.createDirectories(parent);
128 |       } catch (IOException e) {
129 |         throw new IllegalStateException("Unable to create directory: " + parent);
130 |       }
131 |     }
132 | 
133 |     try (OutputStream outputStream = new BufferedOutputStream(Files.newOutputStream(path))) {
134 |       for (ByteBuffer buffer : multipartUploadState.getParts()) {
135 |         outputStream.write(BinaryUtils.copyRemainingBytesFrom(buffer));
136 |       }
137 |     } catch (IOException e) {
138 |       throw new IllegalStateException("Unable to write to file", e);
139 |     }
140 |   }
141 | 
142 |   @Override
143 |   public void abortMultipartUpload(String bucket, String key, String uploadId) {
144 |     MultipartUploadState multipartUploadState = uploadStateMap.remove(uploadId);
145 |     if (!multipartUploadState.getBucket().equals(bucket)) {
146 |       throw new IllegalStateException("Incorrect bucket: " + bucket + " : " + multipartUploadState.getBucket());
147 |     }
148 |     if (!multipartUploadState.getKey().equals(key)) {
149 |       throw new IllegalStateException("Incorrect key: " + key + " : " + multipartUploadState.getKey());
150 |     }
151 |   }
152 | 
153 |   /**
154 |    * Returns a {@link Map} containing a mock representation of pending multipart uploads.
155 |    *
156 |    * @return a {@link Map} containing a mock representation of pending multipart uploads
157 |    */
158 |   public Map<String, MultipartUploadState> getUploadStateMap() {
159 |     return uploadStateMap;
160 |   }
161 | 
162 |   private static class MultipartUploadState {
163 | 
164 |     private final List<ByteBuffer> parts = new ArrayList<>();
165 |     private final String id = UUID.randomUUID().toString();
166 |     private final String bucket;
167 |     private final String key;
168 | 
169 |     private MultipartUploadState(String bucket, String key) {
170 |       this.bucket = bucket;
171 |       this.key = key;
172 |     }
173 | 
174 |     public List<ByteBuffer> getParts() {
175 |       return parts;
176 |     }
177 | 
178 |     public String getId() {
179 |       return id;
180 |     }
181 | 
182 |     public String getBucket() {
183 |       return bucket;
184 |     }
185 | 
186 |     public String getKey() {
187 |       return key;
188 |     }
189 |   }
190 | }
191 | 


--------------------------------------------------------------------------------
/src/test/java/edu/colorado/cires/cmg/s3out/ObjectMetadataTest.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | 
  4 | import static org.junit.jupiter.api.Assertions.assertEquals;
  5 | import static org.junit.jupiter.api.Assertions.assertNull;
  6 | import static org.mockito.ArgumentMatchers.any;
  7 | import static org.mockito.Mockito.mock;
  8 | import static org.mockito.Mockito.times;
  9 | import static org.mockito.Mockito.verify;
 10 | 
 11 | import java.time.Instant;
 12 | import java.time.temporal.ChronoUnit;
 13 | import java.util.HashMap;
 14 | import java.util.Map;
 15 | import org.junit.jupiter.api.Test;
 16 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
 17 | 
 18 | public class ObjectMetadataTest {
 19 | 
 20 |   @Test
 21 |   public void testBuilder() throws Exception {
 22 |     Map<String, String> metadata = new HashMap<>();
 23 |     metadata.put("a", "b");
 24 | 
 25 |     final String acl = "acl";
 26 |     final String cacheControl = "cacheControl";
 27 |     final String contentDisposition = "contentDisposition";
 28 |     final String contentEncoding = "contentEncoding";
 29 |     final String contentLanguage = "contentLanguage";
 30 |     final String contentType = "contentType";
 31 |     final Instant expires = Instant.now();
 32 |     final String grantFullControl = "grantFullControl";
 33 |     final String grantRead = "grantRead";
 34 |     final String grantReadACP = "grantReadACP";
 35 |     final String grantWriteACP = "grantWriteACP";
 36 |     final String serverSideEncryption = "serverSideEncryption";
 37 |     final String storageClass = "storageClass";
 38 |     final String websiteRedirectLocation = "websiteRedirectLocation";
 39 |     final String sseCustomerAlgorithm = "sseCustomerAlgorithm";
 40 |     final String sseCustomerKey = "sseCustomerKey";
 41 |     final String sseCustomerKeyMD5 = "sseCustomerKeyMD5";
 42 |     final String ssekmsKeyId = "ssekmsKeyId";
 43 |     final String ssekmsEncryptionContext = "ssekmsEncryptionContext";
 44 |     final Boolean bucketKeyEnabled = true;
 45 |     final String requestPayer = "requestPayer";
 46 |     final String tagging = "tagging";
 47 |     final String objectLockMode = "objectLockMode";
 48 |     final Instant objectLockRetainUntilDate = expires.plus(5, ChronoUnit.DAYS);
 49 |     final String objectLockLegalHoldStatus = "objectLockLegalHoldStatus";
 50 |     final String expectedBucketOwner = "expectedBucketOwner";
 51 |     final String checksumAlgorithm = "checksumAlgorithm";
 52 | 
 53 |     ObjectMetadata objectMetadata = ObjectMetadata.builder()
 54 |         .metadata(metadata)
 55 |         .metadata("foo", "bar")
 56 |         .acl(acl)
 57 |         .cacheControl(cacheControl)
 58 |         .contentDisposition(contentDisposition)
 59 |         .contentEncoding(contentEncoding)
 60 |         .contentLanguage(contentLanguage)
 61 |         .contentType(contentType)
 62 |         .expires(expires)
 63 |         .grantFullControl(grantFullControl)
 64 |         .grantRead(grantRead)
 65 |         .grantReadACP(grantReadACP)
 66 |         .grantWriteACP(grantWriteACP)
 67 |         .serverSideEncryption(serverSideEncryption)
 68 |         .storageClass(storageClass)
 69 |         .websiteRedirectLocation(websiteRedirectLocation)
 70 |         .sseCustomerAlgorithm(sseCustomerAlgorithm)
 71 |         .sseCustomerKey(sseCustomerKey)
 72 |         .sseCustomerKeyMD5(sseCustomerKeyMD5)
 73 |         .ssekmsKeyId(ssekmsKeyId)
 74 |         .ssekmsEncryptionContext(ssekmsEncryptionContext)
 75 |         .bucketKeyEnabled(bucketKeyEnabled)
 76 |         .requestPayer(requestPayer)
 77 |         .tagging(tagging)
 78 |         .objectLockMode(objectLockMode)
 79 |         .objectLockRetainUntilDate(objectLockRetainUntilDate)
 80 |         .objectLockLegalHoldStatus(objectLockLegalHoldStatus)
 81 |         .expectedBucketOwner(expectedBucketOwner)
 82 |         .checksumAlgorithm(checksumAlgorithm)
 83 |         .build();
 84 | 
 85 |     metadata.put("foo", "bar");
 86 | 
 87 |     assertEquals(metadata, objectMetadata.getMetadata());
 88 |     assertEquals(acl, objectMetadata.getAcl());
 89 |     assertEquals(cacheControl, objectMetadata.getCacheControl());
 90 |     assertEquals(contentDisposition, objectMetadata.getContentDisposition());
 91 |     assertEquals(contentEncoding, objectMetadata.getContentEncoding());
 92 |     assertEquals(contentLanguage, objectMetadata.getContentLanguage());
 93 |     assertEquals(contentType, objectMetadata.getContentType());
 94 |     assertEquals(expires, objectMetadata.getExpires());
 95 |     assertEquals(grantFullControl, objectMetadata.getGrantFullControl());
 96 |     assertEquals(grantRead, objectMetadata.getGrantRead());
 97 |     assertEquals(grantReadACP, objectMetadata.getGrantReadACP());
 98 |     assertEquals(grantWriteACP, objectMetadata.getGrantWriteACP());
 99 |     assertEquals(serverSideEncryption, objectMetadata.getServerSideEncryption());
100 |     assertEquals(storageClass, objectMetadata.getStorageClass());
101 |     assertEquals(websiteRedirectLocation, objectMetadata.getWebsiteRedirectLocation());
102 |     assertEquals(sseCustomerAlgorithm, objectMetadata.getSseCustomerAlgorithm());
103 |     assertEquals(sseCustomerKey, objectMetadata.getSseCustomerKey());
104 |     assertEquals(sseCustomerKeyMD5, objectMetadata.getSseCustomerKeyMD5());
105 |     assertEquals(ssekmsKeyId, objectMetadata.getSsekmsKeyId());
106 |     assertEquals(ssekmsEncryptionContext, objectMetadata.getSsekmsEncryptionContext());
107 |     assertEquals(bucketKeyEnabled, objectMetadata.getBucketKeyEnabled());
108 |     assertEquals(requestPayer, objectMetadata.getRequestPayer());
109 |     assertEquals(tagging, objectMetadata.getTagging());
110 |     assertEquals(objectLockMode, objectMetadata.getObjectLockMode());
111 |     assertEquals(objectLockRetainUntilDate, objectMetadata.getObjectLockRetainUntilDate());
112 |     assertEquals(objectLockLegalHoldStatus, objectMetadata.getObjectLockLegalHoldStatus());
113 |     assertEquals(expectedBucketOwner, objectMetadata.getExpectedBucketOwner());
114 |     assertEquals(checksumAlgorithm, objectMetadata.getChecksumAlgorithm());
115 | 
116 |     CreateMultipartUploadRequest.Builder builder = mock(CreateMultipartUploadRequest.Builder.class);
117 | 
118 |     objectMetadata.apply(builder);
119 |     verify(builder, times(1)).metadata(metadata);
120 |     verify(builder, times(1)).acl(acl);
121 |     verify(builder, times(1)).cacheControl(cacheControl);
122 |     verify(builder, times(1)).contentDisposition(contentDisposition);
123 |     verify(builder, times(1)).contentEncoding(contentEncoding);
124 |     verify(builder, times(1)).contentLanguage(contentLanguage);
125 |     verify(builder, times(1)).contentType(contentType);
126 |     verify(builder, times(1)).expires(expires);
127 |     verify(builder, times(1)).grantFullControl(grantFullControl);
128 |     verify(builder, times(1)).grantRead(grantRead);
129 |     verify(builder, times(1)).grantReadACP(grantReadACP);
130 |     verify(builder, times(1)).grantWriteACP(grantWriteACP);
131 |     verify(builder, times(1)).serverSideEncryption(serverSideEncryption);
132 |     verify(builder, times(1)).storageClass(storageClass);
133 |     verify(builder, times(1)).websiteRedirectLocation(websiteRedirectLocation);
134 |     verify(builder, times(1)).sseCustomerAlgorithm(sseCustomerAlgorithm);
135 |     verify(builder, times(1)).sseCustomerKey(sseCustomerKey);
136 |     verify(builder, times(1)).sseCustomerKeyMD5(sseCustomerKeyMD5);
137 |     verify(builder, times(1)).ssekmsKeyId(ssekmsKeyId);
138 |     verify(builder, times(1)).ssekmsEncryptionContext(ssekmsEncryptionContext);
139 |     verify(builder, times(1)).bucketKeyEnabled(bucketKeyEnabled);
140 |     verify(builder, times(1)).requestPayer(requestPayer);
141 |     verify(builder, times(1)).tagging(tagging);
142 |     verify(builder, times(1)).objectLockMode(objectLockMode);
143 |     verify(builder, times(1)).objectLockRetainUntilDate(objectLockRetainUntilDate);
144 |     verify(builder, times(1)).objectLockLegalHoldStatus(objectLockLegalHoldStatus);
145 |     verify(builder, times(1)).expectedBucketOwner(expectedBucketOwner);
146 |     verify(builder, times(1)).checksumAlgorithm(checksumAlgorithm);
147 |   }
148 | 
149 |   @Test
150 |   public void testBuilderNull() throws Exception {
151 | 
152 |     ObjectMetadata objectMetadata = ObjectMetadata.builder().build();
153 | 
154 |     assertEquals(new HashMap<>(), objectMetadata.getMetadata());
155 |     assertNull(objectMetadata.getAcl());
156 |     assertNull(objectMetadata.getCacheControl());
157 |     assertNull(objectMetadata.getContentDisposition());
158 |     assertNull(objectMetadata.getContentEncoding());
159 |     assertNull(objectMetadata.getContentLanguage());
160 |     assertNull(objectMetadata.getContentType());
161 |     assertNull(objectMetadata.getExpires());
162 |     assertNull(objectMetadata.getGrantFullControl());
163 |     assertNull(objectMetadata.getGrantRead());
164 |     assertNull(objectMetadata.getGrantReadACP());
165 |     assertNull(objectMetadata.getGrantWriteACP());
166 |     assertNull(objectMetadata.getServerSideEncryption());
167 |     assertNull(objectMetadata.getStorageClass());
168 |     assertNull(objectMetadata.getWebsiteRedirectLocation());
169 |     assertNull(objectMetadata.getSseCustomerAlgorithm());
170 |     assertNull(objectMetadata.getSseCustomerKey());
171 |     assertNull(objectMetadata.getSseCustomerKeyMD5());
172 |     assertNull(objectMetadata.getSsekmsKeyId());
173 |     assertNull(objectMetadata.getSsekmsEncryptionContext());
174 |     assertNull(objectMetadata.getBucketKeyEnabled());
175 |     assertNull(objectMetadata.getRequestPayer());
176 |     assertNull(objectMetadata.getTagging());
177 |     assertNull(objectMetadata.getObjectLockMode());
178 |     assertNull(objectMetadata.getObjectLockRetainUntilDate());
179 |     assertNull(objectMetadata.getObjectLockLegalHoldStatus());
180 |     assertNull(objectMetadata.getExpectedBucketOwner());
181 |     assertNull(objectMetadata.getChecksumAlgorithm());
182 | 
183 |     CreateMultipartUploadRequest.Builder builder = mock(CreateMultipartUploadRequest.Builder.class);
184 | 
185 |     objectMetadata.apply(builder);
186 |     verify(builder, times(0)).metadata(any());
187 |     verify(builder, times(0)).acl(any(String.class));
188 |     verify(builder, times(0)).cacheControl(any());
189 |     verify(builder, times(0)).contentDisposition(any());
190 |     verify(builder, times(0)).contentEncoding(any());
191 |     verify(builder, times(0)).contentLanguage(any());
192 |     verify(builder, times(0)).contentType(any());
193 |     verify(builder, times(0)).expires(any());
194 |     verify(builder, times(0)).grantFullControl(any());
195 |     verify(builder, times(0)).grantRead(any());
196 |     verify(builder, times(0)).grantReadACP(any());
197 |     verify(builder, times(0)).grantWriteACP(any());
198 |     verify(builder, times(0)).serverSideEncryption(any(String.class));
199 |     verify(builder, times(0)).storageClass(any(String.class));
200 |     verify(builder, times(0)).websiteRedirectLocation(any());
201 |     verify(builder, times(0)).sseCustomerAlgorithm(any());
202 |     verify(builder, times(0)).sseCustomerKey(any());
203 |     verify(builder, times(0)).sseCustomerKeyMD5(any());
204 |     verify(builder, times(0)).ssekmsKeyId(any());
205 |     verify(builder, times(0)).ssekmsEncryptionContext(any());
206 |     verify(builder, times(0)).bucketKeyEnabled(any());
207 |     verify(builder, times(0)).requestPayer(any(String.class));
208 |     verify(builder, times(0)).tagging(any(String.class));
209 |     verify(builder, times(0)).objectLockMode(any(String.class));
210 |     verify(builder, times(0)).objectLockRetainUntilDate(any(Instant.class));
211 |     verify(builder, times(0)).objectLockLegalHoldStatus(any(String.class));
212 |     verify(builder, times(0)).expectedBucketOwner(any(String.class));
213 |     verify(builder, times(0)).checksumAlgorithm(any(String.class));
214 |   }
215 | }


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/S3OutputStream.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.io.IOException;
  4 | import java.io.OutputStream;
  5 | import java.nio.ByteBuffer;
  6 | import java.util.ArrayList;
  7 | import java.util.List;
  8 | import java.util.concurrent.BlockingQueue;
  9 | import java.util.concurrent.LinkedBlockingDeque;
 10 | import org.slf4j.Logger;
 11 | import org.slf4j.LoggerFactory;
 12 | import software.amazon.awssdk.services.s3.model.CompletedPart;
 13 | 
 14 | /**
 15 |  * An {@link OutputStream} that uses a multipart upload to upload a file to a S3 bucket.
 16 |  */
 17 | public class S3OutputStream extends OutputStream {
 18 | 
 19 |   private static final Logger LOGGER = LoggerFactory.getLogger(S3OutputStream.class);
 20 |   private static final int MiB = 1024 * 1024;
 21 |   private static final int MIN_PART_SIZE_MIB = 5;
 22 | 
 23 |   /**
 24 |    * Creates a new builder for a S3OutputStream.
 25 |    *
 26 |    * @return a new builder for a S3OutputStream
 27 |    */
 28 |   public static Builder builder() {
 29 |     return new Builder();
 30 |   }
 31 | 
 32 |   /**
 33 |    * Builds a {@link S3OutputStream}.
 34 |    */
 35 |   public static class Builder {
 36 | 
 37 |     private S3ClientMultipartUpload s3;
 38 |     private String bucket;
 39 |     private String key;
 40 |     private MultipartUploadRequest uploadRequest;
 41 |     private int partSizeMib = MIN_PART_SIZE_MIB;
 42 |     private boolean autoComplete = true;
 43 |     private int uploadQueueSize = 1;
 44 | 
 45 |     private Builder() {
 46 | 
 47 |     }
 48 | 
 49 |     /**
 50 |      * Sets the {@link S3ClientMultipartUpload} object for the {@link S3OutputStream}. Required.
 51 |      *
 52 |      * @param s3 the {@link S3ClientMultipartUpload}
 53 |      * @return this Builder
 54 |      */
 55 |     public Builder s3(S3ClientMultipartUpload s3) {
 56 |       this.s3 = s3;
 57 |       return this;
 58 |     }
 59 | 
 60 |     /**
 61 |      * Sets the bucket name for the {@link S3OutputStream}. Required.
 62 |      *
 63 |      * @param bucket the bucket name
 64 |      * @return this Builder
 65 |      * @deprecated use {@link #uploadRequest(MultipartUploadRequest)}
 66 |      */
 67 |     @Deprecated
 68 |     public Builder bucket(String bucket) {
 69 |       this.bucket = bucket;
 70 |       return this;
 71 |     }
 72 | 
 73 |     /**
 74 |      * Sets the key where the file will be uploaded to in the bucket. Required.
 75 |      *
 76 |      * @param key the key where the file will be uploaded to in the bucket
 77 |      * @return this Builder
 78 |      * @deprecated use {@link #uploadRequest(MultipartUploadRequest)}
 79 |      */
 80 |     @Deprecated
 81 |     public Builder key(String key) {
 82 |       this.key = key;
 83 |       return this;
 84 |     }
 85 | 
 86 | 
 87 |     /**
 88 |      * Sets upload request with bucket info and object metadata
 89 |      *
 90 |      * @param uploadRequest object
 91 |      * @return this Builder
 92 |      */
 93 |     public Builder uploadRequest(MultipartUploadRequest uploadRequest) {
 94 |       this.uploadRequest = uploadRequest;
 95 |       return this;
 96 |     }
 97 | 
 98 |     /**
 99 |      * Sets the part size to use when uploading in MiB. Must be at least 5. Default value: 5
100 |      *
101 |      * @param partSizeMib the part size to use when uploading in MiB
102 |      * @return this Builder
103 |      */
104 |     public Builder partSizeMib(int partSizeMib) {
105 |       this.partSizeMib = partSizeMib;
106 |       return this;
107 |     }
108 | 
109 |     /**
110 |      * When a multipart file upload is completed, AWS S3 must be notified. Autocompletion is a convenience feature that allows a S3OutputStream to
111 |      * work like a normal {@link OutputStream}.  The main use case for this is where your code generates a S3OutputStream that must be passed to
112 |      * another library as a {@link OutputStream} that you do not control that is responsible for closing it. With autocompletion enabled, the AWS
113 |      * completion notification will always happen when the OutputStream is closed. This is fine, unless an exception occurs and close() is called in a
114 |      * finally block or try-with-resources (as should always be done).  In this scenario, the upload will be completed even if there was an error,
115 |      * rather than aborting the upload.  To ensure compatibility with java.io.OutputStream, autocompletion is enabled by default.
116 |      *
117 |      * If you have control over the code closing the S3OutputStream it is best to disable autocompletion as follows:
118 |      * <pre>
119 |      *     try (
120 |      *         InputStream inputStream = Files.newInputStream(source);
121 |      *         S3OutputStream s3OutputStream = S3OutputStream.builder()
122 |      *             .s3(s3)
123 |      *             .uploadRequest(MultipartUploadRequest.builder().bucket(bucketName).key(key).build())
124 |      *             .autoComplete(false)
125 |      *             .build();
126 |      *     ) {
127 |      *       IOUtils.copy(inputStream, outputStream);
128 |      *       s3OutputStream.done();
129 |      *     }
130 |      * </pre>
131 |      *
132 |      * Note the call to s3OutputStream.done(). This should be called after all data has been uploaded, before calling close or the end of the
133 |      * try-with-resources block. This signals that the upload was successful and when the S3OutputStream is closed, the completion signal will be
134 |      * sent. If done() is not called before close() and error was assumed to have occurred and an abort signal will be send in close() instead.
135 |      *
136 |      * @param autoComplete true to enable autocompletion
137 |      * @return this Builder
138 |      */
139 |     public Builder autoComplete(boolean autoComplete) {
140 |       this.autoComplete = autoComplete;
141 |       return this;
142 |     }
143 | 
144 |     /**
145 |      * A S3OutputStream uses a queue to allow multipart uploads to S3 to happen while additional buffers are being filled concurrently. The
146 |      * uploadQueueSize defines the number of parts to be queued before blocking population of additional parts.  The default value is 1. Specifying a
147 |      * higher value may improve upload speed at the expense of more heap usage. Using a value higher than one should be tested to see if any
148 |      * performance gains are achieved for your situation.
149 |      *
150 |      * @param uploadQueueSize the max number of buffers in the queue before blocking
151 |      * @return this Builder
152 |      */
153 |     public Builder uploadQueueSize(int uploadQueueSize) {
154 |       this.uploadQueueSize = uploadQueueSize;
155 |       return this;
156 |     }
157 | 
158 |     /**
159 |      * Builds a new {@link S3OutputStream}
160 |      *
161 |      * @return a new {@link S3OutputStream}
162 |      */
163 |     public S3OutputStream build() {
164 |       if (partSizeMib < MIN_PART_SIZE_MIB) {
165 |         throw new IllegalArgumentException("Part size MiB must be at least " + MIN_PART_SIZE_MIB);
166 |       }
167 |       MultipartUploadRequest request;
168 |       if (uploadRequest != null) {
169 |         request = uploadRequest;
170 |       } else {
171 |         request = MultipartUploadRequest.builder().bucket(bucket).key(key).build();
172 |       }
173 |       return new S3OutputStream(s3, request, partSizeMib * MiB, autoComplete, uploadQueueSize);
174 |     }
175 |   }
176 | 
177 |   private final S3ClientMultipartUpload s3;
178 |   private final String bucket;
179 |   private final String key;
180 |   private final String checksumAlgorithm;
181 |   private final int maxBufferSize;
182 |   private final String uploadId;
183 |   private final List<CompletedPart> completedParts = new ArrayList<>();
184 |   private final BlockingQueue<UploadConsumerBuffer> uploadQueue;
185 |   private final Thread consumer;
186 | 
187 |   private ByteBuffer buffer;
188 |   private boolean complete;
189 |   private boolean closed;
190 | 
191 | 
192 |   S3OutputStream(S3ClientMultipartUpload s3, MultipartUploadRequest uploadRequest, int maxBufferSize, boolean autoComplete,
193 |       int queueSize) {
194 |     this.uploadQueue = new LinkedBlockingDeque<>(queueSize);
195 |     this.s3 = s3;
196 |     this.bucket = uploadRequest.getBucket();
197 |     this.key = uploadRequest.getKey();
198 |     this.checksumAlgorithm = uploadRequest.getChecksumAlgorithm();
199 |     this.maxBufferSize = maxBufferSize;
200 |     complete = autoComplete;
201 |     uploadId = s3.createMultipartUpload(uploadRequest);
202 |     newBuffer();
203 |     consumer = new Thread(new UploadConsumer());
204 |     consumer.start();
205 |   }
206 | 
207 |   private void newBuffer() {
208 |     buffer = ByteBuffer.allocate(maxBufferSize);
209 |   }
210 | 
211 |   private void uploadPart() {
212 |     if (buffer.position() > 0) {
213 |       buffer.flip();
214 |       try {
215 |         uploadQueue.put(new UploadConsumerBuffer(buffer, false));
216 |       } catch (InterruptedException e) {
217 |         Thread.currentThread().interrupt();
218 |         throw new IllegalStateException("Upload thread was interrupted", e);
219 |       }
220 |     }
221 |   }
222 | 
223 |   private class UploadConsumer implements Runnable {
224 | 
225 |     @Override
226 |     public void run() {
227 |       try {
228 |         while (true) {
229 |           UploadConsumerBuffer buffer = uploadQueue.take();
230 |           if (buffer.isPoison()) {
231 |             return;
232 |           }
233 |           synchronized (completedParts) {
234 |             int partNumber = completedParts.size() + 1;
235 |             UploadPartParams.Builder builder = UploadPartParams.builder()
236 |                     .bucket(bucket)
237 |                     .key(key)
238 |                     .uploadId(uploadId)
239 |                     .partNumber(partNumber)
240 |                     .buffer(buffer.getBuffer())
241 |                     .checksumAlgorithm(checksumAlgorithm);
242 |             completedParts.add(s3.uploadPart(builder.build()));
243 |           }
244 |         }
245 |       } catch (InterruptedException e) {
246 |         Thread.currentThread().interrupt();
247 |       }
248 |     }
249 |   }
250 | 
251 |   private static class UploadConsumerBuffer {
252 | 
253 |     private final ByteBuffer buffer;
254 |     private final boolean poison;
255 | 
256 |     private UploadConsumerBuffer(ByteBuffer buffer, boolean poison) {
257 |       this.buffer = buffer;
258 |       this.poison = poison;
259 |     }
260 | 
261 |     public ByteBuffer getBuffer() {
262 |       return buffer;
263 |     }
264 | 
265 |     public boolean isPoison() {
266 |       return poison;
267 |     }
268 |   }
269 | 
270 |   private void cycleBuffer() {
271 |     uploadPart();
272 |     newBuffer();
273 |   }
274 | 
275 |   private void complete() {
276 |     synchronized (completedParts) {
277 |       s3.completeMultipartUpload(bucket, key, uploadId, completedParts);
278 |     }
279 |   }
280 | 
281 |   private void abort() {
282 |     try {
283 |       s3.abortMultipartUpload(bucket, key, uploadId);
284 |     } catch (Exception e) {
285 |       LOGGER.warn("An error occurred aborting multipart upload: " + bucket + ":" + key, e);
286 |     }
287 |   }
288 | 
289 |   /**
290 |    * If autocomplete is disabled, marks the upload as successful.
291 |    *
292 |    * @see Builder#autoComplete(boolean)
293 |    */
294 |   public void done() {
295 |     complete = true;
296 |   }
297 | 
298 |   @Override
299 |   public void write(byte[] b, int off, int len) throws IOException {
300 |     if (b == null) {
301 |       throw new NullPointerException();
302 |     } else if ((off < 0) || (off > b.length) || (len < 0) ||
303 |         ((off + len) > b.length) || ((off + len) < 0)) {
304 |       throw new IndexOutOfBoundsException();
305 |     } else if (len == 0) {
306 |       return;
307 |     }
308 |     if (buffer.remaining() >= len) {
309 |       buffer.put(b, off, len);
310 |       if (!buffer.hasRemaining()) {
311 |         cycleBuffer();
312 |       }
313 |     } else {
314 |       for (byte[] chunk : chunkBytes(b, off, len)) {
315 |         buffer.put(chunk);
316 |         if (!buffer.hasRemaining()) {
317 |           cycleBuffer();
318 |         }
319 |       }
320 |     }
321 |   }
322 | 
323 |   private List<byte[]> chunkBytes(byte[] b, int initialOffset, int len) {
324 |     List<byte[]> chunks = new ArrayList<>();
325 |     final int bEnd = initialOffset + len;
326 |     int start = initialOffset;
327 |     int end = Math.min(bEnd, start + buffer.remaining());
328 |     byte[] chunk = new byte[end - start];
329 |     System.arraycopy(b, start, chunk, 0, chunk.length);
330 |     chunks.add(chunk);
331 | 
332 |     while (end < bEnd) {
333 |       start = end;
334 |       end = Math.min(bEnd, start + maxBufferSize);
335 |       chunk = new byte[end - start];
336 |       System.arraycopy(b, start, chunk, 0, chunk.length);
337 |       chunks.add(chunk);
338 |     }
339 | 
340 |     return chunks;
341 |   }
342 | 
343 |   @Override
344 |   public void write(int b) throws IOException {
345 |     if (buffer.hasRemaining()) {
346 |       buffer.put((byte) b);
347 |     } else {
348 |       cycleBuffer();
349 |       buffer.put((byte) b);
350 |     }
351 |   }
352 | 
353 |   @Override
354 |   public void close() throws IOException {
355 |     if (!closed) {
356 |       closed = true;
357 |       if (complete) {
358 |         uploadPart();
359 |         try {
360 |           uploadQueue.put(new UploadConsumerBuffer(null, true));
361 |           consumer.join();
362 |         } catch (InterruptedException e) {
363 |           Thread.currentThread().interrupt();
364 |         }
365 |         complete();
366 |       } else {
367 |         try {
368 |           uploadQueue.put(new UploadConsumerBuffer(null, true));
369 |           consumer.join();
370 |         } catch (InterruptedException e) {
371 |           Thread.currentThread().interrupt();
372 |         }
373 |         abort();
374 |       }
375 |     }
376 |   }
377 | }
378 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  3 |   xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  4 |   <modelVersion>4.0.0</modelVersion>
  5 | 
  6 |   <groupId>io.github.ci-cmg</groupId>
  7 |   <artifactId>aws-s3-outputstream</artifactId>
  8 |   <version>1.3.0-SNAPSHOT</version>
  9 |   <packaging>jar</packaging>
 10 | 
 11 |   <name>${project.groupId}:${project.artifactId}</name>
 12 |   <description>Converts AWS SDK v2 multipart upload into an java.io.OutputStream</description>
 13 |   <url>https://github.com/CI-CMG/aws-s3-outputstream</url>
 14 | 
 15 |   <licenses>
 16 |     <license>
 17 |       <name>MIT License</name>
 18 |       <url>http://www.opensource.org/licenses/mit-license.php</url>
 19 |     </license>
 20 |   </licenses>
 21 | 
 22 |   <developers>
 23 |     <developer>
 24 |       <name>CIRES Coastal and Marine Geophysics / Marine Geology and Geophysics Developers</name>
 25 |       <organization>CIRES</organization>
 26 |       <organizationUrl>https://github.com/CI-CMG</organizationUrl>
 27 |     </developer>
 28 |   </developers>
 29 | 
 30 |   <scm>
 31 |     <connection>scm:git:https://github.com/CI-CMG/aws-s3-outputstream.git</connection>
 32 |     <developerConnection>scm:git:https://github.com/CI-CMG/aws-s3-outputstream.git</developerConnection>
 33 |     <url>https://github.com/CI-CMG/aws-s3-outputstream</url>
 34 |     <tag>HEAD</tag>
 35 |   </scm>
 36 | 
 37 |   <properties>
 38 |     <maven.compiler.release>8</maven.compiler.release>
 39 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 40 |     <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
 41 |     <slf4j.version>1.7.32</slf4j.version>
 42 |     <javadoc.version>3.11.2</javadoc.version>
 43 |     <jacoco.version>0.8.12</jacoco.version>
 44 |     <junit.version>5.8.1</junit.version>
 45 |     <nvdApiKey>${env.NVD_API_KEY}</nvdApiKey>
 46 |     <git.username />
 47 |     <git.password />
 48 |   </properties>
 49 | 
 50 |   <dependencyManagement>
 51 |     <dependencies>
 52 |       <dependency>
 53 |         <groupId>software.amazon.awssdk</groupId>
 54 |         <artifactId>bom</artifactId>
 55 |         <version>2.31.41</version>
 56 |         <type>pom</type>
 57 |         <scope>import</scope>
 58 |       </dependency>
 59 |       <dependency>
 60 |         <groupId>io.netty</groupId>
 61 |         <artifactId>netty-bom</artifactId>
 62 |         <version>4.1.121.Final</version>
 63 |         <type>pom</type>
 64 |         <scope>import</scope>
 65 |       </dependency>
 66 |     </dependencies>
 67 |   </dependencyManagement>
 68 | 
 69 |   <dependencies>
 70 |     <dependency>
 71 |       <groupId>software.amazon.awssdk</groupId>
 72 |       <artifactId>s3</artifactId>
 73 |     </dependency>
 74 |     <dependency>
 75 |       <groupId>org.slf4j</groupId>
 76 |       <artifactId>slf4j-api</artifactId>
 77 |       <version>${slf4j.version}</version>
 78 |     </dependency>
 79 |     <dependency>
 80 |       <groupId>org.junit.jupiter</groupId>
 81 |       <artifactId>junit-jupiter-engine</artifactId>
 82 |       <version>${junit.version}</version>
 83 |       <scope>test</scope>
 84 |     </dependency>
 85 |     <dependency>
 86 |       <groupId>org.junit.jupiter</groupId>
 87 |       <artifactId>junit-jupiter-params</artifactId>
 88 |       <version>${junit.version}</version>
 89 |       <scope>test</scope>
 90 |     </dependency>
 91 |     <dependency>
 92 |       <groupId>org.slf4j</groupId>
 93 |       <artifactId>slf4j-simple</artifactId>
 94 |       <version>${slf4j.version}</version>
 95 |       <scope>test</scope>
 96 |     </dependency>
 97 |     <dependency>
 98 |       <groupId>commons-io</groupId>
 99 |       <artifactId>commons-io</artifactId>
100 |       <version>2.11.0</version>
101 |       <scope>test</scope>
102 |     </dependency>
103 |     <dependency>
104 |       <groupId>org.mockito</groupId>
105 |       <artifactId>mockito-core</artifactId>
106 |       <version>3.3.3</version>
107 |       <scope>test</scope>
108 |     </dependency>
109 |   </dependencies>
110 | 
111 |   <reporting>
112 |     <plugins>
113 |       <plugin>
114 |         <groupId>org.apache.maven.plugins</groupId>
115 |         <artifactId>maven-javadoc-plugin</artifactId>
116 |         <version>${javadoc.version}</version>
117 |       </plugin>
118 |       <plugin>
119 |         <groupId>org.jacoco</groupId>
120 |         <artifactId>jacoco-maven-plugin</artifactId>
121 |         <version>${jacoco.version}</version>
122 |       </plugin>
123 |     </plugins>
124 |   </reporting>
125 | 
126 |   <build>
127 |     <pluginManagement>
128 |       <plugins>
129 |         <plugin>
130 |           <groupId>org.apache.maven.plugins</groupId>
131 |           <artifactId>maven-compiler-plugin</artifactId>
132 |           <version>3.13.0</version>
133 |         </plugin>
134 |         <plugin>
135 |           <groupId>org.apache.maven.plugins</groupId>
136 |           <artifactId>maven-surefire-plugin</artifactId>
137 |           <version>3.5.2</version>
138 |         </plugin>
139 |         <plugin>
140 |           <artifactId>maven-antrun-plugin</artifactId>
141 |           <version>3.1.0</version>
142 |         </plugin>
143 |         <plugin>
144 |           <groupId>org.apache.maven.plugins</groupId>
145 |           <artifactId>maven-release-plugin</artifactId>
146 |           <version>3.1.1</version>
147 |         </plugin>
148 |         <plugin>
149 |           <groupId>pl.project13.maven</groupId>
150 |           <artifactId>git-commit-id-plugin</artifactId>
151 |           <version>4.9.10</version>
152 |         </plugin>
153 |         <plugin>
154 |           <groupId>org.jacoco</groupId>
155 |           <artifactId>jacoco-maven-plugin</artifactId>
156 |           <version>${jacoco.version}</version>
157 |         </plugin>
158 |         <plugin>
159 |           <groupId>org.apache.maven.plugins</groupId>
160 |           <artifactId>maven-source-plugin</artifactId>
161 |           <version>3.3.1</version>
162 |         </plugin>
163 |         <plugin>
164 |           <groupId>org.apache.maven.plugins</groupId>
165 |           <artifactId>maven-site-plugin</artifactId>
166 |           <version>3.21.0</version>
167 |         </plugin>
168 |         <plugin>
169 |           <groupId>org.apache.maven.plugins</groupId>
170 |           <artifactId>maven-project-info-reports-plugin</artifactId>
171 |           <version>3.8.0</version>
172 |         </plugin>
173 |         <plugin>
174 |           <groupId>org.apache.maven.plugins</groupId>
175 |           <artifactId>maven-javadoc-plugin</artifactId>
176 |           <version>${javadoc.version}</version>
177 |         </plugin>
178 |         <plugin>
179 |           <groupId>com.github.spotbugs</groupId>
180 |           <artifactId>spotbugs-maven-plugin</artifactId>
181 |           <version>4.8.6.5</version>
182 |         </plugin>
183 |         <plugin>
184 |           <groupId>org.owasp</groupId>
185 |           <artifactId>dependency-check-maven</artifactId>
186 |           <version>12.1.1</version>
187 |         </plugin>
188 |         <plugin>
189 |           <artifactId>maven-resources-plugin</artifactId>
190 |           <version>3.3.1</version>
191 |         </plugin>
192 |         <plugin>
193 |           <groupId>org.simplify4u.plugins</groupId>
194 |           <artifactId>sign-maven-plugin</artifactId>
195 |           <version>1.1.0</version>
196 |         </plugin>
197 |       </plugins>
198 |     </pluginManagement>
199 | 
200 |     <plugins>
201 | 
202 |       <plugin>
203 |         <groupId>org.apache.maven.plugins</groupId>
204 |         <artifactId>maven-javadoc-plugin</artifactId>
205 |         <executions>
206 |           <execution>
207 |             <id>attach-javadocs</id>
208 |             <goals>
209 |               <goal>jar</goal>
210 |             </goals>
211 |           </execution>
212 |         </executions>
213 |       </plugin>
214 | 
215 |       <plugin>
216 |         <groupId>org.apache.maven.plugins</groupId>
217 |         <artifactId>maven-source-plugin</artifactId>
218 |         <executions>
219 |           <execution>
220 |             <id>attach-sources</id>
221 |             <goals>
222 |               <goal>jar-no-fork</goal>
223 |             </goals>
224 |           </execution>
225 |         </executions>
226 |       </plugin>
227 | 
228 |       <plugin>
229 |         <groupId>org.simplify4u.plugins</groupId>
230 |         <artifactId>sign-maven-plugin</artifactId>
231 |         <executions>
232 |           <execution>
233 |             <goals>
234 |               <goal>sign</goal>
235 |             </goals>
236 |           </execution>
237 |         </executions>
238 |       </plugin>
239 | 
240 |       <plugin>
241 |         <groupId>org.sonatype.central</groupId>
242 |         <artifactId>central-publishing-maven-plugin</artifactId>
243 |         <version>0.7.0</version>
244 |         <extensions>true</extensions>
245 |         <configuration>
246 |           <publishingServerId>central</publishingServerId>
247 |           <autoPublish>true</autoPublish>
248 |           <waitUntil>published</waitUntil>
249 |         </configuration>
250 |       </plugin>
251 | 
252 |       <plugin>
253 |         <groupId>org.apache.maven.plugins</groupId>
254 |         <artifactId>maven-release-plugin</artifactId>
255 |         <configuration>
256 |           <autoVersionSubmodules>true</autoVersionSubmodules>
257 |           <useReleaseProfile>false</useReleaseProfile>
258 |           <releaseProfiles>release,site-publish</releaseProfiles>
259 |           <goals>deploy site</goals>
260 |           <tagNameFormat>v@{project.version}</tagNameFormat>
261 |           <arguments>-Dgit.username=${git.username} -Dgit.password=${git.password}</arguments>
262 |           <username>${git.username}</username>
263 |           <password>${git.password}</password>
264 |         </configuration>
265 |       </plugin>
266 | 
267 |       <plugin>
268 |         <groupId>org.jacoco</groupId>
269 |         <artifactId>jacoco-maven-plugin</artifactId>
270 |         <executions>
271 |           <execution>
272 |             <id>default-prepare-agent</id>
273 |             <goals>
274 |               <goal>prepare-agent</goal>
275 |             </goals>
276 |           </execution>
277 |           <execution>
278 |             <id>report</id>
279 |             <phase>verify</phase>
280 |             <goals>
281 |               <goal>report</goal>
282 |             </goals>
283 |           </execution>
284 |         </executions>
285 |       </plugin>
286 |       <plugin>
287 |         <groupId>com.github.spotbugs</groupId>
288 |         <artifactId>spotbugs-maven-plugin</artifactId>
289 |         <executions>
290 |           <execution>
291 |             <goals>
292 |               <goal>check</goal>
293 |             </goals>
294 |           </execution>
295 |         </executions>
296 |         <configuration>
297 |           <excludeFilterFile>spotbugs-exclude.xml</excludeFilterFile>
298 |         </configuration>
299 |       </plugin>
300 |     </plugins>
301 |   </build>
302 | 
303 |   <profiles>
304 | 
305 |     <profile>
306 |       <id>dep-check</id>
307 |       <build>
308 |         <plugins>
309 |           <plugin>
310 |             <groupId>org.owasp</groupId>
311 |             <artifactId>dependency-check-maven</artifactId>
312 |             <configuration>
313 |               <failBuildOnCVSS>0</failBuildOnCVSS>
314 |               <suppressionFiles>
315 |                 <suppressionFile>owasp-dep-check-suppression.xml</suppressionFile>
316 |               </suppressionFiles>
317 |             </configuration>
318 |             <executions>
319 |               <execution>
320 |                 <goals>
321 |                   <goal>check</goal>
322 |                 </goals>
323 |               </execution>
324 |             </executions>
325 |           </plugin>
326 |         </plugins>
327 |       </build>
328 |     </profile>
329 | 
330 |     <profile>
331 |       <id>release</id>
332 |       <properties>
333 |         <sign.skipNoKey>false</sign.skipNoKey>
334 |       </properties>
335 |       <build>
336 |         <plugins>
337 |           <plugin>
338 |             <groupId>pl.project13.maven</groupId>
339 |             <artifactId>git-commit-id-plugin</artifactId>
340 |             <executions>
341 |               <execution>
342 |                 <id>get-the-git-infos</id>
343 |                 <goals>
344 |                   <goal>revision</goal>
345 |                 </goals>
346 |                 <phase>pre-site</phase>
347 |               </execution>
348 |             </executions>
349 |             <configuration>
350 |               <injectAllReactorProjects>true</injectAllReactorProjects>
351 |               <failOnNoGitDirectory>false</failOnNoGitDirectory>
352 |               <offline>true</offline>
353 |               <replacementProperties>
354 |                 <replacementProperty>
355 |                   <property>git.remote.origin.url</property>
356 |                   <propertyOutputSuffix>suffix</propertyOutputSuffix>
357 |                   <token>^.+/(.+).git$</token>
358 |                   <value>$1</value>
359 |                   <regex>true</regex>
360 |                 </replacementProperty>
361 |               </replacementProperties>
362 |             </configuration>
363 |           </plugin>
364 |         </plugins>
365 |       </build>
366 |     </profile>
367 | 
368 | 
369 |     <profile>
370 |       <id>site-publish</id>
371 |       <build>
372 |         <plugins>
373 |           <plugin>
374 |             <groupId>org.apache.maven.plugins</groupId>
375 |             <artifactId>maven-site-plugin</artifactId>
376 |           </plugin>
377 |           <plugin>
378 |             <artifactId>maven-resources-plugin</artifactId>
379 |             <executions>
380 |               <execution>
381 |                 <id>index.html</id>
382 |                 <phase>site</phase>
383 |                 <goals>
384 |                   <goal>copy-resources</goal>
385 |                 </goals>
386 |                 <configuration>
387 |                   <outputDirectory>${project.build.directory}/site-resources</outputDirectory>
388 |                   <useDefaultDelimiters>false</useDefaultDelimiters>
389 |                   <delimiters>
390 |                     <delimiter>@</delimiter>
391 |                   </delimiters>
392 |                   <resources>
393 |                     <resource>
394 |                       <directory>site-resources</directory>
395 |                       <filtering>true</filtering>
396 |                     </resource>
397 |                   </resources>
398 |                 </configuration>
399 |               </execution>
400 |             </executions>
401 |           </plugin>
402 |           <plugin>
403 |             <artifactId>maven-antrun-plugin</artifactId>
404 |             <executions>
405 |               <execution>
406 |                 <id>publish-site</id>
407 |                 <phase>site</phase>
408 |                 <goals>
409 |                   <goal>run</goal>
410 |                 </goals>
411 |                 <configuration>
412 |                   <target>
413 |                     <delete dir="${project.build.directory}/project-documentation" quiet="true"/>
414 |                     <exec dir="${project.build.directory}" executable="git">
415 |                       <arg value="clone" />
416 |                       <arg value="--depth=1" />
417 |                       <arg value="https://${git.username}:${git.password}@github.com/CI-CMG/project-documentation.git" />
418 |                     </exec>
419 |                     <exec dir="${project.build.directory}/project-documentation" executable="git">
420 |                       <arg value="remote" />
421 |                       <arg value="add" />
422 |                       <arg value="javadoc" />
423 |                       <arg value="https://${git.username}:${git.password}@github.com/CI-CMG/project-documentation.git" />
424 |                     </exec>
425 |                     <exec dir="${project.build.directory}/project-documentation" executable="git" failonerror="false">
426 |                       <arg value="rm"/>
427 |                       <arg value="-r"/>
428 |                       <arg value="${git.remote.origin.url.suffix}/${project.version}"/>
429 |                     </exec>
430 |                     <mkdir dir="${project.build.directory}/project-documentation/${git.remote.origin.url.suffix}"/>
431 |                     <copy todir="${project.build.directory}/project-documentation/${git.remote.origin.url.suffix}">
432 |                       <fileset file="${project.build.directory}/site-resources/index.html"/>
433 |                     </copy>
434 |                     <copy todir="${project.build.directory}/project-documentation/${git.remote.origin.url.suffix}/${project.version}">
435 |                       <fileset dir="${project.build.directory}/site"/>
436 |                     </copy>
437 |                     <exec dir="${project.build.directory}/project-documentation" executable="git">
438 |                       <arg value="add"/>
439 |                       <arg value="${git.remote.origin.url.suffix}/index.html"/>
440 |                     </exec>
441 |                     <exec dir="${project.build.directory}/project-documentation" executable="git">
442 |                       <arg value="add"/>
443 |                       <arg value="${git.remote.origin.url.suffix}/${project.version}"/>
444 |                     </exec>
445 |                     <exec dir="${project.build.directory}/project-documentation" executable="git">
446 |                       <arg value="commit"/>
447 |                       <arg value="-m"/>
448 |                       <arg value="Update Documentation: ${git.remote.origin.url.suffix}/${project.version}"/>
449 |                     </exec>
450 |                     <exec dir="${project.build.directory}/project-documentation" executable="git">
451 |                       <arg value="push"/>
452 |                       <arg value="javadoc"/>
453 |                       <arg value="trunk"/>
454 |                     </exec>
455 |                   </target>
456 |                 </configuration>
457 |               </execution>
458 |             </executions>
459 |           </plugin>
460 |         </plugins>
461 |       </build>
462 |     </profile>
463 | 
464 |   </profiles>
465 | 
466 | 
467 | </project>


--------------------------------------------------------------------------------
/src/main/java/edu/colorado/cires/cmg/s3out/ObjectMetadata.java:
--------------------------------------------------------------------------------
  1 | package edu.colorado.cires.cmg.s3out;
  2 | 
  3 | import java.time.Instant;
  4 | import java.util.Collections;
  5 | import java.util.HashMap;
  6 | import java.util.Map;
  7 | import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
  8 | 
  9 | /**
 10 |  * Implementation of {@link ObjectMetadataCustomizer} that allows setting most
 11 |  * AWS SDK object metadata through a builder.
 12 |  */
 13 | public class ObjectMetadata implements ObjectMetadataCustomizer {
 14 | 
 15 |   private final String acl;
 16 |   private final String cacheControl;
 17 |   private final String contentDisposition;
 18 |   private final String contentEncoding;
 19 |   private final String contentLanguage;
 20 |   private final String contentType;
 21 |   private final Instant expires;
 22 |   private final String grantFullControl;
 23 |   private final String grantRead;
 24 |   private final String grantReadACP;
 25 |   private final String grantWriteACP;
 26 |   private final Map<String, String> metadata;
 27 |   private final String serverSideEncryption;
 28 |   private final String storageClass;
 29 |   private final String websiteRedirectLocation;
 30 |   private final String sseCustomerAlgorithm;
 31 |   private final String sseCustomerKey;
 32 |   private final String sseCustomerKeyMD5;
 33 |   private final String ssekmsKeyId;
 34 |   private final String ssekmsEncryptionContext;
 35 |   private final Boolean bucketKeyEnabled;
 36 |   private final String requestPayer;
 37 |   private final String tagging;
 38 |   private final String objectLockMode;
 39 |   private final Instant objectLockRetainUntilDate;
 40 |   private final String objectLockLegalHoldStatus;
 41 |   private final String expectedBucketOwner;
 42 |   private final String checksumAlgorithm;
 43 | 
 44 |   /**
 45 |    * Returns a new builder for an {@link ObjectMetadata}
 46 |    * @return a new builder for an {@link ObjectMetadata}
 47 |    */
 48 |   public static Builder builder() {
 49 |     return new Builder();
 50 |   }
 51 | 
 52 |   private ObjectMetadata(
 53 |       String acl,
 54 |       String cacheControl,
 55 |       String contentDisposition,
 56 |       String contentEncoding,
 57 |       String contentLanguage,
 58 |       String contentType,
 59 |       Instant expires,
 60 |       String grantFullControl,
 61 |       String grantRead,
 62 |       String grantReadACP,
 63 |       String grantWriteACP,
 64 |       Map<String, String> metadata,
 65 |       String serverSideEncryption,
 66 |       String storageClass,
 67 |       String websiteRedirectLocation,
 68 |       String sseCustomerAlgorithm,
 69 |       String sseCustomerKey,
 70 |       String sseCustomerKeyMD5,
 71 |       String ssekmsKeyId,
 72 |       String ssekmsEncryptionContext,
 73 |       Boolean bucketKeyEnabled,
 74 |       String requestPayer,
 75 |       String tagging,
 76 |       String objectLockMode,
 77 |       Instant objectLockRetainUntilDate,
 78 |       String objectLockLegalHoldStatus,
 79 |       String expectedBucketOwner,
 80 |       String checksumAlgorithm) {
 81 |     this.acl = acl;
 82 |     this.cacheControl = cacheControl;
 83 |     this.contentDisposition = contentDisposition;
 84 |     this.contentEncoding = contentEncoding;
 85 |     this.contentLanguage = contentLanguage;
 86 |     this.contentType = contentType;
 87 |     this.expires = expires;
 88 |     this.grantFullControl = grantFullControl;
 89 |     this.grantRead = grantRead;
 90 |     this.grantReadACP = grantReadACP;
 91 |     this.grantWriteACP = grantWriteACP;
 92 |     this.metadata = metadata;
 93 |     this.serverSideEncryption = serverSideEncryption;
 94 |     this.storageClass = storageClass;
 95 |     this.websiteRedirectLocation = websiteRedirectLocation;
 96 |     this.sseCustomerAlgorithm = sseCustomerAlgorithm;
 97 |     this.sseCustomerKey = sseCustomerKey;
 98 |     this.sseCustomerKeyMD5 = sseCustomerKeyMD5;
 99 |     this.ssekmsKeyId = ssekmsKeyId;
100 |     this.ssekmsEncryptionContext = ssekmsEncryptionContext;
101 |     this.bucketKeyEnabled = bucketKeyEnabled;
102 |     this.requestPayer = requestPayer;
103 |     this.tagging = tagging;
104 |     this.objectLockMode = objectLockMode;
105 |     this.objectLockRetainUntilDate = objectLockRetainUntilDate;
106 |     this.objectLockLegalHoldStatus = objectLockLegalHoldStatus;
107 |     this.expectedBucketOwner = expectedBucketOwner;
108 |     this.checksumAlgorithm = checksumAlgorithm;
109 |   }
110 | 
111 |   /**
112 |    * Gets the canned ACL to apply to the object.
113 |    *
114 |    * @return the canned ACL to apply to the object.
115 |    */
116 |   public String getAcl() {
117 |     return acl;
118 |   }
119 | 
120 |   /**
121 |    * Gets the caching behavior along the request/reply chain.
122 |    *
123 |    * @return the caching behavior along the request/reply chain.
124 |    */
125 |   public String getCacheControl() {
126 |     return cacheControl;
127 |   }
128 | 
129 |   /**
130 |    * Gets presentational information for the object.
131 |    *
132 |    * @return presentational information for the object
133 |    */
134 |   public String getContentDisposition() {
135 |     return contentDisposition;
136 |   }
137 | 
138 |   /**
139 |    * Gets content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type
140 |    * referenced by the Content-Type header field.
141 |    *
142 |    * @return content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type
143 |    *    referenced by the Content-Type header field.
144 |    */
145 |   public String getContentEncoding() {
146 |     return contentEncoding;
147 |   }
148 | 
149 |   /**
150 |    * Gets the language the content is in.
151 |    *
152 |    * @return the language the content is in
153 |    */
154 |   public String getContentLanguage() {
155 |     return contentLanguage;
156 |   }
157 | 
158 |   /**
159 |    * Gets the MIME type describing the format of the object data.
160 |    *
161 |    * @return the MIME type describing the format of the object data
162 |    */
163 |   public String getContentType() {
164 |     return contentType;
165 |   }
166 | 
167 |   /**
168 |    * Gets the date and time at which the object is no longer cacheable.
169 |    *
170 |    * @return the date and time at which the object is no longer cacheable
171 |    */
172 |   public Instant getExpires() {
173 |     return expires;
174 |   }
175 | 
176 |   /**
177 |    * Gets the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
178 |    *
179 |    * @return the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
180 |    */
181 |   public String getGrantFullControl() {
182 |     return grantFullControl;
183 |   }
184 | 
185 |   /**
186 |    * Gets grantee state to read the object data and its metadata.
187 |    *
188 |    * @return grantee state to read the object data and its metadata.
189 |    */
190 |   public String getGrantRead() {
191 |     return grantRead;
192 |   }
193 | 
194 |   /**
195 |    * Gets grantee state to read the object ACL.
196 |    *
197 |    * @return Gets grantee state to read the object ACL
198 |    */
199 |   public String getGrantReadACP() {
200 |     return grantReadACP;
201 |   }
202 | 
203 |   /**
204 |    * Gets grantee state to write the ACL for the applicable object.
205 |    *
206 |    * @return grantee state to write the ACL for the applicable object
207 |    */
208 |   public String getGrantWriteACP() {
209 |     return grantWriteACP;
210 |   }
211 | 
212 |   /**
213 |    * Gets an unmodifiable map of metadata to store with the object in S3.
214 |    * @return an unmodifiable map of metadata to store with the object in S3
215 |    */
216 |   public Map<String, String> getMetadata() {
217 |     return metadata;
218 |   }
219 | 
220 |   /**
221 |    * Gets the server-side encryption algorithm used when storing this object in Amazon S3 (for example, AES256, aws:kms).
222 |    *
223 |    * @return the serve-side encryption algorithm used wrhen storing this object in Amazon S3 (for example, AES256, aws:kms)
224 |    */
225 |   public String getServerSideEncryption() {
226 |     return serverSideEncryption;
227 |   }
228 | 
229 |   /**
230 |    * Gets the Storage Class to store newly created objects.
231 |    * @return the Storage Class to store newly created objects
232 |    */
233 |   public String getStorageClass() {
234 |     return storageClass;
235 |   }
236 | 
237 |   /**
238 |    * Gets the redirects requests for this object.
239 |    *
240 |    * @return the redirects requests for this object.
241 |    */
242 |   public String getWebsiteRedirectLocation() {
243 |     return websiteRedirectLocation;
244 |   }
245 | 
246 |   /**
247 |    * Gets the algorithm to use to when encrypting the object (for example, AES256).
248 |    *
249 |    * @return the algorithm to use to when encrypting the object (for example, AES256).
250 |    */
251 |   public String getSseCustomerAlgorithm() {
252 |     return sseCustomerAlgorithm;
253 |   }
254 | 
255 |   /**
256 |    * Gets the customer-provided encryption key for Amazon S3 to use in encrypting data. This value is used to store the object and then it is
257 |    * discarded; Amazon S3 does not store the encryption key. The key must be appropriate for use with the algorithm specified in the
258 |    * x-amz-server-side-encryption-customer-algorithm header.
259 |    *
260 |    * @return the customer-provided encryption key for Amazon S3 to use in encrypting data. This value is used to store the object and then it is
261 |    *    discarded; Amazon S3 does not store the encryption key. The key must be appropriate for use with the algorithm specified in the
262 |    *    x-amz-server-side-encryption-customer-algorithm header.
263 |    */
264 |   public String getSseCustomerKey() {
265 |     return sseCustomerKey;
266 |   }
267 | 
268 |   /**
269 |    * Gets the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a message integrity check to
270 |    * ensure that the encryption key was transmitted without error.
271 |    *
272 |    * @return the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a message integrity check to
273 |    *    ensure that the encryption key was transmitted without error.
274 |    */
275 |   public String getSseCustomerKeyMD5() {
276 |     return sseCustomerKeyMD5;
277 |   }
278 | 
279 |   /**
280 |    * Gets the ID of the symmetric customer managed key to use for object encryption. All GET and PUT requests for an object protected by Amazon
281 |    * Web Services KMS will fail if not made via SSL or using SigV4.
282 |    *
283 |    * @return the ID of the symmetric customer managed key to use for object encryption. All GET and PUT requests for an object protected by Amazon
284 |    *    Web Services KMS will fail if not made via SSL or using SigV4.
285 |    */
286 |   public String getSsekmsKeyId() {
287 |     return ssekmsKeyId;
288 |   }
289 | 
290 |   /**
291 |    * Gets the Amazon Web Services KMS Encryption Context to use for object encryption. The value of this header is a base64-encoded UTF-8
292 |    * string holding JSON with the encryption context key-value pairs.
293 |    *
294 |    * @return the Amazon Web Services KMS Encryption Context to use for object encryption. The value of this header is a base64-encoded UTF-8
295 |    *    string holding JSON with the encryption context key-value pairs.
296 |    */
297 |   public String getSsekmsEncryptionContext() {
298 |     return ssekmsEncryptionContext;
299 |   }
300 | 
301 |   /**
302 |    * Gets whether Amazon S3 should use an S3 Bucket Key for object encryption with server-side encryption using AWS KMS (SSE-KMS). Setting this
303 |    * header to true causes Amazon S3 to use an S3 Bucket Key for object encryption with SSE-KMS.
304 |    *
305 |    * @return whether Amazon S3 should use an S3 Bucket Key for object encryption with server-side encryption using AWS KMS (SSE-KMS). Setting this
306 |    *    header to true causes Amazon S3 to use an S3 Bucket Key for object encryption with SSE-KMS.
307 |    */
308 |   public Boolean getBucketKeyEnabled() {
309 |     return bucketKeyEnabled;
310 |   }
311 | 
312 |   /**
313 |    * Gets the value of the RequestPayer property for this object.
314 |    *
315 |    * @return the value of the RequestPayer property for this object.
316 |    */
317 |   public String getRequestPayer() {
318 |     return requestPayer;
319 |   }
320 | 
321 |   /**
322 |    * Gets the tag-set for the object. The tag-set must be encoded as URL Query parameters.
323 |    *
324 |    * @return the tag-set for the object. The tag-set must be encoded as URL Query parameters.
325 |    */
326 |   public String getTagging() {
327 |     return tagging;
328 |   }
329 | 
330 |   /**
331 |    * Gets the Object Lock mode that you want to apply to the uploaded object.
332 |    *
333 |    * @return the Object Lock mode that you want to apply to the uploaded object.
334 |    */
335 |   public String getObjectLockMode() {
336 |     return objectLockMode;
337 |   }
338 | 
339 |   /**
340 |    * Gets the date and time when you want the Object Lock to expire.
341 |    *
342 |    * @return the date and time when you want the Object Lock to expire.
343 |    */
344 |   public Instant getObjectLockRetainUntilDate() {
345 |     return objectLockRetainUntilDate;
346 |   }
347 | 
348 |   /**
349 |    * Gets whether you want to apply a legal hold to the uploaded object.
350 |    *
351 |    * @return whether you want to apply a legal hold to the uploaded object.
352 |    */
353 |   public String getObjectLockLegalHoldStatus() {
354 |     return objectLockLegalHoldStatus;
355 |   }
356 | 
357 |   /**
358 |    * Gets the account ID of the expected bucket owner. If the bucket is owned by a different account, the request fails with the HTTP status code 403
359 |    * Forbidden (access denied).
360 |    *
361 |    * @return the account ID of the expected bucket owner. If the bucket is owned by a different account, the request fails with the HTTP status code 403
362 |    *    Forbidden (access denied).
363 |    */
364 |   public String getExpectedBucketOwner() {
365 |     return expectedBucketOwner;
366 |   }
367 | 
368 |   /**
369 |    * Get the algorithm you want Amazon S3 to use to create the checksum for the object.
370 |    *
371 |    * @return the algorithm you want Amazon S3 to use to create the checksum for the object
372 |    */
373 |   public String getChecksumAlgorithm() {
374 |     return checksumAlgorithm;
375 |   }
376 | 
377 |   @Override
378 |   public void apply(CreateMultipartUploadRequest.Builder builder) {
379 |     if (acl != null) {
380 |       builder.acl(acl);
381 |     }
382 |     if (cacheControl != null) {
383 |       builder.cacheControl(cacheControl);
384 |     }
385 |     if (contentDisposition != null) {
386 |       builder.contentDisposition(contentDisposition);
387 |     }
388 |     if (contentEncoding != null) {
389 |       builder.contentEncoding(contentEncoding);
390 |     }
391 |     if (contentLanguage != null) {
392 |       builder.contentLanguage(contentLanguage);
393 |     }
394 |     if (contentType != null) {
395 |       builder.contentType(contentType);
396 |     }
397 |     if (expires != null) {
398 |       builder.expires(expires);
399 |     }
400 |     if (grantFullControl != null) {
401 |       builder.grantFullControl(grantFullControl);
402 |     }
403 |     if (grantRead != null) {
404 |       builder.grantRead(grantRead);
405 |     }
406 |     if (grantReadACP != null) {
407 |       builder.grantReadACP(grantReadACP);
408 |     }
409 |     if (grantWriteACP != null) {
410 |       builder.grantWriteACP(grantWriteACP);
411 |     }
412 |     if (!metadata.isEmpty()) {
413 |       builder.metadata(metadata);
414 |     }
415 |     if (serverSideEncryption != null) {
416 |       builder.serverSideEncryption(serverSideEncryption);
417 |     }
418 |     if (storageClass != null) {
419 |       builder.storageClass(storageClass);
420 |     }
421 |     if (websiteRedirectLocation != null) {
422 |       builder.websiteRedirectLocation(websiteRedirectLocation);
423 |     }
424 |     if (sseCustomerAlgorithm != null) {
425 |       builder.sseCustomerAlgorithm(sseCustomerAlgorithm);
426 |     }
427 |     if (sseCustomerKey != null) {
428 |       builder.sseCustomerKey(sseCustomerKey);
429 |     }
430 |     if (sseCustomerKeyMD5 != null) {
431 |       builder.sseCustomerKeyMD5(sseCustomerKeyMD5);
432 |     }
433 |     if (ssekmsKeyId != null) {
434 |       builder.ssekmsKeyId(ssekmsKeyId);
435 |     }
436 |     if (ssekmsEncryptionContext != null) {
437 |       builder.ssekmsEncryptionContext(ssekmsEncryptionContext);
438 |     }
439 |     if (bucketKeyEnabled != null) {
440 |       builder.bucketKeyEnabled(bucketKeyEnabled);
441 |     }
442 |     if (requestPayer != null) {
443 |       builder.requestPayer(requestPayer);
444 |     }
445 |     if (tagging != null) {
446 |       builder.tagging(tagging);
447 |     }
448 |     if (objectLockMode != null) {
449 |       builder.objectLockMode(objectLockMode);
450 |     }
451 |     if (objectLockRetainUntilDate != null) {
452 |       builder.objectLockRetainUntilDate(objectLockRetainUntilDate);
453 |     }
454 |     if (objectLockLegalHoldStatus != null) {
455 |       builder.objectLockLegalHoldStatus(objectLockLegalHoldStatus);
456 |     }
457 |     if (expectedBucketOwner != null) {
458 |       builder.expectedBucketOwner(expectedBucketOwner);
459 |     }
460 |     if (checksumAlgorithm != null) {
461 |       builder.checksumAlgorithm(checksumAlgorithm);
462 |     }
463 |   }
464 | 
465 |   /**
466 |    * Builder for an {@link ObjectMetadata}
467 |    */
468 |   public static class Builder {
469 | 
470 |     private Map<String, String> metadata = new HashMap<>();
471 |     private String acl;
472 |     private String cacheControl;
473 |     private String contentDisposition;
474 |     private String contentEncoding;
475 |     private String contentLanguage;
476 |     private String contentType;
477 |     private Instant expires;
478 |     private String grantFullControl;
479 |     private String grantRead;
480 |     private String grantReadACP;
481 |     private String grantWriteACP;
482 |     private String serverSideEncryption;
483 |     private String storageClass;
484 |     private String websiteRedirectLocation;
485 |     private String sseCustomerAlgorithm;
486 |     private String sseCustomerKey;
487 |     private String sseCustomerKeyMD5;
488 |     private String ssekmsKeyId;
489 |     private String ssekmsEncryptionContext;
490 |     private Boolean bucketKeyEnabled;
491 |     private String requestPayer;
492 |     private String tagging;
493 |     private String objectLockMode;
494 |     private Instant objectLockRetainUntilDate;
495 |     private String objectLockLegalHoldStatus;
496 |     private String expectedBucketOwner;
497 |     private String checksumAlgorithm;
498 | 
499 |     /**
500 |      * Sets the canned ACL to apply to the object.
501 |      *
502 |      * @param acl The canned ACL to apply to the object
503 |      * @return this Builder
504 |      */
505 |     public Builder acl(String acl) {
506 |       this.acl = acl;
507 |       return this;
508 |     }
509 | 
510 |     /**
511 |      * Specifies caching behavior along the request/reply chain.
512 |      *
513 |      * @param cacheControl Specifies caching behavior along the request/reply chain.
514 |      * @return this Builder
515 |      */
516 |     public Builder cacheControl(String cacheControl) {
517 |       this.cacheControl = cacheControl;
518 |       return this;
519 |     }
520 | 
521 |     /**
522 |      * Specifies presentational information for the object.
523 |      *
524 |      * @param contentDisposition Specifies presentational information for the object.
525 |      * @return this Builder
526 |      */
527 |     public Builder contentDisposition(String contentDisposition) {
528 |       this.contentDisposition = contentDisposition;
529 |       return this;
530 |     }
531 | 
532 |     /**
533 |      * Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to obtain the media-type
534 |      * referenced by the Content-Type header field.
535 |      *
536 |      * @param contentEncoding Specifies what content encodings have been applied to the object and thus what decoding mechanisms must be applied to
537 |      * obtain the media-type referenced by the Content-Type header field.
538 |      * @return this Builder
539 |      */
540 |     public Builder contentEncoding(String contentEncoding) {
541 |       this.contentEncoding = contentEncoding;
542 |       return this;
543 |     }
544 | 
545 |     /**
546 |      * The language the content is in.
547 |      *
548 |      * @param contentLanguage The language the content is in.
549 |      * @return this Builder
550 |      */
551 |     public Builder contentLanguage(String contentLanguage) {
552 |       this.contentLanguage = contentLanguage;
553 |       return this;
554 |     }
555 | 
556 |     /**
557 |      * A standard MIME type describing the format of the object data. Note: Setting this will override any value set by a {@link
558 |      * ContentTypeResolver}.
559 |      *
560 |      * @param contentType A standard MIME type describing the format of the object data.
561 |      * @return this Builder
562 |      */
563 |     public Builder contentType(String contentType) {
564 |       this.contentType = contentType;
565 |       return this;
566 |     }
567 | 
568 |     /**
569 |      * The date and time at which the object is no longer cacheable.
570 |      *
571 |      * @param expires The date and time at which the object is no longer cacheable.
572 |      * @return this Builder
573 |      */
574 |     public Builder expires(Instant expires) {
575 |       this.expires = expires;
576 |       return this;
577 |     }
578 | 
579 |     /**
580 |      * Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
581 |      *
582 |      * @param grantFullControl Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
583 |      * @return this Builder
584 |      */
585 |     public Builder grantFullControl(String grantFullControl) {
586 |       this.grantFullControl = grantFullControl;
587 |       return this;
588 |     }
589 | 
590 |     /**
591 |      * Allows grantee to read the object data and its metadata.
592 |      *
593 |      * @param grantRead Allows grantee to read the object data and its metadata.
594 |      * @return this Builder
595 |      */
596 |     public Builder grantRead(String grantRead) {
597 |       this.grantRead = grantRead;
598 |       return this;
599 |     }
600 | 
601 |     /**
602 |      * Allows grantee to read the object ACL.
603 |      *
604 |      * @param grantReadACP Allows grantee to read the object ACL.
605 |      * @return this Builder
606 |      */
607 |     public Builder grantReadACP(String grantReadACP) {
608 |       this.grantReadACP = grantReadACP;
609 |       return this;
610 |     }
611 | 
612 |     /**
613 |      * Allows grantee to write the ACL for the applicable object.
614 |      *
615 |      * @param grantWriteACP Allows grantee to write the ACL for the applicable object.
616 |      * @return this Builder
617 |      */
618 |     public Builder grantWriteACP(String grantWriteACP) {
619 |       this.grantWriteACP = grantWriteACP;
620 |       return this;
621 |     }
622 | 
623 |     /**
624 |      * A map of metadata to store with the object in S3. Note: setting this clears any values set by {@link #metadata(String, String)}
625 |      *
626 |      * @param metadata A map of metadata to store with the object in S3.
627 |      * @return this Builder
628 |      */
629 |     public Builder metadata(Map<String, String> metadata) {
630 |       if (metadata == null) {
631 |         metadata = new HashMap<>();
632 |       }
633 |       this.metadata = new HashMap<>(metadata);
634 |       return this;
635 |     }
636 | 
637 |     /**
638 |      * Adds to the map of metadata to store with the object in S3.
639 |      *
640 |      * @param key the metadata key
641 |      * @param value the metadata value
642 |      * @return this Builder
643 |      */
644 |     public Builder metadata(String key, String value) {
645 |       this.metadata.put(key, value);
646 |       return this;
647 |     }
648 | 
649 |     /**
650 |      * The server-side encryption algorithm used when storing this object in Amazon S3 (for example, AES256, aws:kms).
651 |      *
652 |      * @param serverSideEncryption The server-side encryption algorithm used when storing this object in Amazon S3 (for example, AES256, aws:kms).
653 |      * @return this Builder
654 |      */
655 |     public Builder serverSideEncryption(String serverSideEncryption) {
656 |       this.serverSideEncryption = serverSideEncryption;
657 |       return this;
658 |     }
659 | 
660 |     /**
661 |      * By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides high durability and
662 |      * high availability. Depending on performance needs, you can specify a different Storage Class.
663 |      *
664 |      * @param storageClass By default, Amazon S3 uses the STANDARD Storage Class to store newly created objects. The STANDARD storage class provides
665 |      * high durability and high availability. Depending on performance needs, you can specify a different Storage Class.
666 |      * @return this Builder
667 |      */
668 |     public Builder storageClass(String storageClass) {
669 |       this.storageClass = storageClass;
670 |       return this;
671 |     }
672 | 
673 |     /**
674 |      * If the bucket is configured as a website, redirects requests for this object to another object in the same bucket or to an external URL. Amazon
675 |      * S3 stores the value of this header in the object metadata.
676 |      *
677 |      * @param websiteRedirectLocation If the bucket is configured as a website, redirects requests for this object to another object in the same
678 |      * bucket or to an external URL. Amazon S3 stores the value of this header in the object metadata.
679 |      * @return this Builder
680 |      */
681 |     public Builder websiteRedirectLocation(String websiteRedirectLocation) {
682 |       this.websiteRedirectLocation = websiteRedirectLocation;
683 |       return this;
684 |     }
685 | 
686 |     /**
687 |      * Specifies the algorithm to use to when encrypting the object (for example, AES256).
688 |      *
689 |      * @param sseCustomerAlgorithm Specifies the algorithm to use to when encrypting the object (for example, AES256).
690 |      * @return this Builder
691 |      */
692 |     public Builder sseCustomerAlgorithm(String sseCustomerAlgorithm) {
693 |       this.sseCustomerAlgorithm = sseCustomerAlgorithm;
694 |       return this;
695 |     }
696 | 
697 |     /**
698 |      * Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data. This value is used to store the object and then it is
699 |      * discarded; Amazon S3 does not store the encryption key. The key must be appropriate for use with the algorithm specified in the
700 |      * x-amz-server-side-encryption-customer-algorithm header.
701 |      *
702 |      * @param sseCustomerKey Specifies the customer-provided encryption key for Amazon S3 to use in encrypting data. This value is used to store the
703 |      * object and then it is discarded; Amazon S3 does not store the encryption key. The key must be appropriate for use with the algorithm specified
704 |      * in the x-amz-server-side-encryption-customer-algorithm header.
705 |      * @return this Builder
706 |      */
707 |     public Builder sseCustomerKey(String sseCustomerKey) {
708 |       this.sseCustomerKey = sseCustomerKey;
709 |       return this;
710 |     }
711 | 
712 |     /**
713 |      * Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a message integrity check to
714 |      * ensure that the encryption key was transmitted without error.
715 |      *
716 |      * @param sseCustomerKeyMD5 Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321. Amazon S3 uses this header for a message
717 |      * integrity check to ensure that the encryption key was transmitted without error.
718 |      * @return this Builder
719 |      */
720 |     public Builder sseCustomerKeyMD5(String sseCustomerKeyMD5) {
721 |       this.sseCustomerKeyMD5 = sseCustomerKeyMD5;
722 |       return this;
723 |     }
724 | 
725 |     /**
726 |      * Specifies the ID of the symmetric customer managed key to use for object encryption. All GET and PUT requests for an object protected by Amazon
727 |      * Web Services KMS will fail if not made via SSL or using SigV4.
728 |      *
729 |      * @param ssekmsKeyId Specifies the ID of the symmetric customer managed key to use for object encryption. All GET and PUT requests for an object
730 |      * protected by Amazon Web Services KMS will fail if not made via SSL or using SigV4.
731 |      * @return this Builder
732 |      */
733 |     public Builder ssekmsKeyId(String ssekmsKeyId) {
734 |       this.ssekmsKeyId = ssekmsKeyId;
735 |       return this;
736 |     }
737 | 
738 |     /**
739 |      * Specifies the Amazon Web Services KMS Encryption Context to use for object encryption. The value of this header is a base64-encoded UTF-8
740 |      * string holding JSON with the encryption context key-value pairs.
741 |      *
742 |      * @param ssekmsEncryptionContext Specifies the Amazon Web Services KMS Encryption Context to use for object encryption. The value of this header
743 |      * is a base64-encoded UTF-8 string holding JSON with the encryption context key-value pairs.
744 |      * @return this Builder
745 |      */
746 |     public Builder ssekmsEncryptionContext(String ssekmsEncryptionContext) {
747 |       this.ssekmsEncryptionContext = ssekmsEncryptionContext;
748 |       return this;
749 |     }
750 | 
751 |     /**
752 |      * Specifies whether Amazon S3 should use an S3 Bucket Key for object encryption with server-side encryption using AWS KMS (SSE-KMS). Setting this
753 |      * header to true causes Amazon S3 to use an S3 Bucket Key for object encryption with SSE-KMS.
754 |      *
755 |      * @param bucketKeyEnabled Specifies whether Amazon S3 should use an S3 Bucket Key for object encryption with server-side encryption using AWS KMS
756 |      * (SSE-KMS). Setting this header to true causes Amazon S3 to use an S3 Bucket Key for object encryption with SSE-KMS.
757 |      * @return this Builder
758 |      */
759 |     public Builder bucketKeyEnabled(Boolean bucketKeyEnabled) {
760 |       this.bucketKeyEnabled = bucketKeyEnabled;
761 |       return this;
762 |     }
763 | 
764 |     /**
765 |      * Sets the value of the RequestPayer property for this object.
766 |      *
767 |      * @param requestPayer The new value for the RequestPayer property for this object.
768 |      * @return this Builder
769 |      */
770 |     public Builder requestPayer(String requestPayer) {
771 |       this.requestPayer = requestPayer;
772 |       return this;
773 |     }
774 | 
775 |     /**
776 |      * The tag-set for the object. The tag-set must be encoded as URL Query parameters.
777 |      *
778 |      * @param tagging The tag-set for the object. The tag-set must be encoded as URL Query parameters.
779 |      * @return this Builder
780 |      */
781 |     public Builder tagging(String tagging) {
782 |       this.tagging = tagging;
783 |       return this;
784 |     }
785 | 
786 |     /**
787 |      * Specifies the Object Lock mode that you want to apply to the uploaded object.
788 |      *
789 |      * @param objectLockMode Specifies the Object Lock mode that you want to apply to the uploaded object.
790 |      * @return this Builder
791 |      */
792 |     public Builder objectLockMode(String objectLockMode) {
793 |       this.objectLockMode = objectLockMode;
794 |       return this;
795 |     }
796 | 
797 |     /**
798 |      * Specifies the date and time when you want the Object Lock to expire.
799 |      *
800 |      * @param objectLockRetainUntilDate Specifies the date and time when you want the Object Lock to expire.
801 |      * @return this Builder
802 |      */
803 |     public Builder objectLockRetainUntilDate(Instant objectLockRetainUntilDate) {
804 |       this.objectLockRetainUntilDate = objectLockRetainUntilDate;
805 |       return this;
806 |     }
807 | 
808 |     /**
809 |      * Specifies whether you want to apply a legal hold to the uploaded object.
810 |      *
811 |      * @param objectLockLegalHoldStatus Specifies whether you want to apply a legal hold to the uploaded object.
812 |      * @return this Builder
813 |      */
814 |     public Builder objectLockLegalHoldStatus(String objectLockLegalHoldStatus) {
815 |       this.objectLockLegalHoldStatus = objectLockLegalHoldStatus;
816 |       return this;
817 |     }
818 | 
819 |     /**
820 |      * The account ID of the expected bucket owner. If the bucket is owned by a different account, the request fails with the HTTP status code 403
821 |      * Forbidden (access denied).
822 |      *
823 |      * @param expectedBucketOwner The account ID of the expected bucket owner. If the bucket is owned by a different account, the request fails with
824 |      * the HTTP status code 403 Forbidden (access denied).
825 |      * @return this Builder
826 |      */
827 |     public Builder expectedBucketOwner(String expectedBucketOwner) {
828 |       this.expectedBucketOwner = expectedBucketOwner;
829 |       return this;
830 |     }
831 | 
832 |     /**
833 |      * Indicates the algorithm you want Amazon S3 to use to create the checksum for the object.
834 |      *
835 |      * @param checksumAlgorithm Indicates the algorithm you want Amazon S3 to use to create the checksum for the object.
836 |      * @return this Builder
837 |      * @deprecated Use {@link MultipartUploadRequest.Builder#checksumAlgorithm(String)} instead
838 |      */
839 |     @Deprecated
840 |     public Builder checksumAlgorithm(String checksumAlgorithm) {
841 |       this.checksumAlgorithm = checksumAlgorithm;
842 |       return this;
843 |     }
844 | 
845 |     /**
846 |      * Builds a new {@link ObjectMetadata} object
847 |      *
848 |      * @return a new {@link ObjectMetadata} object
849 |      */
850 |     public ObjectMetadata build() {
851 |       return new ObjectMetadata(
852 |           acl,
853 |           cacheControl,
854 |           contentDisposition,
855 |           contentEncoding,
856 |           contentLanguage,
857 |           contentType,
858 |           expires,
859 |           grantFullControl,
860 |           grantRead,
861 |           grantReadACP,
862 |           grantWriteACP,
863 |           Collections.unmodifiableMap(metadata),
864 |           serverSideEncryption,
865 |           storageClass,
866 |           websiteRedirectLocation,
867 |           sseCustomerAlgorithm,
868 |           sseCustomerKey,
869 |           sseCustomerKeyMD5,
870 |           ssekmsKeyId,
871 |           ssekmsEncryptionContext,
872 |           bucketKeyEnabled,
873 |           requestPayer,
874 |           tagging,
875 |           objectLockMode,
876 |           objectLockRetainUntilDate,
877 |           objectLockLegalHoldStatus,
878 |           expectedBucketOwner,
879 |           checksumAlgorithm);
880 |     }
881 |   }
882 | }
883 | 


--------------------------------------------------------------------------------