sessionToken();
15 |
16 | static Credentials of(String accessKey, String secretKey) {
17 | return new CredentialsImpl(accessKey, secretKey, Optional.empty());
18 | }
19 |
20 | static Credentials of(String accessKey, String secretKey, String sessionToken) {
21 | return new CredentialsImpl(accessKey, secretKey, Optional.of(sessionToken));
22 | }
23 |
24 | static Credentials fromEnvironment() {
25 | return Environment.instance().credentials();
26 | }
27 |
28 | static Credentials fromSystemProperties() {
29 | return new CredentialsImpl(System.getProperty("aws.accessKeyId"),
30 | System.getProperty("aws.secretKey"),
31 | Optional.empty());
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/ExceptionFactory.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.util.Optional;
4 |
5 | import com.github.davidmoten.aws.lw.client.internal.ExceptionFactoryDefault;
6 |
7 | @FunctionalInterface
8 | public interface ExceptionFactory {
9 |
10 | /**
11 | * Returns a {@link RuntimeException} (or subclass) if the response error
12 | * condition is met (usually {@code !response.isOk()}. If no exception to be
13 | * thrown then returns {@code Optional.empty()}.
14 | *
15 | * @param response response to map into exception
16 | * @return optional runtime exception
17 | */
18 | Optional create(Response response);
19 |
20 | ExceptionFactory DEFAULT = new ExceptionFactoryDefault();
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/Formats.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.time.format.DateTimeFormatter;
4 | import java.util.Locale;
5 |
6 | public final class Formats {
7 |
8 | private Formats() {
9 | // prevent instantiation
10 | }
11 |
12 | /**
13 | * A common date-time format used in the AWS API. For example {@code Last-Modified}
14 | * header for an S3 object uses this format.
15 | *
16 | *
17 | * See Common
19 | * Request Headers and Common
21 | * Response Headers .
22 | */
23 | public static final DateTimeFormatter FULL_DATE = DateTimeFormatter //
24 | .ofPattern("EEE, d MMM yyyy HH:mm:ss z", Locale.ENGLISH);
25 |
26 | }
27 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/HttpClient.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.IOException;
4 | import java.net.URL;
5 | import java.util.Map;
6 |
7 | import com.github.davidmoten.aws.lw.client.internal.HttpClientDefault;
8 |
9 | public interface HttpClient {
10 |
11 | ResponseInputStream request(URL endpointUrl, String httpMethod, Map headers,
12 | byte[] requestBody, int connectTimeoutMs, int readTimeoutMs) throws IOException;
13 |
14 | static HttpClient defaultClient() {
15 | return HttpClientDefault.INSTANCE;
16 | }
17 | }
18 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/HttpMethod.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public enum HttpMethod {
4 |
5 | GET, PUT, PATCH, DELETE, POST, HEAD;
6 |
7 | }
8 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/MaxAttemptsExceededException.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public final class MaxAttemptsExceededException extends RuntimeException {
4 |
5 | /**
6 | *
7 | */
8 | private static final long serialVersionUID = -5945914615129555985L;
9 |
10 | public MaxAttemptsExceededException(String message, Throwable e) {
11 | super(message, e);
12 | }
13 |
14 | }
15 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/Metadata.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.util.Map;
4 | import java.util.Map.Entry;
5 | import java.util.Optional;
6 | import java.util.Set;
7 |
8 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
9 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
10 |
11 | public final class Metadata {
12 |
13 | private final Map map;
14 |
15 | Metadata(Map map) {
16 | this.map = map;
17 | }
18 |
19 | public Optional value(String key) {
20 | Preconditions.checkNotNull(key);
21 | return Optional.ofNullable(map.get(Util.canonicalMetadataKey(key)));
22 | }
23 |
24 | public Set> entrySet() {
25 | return map.entrySet();
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/Multipart.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.BufferedInputStream;
4 | import java.io.File;
5 | import java.io.FileInputStream;
6 | import java.io.IOException;
7 | import java.io.InputStream;
8 | import java.io.OutputStream;
9 | import java.io.UncheckedIOException;
10 | import java.util.concurrent.Callable;
11 | import java.util.concurrent.ExecutorService;
12 | import java.util.concurrent.Executors;
13 | import java.util.concurrent.TimeUnit;
14 | import java.util.function.Function;
15 |
16 | import com.github.davidmoten.aws.lw.client.internal.Retries;
17 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
18 |
19 | public final class Multipart {
20 |
21 | private Multipart() {
22 | // prevent instantiation
23 | }
24 |
25 | public static Builder s3(Client s3) {
26 | Preconditions.checkNotNull(s3);
27 | return new Builder(s3);
28 | }
29 |
30 | public static final class Builder {
31 |
32 | private final Client s3;
33 | private String bucket;
34 | public String key;
35 | public ExecutorService executor;
36 | public long timeoutMs = TimeUnit.HOURS.toMillis(1);
37 | public Function transform = x -> x;
38 | public int partSize = 5 * 1024 * 1024;
39 | public Retries retries;
40 |
41 | Builder(Client s3) {
42 | this.s3 = s3;
43 | this.retries = s3.retries().withValueShouldRetry(values -> false);
44 | }
45 |
46 | public Builder2 bucket(String bucket) {
47 | Preconditions.checkNotNull(bucket, "bucket cannot be null");
48 | this.bucket = bucket;
49 | return new Builder2(this);
50 | }
51 | }
52 |
53 | public static final class Builder2 {
54 |
55 | private final Builder b;
56 |
57 | Builder2(Builder b) {
58 | this.b = b;
59 | }
60 |
61 | public Builder3 key(String key) {
62 | Preconditions.checkNotNull(key, "key cannot be null");
63 | b.key = key;
64 | return new Builder3(b);
65 | }
66 | }
67 |
68 | public static final class Builder3 {
69 |
70 | private final Builder b;
71 |
72 | Builder3(Builder b) {
73 | this.b = b;
74 | }
75 |
76 | public Builder3 executor(ExecutorService executor) {
77 | Preconditions.checkNotNull(executor, "executor cannot be null");
78 | b.executor = executor;
79 | return this;
80 | }
81 |
82 | public Builder3 partTimeout(long duration, TimeUnit unit) {
83 | Preconditions.checkArgument(duration > 0, "duration must be positive");
84 | Preconditions.checkNotNull(unit, "unit cannot be null");
85 | b.timeoutMs = unit.toMillis(duration);
86 | return this;
87 | }
88 |
89 | public Builder3 partSize(int partSize) {
90 | Preconditions.checkArgument(partSize >= 5 * 1024 * 1024);
91 | b.partSize = partSize;
92 | return this;
93 | }
94 |
95 | public Builder3 partSizeMb(int partSizeMb) {
96 | return partSize(partSizeMb * 1024 * 1024);
97 | }
98 |
99 | public Builder3 maxAttemptsPerAction(int maxAttempts) {
100 | Preconditions.checkArgument(maxAttempts >= 1, "maxAttempts must be at least one");
101 | b.retries = b.retries.withMaxAttempts(maxAttempts);
102 | return this;
103 | }
104 |
105 | public Builder3 retryInitialInterval(long duration, TimeUnit unit) {
106 | Preconditions.checkArgument(duration >= 0, "duration cannot be negative");
107 | Preconditions.checkNotNull(unit, "unit cannot be null");
108 | b.retries = b.retries.withInitialIntervalMs(unit.toMillis(duration));
109 | return this;
110 | }
111 |
112 | public Builder3 retryBackoffFactor(double factor) {
113 | Preconditions.checkArgument(factor >= 0, "retryBackoffFactory cannot be negative");
114 | b.retries = b.retries.withBackoffFactor(factor);
115 | return this;
116 | }
117 |
118 | public Builder3 retryMaxInterval(long duration, TimeUnit unit) {
119 | Preconditions.checkArgument(duration >= 0, "duration cannot be negative");
120 | Preconditions.checkNotNull(unit, "unit cannot be null");
121 | b.retries = b.retries.withMaxIntervalMs(unit.toMillis(duration));
122 | return this;
123 | }
124 |
125 | /**
126 | * Sets the level of randomness applied to the next retry interval. The next
127 | * calculated retry interval is multiplied by
128 | * {@code (1 - jitter * Math.random())}. A value of zero means no jitter, 1
129 | * means max jitter.
130 | *
131 | * @param jitter level of randomness applied to the retry interval
132 | * @return this
133 | */
134 | public Builder3 retryJitter(double jitter) {
135 | Preconditions.checkArgument(jitter >= 0 && jitter <= 1, "jitter must be between 0 and 1");
136 | b.retries = b.retries.withJitter(jitter);
137 | return this;
138 | }
139 |
140 |
141 | public Builder3 transformCreateRequest(Function transform) {
142 | Preconditions.checkNotNull(transform, "transform cannot be null");
143 | b.transform = transform;
144 | return this;
145 | }
146 |
147 | public void upload(byte[] bytes, int offset, int length) {
148 | Preconditions.checkNotNull(bytes, "bytes cannot be null");
149 | try (OutputStream out = outputStream()) {
150 | out.write(bytes, offset, length);
151 | } catch (IOException e) {
152 | throw new UncheckedIOException(e);
153 | }
154 | }
155 |
156 | public void upload(byte[] bytes) {
157 | upload(bytes, 0, bytes.length);
158 | }
159 |
160 | public void upload(File file) {
161 | Preconditions.checkNotNull(file, "file cannot be null");
162 | upload(() -> new BufferedInputStream(new FileInputStream(file)));
163 | }
164 |
165 | public void upload(Callable factory) {
166 | Preconditions.checkNotNull(factory, "factory cannot be null");
167 | try (InputStream in = factory.call(); MultipartOutputStream out = outputStream()) {
168 | copy(in, out);
169 | } catch (IOException e) {
170 | throw new UncheckedIOException(e);
171 | } catch (Exception e) {
172 | throw new RuntimeException(e);
173 | }
174 | }
175 |
176 | public MultipartOutputStream outputStream() {
177 | if (b.executor == null) {
178 | b.executor = Executors.newCachedThreadPool();
179 | }
180 | return new MultipartOutputStream(b.s3, b.bucket, b.key, b.transform, b.executor, b.timeoutMs, b.retries,
181 | b.partSize);
182 | }
183 | }
184 |
185 | private static void copy(InputStream in, OutputStream out) throws IOException {
186 | byte[] buffer = new byte[8192];
187 | int n;
188 | while ((n = in.read(buffer)) != -1) {
189 | out.write(buffer, 0, n);
190 | }
191 | }
192 | }
193 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/MultipartOutputStream.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.ByteArrayOutputStream;
4 | import java.io.IOException;
5 | import java.io.OutputStream;
6 | import java.util.List;
7 | import java.util.concurrent.Callable;
8 | import java.util.concurrent.CopyOnWriteArrayList;
9 | import java.util.concurrent.ExecutorService;
10 | import java.util.concurrent.Future;
11 | import java.util.concurrent.TimeUnit;
12 | import java.util.function.Function;
13 | import java.util.stream.Collectors;
14 |
15 | import com.github.davidmoten.aws.lw.client.internal.Retries;
16 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
17 | import com.github.davidmoten.aws.lw.client.xml.builder.Xml;
18 |
19 | // NotThreadSafe
20 | public final class MultipartOutputStream extends OutputStream {
21 |
22 | private final Client s3;
23 | private final String bucket;
24 | private final String key;
25 | private final String uploadId;
26 | private final ExecutorService executor;
27 | private final ByteArrayOutputStream bytes;
28 | private final byte[] singleByte = new byte[1]; // for reuse in write(int) method
29 | private final long partTimeoutMs;
30 | private final Retries retries;
31 | private final int partSize;
32 | private final List> futures = new CopyOnWriteArrayList<>();
33 | private int nextPart = 1;
34 |
35 | MultipartOutputStream(Client s3, String bucket, String key,
36 | Function transformCreate, ExecutorService executor,
37 | long partTimeoutMs, Retries retries, int partSize) {
38 | Preconditions.checkNotNull(s3);
39 | Preconditions.checkNotNull(bucket);
40 | Preconditions.checkNotNull(key);
41 | Preconditions.checkNotNull(transformCreate);
42 | Preconditions.checkNotNull(executor);
43 | Preconditions.checkArgument(partTimeoutMs > 0);
44 | Preconditions.checkNotNull(retries);
45 | Preconditions.checkArgument(partSize >= 5 * 1024 * 1024);
46 | this.s3 = s3;
47 | this.bucket = bucket;
48 | this.key = key;
49 | this.executor = executor;
50 | this.partTimeoutMs = partTimeoutMs;
51 | this.retries = retries;
52 | this.partSize = partSize;
53 | this.bytes = new ByteArrayOutputStream();
54 | this.uploadId = transformCreate.apply(s3 //
55 | .path(bucket, key) //
56 | .query("uploads") //
57 | .method(HttpMethod.POST)) //
58 | .responseAsXml() //
59 | .content("UploadId");
60 | }
61 |
62 | public void abort() {
63 | futures.forEach(f -> f.cancel(true));
64 | s3 //
65 | .path(bucket, key) //
66 | .query("uploadId", uploadId) //
67 | .method(HttpMethod.DELETE) //
68 | .execute();
69 | }
70 |
71 | @Override
72 | public void write(byte[] b, int off, int len) throws IOException {
73 | while (len > 0) {
74 | int remaining = partSize - bytes.size();
75 | int n = Math.min(remaining, len);
76 | bytes.write(b, off, n);
77 | off += n;
78 | len -= n;
79 | if (bytes.size() == partSize) {
80 | submitPart();
81 | }
82 | }
83 | }
84 |
85 | @Override
86 | public void write(byte[] b) throws IOException {
87 | write(b, 0, b.length);
88 | }
89 |
90 | private void submitPart() {
91 | int part = nextPart;
92 | nextPart++;
93 | byte[] body = bytes.toByteArray();
94 | bytes.reset();
95 | Future future = executor.submit(() -> retry(() -> s3 //
96 | .path(bucket, key) //
97 | .method(HttpMethod.PUT) //
98 | .query("partNumber", "" + part) //
99 | .query("uploadId", uploadId) //
100 | .requestBody(body) //
101 | .readTimeout(partTimeoutMs, TimeUnit.MILLISECONDS) //
102 | .responseExpectStatusCode(200) //
103 | .firstHeader("ETag") //
104 | .get() //
105 | .replace("\"", ""), //
106 | "on part " + part));
107 | futures.add(future);
108 | }
109 |
110 | private T retry(Callable callable, String description) {
111 | //TODO use description
112 | return retries.call(callable, x -> false);
113 | }
114 |
115 | @Override
116 | public void close() throws IOException {
117 | // submit whatever's left
118 | if (bytes.size() > 0) {
119 | submitPart();
120 | }
121 | List etags = futures //
122 | .stream() //
123 | .map(future -> getResult(future)) //
124 | .collect(Collectors.toList());
125 |
126 | Xml xml = Xml //
127 | .create("CompleteMultipartUpload") //
128 | .attribute("xmlns", "http:s3.amazonaws.com/doc/2006-03-01/");
129 | for (int i = 0; i < etags.size(); i++) {
130 | xml = xml //
131 | .element("Part") //
132 | .element("ETag").content(etags.get(i)) //
133 | .up() //
134 | .element("PartNumber").content(String.valueOf(i + 1)) //
135 | .up().up();
136 | }
137 | String xmlFinal = xml.toString();
138 | retry(() -> {
139 | s3.path(bucket, key) //
140 | .method(HttpMethod.POST) //
141 | .query("uploadId", uploadId) //
142 | .header("Content-Type", "application/xml") //
143 | .unsignedPayload() //
144 | .requestBody(xmlFinal) //
145 | .execute();
146 | return null;
147 | }, "while completing multipart upload");
148 | }
149 |
150 | private String getResult(Future future) {
151 | try {
152 | return future.get(partTimeoutMs, TimeUnit.MILLISECONDS);
153 | } catch (Throwable e) {
154 | abort();
155 | throw new RuntimeException(e);
156 | }
157 | }
158 |
159 | @Override
160 | public void write(int b) throws IOException {
161 | singleByte[0] = (byte) b;
162 | write(singleByte, 0, 1);
163 | }
164 | }
165 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/RequestHelper.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.IOException;
4 | import java.io.UnsupportedEncodingException;
5 | import java.net.URL;
6 | import java.net.URLDecoder;
7 | import java.util.ArrayList;
8 | import java.util.Collections;
9 | import java.util.HashMap;
10 | import java.util.List;
11 | import java.util.Map;
12 | import java.util.Optional;
13 | import java.util.stream.Collectors;
14 |
15 | import com.github.davidmoten.aws.lw.client.internal.Clock;
16 | import com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4;
17 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
18 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
19 |
20 | final class RequestHelper {
21 |
22 | private RequestHelper() {
23 | // prevent instantiation
24 | }
25 |
26 | static void put(Map> map, String name, String value) {
27 | Preconditions.checkNotNull(map);
28 | Preconditions.checkNotNull(name);
29 | Preconditions.checkNotNull(value);
30 | List list = map.get(name);
31 | if (list == null) {
32 | list = new ArrayList<>();
33 | map.put(name, list);
34 | }
35 | list.add(value);
36 | }
37 |
38 | static Map combineHeaders(Map> headers) {
39 | Preconditions.checkNotNull(headers);
40 | return headers.entrySet().stream().collect(Collectors.toMap(x -> x.getKey(),
41 | x -> x.getValue().stream().collect(Collectors.joining(","))));
42 | }
43 |
44 | static String presignedUrl(Clock clock, String url, String method, Map headers,
45 | byte[] requestBody, String serviceName, Optional regionName, Credentials credentials,
46 | int connectTimeoutMs, int readTimeoutMs, long expirySeconds, boolean signPayload) {
47 |
48 | // the region-specific endpoint to the target object expressed in path style
49 | URL endpointUrl = Util.toUrl(url);
50 |
51 | Map h = new HashMap<>(headers);
52 | final String contentHashString;
53 | if (isEmpty(requestBody)) {
54 | contentHashString = AwsSignatureVersion4.UNSIGNED_PAYLOAD;
55 | h.put("x-amz-content-sha256", "");
56 | } else if (!signPayload) {
57 | contentHashString = AwsSignatureVersion4.UNSIGNED_PAYLOAD;
58 | h.put("x-amz-content-sha256", contentHashString);
59 | } else {
60 | // compute hash of the body content
61 | byte[] contentHash = Util.sha256(requestBody);
62 | contentHashString = Util.toHex(contentHash);
63 | h.put("content-length", "" + requestBody.length);
64 | h.put("x-amz-content-sha256", contentHashString);
65 | }
66 |
67 | List parameters = extractQueryParameters(endpointUrl);
68 | // don't use Collectors.toMap because it doesn't accept null values in map
69 | Map q = new HashMap<>();
70 | parameters.forEach(p -> q.put(p.name, p.value));
71 |
72 | // construct the query parameter string to accompany the url
73 |
74 | // for SignatureV4, the max expiry for a presigned url is 7 days,
75 | // expressed in seconds
76 | q.put("X-Amz-Expires", "" + expirySeconds);
77 |
78 | String authorizationQueryParameters = AwsSignatureVersion4.computeSignatureForQueryAuth(
79 | endpointUrl, method, serviceName, regionName, clock, h, q, contentHashString,
80 | credentials.accessKey(), credentials.secretKey(), credentials.sessionToken());
81 |
82 | // build the presigned url to incorporate the authorization elements as query
83 | // parameters
84 | String u = endpointUrl.toString();
85 | final String presignedUrl;
86 | if (u.contains("?")) {
87 | presignedUrl = u + "&" + authorizationQueryParameters;
88 | } else {
89 | presignedUrl = u + "?" + authorizationQueryParameters;
90 | }
91 | return presignedUrl;
92 | }
93 |
94 | private static void includeTokenIfPresent(Credentials credentials, Map h) {
95 | if (credentials.sessionToken().isPresent()) {
96 | h.put("x-amz-security-token", credentials.sessionToken().get());
97 | }
98 | }
99 |
100 | static ResponseInputStream request(Clock clock, HttpClient httpClient, String url,
101 | HttpMethod method, Map headers, byte[] requestBody, String serviceName,
102 | Optional regionName, Credentials credentials, int connectTimeoutMs, int readTimeoutMs, //
103 | boolean signPayload) throws IOException {
104 |
105 | // the region-specific endpoint to the target object expressed in path style
106 | URL endpointUrl = Util.toUrl(url);
107 |
108 | Map h = new HashMap<>(headers);
109 | final String contentHashString;
110 | if (isEmpty(requestBody)) {
111 | contentHashString = AwsSignatureVersion4.EMPTY_BODY_SHA256;
112 | } else {
113 | if (!signPayload) {
114 | contentHashString = AwsSignatureVersion4.UNSIGNED_PAYLOAD;
115 | } else {
116 | // compute hash of the body content
117 | byte[] contentHash = Util.sha256(requestBody);
118 | contentHashString = Util.toHex(contentHash);
119 | }
120 | h.put("content-length", "" + requestBody.length);
121 | }
122 | h.put("x-amz-content-sha256", contentHashString);
123 |
124 | includeTokenIfPresent(credentials, h);
125 |
126 | List parameters = extractQueryParameters(endpointUrl);
127 | // don't use Collectors.toMap because it doesn't accept null values in map
128 | Map q = new HashMap<>();
129 | parameters.forEach(p -> q.put(p.name, p.value));
130 | String authorization = AwsSignatureVersion4.computeSignatureForAuthorizationHeader(
131 | endpointUrl, method.toString(), serviceName, regionName.orElse("us-east-1"), clock, h, q,
132 | contentHashString, credentials.accessKey(), credentials.secretKey());
133 |
134 | // place the computed signature into a formatted 'Authorization' header
135 | // and call S3
136 | h.put("Authorization", authorization);
137 | return httpClient.request(endpointUrl, method.toString(), h, requestBody, connectTimeoutMs,
138 | readTimeoutMs);
139 | }
140 |
141 | private static List extractQueryParameters(URL endpointUrl) {
142 | String query = endpointUrl.getQuery();
143 | if (query == null) {
144 | return Collections.emptyList();
145 | } else {
146 | return extractQueryParameters(query);
147 | }
148 | }
149 |
150 | private static final char QUERY_PARAMETER_SEPARATOR = '&';
151 | private static final char QUERY_PARAMETER_VALUE_SEPARATOR = '=';
152 |
153 | /**
154 | * Extract parameters from a query string, preserving encoding.
155 | *
156 | * We can't use Apache HTTP Client's URLEncodedUtils.parse, mainly because we
157 | * don't want to decode names/values.
158 | *
159 | * @param rawQuery the query to parse
160 | * @return The list of parameters, in the order they were found.
161 | */
162 | // VisibleForTesting
163 | static List extractQueryParameters(String rawQuery) {
164 | List results = new ArrayList<>();
165 | int endIndex = rawQuery.length() - 1;
166 | int index = 0;
167 | while (index <= endIndex) {
168 | /*
169 | * Ideally we should first look for '&', then look for '=' before the '&', but
170 | * obviously that's not how AWS understand query parsing; see the test
171 | * "post-vanilla-query-nonunreserved" in the test suite. A string such as
172 | * "?foo&bar=qux" will be understood as one parameter with name "foo&bar" and
173 | * value "qux". Don't ask me why.
174 | */
175 | String name;
176 | String value;
177 | int nameValueSeparatorIndex = rawQuery.indexOf(QUERY_PARAMETER_VALUE_SEPARATOR, index);
178 | if (nameValueSeparatorIndex < 0) {
179 | // No value
180 | name = rawQuery.substring(index);
181 | value = null;
182 |
183 | index = endIndex + 1;
184 | } else {
185 | int parameterSeparatorIndex = rawQuery.indexOf(QUERY_PARAMETER_SEPARATOR,
186 | nameValueSeparatorIndex);
187 | if (parameterSeparatorIndex < 0) {
188 | parameterSeparatorIndex = endIndex + 1;
189 | }
190 | name = rawQuery.substring(index, nameValueSeparatorIndex);
191 | value = rawQuery.substring(nameValueSeparatorIndex + 1, parameterSeparatorIndex);
192 |
193 | index = parameterSeparatorIndex + 1;
194 | }
195 | // note that value = null is valid as we can have a parameter without a value in
196 | // a query string (legal http)
197 | results.add(parameter(name, value, "UTF-8"));
198 | }
199 | return results;
200 | }
201 |
202 | // VisibleForTesting
203 | static Parameter parameter(String name, String value, String charset) {
204 | try {
205 | return new Parameter(URLDecoder.decode(name, charset),
206 | value == null ? value : URLDecoder.decode(value, charset));
207 | } catch (UnsupportedEncodingException e) {
208 | throw new RuntimeException(e);
209 | }
210 | }
211 |
212 | // VisibleForTesting
213 | static final class Parameter {
214 | final String name;
215 | final String value;
216 |
217 | Parameter(String name, String value) {
218 | this.name = name;
219 | this.value = value;
220 | }
221 | }
222 |
223 | static boolean isEmpty(byte[] array) {
224 | return array == null || array.length == 0;
225 | }
226 |
227 | }
228 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/Response.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.net.HttpURLConnection;
4 | import java.nio.charset.StandardCharsets;
5 | import java.time.Instant;
6 | import java.time.ZonedDateTime;
7 | import java.util.List;
8 | import java.util.Locale;
9 | import java.util.Map;
10 | import java.util.Optional;
11 | import java.util.stream.Collectors;
12 |
13 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
14 |
15 | public final class Response {
16 |
17 | private final Map> headers;
18 | private final Map> headersLowerCaseKey;
19 | private final byte[] content;
20 | private final int statusCode;
21 |
22 | public Response(Map> headers, byte[] content, int statusCode) {
23 | this.headers = headers;
24 | this.headersLowerCaseKey = lowerCaseKey(headers);
25 | this.content = content;
26 | this.statusCode = statusCode;
27 | }
28 |
29 | public Map> headers() {
30 | return headers;
31 | }
32 |
33 | public Map> headersLowerCaseKey() {
34 | return headersLowerCaseKey;
35 | }
36 |
37 | public Optional firstHeader(String name) {
38 | List h = headersLowerCaseKey.get(lowerCase(name));
39 | if (h == null || h.isEmpty()) {
40 | return Optional.empty();
41 | } else {
42 | return Optional.of(h.get(0));
43 | }
44 | }
45 |
46 | public Optional firstHeaderFullDate(String name) {
47 | return firstHeader(name) //
48 | .map(x -> ZonedDateTime.parse(x, Formats.FULL_DATE).toInstant());
49 | }
50 |
51 | /**
52 | * Returns those headers that start with {@code x-amz-meta-} (and removes that
53 | * prefix).
54 | *
55 | * @return headers that start with {@code x-amz-meta-} (and removes that prefix)
56 | */
57 | public Metadata metadata() {
58 | return new Metadata(headersLowerCaseKey //
59 | .entrySet() //
60 | .stream() //
61 | .filter(x -> x.getKey() != null) //
62 | .filter(x -> x.getKey().startsWith("x-amz-meta-")) //
63 | .collect(Collectors.toMap( //
64 | x -> x.getKey().substring(11), //
65 | x -> x.getValue().get(0))));
66 | }
67 |
68 | public Optional metadata(String name) {
69 | Preconditions.checkNotNull(name);
70 | // value() method does conversion of name to lower case
71 | return metadata().value(name);
72 | }
73 |
74 | public byte[] content() {
75 | return content;
76 | }
77 |
78 | public String contentUtf8() {
79 | return new String(content, StandardCharsets.UTF_8);
80 | }
81 |
82 | public int statusCode() {
83 | return statusCode;
84 | }
85 |
86 | public boolean isOk() {
87 | return statusCode >= 200 && statusCode <= 299;
88 | }
89 |
90 | /**
91 | * Returns true if and only if status code is 2xx. Returns false if status code
92 | * is 404 (NOT_FOUND) and throws a {@link ServiceException} otherwise.
93 | *
94 | * @return true if status code 2xx, false if 404 otherwise throws
95 | * ServiceException
96 | * @throws ServiceException if status code other than 2xx or 404
97 | */
98 | public boolean exists() {
99 | if (statusCode >= 200 && statusCode <= 299) {
100 | return true;
101 | } else if (statusCode == HttpURLConnection.HTTP_NOT_FOUND) {
102 | return false;
103 | } else {
104 | throw new ServiceException(statusCode, "call failed");
105 | }
106 | }
107 |
108 | private static Map> lowerCaseKey(Map> m) {
109 | return m.entrySet().stream().collect( //
110 | Collectors.toMap( //
111 | entry -> lowerCase(entry.getKey()), //
112 | entry -> entry.getValue()));
113 | }
114 |
115 | private static String lowerCase(String s) {
116 | return s == null ? s : s.toLowerCase(Locale.ENGLISH);
117 | }
118 |
119 | // TODO add toString method
120 | }
121 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/ResponseInputStream.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.Closeable;
4 | import java.io.IOException;
5 | import java.io.InputStream;
6 | import java.net.HttpURLConnection;
7 | import java.util.List;
8 | import java.util.Map;
9 | import java.util.Optional;
10 | import java.util.stream.Collectors;
11 |
12 | public final class ResponseInputStream extends InputStream {
13 |
14 | private final Closeable closeable; // nullable
15 | private final int statusCode;
16 | private final Map> headers;
17 | private final InputStream content;
18 |
19 | public ResponseInputStream(HttpURLConnection connection, int statusCode,
20 | Map> headers, InputStream content) {
21 | this(() -> connection.disconnect(), statusCode, headers, content);
22 | }
23 |
24 | public ResponseInputStream(Closeable closeable, int statusCode,
25 | Map> headers, InputStream content) {
26 | this.closeable = closeable;
27 | this.statusCode = statusCode;
28 | this.headers = headers;
29 | this.content = content;
30 | }
31 |
32 | @Override
33 | public int read(byte[] b, int off, int len) throws IOException {
34 | return content.read(b, off, len);
35 | }
36 |
37 | @Override
38 | public int read() throws IOException {
39 | return content.read();
40 | }
41 |
42 | @Override
43 | public void close() throws IOException {
44 | try {
45 | content.close();
46 | } finally {
47 | closeable.close();
48 | }
49 | }
50 |
51 | public int statusCode() {
52 | return statusCode;
53 | }
54 |
55 | public Map> headers() {
56 | return headers;
57 | }
58 |
59 | public Optional header(String name) {
60 | for (String key : headers.keySet()) {
61 | if (name.equalsIgnoreCase(key)) {
62 | return Optional.of(headers.get(key).stream().collect(Collectors.joining(",")));
63 | }
64 | }
65 | return Optional.empty();
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/ServiceException.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public final class ServiceException extends RuntimeException {
4 |
5 | private static final long serialVersionUID = -6963816822115090962L;
6 | private final int statusCode;
7 | private final String message;
8 |
9 | public ServiceException(int statusCode, String message) {
10 | super("statusCode=" + statusCode + ": " + message);
11 | this.statusCode = statusCode;
12 | this.message = message;
13 | }
14 |
15 | public int statusCode() {
16 | return statusCode;
17 | }
18 |
19 | public String message() {
20 | return message;
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/Clock.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | @FunctionalInterface
4 | public interface Clock {
5 |
6 | long time();
7 |
8 | Clock DEFAULT = new ClockDefault();
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/ClockDefault.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | public final class ClockDefault implements Clock {
4 |
5 | @Override
6 | public long time() {
7 | return System.currentTimeMillis();
8 | }
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/CredentialsHelper.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.io.IOException;
4 | import java.io.UncheckedIOException;
5 | import java.net.URL;
6 | import java.nio.charset.StandardCharsets;
7 | import java.nio.file.FileSystems;
8 | import java.nio.file.Files;
9 | import java.util.HashMap;
10 | import java.util.Map;
11 | import java.util.Optional;
12 |
13 | import com.github.davidmoten.aws.lw.client.Credentials;
14 | import com.github.davidmoten.aws.lw.client.HttpClient;
15 | import com.github.davidmoten.aws.lw.client.ResponseInputStream;
16 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
17 |
18 | final class CredentialsHelper {
19 |
20 | private static final int CONNECT_TIMEOUT_MS = 10000;
21 | private static final int READ_TIMEOUT_MS = 10000;
22 |
23 | private CredentialsHelper() {
24 | // prevent instantiation
25 | }
26 |
27 | static Credentials credentialsFromEnvironment(Environment env, HttpClient client) {
28 | // if using SnapStart we need to get the credentials from a local container
29 | // it is a precondition that SnapStart snapshot has happened before credentials get loaded
30 | // so we get a chance to refresh creds from the local container
31 | String containerCredentialsUri = env.get("AWS_CONTAINER_CREDENTIALS_FULL_URI");
32 | if (containerCredentialsUri != null) {
33 | String containerToken = env.get("AWS_CONTAINER_AUTHORIZATION_TOKEN");
34 | String containerTokenFile = env.get("AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE");
35 | containerToken = resolveContainerToken(containerToken, containerTokenFile);
36 | try {
37 | // Create a connection to the credentials URI
38 | URL url = new URL(containerCredentialsUri);
39 | Map headers = new HashMap<>();
40 | headers.put("Authorization", containerToken);
41 | ResponseInputStream response = client.request(url, "GET", headers, null, CONNECT_TIMEOUT_MS,
42 | READ_TIMEOUT_MS);
43 |
44 | if (response.statusCode() != 200) {
45 | throw new RuntimeException("Failed to retrieve credentials: HTTP " + response.statusCode());
46 | }
47 |
48 | String json = new String(Util.readBytesAndClose(response), StandardCharsets.UTF_8);
49 |
50 | // Parse the JSON response
51 | String accessKeyId = Util.jsonFieldText(json, "AccessKeyId").get();
52 | String secretAccessKey = Util.jsonFieldText(json, "SecretAccessKey").get();
53 | String sessionToken = Util.jsonFieldText(json, "Token").get();
54 | return new CredentialsImpl(accessKeyId, secretAccessKey, Optional.of(sessionToken));
55 | } catch (IOException e) {
56 | throw new UncheckedIOException(e);
57 | }
58 | } else {
59 | String accessKey = env.get("AWS_ACCESS_KEY_ID");
60 | String secretKey = env.get("AWS_SECRET_ACCESS_KEY");
61 | Optional token = Optional.ofNullable(env.get("AWS_SESSION_TOKEN"));
62 | return new CredentialsImpl(accessKey, secretKey, token);
63 | }
64 | }
65 |
66 | // VisibleForTesting
67 | static String resolveContainerToken(String containerToken, String containerTokenFile) {
68 | if (containerToken == null && containerTokenFile != null) {
69 | return readUtf8(containerTokenFile);
70 | } else if (containerToken == null) {
71 | throw new IllegalStateException("token not found to retrieve credentials from local container");
72 | } else {
73 | return containerToken;
74 | }
75 | }
76 |
77 | // VisibleForTesting
78 | static String readUtf8(String file) {
79 | try {
80 | byte[] bytes = Files.readAllBytes(FileSystems.getDefault().getPath(file));
81 | return new String(bytes, StandardCharsets.UTF_8);
82 | } catch (IOException e) {
83 | throw new IllegalStateException("Cannot read string contents from file " + file);
84 | }
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/CredentialsImpl.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.util.Optional;
4 |
5 | import com.github.davidmoten.aws.lw.client.Credentials;
6 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
7 |
8 | public final class CredentialsImpl implements Credentials {
9 |
10 | private final String accessKey;
11 | private final String secretKey;
12 | private final Optional sessionToken;
13 |
14 | public CredentialsImpl(String accessKey, String secretKey, Optional sessionToken) {
15 | Preconditions.checkNotNull(accessKey);
16 | Preconditions.checkNotNull(secretKey);
17 | Preconditions.checkNotNull(sessionToken);
18 | this.accessKey = accessKey;
19 | this.secretKey = secretKey;
20 | this.sessionToken = sessionToken;
21 | }
22 |
23 | @Override
24 | public String accessKey() {
25 | return accessKey;
26 | }
27 |
28 | @Override
29 | public String secretKey() {
30 | return secretKey;
31 | }
32 |
33 | @Override
34 | public Optional sessionToken() {
35 | return sessionToken;
36 | }
37 |
38 | }
39 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/Environment.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import com.github.davidmoten.aws.lw.client.Credentials;
4 | import com.github.davidmoten.aws.lw.client.HttpClient;
5 |
6 | @FunctionalInterface
7 | public interface Environment {
8 |
9 | String get(String name);
10 |
11 | default Credentials credentials() {
12 | return CredentialsHelper.credentialsFromEnvironment(this, HttpClient.defaultClient());
13 | }
14 |
15 | static Environment instance() {
16 | return EnvironmentDefault.INSTANCE;
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/EnvironmentDefault.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | public final class EnvironmentDefault implements Environment {
4 |
5 | // mutable for testing
6 | public static Environment INSTANCE = new EnvironmentDefault();
7 |
8 | private EnvironmentDefault() {
9 | // prevent instantiation
10 | }
11 |
12 | @Override
13 | public String get(String name) {
14 | return System.getenv(name);
15 | }
16 |
17 | }
18 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/ExceptionFactoryDefault.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.nio.charset.StandardCharsets;
4 | import java.util.Optional;
5 |
6 | import com.github.davidmoten.aws.lw.client.ExceptionFactory;
7 | import com.github.davidmoten.aws.lw.client.Response;
8 | import com.github.davidmoten.aws.lw.client.ServiceException;
9 |
10 | public class ExceptionFactoryDefault implements ExceptionFactory{
11 |
12 | @Override
13 | public Optional create(Response r) {
14 | if (r.isOk()) {
15 | return Optional.empty();
16 | } else {
17 | return Optional.of(new ServiceException(r.statusCode(),
18 | new String(r.content(), StandardCharsets.UTF_8)));
19 | }
20 | }
21 |
22 | }
23 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/ExceptionFactoryExtended.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.util.Optional;
4 | import java.util.function.Function;
5 | import java.util.function.Predicate;
6 |
7 | import com.github.davidmoten.aws.lw.client.ExceptionFactory;
8 | import com.github.davidmoten.aws.lw.client.Response;
9 |
10 | public class ExceptionFactoryExtended implements ExceptionFactory {
11 |
12 | private final ExceptionFactory factory;
13 | private final Predicate predicate;
14 | private final Function function;
15 |
16 | public ExceptionFactoryExtended(ExceptionFactory factory, Predicate predicate, Function function) {
17 | this.factory = factory;
18 | this.predicate = predicate;
19 | this.function = function;
20 | }
21 |
22 | @Override
23 | public Optional create(Response response) {
24 | if (predicate.test(response)) {
25 | return Optional.of(function.apply(response));
26 | } else {
27 | return factory.create(response);
28 | }
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/HttpClientDefault.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.io.IOException;
4 | import java.io.InputStream;
5 | import java.io.OutputStream;
6 | import java.io.UncheckedIOException;
7 | import java.net.HttpURLConnection;
8 | import java.net.URL;
9 | import java.util.List;
10 | import java.util.Map;
11 |
12 | import com.github.davidmoten.aws.lw.client.HttpClient;
13 | import com.github.davidmoten.aws.lw.client.ResponseInputStream;
14 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
15 |
16 | public final class HttpClientDefault implements HttpClient {
17 |
18 | public static final HttpClientDefault INSTANCE = new HttpClientDefault();
19 |
20 | private HttpClientDefault() {
21 | }
22 |
23 | @Override
24 | public ResponseInputStream request(URL endpointUrl, String httpMethod,
25 | Map headers, byte[] requestBody, int connectTimeoutMs,
26 | int readTimeoutMs) throws IOException {
27 | HttpURLConnection connection = Util.createHttpConnection(endpointUrl, httpMethod, headers,
28 | connectTimeoutMs, readTimeoutMs);
29 | return request(connection, requestBody);
30 | }
31 |
32 | // VisibleForTesting
33 | static ResponseInputStream request(HttpURLConnection connection, byte[] requestBody) {
34 | int responseCode;
35 | Map> responseHeaders;
36 | InputStream is;
37 | try {
38 | if (requestBody != null) {
39 | OutputStream out = connection.getOutputStream();
40 | out.write(requestBody);
41 | out.flush();
42 | }
43 | responseHeaders = connection.getHeaderFields();
44 | responseCode = connection.getResponseCode();
45 | if (isOk(responseCode)) {
46 | is = connection.getInputStream();
47 | } else {
48 | is = connection.getErrorStream();
49 | }
50 | if (is == null) {
51 | is = Util.emptyInputStream();
52 | }
53 | } catch (IOException e) {
54 | try {
55 | connection.disconnect();
56 | } catch (Throwable e2) {
57 | // ignore
58 | }
59 | throw new UncheckedIOException(e);
60 | }
61 | return new ResponseInputStream(connection, responseCode, responseHeaders, is);
62 | }
63 |
64 | private static boolean isOk(int responseCode) {
65 | return responseCode >= 200 && responseCode <= 299;
66 | }
67 |
68 | }
69 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/Retries.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import java.io.IOException;
4 | import java.io.UncheckedIOException;
5 | import java.util.concurrent.Callable;
6 | import java.util.function.Predicate;
7 |
8 | import com.github.davidmoten.aws.lw.client.MaxAttemptsExceededException;
9 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
10 |
11 | public final class Retries {
12 |
13 | private final long initialIntervalMs;
14 | private final int maxAttempts;
15 | private final double backoffFactor;
16 | private final long maxIntervalMs;
17 | private final double jitter;
18 | private final Predicate valueShouldRetry;
19 | private final Predicate throwableShouldRetry;
20 |
21 | public Retries(long initialIntervalMs, int maxAttempts, double backoffFactor, double jitter, long maxIntervalMs,
22 | Predicate valueShouldRetry, Predicate throwableShouldRetry) {
23 | Preconditions.checkArgument(jitter >= 0 && jitter <= 1, "jitter must be between 0 and 1 inclusive");
24 | this.initialIntervalMs = initialIntervalMs;
25 | this.maxAttempts = maxAttempts;
26 | this.backoffFactor = backoffFactor;
27 | this.jitter = jitter;
28 | this.maxIntervalMs = maxIntervalMs;
29 | this.valueShouldRetry = valueShouldRetry;
30 | this.throwableShouldRetry = throwableShouldRetry;
31 | }
32 |
33 | public static Retries create(Predicate valueShouldRetry,
34 | Predicate throwableShouldRetry) {
35 | return new Retries( //
36 | 100, //
37 | 4, //
38 | 2.0, //
39 | 0.0, // no jitter
40 | 20000, //
41 | valueShouldRetry, //
42 | throwableShouldRetry);
43 | }
44 |
45 | public T call(Callable callable) {
46 | return call(callable, valueShouldRetry);
47 | }
48 |
49 | public S call(Callable callable, Predicate valueShouldRetry) {
50 | long intervalMs = initialIntervalMs;
51 | int attempt = 0;
52 | while (true) {
53 | S value;
54 | try {
55 | attempt++;
56 | value = callable.call();
57 | if (!valueShouldRetry.test(value)) {
58 | return value;
59 | }
60 | if (reachedMaxAttempts(attempt, maxAttempts)) {
61 | // note that caller is not aware that maxAttempts were reached, the caller just
62 | // receives the last error response
63 | return value;
64 | }
65 | } catch (Throwable t) {
66 | if (!throwableShouldRetry.test(t)) {
67 | rethrow(t);
68 | }
69 | if (reachedMaxAttempts(attempt, maxAttempts)) {
70 | throw new MaxAttemptsExceededException("exceeded max attempts " + maxAttempts, t);
71 | }
72 | }
73 | sleep(intervalMs);
74 | //calculate the interval for the next retry
75 | intervalMs = Math.round(backoffFactor * intervalMs);
76 | if (maxIntervalMs > 0) {
77 | intervalMs = Math.min(maxIntervalMs, intervalMs);
78 | }
79 | // apply jitter (if 0 then no change)
80 | intervalMs = Math.round((1 - jitter * Math.random()) * intervalMs);
81 | }
82 | }
83 |
84 | // VisibleForTesting
85 | static boolean reachedMaxAttempts(int attempt, int maxAttempts) {
86 | return maxAttempts > 0 && attempt >= maxAttempts;
87 | }
88 |
89 | static void sleep(long intervalMs) {
90 | try {
91 | Thread.sleep(intervalMs);
92 | } catch (InterruptedException e) {
93 | throw new RuntimeException(e);
94 | }
95 | }
96 |
97 | public Retries withValueShouldRetry(Predicate valueShouldRetry) {
98 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
99 | throwableShouldRetry);
100 | }
101 |
102 | public Retries withInitialIntervalMs(long initialIntervalMs) {
103 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
104 | throwableShouldRetry);
105 | }
106 |
107 | public Retries withMaxAttempts(int maxAttempts) {
108 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
109 | throwableShouldRetry);
110 | }
111 |
112 | public Retries withBackoffFactor(double backoffFactor) {
113 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
114 | throwableShouldRetry);
115 | }
116 |
117 | public Retries withMaxIntervalMs(long maxIntervalMs) {
118 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
119 | throwableShouldRetry);
120 | }
121 |
122 | public Retries withJitter(double jitter) {
123 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
124 | throwableShouldRetry);
125 | }
126 |
127 | public Retries withThrowableShouldRetry(Predicate throwableShouldRetry) {
128 | return new Retries(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
129 | throwableShouldRetry);
130 | }
131 |
132 | public Retries copy() {
133 | return new Retries<>(initialIntervalMs, maxAttempts, backoffFactor, jitter, maxIntervalMs, valueShouldRetry,
134 | throwableShouldRetry);
135 | }
136 |
137 | // VisibleForTesting
138 | static void rethrow(Throwable t) throws Error {
139 | if (t instanceof RuntimeException) {
140 | throw (RuntimeException) t;
141 | } else if (t instanceof Error) {
142 | throw (Error) t;
143 | } else if (t instanceof IOException) {
144 | throw new UncheckedIOException((IOException) t);
145 | } else {
146 | throw new RuntimeException(t);
147 | }
148 | }
149 |
150 | }
151 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/util/Preconditions.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.util;
2 |
3 | public final class Preconditions {
4 |
5 | private Preconditions() {
6 | // prevent instantiation
7 | }
8 |
9 | public static T checkNotNull(T t) {
10 | return checkNotNull(t, "argument cannot be null");
11 | }
12 |
13 | public static T checkNotNull(T t, String message) {
14 | if (t == null) {
15 | throw new IllegalArgumentException(message);
16 | }
17 | return t;
18 | }
19 |
20 | public static void checkArgument(boolean b, String message) {
21 | if (!b)
22 | throw new IllegalArgumentException(message);
23 | }
24 |
25 | public static void checkArgument(boolean b) {
26 | if (!b)
27 | throw new IllegalArgumentException();
28 | }
29 |
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/internal/util/Util.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.util;
2 |
3 | import java.io.ByteArrayOutputStream;
4 | import java.io.IOException;
5 | import java.io.InputStream;
6 | import java.io.UncheckedIOException;
7 | import java.io.UnsupportedEncodingException;
8 | import java.net.HttpURLConnection;
9 | import java.net.MalformedURLException;
10 | import java.net.URL;
11 | import java.net.URLEncoder;
12 | import java.nio.charset.StandardCharsets;
13 | import java.security.MessageDigest;
14 | import java.security.NoSuchAlgorithmException;
15 | import java.util.Locale;
16 | import java.util.Map;
17 | import java.util.Map.Entry;
18 | import java.util.Optional;
19 |
20 | /**
21 | * Utilities for encoding and decoding binary data to and from different forms.
22 | */
23 | public final class Util {
24 |
25 | private Util() {
26 | // prevent instantiation
27 | }
28 |
29 | public static HttpURLConnection createHttpConnection(URL endpointUrl, String httpMethod,
30 | Map headers, int connectTimeoutMs, int readTimeoutMs) throws IOException {
31 | Preconditions.checkNotNull(headers);
32 | HttpURLConnection connection = (HttpURLConnection) endpointUrl.openConnection();
33 | connection.setRequestMethod(httpMethod);
34 |
35 | for (Entry entry : headers.entrySet()) {
36 | connection.setRequestProperty(entry.getKey(), entry.getValue());
37 | }
38 |
39 | connection.setUseCaches(false);
40 | connection.setDoInput(true);
41 | connection.setDoOutput(true);
42 | connection.setConnectTimeout(connectTimeoutMs);
43 | connection.setReadTimeout(readTimeoutMs);
44 | return connection;
45 | }
46 |
47 | public static String canonicalMetadataKey(String meta) {
48 | StringBuilder b = new StringBuilder();
49 | String s = meta.toLowerCase(Locale.ENGLISH);
50 | for (int ch : s.toCharArray()) {
51 | if (Character.isDigit(ch) || Character.isAlphabetic(ch)) {
52 | b.append((char) ch);
53 | }
54 | }
55 | return b.toString();
56 | }
57 |
58 | /**
59 | * Converts byte data to a Hex-encoded string.
60 | *
61 | * @param data data to hex encode.
62 | *
63 | * @return hex-encoded string.
64 | */
65 | public static String toHex(byte[] data) {
66 | StringBuilder sb = new StringBuilder(data.length * 2);
67 | for (int i = 0; i < data.length; i++) {
68 | String hex = Integer.toHexString(data[i]);
69 | if (hex.length() == 1) {
70 | // Append leading zero.
71 | sb.append("0");
72 | } else if (hex.length() == 8) {
73 | // Remove ff prefix from negative numbers.
74 | hex = hex.substring(6);
75 | }
76 | sb.append(hex);
77 | }
78 | return sb.toString().toLowerCase(Locale.getDefault());
79 | }
80 |
81 | public static URL toUrl(String url) {
82 | try {
83 | return new URL(url);
84 | } catch (MalformedURLException e) {
85 | throw new RuntimeException(e);
86 | }
87 | }
88 |
89 | public static String urlEncode(String url, boolean keepPathSlash) {
90 | return urlEncode(url, keepPathSlash, "UTF-8");
91 | }
92 |
93 | // VisibleForTesting
94 | static String urlEncode(String url, boolean keepPathSlash, String charset) {
95 | String encoded;
96 | try {
97 | encoded = URLEncoder.encode(url, charset).replace("+", "%20").replace("*", "%2A").replace("%7E", "~");
98 | } catch (UnsupportedEncodingException e) {
99 | throw new RuntimeException(e);
100 | }
101 | if (keepPathSlash) {
102 | return encoded.replace("%2F", "/");
103 | } else {
104 | return encoded;
105 | }
106 | }
107 |
108 | /**
109 | * Hashes the string contents (assumed to be UTF-8) using the SHA-256 algorithm.
110 | */
111 | public static byte[] sha256(String text) {
112 | return sha256(text.getBytes(StandardCharsets.UTF_8));
113 | }
114 |
115 | public static byte[] sha256(byte[] data) {
116 | return hash(data, "SHA-256");
117 | }
118 |
119 | // VisibleForTesting
120 | static byte[] hash(byte[] data, String algorithm) {
121 | try {
122 | MessageDigest md = MessageDigest.getInstance(algorithm);
123 | md.update(data);
124 | return md.digest();
125 | } catch (NoSuchAlgorithmException e) {
126 | throw new RuntimeException(e);
127 | }
128 | }
129 |
130 | public static byte[] readBytesAndClose(InputStream in) {
131 | try {
132 | byte[] buffer = new byte[8192];
133 | int n;
134 | ByteArrayOutputStream bytes = new ByteArrayOutputStream();
135 | while ((n = in.read(buffer)) != -1) {
136 | bytes.write(buffer, 0, n);
137 | }
138 | return bytes.toByteArray();
139 | } catch (IOException e) {
140 | throw new UncheckedIOException(e);
141 | } finally {
142 | try {
143 | in.close();
144 | } catch (IOException e) {
145 | throw new UncheckedIOException(e);
146 | }
147 | }
148 | }
149 |
150 | private static final InputStream EMPTY_INPUT_STREAM = new InputStream() {
151 | @Override
152 | public int read() throws IOException {
153 | return -1;
154 | }
155 | };
156 |
157 | public static final InputStream emptyInputStream() {
158 | return EMPTY_INPUT_STREAM;
159 | }
160 |
161 | public static Optional jsonFieldText(String json, String fieldName) {
162 | // it is assumed that the json field is valid object json
163 | String key = "\"" + fieldName + "\"";
164 | int keyPosition = json.indexOf(key);
165 | if (keyPosition == -1) {
166 | return Optional.empty(); // Field not found
167 | }
168 |
169 | // Find the position of the colon after the key and skip any whitespace
170 | int colonPosition = json.indexOf(":", keyPosition + key.length());
171 | if (colonPosition == -1) {
172 | return Optional.empty(); // Colon not found, malformed JSON
173 | }
174 |
175 | // Skip whitespace after the colon
176 | int valueStart = colonPosition + 1;
177 | while (valueStart < json.length() && Character.isWhitespace(json.charAt(valueStart))) {
178 | valueStart++;
179 | }
180 |
181 | // Check if the value is a string
182 | boolean isString = json.charAt(valueStart) == '"';
183 | StringBuilder value = new StringBuilder();
184 | boolean isEscaped = false;
185 |
186 | // Parse the value, handling escaped quotes
187 | for (int i = valueStart + (isString ? 1 : 0); i < json.length(); i++) {
188 | char c = json.charAt(i);
189 |
190 | if (isString) {
191 | // Handle string value
192 | if (isEscaped) {
193 | // Append escaped character and reset flag
194 | value.append(c);
195 | isEscaped = false;
196 | } else if (c == '\\') {
197 | // Next character is escaped
198 | isEscaped = true;
199 | } else if (c == '"') {
200 | // End of string
201 | break;
202 | } else {
203 | value.append(c);
204 | }
205 | } else {
206 | // Handle non-string value
207 | if (c == ',' || c == '}') {
208 | // End of non-string value
209 | break;
210 | } else {
211 | value.append(c);
212 | }
213 | }
214 | }
215 | String v = value.toString().trim();
216 | if (!isString && "null".equals(v)) {
217 | return Optional.empty();
218 | } else {
219 | return Optional.of(v);
220 | }
221 | }
222 | }
223 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/xml/XmlParseException.java:
--------------------------------------------------------------------------------
1 | /*
2 | * The NanoXML 2 Lite licence blurb is included here. The classes have been
3 | * completely butchered but the core xml parsing routines are thanks to
4 | * the NanoXML authors.
5 | *
6 | **/
7 |
8 | /* XMLParseException.java
9 | *
10 | * $Revision: 1.4 $
11 | * $Date: 2002/03/24 10:27:59 $
12 | * $Name: RELEASE_2_2_1 $
13 | *
14 | * This file is part of NanoXML 2 Lite.
15 | * Copyright (C) 2000-2002 Marc De Scheemaecker, All Rights Reserved.
16 | *
17 | * This software is provided 'as-is', without any express or implied warranty.
18 | * In no event will the authors be held liable for any damages arising from the
19 | * use of this software.
20 | *
21 | * Permission is granted to anyone to use this software for any purpose,
22 | * including commercial applications, and to alter it and redistribute it
23 | * freely, subject to the following restrictions:
24 | *
25 | * 1. The origin of this software must not be misrepresented; you must not
26 | * claim that you wrote the original software. If you use this software in
27 | * a product, an acknowledgment in the product documentation would be
28 | * appreciated but is not required.
29 | *
30 | * 2. Altered source versions must be plainly marked as such, and must not be
31 | * misrepresented as being the original software.
32 | *
33 | * 3. This notice may not be removed or altered from any source distribution.
34 | *****************************************************************************/
35 |
36 |
37 | package com.github.davidmoten.aws.lw.client.xml;
38 |
39 | /**
40 | * An XMLParseException is thrown when an error occures while parsing an XML
41 | * string.
42 | *
43 | * $Revision: 1.4 $
45 | *
46 | * @see com.github.davidmoten.aws.lw.client.xml.XmlElement
47 | *
48 | * @author Marc De Scheemaecker
49 | * @version $Name: RELEASE_2_2_1 $, $Revision: 1.4 $
50 | */
51 | public class XmlParseException
52 | extends RuntimeException
53 | {
54 |
55 | /**
56 | *
57 | */
58 | private static final long serialVersionUID = 2719032602966457493L;
59 |
60 |
61 | /**
62 | * Indicates that no line number has been associated with this exception.
63 | */
64 | public static final int NO_LINE = -1;
65 |
66 |
67 | /**
68 | * The line number in the source code where the error occurred, or
69 | * NO_LINE if the line number is unknown.
70 | *
71 | *
Invariants:
72 | * lineNumber > 0 || lineNumber == NO_LINE
73 | * Preconditions:
85 | * message != null
86 | * lineNumber > 0
87 | * Postconditions:
90 | * getLineNumber() => lineNr
91 | *
92 | */
93 | public XmlParseException(String name,
94 | int lineNumber,
95 | String message)
96 | {
97 | super("Problem parsing "
98 | + ((name == null) ? "the XML definition"
99 | : ("a " + name + " element"))
100 | + " at line " + lineNumber + ": " + message);
101 | this.lineNumber = lineNumber;
102 | }
103 |
104 |
105 | /**
106 | * Where the error occurred, or NO_LINE if the line number is
107 | * unknown.
108 | *
109 | */
110 | public int lineNumber()
111 | {
112 | return this.lineNumber;
113 | }
114 |
115 | }
116 |
--------------------------------------------------------------------------------
/src/main/java/com/github/davidmoten/aws/lw/client/xml/builder/Xml.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.xml.builder;
2 |
3 | import java.util.ArrayList;
4 | import java.util.HashMap;
5 | import java.util.List;
6 | import java.util.Map;
7 | import java.util.stream.Collectors;
8 |
9 | import com.github.davidmoten.aws.lw.client.internal.util.Preconditions;
10 |
11 | public final class Xml {
12 |
13 | private final String name;
14 | private final Xml parent;
15 | private Map attributes = new HashMap<>();
16 | private List children = new ArrayList<>();
17 | private String content;
18 | private boolean prelude = true;
19 |
20 | private Xml(String name) {
21 | this(name, null);
22 | }
23 |
24 | private Xml(String name, Xml parent) {
25 | checkPresent(name, "name");
26 | this.name = name;
27 | this.parent = parent;
28 | }
29 |
30 | public static Xml create(String name) {
31 | return new Xml(name);
32 | }
33 |
34 | public Xml excludePrelude() {
35 | Xml xml = this;
36 | while (xml.parent != null) {
37 | xml = xml.parent;
38 | }
39 | xml.prelude = false;
40 | return this;
41 | }
42 |
43 | public Xml element(String name) {
44 | checkPresent(name, "name");
45 | Preconditions.checkArgument(content == null,
46 | "content cannot be already specified if starting a child element");
47 | Xml xml = new Xml(name, this);
48 | this.children.add(xml);
49 | return xml;
50 | }
51 |
52 | public Xml e(String name) {
53 | return element(name);
54 | }
55 |
56 | public Xml attribute(String name, String value) {
57 | checkPresent(name, "name");
58 | Preconditions.checkNotNull(value);
59 | this.attributes.put(name, value);
60 | return this;
61 | }
62 |
63 | public Xml a(String name, String value) {
64 | return attribute(name, value);
65 | }
66 |
67 | public Xml content(String content) {
68 | Preconditions.checkArgument(children.isEmpty());
69 | this.content = content;
70 | return this;
71 | }
72 |
73 | public Xml up() {
74 | return parent;
75 | }
76 |
77 | private static void checkPresent(String s, String name) {
78 | if (s == null || s.trim().isEmpty()) {
79 | throw new IllegalArgumentException(name + " must be non-null and non-blank");
80 | }
81 | }
82 |
83 | private String toString(String indent) {
84 | StringBuilder b = new StringBuilder();
85 | if (indent.length() == 0 && prelude) {
86 | b.append("\n");
87 | }
88 | // TODO encode attributes and content for xml
89 | String atts = attributes.entrySet().stream().map(
90 | entry -> " " + entry.getKey() + "=\"" + encodeXml(entry.getValue(), true) + "\"")
91 | .collect(Collectors.joining());
92 | b.append(String.format("%s<%s%s>", indent, name, atts));
93 | if (content != null) {
94 | b.append(encodeXml(content, false));
95 | b.append(String.format("", name));
96 | if (parent != null) {
97 | b.append("\n");
98 | }
99 | } else {
100 | b.append("\n");
101 | for (Xml xml : children) {
102 | b.append(xml.toString(indent + " "));
103 | }
104 | b.append(String.format("%s", indent, name));
105 | if (parent != null) {
106 | b.append("\n");
107 | }
108 | }
109 | return b.toString();
110 | }
111 |
112 | public String toString() {
113 | Xml xml = this;
114 | while (xml.parent != null) {
115 | xml = xml.parent;
116 | }
117 | return xml.toString("");
118 | }
119 |
120 | private static final Map CONTENT_CHARACTER_MAP = createContentCharacterMap();
121 | private static final Map ATTRIBUTE_CHARACTER_MAP = createAttributeCharacterMap();
122 |
123 | private static Map createContentCharacterMap() {
124 | Map m = new HashMap<>();
125 | m.put((int) '&', "&");
126 | m.put((int) '>', ">");
127 | m.put((int) '<', "<");
128 | return m;
129 | }
130 |
131 | private static Map createAttributeCharacterMap() {
132 | Map m = new HashMap<>();
133 | m.put((int) '\'', "'");
134 | m.put((int) '\"', """);
135 | return m;
136 | }
137 |
138 | private static String encodeXml(CharSequence s, boolean isAttribute) {
139 | StringBuilder b = new StringBuilder();
140 | int len = s.length();
141 | for (int i = 0; i < len; i++) {
142 | int c = s.charAt(i);
143 | if (c >= 0xd800 && c <= 0xdbff && i + 1 < len) {
144 | c = ((c - 0xd7c0) << 10) | (s.charAt(++i) & 0x3ff); // UTF16 decode
145 | }
146 | if (c < 0x80) { // ASCII range: test most common case first
147 | if (c < 0x20 && (c != '\t' && c != '\r' && c != '\n')) {
148 | // Illegal XML character, even encoded. Skip or substitute
149 | b.append("�"); // Unicode replacement character
150 | } else {
151 | String r = CONTENT_CHARACTER_MAP.get(c);
152 | if (r != null) {
153 | b.append(r);
154 | } else if (isAttribute) {
155 | String r2 = ATTRIBUTE_CHARACTER_MAP.get(c);
156 | if (r2 != null) {
157 | b.append(r2);
158 | } else {
159 | b.append((char) c);
160 | }
161 | } else {
162 | b.append((char) c);
163 | }
164 |
165 | }
166 | } else if ((c >= 0xd800 && c <= 0xdfff) || c == 0xfffe || c == 0xffff) {
167 | // Illegal XML character, even encoded. Skip or substitute
168 | b.append("�"); // Unicode replacement character
169 | } else {
170 | b.append("&#x");
171 | b.append(Integer.toHexString(c));
172 | b.append(';');
173 | }
174 | }
175 | return b.toString();
176 | }
177 |
178 | }
179 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/AwsSdkV2Main.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.IOException;
4 | import java.util.Arrays;
5 | import java.util.Collection;
6 |
7 | import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider;
8 | import software.amazon.awssdk.awscore.exception.AwsServiceException;
9 | import software.amazon.awssdk.core.exception.SdkClientException;
10 | import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
11 | import software.amazon.awssdk.regions.Region;
12 | import software.amazon.awssdk.services.s3.S3Client;
13 | import software.amazon.awssdk.services.s3.model.CompleteMultipartUploadRequest;
14 | import software.amazon.awssdk.services.s3.model.CompletedMultipartUpload;
15 | import software.amazon.awssdk.services.s3.model.CompletedPart;
16 | import software.amazon.awssdk.services.s3.model.GetObjectRequest;
17 | import software.amazon.awssdk.services.s3.model.InvalidObjectStateException;
18 | import software.amazon.awssdk.services.s3.model.NoSuchKeyException;
19 | import software.amazon.awssdk.services.s3.model.S3Exception;
20 | import software.amazon.awssdk.utils.IoUtils;
21 |
22 | public class AwsSdkV2Main {
23 |
24 | public static void main(String[] args) throws NoSuchKeyException, InvalidObjectStateException, S3Exception, AwsServiceException, SdkClientException, IOException {
25 | S3Client client = S3Client.builder()
26 | .region(Region.AP_SOUTHEAST_2)
27 | .credentialsProvider(SystemPropertyCredentialsProvider.create()) //
28 | .httpClient(UrlConnectionHttpClient.builder().build())
29 | .build();
30 | client.getObject(GetObjectRequest.builder().bucket("mybucket").key("mykey").build());
31 | String r = IoUtils.toUtf8String(client.getObject(GetObjectRequest.builder().bucket("amsa-xml-in").key("ExampleObject.txt").build()));
32 | System.out.println(r);
33 | CompletedPart part = CompletedPart.builder().eTag("et123").partNumber(1).build();
34 | Collection parts = Arrays.asList(part);
35 | CompletedMultipartUpload m = CompletedMultipartUpload.builder().parts(parts).build();
36 | // client.putObject(PutObjectRequest.builder().bucket("amsa-xml-in").key("ExampleObject.txt").build(), //
37 | // RequestBody.fromString("hi there"));
38 | CompleteMultipartUploadRequest request = CompleteMultipartUploadRequest.builder().bucket("amsa-xml-in").key("mykey").uploadId("abc") //
39 | .multipartUpload(m).build();
40 | client.completeMultipartUpload(request);
41 | }
42 |
43 | }
44 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/ClientCountLoadedClassesMain.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public class ClientCountLoadedClassesMain {
4 |
5 | public static void main(String[] args) {
6 | // run with -verbose:class
7 | String regionName = "ap-southeast-2";
8 | String accessKey = System.getProperty("accessKey");
9 | String secretKey = System.getProperty("secretKey");
10 |
11 | Credentials credentials = Credentials.of(accessKey, secretKey);
12 | Client s3 = Client //
13 | .s3() //
14 | .region(regionName) //
15 | .credentials(credentials) //
16 | .build();
17 | System.out.println(s3.path("amsa-xml-in", "ExampleObject.txt").responseAsUtf8());
18 | System.out.println("2350 classes loaded");
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/ClientMain.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.BufferedOutputStream;
4 | import java.io.FileNotFoundException;
5 | import java.io.FileOutputStream;
6 | import java.io.IOException;
7 | import java.io.OutputStream;
8 | import java.nio.charset.StandardCharsets;
9 | import java.util.List;
10 | import java.util.Map;
11 | import java.util.concurrent.TimeUnit;
12 | import java.util.stream.Collectors;
13 |
14 | import org.davidmoten.kool.Stream;
15 |
16 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
17 | import com.github.davidmoten.aws.lw.client.xml.XmlElement;
18 | import com.github.davidmoten.aws.lw.client.xml.builder.Xml;
19 |
20 | public final class ClientMain {
21 |
22 | private static final boolean TEST_CHUNKED = false;
23 |
24 | public static void main(String[] args)
25 | throws InterruptedException, FileNotFoundException, IOException {
26 | String regionName = "ap-southeast-2";
27 | String accessKey = System.getProperty("accessKey");
28 | String secretKey = System.getProperty("secretKey");
29 |
30 | Credentials credentials = Credentials.of(accessKey, secretKey);
31 | Client sqs = Client //
32 | .sqs() //
33 | .region(regionName) //
34 | .credentials(credentials) //
35 | .build();
36 | Client s3 = Client.s3().from(sqs).build();
37 | System.out.println(s3.path("moten-fixes", "name with spaces.txt").responseAsUtf8());
38 | // System.exit(0);
39 | System.out.println(s3.path("moten-fixes", "Neo4j_Graph_Algorithms_r3.mobi").presignedUrl(5,
40 | TimeUnit.MINUTES));
41 | {
42 | // create bucket
43 | String bucketName = "temp-bucket-" + System.currentTimeMillis();
44 | String createXml = "\n"
45 | + "\n"
46 | + " " + regionName + " \n"
47 | + " ";
48 | s3.path(bucketName) //
49 | .method(HttpMethod.PUT) //
50 | .requestBody(createXml) //
51 | .execute();
52 |
53 | String objectName = "ExampleObject.txt";
54 | Map> h = s3 //
55 | .path(bucketName, objectName) //
56 | .method(HttpMethod.PUT) //
57 | .requestBody("hi there") //
58 | .metadata("category", "something") //
59 | .response() //
60 | .headers();
61 | System.out.println("put object completed, headers:");
62 | h.entrySet().stream().forEach(x -> System.out.println(" " + x));
63 |
64 | try {
65 | String uploadId = s3 //
66 | .path(bucketName, objectName) //
67 | .query("uploads") //
68 | .method(HttpMethod.POST) //
69 | .responseAsXml() //
70 | .content("UploadId");
71 | System.out.println("uploadId=" + uploadId);
72 |
73 | // upload part 1
74 | String text1 = Stream.repeatElement("hello").take(1200000).join(" ").get();
75 | String tag1 = s3.path(bucketName, objectName) //
76 | .method(HttpMethod.PUT) //
77 | .query("partNumber", "1") //
78 | .query("uploadId", uploadId) //
79 | .requestBody(text1) //
80 | .response() //
81 | .headers() //
82 | .get("ETag") //
83 | .get(0);
84 |
85 | // upload part 2
86 | String text2 = Stream.repeatElement("there").take(1200000).join(" ").get();
87 | String tag2 = s3.path(bucketName, objectName) //
88 | .method(HttpMethod.PUT) //
89 | .query("partNumber", "2") //
90 | .query("uploadId", uploadId) //
91 | .requestBody(text2) //
92 | .response() //
93 | .headers() //
94 | .get("ETag") //
95 | .get(0);
96 | System.out.println("tags=" + tag1 + " " + tag2);
97 | String xml = Xml //
98 | .create("CompleteMultipartUpload") //
99 | .attribute("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/") //
100 | .element("Part") //
101 | .element("ETag").content(tag1.substring(1, tag1.length() - 1)) //
102 | .up() //
103 | .element("PartNumber").content("1") //
104 | .up().up() //
105 | .element("Part") //
106 | .element("ETag").content(tag2.substring(1, tag2.length() - 1)) //
107 | .up() //
108 | .element("PartNumber").content("2") //
109 | .toString();
110 | System.out.println(xml);
111 |
112 | s3.path(bucketName, objectName) //
113 | .method(HttpMethod.POST) //
114 | .query("uploadId", uploadId) //
115 | .header("Content-Type", "application/xml") //
116 | .unsignedPayload() //
117 | .requestBody(xml) //
118 | .execute();
119 |
120 | } catch (Throwable e) {
121 | e.printStackTrace();
122 | }
123 |
124 | // {
125 | // Response r = s3.path(bucketName, objectName) //
126 | // .method(HttpMethod.HEAD) //
127 | // .response();
128 | // DateTimeFormatter dtf = DateTimeFormatter.ofPattern("E, d MMM Y hh:mm:ss z",Locale.ENGLISH).withZone(ZoneId.of("GMT"));
129 | // System.out.println("parsed=" + dtf.parse(r.headers().get("Last-Modified").get(0)));
130 | // System.exit(0);
131 | //
132 | // }
133 |
134 | try {
135 | s3.path(bucketName + "/" + "not-there") //
136 | .responseAsUtf8();
137 | } catch (Throwable e) {
138 | e.printStackTrace(System.out);
139 | }
140 |
141 | try {
142 | Client.s3() //
143 | .region(regionName) //
144 | .credentials(credentials) //
145 | .exception(r -> !r.isOk(), r -> new NoSuchKeyException(r.contentUtf8()))
146 | .build() //
147 | .path(bucketName + "/" + "not-there") //
148 | .responseAsUtf8();
149 | } catch (NoSuchKeyException e) {
150 | e.printStackTrace(System.out);
151 | }
152 |
153 | {
154 | Response r = s3.path(bucketName, "notThere").response();
155 | System.out.println("ok=" + r.isOk() + ", statusCode=" + r.statusCode()
156 | + ", message=" + r.contentUtf8());
157 | }
158 | {
159 | // read bucket object
160 | String text = s3.path(bucketName, objectName).responseAsUtf8();
161 | System.out.println(text);
162 | System.out.println("presignedUrl="
163 | + s3.path("amsa-xml-in" + "/" + objectName).presignedUrl(1, TimeUnit.DAYS));
164 | }
165 | {
166 | // read bucket object as stream
167 | byte[] bytes = Util
168 | .readBytesAndClose(s3.path(bucketName, objectName).responseInputStream());
169 | System.out.println(new String(bytes, StandardCharsets.UTF_8));
170 | System.out.println("presignedUrl="
171 | + s3.path("amsa-xml-in" + "/" + objectName).presignedUrl(1, TimeUnit.DAYS));
172 | }
173 | {
174 | // read bucket object with metadata
175 | Response r = s3 //
176 | .path(bucketName, objectName) //
177 | .response();
178 | System.out.println(r.content().length + " chars read");
179 | r //
180 | .metadata() //
181 | .entrySet() //
182 | .stream() //
183 | .map(x -> x.getKey() + "=" + x.getValue()) //
184 | .forEach(System.out::println);
185 |
186 | System.out.println("category[0]=" + r.metadata("category").orElse(""));
187 | }
188 |
189 | List keys = s3 //
190 | .url("https://" + bucketName + ".s3." + regionName + ".amazonaws.com") //
191 | .query("list-type", "2") //
192 | .responseAsXml() //
193 | .childrenWithName("Contents") //
194 | .stream() //
195 | .map(x -> x.content("Key")) //
196 | .collect(Collectors.toList());
197 |
198 | System.out.println(keys);
199 |
200 | // delete object
201 | s3.path(bucketName, objectName) //
202 | .method(HttpMethod.DELETE) //
203 | .execute();
204 |
205 | // delete bucket
206 | s3.path(bucketName) //
207 | .method(HttpMethod.DELETE) //
208 | .execute();
209 | System.out.println("bucket deleted");
210 |
211 | System.out.println("all actions complete on s3");
212 | }
213 |
214 | {
215 | String queueName = "MyQueue-" + System.currentTimeMillis();
216 |
217 | sqs.query("Action", "CreateQueue") //
218 | .query("QueueName", queueName) //
219 | .execute();
220 |
221 | String queueUrl = sqs //
222 | .query("Action", "GetQueueUrl") //
223 | .query("QueueName", queueName) //
224 | .responseAsXml() //
225 | .content("GetQueueUrlResult", "QueueUrl");
226 |
227 | for (int i = 1; i <= 3; i++) {
228 | sqs.url(queueUrl) //
229 | .query("Action", "SendMessage") //
230 | .query("MessageBody", "hi there --> " + i) //
231 | .execute();
232 | }
233 |
234 | // read all messages, print to console and delete them
235 | List list;
236 | Request request = sqs //
237 | .url(queueUrl) //
238 | .query("Action", "ReceiveMessage");
239 | do {
240 | list = request //
241 | .responseAsXml() //
242 | .child("ReceiveMessageResult") //
243 | .children();
244 |
245 | list.forEach(x -> {
246 | String msg = x.child("Body").content();
247 | System.out.println(msg);
248 | // mark message as read
249 | sqs.url(queueUrl) //
250 | .query("Action", "DeleteMessage") //
251 | .query("ReceiptHandle", x.child("ReceiptHandle").content()) //
252 | .execute();
253 | });
254 | } while (!list.isEmpty());
255 |
256 | sqs.url(queueUrl) //
257 | .query("Action", "DeleteQueue") //
258 | .execute();
259 |
260 | System.out.println("all actions complete on " + queueUrl);
261 | }
262 | {
263 | // test chunked response
264 | if (TEST_CHUNKED) {
265 | try (ResponseInputStream in = s3
266 | .path("moten-fixes", "Neo4j_Graph_Algorithms_r3.mobi")
267 | .responseInputStream()) {
268 | try (OutputStream out = new BufferedOutputStream(
269 | new FileOutputStream("target/thing.mobi"))) {
270 | byte[] buffer = new byte[8192];
271 | int n;
272 | while ((n = in.read(buffer)) != -1) {
273 | out.write(buffer, 0, n);
274 | }
275 | }
276 | }
277 | }
278 | }
279 | }
280 | }
281 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/CredentialsTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import org.junit.Test;
4 |
5 | import com.github.davidmoten.aws.lw.client.internal.Environment;
6 | import com.github.davidmoten.aws.lw.client.internal.EnvironmentDefault;
7 |
8 | public class CredentialsTest {
9 |
10 | @Test
11 | public void testFromEnvironment() {
12 | Environment instance = EnvironmentDefault.INSTANCE;
13 | EnvironmentDefault.INSTANCE = x -> "AWS_CONTAINER_CREDENTIALS_FULL_URI".equals(x) ? null : "thing";
14 | try {
15 | Credentials.fromEnvironment();
16 | } finally {
17 | EnvironmentDefault.INSTANCE = instance;
18 | }
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/FormatsTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import static org.junit.Assert.assertEquals;
4 |
5 | import java.time.Instant;
6 | import java.time.temporal.TemporalAccessor;
7 |
8 | import org.junit.Test;
9 |
10 | public class FormatsTest {
11 |
12 | @Test
13 | public void testFullDate() {
14 | String s = "Wed, 25 Aug 2021 21:55:47 GMT";
15 | TemporalAccessor t = Formats.FULL_DATE.parse(s);
16 | assertEquals(1629928547000L, Instant.from(t).toEpochMilli());
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/HttpClientTesting.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.IOException;
4 | import java.net.URL;
5 | import java.nio.charset.StandardCharsets;
6 | import java.util.Collections;
7 | import java.util.Map;
8 |
9 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
10 |
11 | public final class HttpClientTesting implements HttpClient {
12 |
13 | public static final HttpClientTesting INSTANCE = new HttpClientTesting(false);
14 | public static final HttpClientTesting THROWING = new HttpClientTesting(true);
15 |
16 | private final boolean throwing;
17 | public URL endpointUrl;
18 | public String httpMethod;
19 | public Map headers;
20 | public byte[] requestBody;
21 | public int connectTimeoutMs;
22 | public int readTimeoutMs;
23 |
24 | private HttpClientTesting(boolean throwing) {
25 | this.throwing = throwing;
26 | }
27 |
28 | @Override
29 | public ResponseInputStream request(URL endpointUrl, String httpMethod,
30 | Map headers, byte[] requestBody, int connectTimeoutMs,
31 | int readTimeoutMs) throws IOException {
32 | this.endpointUrl = endpointUrl;
33 | this.httpMethod = httpMethod;
34 | this.headers = headers;
35 | this.requestBody = requestBody;
36 | this.connectTimeoutMs = connectTimeoutMs;
37 | this.readTimeoutMs = readTimeoutMs;
38 | if (throwing) {
39 | throw new IOException("bingo");
40 | } else {
41 | return new ResponseInputStream(() -> {}, 200, Collections.emptyMap(),
42 | Util.emptyInputStream());
43 | }
44 | }
45 |
46 | @Override
47 | public String toString() {
48 | return "HttpClientTesting [\n endpointUrl=" + endpointUrl + "\n httpMethod=" + httpMethod
49 | + "\n headers=" + headers + "\n requestBody="
50 | + new String(requestBody, StandardCharsets.UTF_8) + "\n connectTimeoutMs="
51 | + connectTimeoutMs + "\n readTimeoutMs=" + readTimeoutMs + "\n]";
52 | }
53 |
54 | public String requestBodyString() {
55 | return new String(requestBody, StandardCharsets.UTF_8);
56 | }
57 |
58 | }
59 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/HttpClientTestingWithQueue.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.ByteArrayOutputStream;
4 | import java.io.IOException;
5 | import java.net.URL;
6 | import java.util.LinkedList;
7 | import java.util.List;
8 | import java.util.Map;
9 | import java.util.Queue;
10 | import java.util.concurrent.CopyOnWriteArrayList;
11 |
12 | public class HttpClientTestingWithQueue implements HttpClient {
13 |
14 | // needs to be volatile to work with Multipart async operations
15 | private final Queue queue = new LinkedList<>();
16 | private final List urls = new CopyOnWriteArrayList<>();
17 | private final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
18 |
19 | public void add(ResponseInputStream r) {
20 | queue.add(r);
21 | }
22 |
23 | public void add(IOException e) {
24 | queue.add(e);
25 | }
26 |
27 | public List urls() {
28 | return urls;
29 | }
30 |
31 | public byte[] bytes() {
32 | return bytes.toByteArray();
33 | }
34 |
35 | @Override
36 | public synchronized ResponseInputStream request(URL endpointUrl, String httpMethod, Map headers,
37 | byte[] requestBody, int connectTimeoutMs, int readTimeoutMs) throws IOException {
38 | urls.add(httpMethod + ":" + endpointUrl.toString());
39 | Object o = queue.poll();
40 | if (o instanceof ResponseInputStream) {
41 | ResponseInputStream r = (ResponseInputStream) o;
42 | if (r.statusCode() == 200 && requestBody != null && httpMethod == "PUT") {
43 | bytes.write(requestBody);
44 | }
45 | return r;
46 | } else {
47 | throw (IOException) o;
48 | }
49 | }
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/LightweightMain.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public class LightweightMain {
4 |
5 | public static void main(String[] args) {
6 | Client s3 = Client.s3().region("ap-southeast-2").credentialsFromSystemProperties().build();
7 | System.out.println(s3.path("amsa-xml-in", "ExampleObject.txt").responseAsUtf8());
8 | }
9 |
10 | }
11 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/MultipartMain.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.File;
4 |
5 | public class MultipartMain {
6 |
7 | public static void main(String[] args) {
8 | String regionName = "ap-southeast-2";
9 | String accessKey = System.getProperty("accessKey");
10 | String secretKey = System.getProperty("secretKey");
11 |
12 | Credentials credentials = Credentials.of(accessKey, secretKey);
13 | Client sqs = Client //
14 | .sqs() //
15 | .region(regionName) //
16 | .credentials(credentials) //
17 | .build();
18 | Client s3 = Client.s3().from(sqs).build();
19 |
20 | Multipart //
21 | .s3(s3) //
22 | .bucket("moten-fixes") //
23 | .key("part001.json") //
24 | .upload(new File("/home/dave/part001.json"));
25 | System.out.println("completed upload");
26 | }
27 |
28 | }
29 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/NoSuchKeyException.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | public class NoSuchKeyException extends RuntimeException {
4 |
5 | private static final long serialVersionUID = -1589744530315669585L;
6 |
7 | public NoSuchKeyException(String message) {
8 | super(message);
9 | }
10 |
11 | }
12 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/RequestHelperTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import static org.junit.Assert.assertEquals;
4 | import static org.junit.Assert.assertFalse;
5 | import static org.junit.Assert.assertNull;
6 | import static org.junit.Assert.assertTrue;
7 |
8 | import java.util.List;
9 |
10 | import org.junit.Test;
11 |
12 | import com.github.davidmoten.aws.lw.client.RequestHelper.Parameter;
13 | import com.github.davidmoten.junit.Asserts;
14 |
15 | public class RequestHelperTest {
16 |
17 | @Test
18 | public void isUtilityClass() {
19 | Asserts.assertIsUtilityClass(RequestHelper.class);
20 | }
21 |
22 | @Test
23 | public void testIsEmpty() {
24 | assertTrue(RequestHelper.isEmpty(null));
25 | assertTrue(RequestHelper.isEmpty(new byte[0]));
26 | assertFalse(RequestHelper.isEmpty(new byte[2]));
27 | }
28 |
29 | @Test
30 | public void testExtractQueryParameters() {
31 | List list = RequestHelper.extractQueryParameters("a=1&b=2");
32 | assertEquals(2, list.size());
33 | assertEquals("a", list.get(0).name);
34 | assertEquals("1", list.get(0).value);
35 | assertEquals("b", list.get(1).name);
36 | assertEquals("2", list.get(1).value);
37 | }
38 |
39 | @Test
40 | public void testExtractQueryParametersDoesNotHaveToHaveValue() {
41 | List list = RequestHelper.extractQueryParameters("hello");
42 | assertEquals("hello", list.get(0).name);
43 | assertNull(list.get(0).value);
44 | }
45 |
46 | @Test(expected = RuntimeException.class)
47 | public void testEncoding() {
48 | RequestHelper.parameter("name", "fred", "");
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/RequestTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import static org.junit.Assert.assertEquals;
4 | import static org.junit.Assert.assertFalse;
5 | import static org.junit.Assert.assertTrue;
6 |
7 | import java.io.ByteArrayInputStream;
8 | import java.io.IOException;
9 | import java.util.Collections;
10 | import java.util.HashMap;
11 | import java.util.List;
12 | import java.util.Map;
13 |
14 | import org.junit.Test;
15 |
16 | public class RequestTest {
17 |
18 | @Test
19 | public void testTrim() {
20 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes("abc"));
21 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes("/abc"));
22 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes(" /abc"));
23 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes("abc/"));
24 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes("abc/ "));
25 | assertEquals("abc", Request.trimAndRemoveLeadingAndTrailingSlashes("/abc/"));
26 | }
27 |
28 | @Test
29 | public void testHasBodyWhenContentLengthPresent() throws IOException {
30 | Map> headers = new HashMap<>();
31 | headers.put("content-length", Collections.singletonList("3"));
32 | try (ResponseInputStream r = new ResponseInputStream(() ->{}, 200, headers,
33 | new ByteArrayInputStream(new byte[] { 1, 2, 3 }))) {
34 | assertTrue(Request.hasBody(r));
35 | }
36 | }
37 |
38 | @Test
39 | public void testHasBodyWhenChunked() throws IOException {
40 | Map> headers = new HashMap<>();
41 | headers.put("transfer-encoding", Collections.singletonList("chunkeD"));
42 | try (ResponseInputStream r = new ResponseInputStream(() ->{}, 200, headers,
43 | new ByteArrayInputStream(new byte[] { 1, 2, 3 }))) {
44 | assertTrue(Request.hasBody(r));
45 | }
46 | }
47 |
48 | @Test
49 | public void testHasBodyButNoHeader() throws IOException {
50 | Map> headers = new HashMap<>();
51 | try (ResponseInputStream r = new ResponseInputStream(() ->{}, 200, headers,
52 | new ByteArrayInputStream(new byte[] { 1, 2, 3 }))) {
53 | assertFalse(Request.hasBody(r));
54 | }
55 | }
56 |
57 | @Test
58 | public void testTrimAndEnsureHasTrailingSlash() {
59 | assertEquals("/",Request.trimAndEnsureHasTrailingSlash(""));
60 | assertEquals("/",Request.trimAndEnsureHasTrailingSlash("/"));
61 | assertEquals("abc/",Request.trimAndEnsureHasTrailingSlash("abc"));
62 | assertEquals("abc/",Request.trimAndEnsureHasTrailingSlash("abc/"));
63 | }
64 |
65 | }
66 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/ResponseInputStreamTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.IOException;
4 | import java.util.Collections;
5 |
6 | import org.junit.Test;
7 |
8 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
9 |
10 | public class ResponseInputStreamTest {
11 |
12 | @Test
13 | public void test() throws IOException {
14 | new ResponseInputStream(() ->{}, 200, Collections.emptyMap(), Util.emptyInputStream()).close();
15 | }
16 |
17 | }
18 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/ResponseTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import static org.junit.Assert.assertEquals;
4 | import static org.junit.Assert.assertFalse;
5 | import static org.junit.Assert.assertTrue;
6 |
7 | import java.nio.charset.StandardCharsets;
8 | import java.util.Arrays;
9 | import java.util.Collections;
10 | import java.util.HashMap;
11 | import java.util.List;
12 | import java.util.Map;
13 |
14 | import org.junit.Test;
15 |
16 | public class ResponseTest {
17 |
18 | @Test
19 | public void test() {
20 | byte[] content = "hi there".getBytes(StandardCharsets.UTF_8);
21 | Map> headers = new HashMap<>();
22 | headers.put("x-amz-meta-color", Collections.singletonList("red"));
23 | headers.put("x-amz-meta-thing", Collections.singletonList("under"));
24 | headers.put("blah", Collections.singletonList("stuff"));
25 | Response r = new Response(headers, content, 200);
26 | assertEquals("hi there", r.contentUtf8());
27 | assertTrue(r.isOk());
28 | assertEquals("red", r.metadata("color").get());
29 | assertEquals("under", r.metadata("thing").get());
30 | assertEquals(2, r.metadata().entrySet().size());
31 | assertEquals(3, r.headers().size());
32 | assertEquals(200, r.statusCode());
33 | }
34 |
35 | @Test
36 | public void testFilterNullKeys() {
37 | byte[] content = "hi there".getBytes(StandardCharsets.UTF_8);
38 | Map> headers = new HashMap<>();
39 | headers.put("x-amz-meta-color", Collections.singletonList("red"));
40 | headers.put((String) null, Collections.singletonList("thing"));
41 | Response r = new Response(headers, content, 200);
42 | assertEquals(1, r.metadata().entrySet().size());
43 | assertEquals(2, r.headers().size());
44 | }
45 |
46 | @Test
47 | public void testResponseCodeOk() {
48 | Response r = new Response(Collections.emptyMap(), new byte[0], 210);
49 | assertTrue(r.isOk());
50 | }
51 |
52 | @Test
53 | public void testResponseCode199() {
54 | Response r = new Response(Collections.emptyMap(), new byte[0], 199);
55 | assertTrue(!r.isOk());
56 | }
57 |
58 | @Test
59 | public void testResponseCode300() {
60 | Response r = new Response(Collections.emptyMap(), new byte[0], 300);
61 | assertTrue(!r.isOk());
62 | }
63 |
64 | @Test
65 | public void testExists200() {
66 | Response r = new Response(Collections.emptyMap(), new byte[0], 200);
67 | assertTrue(r.exists());
68 | }
69 |
70 | @Test
71 | public void testExists299() {
72 | Response r = new Response(Collections.emptyMap(), new byte[0], 299);
73 | assertTrue(r.exists());
74 | }
75 |
76 | @Test
77 | public void testExists404() {
78 | Response r = new Response(Collections.emptyMap(), new byte[0], 404);
79 | assertFalse(r.exists());
80 | }
81 |
82 | @Test(expected = ServiceException.class)
83 | public void testExists500() {
84 | Response r = new Response(Collections.emptyMap(), new byte[0], 500);
85 | r.exists();
86 | }
87 |
88 | @Test(expected = ServiceException.class)
89 | public void testExists100() {
90 | Response r = new Response(Collections.emptyMap(), new byte[0], 100);
91 | r.exists();
92 | }
93 |
94 | @Test
95 | public void testFirstHeader() {
96 | Map> map = new HashMap<>();
97 | map.put("thing", Arrays.asList("a", "b"));
98 | Response r = new Response(map, new byte[0], 100);
99 | assertEquals("a", r.firstHeader("thing").get());
100 | }
101 |
102 | @Test
103 | public void testFirstHeaderIsCaseInsensitive() {
104 | Map> map = new HashMap<>();
105 | List v = Arrays.asList("a", "b");
106 | map.put("Thing", v);
107 | Response r = new Response(map, new byte[0], 100);
108 | assertEquals("a", r.firstHeader("THING").get());
109 | assertEquals(v, r.headers().get("Thing"));
110 | assertEquals(v, r.headersLowerCaseKey().get("thing"));
111 | }
112 |
113 | @Test
114 | public void testFirstHeaderFullDateNoHeaders() {
115 | Map> map = new HashMap<>();
116 | Response r = new Response(map, new byte[0], 100);
117 | assertFalse(r.firstHeaderFullDate("date").isPresent());
118 | }
119 |
120 | @Test
121 | public void testFirstHeaderFullDateBlank() {
122 | Map> map = new HashMap<>();
123 | map.put("date", Collections.emptyList());
124 | Response r = new Response(map, new byte[0], 100);
125 | assertFalse(r.firstHeaderFullDate("date").isPresent());
126 | }
127 |
128 | @Test
129 | public void testFirstHeaderFullDate() {
130 | Map> map = new HashMap<>();
131 | map.put("date",
132 | Arrays.asList("Wed, 25 Aug 2021 21:55:47 GMT", "Thu, 26 Aug 2021 21:55:47 GMT"));
133 | Response r = new Response(map, new byte[0], 100);
134 | assertEquals(1629928547000L, r.firstHeaderFullDate("date").get().toEpochMilli());
135 | }
136 | }
137 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/RuntimeAnalysisTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client;
2 |
3 | import java.io.File;
4 | import java.text.DecimalFormat;
5 | import java.util.HashMap;
6 | import java.util.List;
7 |
8 | import org.davidmoten.kool.Statistics;
9 | import org.davidmoten.kool.Stream;
10 | import org.junit.Test;
11 |
12 | public class RuntimeAnalysisTest {
13 |
14 | @Test
15 | public void test() {
16 | report("src/test/resources/one-time-link-lambda-runtimes.txt");
17 | report("src/test/resources/one-time-link-lambda-runtimes-sdk-v1.txt");
18 | report("src/test/resources/one-time-link-lambda-runtimes-sdk-v2.txt");
19 | }
20 |
21 | private void report(String filename) {
22 | List list = Stream.lines(new File(filename)) //
23 | .map(line -> line.trim()) //
24 | .filter(line -> !line.isEmpty()) //
25 | .map(line -> line.replaceAll("\\s+", " ")) //
26 | .map(line -> line.split(" ")) //
27 | .map(x -> new Record(Double.parseDouble(x[0]), Double.parseDouble(x[1]),
28 | Double.parseDouble(x[2]) * 1000)) //
29 | .toList().get();
30 |
31 | Stream.from(list) //
32 | .statistics(x -> x.coldStartRuntime2GBLight)//
33 | .println().go();
34 |
35 | Stream.from(list) //
36 | .statistics(x -> x.actualWarmStartRuntime2GBLightAverage()).println().go();
37 | ;
38 |
39 | Stream.from(list) //
40 | .statistics(x -> x.apigLambdaRequestTimeMs).println().go();
41 |
42 | Stream.from(list) //
43 | .statistics(x -> x.apigLambdaRequestTimeMs - x.coldStartRuntime2GBLight).println()
44 | .go();
45 | }
46 |
47 | static final class Record {
48 | final double coldStartRuntime2GBLight;
49 | final double warmStartRuntime2GBLight10SampleAverage;
50 | final double apigLambdaRequestTimeMs;
51 |
52 | public Record(double coldStartRuntime2GBLight,
53 | double warmStartRuntime2GBLight9SampleAverage, double apigLambdaRequestTimeMs) {
54 | this.coldStartRuntime2GBLight = coldStartRuntime2GBLight;
55 | this.warmStartRuntime2GBLight10SampleAverage = warmStartRuntime2GBLight9SampleAverage;
56 | this.apigLambdaRequestTimeMs = apigLambdaRequestTimeMs;
57 | }
58 |
59 | public double actualWarmStartRuntime2GBLightAverage() {
60 | return (warmStartRuntime2GBLight10SampleAverage * 10 - coldStartRuntime2GBLight) / 9;
61 | }
62 |
63 | @Override
64 | public String toString() {
65 | StringBuilder builder = new StringBuilder();
66 | builder.append("Record [coldStartRuntime2GBLight=");
67 | builder.append(coldStartRuntime2GBLight);
68 | builder.append(", warmStartRuntime2GBLight9SampleAverage=");
69 | builder.append(warmStartRuntime2GBLight10SampleAverage);
70 | builder.append("]");
71 | return builder.toString();
72 | }
73 |
74 | }
75 |
76 | private static Stream lines(String filename) {
77 | return Stream.lines(new File(filename)) //
78 | .map(line -> line.trim()) //
79 | .filter(line -> line.length() > 0) //
80 | .filter(line -> !line.startsWith("#"));
81 | }
82 |
83 | @Test
84 | public void testStaticFields() {
85 | System.out.println("// results with static fields");
86 | lines("src/test/resources/one-time-link-lambda-runtimes-2.txt") //
87 | .map(line -> line.split("\\s+")) //
88 | .skip(1) //
89 | .map(items -> Double.parseDouble(items[0])) //
90 | .statistics(x -> x) //
91 | .println().go();
92 | lines("src/test/resources/one-time-link-lambda-runtimes-sdk-v2-2.txt") //
93 | .filter(line -> line.startsWith("C")) //
94 | .map(line -> line.split(",")) //
95 | .skip(1) //
96 | .map(items -> Double.parseDouble(items[2])) //
97 | .statistics(x -> x).println().go();
98 | lines("src/test/resources/one-time-link-lambda-runtimes-sdk-v2-2.txt") //
99 | .filter(line -> line.startsWith("W")) //
100 | .map(line -> line.split(",")) //
101 | .map(items -> Double.parseDouble(items[2])) //
102 | .statistics(x -> x).println().go();
103 |
104 | System.out.println("request time analysis with static fields");
105 | System.out.println("| | Average | Stdev | Min | Max | n |");
106 | System.out.println("|-------|-------|-------|------|-------|------|");
107 | reportRequestTimeStats("AWS SDK v1", 0);
108 | reportRequestTimeStats("AWS SDK v2", 1);
109 | reportRequestTimeStats("lightweight", 2);
110 | }
111 |
112 | @Test
113 | public void testStaticFields2() {
114 | // lines(
115 | // "src/test/resources/one-time-link-hourly-store-request-times-raw.txt").skip(1) //
116 | // .bufferUntil((list, x) -> x.contains("AEST"), true) //
117 | // .map(x -> x.subList(1, x.size())) //
118 | // .println().go();
119 | Stream>> o = lines(
120 | "src/test/resources/one-time-link-hourly-store-request-times-raw.txt") //
121 | .skip(1) //
122 | .bufferUntil((list, x) -> x.contains("AEST"), true) //
123 | .map(list -> list.subList(1, list.size())) //
124 | .map(list -> Stream //
125 | .from(list) //
126 | .filter(x -> !x.contains("AEST")) //
127 | .map(y -> y.substring(10, y.length() - 1)) //
128 | .toList().get()) //
129 | .filter(list -> !list.stream().anyMatch(x -> Double.parseDouble(x) > 10))
130 | .map(x -> Stream.from(x) //
131 | .mapWithIndex() //
132 | .groupByList( //
133 | HashMap::new, //
134 | y -> (int) (y.index() % 3), //
135 | y -> y.value())
136 | .get());
137 |
138 | System.out.println("cold start");
139 | for (int i = 0; i < 3; i++) {
140 | int j = i;
141 | o.map(x -> x.get(j)).map(x -> x.get(0)).statistics(Double::parseDouble).println().go();
142 | }
143 |
144 | System.out.println("warm start");
145 | Statistics light = o.map(x -> x.get(0)).flatMap(x -> Stream.from(x.subList(1, x.size())))
146 | .statistics(Double::parseDouble).get();
147 | for (int i = 0; i < 3; i++) {
148 | int j = i;
149 | Statistics stats = o.map(x -> x.get(j))
150 | .flatMap(x -> Stream.from(x.subList(1, x.size())))
151 | .statistics(Double::parseDouble).println().get();
152 | System.out.println("z score=" + Math.abs(light.mean() - stats.mean())
153 | / stats.standardDeviation() * Math.sqrt(light.count()));
154 | }
155 | for (int i = 0; i < 3; i++) {
156 | int j = i;
157 | o.map(x -> x.get(j)).flatMap(x -> Stream //
158 | .from(x.subList(1, x.size()))) //
159 | .statistics(Double::parseDouble) //
160 | .map(x -> markdownRow(j + "", x)) //
161 | .println().go();
162 | }
163 | }
164 |
165 | private static void reportRequestTimeStats(String name, int index) {
166 | lines("src/test/resources/one-time-link-hourly-store-request-times.txt") //
167 | .map(line -> line.split("\\s+")) //
168 | .map(items -> Double.parseDouble(items[index])) //
169 | .statistics(x -> x) //
170 | .map(x -> markdownRow(name, x)) //
171 | .println() //
172 | .go();
173 | }
174 |
175 | public static String markdownRow(String name, Statistics x) {
176 | DecimalFormat df = new DecimalFormat("0.000");
177 | return "| **" + name + "** | " + df.format(x.mean()) + " | "
178 | + df.format(x.standardDeviation()) + " | " + df.format(x.min()) + " | "
179 | + df.format(x.max()) + " | " + x.count() + " |";
180 | }
181 |
182 | }
183 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/CredentialsHelperTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import static org.junit.Assert.assertEquals;
4 |
5 | import java.io.UncheckedIOException;
6 | import java.util.Map;
7 |
8 | import org.junit.Test;
9 |
10 | import com.github.davidmoten.aws.lw.client.Credentials;
11 | import com.github.davidmoten.aws.lw.client.HttpClient;
12 | import com.github.davidmoten.aws.lw.client.HttpClientTesting;
13 | import com.github.davidmoten.guavamini.Maps;
14 | import com.github.davidmoten.http.test.server.Server;
15 | import com.github.davidmoten.junit.Asserts;
16 |
17 | public class CredentialsHelperTest {
18 |
19 | @Test
20 | public void testIsUtilityClass() {
21 | Asserts.assertIsUtilityClass(CredentialsHelper.class);
22 | }
23 |
24 | @Test
25 | public void testFromContainer() {
26 | try (Server server = Server.start()) {
27 | Map map = Maps //
28 | .put("AWS_CONTAINER_CREDENTIALS_FULL_URI", server.baseUrl()) //
29 | .put("AWS_CONTAINER_AUTHORIZATION_TOKEN", "abcde") //
30 | .buildImmutable();
31 | server.response().statusCode(200)
32 | .body("{\"AccessKeyId\":\"123\", \"SecretAccessKey\":\"secret\", \"Token\": \"token\"}").add();
33 | Environment env = x -> map.get(x);
34 | Credentials c = CredentialsHelper.credentialsFromEnvironment(env, HttpClient.defaultClient());
35 | assertEquals("123", c.accessKey());
36 | assertEquals("secret", c.secretKey());
37 | assertEquals("token", c.sessionToken().get());
38 | }
39 | }
40 |
41 | @Test(expected = UncheckedIOException.class)
42 | public void testFromContainerIOException() {
43 | try (Server server = Server.start()) {
44 | Map map = Maps //
45 | .put("AWS_CONTAINER_CREDENTIALS_FULL_URI", server.baseUrl()) //
46 | .put("AWS_CONTAINER_AUTHORIZATION_TOKEN", "abcde") //
47 | .buildImmutable();
48 | server.response().statusCode(200)
49 | .body("{\"AccessKeyId\":\"123\", \"SecretAccessKey\":\"secret\", \"Token\": \"token\"}").add();
50 | Environment env = x -> map.get(x);
51 | CredentialsHelper.credentialsFromEnvironment(env, HttpClientTesting.THROWING);
52 | }
53 | }
54 |
55 | @Test(expected = RuntimeException.class)
56 | public void testFromContainerHttpError() {
57 | try (Server server = Server.start()) {
58 | Map map = Maps //
59 | .put("AWS_CONTAINER_CREDENTIALS_FULL_URI", server.baseUrl()) //
60 | .put("AWS_CONTAINER_AUTHORIZATION_TOKEN", "abcde") //
61 | .buildImmutable();
62 | server.response().statusCode(500).add();
63 | Environment env = x -> map.get(x);
64 | CredentialsHelper.credentialsFromEnvironment(env, HttpClient.defaultClient());
65 | }
66 | }
67 |
68 | @Test
69 | public void testTokenFromFile() {
70 | assertEquals("something", CredentialsHelper.readUtf8("src/test/resources/test.txt"));
71 | }
72 |
73 | @Test(expected = IllegalStateException.class)
74 | public void testTokenFromFileDoesNotExist() {
75 | CredentialsHelper.readUtf8("doesNotExist");
76 | }
77 |
78 | @Test
79 | public void testResolveContainerTokenFromFile() {
80 | assertEquals("something", CredentialsHelper.resolveContainerToken(null, "src/test/resources/test.txt"));
81 | }
82 |
83 | @Test
84 | public void testResolveContainerTokenIfAlreadyPresent() {
85 | assertEquals("something", CredentialsHelper.resolveContainerToken("something", null));
86 | }
87 |
88 | @Test(expected = IllegalStateException.class)
89 | public void testResolveContainerTokenNeitherPresent() {
90 | CredentialsHelper.resolveContainerToken(null, null);
91 | }
92 |
93 | }
94 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/EnvironmentDefaultTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import static org.junit.Assert.assertNotNull;
4 |
5 | import org.junit.Test;
6 |
7 | public class EnvironmentDefaultTest {
8 |
9 | @Test
10 | public void testGet() {
11 | String key = System.getenv().keySet().stream().findFirst().get();
12 | assertNotNull(EnvironmentDefault.INSTANCE.get(key));
13 | }
14 |
15 | }
16 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/HttpClientDefaultTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import static org.junit.Assert.assertEquals;
4 | import static org.junit.Assert.assertTrue;
5 | import static org.mockito.Mockito.when;
6 |
7 | import java.io.ByteArrayOutputStream;
8 | import java.io.IOException;
9 | import java.io.UncheckedIOException;
10 | import java.net.HttpURLConnection;
11 |
12 | import org.junit.Assert;
13 | import org.junit.Test;
14 | import org.mockito.Mockito;
15 |
16 | import com.github.davidmoten.aws.lw.client.ResponseInputStream;
17 |
18 | public class HttpClientDefaultTest {
19 |
20 | @Test
21 | public void testGetInputStreamThrows() throws IOException {
22 | HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
23 | when(connection.getInputStream()).thenThrow(IOException.class);
24 | when(connection.getOutputStream()).thenReturn(new ByteArrayOutputStream());
25 | when(connection.getResponseCode()).thenReturn(200);
26 | try {
27 | HttpClientDefault.request(connection, new byte[0]);
28 | Assert.fail();
29 | } catch (UncheckedIOException e) {
30 | // expected
31 | }
32 | Mockito.verify(connection, Mockito.times(1)).disconnect();
33 | }
34 |
35 | @Test
36 | public void testDisconnectThrows() throws IOException {
37 | HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
38 | when(connection.getInputStream()).thenThrow(IOException.class);
39 | when(connection.getOutputStream()).thenReturn(new ByteArrayOutputStream());
40 | when(connection.getResponseCode()).thenReturn(200);
41 | Mockito.doThrow(RuntimeException.class).when(connection).disconnect();
42 | try {
43 | HttpClientDefault.request(connection, new byte[0]);
44 | Assert.fail();
45 | } catch (UncheckedIOException e) {
46 | // expected
47 | }
48 | Mockito.verify(connection, Mockito.times(1)).disconnect();
49 | }
50 |
51 | @Test
52 | public void testGetInputStreamReturnsNull() throws IOException {
53 | HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
54 | when(connection.getInputStream()).thenReturn(null);
55 | when(connection.getOutputStream()).thenReturn(new ByteArrayOutputStream());
56 | when(connection.getResponseCode()).thenReturn(200);
57 | try (ResponseInputStream response = HttpClientDefault.request(connection, new byte[0])) {
58 | assertEquals(200, response.statusCode());
59 | assertEquals(-1, response.read());
60 | assertTrue(response.headers().isEmpty());
61 | }
62 | }
63 |
64 | @Test
65 | public void testIsOKWhenNotOk() throws IOException {
66 | HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
67 | when(connection.getInputStream()).thenReturn(null);
68 | when(connection.getOutputStream()).thenReturn(new ByteArrayOutputStream());
69 | when(connection.getResponseCode()).thenReturn(500);
70 | try (ResponseInputStream response = HttpClientDefault.request(connection, new byte[0])) {
71 | assertEquals(500, response.statusCode());
72 | assertEquals(-1, response.read());
73 | assertTrue(response.headers().isEmpty());
74 | }
75 | }
76 |
77 | @Test
78 | public void testIsOKWhenNotOkStatusCodeLessThan200() throws IOException {
79 | HttpURLConnection connection = Mockito.mock(HttpURLConnection.class);
80 | when(connection.getInputStream()).thenReturn(null);
81 | when(connection.getOutputStream()).thenReturn(new ByteArrayOutputStream());
82 | when(connection.getResponseCode()).thenReturn(100);
83 | try (ResponseInputStream response = HttpClientDefault.request(connection, new byte[0])) {
84 | assertEquals(100, response.statusCode());
85 | assertEquals(-1, response.read());
86 | assertTrue(response.headers().isEmpty());
87 | }
88 | }
89 |
90 | }
91 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/RetriesTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal;
2 |
3 | import static org.junit.Assert.assertFalse;
4 | import static org.junit.Assert.assertTrue;
5 |
6 | import java.io.IOException;
7 | import java.io.UncheckedIOException;
8 |
9 | import org.junit.Test;
10 |
11 | public class RetriesTest {
12 |
13 | @Test(expected = IllegalArgumentException.class)
14 | public void testBadJitter() {
15 | double jitter = -1;
16 | new Retries(100, 10, 2.0, jitter, 30000, x -> false, x -> false);
17 | }
18 |
19 | @Test(expected = IllegalArgumentException.class)
20 | public void testBadJitter2() {
21 | double jitter = 2;
22 | new Retries(100, 10, 2.0, jitter, 30000, x -> false, x -> false);
23 | }
24 |
25 | @Test(expected = OutOfMemoryError.class)
26 | public void testRethrowError() {
27 | Retries.rethrow(new OutOfMemoryError());
28 | }
29 |
30 | @Test(expected = NullPointerException.class)
31 | public void testRethrowRuntimeError() {
32 | Retries.rethrow(new NullPointerException());
33 | }
34 |
35 | @Test(expected = UncheckedIOException.class)
36 | public void testRethrowIOException() {
37 | Retries.rethrow(new IOException());
38 | }
39 |
40 | @Test(expected = RuntimeException.class)
41 | public void testRethrowException() {
42 | Retries.rethrow(new Exception());
43 | }
44 |
45 | @Test
46 | public void testNotYetReachedMaxAttempts() {
47 | assertFalse(Retries.reachedMaxAttempts(1, 2));
48 | }
49 |
50 | @Test
51 | public void testReachedMaxAttemptsExactly() {
52 | assertTrue(Retries.reachedMaxAttempts(2, 2));
53 | }
54 |
55 | @Test
56 | public void testExceededMaxAttempts() {
57 | assertTrue(Retries.reachedMaxAttempts(3, 2));
58 | }
59 |
60 | @Test
61 | public void testUnlimitedAtempts() {
62 | assertFalse(Retries.reachedMaxAttempts(3, 0));
63 | }
64 |
65 | }
66 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/auth/Aws4SignerForChunkedUpload.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.auth;
2 |
3 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.ALGORITHM;
4 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.SCHEME;
5 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.TERMINATOR;
6 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.dateStampFormat;
7 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getCanonicalRequest;
8 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getCanonicalizeHeaderNames;
9 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getCanonicalizedHeaderString;
10 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getCanonicalizedQueryString;
11 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getStringToSign;
12 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.sign;
13 |
14 | import java.net.URL;
15 | import java.nio.charset.StandardCharsets;
16 | import java.util.Date;
17 | import java.util.Map;
18 |
19 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
20 |
21 | /**
22 | * Sample AWS4 signer demonstrating how to sign 'chunked' uploads
23 | */
24 | public final class Aws4SignerForChunkedUpload {
25 |
26 | /**
27 | * SHA256 substitute marker used in place of x-amz-content-sha256 when employing
28 | * chunked uploads
29 | */
30 | public static final String STREAMING_BODY_SHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD";
31 |
32 | private static final String CLRF = "\r\n";
33 | private static final String CHUNK_STRING_TO_SIGN_PREFIX = "AWS4-HMAC-SHA256-PAYLOAD";
34 | private static final String CHUNK_SIGNATURE_HEADER = ";chunk-signature=";
35 | private static final int SIGNATURE_LENGTH = 64;
36 | private static final byte[] FINAL_CHUNK = new byte[0];
37 |
38 | /**
39 | * Tracks the previously computed signature value; for chunk 0 this will contain
40 | * the signature included in the Authorization header. For subsequent chunks it
41 | * contains the computed signature of the prior chunk.
42 | */
43 | private String lastComputedSignature;
44 |
45 | /**
46 | * Date and time of the original signing computation, in ISO 8601 basic format,
47 | * reused for each chunk
48 | */
49 | private String dateTimeStamp;
50 |
51 | /**
52 | * The scope value of the original signing computation, reused for each chunk
53 | */
54 | private String scope;
55 |
56 | /**
57 | * The derived signing key used in the original signature computation and
58 | * re-used for each chunk
59 | */
60 | private byte[] signingKey;
61 | private final URL endpointUrl;
62 | private final String httpMethod;
63 | private final String serviceName;
64 | private final String regionName;
65 |
66 | public Aws4SignerForChunkedUpload(URL endpointUrl, String httpMethod, String serviceName,
67 | String regionName) {
68 | this.endpointUrl = endpointUrl;
69 | this.httpMethod = httpMethod;
70 | this.serviceName = serviceName;
71 | this.regionName = regionName;
72 | }
73 |
74 | /**
75 | * Computes an AWS4 signature for a request, ready for inclusion as an
76 | * 'Authorization' header.
77 | *
78 | * @param headers The request headers; 'Host' and 'X-Amz-Date' will be
79 | * added to this set.
80 | * @param queryParameters Any query parameters that will be added to the
81 | * endpoint. The parameters should be specified in
82 | * canonical format.
83 | * @param bodyHash Precomputed SHA256 hash of the request body content;
84 | * this value should also be set as the header
85 | * 'X-Amz-Content-SHA256' for non-streaming uploads.
86 | * @param awsAccessKey The user's AWS Access Key.
87 | * @param awsSecretKey The user's AWS Secret Key.
88 | * @return The computed authorization string for the request. This value needs
89 | * to be set as the header 'Authorization' on the subsequent HTTP
90 | * request.
91 | */
92 | public String computeSignature(Map headers, Map queryParameters,
93 | String bodyHash, String awsAccessKey, String awsSecretKey) {
94 | // first get the date and time for the subsequent request, and convert
95 | // to ISO 8601 format for use in signature generation
96 | Date now = new Date();
97 | this.dateTimeStamp = AwsSignatureVersion4.dateTimeFormat().format(now);
98 |
99 | // update the headers with required 'x-amz-date' and 'host' values
100 | headers.put("x-amz-date", dateTimeStamp);
101 |
102 | String hostHeader = endpointUrl.getHost();
103 | int port = endpointUrl.getPort();
104 | if (port > -1) {
105 | hostHeader = hostHeader.concat(":" + port);
106 | }
107 | headers.put("Host", hostHeader);
108 |
109 | // canonicalize the headers; we need the set of header names as well as the
110 | // names and values to go into the signature process
111 | String canonicalizedHeaderNames = getCanonicalizeHeaderNames(headers);
112 | String canonicalizedHeaders = getCanonicalizedHeaderString(headers);
113 |
114 | // if any query string parameters have been supplied, canonicalize them
115 | String canonicalizedQueryParameters = getCanonicalizedQueryString(queryParameters);
116 |
117 | // canonicalize the various components of the request
118 | String canonicalRequest = getCanonicalRequest(endpointUrl, httpMethod,
119 | canonicalizedQueryParameters, canonicalizedHeaderNames, canonicalizedHeaders,
120 | bodyHash);
121 | System.out.println("--------- Canonical request --------");
122 | System.out.println(canonicalRequest);
123 | System.out.println("------------------------------------");
124 |
125 | // construct the string to be signed
126 | String dateStamp = dateStampFormat().format(now);
127 | this.scope = dateStamp + "/" + regionName + "/" + serviceName + "/" + TERMINATOR;
128 | String stringToSign = getStringToSign(SCHEME, ALGORITHM, dateTimeStamp, scope,
129 | canonicalRequest);
130 | System.out.println("--------- String to sign -----------");
131 | System.out.println(stringToSign);
132 | System.out.println("------------------------------------");
133 |
134 | // compute the signing key
135 | byte[] kSecret = (SCHEME + awsSecretKey).getBytes(StandardCharsets.UTF_8);
136 | byte[] kDate = sign(dateStamp, kSecret);
137 | byte[] kRegion = sign(regionName, kDate);
138 | byte[] kService = sign(serviceName, kRegion);
139 | this.signingKey = sign(TERMINATOR, kService);
140 | byte[] signature = sign(stringToSign, signingKey);
141 |
142 | // cache the computed signature ready for chunk 0 upload
143 | lastComputedSignature = Util.toHex(signature);
144 |
145 | String credentialsAuthorizationHeader = "Credential=" + awsAccessKey + "/" + scope;
146 | String signedHeadersAuthorizationHeader = "SignedHeaders=" + canonicalizedHeaderNames;
147 | String signatureAuthorizationHeader = "Signature=" + lastComputedSignature;
148 |
149 | String authorizationHeader = SCHEME + "-" + ALGORITHM + " " + credentialsAuthorizationHeader
150 | + ", " + signedHeadersAuthorizationHeader + ", " + signatureAuthorizationHeader;
151 |
152 | return authorizationHeader;
153 | }
154 |
155 | /**
156 | * Calculates the expanded payload size of our data when it is chunked
157 | *
158 | * @param originalLength The true size of the data payload to be uploaded
159 | * @param chunkSize The size of each chunk we intend to send; each chunk
160 | * will be prefixed with signed header data, expanding the
161 | * overall size by a determinable amount
162 | * @return The overall payload size to use as content-length on a chunked upload
163 | */
164 | public static long calculateChunkedContentLength(long originalLength, long chunkSize) {
165 | if (originalLength <= 0) {
166 | throw new IllegalArgumentException("Nonnegative content length expected.");
167 | }
168 |
169 | long maxSizeChunks = originalLength / chunkSize;
170 | long remainingBytes = originalLength % chunkSize;
171 | return maxSizeChunks * calculateChunkHeaderLength(chunkSize)
172 | + (remainingBytes > 0 ? calculateChunkHeaderLength(remainingBytes) : 0)
173 | + calculateChunkHeaderLength(0);
174 | }
175 |
176 | /**
177 | * Returns the size of a chunk header, which only varies depending on the
178 | * selected chunk size
179 | *
180 | * @param chunkDataSize The intended size of each chunk; this is placed into the
181 | * chunk header
182 | * @return The overall size of the header that will prefix the user data in each
183 | * chunk
184 | */
185 | private static long calculateChunkHeaderLength(long chunkDataSize) {
186 | return Long.toHexString(chunkDataSize).length() + CHUNK_SIGNATURE_HEADER.length()
187 | + SIGNATURE_LENGTH + CLRF.length() + chunkDataSize + CLRF.length();
188 | }
189 |
190 | /**
191 | * Returns a chunk for upload consisting of the signed 'header' or chunk prefix
192 | * plus the user data. The signature of the chunk incorporates the signature of
193 | * the previous chunk (or, if the first chunk, the signature of the headers
194 | * portion of the request).
195 | *
196 | * @param userDataLen The length of the user data contained in userData
197 | * @param userData Contains the user data to be sent in the upload chunk
198 | * @return A new buffer of data for upload containing the chunk header plus user
199 | * data
200 | */
201 | public byte[] constructSignedChunk(int userDataLen, byte[] userData) {
202 | // to keep our computation routine signatures simple, if the userData
203 | // buffer contains less data than it could, shrink it. Note the special case
204 | // to handle the requirement that we send an empty chunk to complete
205 | // our chunked upload.
206 | byte[] dataToChunk;
207 | if (userDataLen == 0) {
208 | dataToChunk = FINAL_CHUNK;
209 | } else {
210 | if (userDataLen < userData.length) {
211 | // shrink the chunkdata to fit
212 | dataToChunk = new byte[userDataLen];
213 | System.arraycopy(userData, 0, dataToChunk, 0, userDataLen);
214 | } else {
215 | dataToChunk = userData;
216 | }
217 | }
218 |
219 | StringBuilder chunkHeader = new StringBuilder();
220 |
221 | // start with size of user data
222 | chunkHeader.append(Integer.toHexString(dataToChunk.length));
223 |
224 | // nonsig-extension; we have none in these samples
225 | String nonsigExtension = "";
226 |
227 | // if this is the first chunk, we package it with the signing result
228 | // of the request headers, otherwise we use the cached signature
229 | // of the previous chunk
230 |
231 | // sig-extension
232 | String chunkStringToSign = CHUNK_STRING_TO_SIGN_PREFIX + "\n" + dateTimeStamp + "\n" + scope
233 | + "\n" + lastComputedSignature + "\n" + Util.toHex(Util.sha256(nonsigExtension))
234 | + "\n" + Util.toHex(Util.sha256(dataToChunk));
235 |
236 | // compute the V4 signature for the chunk
237 | String chunkSignature = Util.toHex(AwsSignatureVersion4.sign(chunkStringToSign, signingKey));
238 |
239 | // cache the signature to include with the next chunk's signature computation
240 | lastComputedSignature = chunkSignature;
241 |
242 | // construct the actual chunk, comprised of the non-signed extensions, the
243 | // 'headers' we just signed and their signature, plus a newline then copy
244 | // that plus the user's data to a payload to be written to the request stream
245 | chunkHeader.append(nonsigExtension + CHUNK_SIGNATURE_HEADER + chunkSignature);
246 | chunkHeader.append(CLRF);
247 |
248 | byte[] header = chunkHeader.toString().getBytes(StandardCharsets.UTF_8);
249 | byte[] trailer = CLRF.getBytes(StandardCharsets.UTF_8);
250 | byte[] signedChunk = new byte[header.length + dataToChunk.length + trailer.length];
251 | System.arraycopy(header, 0, signedChunk, 0, header.length);
252 | System.arraycopy(dataToChunk, 0, signedChunk, header.length, dataToChunk.length);
253 | System.arraycopy(trailer, 0, signedChunk, header.length + dataToChunk.length,
254 | trailer.length);
255 |
256 | // this is the total data for the chunk that will be sent to the request stream
257 | return signedChunk;
258 | }
259 | }
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/auth/AwsSignatureVersion4Test.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.auth;
2 |
3 | import static com.github.davidmoten.aws.lw.client.internal.auth.AwsSignatureVersion4.getCanonicalizedResourcePath;
4 | import static org.junit.Assert.assertEquals;
5 |
6 | import java.net.MalformedURLException;
7 | import java.net.URL;
8 |
9 | import org.junit.Test;
10 |
11 | public class AwsSignatureVersion4Test {
12 |
13 | @Test
14 | public void testPath() throws MalformedURLException {
15 | assertEquals("/", getCanonicalizedResourcePath(new URL("https://")));
16 | }
17 |
18 | @Test
19 | public void testPath2() throws MalformedURLException {
20 | assertEquals("/hi", getCanonicalizedResourcePath(new URL("https://blah.com/hi")));
21 | }
22 |
23 | @Test(expected=RuntimeException.class)
24 | public void testSignBadAlgorithmThrows() {
25 | AwsSignatureVersion4.sign("hi there", new byte[] {1,2,3,4}, "doesnotexist");
26 | }
27 |
28 | @Test(expected=RuntimeException.class)
29 | public void testSignBadKeyThrows2() {
30 | AwsSignatureVersion4.sign("hi there", new byte[] {}, AwsSignatureVersion4.ALGORITHM_HMAC_SHA256);
31 | }
32 |
33 | }
34 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/auth/HttpUtils.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.auth;
2 |
3 | import java.io.BufferedReader;
4 | import java.io.IOException;
5 | import java.io.InputStream;
6 | import java.io.InputStreamReader;
7 | import java.net.HttpURLConnection;
8 | import java.net.URL;
9 | import java.nio.charset.StandardCharsets;
10 | import java.util.Map;
11 |
12 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
13 |
14 | /**
15 | * Various Http helper routines
16 | */
17 | public class HttpUtils {
18 |
19 | private static final int CONNECT_TIMEOUT_MS = 30000;
20 | private static final int READ_TIMEOUT_MS = 5 * 60000;
21 |
22 | public static String executeHttpRequest(HttpURLConnection connection) {
23 | try {
24 | // Get Response
25 | InputStream is;
26 | try {
27 | is = connection.getInputStream();
28 | } catch (IOException e) {
29 | is = connection.getErrorStream();
30 | }
31 |
32 | try (BufferedReader rd = new BufferedReader(
33 | new InputStreamReader(is, StandardCharsets.UTF_8))) {
34 | String line;
35 | StringBuffer response = new StringBuffer();
36 | while ((line = rd.readLine()) != null) {
37 | response.append(line);
38 | response.append('\r');
39 | }
40 | return response.toString();
41 | }
42 | } catch (IOException | RuntimeException e) {
43 | throw new RuntimeException("Request failed. " + e.getMessage(), e);
44 | } finally {
45 | if (connection != null) {
46 | connection.disconnect();
47 | }
48 | }
49 | }
50 |
51 | public static HttpURLConnection createHttpConnection(URL endpointUrl, String httpMethod,
52 | Map headers) throws IOException {
53 | return Util.createHttpConnection(endpointUrl, httpMethod, headers, CONNECT_TIMEOUT_MS,
54 | READ_TIMEOUT_MS);
55 | }
56 |
57 | }
58 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/auth/PutS3ObjectChunkedSample.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.auth;
2 |
3 | import java.io.ByteArrayInputStream;
4 | import java.io.DataOutputStream;
5 | import java.net.HttpURLConnection;
6 | import java.net.URL;
7 | import java.util.HashMap;
8 | import java.util.Map;
9 |
10 | import com.github.davidmoten.aws.lw.client.internal.util.Util;
11 |
12 | /**
13 | * Sample code showing how to PUT objects to Amazon S3 using chunked uploading
14 | * with Signature V4 authorization
15 | */
16 | public class PutS3ObjectChunkedSample {
17 |
18 | private static final String contentSeed =
19 | "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Nunc tortor metus, sagittis eget augue ut,\n"
20 | + "feugiat vehicula risus. Integer tortor mauris, vehicula nec mollis et, consectetur eget tortor. In ut\n"
21 | + "elit sagittis, ultrices est ut, iaculis turpis. In hac habitasse platea dictumst. Donec laoreet tellus\n"
22 | + "at auctor tempus. Praesent nec diam sed urna sollicitudin vehicula eget id est. Vivamus sed laoreet\n"
23 | + "lectus. Aliquam convallis condimentum risus, vitae porta justo venenatis vitae. Phasellus vitae nunc\n"
24 | + "varius, volutpat quam nec, mollis urna. Donec tempus, nisi vitae gravida facilisis, sapien sem malesuada\n"
25 | + "purus, id semper libero ipsum condimentum nulla. Suspendisse vel mi leo. Morbi pellentesque placerat congue.\n"
26 | + "Nunc sollicitudin nunc diam, nec hendrerit dui commodo sed. Duis dapibus commodo elit, id commodo erat\n"
27 | + "congue id. Aliquam erat volutpat.\n";
28 |
29 | /**
30 | * Uploads content to an Amazon S3 object in a series of signed 'chunks' using Signature V4 authorization.
31 | */
32 | public static void putS3ObjectChunked(String bucketName, String regionName, String awsAccessKey, String awsSecretKey) {
33 | System.out.println("***************************************************");
34 | System.out.println("* Executing sample 'PutS3ObjectChunked' *");
35 | System.out.println("***************************************************");
36 |
37 | // this sample uses a chunk data length of 64K; this should yield one
38 | // 64K chunk, one partial chunk and the final 0 byte payload terminator chunk
39 | final int userDataBlockSize = 64 * 1024;
40 | String sampleContent = make65KPayload();
41 |
42 | URL endpointUrl;
43 | if (regionName.equals("us-east-1")) {
44 | endpointUrl = Util.toUrl("https://s3.amazonaws.com/" + bucketName + "/ExampleChunkedObject.txt");
45 | } else {
46 | endpointUrl = Util.toUrl("https://s3-" + regionName + ".amazonaws.com/" + bucketName + "/ExampleChunkedObject.txt");
47 | }
48 |
49 | // set the markers indicating we're going to send the upload as a series
50 | // of chunks:
51 | // -- 'x-amz-content-sha256' is the fixed marker indicating chunked
52 | // upload
53 | // -- 'content-length' becomes the total size in bytes of the upload
54 | // (including chunk headers),
55 | // -- 'x-amz-decoded-content-length' is used to transmit the actual
56 | // length of the data payload, less chunk headers
57 |
58 | Map headers = new HashMap();
59 | headers.put("x-amz-storage-class", "REDUCED_REDUNDANCY");
60 | headers.put("x-amz-content-sha256", Aws4SignerForChunkedUpload.STREAMING_BODY_SHA256);
61 | headers.put("content-encoding", "" + "aws-chunked");
62 | headers.put("x-amz-decoded-content-length", "" + sampleContent.length());
63 |
64 | Aws4SignerForChunkedUpload signer = new Aws4SignerForChunkedUpload(
65 | endpointUrl, "PUT", "s3", regionName);
66 |
67 | // how big is the overall request stream going to be once we add the signature
68 | // 'headers' to each chunk?
69 | long totalLength = Aws4SignerForChunkedUpload.calculateChunkedContentLength(sampleContent.length(), userDataBlockSize);
70 | headers.put("content-length", "" + totalLength);
71 |
72 | String authorization = signer.computeSignature(headers,
73 | null, // no query parameters
74 | Aws4SignerForChunkedUpload.STREAMING_BODY_SHA256,
75 | awsAccessKey,
76 | awsSecretKey);
77 |
78 | // place the computed signature into a formatted 'Authorization' header
79 | // and call S3
80 | headers.put("Authorization", authorization);
81 |
82 | // start consuming the data payload in blocks which we subsequently chunk; this prefixes
83 | // the data with a 'chunk header' containing signature data from the prior chunk (or header
84 | // signing, if the first chunk) plus length and other data. Each completed chunk is
85 | // written to the request stream and to complete the upload, we send a final chunk with
86 | // a zero-length data payload.
87 |
88 | try {
89 | // first set up the connection
90 | HttpURLConnection connection = HttpUtils.createHttpConnection(endpointUrl, "PUT", headers);
91 |
92 | // get the request stream and start writing the user data as chunks, as outlined
93 | // above;
94 | byte[] buffer = new byte[userDataBlockSize];
95 | DataOutputStream outputStream = new DataOutputStream(connection.getOutputStream());
96 |
97 | // get the data stream
98 | ByteArrayInputStream inputStream = new ByteArrayInputStream(sampleContent.getBytes("UTF-8"));
99 |
100 | int bytesRead = 0;
101 | while ( (bytesRead = inputStream.read(buffer, 0, buffer.length)) != -1 ) {
102 | // process into a chunk
103 | byte[] chunk = signer.constructSignedChunk(bytesRead, buffer);
104 |
105 | // send the chunk
106 | outputStream.write(chunk);
107 | outputStream.flush();
108 | }
109 |
110 | // last step is to send a signed zero-length chunk to complete the upload
111 | byte[] finalChunk = signer.constructSignedChunk(0, buffer);
112 | outputStream.write(finalChunk);
113 | outputStream.flush();
114 | outputStream.close();
115 |
116 | // make the call to Amazon S3
117 | String response = HttpUtils.executeHttpRequest(connection);
118 | System.out.println("--------- Response content ---------");
119 | System.out.println(response);
120 | System.out.println("------------------------------------");
121 | } catch (Exception e) {
122 | throw new RuntimeException("Error when sending chunked upload request. " + e.getMessage(), e);
123 | }
124 | }
125 |
126 | /**
127 | * Want sample to upload 3 chunks for our selected chunk size of 64K; one
128 | * full size chunk, one partial chunk and then the 0-byte terminator chunk.
129 | * This routine just takes 1K of seed text and turns it into a 65K-or-so
130 | * string for sample use.
131 | */
132 | private static String make65KPayload() {
133 | StringBuilder oneKSeed = new StringBuilder();
134 | while ( oneKSeed.length() < 1024 ) {
135 | oneKSeed.append(contentSeed);
136 | }
137 |
138 | // now scale up to meet/exceed our requirement
139 | StringBuilder output = new StringBuilder();
140 | for (int i = 0; i < 66; i++) {
141 | output.append(oneKSeed);
142 | }
143 | return output.toString();
144 | }
145 | }
146 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/util/PreconditionsTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.util;
2 |
3 | import org.junit.Test;
4 |
5 | import com.github.davidmoten.junit.Asserts;
6 |
7 | public class PreconditionsTest {
8 |
9 | @Test
10 | public void isUtilityClass() {
11 | Asserts.assertIsUtilityClass(Preconditions.class);
12 | }
13 |
14 | @Test(expected=IllegalArgumentException.class)
15 | public void wantIAEnotNPE() {
16 | Preconditions.checkNotNull(null, "hey!");
17 | }
18 |
19 | }
20 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/internal/util/UtilTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.internal.util;
2 |
3 | import static org.junit.Assert.assertArrayEquals;
4 | import static org.junit.Assert.assertEquals;
5 | import static org.junit.Assert.assertFalse;
6 | import static org.junit.Assert.assertTrue;
7 |
8 | import java.io.ByteArrayInputStream;
9 | import java.io.IOException;
10 | import java.io.InputStream;
11 | import java.io.UncheckedIOException;
12 | import java.net.URL;
13 | import java.nio.charset.StandardCharsets;
14 | import java.util.Collections;
15 | import java.util.Optional;
16 | import java.util.concurrent.atomic.AtomicBoolean;
17 |
18 | import org.junit.Assert;
19 | import org.junit.Ignore;
20 | import org.junit.Test;
21 |
22 | import com.github.davidmoten.junit.Asserts;
23 |
24 | public class UtilTest {
25 |
26 | @Test
27 | public void isUtilityClass() {
28 | Asserts.assertIsUtilityClass(Util.class);
29 | }
30 |
31 | @Test(expected = RuntimeException.class)
32 | public void testHash() {
33 | Util.hash("hi there".getBytes(StandardCharsets.UTF_8), "does not exist");
34 | }
35 |
36 | @Test(expected = RuntimeException.class)
37 | public void testToUrl() {
38 | Util.toUrl("bad");
39 | }
40 |
41 | @Test(expected = RuntimeException.class)
42 | public void testUrlEncode() {
43 | Util.urlEncode("abc://google.com", true, "does not exist");
44 | }
45 |
46 | @Test
47 | public void testUrlEncodeAsterisk() {
48 | assertEquals("%2A", Util.urlEncode("*", true));
49 | }
50 |
51 | @Test
52 | public void testUrlEncodeAllSpecialChars() {
53 | String nonEncodedCharacters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_.~";
54 | String encodedCharactersInput = "\t\n\r !\"#$%&'()*+,/:;<=>?@[\\]^`{|}";
55 | String encodedCharactersOutput = "%09%0A%0D%20%21%22%23%24%25%26%27%28%29%2A%2B%2C%2F%3A%3B%3C%3D%3E%3F%40%5B%5C%5D%5E%60%7B%7C%7D";
56 |
57 | assertEquals(Util.urlEncode("", true), "");
58 | assertEquals(nonEncodedCharacters, Util.urlEncode(nonEncodedCharacters, false));
59 | assertEquals(encodedCharactersOutput, Util.urlEncode(encodedCharactersInput, false));
60 | }
61 |
62 | @Test
63 | public void testCreateConnectionBad() throws IOException {
64 | Util.createHttpConnection(new URL("https://doesnotexist.never12345"), "GET",
65 | Collections.emptyMap(), 100, 100);
66 | }
67 |
68 | @Test
69 | public void testReadAndClose() {
70 | byte[] b = "hi there".getBytes(StandardCharsets.UTF_8);
71 | ByteArrayInputStream in = new ByteArrayInputStream(b);
72 | assertArrayEquals(b, Util.readBytesAndClose(in));
73 | }
74 |
75 | @Test
76 | public void testReadAndCloseReadThrows() {
77 | AtomicBoolean closed = new AtomicBoolean();
78 | InputStream in = new InputStream() {
79 |
80 | @Override
81 | public int read() throws IOException {
82 | throw new IOException("boo");
83 | }
84 |
85 | @Override
86 | public void close() {
87 | closed.set(true);
88 | }
89 | };
90 | try {
91 | Util.readBytesAndClose(in);
92 | Assert.fail();
93 | } catch (UncheckedIOException e) {
94 | // expected
95 | assertTrue(closed.get());
96 | }
97 | }
98 |
99 | @Test
100 | public void testReadAndCloseThrows() {
101 | InputStream in = new InputStream() {
102 |
103 | @Override
104 | public int read() throws IOException {
105 | return -1;
106 | }
107 |
108 | @Override
109 | public void close() throws IOException {
110 | throw new IOException("boo");
111 | }
112 | };
113 | try {
114 | Util.readBytesAndClose(in);
115 | Assert.fail();
116 | } catch (UncheckedIOException e) {
117 | // expected
118 | }
119 | }
120 |
121 | @Test
122 | public void testCanonicalMetadata() {
123 | assertEquals("abc123", Util.canonicalMetadataKey("abc123"));
124 | }
125 |
126 | @Test
127 | public void testCanonicalMetadataIgnoresDisallowedCharacters() {
128 | assertEquals("abc123", Util.canonicalMetadataKey("abc123@!"));
129 | }
130 |
131 | // json tests generated by chatgpt
132 |
133 | @Test
134 | public void testJsonFieldTextExtractStringField() {
135 | String json = "{\"name\":\"John\"}";
136 | assertEquals(Optional.of("John"), Util.jsonFieldText(json, "name"));
137 | }
138 |
139 | @Test
140 | public void testJsonFieldTextExtractStringFieldWithEscapedQuotes() {
141 | String json = "{\"name\":\"John \\\"Doe\\\"\"}";
142 | assertEquals(Optional.of("John \"Doe\""), Util.jsonFieldText(json, "name"));
143 | }
144 |
145 | @Test
146 | public void testJsonFieldTextExtractNonStringField() {
147 | String json = "{\"age\":30}";
148 | assertEquals(Optional.of("30"), Util.jsonFieldText(json, "age"));
149 | }
150 |
151 | @Test
152 | public void testJsonFieldTextExtractBooleanField() {
153 | String json = "{\"isActive\":true}";
154 | assertEquals(Optional.of("true"), Util.jsonFieldText(json, "isActive"));
155 | }
156 |
157 | @Test
158 | @Ignore // should return empty
159 | public void testJsonFieldTextExtractNullField() {
160 | String json = "{\"middleName\":null}";
161 | assertEquals(Optional.of("null"), Util.jsonFieldText(json, "middleName"));
162 | }
163 |
164 | @Test
165 | public void testJsonFieldTextFieldNotFound() {
166 | String json = "{\"name\":\"John\"}";
167 | assertEquals(Optional.empty(), Util.jsonFieldText(json, "age"));
168 | }
169 |
170 | @Test
171 | public void testJsonFieldTextEmptyJson() {
172 | String json = "{}";
173 | assertEquals(Optional.empty(), Util.jsonFieldText(json, "name"));
174 | }
175 |
176 | @Test
177 | public void testJsonFieldTextJsonWithWhitespace() {
178 | String json = " { \"name\" : \"John\" , \"age\" : 30 } ";
179 | assertEquals(Optional.of("John"), Util.jsonFieldText(json, "name"));
180 | assertEquals(Optional.of("30"), Util.jsonFieldText(json, "age"));
181 | }
182 |
183 | @Test
184 | public void testJsonFieldTextJsonWithExtraFields() {
185 | String json = "{\"name\":\"John\", \"age\":30, \"city\":\"New York\"}";
186 | assertEquals(Optional.of("John"), Util.jsonFieldText(json, "name"));
187 | assertEquals(Optional.of("30"), Util.jsonFieldText(json, "age"));
188 | assertEquals(Optional.of("New York"), Util.jsonFieldText(json, "city"));
189 | }
190 |
191 | @Test
192 | public void testJsonFieldTextJsonWithNestedQuotesInStringField() {
193 | String json = "{\"quote\":\"\\\"To be or not to be\\\"\"}";
194 | assertEquals(Optional.of("\"To be or not to be\""), Util.jsonFieldText(json, "quote"));
195 | }
196 |
197 | @Test
198 | public void testJsonFieldTextJsonWithTrailingComma() {
199 | String json = "{\"name\":\"John\",}";
200 | assertEquals(Optional.of("John"), Util.jsonFieldText(json, "name"));
201 | }
202 |
203 | @Test
204 | public void testJsonFieldTextFieldWithSpecialCharacters() {
205 | String json = "{\"key-1\":\"value!@#$%^&*()\"}";
206 | assertEquals(Optional.of("value!@#$%^&*()"), Util.jsonFieldText(json, "key-1"));
207 | }
208 |
209 | @Test
210 | public void testJsonFieldTextMultipleFieldsWithSameName() {
211 | String json = "{\"name\":\"John\", \"other\":{\"name\":\"Doe\"}}";
212 | assertEquals(Optional.of("John"), Util.jsonFieldText(json, "name"));
213 | }
214 |
215 | @Test
216 | public void testJsonFieldNoColon() {
217 | String json = "[\"name\"]";
218 | assertFalse(Util.jsonFieldText(json, "name").isPresent());
219 | }
220 |
221 | @Test
222 | public void testJsonFieldIsNull() {
223 | String json = "{\"name\": null}";
224 | assertFalse(Util.jsonFieldText(json, "name").isPresent());
225 | }
226 | }
227 |
--------------------------------------------------------------------------------
/src/test/java/com/github/davidmoten/aws/lw/client/xml/builder/XmlTest.java:
--------------------------------------------------------------------------------
1 | package com.github.davidmoten.aws.lw.client.xml.builder;
2 |
3 | import static org.junit.Assert.assertEquals;
4 |
5 | import org.junit.Test;
6 |
7 | public class XmlTest {
8 |
9 | @Test
10 | public void test() {
11 | String xml = Xml //
12 | .create("CompleteMultipartUpload") //
13 | .a("xmlns", "http://s3.amazonaws.com/doc/2006-03-01/") //
14 | .a("weird", "&<>'\"") //
15 | .e("Part") //
16 | .e("ETag").content("1234&") //
17 | .up() //
18 | .e("PartNumber").content("1") //
19 | .toString();
20 | assertEquals("\n"
21 | + "\n"
22 | + " \n" + " 1234& \n" + " 1 \n"
23 | + " \n" + " ", xml);
24 | }
25 |
26 | @Test(expected = IllegalArgumentException.class)
27 | public void testContentAndChild() {
28 | Xml.create("root").content("boo").element("child");
29 | }
30 |
31 | @Test
32 | public void testPrelude() {
33 | assertEquals("\n" + "\n" + " ",
34 | Xml.create("root").toString());
35 | }
36 |
37 | @Test
38 | public void testNoPrelude() {
39 | assertEquals("\n" + " ", Xml.create("root").excludePrelude().toString());
40 | }
41 |
42 | @Test
43 | public void testNoPreludeOnChild() {
44 | assertEquals("\n ",
45 | Xml.create("root").element("thing").content("").excludePrelude().toString());
46 | }
47 |
48 | @Test(expected=IllegalArgumentException.class)
49 | public void testNullName() {
50 | Xml.create(null);
51 | }
52 |
53 | @Test(expected=IllegalArgumentException.class)
54 | public void testBlankName() {
55 | Xml.create(" ");
56 | }
57 |
58 | @Test
59 | public void testUnusualCharacters1() {
60 | assertEquals(
61 | "𐑡bc ", Xml.create("root").excludePrelude().content("" + (char) 0xd801 + "abc").toString());
62 | assertEquals(
63 | "� ", Xml.create("root").excludePrelude().content("" + (char) 0xd801).toString());
64 | }
65 |
66 | @Test
67 | public void testUnusualCharacters2() {
68 | assertEquals(
69 | "�abc ", Xml.create("root").excludePrelude().content("" + (char) 0xdc00 + "abc").toString());
70 | }
71 |
72 | @Test
73 | public void testIllegalCharacters() {
74 | assertEquals(
75 | "�abc ", Xml.create("root").excludePrelude().content("" + (char) 0x00 + "abc").toString());
76 | }
77 |
78 | @Test
79 | public void testLegalWhitespace() {
80 | assertEquals(
81 | "\t\n\rabc ", Xml.create("root").excludePrelude().content("\t\n\rabc").toString());
82 | }
83 |
84 | @Test
85 | public void testUnusualCharacters3() {
86 | assertEquals(
87 | "�� ", Xml.create("root").excludePrelude().content("" + (char) 0xd800 + (char) 0xdfff + (char)0xfffe + (char) 0xffff + (char) 0xefff + (char) 0xd7ff).toString());
88 | }
89 |
90 |
91 | }
92 |
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-hourly-store-request-times.txt:
--------------------------------------------------------------------------------
1 | 3.823 2.969 1.787
2 | 3.890 3.070 1.906
3 | 4.031 3.060 1.792
4 | 4.095 3.480 1.938
5 | 3.718 4.060 1.961
6 | 3.785 3.993 1.899
7 | 3.583 3.072 2.376
8 | 5.280 3.094 1.796
9 | 3.891 3.002 1.795
10 | 4.096 3.071 1.827
11 | 3.694 2.997 1.789
12 | 4.301 3.174 1.844
13 | 3.583 2.941 1.919
14 | 3.990 3.072 2.056
15 | 4.299 3.275 1.861
16 | 4.198 2.968 1.861
17 | 3.892 3.070 1.835
18 | 4.094 3.091 1.962
19 | 3.887 3.029 2.093
20 | 4.198 3.175 2.305
21 | 3.779 3.158 2.138
22 | 4.056 3.016 1.952
23 | 3.889 3.072 1.739
24 | 4.300 2.969 2.039
25 | 3.787 2.918 1.873
26 | 3.788 3.170 1.915
27 | 3.911 3.152 2.023
28 | 3.788 3.174 1.970
29 |
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-lambda-runtimes-2.txt:
--------------------------------------------------------------------------------
1 | 1010 614 0.910
2 | 156 20.459 0.917
3 | 153 22.208 0.981
4 | 164 21.393 0.920
5 | 148 19.924 0.904
6 | 149 19.742 0.910
7 | 159 20.913 1.012
8 | 214 29.223 1.404
9 | 160 20.894 0.947
10 | 155 20.276 0.912
11 | 149 20.104 0.922
12 | 152 19.89 0.951
13 | 191 25.314 1.079
14 | 143 19.544 0.866
15 | 156 20.635 0.960
16 | 158 20.792 0.974
17 | 152 21.402 1.005
18 | 165 23.166 1.158
19 | 151 21.162 0.964
20 | 158 20.597 0.868
21 | 154 20.31 0.942
22 | 147 19.751 0.878
23 | 179 22.738 1.027
24 |
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-lambda-runtimes-sdk-v1.txt:
--------------------------------------------------------------------------------
1 | 2635 363 3.419
2 | 2555 355 3.372
3 | 4315 564 5.504
4 | 2471 344 3.310
5 | 2594 361 3.567
6 | 2616 359 3.476
7 | 2653 385 3.560
8 | 2607 357 3.442
9 | 2629 357 3.525
10 | 2498 358 3.274
11 | 2582 357 3.469
12 | 3683 507 4.615
13 | 2496 350 3.339
14 | 2540 354 3.593
15 | 2532 348 3.514
16 | 2862 403 3.821
17 | 3216 417 4.448
18 | 2592 353 3.573
19 | 2956 399 4.063
20 | 2493 343 3.373
21 | 2473 332 3.318
22 | 3464 495 4.611
23 | 2594 361 3.445
24 | 2483 345 3.384
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-lambda-runtimes-sdk-v2-2.txt:
--------------------------------------------------------------------------------
1 | # cold/warm, time, runtime with static fields sdk v2
2 | C,2021-06-15 11:11:21.255,1320.67
3 | W,2021-06-15 11:17:02.263,160.33
4 | W,2021-06-15 11:17:02.693,167.6
5 | W,2021-06-15 11:17:02.995,111.72
6 | W,2021-06-15 11:17:03.227,88.67
7 | W,2021-06-15 11:17:03.494,103.69
8 | W,2021-06-15 11:17:03.866,136.91
9 | W,2021-06-15 11:17:04.148,84.41
10 | W,2021-06-15 11:17:04.445,91.16
11 | W,2021-06-15 11:17:04.679,84.56
12 | W,2021-06-15 11:17:04.912,81.1
13 | C,2021-06-15 12:17:04.132,1379.87
14 | W,2021-06-15 12:17:04.524,204.21
15 | W,2021-06-15 12:17:04.838,106.58
16 | W,2021-06-15 12:17:05.090,104.93
17 | W,2021-06-15 12:17:05.305,75.84
18 | W,2021-06-15 12:17:05.587,138.7
19 | W,2021-06-15 12:17:05.943,100.39
20 | W,2021-06-15 12:17:06.183,87.1
21 | W,2021-06-15 12:17:06.505,146.15
22 | W,2021-06-15 12:17:06.765,86.97
23 | C,2021-06-15 13:17:04.854,1271.97
24 | W,2021-06-15 13:17:05.129,99.86
25 | W,2021-06-15 13:17:05.546,197.37
26 | W,2021-06-15 13:17:05.831,86.63
27 | W,2021-06-15 13:17:06.069,95.58
28 | W,2021-06-15 13:17:06.319,104.63
29 | W,2021-06-15 13:17:06.598,135.95
30 | W,2021-06-15 13:17:06.870,77.26
31 | W,2021-06-15 13:17:07.164,82.16
32 | W,2021-06-15 13:17:07.398,81.64
33 | C,2021-06-15 14:17:04.346,1150.13
34 | W,2021-06-15 14:17:04.625,110.44
35 | W,2021-06-15 14:17:04.954,188.5
36 | W,2021-06-15 14:17:05.245,98.23
37 | W,2021-06-15 14:17:05.541,86.01
38 | W,2021-06-15 14:17:05.817,133.97
39 | W,2021-06-15 14:17:06.090,141.19
40 | W,2021-06-15 14:17:06.356,77.89
41 | W,2021-06-15 14:17:06.681,143.45
42 | W,2021-06-15 14:17:06.972,77.88
43 | C,2021-06-15 15:17:06.318,1691.01
44 | W,2021-06-15 15:17:06.855,387.98
45 | W,2021-06-15 15:17:07.331,341.33
46 | W,2021-06-15 15:17:07.677,171.26
47 | W,2021-06-15 15:17:07.999,98.55
48 | W,2021-06-15 15:17:08.300,147.1
49 | W,2021-06-15 15:17:08.748,220.24
50 | W,2021-06-15 15:17:09.040,103.81
51 | W,2021-06-15 15:17:09.286,97.12
52 | W,2021-06-15 15:17:09.559,131.11
53 | C,2021-06-15 16:17:04.639,1280.04
54 | W,2021-06-15 16:17:04.976,139.51
55 | W,2021-06-15 16:17:05.258,111.97
56 | W,2021-06-15 16:17:05.508,105.85
57 | W,2021-06-15 16:17:05.749,95.58
58 | W,2021-06-15 16:17:06.001,115.13
59 | W,2021-06-15 16:17:06.252,113.16
60 | W,2021-06-15 16:17:06.483,79.93
61 | W,2021-06-15 16:17:06.726,104.4
62 | W,2021-06-15 16:17:06.958,85.9
63 | C,2021-06-15 17:17:04.933,1181.81
64 | W,2021-06-15 17:17:05.295,139.89
65 | W,2021-06-15 17:17:05.546,93.66
66 | W,2021-06-15 17:17:05.768,82.63
67 | W,2021-06-15 17:17:05.992,74.32
68 | W,2021-06-15 17:17:06.250,113.97
69 | W,2021-06-15 17:17:06.489,97.94
70 | W,2021-06-15 17:17:06.749,116.95
71 | W,2021-06-15 17:17:07.096,103.41
72 | W,2021-06-15 17:17:07.314,78.04
73 | C,2021-06-15 18:17:04.335,1293.44
74 | W,2021-06-15 18:17:04.609,131.29
75 | W,2021-06-15 18:17:04.921,91.6
76 | W,2021-06-15 18:17:05.164,99.73
77 | W,2021-06-15 18:17:05.419,113.35
78 | W,2021-06-15 18:17:05.810,144.81
79 | W,2021-06-15 18:17:06.071,86.3
80 | W,2021-06-15 18:17:06.285,78.93
81 | W,2021-06-15 18:17:06.508,81.84
82 | W,2021-06-15 18:17:06.725,73.38
83 | C,2021-06-15 19:17:04.728,1241.26
84 | W,2021-06-15 19:17:05.034,127.54
85 | W,2021-06-15 19:17:05.328,159.78
86 | W,2021-06-15 19:17:05.561,98.96
87 | W,2021-06-15 19:17:05.815,88.19
88 | W,2021-06-15 19:17:06.076,134.72
89 | W,2021-06-15 19:17:06.425,130.27
90 | W,2021-06-15 19:17:06.679,96.66
91 | W,2021-06-15 19:17:06.916,92.27
92 | W,2021-06-15 19:17:07.144,90.91
93 | C,2021-06-15 20:17:04.119,1247.01
94 | W,2021-06-15 20:17:04.510,124.67
95 | W,2021-06-15 20:17:04.846,79.49
96 | W,2021-06-15 20:17:05.129,93.83
97 | W,2021-06-15 20:17:05.396,86.72
98 | W,2021-06-15 20:17:05.712,127.66
99 | W,2021-06-15 20:17:06.006,93.66
100 | W,2021-06-15 20:17:06.221,75.4
101 | W,2021-06-15 20:17:06.471,81.78
102 | W,2021-06-15 20:17:06.693,95.47
103 | C,2021-06-15 21:17:05.282,1321.76
104 | W,2021-06-15 21:17:05.584,134.14
105 | W,2021-06-15 21:17:06.512,96.31
106 | W,2021-06-15 21:17:07.073,80.15
107 | W,2021-06-15 21:17:07.449,84.6
108 | W,2021-06-15 21:17:08.164,145.11
109 | W,2021-06-15 21:17:08.590,89.37
110 | W,2021-06-15 21:17:08.883,153.14
111 | W,2021-06-15 21:17:09.136,82.68
112 | W,2021-06-15 21:17:09.422,74.66
113 | C,2021-06-15 22:17:04.244,1178.37
114 | W,2021-06-15 22:17:04.584,133.81
115 | W,2021-06-15 22:17:04.877,151.69
116 | W,2021-06-15 22:17:05.148,94.24
117 | W,2021-06-15 22:17:05.384,83.87
118 | W,2021-06-15 22:17:05.669,92.86
119 | W,2021-06-15 22:17:05.936,120.13
120 | W,2021-06-15 22:17:06.169,103.61
121 | W,2021-06-15 22:17:06.415,92.3
122 | W,2021-06-15 22:17:06.641,71.07
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-lambda-runtimes-sdk-v2.txt:
--------------------------------------------------------------------------------
1 | 2224 339 0
2 | 2362 351 0
3 | 2347 343 0
4 | 2941 452 0
5 | 2221 338 0
6 | 2281 336 0
7 | 2159 319 0
8 | 2146 329 0
9 | 2263 336 0
10 | 2227 333 0
11 | 2222 334 0
12 | 2292 344 0
13 | 2183 331 0
14 | 2431 364 0
15 | 2050 313 0
16 | 2438 375 0
17 | 2260 337 0
18 | 2327 340 0
19 | 2283 354 0
20 | 2094 316 0
21 | 2548 383 0
22 | 2228 335 0
23 | 2126 318 0
24 | 1976 313 0
25 | 2422 373 0
26 | 2194 324 0
27 | 2527 417 0
28 | 2496 381 0
29 | 2132 326 0
30 | 2297 338 0
--------------------------------------------------------------------------------
/src/test/resources/one-time-link-lambda-runtimes.txt:
--------------------------------------------------------------------------------
1 | 1102 197.32 2.150
2 | 1047 214 2.199
3 | 1002 185 1.949
4 | 913.99 176 1.734
5 | 1292 245 2.593
6 | 1031 204 2.524
7 | 1126 218 2.182
8 | 1058 200 1.920
9 | 922.79 195 1.840
10 | 1306 231 2.194
11 | 1050 186 2.014
12 | 913.42 163 1.735
13 | 1039 191 1.876
14 | 1273 224 2.355
15 | 950.53 177 1.738
16 | 984.68 185 1.742
17 | 912.26 163 1.659
18 | 1242 242 2.512
19 | 1025 185 1.944
20 | 966 194 1.769
21 | 960.78 179 1.785
22 | 986.6 179 1.824
23 | 988 176 2.129
24 | 984 194 1.726
25 | 980 186 1.787
--------------------------------------------------------------------------------
/src/test/resources/test.txt:
--------------------------------------------------------------------------------
1 | something
--------------------------------------------------------------------------------