├── .gitignore
├── CHANGELOG.md
├── README.md
├── build.sbt
├── cloudwatch
    └── src
    │   ├── main
    │       └── scala
    │       │   └── cloudwatch.scala
    │   └── test
    │       └── scala
    │           └── CloudwatchSpec.scala
├── commons
    └── src
    │   └── main
    │       └── scala
    │           ├── AWSThreadFactory.scala
    │           └── FutureHelper.scala
├── project
    ├── Dependencies.scala
    ├── build.properties
    └── plugins.sbt
├── s3
    └── src
    │   ├── main
    │       └── scala
    │       │   ├── AmazonS3Client.scala
    │       │   ├── AmazonS3ClientMaterialized.scala
    │       │   ├── AmazonS3Wrapper.scala
    │       │   └── futureTransfer.scala
    │   └── test
    │       ├── resources
    │           ├── big.txt
    │           ├── medium.txt
    │           └── small.txt
    │       └── scala
    │           └── S3Spec.scala
├── scaladoc.sbt
└── sqs
    └── src
        ├── main
            └── scala
            │   ├── AmazonSQSClient.scala
            │   └── AmazonSQSWrapper.scala
        └── test
            └── scala
                └── SQSSpec.scala


/.gitignore:
--------------------------------------------------------------------------------
 1 | .logs
 2 | project/project
 3 | project/target
 4 | target
 5 | tmp
 6 | .history
 7 | dist
 8 | **/.idea
 9 | /*.iml
10 | /out
11 | /.idea_modules
12 | /.classpath
13 | /.project
14 | /RUNNING_PID
15 | /.settings


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | # Changelog
 2 | 
 3 | This file summarizes the main changes for each release.
 4 | 
 5 | # <a name="0.12.2"></a>Version 0.12.2
 6 | 
 7 |  - Add cross compilation to scala 2.12 ([#24](https://github.com/MfgLabs/common-aws/pull/24))
 8 |  - In `uploadStreamAsFile` expose the `InitiateMultipartUploadRequest` as suggested in [#13](hhttps://github.com/MfgLabs/commons-aws/issues/13))
 9 |  - Upgrade `akka-stream-extensions` version to `0.11.2`
10 | 
11 | ## <a name="0.12.1"></a>Version 0.12.1
12 | 
13 |  - Add the region as a mandatory parameter for all clients.
14 |  - Add the `apply` and `from` constructor for `AmazonCloudwatchClient`.
15 | 
16 | ### <a name="0.12.0"></a>Version 0.12.0
17 | 
18 |  - Rework the class hierarchy, for `S3` and `SQS`
19 |    - The `Wrapper` wrap the amazon Java sdk function with no additional logic.
20 |    - The `Client` extend it
21 |    - For `ClientMaterialized` extend the `Client` and need an additional `Materializer` to transform a stream to `Future`.
22 |  - Move `deleteObjects(bucket, prefix)` out of `AmazonS3Wrapper` to be able to use `AmazonS3ClientMaterialized.listFiles`
23 |  - Switch to `ForkJoinPool` with daemon thread as default.
24 | 
25 | 
26 | ### <a name="0.11.0"></a>Version 0.11.0
27 | 
28 |  - Remove dependency to the unmaintained `dwhjames/aws-wrap` by including the necessary code
29 |  - Split the project in three : `commons-aws-cloudwatch`, `commons-aws-s3`  and `commons-aws-sqs`
30 |  - Upgrade dependencies including `aws-java-sdk` and `akka-stream-extensions`
31 |  - Remove `CloudwatchAkkaHeartbeat`
32 |  - Replace `AmazonCloudwatchClient` and `AmazonS3AsyncClient` constructors by an `apply` with default values.
33 |  - The followind duplicated methods from`AmazonS3AsyncClient` were deleted :
34 |    - `uploadFile` use `putObject` with `new PutObjectRequest().withCannedAcl(CannedAccessControlList.PublicReadWrite)`
35 |    - `deleteFile` use `deleteObject`
36 |    - `deleteFiles` use `deleteObjects`
37 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # Streaming / asynchronous Scala client for common AWS services
  2 | 
  3 | Streaming / asynchronous Scala client for common AWS services.
  4 | When possible, clients expose methods that return Akka Stream's Sources / Flows / Sinks to provide streaming facilities.
  5 | 
  6 | Clients use a pool of threads managed internally and optimized for blocking IO operations.
  7 | 
  8 | This library makes heavy use of our extension library for Akka Stream
  9 | [MfgLabs/akka-stream-extensions](https://github.com/MfgLabs/akka-stream-extensions).
 10 | 
 11 | ## Resolver
 12 | 
 13 | ```scala
 14 | resolvers ++= Seq(
 15 |   Resolver.bintrayRepo("mfglabs", "maven")
 16 | )
 17 | ```
 18 | 
 19 | ## Dependencies
 20 | 
 21 | Three packages are available :
 22 | ```scala
 23 | libraryDependencies += "com.mfglabs" %% "commons-aws-cloudwatch" % "0.12.2"
 24 | libraryDependencies += "com.mfglabs" %% "commons-aws-s3" % "0.12.2"
 25 | libraryDependencies += "com.mfglabs" %% "commons-aws-sqs" % "0.12.2"
 26 | ```
 27 | 
 28 | Changelog [here](CHANGELOG.md)
 29 | 
 30 | ## Usage
 31 | 
 32 | > Scaladoc is available [there](http://mfglabs.github.io/commons-aws/api/current/)
 33 | 
 34 | ### Commons
 35 | 
 36 | #### S3
 37 | 
 38 | ```scala
 39 | import com.mfglabs.commons.aws.s3._
 40 | 
 41 | val client = AmazonS3Client()()) // client with un-materialized composable Source / Flow / Sink
 42 | 
 43 | val fileStream: Source[ByteString, Unit] = client.getFileAsStream(bucket, key)
 44 | 
 45 | val multipartfileStream: Source[ByteString, Unit] = client.getMultipartFileAsStream(bucket, prefix)
 46 | 
 47 | someBinaryStream.via(
 48 |   client.uploadStreamAsFile(bucket, key, chunkUploadConcurrency = 2)
 49 | )
 50 | 
 51 | someBinaryStream.via(
 52 |   client.uploadStreamAsMultipartFile(
 53 |     bucket,
 54 |     prefix,
 55 |     nbChunkPerFile = 10000,
 56 |     chunkUploadConcurrency = 2
 57 |   )
 58 | )
 59 | 
 60 | val ops = client.materialized(flowMaterializer) // client with added materialized methods
 61 | 
 62 | val file: Future[ByteString] = ops.getFile(bucket, key)
 63 | 
 64 | // More methods, check the source code
 65 | ```
 66 | 
 67 | Please remark that you don't need any implicit `scala.concurrent.ExecutionContext` as it's directly provided
 68 | and managed by [[AmazonS3Client]] itself.
 69 | 
 70 | There are also smart `AmazonS3Client` constructors that can be provided with custom.
 71 | `java.util.concurrent.ExecutorService` if you want to manage your pools of threads.
 72 | 
 73 | 
 74 | #### SQS
 75 | 
 76 | ```scala
 77 | import com.mfglabs.commons.aws.sqs._
 78 | 
 79 | val sqs = AmazonSQSClient()()
 80 | 
 81 | val sender: Flow[String, SendMessageResult, Unit] =
 82 |   Flow[String].map { body =>
 83 |     val req = new SendMessageRequest()
 84 |     req.setMessageBody(body)
 85 |     req.setQueueUrl(queueUrl)
 86 |     req
 87 |   }
 88 |   .via(sqs.sendMessageAsStream())
 89 | 
 90 | val receiver: Source[Message, Unit] = sqs.receiveMessageAsStream(queueUrl, autoAck = false)
 91 | ```
 92 | 
 93 | Please remark that you don't need any implicit `scala.concurrent.ExecutionContext` as it's directly provided
 94 | and managed by [[AmazonSQSClient]] itself.
 95 | 
 96 | There are also smart `AmazonSQSClient` constructors that can be provided with custom.
 97 | `java.util.concurrent.ExecutorService` if you want to manage your pools of threads.
 98 | 
 99 | #### Cloudwatch
100 | 
101 | In your code:
102 | 
103 | ```scala
104 | import com.mfglabs.commons.aws.cloudwatch
105 | import cloudwatch._ // brings implicit extensions
106 | 
107 | // Create the client
108 | val CW = cloudwatch.AmazonCloudwatchClient()()
109 | 
110 | // Use it
111 | for {
112 |   metrics  <- CW.listMetrics()
113 | } yield (metrics)
114 | ```
115 | 
116 | Please remark that you don't need any implicit `scala.concurrent.ExecutionContext` as it's directly provided
117 | and managed by [[AmazonCloudwatchClient]] itself.
118 | 
119 | There are also smart `AmazonCloudwatchClient` constructors that can be provided with custom.
120 | `java.util.concurrent.ExecutorService` if you want to manage your pools of threads.
121 | 
122 | ## License
123 | 
124 | This software is licensed under the Apache 2 license, quoted below.
125 | 
126 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
127 | 
128 | http://www.apache.org/licenses/LICENSE-2.0
129 | 
130 | Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
131 | 


--------------------------------------------------------------------------------
/build.sbt:
--------------------------------------------------------------------------------
  1 | import bintray.Plugin._
  2 | 
  3 | organization in ThisBuild := "com.mfglabs"
  4 | 
  5 | scalaVersion in ThisBuild := "2.11.11"
  6 | 
  7 | version in ThisBuild := "0.12.2"
  8 | 
  9 | crossScalaVersions := Seq("2.11.11", "2.12.2")
 10 | 
 11 | resolvers in ThisBuild ++= Seq(
 12 |   "Typesafe repository" at "http://repo.typesafe.com/typesafe/releases/",
 13 |   Resolver.sonatypeRepo("releases"),
 14 |   DefaultMavenRepository,
 15 |   Resolver.bintrayRepo("mfglabs", "maven")
 16 | )
 17 | 
 18 | scalacOptions in ThisBuild ++= Seq(
 19 |   "-encoding", "UTF-8",
 20 |   "-target:jvm-1.8",
 21 |   "-Ydelambdafy:method",
 22 |   "-Yno-adapted-args",
 23 |   "-deprecation",
 24 |   "-feature",
 25 |   "-language:postfixOps",
 26 |   "-unchecked",
 27 |   "-Xfuture",
 28 |   "-Xlint",
 29 |   "-Xlint:-missing-interpolator",
 30 |   "-Xlint:private-shadow",
 31 |   "-Xlint:type-parameter-shadow",
 32 |   "-Ywarn-dead-code",
 33 |   "-Ywarn-unused",
 34 |   "-Ywarn-unused-import",
 35 |   "-Ywarn-numeric-widen",
 36 |   "-Ywarn-value-discard",
 37 |   "-Xcheckinit"
 38 | )
 39 | 
 40 | publishMavenStyle in ThisBuild := true
 41 | 
 42 | lazy val commonSettings = Seq(
 43 |   scmInfo := Some(ScmInfo(
 44 |     url("https://github.com/MfgLabs/commons-aws"),
 45 |     "git@github.com:MfgLabs/commons-aws.git"
 46 |   ))
 47 | )
 48 | 
 49 | lazy val publishSettings = Seq(
 50 |   homepage := Some(url("https://github.com/MfgLabs/commons-aws")),
 51 |   licenses := Seq("Apache-2.0" -> url("http://www.apache.org/licenses/LICENSE-2.0.html")),
 52 |   autoAPIMappings := true,
 53 |   publishMavenStyle := true,
 54 |   publishArtifact in packageDoc := false,
 55 |   publishArtifact in Test := false,
 56 |   pomIncludeRepository := { _ => false }
 57 | ) ++ bintrayPublishSettings
 58 | 
 59 | lazy val noPublishSettings = Seq(
 60 |   publish := (),
 61 |   publishLocal := (),
 62 |   publishArtifact := false
 63 | )
 64 | 
 65 | lazy val all = (project in file("."))
 66 |   .aggregate(commons)
 67 |   .aggregate(cloudwatch)
 68 |   .aggregate(s3)
 69 |   .aggregate(sqs)
 70 |   .settings(name := "commons-aws-all")
 71 |   .enablePlugins(ScalaUnidocPlugin)
 72 |   .settings(name := "commons-aws-all")
 73 |   .settings(noPublishSettings)
 74 | 
 75 | lazy val commons = project.in(file("commons"))
 76 |   .settings   (
 77 |     name := "commons-aws",
 78 |     libraryDependencies ++= Seq(
 79 |       Dependencies.Compile.awsJavaSDKcore
 80 |     ),
 81 |     commonSettings,
 82 |     publishSettings
 83 |   )
 84 | 
 85 | lazy val cloudwatch = project.in(file("cloudwatch"))
 86 |   .settings   (
 87 |     name := "commons-aws-cloudwatch",
 88 |     libraryDependencies ++= Seq(
 89 |       Dependencies.Compile.awsJavaSDKcw,
 90 |       Dependencies.Compile.akkaStreamExt,
 91 |       Dependencies.Compile.slf4j,
 92 |       Dependencies.Test.scalaTest
 93 |     ),
 94 |     commonSettings,
 95 |     publishSettings
 96 |   ).dependsOn(commons)
 97 | 
 98 | lazy val s3 = project.in(file("s3"))
 99 |   .settings   (
100 |     name := "commons-aws-s3",
101 |     libraryDependencies ++= Seq(
102 |       Dependencies.Compile.awsJavaSDKs3,
103 |       Dependencies.Compile.akkaStreamExt,
104 |       Dependencies.Compile.slf4j,
105 |       Dependencies.Test.scalaTest
106 |     ),
107 |     commonSettings,
108 |     publishSettings
109 |   ).dependsOn(commons)
110 | 
111 | lazy val sqs = project.in(file("sqs"))
112 |   .settings   (
113 |     name := "commons-aws-sqs",
114 |     libraryDependencies ++= Seq(
115 |       Dependencies.Compile.awsJavaSDKsqs,
116 |       Dependencies.Compile.akkaStreamExt,
117 |       Dependencies.Compile.slf4j,
118 |       Dependencies.Test.scalaTest
119 |     ),
120 |     commonSettings,
121 |     publishSettings
122 |   ).dependsOn(commons)
123 | 


--------------------------------------------------------------------------------
/cloudwatch/src/main/scala/cloudwatch.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2012-2015 Pellucid Analytics
  3 |  * Copyright 2015 Daniel W. H. James
  4 |  *
  5 |  * Licensed under the Apache License, Version 2.0 (the "License");
  6 |  * you may not use this file except in compliance with the License.
  7 |  * You may obtain a copy of the License at
  8 |  *
  9 |  *   http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package com.mfglabs.commons.aws
 19 | package cloudwatch
 20 | 
 21 | import java.util.concurrent.ExecutorService
 22 | import scala.concurrent.Future
 23 | 
 24 | import com.amazonaws.auth.{AWSCredentialsProvider, DefaultAWSCredentialsProviderChain}
 25 | import com.amazonaws.ClientConfiguration
 26 | import com.amazonaws.client.builder.ExecutorFactory
 27 | 
 28 | import com.amazonaws.services.cloudwatch.{AmazonCloudWatchAsync, AmazonCloudWatchAsyncClientBuilder}
 29 | import com.amazonaws.services.cloudwatch.model._
 30 | 
 31 | 
 32 | object AmazonCloudwatchClient {
 33 |   import com.amazonaws.auth.{AWSCredentials, AWSStaticCredentialsProvider}
 34 |   import FutureHelper.defaultExecutorService
 35 |   import com.amazonaws.regions.Regions
 36 | 
 37 |   def apply(
 38 |     region              : Regions,
 39 |     awsCredentials      : AWSCredentials,
 40 |     clientConfiguration : ClientConfiguration = new ClientConfiguration()
 41 |   )(
 42 |     executorService     : ExecutorService     = defaultExecutorService(clientConfiguration, "aws.wrap.cloudwatch")
 43 |   ): AmazonCloudwatchClient = {
 44 |     from(region, new AWSStaticCredentialsProvider(awsCredentials), clientConfiguration)(executorService)
 45 |   }
 46 | 
 47 |   def from(
 48 |     region                 : Regions,
 49 |     awsCredentialsProvider : AWSCredentialsProvider = new DefaultAWSCredentialsProviderChain,
 50 |     clientConfiguration    : ClientConfiguration    = new ClientConfiguration()
 51 |   )(
 52 |     executorService        : ExecutorService        = defaultExecutorService(clientConfiguration, "aws.wrap.cloudwatch")
 53 |   ): AmazonCloudwatchClient = new AmazonCloudwatchClient(
 54 |     AmazonCloudWatchAsyncClientBuilder
 55 |       .standard()
 56 |       .withRegion(region)
 57 |       .withCredentials(awsCredentialsProvider)
 58 |       .withClientConfiguration(clientConfiguration)
 59 |       .withExecutorFactory(new ExecutorFactory { def newExecutor() = executorService })
 60 |       .build()
 61 |   )
 62 | 
 63 | }
 64 | 
 65 | /**
 66 |  * A lightweight wrapper for [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/dynamodbv2/AmazonDynamoDBAsyncClient.html AmazonCloudWatchAsyncClient]].
 67 |  *
 68 |  * @constructor construct a wrapper client from an Amazon async client.
 69 |  * @param client
 70 |   *     the underlying [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/dynamodbv2/AmazonDynamoDBAsyncClient.html AmazonCloudWatchAsyncClient]].
 71 |  * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/dynamodbv2/AmazonDynamoDBAsyncClient.html AmazonCloudWatchAsyncClient]]
 72 |  */
 73 | class AmazonCloudwatchClient(val client: AmazonCloudWatchAsync) {
 74 |   import FutureHelper._
 75 | 
 76 |   /**
 77 |   * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#deleteAlarms(com.amazonaws.services.cloudwatch.model.DeleteAlarmsRequest) AWS Java SDK]]
 78 |   */
 79 |   def deleteAlarms(
 80 |     deleteAlarmsRequest: DeleteAlarmsRequest
 81 |   ): Future[DeleteAlarmsResult] =
 82 |     wrapAsyncMethod(client.deleteAlarmsAsync, deleteAlarmsRequest)
 83 | 
 84 |   /**
 85 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#deleteAlarms(com.amazonaws.services.cloudwatch.model.DeleteAlarmsRequest) AWS Java SDK]]
 86 |     */
 87 |   def deleteAlarms(alarmNames: String*): Future[DeleteAlarmsResult] =
 88 |     deleteAlarms(new DeleteAlarmsRequest().withAlarmNames(alarmNames: _*))
 89 | 
 90 |   /**
 91 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#describeAlarmHistory(com.amazonaws.services.cloudwatch.model.DescribeAlarmHistoryRequest) AWS Java SDK]]
 92 |     */
 93 |   def describeAlarmHistory(
 94 |     describeAlarmHistoryRequest: DescribeAlarmHistoryRequest
 95 |   ): Future[DescribeAlarmHistoryResult] =
 96 |     wrapAsyncMethod(client.describeAlarmHistoryAsync, describeAlarmHistoryRequest)
 97 | 
 98 |   /**
 99 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#describeAlarmHistory() AWS Java SDK]]
100 |     */
101 |   def describeAlarmHistory(): Future[DescribeAlarmHistoryResult] =
102 |     describeAlarmHistory(new DescribeAlarmHistoryRequest())
103 | 
104 |   /**
105 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#describeAlarms(com.amazonaws.services.cloudwatch.model.DescribeAlarmsRequest) AWS Java SDK]]
106 |     */
107 |   def describeAlarms(
108 |     describeAlarmRequest: DescribeAlarmsRequest
109 |   ): Future[DescribeAlarmsResult] =
110 |     wrapAsyncMethod(client.describeAlarmsAsync, describeAlarmRequest)
111 | 
112 |   /**
113 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#describeAlarms() AWS Java SDK]]
114 |     */
115 |   def describeAlarms(): Future[DescribeAlarmsResult] =
116 |     describeAlarms(new DescribeAlarmsRequest())
117 | 
118 |   /**
119 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#describeAlarmsForMetric(com.amazonaws.services.cloudwatch.model.DescribeAlarmsForMetricRequest) AWS Java SDK]]
120 |     */
121 |   def describeAlarmsForMetric(
122 |     describeAlarmsForMetricRequest: DescribeAlarmsForMetricRequest
123 |   ): Future[DescribeAlarmsForMetricResult] =
124 |     wrapAsyncMethod(client.describeAlarmsForMetricAsync, describeAlarmsForMetricRequest)
125 | 
126 |   /**
127 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#disableAlarmActions(com.amazonaws.services.cloudwatch.model.DisableAlarmActionsRequest) AWS Java SDK]]
128 |     */
129 |   def disableAlarmActions(
130 |     disableAlarmActionsRequest: DisableAlarmActionsRequest
131 |   ): Future[DisableAlarmActionsResult] =
132 |     wrapAsyncMethod(client.disableAlarmActionsAsync, disableAlarmActionsRequest)
133 | 
134 |   /**
135 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#disableAlarmActions(com.amazonaws.services.cloudwatch.model.DisableAlarmActionsRequest) AWS Java SDK]]
136 |     */
137 |   def disableAlarmActions(alarmNames: String*): Future[DisableAlarmActionsResult] =
138 |     disableAlarmActions(new DisableAlarmActionsRequest().withAlarmNames(alarmNames: _*))
139 | 
140 |   /**
141 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#enableAlarmActions(com.amazonaws.services.cloudwatch.model.EnableAlarmActionsRequest) AWS Java SDK]]
142 |     */
143 |   def enableAlarmActions(
144 |     enableAlarmActionsRequest: EnableAlarmActionsRequest
145 |   ): Future[EnableAlarmActionsResult] =
146 |     wrapAsyncMethod(client.enableAlarmActionsAsync, enableAlarmActionsRequest)
147 | 
148 |   /**
149 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#enableAlarmActions(com.amazonaws.services.cloudwatch.model.EnableAlarmActionsRequest) AWS Java SDK]]
150 |     */
151 |   def enableAlarmActions(alarmNames: String*): Future[EnableAlarmActionsResult] =
152 |     enableAlarmActions(new EnableAlarmActionsRequest().withAlarmNames(alarmNames: _*))
153 | 
154 |   /**
155 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#getMetricStatistics(com.amazonaws.services.cloudwatch.model.GetMetricStatisticsRequest) AWS Java SDK]]
156 |     */
157 |   def getMetricStatistics(
158 |     getMetricStatisticsRequest: GetMetricStatisticsRequest
159 |   ): Future[GetMetricStatisticsResult] =
160 |     wrapAsyncMethod(client.getMetricStatisticsAsync, getMetricStatisticsRequest)
161 | 
162 |   /**
163 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#listMetrics(com.amazonaws.services.cloudwatch.model.ListMetricsRequest) AWS Java SDK]]
164 |     */
165 |   def listMetrics(
166 |     listMetricsRequest: ListMetricsRequest
167 |   ): Future[ListMetricsResult] =
168 |     wrapAsyncMethod(client.listMetricsAsync, listMetricsRequest)
169 | 
170 |   /**
171 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#listMetrics() AWS Java SDK]]
172 |     */
173 |   def listMetrics(): Future[ListMetricsResult] =
174 |     listMetrics(new ListMetricsRequest)
175 | 
176 |   /**
177 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#putMetricAlarm(com.amazonaws.services.cloudwatch.model.PutMetricAlarmRequest) AWS Java SDK]]
178 |     */
179 |   def putMetricAlarm(
180 |     putMetricAlarmRequest: PutMetricAlarmRequest
181 |   ): Future[PutMetricAlarmResult] =
182 |     wrapAsyncMethod(client.putMetricAlarmAsync, putMetricAlarmRequest)
183 | 
184 |   /**
185 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#putMetricData(com.amazonaws.services.cloudwatch.model.PutMetricDataRequest) AWS Java SDK]]
186 |     */
187 |   def putMetricData(
188 |     putMetricDataRequest: PutMetricDataRequest
189 |   ): Future[PutMetricDataResult] =
190 |     wrapAsyncMethod(client.putMetricDataAsync, putMetricDataRequest)
191 | 
192 |   /**
193 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#putMetricData(com.amazonaws.services.cloudwatch.model.PutMetricDataRequest) AWS Java SDK]]
194 |     */
195 |   def putMetricData(
196 |     namespace:  String,
197 |     metricData: Iterable[MetricDatum]
198 |   ): Future[PutMetricDataResult] = {
199 |     import scala.collection.JavaConverters.asJavaCollectionConverter
200 | 
201 |     putMetricData(
202 |       new PutMetricDataRequest()
203 |       .withNamespace(namespace)
204 |       .withMetricData(metricData.asJavaCollection)
205 |     )
206 |   }
207 | 
208 |   /**
209 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#setAlarmState(com.amazonaws.services.cloudwatch.model.SetAlarmStateRequest) AWS Java SDK]]
210 |     */
211 |   def setAlarmState(
212 |     setAlarmStateRequest: SetAlarmStateRequest
213 |   ): Future[SetAlarmStateResult] =
214 |     wrapAsyncMethod(client.setAlarmStateAsync, setAlarmStateRequest)
215 | 
216 |   /**
217 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#setAlarmState(com.amazonaws.services.cloudwatch.model.SetAlarmStateRequest) AWS Java SDK]]
218 |     */
219 |   def setAlarmState(
220 |     alarmName: String,
221 |     stateReason: String,
222 |     stateValue: StateValue,
223 |     stateReasonData: String = ""
224 |   ): Future[SetAlarmStateResult] =
225 |     setAlarmState(
226 |       new SetAlarmStateRequest()
227 |       .withAlarmName(alarmName)
228 |       .withStateReason(stateReason)
229 |       .withStateValue(stateValue)
230 |       .withStateReasonData(stateReasonData)
231 |     )
232 | 
233 |   /**
234 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/cloudwatch/AmazonCloudWatch.html#shutdown() AWS Java SDK]]
235 |     */
236 |   def shutdown(): Unit =
237 |     client.shutdown()
238 | 
239 | }
240 | 


--------------------------------------------------------------------------------
/cloudwatch/src/test/scala/CloudwatchSpec.scala:
--------------------------------------------------------------------------------
 1 | package com.mfglabs.commons.aws
 2 | 
 3 | import com.amazonaws.regions.Regions
 4 | import org.scalatest.{FlatSpec, Matchers}
 5 | import org.scalatest.concurrent.ScalaFutures
 6 | import org.scalatest.time.{Minutes, Millis, Span}
 7 | 
 8 | class CloudwatchSpec extends FlatSpec with Matchers with ScalaFutures {
 9 |   implicit override val patienceConfig =
10 |     PatienceConfig(timeout = Span(2, Minutes), interval = Span(5, Millis))
11 | 
12 |   val CW = cloudwatch.AmazonCloudwatchClient.from(Regions.EU_WEST_1)()
13 | 
14 |   "Cloudwatch client" should "retrieve all metrics" in {
15 |     whenReady(CW.listMetrics()) { s => s.getMetrics should not be 'empty }
16 |   }
17 | 
18 | }
19 | 


--------------------------------------------------------------------------------
/commons/src/main/scala/AWSThreadFactory.scala:
--------------------------------------------------------------------------------
 1 | package com.mfglabs.commons.aws
 2 | 
 3 | import java.util.concurrent._
 4 | 
 5 | class AWSThreadFactory(name: String) extends ForkJoinPool.ForkJoinWorkerThreadFactory {
 6 |   private val backingThreadFactory = ForkJoinPool.defaultForkJoinWorkerThreadFactory
 7 | 
 8 |   override def newThread(pool: ForkJoinPool): ForkJoinWorkerThread = {
 9 |     val thread = backingThreadFactory.newThread(pool)
10 |     thread.setName(s"$name-${thread.getPoolIndex()}")
11 |     thread
12 |   }
13 | }
14 | 


--------------------------------------------------------------------------------
/commons/src/main/scala/FutureHelper.scala:
--------------------------------------------------------------------------------
 1 | package com.mfglabs.commons.aws
 2 | 
 3 | import com.amazonaws.AmazonWebServiceRequest
 4 | import com.amazonaws.ClientConfiguration
 5 | import com.amazonaws.handlers.AsyncHandler
 6 | 
 7 | import scala.concurrent.{Future, Promise}
 8 | import scala.util.Try
 9 | import java.util.concurrent.{Future => JFuture, ForkJoinPool}
10 | 
11 | object FutureHelper {
12 | 
13 |   def promiseToAsyncHandler[Request <: AmazonWebServiceRequest, Result](p: Promise[Result]) =
14 |     new AsyncHandler[Request, Result] {
15 |       override def onError(exception: Exception): Unit = { p.failure(exception); () }
16 |       override def onSuccess(request: Request, result: Result): Unit = { p.success(result); () }
17 |     }
18 | 
19 |   @inline
20 |   def wrapAsyncMethod[Request <: AmazonWebServiceRequest, Result](
21 |     f:       (Request, AsyncHandler[Request, Result]) => JFuture[Result],
22 |     request: Request
23 |   ): Future[Result] = {
24 |     val p = Promise[Result]
25 |     f(request, promiseToAsyncHandler(p))
26 |     p.future
27 |   }
28 | 
29 |   def defaultExecutorService(clientConfiguration: ClientConfiguration, factoryName: String) = {
30 |     new ForkJoinPool(
31 |       clientConfiguration.getMaxConnections + 1,
32 |       new AWSThreadFactory(factoryName),
33 |       null, // We do not override the default Thread.UncaughtExceptionHandler
34 |       true  // activate asyncMode
35 |     )
36 |   }
37 | 
38 |   trait MethodWrapper {
39 | 
40 |     @inline
41 |     def executorService: java.util.concurrent.ExecutorService
42 | 
43 |     @inline
44 |     def wrapMethod[Request, Result](f: Request => Result, request: Request): Future[Result] = {
45 |       val p = Promise[Result]
46 |       executorService.execute(new Runnable {
47 |         override def run(): Unit = {
48 |           p.complete(Try { f(request) })
49 |           ()
50 |         }
51 |       })
52 |       p.future
53 |     }
54 |   }
55 | 
56 | }
57 | 


--------------------------------------------------------------------------------
/project/Dependencies.scala:
--------------------------------------------------------------------------------
 1 | import sbt._
 2 | 
 3 | object Dependencies {
 4 | 
 5 |   object V {
 6 |     val awsJavaSDK        = "1.11.132"
 7 |     val akkaStreamExt     = "0.11.2"
 8 |     val scalaTest         = "3.0.3"
 9 |     val slf4j             = "1.7.12"
10 |   }
11 | 
12 |   object Compile {
13 |     val awsJavaSDKcore    = "com.amazonaws"       %   "aws-java-sdk-core"             % V.awsJavaSDK
14 |     val awsJavaSDKcw      = "com.amazonaws"       %   "aws-java-sdk-cloudwatch"       % V.awsJavaSDK
15 |     val awsJavaSDKs3      = "com.amazonaws"       %   "aws-java-sdk-s3"               % V.awsJavaSDK
16 |     val awsJavaSDKsqs     = "com.amazonaws"       %   "aws-java-sdk-sqs"              % V.awsJavaSDK
17 | 
18 |     val akkaStreamExt     = "com.mfglabs"         %%  "akka-stream-extensions"        % V.akkaStreamExt
19 | 
20 |     val slf4j             = "org.slf4j"           %   "slf4j-api"                     % V.slf4j
21 |   }
22 | 
23 |   object Test {
24 |     val scalaTest         = "org.scalatest"       %% "scalatest"            % V.scalaTest     % "test"
25 |   }
26 | }
27 | 


--------------------------------------------------------------------------------
/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version=0.13.15
2 | 


--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | addSbtPlugin("com.eed3si9n" % "sbt-unidoc" % "0.4.0")
2 | 
3 | addSbtPlugin("me.lessis" % "bintray-sbt" % "0.2.1")
4 | 


--------------------------------------------------------------------------------
/s3/src/main/scala/AmazonS3Client.scala:
--------------------------------------------------------------------------------
  1 | package com.mfglabs.commons.aws
  2 | package s3
  3 | 
  4 | import akka.stream.ActorMaterializer
  5 | import akka.stream.scaladsl._
  6 | import akka.util.ByteString
  7 | 
  8 | import com.amazonaws.services.s3.AmazonS3
  9 | import com.amazonaws.services.s3.model._
 10 | import com.mfglabs.stream.{ExecutionContextForBlockingOps, FlowExt, SourceExt}
 11 | 
 12 | import java.io.{ByteArrayInputStream, InputStream}
 13 | import java.util.concurrent.ExecutorService
 14 | import java.util.zip.GZIPInputStream
 15 | 
 16 | import scala.concurrent.Future
 17 | 
 18 | object AmazonS3Client {
 19 |   import com.amazonaws.auth._
 20 |   import com.amazonaws.ClientConfiguration
 21 |   import com.amazonaws.regions.Regions
 22 |   import com.amazonaws.services.s3.AmazonS3ClientBuilder
 23 |   import FutureHelper.defaultExecutorService
 24 | 
 25 |   def apply(
 26 |     region              : Regions,
 27 |     awsCredentials      : AWSCredentials,
 28 |     clientConfiguration : ClientConfiguration = new ClientConfiguration()
 29 |   )(
 30 |     executorService     : ExecutorService     = defaultExecutorService(clientConfiguration, "aws.wrap.s3")
 31 |   ): AmazonS3Client = {
 32 |     from(region, new AWSStaticCredentialsProvider(awsCredentials), clientConfiguration)(executorService)
 33 |   }
 34 | 
 35 |   def from(
 36 |     region                 : Regions,
 37 |     awsCredentialsProvider : AWSCredentialsProvider = new DefaultAWSCredentialsProviderChain,
 38 |     clientConfiguration    : ClientConfiguration    = new ClientConfiguration()
 39 |   )(
 40 |     executorService        : ExecutorService        = defaultExecutorService(clientConfiguration, "aws.wrap.s3")
 41 |   ): AmazonS3Client = {
 42 | 
 43 |    val client = AmazonS3ClientBuilder
 44 |       .standard()
 45 |       .withRegion(region)
 46 |       .withCredentials(awsCredentialsProvider)
 47 |       .withClientConfiguration(clientConfiguration)
 48 |       .build()
 49 | 
 50 |     new AmazonS3Client(client, executorService)
 51 |   }
 52 | 
 53 |   def build(builder: AmazonS3ClientBuilder) = {
 54 |     val client = builder.build()
 55 | 
 56 |     new AmazonS3Client(client, defaultExecutorService(builder.getClientConfiguration, "aws.wrap.s3"))
 57 |   }
 58 | 
 59 | }
 60 | 
 61 | class AmazonS3Client(
 62 |   val client          : AmazonS3,
 63 |   val executorService : ExecutorService
 64 | ) extends AmazonS3Wrapper {
 65 |   import scala.collection.immutable.Seq
 66 | 
 67 |   implicit lazy val ecForBlockingOps = ExecutionContextForBlockingOps(ec)
 68 | 
 69 |   /**
 70 |    * Return a client with additional logic to obtain a Stream logic as a Future.
 71 |    */
 72 |   def materialized(flowMaterializer : ActorMaterializer): AmazonS3ClientMaterialized =
 73 |     new AmazonS3ClientMaterialized(client, executorService, flowMaterializer)
 74 | 
 75 |   /** List all files with a given prefix of a S3 bucket.
 76 |    * @param bucket S3 bucket
 77 |    * @param maybePrefix S3 prefix. If not present, all the files of the bucket will be returned.
 78 |    */
 79 |   def listFilesAsStream(bucket: String, maybePrefix: Option[String] = None): Source[S3ObjectSummary, akka.NotUsed] = {
 80 |     import collection.JavaConverters._
 81 | 
 82 |     def getFirstListing: Future[ObjectListing] = maybePrefix match {
 83 |       case Some(prefix) => listObjects(bucket, prefix)
 84 |       case None => listObjects(bucket)
 85 |     }
 86 | 
 87 |     def unfold(listing: ObjectListing) =
 88 |       Some(Some(listing) -> listing.getObjectSummaries.asScala.to[Seq])
 89 | 
 90 |     Source.unfoldAsync[Option[ObjectListing], Seq[S3ObjectSummary]](None) {
 91 |       case None             => getFirstListing.map(unfold)
 92 |       case Some(oldListing) if oldListing.isTruncated => listNextBatchOfObjects(oldListing).map(unfold)
 93 |       case Some(_)          => Future.successful(None)
 94 |     }.mapConcat(identity)
 95 |   }
 96 | 
 97 |   /** Stream a S3 file.
 98 |     * @param bucket the bucket name
 99 |     * @param key the key of file
100 |     * @param inputStreamTransform optional inputstream transformation (for example GZIP decompression, ...)
101 |     */
102 |   def getFileAsStream(bucket: String, key: String, inputStreamTransform: InputStream => InputStream = identity): Source[ByteString, akka.NotUsed] = {
103 |     SourceExt.seededLazyAsync(getObject(bucket, key)) { o =>
104 |       StreamConverters.fromInputStream(() => inputStreamTransform(o.getObjectContent))
105 |     }
106 |   }
107 | 
108 |   /**
109 |    * Stream a gzipped S3 file and decompress it on the fly.
110 |    * @param bucket S3 bucket
111 |    * @param key S3 key
112 |    * @return
113 |    */
114 |   def uncompressGzippedFileAsStream(bucket: String, key: String) =
115 |     getFileAsStream(bucket, key, is => new GZIPInputStream(is))
116 | 
117 |   /**
118 |    * Stream sequentially a SE multipart file.
119 |    * @param bucket S3 bucket
120 |    * @param prefix S3 prefix. Files of path prefix* will be taken into account.
121 |    * @param inputStreamTransform optional inputstream transformation (for example GZIP decompression, ...)
122 |    */
123 |   def getMultipartFileAsStream(bucket: String, prefix: String, inputStreamTransform: InputStream => InputStream = identity): Source[ByteString, akka.NotUsed] = {
124 |     listFilesAsStream(bucket, Some(prefix))
125 |       .flatMapConcat { s3Object => getFileAsStream(bucket, s3Object.getKey, inputStreamTransform) }
126 |   }
127 | 
128 |   /**
129 |    * Upload a stream of bytes as a S3 file and returns an element of type CompleteMultipartUploadResult (or nothing if upstream was empty).
130 |    * @param bucket S3 bucket
131 |    * @param key S3 key
132 |    * @param chunkUploadConcurrency chunk upload concurrency. Order is guaranteed even with chunkUploadConcurrency > 1.
133 |    */
134 |   def uploadStreamAsFile(bucket: String, key: String, chunkUploadConcurrency: Int = 1): Flow[ByteString, CompleteMultipartUploadResult, akka.NotUsed] = {
135 |     val request = new InitiateMultipartUploadRequest(bucket, key)
136 |     uploadStreamAsFile(request, chunkUploadConcurrency)
137 |   }
138 | 
139 |   def uploadStreamAsFile(intiate: InitiateMultipartUploadRequest, chunkUploadConcurrency: Int): Flow[ByteString, CompleteMultipartUploadResult, akka.NotUsed] = {
140 |     import scala.collection.JavaConverters._
141 | 
142 |     val uploadChunkSize = 8 * 1024 * 1024 // recommended by AWS
143 | 
144 |     def initiateUpload: Future[String] = initiateMultipartUpload(intiate).map(_.getUploadId)
145 | 
146 |     Flow[ByteString]
147 |       .via(FlowExt.rechunkByteStringBySize(uploadChunkSize))
148 |       .via(FlowExt.zipWithConstantLazyAsync(initiateUpload))
149 |       .via(FlowExt.zipWithIndex)
150 |       .mapAsyncUnordered(chunkUploadConcurrency) { case ((bytes, uploadId), partNumber) =>
151 |         val uploadRequest = new UploadPartRequest()
152 |           .withBucketName(intiate.getBucketName)
153 |           .withKey(intiate.getKey)
154 |           .withPartNumber((partNumber + 1).toInt)
155 |           .withUploadId(uploadId)
156 |           .withInputStream(new ByteArrayInputStream(bytes.toArray))
157 |           .withPartSize(bytes.length.toLong)
158 | 
159 |         uploadPart(uploadRequest).map(r => (r.getPartETag, uploadId)).recoverWith {
160 |           case e: Exception =>
161 |             abortMultipartUpload(new AbortMultipartUploadRequest(intiate.getBucketName, intiate.getKey, uploadId))
162 |             Future.failed(e)
163 |         }
164 |       }
165 |       .via(FlowExt.fold(Vector.empty[(PartETag, String)])(_ :+ _))
166 |       .mapAsync(1) { etags =>
167 |         etags.headOption match {
168 |           case Some((_, uploadId)) =>
169 |             val compRequest = new CompleteMultipartUploadRequest(
170 |               intiate.getBucketName, intiate.getKey, uploadId, etags.map(_._1).asJava
171 |             )
172 | 
173 |             val futResult = completeMultipartUpload(compRequest).map(Option.apply).recoverWith { case e: Exception =>
174 |               abortMultipartUpload(new AbortMultipartUploadRequest(intiate.getBucketName, intiate.getKey, uploadId))
175 |               Future.failed(e)
176 |             }
177 |             futResult
178 | 
179 |           case None => Future.successful(None)
180 |         }
181 |       }
182 |       .mapConcat(_.to[scala.collection.immutable.Seq])
183 |   }
184 | 
185 |   /**
186 |    * Upload a stream as a multipart file with a given number of upstream chunk per file. Part file are uploaded sequentially but
187 |    * chunk inside a part file can be uploaded concurrently (tuned with chunkUploadConcurrency).
188 |    * The Flow returns a CompleteMultipartUploadResult for each part file uploaded.
189 |    * @param bucket S3 bucket
190 |    * @param prefix S3 prefix. The actual part files will be named prefix.part.00000001, prefix.part.00000002, ...
191 |    * @param nbChunkPerFile the number of upstream chunks (for example lines) to include in each part file
192 |    * @param chunkUploadConcurrency chunk upload concurrency. Order is guaranteed even with chunkUploadConcurrency > 1.
193 |    */
194 |   def uploadStreamAsMultipartFile(bucket: String, prefix: String, nbChunkPerFile: Int,
195 |                                   chunkUploadConcurrency: Int = 1): Flow[ByteString, CompleteMultipartUploadResult, akka.NotUsed] = {
196 | 
197 |     def formatKey(fileNb: Long) = {
198 |       val pad = "%08d".format(fileNb)
199 |       s"$prefix.part.$pad"
200 |     }
201 | 
202 |     uploadStreamAsMultipartFile(bucket, i => formatKey(i / nbChunkPerFile), nbChunkPerFile, chunkUploadConcurrency)
203 |   }
204 | 
205 |   /**
206 |    * Upload a stream as a multipart file with a given number of upstream chunk per file. Part file are uploaded sequentially but
207 |    * chunk inside a part file can be uploaded concurrently (tuned with chunkUploadConcurrency).
208 |    * The Flow returns a CompleteMultipartUploadResult for each part file uploaded.
209 |    * @param bucket S3 bucket
210 |    * @param getKey S3 key factory that will generate a new key for each part file based on the index of upstream chunks...
211 |    * @param nbChunkPerFile the number of upstream chunks (for example lines) to include in each part file
212 |    * @param chunkUploadConcurrency chunk upload concurrency. Order is guaranteed even with chunkUploadConcurrency > 1.
213 |    */
214 |   def uploadStreamAsMultipartFile(bucket: String, getKey: Long => String, nbChunkPerFile: Int,
215 |       chunkUploadConcurrency : Int): Flow[ByteString, CompleteMultipartUploadResult, akka.NotUsed] = {
216 | 
217 |     Flow[ByteString]
218 |       .via(FlowExt.zipWithIndex)
219 |       .splitWhen { x =>
220 |         val i = x._2
221 |         i != 0 && i % nbChunkPerFile == 0
222 |       }
223 |       .via(
224 |           FlowExt.withHead(includeHeadInUpStream = true) { case (_, i) =>
225 |             Flow[(ByteString, Long)].map(_._1).via(uploadStreamAsFile(bucket, getKey(i), chunkUploadConcurrency))
226 |           }
227 |         )
228 |       .concatSubstreams
229 |   }
230 |   def uploadStreamAsMultipartFile(bucket: String, getKey: Long => String, nbChunkPerFile: Int) : Flow[ByteString, CompleteMultipartUploadResult, akka.NotUsed] =
231 |     uploadStreamAsMultipartFile(bucket,getKey,nbChunkPerFile,1)
232 | }
233 | 


--------------------------------------------------------------------------------
/s3/src/main/scala/AmazonS3ClientMaterialized.scala:
--------------------------------------------------------------------------------
 1 | package com.mfglabs.commons.aws
 2 | package s3
 3 | 
 4 | import akka.stream._
 5 | import akka.stream.scaladsl._
 6 | import akka.util.ByteString
 7 | import com.amazonaws.services.s3.model.{DeleteObjectsResult, S3ObjectSummary}
 8 | import scala.concurrent.Future
 9 | 
10 | /**
11 |  * Additional functions which materialize stream to Future.
12 |  */
13 | class AmazonS3ClientMaterialized(
14 |   client                        : com.amazonaws.services.s3.AmazonS3,
15 |   executorService               : java.util.concurrent.ExecutorService,
16 |   implicit val flowMaterializer : ActorMaterializer
17 | ) extends AmazonS3Client(client, executorService) {
18 | 
19 |   def deleteObjects(bucket: String, commonPrefix: String): Future[Seq[DeleteObjectsResult.DeletedObject]] = {
20 |     for {
21 |       objects   <- listFiles(bucket, Some(commonPrefix))
22 |       deleted   <- deleteObjects(bucket, objects.map(_.getKey):_*)
23 |     } yield deleted
24 |   }
25 | 
26 |   /** List all files with a given prefix of a S3 bucket.
27 |     *
28 |     * @param  bucket the bucket name
29 |     * @param  path an optional path to search in bucket
30 |     * @return a future of seq of file keys & last modified dates (or a failure)
31 |     */
32 |   def listFiles(bucket: String, path: Option[String] = None): Future[Seq[S3ObjectSummary]] = {
33 |     listFilesAsStream(bucket, path).runWith(Sink.seq)
34 |   }
35 | 
36 |   /**
37 |    * Get a S3 file.
38 |    * @param bucket
39 |    * @param key
40 |    * @return binary file
41 |    */
42 |   def getFile(bucket: String, key: String): Future[ByteString] = {
43 |     getFileAsStream(bucket, key).runFold(ByteString.empty)(_ ++ _).map(_.compact)
44 |   }
45 | 
46 |   override def shutdown(): Unit = {
47 |     flowMaterializer.shutdown()
48 |     super.shutdown()
49 |   }
50 | 
51 | }
52 | 


--------------------------------------------------------------------------------
/s3/src/main/scala/AmazonS3Wrapper.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2012-2015 Pellucid Analytics
  3 |  * Copyright 2015 Daniel W. H. James
  4 |  *
  5 |  * Licensed under the Apache License, Version 2.0 (the "License");
  6 |  * you may not use this file except in compliance with the License.
  7 |  * You may obtain a copy of the License at
  8 |  *
  9 |  *   http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package com.mfglabs.commons.aws
 19 | package s3
 20 | 
 21 | import java.io.{InputStream, File}
 22 | import java.net.URL
 23 | 
 24 | import scala.collection.JavaConverters._
 25 | import scala.concurrent.{ExecutionContext, Future}
 26 | 
 27 | import com.amazonaws.services.s3._
 28 | import com.amazonaws.services.s3.model._
 29 | 
 30 | /**
 31 |   * A lightweight wrapper for [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html AmazonS3Client]]
 32 |   *
 33 |   * The AWS Java SDK does not provide an asynchronous S3 client,
 34 |   * so this class follows the approach of the asynchronous clients
 35 |   * that are provided by the SDK. Namely, to make the synchronous
 36 |   * calls within an executor service. The methods in this class
 37 |   * all return Scala futures.
 38 |   *
 39 |   * @param client: the underlying AmazonS3Client
 40 |   * @param clientConfiguration
 41 |   * @param executorService
 42 |   *     an executor service for synchronous calls to the underlying AmazonS3Client.
 43 |   * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3Client.html AmazonS3Client]]
 44 |   * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/auth/AWSCredentialsProvider.html AWSCredentialsProvider]]
 45 |   * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/ClientConfiguration.html ClientConfiguration]]
 46 |   * @see java.util.concurrent.ExecutorService
 47 |   */
 48 | trait AmazonS3Wrapper extends FutureHelper.MethodWrapper {
 49 |   def client          : AmazonS3
 50 |   def executorService : java.util.concurrent.ExecutorService
 51 | 
 52 |   implicit val ec = ExecutionContext.fromExecutorService(executorService)
 53 | 
 54 |   /**
 55 |     * Shutdown the client and the executor service.
 56 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/AmazonWebServiceClient.html#shutdown() AmazonWebServiceClient.shutdown()]]
 57 |     */
 58 |   def shutdown(): Unit = {
 59 |     client.shutdown()
 60 |     executorService.shutdownNow()
 61 |     ()
 62 |   }
 63 | 
 64 |   /**
 65 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#abortMultipartUpload(com.amazonaws.services.s3.model.AbortMultipartUploadRequest) AWS Java SDK]]
 66 |    */
 67 |   def abortMultipartUpload(req: AbortMultipartUploadRequest): Future[Unit] =
 68 |     wrapMethod[AbortMultipartUploadRequest, Unit](client.abortMultipartUpload _, req)
 69 | 
 70 |   /**
 71 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#completeMultipartUpload(com.amazonaws.services.s3.model.CompleteMultipartUploadRequest) AWS Java SDK]]
 72 |    */
 73 |   def completeMultipartUpload(
 74 |     completeMultipartUploadRequest: CompleteMultipartUploadRequest
 75 |   ): Future[CompleteMultipartUploadResult] =
 76 |     wrapMethod(client.completeMultipartUpload, completeMultipartUploadRequest)
 77 | 
 78 |   /**
 79 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#copyObject(com.amazonaws.services.s3.model.CopyObjectRequest) AWS Java SDK]]
 80 |     */
 81 |   def copyObject(
 82 |     copyObjectRequest: CopyObjectRequest
 83 |   ): Future[CopyObjectResult] =
 84 |     wrapMethod(client.copyObject, copyObjectRequest)
 85 | 
 86 |   /**
 87 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#copyObject(com.amazonaws.services.s3.model.CopyObjectRequest) AWS Java SDK]]
 88 |     */
 89 |   def copyObject(
 90 |     sourceBucketName:      String,
 91 |     sourceKey:             String,
 92 |     destinationBucketName: String,
 93 |     destinationKey:        String
 94 |   ): Future[CopyObjectResult] =
 95 |     copyObject(new CopyObjectRequest(sourceBucketName, sourceKey, destinationBucketName, destinationKey))
 96 | 
 97 |   /**
 98 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#copyPart(com.amazonaws.services.s3.model.CopyPartRequest) AWS Java SDK]]
 99 |    */
100 |   def copyPart(
101 |     copyPartRequest: CopyPartRequest
102 |   ): Future[CopyPartResult] =
103 |     wrapMethod(client.copyPart, copyPartRequest)
104 | 
105 |   /**
106 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#createBucket(com.amazonaws.services.s3.model.CreateBucketRequest) AWS Java SDK]]
107 |     */
108 |   def createBucket(
109 |     createBucketRequest: CreateBucketRequest
110 |   ): Future[Bucket] =
111 |     wrapMethod[CreateBucketRequest, Bucket](client.createBucket, createBucketRequest)
112 | 
113 |   /**
114 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#createBucket(com.amazonaws.services.s3.model.CreateBucketRequest) AWS Java SDK]]
115 |     */
116 |   def createBucket(
117 |     bucketName: String
118 |   ): Future[Bucket] =
119 |     createBucket(new CreateBucketRequest(bucketName))
120 | 
121 |   /**
122 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#createBucket(com.amazonaws.services.s3.model.CreateBucketRequest) AWS Java SDK]]
123 |     */
124 |   def createBucket(
125 |     bucketName: String,
126 |     region:     Region
127 |   ): Future[Bucket] =
128 |     createBucket(new CreateBucketRequest(bucketName, region))
129 | 
130 |   /**
131 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#createBucket(com.amazonaws.services.s3.model.CreateBucketRequest) AWS Java SDK]]
132 |     */
133 |   def createBucket(
134 |     bucketName: String,
135 |     region:     String
136 |   ): Future[Bucket] =
137 |     createBucket(new CreateBucketRequest(bucketName, region))
138 | 
139 |   /**
140 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteBucket(com.amazonaws.services.s3.model.DeleteBucketRequest) AWS Java SDK]]
141 |     */
142 |   def deleteBucket(
143 |     deleteBucketRequest: DeleteBucketRequest
144 |   ): Future[Unit] =
145 |     wrapMethod[DeleteBucketRequest, Unit](client.deleteBucket, deleteBucketRequest)
146 | 
147 |   /**
148 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteBucket(com.amazonaws.services.s3.model.DeleteBucketRequest) AWS Java SDK]]
149 |     */
150 |   def deleteBucket(
151 |     bucketName: String
152 |   ): Future[Unit] =
153 |     deleteBucket(new DeleteBucketRequest(bucketName))
154 | 
155 |   /**
156 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteObject(com.amazonaws.services.s3.model.DeleteObjectRequest) AWS Java SDK]]
157 |     */
158 |   def deleteObject(
159 |     deleteObjectRequest: DeleteObjectRequest
160 |   ): Future[Unit] =
161 |     wrapMethod(client.deleteObject, deleteObjectRequest)
162 | 
163 |   /**
164 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteObject(com.amazonaws.services.s3.model.DeleteObjectRequest) AWS Java SDK]]
165 |     */
166 |   def deleteObject(
167 |     bucketName: String,
168 |     key:        String
169 |   ): Future[Unit] =
170 |     deleteObject(new DeleteObjectRequest(bucketName, key))
171 | 
172 |   /**
173 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteObjects(com.amazonaws.services.s3.model.DeleteObjectsRequest) AWS Java SDK]]
174 |     */
175 |   def deleteObjects(
176 |     deleteObjectsRequest: DeleteObjectsRequest
177 |   ): Future[Seq[DeleteObjectsResult.DeletedObject]] =
178 |     wrapMethod((req: DeleteObjectsRequest) => client.deleteObjects(req).getDeletedObjects.asScala.toSeq, deleteObjectsRequest)
179 | 
180 |   /**
181 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteObjects(com.amazonaws.services.s3.model.DeleteObjectsRequest) AWS Java SDK]]
182 |     */
183 |   def deleteObjects(
184 |     bucketName: String,
185 |     keys:       String*
186 |   ): Future[Seq[DeleteObjectsResult.DeletedObject]] = {
187 |     deleteObjects(new DeleteObjectsRequest(bucketName).withKeys(keys:_ *))
188 |   }
189 | 
190 |   /**
191 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteVersion(com.amazonaws.services.s3.model.DeleteVersionRequest) AWS Java SDK]]
192 |     */
193 |   def deleteVersion(
194 |     deleteVersionRequest: DeleteVersionRequest
195 |   ): Future[Unit] =
196 |     wrapMethod(client.deleteVersion, deleteVersionRequest)
197 | 
198 |   /**
199 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#deleteVersion(com.amazonaws.services.s3.model.DeleteVersionRequest) AWS Java SDK]]
200 |     */
201 |   def deleteVersion(
202 |     bucketName: String,
203 |     key:        String,
204 |     versionId:  String
205 |   ): Future[Unit] =
206 |     deleteVersion(new DeleteVersionRequest(bucketName, key, versionId))
207 | 
208 |   /**
209 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#doesBucketExist(java.lang.String) AWS Java SDK]]
210 |     */
211 |   def doesBucketExist(
212 |     bucketName: String
213 |   ): Future[Boolean] =
214 |     wrapMethod(client.doesBucketExist, bucketName)
215 | 
216 |   /**
217 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getBucketLocation(com.amazonaws.services.s3.model.GetBucketLocationRequest) AWS Java SDK]]
218 |     */
219 |   def getBucketLocation(
220 |     getBucketLocationRequest: GetBucketLocationRequest
221 |   ): Future[String] =
222 |     wrapMethod[GetBucketLocationRequest, String](client.getBucketLocation, getBucketLocationRequest)
223 | 
224 |   /**
225 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getBucketLocation(com.amazonaws.services.s3.model.GetBucketLocationRequest) AWS Java SDK]]
226 |     */
227 |   def getBucketLocation(
228 |     bucketName: String
229 |   ): Future[String] =
230 |     getBucketLocation(new GetBucketLocationRequest(bucketName))
231 | 
232 |   /**
233 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#generatePresignedUrl(com.amazonaws.services.s3.model.GeneratePresignedUrlRequest) AWS Java SDK]]
234 |    */
235 |   def generatePresignedUrlRequest(
236 |     generatePresignedUrlRequest: GeneratePresignedUrlRequest
237 |   ): Future[URL] =
238 |     wrapMethod(client.generatePresignedUrl, generatePresignedUrlRequest)
239 | 
240 |   /**
241 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getObject(com.amazonaws.services.s3.model.GetObjectRequest) AWS Java SDK]]
242 |    */
243 |   def getObject(
244 |     getObjectRequest: GetObjectRequest
245 |   ): Future[S3Object] =
246 |     wrapMethod[GetObjectRequest, S3Object](client.getObject, getObjectRequest)
247 | 
248 |   /**
249 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getObject(com.amazonaws.services.s3.model.GetObjectRequest) AWS Java SDK]]
250 |    */
251 |   def getObject(
252 |     bucketName: String,
253 |     key: String
254 |   ): Future[S3Object] =
255 |     getObject(new GetObjectRequest(bucketName, key))
256 | 
257 |   /**
258 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getObject(com.amazonaws.services.s3.model.GetObjectRequest) AWS Java SDK]]
259 |    */
260 |   def getObject(
261 |     getObjectRequest: GetObjectRequest,
262 |     destinationFile: File
263 |   ): Future[ObjectMetadata] =
264 |     wrapMethod[(GetObjectRequest, File), ObjectMetadata]({ case (r: GetObjectRequest, f: File) => client.getObject(r, f) }, (getObjectRequest, destinationFile))
265 | 
266 |   /**
267 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getObjectMetadata(com.amazonaws.services.s3.model.GetObjectMetadataRequest) AWS Java SDK]]
268 |     */
269 |   def getObjectMetadata(
270 |     getObjectMetadataRequest: GetObjectMetadataRequest
271 |   ): Future[ObjectMetadata] =
272 |     wrapMethod(client.getObjectMetadata, getObjectMetadataRequest)
273 | 
274 |   /**
275 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#getObjectMetadata(com.amazonaws.services.s3.model.GetObjectMetadataRequest) AWS Java SDK]]
276 |     */
277 |   def getObjectMetadata(
278 |     bucketName: String,
279 |     key:        String
280 |   ): Future[ObjectMetadata] =
281 |     getObjectMetadata(new GetObjectMetadataRequest(bucketName, key))
282 | 
283 |   /**
284 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#initiateMultipartUpload(com.amazonaws.services.s3.model.InitiateMultipartUploadRequest) AWS Java SDK]]
285 |    */
286 |   def initiateMultipartUpload(
287 |     initiateMultipartUploadRequest: InitiateMultipartUploadRequest
288 |   ): Future[InitiateMultipartUploadResult] =
289 |     wrapMethod(client.initiateMultipartUpload, initiateMultipartUploadRequest)
290 | 
291 |   /**
292 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listBuckets(com.amazonaws.services.s3.model.ListBucketsRequest) AWS Java SDK]]
293 |     */
294 |   def listBuckets(
295 |     listBucketsRequest: ListBucketsRequest
296 |   ): Future[Seq[Bucket]] =
297 |     wrapMethod((req: ListBucketsRequest) => client.listBuckets(req).asScala.toSeq, listBucketsRequest)
298 | 
299 |   /**
300 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listBuckets(com.amazonaws.services.s3.model.ListBucketsRequest) AWS Java SDK]]
301 |     */
302 |   def listBuckets(): Future[Seq[Bucket]] =
303 |     listBuckets(new ListBucketsRequest())
304 | 
305 |   /**
306 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listNextBatchOfObjects(com.amazonaws.services.s3.model.ListNextBatchOfObjectsRequest)]]
307 |    */
308 |   def listNextBatchOfObjects(req: ListNextBatchOfObjectsRequest): Future[ObjectListing] =
309 |     wrapMethod[ListNextBatchOfObjectsRequest, ObjectListing](client.listNextBatchOfObjects, req)
310 | 
311 |   /**
312 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listNextBatchOfObjects(com.amazonaws.services.s3.model.ObjectListing) AWS Java SDK]]
313 |    */
314 |   def listNextBatchOfObjects(req: ObjectListing): Future[ObjectListing] =
315 |     wrapMethod[ObjectListing, ObjectListing](client.listNextBatchOfObjects _, req)
316 | 
317 |   /**
318 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listObjects(com.amazonaws.services.s3.model.ListObjectsRequest) AWS Java SDK]]
319 |     */
320 |   def listObjects(
321 |     listObjectsRequest: ListObjectsRequest
322 |   ): Future[ObjectListing] =
323 |     wrapMethod[ListObjectsRequest, ObjectListing](client.listObjects, listObjectsRequest)
324 | 
325 |   /**
326 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listObjects(com.amazonaws.services.s3.model.ListObjectsRequest) AWS Java SDK]]
327 |     */
328 |   def listObjects(
329 |     bucketName: String
330 |   ): Future[ObjectListing] =
331 |     listObjects(
332 |       new ListObjectsRequest()
333 |       .withBucketName(bucketName)
334 |     )
335 | 
336 |   /**
337 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listObjects(com.amazonaws.services.s3.model.ListObjectsRequest) AWS Java SDK]]
338 |     */
339 |   def listObjects(
340 |     bucketName: String,
341 |     prefix:     String
342 |   ): Future[ObjectListing] =
343 |     listObjects(
344 |       new ListObjectsRequest()
345 |       .withBucketName(bucketName)
346 |       .withPrefix(prefix)
347 |     )
348 | 
349 |   /**
350 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listVersions(com.amazonaws.services.s3.model.ListVersionsRequest) AWS Java SDK]]
351 |     */
352 |   def listVersions(
353 |     listVersionsRequest: ListVersionsRequest
354 |   ): Future[VersionListing] =
355 |       wrapMethod(client.listVersions, listVersionsRequest)
356 | 
357 |   /**
358 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listVersions(com.amazonaws.services.s3.model.ListVersionsRequest) AWS Java SDK]]
359 |     */
360 |   def listVersions(
361 |     bucketName: String,
362 |     prefix:     String
363 |   ): Future[VersionListing] =
364 |     listVersions(
365 |       new ListVersionsRequest()
366 |       .withBucketName(bucketName)
367 |       .withPrefix(prefix)
368 |     )
369 | 
370 |   /**
371 |     * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#listVersions(com.amazonaws.services.s3.model.ListVersionsRequest) AWS Java SDK]]
372 |     */
373 |   def listVersions(
374 |     bucketName:      String,
375 |     prefix:          String,
376 |     keyMarker:       String,
377 |     versionIdMarker: String,
378 |     delimiter:       String,
379 |     maxKeys:         Int
380 |   ): Future[VersionListing] =
381 |     listVersions(new ListVersionsRequest(bucketName, prefix, keyMarker, versionIdMarker, delimiter, maxKeys))
382 | 
383 |   /**
384 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#putObject(com.amazonaws.services.s3.model.PutObjectRequest) AWS Java SDK]]
385 |    */
386 |   def putObject(
387 |     putObjectRequest: PutObjectRequest
388 |   ): Future[PutObjectResult] =
389 |     wrapMethod[PutObjectRequest, PutObjectResult](client.putObject, putObjectRequest)
390 | 
391 |   /**
392 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#putObject(com.amazonaws.services.s3.model.PutObjectRequest) AWS Java SDK]]
393 |    */
394 |   def putObject(
395 |     bucketName: String,
396 |     key:        String,
397 |     file:       File
398 |   ): Future[PutObjectResult] =
399 |     putObject(new PutObjectRequest(bucketName, key, file))
400 | 
401 |   /**
402 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#putObject(com.amazonaws.services.s3.model.PutObjectRequest) AWS Java SDK]]
403 |    */
404 |   def putObject(
405 |     bucketName: String,
406 |     key: String,
407 |     input: InputStream,
408 |     metadata: ObjectMetadata
409 |   ): Future[PutObjectResult] =
410 |     putObject(new PutObjectRequest(bucketName, key, input, metadata))
411 | 
412 |   /**
413 |    * @see [[http://docs.aws.amazon.com/AWSJavaSDK/latest/javadoc/com/amazonaws/services/s3/AmazonS3.html#uploadPart(com.amazonaws.services.s3.model.UploadPartRequest) AWS Java SDK]]
414 |    */
415 |   def uploadPart(req: UploadPartRequest): Future[UploadPartResult] =
416 |     wrapMethod[UploadPartRequest, UploadPartResult](client.uploadPart _, req)
417 | 
418 | }
419 | 


--------------------------------------------------------------------------------
/s3/src/main/scala/futureTransfer.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2012-2015 Pellucid Analytics
  3 |  * Copyright 2015 Daniel W. H. James
  4 |  *
  5 |  * Licensed under the Apache License, Version 2.0 (the "License");
  6 |  * you may not use this file except in compliance with the License.
  7 |  * You may obtain a copy of the License at
  8 |  *
  9 |  *   http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package com.mfglabs.commons.aws
 19 | package s3
 20 | 
 21 | import scala.concurrent.{Future, Promise}
 22 | 
 23 | import com.amazonaws.event.{ProgressListener, ProgressEvent, ProgressEventType}
 24 | import com.amazonaws.services.s3.transfer.Transfer
 25 | 
 26 | import org.slf4j.{Logger, LoggerFactory}
 27 | 
 28 | 
 29 | /**
 30 |   * A helper object providing a Scala Future interface for S3 Transfers.
 31 |   *
 32 |   * Transfers to and from S3 using the TransferManager provider a listener
 33 |   * interface, and [[FutureTransfer.listenFor]] adapts this interface to
 34 |   * Scala futures.
 35 |   *
 36 |   * @see [[com.amazonaws.services.s3.transfer.TransferManager TransferManager]]
 37 |   * @see [[com.amazonaws.services.s3.transfer.Transfer Transfer]]
 38 |   */
 39 | object FutureTransfer {
 40 | 
 41 |   private val logger: Logger = LoggerFactory.getLogger("com.mfglabs.commons.aws.s3.FutureTransfer")
 42 | 
 43 |   /**
 44 |     * Attach a listener to an S3 Transfer and return it as a Future.
 45 |     *
 46 |     * This helper method attaches a progress and state change listeners to the given
 47 |     * Transfer object. The returned future is completed with the
 48 |     * same transfer when the transfer is ‘done’ (canceled, completed,
 49 |     * or failed). The future will always been completed successfully
 50 |     * even if the transfer itself has failed. It is up to the caller
 51 |     * to extract the result of the transfer and perform any error
 52 |     * handling.
 53 |     *
 54 |     * In essence, this helper just gives back the transfer when it is done.
 55 |     *
 56 |     * The detailed progress of the transfer is logged at debug level to the
 57 |     * `com.github.dwhjames.awswrap.s3.FutureTransfer` logger.
 58 |     *
 59 |     * @tparam T
 60 |     *     a subtype of Transfer.
 61 |     * @param transfer
 62 |     *     an S3 Transfer to listen for progress.
 63 |     * @return the transfer in a future.
 64 |     * @see [[com.amazonaws.services.s3.transfer.Transfer Transfer]]
 65 |     * @see [[com.amazonaws.event.ProgressListener ProgressListener]]
 66 |     */
 67 |   def listenFor[T <: Transfer](transfer: T): Future[transfer.type] = {
 68 |     import com.amazonaws.services.s3.transfer.internal.{ AbstractTransfer, TransferStateChangeListener }
 69 |     val transferDescription = transfer.getDescription
 70 |     def debugLog(eventType: String): Unit = {
 71 |       logger.debug(s"$eventType : $transferDescription")
 72 |     }
 73 |     def logTransferState(state: Transfer.TransferState): Unit = {
 74 |       if (logger.isDebugEnabled) {
 75 |         state match {
 76 |           case Transfer.TransferState.Waiting =>
 77 |             debugLog("Waiting")
 78 |           case Transfer.TransferState.InProgress =>
 79 |             debugLog("InProgress")
 80 |           case Transfer.TransferState.Completed =>
 81 |             debugLog("Completed")
 82 |           case Transfer.TransferState.Canceled =>
 83 |             debugLog("Canceled")
 84 |           case Transfer.TransferState.Failed =>
 85 |             debugLog("Failed")
 86 |           case _ =>
 87 |             logger.warn(s"unrecognized transfer state for transfer $transferDescription")
 88 |         }
 89 |       }
 90 |     }
 91 |     def logProgressEvent(progressEvent: ProgressEvent): Unit = {
 92 |       if (logger.isDebugEnabled) {
 93 |         progressEvent.getEventType match {
 94 |           case ProgressEventType.CLIENT_REQUEST_FAILED_EVENT =>
 95 |             debugLog("CLIENT_REQUEST_FAILED_EVENT")
 96 |           case ProgressEventType.CLIENT_REQUEST_RETRY_EVENT =>
 97 |             debugLog("CLIENT_REQUEST_RETRY_EVENT")
 98 |           case ProgressEventType.CLIENT_REQUEST_STARTED_EVENT =>
 99 |             debugLog("CLIENT_REQUEST_STARTED_EVENT")
100 |           case ProgressEventType.CLIENT_REQUEST_SUCCESS_EVENT =>
101 |             debugLog("CLIENT_REQUEST_SUCCESS_EVENT")
102 |           case ProgressEventType.HTTP_REQUEST_COMPLETED_EVENT =>
103 |             debugLog("HTTP_REQUEST_COMPLETED_EVENT")
104 |           case ProgressEventType.HTTP_REQUEST_CONTENT_RESET_EVENT =>
105 |             debugLog("HTTP_REQUEST_CONTENT_RESET_EVENT")
106 |           case ProgressEventType.HTTP_REQUEST_STARTED_EVENT =>
107 |             debugLog("HTTP_REQUEST_STARTED_EVENT")
108 |           case ProgressEventType.HTTP_RESPONSE_COMPLETED_EVENT =>
109 |             debugLog("HTTP_RESPONSE_COMPLETED_EVENT")
110 |           case ProgressEventType.HTTP_RESPONSE_CONTENT_RESET_EVENT =>
111 |             debugLog("HTTP_RESPONSE_CONTENT_RESET_EVENT")
112 |           case ProgressEventType.HTTP_RESPONSE_STARTED_EVENT =>
113 |             debugLog("HTTP_RESPONSE_STARTED_EVENT")
114 |           case ProgressEventType.REQUEST_BYTE_TRANSFER_EVENT =>
115 |             debugLog("REQUEST_BYTE_TRANSFER_EVENT")
116 |           case ProgressEventType.REQUEST_CONTENT_LENGTH_EVENT =>
117 |             debugLog("REQUEST_CONTENT_LENGTH_EVENT")
118 |           case ProgressEventType.RESPONSE_BYTE_DISCARD_EVENT =>
119 |             debugLog("RESPONSE_BYTE_DISCARD_EVENT")
120 |           case ProgressEventType.RESPONSE_BYTE_TRANSFER_EVENT =>
121 |             debugLog("RESPONSE_BYTE_TRANSFER_EVENT")
122 |           case ProgressEventType.RESPONSE_CONTENT_LENGTH_EVENT =>
123 |             debugLog("RESPONSE_CONTENT_LENGTH_EVENT")
124 |           case ProgressEventType.TRANSFER_CANCELED_EVENT =>
125 |             debugLog("TRANSFER_CANCELED_EVENT")
126 |           case ProgressEventType.TRANSFER_COMPLETED_EVENT =>
127 |             debugLog("TRANSFER_COMPLETED_EVENT")
128 |           case ProgressEventType.TRANSFER_FAILED_EVENT =>
129 |             debugLog("TRANSFER_FAILED_EVENT")
130 |           case ProgressEventType.TRANSFER_PART_COMPLETED_EVENT =>
131 |             debugLog("TRANSFER_PART_COMPLETED_EVENT")
132 |           case ProgressEventType.TRANSFER_PART_FAILED_EVENT =>
133 |             debugLog("TRANSFER_PART_FAILED_EVENT")
134 |           case ProgressEventType.TRANSFER_PART_STARTED_EVENT =>
135 |             debugLog("TRANSFER_PART_STARTED_EVENT")
136 |           case ProgressEventType.TRANSFER_PREPARING_EVENT =>
137 |             debugLog("TRANSFER_PREPARING_EVENT")
138 |           case ProgressEventType.TRANSFER_STARTED_EVENT =>
139 |             debugLog("TRANSFER_STARTED_EVENT")
140 |           case _ =>
141 |             logger.warn(s"unrecognized progress event type for transfer $transferDescription")
142 |         }
143 |       }
144 |     }
145 | 
146 |     val p = Promise[transfer.type]
147 | 
148 |     if (transfer.isInstanceOf[AbstractTransfer]) {
149 |       /* Attach a state change listener to the transfer.
150 |        * At this point, the transfer is already in progress
151 |        * and may even have already completed. We will have
152 |        * missed any state change events that have already been
153 |        * fired, including the completion event!
154 |        */
155 |       transfer.asInstanceOf[AbstractTransfer].addStateChangeListener(new TransferStateChangeListener {
156 |         /* Note that the transferStateChanged will be called in the Java SDK’s
157 |          * special thread for callbacks, so any blocking calls here have
158 |          * the potential to induce deadlock.
159 |          */
160 |         override def transferStateChanged(t: Transfer, state: Transfer.TransferState): Unit = {
161 |           logTransferState(state)
162 | 
163 |           if (state == Transfer.TransferState.Completed ||
164 |               state == Transfer.TransferState.Canceled ||
165 |               state == Transfer.TransferState.Failed) {
166 |             val success = p trySuccess transfer
167 |             if (logger.isDebugEnabled) {
168 |               if (success) {
169 |                 logger.debug(s"promise successfully completed from transfer state change listener for $transferDescription")
170 |               }
171 |             }
172 |           }
173 |         }
174 |       })
175 |     }
176 | 
177 |     /* Attach a progress listener to the transfer.
178 |      * At this point, the transfer is already in progress
179 |      * and may even have already completed. We will have
180 |      * missed any progress events that have already been
181 |      * fired, including the completion event!
182 |      */
183 |     transfer.addProgressListener(new ProgressListener {
184 |       /* Note that the progressChanged will be called in the Java SDK’s
185 |        * `java-sdk-progress-listener-callback-thread` special thread
186 |        * for progress event callbacks, so any blocking calls here have
187 |        * the potential to induce deadlock.
188 |        */
189 |       override def progressChanged(progressEvent: ProgressEvent): Unit = {
190 |         logProgressEvent(progressEvent)
191 | 
192 |         val code = progressEvent.getEventType()
193 |         if (code == ProgressEventType.TRANSFER_CANCELED_EVENT ||
194 |             code == ProgressEventType.TRANSFER_COMPLETED_EVENT ||
195 |             code == ProgressEventType.TRANSFER_FAILED_EVENT) {
196 |           val success = p trySuccess transfer
197 |           if (logger.isDebugEnabled) {
198 |             if (success) {
199 |               logger.debug(s"promise successfully completed from progress listener for $transferDescription")
200 |             }
201 |           }
202 |         }
203 |       }
204 |     })
205 | 
206 |     /* In case the progress listener never fires due to the
207 |      * transfer already being done, poll the transfer once.
208 |      */
209 |     if (transfer.isDone) {
210 |       val success = p trySuccess transfer
211 |       if (logger.isDebugEnabled) {
212 |         if (success) {
213 |           logger.debug(s"promise successfully completed from outside of callbacks for $transferDescription")
214 |         }
215 |       }
216 |     }
217 | 
218 |     p.future
219 |   }
220 | }
221 | 


--------------------------------------------------------------------------------
/s3/src/test/resources/medium.txt:
--------------------------------------------------------------------------------
1 | m
2 | f
3 | g
4 | labs
5 | a
6 | b
7 | s
8 | 


--------------------------------------------------------------------------------
/s3/src/test/resources/small.txt:
--------------------------------------------------------------------------------
1 | This is toto.
2 | 


--------------------------------------------------------------------------------
/s3/src/test/scala/S3Spec.scala:
--------------------------------------------------------------------------------
  1 | package com.mfglabs.commons.aws
  2 | package s3
  3 | 
  4 | import akka.actor.ActorSystem
  5 | import akka.stream._
  6 | import akka.stream.scaladsl._
  7 | import akka.util.ByteString
  8 | import com.amazonaws.services.s3.model.{AmazonS3Exception, CompleteMultipartUploadResult}
  9 | import com.mfglabs.stream._
 10 | import org.scalatest._
 11 | import concurrent.ScalaFutures
 12 | import org.scalatest.time.{Minutes, Millis, Span}
 13 | 
 14 | import scala.concurrent._
 15 | import scala.concurrent.duration._
 16 | 
 17 | class S3Spec extends FlatSpec with Matchers with ScalaFutures with BeforeAndAfterAll {
 18 |   import s3._
 19 |   import scala.concurrent.ExecutionContext.Implicits.global
 20 | 
 21 |   val bucket = "mfg-commons-aws"
 22 |   val keyPrefix = "test/core"
 23 |   val multipartkeyPrefix = "test/multipart-upload-core"
 24 | 
 25 |   implicit override val patienceConfig =
 26 |     PatienceConfig(timeout = Span(3, Minutes), interval = Span(20, Millis))
 27 | 
 28 |   implicit val system = ActorSystem()
 29 |   implicit val fm = ActorMaterializer()
 30 | 
 31 |   val s3Client = AmazonS3Client.from(
 32 |     com.amazonaws.regions.Regions.EU_WEST_1,
 33 |     new com.amazonaws.auth.profile.ProfileCredentialsProvider("mfg")
 34 |   )().materialized(fm)
 35 | 
 36 |   it should "upload/list/delete small files" in {
 37 |     s3Client.deleteObjects(bucket, s"$keyPrefix").futureValue
 38 |     s3Client.putObject(bucket, s"$keyPrefix/small.txt", new java.io.File(getClass.getResource("/small.txt").getPath)).futureValue
 39 |     val l = s3Client.listFiles(bucket, Some(keyPrefix)).futureValue
 40 |     s3Client.deleteObjects(bucket, s"$keyPrefix/small.txt").futureValue
 41 |     val l2 = s3Client.listFiles(bucket, Some(keyPrefix)).futureValue
 42 | 
 43 |     (l map (_.getKey)) should equal(List(s"$keyPrefix/small.txt"))
 44 |     l2 should be('empty)
 45 |   }
 46 | 
 47 |   it should "throw an exception when a file is non-existent" in {
 48 |     val source = s3Client.getFileAsStream(bucket, "foo")
 49 |     intercept[AmazonS3Exception] {
 50 |       Await.result(source.runFold(ByteString.empty)(_ ++ _).map(_.compact), 5 seconds)
 51 |     }
 52 |   }
 53 | 
 54 |   it should "upload and download a big file as a single file" in {
 55 |     val futBytes = StreamConverters.fromInputStream(() => getClass.getResourceAsStream("/big.txt"))
 56 |       .via(s3Client.uploadStreamAsFile(bucket, s"$keyPrefix/big", chunkUploadConcurrency = 2))
 57 |       .flatMapConcat(_ => s3Client.getFileAsStream(bucket, s"$keyPrefix/big"))
 58 |       .runFold(ByteString.empty)(_ ++ _)
 59 |       .map(_.compact)
 60 | 
 61 |     val expectedBytes = StreamConverters.fromInputStream(() => getClass.getResourceAsStream("/big.txt"))
 62 |       .runFold(ByteString.empty)(_ ++ _).map(_.compact)
 63 | 
 64 |     whenReady(futBytes zip expectedBytes) { case (bytes, expectedBytes) =>
 65 |       bytes shouldEqual expectedBytes
 66 |     }
 67 |   }
 68 | 
 69 |   it should "download a big file and chunk it by line" in {
 70 |     val futLines = s3Client.getFileAsStream(bucket, s"$keyPrefix/big")
 71 |       .via(FlowExt.rechunkByteStringBySize(2 * 1024 * 1024))
 72 |       .via(FlowExt.rechunkByteStringBySeparator(ByteString("\n"), 8 * 1024))
 73 |       .map(_.utf8String)
 74 |       .runWith(Sink.seq)
 75 | 
 76 |     val futExpectedLines = StreamConverters
 77 |         .fromInputStream(() => getClass.getResourceAsStream("/big.txt"))
 78 |         .runFold(ByteString.empty)(_ ++ _)
 79 |         .map(_.compact.utf8String)
 80 |         .map(_.split("\n").to[scala.collection.immutable.Seq])
 81 | 
 82 |     whenReady(futLines zip futExpectedLines) { case (lines, expectedLines) =>
 83 |       lines shouldEqual expectedLines
 84 |     }
 85 |   }
 86 | 
 87 | 
 88 |   it should "upload and download a big file as a multipart file" in {
 89 |     val bytes = StreamConverters
 90 |       .fromInputStream(() => getClass.getResourceAsStream("/big.txt"), chunkSize = 2 * 1024 * 1024)
 91 |       .via(s3Client.uploadStreamAsMultipartFile(bucket, s"$keyPrefix/big", nbChunkPerFile = 1, chunkUploadConcurrency = 2))
 92 |       .via(FlowExt.fold[CompleteMultipartUploadResult, Vector[CompleteMultipartUploadResult]](Vector.empty)(_ :+ _))
 93 |       .flatMapConcat(_ => s3Client.getMultipartFileAsStream(bucket, s"$keyPrefix/big.part"))
 94 |       .runFold(ByteString.empty)(_ ++ _)
 95 |       .map(_.compact).futureValue
 96 | 
 97 |    val expectedBytes = StreamConverters
 98 |       .fromInputStream(() => getClass.getResourceAsStream("/big.txt"))
 99 |       .runFold(ByteString.empty)(_ ++ _).map(_.compact).futureValue
100 | 
101 |     bytes shouldEqual expectedBytes
102 |   }
103 | 
104 | 
105 |   override def afterAll() = {
106 |     s3Client.shutdown()
107 |     val _ = system.terminate().futureValue
108 |   }
109 | }
110 | 


--------------------------------------------------------------------------------
/scaladoc.sbt:
--------------------------------------------------------------------------------
 1 | import scala.util.matching.Regex.Match
 2 | 
 3 | packageDoc in Compile <<= packageDoc in ScalaUnidoc
 4 | 
 5 | artifact in (ScalaUnidoc, packageDoc) := {
 6 |   val previous: Artifact = (artifact in (ScalaUnidoc, packageDoc)).value
 7 |   previous.copy(classifier = Some("javadoc"))
 8 | }
 9 | 
10 | scalacOptions in (Compile, doc) ++= Seq(
11 |   "-implicits",
12 |   "-sourcepath", baseDirectory.value.getAbsolutePath,
13 |   "-doc-source-url", s"https://github.com/MfgLabs/commons-aws/tree/${version.value}€{FILE_PATH}.scala"
14 | )
15 | 
16 | autoAPIMappings := true
17 | 
18 | apiURL := Some(url("https://MfgLabs.github.io/commons-aws/api/current/"))
19 | 


--------------------------------------------------------------------------------
/sqs/src/main/scala/AmazonSQSClient.scala:
--------------------------------------------------------------------------------
  1 | package com.mfglabs.commons.aws
  2 | package sqs
  3 | 
  4 | import akka.actor._
  5 | import akka.stream.scaladsl._
  6 | import com.amazonaws.client.builder.ExecutorFactory
  7 | import com.amazonaws.services.sqs.model._
  8 | import com.mfglabs.stream._
  9 | import java.util.concurrent.ExecutorService
 10 | import scala.concurrent.ExecutionContext
 11 | import scala.concurrent.duration._
 12 | 
 13 | 
 14 | object AmazonSQSClient {
 15 |   import com.amazonaws.auth._
 16 |   import com.amazonaws.ClientConfiguration
 17 |   import com.amazonaws.regions.Regions
 18 |   import com.amazonaws.services.sqs.AmazonSQSAsyncClientBuilder
 19 |   import FutureHelper.defaultExecutorService
 20 | 
 21 |   def apply(
 22 |     region              : Regions,
 23 |     awsCredentials      : AWSCredentials,
 24 |     clientConfiguration : ClientConfiguration = new ClientConfiguration()
 25 |   )(
 26 |     executorService     : ExecutorService     = defaultExecutorService(clientConfiguration, "aws.wrap.sqs")
 27 |   ): AmazonSQSClient = {
 28 |     from(region, new AWSStaticCredentialsProvider(awsCredentials), clientConfiguration)(executorService)
 29 |   }
 30 | 
 31 |   def from(
 32 |     region                 : Regions,
 33 |     awsCredentialsProvider : AWSCredentialsProvider = new DefaultAWSCredentialsProviderChain,
 34 |     clientConfiguration    : ClientConfiguration    = new ClientConfiguration()
 35 |   )(
 36 |     executorService     : ExecutorService     = defaultExecutorService(clientConfiguration, "aws.wrap.sqs")
 37 |   ): AmazonSQSClient = {
 38 |    val client = AmazonSQSAsyncClientBuilder
 39 |       .standard()
 40 |       .withRegion(region)
 41 |       .withCredentials(awsCredentialsProvider)
 42 |       .withClientConfiguration(clientConfiguration)
 43 |       .withExecutorFactory(new ExecutorFactory { def newExecutor() = executorService })
 44 |       .build()
 45 | 
 46 |     new AmazonSQSClient(client, executorService)
 47 |   }
 48 | 
 49 |   def build(builder: AmazonSQSAsyncClientBuilder)(
 50 |     executorService : ExecutorService = defaultExecutorService(builder.getClientConfiguration, "aws.wrap.sqs")
 51 |   ) = {
 52 |    val client = builder
 53 |       .withExecutorFactory(new ExecutorFactory { def newExecutor() = executorService })
 54 |       .build()
 55 | 
 56 |     new AmazonSQSClient(client, executorService)
 57 |   }
 58 | 
 59 | }
 60 | 
 61 | 
 62 | class AmazonSQSClient(
 63 |   val client          : com.amazonaws.services.sqs.AmazonSQSAsync,
 64 |   val executorService : ExecutorService
 65 | ) extends AmazonSQSWrapper {
 66 | 
 67 |   import scala.collection.JavaConverters._
 68 | 
 69 |   val defaultMessageOpsConcurrency = 16
 70 | 
 71 |   implicit val ec = ExecutionContext.fromExecutorService(executorService)
 72 | 
 73 |   /**
 74 |    * Send SQS messages as a stream
 75 |    * Important note: SQS does not ensure message ordering, so setting messageSendingConcurrency parameter equal to 1 does not
 76 |    * guarantee total message ordering.
 77 |    * @param messageSendingConcurrency
 78 |    */
 79 |   def sendMessageAsStream(messageSendingConcurrency: Int = defaultMessageOpsConcurrency): Flow[SendMessageRequest, SendMessageResult, akka.NotUsed] = {
 80 |     Flow[SendMessageRequest].mapAsync(messageSendingConcurrency) { msg =>
 81 |       sendMessage(msg)
 82 |     }
 83 |   }
 84 | 
 85 |   /**
 86 |    * Receive messages from a SQS queue as a stream.
 87 |    * @param queueUrl SQS queue url
 88 |    * @param longPollingMaxWait SQS long-polling parameter.
 89 |    * @param autoAck If true, the SQS messages will be automatically ack once they are received. If false, you must call
 90 |    *                deleteMessage yourself when you want to ack the message.
 91 |    * @param messageAttributeNames the message attribute names asked to be returned
 92 |    */
 93 |   def receiveMessageAsStream(queueUrl: String, messageAckingConcurrency: Int = defaultMessageOpsConcurrency,
 94 |                              longPollingMaxWait: FiniteDuration = 20 seconds, autoAck: Boolean = false,
 95 |                              messageAttributeNames: Seq[String] = Seq.empty): Source[Message, ActorRef] = {
 96 |     val source = SourceExt.bulkPullerAsync(0L) { (total, currentDemand) =>
 97 |       val msg = new ReceiveMessageRequest(queueUrl)
 98 |       msg.setWaitTimeSeconds(longPollingMaxWait.toSeconds.toInt) // > 0 seconds allow long-polling. 20 seconds is the maximum
 99 |       msg.setMaxNumberOfMessages(Math.min(currentDemand, 10)) // 10 is SQS limit
100 |       msg.setMessageAttributeNames(messageAttributeNames.asJava)
101 |       receiveMessage(msg).map(res => (res.getMessages.asScala, false))
102 |     }
103 | 
104 |     if (autoAck)
105 |       source.mapAsync(messageAckingConcurrency) { msg =>
106 |         deleteMessage(queueUrl, msg.getReceiptHandle).map(_ => msg)
107 |       }
108 |     else source
109 |   }
110 | 
111 | 
112 |   /**
113 |    * Receive messages from a SQS queue as a stream and if connection closes,
114 |    * retry using an exponential backoff reconnection strategy (2^retryNb * retryMinInterval).
115 |    *
116 |    * @param queueUrl SQS queue url
117 |    * @param maxRetryDuration maximum retry duration.
118 |    * @param retryMinInterval minimum delay before retrying.
119 |    * @param longPollingMaxWait SQS long-polling parameter.
120 |    * @param autoAck If true, the SQS messages will be automatically ack once they are received. If false, you must call
121 |    *                deleteMessage yourself when you want to ack the message.
122 |    * @param messageAttributeNames the message attribute names asked to be returned
123 |    */
124 |   def receiveMessageAsStreamWithRetryExpBackoff(
125 |       queueUrl: String, messageAckingConcurrency: Int = defaultMessageOpsConcurrency,
126 |       maxRetryDuration: FiniteDuration = 540.seconds, retryMinInterval: FiniteDuration = 1.second,
127 |       longPollingMaxWait: FiniteDuration = 20 seconds, autoAck: Boolean = false,
128 |       messageAttributeNames: Seq[String] = Seq.empty): Source[Message, ActorRef] = {
129 |     val source = SourceExt.bulkPullerAsyncWithErrorExpBackoff(0L, maxRetryDuration, retryMinInterval) { (total, currentDemand) =>
130 |       val msg = new ReceiveMessageRequest(queueUrl)
131 |       msg.setWaitTimeSeconds(longPollingMaxWait.toSeconds.toInt) // > 0 seconds allow long-polling. 20 seconds is the maximum
132 |       msg.setMaxNumberOfMessages(Math.min(currentDemand, 10)) // 10 is SQS limit
133 |       msg.setMessageAttributeNames(messageAttributeNames.asJava)
134 |       receiveMessage(msg).map(res => (res.getMessages.asScala, false))
135 |     }
136 | 
137 |     if (autoAck)
138 |       source.mapAsync(messageAckingConcurrency) { msg =>
139 |         deleteMessage(queueUrl, msg.getReceiptHandle).map(_ => msg)
140 |       }
141 |     else source
142 |   }
143 | }
144 | 


--------------------------------------------------------------------------------
/sqs/src/main/scala/AmazonSQSWrapper.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Copyright 2012-2015 Pellucid Analytics
  3 |  * Copyright 2015 Daniel W. H. James
  4 |  *
  5 |  * Licensed under the Apache License, Version 2.0 (the "License");
  6 |  * you may not use this file except in compliance with the License.
  7 |  * You may obtain a copy of the License at
  8 |  *
  9 |  *   http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package com.mfglabs.commons.aws
 19 | package sqs
 20 | 
 21 | import scala.concurrent.{Future, ExecutionContext}
 22 | import scala.collection.JavaConverters._
 23 | 
 24 | import com.amazonaws.services.sqs.AmazonSQSAsync
 25 | import com.amazonaws.services.sqs.model._
 26 | 
 27 | trait AmazonSQSWrapper {
 28 |   import FutureHelper._
 29 | 
 30 |   def client: AmazonSQSAsync
 31 |   implicit def ec: ExecutionContext
 32 | 
 33 |   def addPermission(
 34 |     addPermissionRequest: AddPermissionRequest
 35 |   ): Future[AddPermissionResult] =
 36 |     wrapAsyncMethod(client.addPermissionAsync, addPermissionRequest)
 37 | 
 38 |   def addPermission(
 39 |     queueUrl:       String,
 40 |     label:          String,
 41 |     accountActions: Map[String, String]
 42 |   ): Future[AddPermissionResult] = {
 43 |     val (accounts, actions) = accountActions.unzip
 44 |     addPermission(
 45 |       new AddPermissionRequest(
 46 |         queueUrl,
 47 |         label,
 48 |         accounts.toSeq.asJava,
 49 |         actions.toSeq.asJava
 50 |       )
 51 |     )
 52 |   }
 53 | 
 54 |   def changeMessageVisibility(
 55 |     changeMessageVisibilityRequest: ChangeMessageVisibilityRequest
 56 |   ): Future[ChangeMessageVisibilityResult] =
 57 |     wrapAsyncMethod(client.changeMessageVisibilityAsync, changeMessageVisibilityRequest)
 58 | 
 59 |   def changeMessageVisibility(
 60 |     queueUrl:          String,
 61 |     receiptHandle:     String,
 62 |     visibilityTimeout: Int
 63 |   ): Future[ChangeMessageVisibilityResult] =
 64 |     changeMessageVisibility(new ChangeMessageVisibilityRequest(queueUrl, receiptHandle, visibilityTimeout))
 65 | 
 66 |   def changeMessageVisibilityBatch(
 67 |     changeMessageVisibilityBatchRequest: ChangeMessageVisibilityBatchRequest
 68 |   ): Future[ChangeMessageVisibilityBatchResult] =
 69 |     wrapAsyncMethod(client.changeMessageVisibilityBatchAsync, changeMessageVisibilityBatchRequest)
 70 | 
 71 |   def changeMessageVisibilityBatch(
 72 |     queueUrl:            String,
 73 |     messageVisibilities: Seq[(String, String, Int)]
 74 |   ): Future[ChangeMessageVisibilityBatchResult] =
 75 |     changeMessageVisibilityBatch(
 76 |       new ChangeMessageVisibilityBatchRequest(
 77 |         queueUrl,
 78 |         messageVisibilities.map { case (id, receiptHandle, visibilityTimeout) =>
 79 |           new ChangeMessageVisibilityBatchRequestEntry(id, receiptHandle)
 80 |           .withVisibilityTimeout(visibilityTimeout)
 81 |         } .asJava
 82 |       )
 83 |     )
 84 | 
 85 |   def createQueue(
 86 |     createQueueRequest: CreateQueueRequest
 87 |   ): Future[CreateQueueResult] =
 88 |     wrapAsyncMethod[CreateQueueRequest, CreateQueueResult](client.createQueueAsync, createQueueRequest)
 89 | 
 90 |   def createQueue(
 91 |     queueName: String,
 92 |     attributes: Map[QueueAttributeName, Any] = Map.empty
 93 |   ): Future[CreateQueueResult] =
 94 |     createQueue(
 95 |       new CreateQueueRequest(queueName)
 96 |       .withAttributes(
 97 |         attributes.map{ case (n, v) =>
 98 |           (n.toString, v.toString)
 99 |         }.asJava
100 |       )
101 |     )
102 | 
103 |   def deleteMessage(
104 |     deleteMessageRequest: DeleteMessageRequest
105 |   ): Future[DeleteMessageResult] =
106 |     wrapAsyncMethod(client.deleteMessageAsync, deleteMessageRequest)
107 | 
108 |   def deleteMessage(
109 |     queueUrl:      String,
110 |     receiptHandle: String
111 |   ): Future[DeleteMessageResult] =
112 |     deleteMessage(new DeleteMessageRequest(queueUrl, receiptHandle))
113 | 
114 |   def deleteMessageBatch(
115 |     deleteMessageBatchRequest: DeleteMessageBatchRequest
116 |   ): Future[DeleteMessageBatchResult] =
117 |     wrapAsyncMethod(client.deleteMessageBatchAsync, deleteMessageBatchRequest)
118 | 
119 |   def deleteMessageBatch(
120 |     queueUrl: String,
121 |     entries:  Seq[(String, String)]
122 |   ): Future[DeleteMessageBatchResult] =
123 |     deleteMessageBatch(
124 |       new DeleteMessageBatchRequest(
125 |         queueUrl,
126 |         entries.map{ case (id, receiptHandle) =>
127 |           new DeleteMessageBatchRequestEntry(id, receiptHandle)
128 |         }.asJava
129 |       )
130 |     )
131 | 
132 |   def deleteQueue(
133 |     deleteQueueRequest: DeleteQueueRequest
134 |   ): Future[DeleteQueueResult] =
135 |     wrapAsyncMethod[DeleteQueueRequest, DeleteQueueResult](client.deleteQueueAsync, deleteQueueRequest)
136 | 
137 |   def deleteQueue(
138 |     queueUrl: String
139 |   ): Future[DeleteQueueResult] =
140 |     deleteQueue(new DeleteQueueRequest(queueUrl))
141 | 
142 |   def getQueueAttributes(
143 |     getQueueAttributesRequest: GetQueueAttributesRequest
144 |   ): Future[GetQueueAttributesResult] =
145 |     wrapAsyncMethod(client.getQueueAttributesAsync, getQueueAttributesRequest)
146 | 
147 |   def getQueueAttributes(
148 |     queueUrl: String,
149 |     attributeNames: Seq[String]
150 |   ): Future[Map[String, String]] =
151 |     getQueueAttributes(
152 |       new GetQueueAttributesRequest(queueUrl)
153 |         .withAttributeNames(attributeNames: _*)
154 |     ).map(_.getAttributes.asScala.toMap)
155 | 
156 |   def getQueueUrl(
157 |     getQueueUrlRequest: GetQueueUrlRequest
158 |   ): Future[GetQueueUrlResult] =
159 |     wrapAsyncMethod[GetQueueUrlRequest, GetQueueUrlResult](client.getQueueUrlAsync, getQueueUrlRequest)
160 | 
161 |   def getQueueUrl(
162 |     queueName: String
163 |   ): Future[String] =
164 |     getQueueUrl(new GetQueueUrlRequest(queueName)).map(_.getQueueUrl)
165 | 
166 |   def listQueues(
167 |     listQueuesRequest: ListQueuesRequest
168 |   ): Future[ListQueuesResult] =
169 |     wrapAsyncMethod[ListQueuesRequest, ListQueuesResult](client.listQueuesAsync, listQueuesRequest)
170 | 
171 |   def listQueues(
172 |     queueNamePrefix: String = null
173 |   ): Future[Seq[String]] =
174 |     listQueues(new ListQueuesRequest(queueNamePrefix)).map(_.getQueueUrls.asScala.toSeq)
175 | 
176 |   def receiveMessage(
177 |     receiveMessageRequest: ReceiveMessageRequest
178 |   ): Future[ReceiveMessageResult] =
179 |     wrapAsyncMethod[ReceiveMessageRequest, ReceiveMessageResult](client.receiveMessageAsync, receiveMessageRequest)
180 | 
181 |   def receiveMessage(
182 |     queueUrl: String
183 |   ): Future[Message] =
184 |     receiveMessage(new ReceiveMessageRequest(queueUrl)).map(_.getMessages.get(0))
185 | 
186 |   def receiveMessage(
187 |     queueUrl:            String,
188 |     maxNumberOfMessages: Int
189 |   ): Future[Seq[Message]] =
190 |     receiveMessage(
191 |       new ReceiveMessageRequest(queueUrl)
192 |       .withMaxNumberOfMessages(maxNumberOfMessages)
193 |     ).map(_.getMessages.asScala.toSeq)
194 | 
195 |   def removePermission(
196 |     removePermissionRequest: RemovePermissionRequest
197 |   ): Future[RemovePermissionResult] =
198 |     wrapAsyncMethod(client.removePermissionAsync, removePermissionRequest)
199 | 
200 |   def removePermission(
201 |     queueUrl: String,
202 |     label:    String
203 |   ): Future[RemovePermissionResult] =
204 |     removePermission(new RemovePermissionRequest(queueUrl, label))
205 | 
206 |   def sendMessage(
207 |     sendMessageRequest: SendMessageRequest
208 |   ): Future[SendMessageResult] =
209 |     wrapAsyncMethod(client.sendMessageAsync, sendMessageRequest)
210 | 
211 |   def sendMessage(
212 |     queueUrl:    String,
213 |     messageBody: String
214 |   ): Future[SendMessageResult] =
215 |     sendMessage(new SendMessageRequest(queueUrl, messageBody))
216 | 
217 |   def sendMessageBatch(
218 |     sendMessageBatchRequest: SendMessageBatchRequest
219 |   ): Future[SendMessageBatchResult] =
220 |     wrapAsyncMethod(client.sendMessageBatchAsync, sendMessageBatchRequest)
221 | 
222 |   def sendMessageBatch(
223 |     queueUrl: String,
224 |     entries:  Seq[(String, String)]
225 |   ): Future[SendMessageBatchResult] =
226 |     sendMessageBatch(
227 |       new SendMessageBatchRequest(
228 |         queueUrl,
229 |         entries.map{ case (id, messageBody) =>
230 |           new SendMessageBatchRequestEntry(id, messageBody)
231 |         }.asJava
232 |       )
233 |     )
234 | 
235 |   def setQueueAttributes(
236 |     setQueueAttributesRequest: SetQueueAttributesRequest
237 |   ): Future[SetQueueAttributesResult] =
238 |     wrapAsyncMethod(client.setQueueAttributesAsync, setQueueAttributesRequest)
239 | 
240 |   def setQueueAttributes(
241 |     queueUrl:   String,
242 |     attributes: Map[String, String]
243 |   ): Future[SetQueueAttributesResult] =
244 |     setQueueAttributes(new SetQueueAttributesRequest(queueUrl, attributes.asJava))
245 | 
246 |   def shutdown(): Unit =
247 |     client.shutdown()
248 | 
249 | }
250 | 


--------------------------------------------------------------------------------
/sqs/src/test/scala/SQSSpec.scala:
--------------------------------------------------------------------------------
  1 | package com.mfglabs.commons.aws
  2 | package sqs
  3 | 
  4 | import akka.actor._
  5 | import akka.stream._
  6 | import akka.stream.scaladsl._
  7 | import com.amazonaws.services.sqs.model.{CreateQueueRequest, MessageAttributeValue, SendMessageRequest}
  8 | import org.scalatest.concurrent.ScalaFutures
  9 | import org.scalatest.time.{Millis, Seconds, Span}
 10 | import org.scalatest.{FlatSpec, Matchers}
 11 | 
 12 | import scala.concurrent.duration._
 13 | import scala.util.Random
 14 | 
 15 | class SQSSpec extends FlatSpec with Matchers with ScalaFutures {
 16 |   import com.amazonaws.regions.Regions
 17 |   import scala.collection.JavaConverters._
 18 | 
 19 |   implicit override val patienceConfig =
 20 |     PatienceConfig(timeout = Span(60, Seconds), interval = Span(5, Millis))
 21 | 
 22 |   implicit val as = ActorSystem()
 23 |   implicit val fm = ActorMaterializer()
 24 | 
 25 |   val sqsClient  = AmazonSQSClient.from(Regions.EU_WEST_1)()
 26 | 
 27 |   val testQueueName = "commons-aws-sqs-test-" + Random.nextInt()
 28 |   val testQueueName2 = "commons-aws-sqs-test-" + Random.nextInt()
 29 | 
 30 |   val messageAttributeKey = "attribute_key"
 31 | 
 32 |   "SQS client" should "delete all test queue if any" in {
 33 |     sqsClient.listQueues("commons-aws-sqs-test-").futureValue.headOption.foreach(queueUrl => sqsClient.deleteQueue(queueUrl).futureValue)
 34 |   }
 35 | 
 36 |   it should "send message and receive them as streams" in {
 37 |     val newQueueReq = new CreateQueueRequest()
 38 |     newQueueReq.setAttributes(Map("VisibilityTimeout" -> 10.toString).asJava) // 10 seconds
 39 |     newQueueReq.setQueueName(testQueueName)
 40 |     val queueUrl = sqsClient.createQueue(newQueueReq).futureValue.getQueueUrl
 41 | 
 42 |     val msgs = for (i <- 1 to 200) yield s"Message $i"
 43 |     val futSent = Source(msgs)
 44 |       .map { body =>
 45 |       val req = new SendMessageRequest()
 46 |       req.addMessageAttributesEntry(messageAttributeKey, new MessageAttributeValue().withDataType("String").withStringValue(body))
 47 |       req.setMessageBody(body)
 48 |       req.setQueueUrl(queueUrl)
 49 |       req
 50 |     }
 51 |       .via(sqsClient.sendMessageAsStream())
 52 |       .take(200)
 53 |       .runWith(Sink.seq)
 54 |     val futReceived = sqsClient.receiveMessageAsStream(queueUrl, autoAck = true, messageAttributeNames = List(messageAttributeKey)).take(200).runWith(Sink.seq)
 55 | 
 56 |     val (_, received) = futSent.zip(futReceived).futureValue
 57 | 
 58 |     received.map(_.getBody).sorted shouldEqual msgs.sorted
 59 |     received.map(_.getMessageAttributes.get(messageAttributeKey).getStringValue).sorted shouldEqual msgs.sorted
 60 | 
 61 |     // testing auto-ack (queue must be empty)
 62 |     val res = sqsClient
 63 |       .receiveMessageAsStream(queueUrl, longPollingMaxWait = 1 second)
 64 |       .takeWithin(10 seconds)
 65 |       .runWith(Sink.seq)
 66 |       .futureValue
 67 |     res shouldBe empty
 68 | 
 69 |     sqsClient.deleteQueue(queueUrl).futureValue
 70 |   }
 71 | 
 72 | 
 73 |   it should "send message and receive them as streams with retry mechanism" in {
 74 |     val newQueueReq = new CreateQueueRequest()
 75 |     newQueueReq.setAttributes(Map("VisibilityTimeout" -> 10.toString).asJava) // 10 seconds
 76 |     newQueueReq.setQueueName(testQueueName2)
 77 |     val queueUrl = sqsClient.createQueue(newQueueReq).futureValue.getQueueUrl
 78 | 
 79 |     val msgs = for (i <- 1 to 200) yield s"Message $i"
 80 |     val futSent = Source(msgs)
 81 |       .map { body =>
 82 |         val req = new SendMessageRequest()
 83 |         req.setMessageBody(body)
 84 |         req.setQueueUrl(queueUrl)
 85 |         req
 86 |       }
 87 |       .via(sqsClient.sendMessageAsStream())
 88 |       .take(200)
 89 |       .runWith(Sink.seq)
 90 |     val futReceived = sqsClient.receiveMessageAsStreamWithRetryExpBackoff(queueUrl, autoAck = true).take(200).runWith(Sink.seq)
 91 | 
 92 |     val (_, received) = futSent.zip(futReceived).futureValue
 93 | 
 94 |     received.map(_.getBody).sorted shouldEqual msgs.sorted
 95 | 
 96 |     // testing auto-ack (queue must be empty)
 97 |     val res = sqsClient
 98 |       .receiveMessageAsStream(queueUrl, longPollingMaxWait = 1 second)
 99 |       .takeWithin(10 seconds)
100 |       .runWith(Sink.seq)
101 |       .futureValue
102 |     res shouldBe empty
103 | 
104 |     sqsClient.deleteQueue(queueUrl).futureValue
105 |   }
106 | 
107 | }
108 | 
109 | 


--------------------------------------------------------------------------------