├── project ├── version.properties ├── build.properties └── plugins.sbt ├── .gitignore ├── jitpack.yml ├── sonatype.sbt ├── .travis.yml ├── CHANGES.md ├── DEPLOY.md ├── src ├── main │ └── scala │ │ └── com │ │ └── timcharper │ │ └── acked │ │ ├── DummyPromise.scala │ │ ├── package.scala │ │ ├── FlowHelpers.scala │ │ ├── AckedSink.scala │ │ ├── AckedGraph.scala │ │ ├── AckedSubFlow.scala │ │ ├── AckedSource.scala │ │ ├── Components.scala │ │ └── AckedFlowOps.scala └── test │ └── scala │ └── com │ └── timcharper │ └── acked │ ├── Helpers.scala │ ├── AckedSinkSpec.scala │ ├── ComponentsSpec.scala │ └── AckedSourceSpec.scala ├── LICENSE └── README.md /project/version.properties: -------------------------------------------------------------------------------- 1 | version=2.1.1 -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .ensime 2 | .ensime_cache 3 | target -------------------------------------------------------------------------------- /jitpack.yml: -------------------------------------------------------------------------------- 1 | install: 2 | - sbt +publishM2 3 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.3.12 2 | -------------------------------------------------------------------------------- /sonatype.sbt: -------------------------------------------------------------------------------- 1 | sonatypeProfileName := "com.timcharper" 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.2") 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | scala: 3 | - 2.13.2 4 | jdk: 5 | - oraclejdk11 6 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Notable changes: 2 | 3 | ## 2.0 4 | 5 | - AckedSink.fold / AckedSource.runFold were removed. They are too awkward. You may use AckedFlowOps.fold with AckedSink.head, if you wish. The behavior of the combination of those elements is more clear. 6 | 7 | - Naturally, API compatibility was brought into alignment with Akka Streams. 8 | -------------------------------------------------------------------------------- /DEPLOY.md: -------------------------------------------------------------------------------- 1 | - bump version in `version.sbt` 2 | - `sbt +test` 3 | - `sbt +publishSigned` 4 | - `sbt sonatypeRelease` 5 | 6 | - commit tag and push 7 | 8 | ``` 9 | source project/version.properties 10 | git add . 11 | git commit -m "release $version" 12 | git tag v$version 13 | git push origin v$version 14 | 15 | 16 | Temporarily set version: 17 | 18 | `++2.10.5` 19 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/DummyPromise.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import scala.concurrent.Future 4 | import scala.concurrent.Promise 5 | 6 | private [acked] object DummyPromise extends Promise[Unit] { 7 | import scala.util.Try 8 | def future = Future.successful(()) 9 | def isCompleted = true 10 | def tryComplete(result: Try[Unit]): Boolean = 11 | true 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/package.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper 2 | 3 | import scala.concurrent.ExecutionContext 4 | 5 | package object acked { 6 | import scala.concurrent.Promise 7 | 8 | type AckTup[+T] = (Promise[Unit], T) 9 | 10 | // WARNING!!! Don't block inside of Runnable (Future) that uses this. 11 | private[acked] object SameThreadExecutionContext extends ExecutionContext { 12 | def execute(r: Runnable): Unit = 13 | r.run() 14 | 15 | override def reportFailure(t: Throwable): Unit = 16 | throw new IllegalStateException("problem in op_rabbit internal callback", t) 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is licensed under the Apache 2 license, quoted below. 2 | 3 | Copyright 2015 Tim Harper. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); you may not 6 | use this file except in compliance with the License. You may obtain a copy of 7 | the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, WITHOUT 13 | WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 14 | License for the specific language governing permissions and limitations under 15 | the License. -------------------------------------------------------------------------------- /src/test/scala/com/timcharper/acked/Helpers.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.actor.ActorSystem 4 | import org.scalatest.{BeforeAndAfterEach, TestData, Suite} 5 | import scala.concurrent.{Future, Await} 6 | import scala.concurrent.duration._ 7 | 8 | trait ActorSystemTest extends Suite with org.scalatest.BeforeAndAfterEachTestData { 9 | implicit var actorSystem: ActorSystem = null 10 | override def beforeEach(testData: TestData): Unit = { 11 | actorSystem = ActorSystem("testing") 12 | super.beforeEach(testData) 13 | } 14 | 15 | override def afterEach(testData: TestData): Unit = { 16 | actorSystem.terminate() 17 | super.afterEach(testData) 18 | } 19 | 20 | def await[T](f: Future[T], duration: FiniteDuration = 5.seconds) = { 21 | Await.result(f, duration) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/FlowHelpers.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import scala.concurrent.{Promise, Future} 4 | 5 | object FlowHelpers { 6 | // propagate exception, doesn't recover 7 | def propFutureException[T](p: Promise[Unit])(f: => Future[T]): Future[T] = { 8 | implicit val ec = SameThreadExecutionContext 9 | val result = propException(p)(f) 10 | result.failed 11 | .foreach { case e => p.failure(e) } 12 | result 13 | } 14 | 15 | // Catch and propagate exception; exception is still thrown 16 | // TODO - rather than catching the exception, wrap it, with the promise, and wrap the provided handler. If the handler is invoked, then nack the message with the exception. This way, .recover can be supported. 17 | def propException[T](p: Promise[Unit])(t: => T): T = { 18 | try { 19 | t 20 | } catch { 21 | case e: Throwable => 22 | p.failure(e) 23 | throw (e) 24 | } 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/test/scala/com/timcharper/acked/AckedSinkSpec.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.pattern.ask 4 | import akka.stream.ActorMaterializer 5 | import org.scalatest.{FunSpec, Matchers} 6 | import scala.concurrent.{Future, Promise} 7 | import scala.util.{Failure, Success, Try} 8 | 9 | class AckedSinkSpec extends FunSpec with Matchers with ActorSystemTest { 10 | 11 | describe("head") { 12 | it("acknowledges only the first element") { 13 | case class LeException(msg: String) extends Exception(msg) 14 | val input = (Stream.continually(Promise[Unit]) zip Range.inclusive(1, 5)).toList 15 | implicit val materializer = ActorMaterializer() 16 | Try(await(AckedSource(input).runWith(AckedSink.head))) 17 | input.map { case (p, _) => 18 | p.tryFailure(LeException("didn't complete")) 19 | Try(await(p.future)) match { 20 | case Success(_) => None 21 | case Failure(LeException(msg)) => Some(msg) 22 | case Failure(e) => throw e 23 | } 24 | } should be (Seq(None, Some("didn't complete"), Some("didn't complete"), Some("didn't complete"), Some("didn't complete"))) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/AckedSink.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.Done 4 | import akka.actor._ 5 | import akka.stream.Attributes 6 | import akka.stream.Graph 7 | import akka.stream.SinkShape 8 | import akka.stream.scaladsl.{Flow, Keep, Sink} 9 | 10 | import scala.annotation.unchecked.uncheckedVariance 11 | import scala.concurrent.{ExecutionContext, Future} 12 | import scala.concurrent.duration._ 13 | 14 | // Simply a container class which signals "this is safe to use for acknowledgement" 15 | case class AckedSink[-In, +Mat](akkaSink: Graph[SinkShape[AckTup[In]], Mat]) extends AckedGraph[AckedSinkShape[In], Mat] { 16 | val shape = new AckedSinkShape(akkaSink.shape) // lazy val shape = new AckedSinkShape(akkaSink.shape) 17 | val akkaGraph = akkaSink 18 | 19 | override def withAttributes(attr: Attributes): AckedSink[In, Mat] = 20 | AckedSink(akkaGraph.withAttributes(attr)) 21 | 22 | override def addAttributes(attr: Attributes): AckedSink[In, Mat] = 23 | AckedSink(akkaGraph.addAttributes(attr)) 24 | } 25 | 26 | case object MessageNacked extends Exception(s"A published message was nacked by the broker.") 27 | 28 | object AckedSink { 29 | import FlowHelpers.propException 30 | def foreach[T](fn: T => Unit) = AckedSink[T, Future[Done]] { 31 | Sink.foreach { case (p, data) => 32 | propException(p) { fn(data) } 33 | p.success(()) 34 | } 35 | } 36 | 37 | def head[T] = AckedSink[T, Future[T]] { 38 | implicit val ec = SameThreadExecutionContext 39 | val s = Sink.head[AckTup[T]] 40 | s.mapMaterializedValue { 41 | _.map{ case (p, out) => 42 | p.success(()) 43 | out 44 | } 45 | } 46 | } 47 | 48 | def ack[T] = AckedSink[T, Future[Done]] { 49 | Sink.foreach { case (p, data) => 50 | p.success(()) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/AckedGraph.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.stream._ 4 | import scala.annotation.unchecked.uncheckedVariance 5 | 6 | trait AckedShape { self => 7 | type Self <: AckedShape 8 | type AkkaShape <: akka.stream.Shape 9 | val akkaShape: AkkaShape 10 | def wrapShape(akkaShape: AkkaShape): Self 11 | } 12 | 13 | trait AckedGraph[+S <: AckedShape, +M] { 14 | type Shape = S @uncheckedVariance 15 | protected [acked] val shape: Shape 16 | type AkkaShape = shape.AkkaShape 17 | val akkaGraph: Graph[AkkaShape, M] 18 | def wrapShape(akkaShape: akkaGraph.Shape): shape.Self = 19 | shape.wrapShape(akkaShape) 20 | 21 | def withAttributes(attr: Attributes): AckedGraph[S, M] 22 | 23 | def named(name: String): AckedGraph[S, M] = withAttributes(Attributes.name(name)) 24 | 25 | def addAttributes(attr: Attributes): AckedGraph[S, M] 26 | } 27 | 28 | final class AckedSourceShape[+T](s: SourceShape[AckTup[T]]) extends AckedShape { 29 | type Self = AckedSourceShape[T] @uncheckedVariance 30 | type AkkaShape = SourceShape[AckTup[T]] @uncheckedVariance 31 | val akkaShape = s 32 | def wrapShape(akkaShape: AkkaShape @uncheckedVariance): Self = 33 | new AckedSourceShape(akkaShape) 34 | } 35 | 36 | final class AckedSinkShape[-T](s: SinkShape[AckTup[T]]) extends AckedShape { 37 | type Self = AckedSinkShape[T] @uncheckedVariance 38 | type AkkaShape = SinkShape[AckTup[T]] @uncheckedVariance 39 | val akkaShape = s 40 | def wrapShape(akkaShape: AkkaShape @uncheckedVariance): Self = 41 | new AckedSinkShape(akkaShape) 42 | } 43 | 44 | class AckedFlowShape[-I, +O](s: FlowShape[AckTup[I], AckTup[O]]) extends AckedShape { 45 | type Self = AckedFlowShape[I, O] @uncheckedVariance 46 | type AkkaShape = FlowShape[AckTup[I], AckTup[O]] @uncheckedVariance 47 | val akkaShape = s 48 | def wrapShape(akkaShape: AkkaShape @uncheckedVariance): Self = 49 | new AckedFlowShape(akkaShape) 50 | } 51 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/AckedSubFlow.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.stream.scaladsl.FlowOps 4 | import akka.stream.scaladsl.SubFlow 5 | import language.higherKinds 6 | import scala.annotation.unchecked.uncheckedVariance 7 | 8 | trait AckedSubFlow[+Out, +Mat, +F[+_]] extends AckedFlowOps[Out, Mat] { 9 | 10 | override type Repr[+T] = AckedSubFlow[T, Mat @uncheckedVariance, F @uncheckedVariance] 11 | // override type Closed = C 12 | 13 | // /** 14 | // * Attach a [[Sink]] to each sub-flow, closing the overall Graph that is being 15 | // * constructed. 16 | // */ 17 | // def to[M](sink: AckedGraph[AckedSinkShape[Out], M]): C 18 | 19 | /** 20 | * Flatten the sub-flows back into the super-flow by performing a merge 21 | * without parallelism limit (i.e. having an unbounded number of sub-flows 22 | * active concurrently). 23 | * 24 | * This is identical in effect to `mergeSubstreamsWithParallelism(Integer.MAX_VALUE)`. 25 | */ 26 | def mergeSubstreams: F[Out] = mergeSubstreamsWithParallelism(Int.MaxValue) 27 | 28 | /** 29 | * Flatten the sub-flows back into the super-flow by performing a merge 30 | * with the given parallelism limit. This means that only up to `parallelism` 31 | * substreams will be executed at any given time. Substreams that are not 32 | * yet executed are also not materialized, meaning that back-pressure will 33 | * be exerted at the operator that creates the substreams when the parallelism 34 | * limit is reached. 35 | */ 36 | def mergeSubstreamsWithParallelism(parallelism: Int): F[Out] 37 | 38 | /** 39 | * Flatten the sub-flows back into the super-flow by concatenating them. 40 | * This is usually a bad idea when combined with `groupBy` since it can 41 | * easily lead to deadlock—the concatenation does not consume from the second 42 | * substream until the first has finished and the `groupBy` stage will get 43 | * back-pressure from the second stream. 44 | * 45 | * This is identical in effect to `mergeSubstreamsWithParallelism(1)`. 46 | */ 47 | def concatSubstreams: F[Out] = mergeSubstreamsWithParallelism(1) 48 | } 49 | 50 | object AckedSubFlow { 51 | trait Converter[G[+_], F[+_]] { 52 | def apply[O](o: G[AckTup[O]]): F[O] 53 | } 54 | 55 | class Impl[+Out, +Mat, +F[+_], +G[+_]]( 56 | val wrappedRepr: SubFlow[AckTup[Out], Mat, G, _], 57 | rewrap: Converter[G, F] 58 | ) extends AckedSubFlow[Out, Mat, F] { 59 | 60 | type UnwrappedRepr[+O] <: SubFlow[O, Mat, G, _] 61 | type WrappedRepr[+O] = SubFlow[AckTup[O], Mat @uncheckedVariance, G @uncheckedVariance, _] 62 | 63 | protected def andThen[U](next: WrappedRepr[U] @uncheckedVariance): Repr[U] = { 64 | new Impl(next, rewrap) 65 | } 66 | 67 | def mergeSubstreamsWithParallelism(parallelism: Int): F[Out] = 68 | rewrap(wrappedRepr.mergeSubstreamsWithParallelism(parallelism)) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/test/scala/com/timcharper/acked/ComponentsSpec.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import org.scalatest.{FunSpec, Matchers} 4 | 5 | import akka.stream.Attributes 6 | import akka.stream.OverflowStrategy 7 | import akka.stream.scaladsl.Keep 8 | import akka.stream.scaladsl.Source 9 | 10 | import scala.concurrent.Promise 11 | import scala.collection.mutable._ 12 | import scala.util.{Try, Success, Failure} 13 | import scala.concurrent.duration._ 14 | 15 | class ComponentsSpec extends FunSpec with Matchers with ActorSystemTest { 16 | trait Fixtures { 17 | implicit val materializer = akka.stream.ActorMaterializer() 18 | 19 | val data: List[(Promise[Unit], Int)] = 20 | Stream 21 | .continually(Promise[Unit]()) 22 | .zip(Stream.continually(1 to 50).take(10).flatten) 23 | .toList 24 | } 25 | 26 | describe("BundlingBuffer") { 27 | it("bundles together items when back pressured") { 28 | new Fixtures { 29 | val seen: Set[Int] = Set.empty 30 | 31 | val sink = AckedFlow[Int] 32 | .fold(0) { (cnt, x) => 33 | Thread.sleep(10 + x % 10) 34 | seen += x 35 | cnt + 1 36 | } 37 | .toMat(AckedSink.head)(Keep.right) 38 | .withAttributes(Attributes.asyncBoundary) 39 | 40 | val f = 41 | AckedSource(data) 42 | .via(Components.bundlingBuffer(500, OverflowStrategy.fail)) 43 | .runWith(sink) 44 | 45 | val count = await(f, 20.seconds) 46 | 47 | // 500 elements went into it. Significantly less should have made it through. 48 | count should be < 100 49 | 50 | // Every unique item should have made it through at least once. 51 | seen.toList.sorted should be(1 to 50) 52 | 53 | // Every promise should be acknowledged 54 | for ((p, i) <- data.map(_._1).zipWithIndex) { 55 | (p.future.isCompleted, i) shouldBe (true, i) 56 | } 57 | } 58 | } 59 | 60 | it("doesn't bundle when items aren't backpressured") { 61 | new Fixtures { 62 | val f = AckedSource(data) 63 | .via(Components.bundlingBuffer(500, OverflowStrategy.fail)) 64 | .fold(0) { (cnt, x) => 65 | cnt + 1 66 | } 67 | . // By not making the sink async (default with 2.0.1), we guarantee no backpressure will happen 68 | runWith(AckedSink.head) 69 | 70 | val count = await(f) 71 | 72 | for ((p, i) <- data.map(_._1).zipWithIndex) { 73 | p.future.isCompleted shouldBe true 74 | } 75 | count shouldBe 500 76 | 77 | } 78 | } 79 | 80 | it("drops new elements when buffer is overrun, failing the promises") { 81 | new Fixtures { 82 | var seen = Stack.empty[Int] 83 | val f = AckedSource(data) 84 | .via(Components.bundlingBuffer(10, OverflowStrategy.dropHead)) 85 | .runWith( 86 | AckedSink 87 | .foreach[Int] { n => 88 | Thread.sleep(10) 89 | seen.push(n) 90 | } 91 | .withAttributes(Attributes.asyncBoundary) 92 | ) 93 | 94 | await(f) 95 | 96 | seen.length should be < 100 97 | 98 | val results = for (p <- data.map(_._1)) yield { 99 | Try(await(p.future)) 100 | } 101 | results.filter(_.isInstanceOf[Success[_]]).length shouldBe seen.length 102 | } 103 | } 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Acknowledged Streams 2 | 3 | Author: Tim Harper 4 | 5 | 6 | ## Versions 7 | 8 | Please use the following table to guide you with Akka comptability: 9 | 10 | | acked-streams version | akka version | Scala | 11 | | --------------------- | ------------------------------ | -------------- | 12 | | 2.0.x | akka-stream-experimental 2.0.x | 2.10.x | 13 | | 2.1.x | akka-stream 2.4.x | 2.11.x, 2.12.x | 14 | 15 | Note that `2.1.x` and onward do not support Scala `2.10`, as Akka `2.4.x` does 16 | not support it, either. 17 | 18 | ## Installation 19 | 20 | Acknowledged Streams builds against akka-stream-experimental `1.0`. 21 | 22 | Add the following to `build.sbt`: 23 | 24 | libraryDependencies += "com.timcharper" %% "acked-streams" % "1.0-RC1" 25 | 26 | ## Motivation 27 | 28 | TL; DR - http://tim.theenchanter.com/2015/07/the-need-for-acknowledgement-in-streams.html 29 | 30 | Acknowledged Streams builds on Akka streams and provides the mechansim to receive signal when a message has completed it's flight through a stream pipeline (was filtered, or processed by some sink). By so doing, it provides the underlying component necessary for stream persistence. It safely supports operations that modify element cardinality, such as `grouped`, `groupedWithin`, `mapConcat`, etc. It's heavily tested to ensure that it is impossible for an element to complete its flight through the stream and not be acknowledged. 31 | 32 | The first version uses Promise objects to signal acknowledgement upstream. While there are situations in which this is not ideal (IE: acknolwedgement does not backpressure), it is incredibly simple to implement, and Akka streams presently lacks the extensibility to implement acknowledgment as generally as it is implemented in this library. 33 | 34 | ## Usage 35 | 36 | You can construct an acknowledged source by providing it an Iterable or a Source of `(Promise[Unit], T)`. The `Promise[Unit]` must not be acknowledged. Any exceptions in the stream will be propagated to this promise, regardless of the stream Supervision Decider. On acknowledgement, the Promise is completed. 37 | 38 | Examples: 39 | 40 | val data = Stream.continually(Promise[Unit]) zip Range(1, Math.max(40, (Random.nextInt(200)))) 41 | AckedSource(data). 42 | map(...). 43 | ... etc 44 | 45 | See the [demo project](https://github.com/timcharper/acked-stream-demo), or the [acked-stream tests](https://github.com/timcharper/acked-stream/tree/master/src/test/scala/com/timcharper/acked). 46 | 47 | ## Supported operations 48 | 49 | The API mirrors the Akka Stream API where it is possible to positively correlate a stream element with an input element. AckedFlow and AckedSink are implemented and behave accordingly (where an AckedSink is responsible for message acknowledgmeent). 50 | 51 | Notes on operations: 52 | 53 | - `filter`: If a element is filtered, it is acknowledged. 54 | - `collect`: If a element is filtered, it is acknowledged. 55 | - `mapConcat`: If the output is 0 elements, the element is acknowledged. If the output is >= 2 elements, the original element is acknowledged after all resulting elements are acknowledged. 56 | - `groupedWithin`, `grouped`: If `n` elements go into the a single group, then all `n` elements are acknowledged after the grouped element is acknowledged. 57 | - `unsafe`: If you wish to have manual control over acknowledgement, call `unsafe` to get a Source[(Promise[Unit], T)]. Note, as the method name suggests, and if you forget to acknowledge a Promise, then it will be forever unacknowledged. 58 | 59 | ## Future plans 60 | 61 | Rather than using `Promise` to signal acknowledgement, convert to use a series of `BidiFlow`. 62 | 63 | 64 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/AckedSource.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.{Done, NotUsed} 4 | import akka.pattern.pipe 5 | import akka.stream.Attributes 6 | import akka.stream.{Graph, Materializer} 7 | import akka.stream.scaladsl.{Keep, RunnableGraph, Source} 8 | 9 | import scala.annotation.tailrec 10 | import scala.annotation.unchecked.uncheckedVariance 11 | import scala.concurrent.{Future, Promise} 12 | import scala.language.implicitConversions 13 | 14 | class AckedSource[+Out, +Mat](val wrappedRepr: Source[AckTup[Out], Mat]) extends AckedFlowOpsMat[Out, Mat] with AckedGraph[AckedSourceShape[Out], Mat] { 15 | type UnwrappedRepr[+O] = Source[O, Mat @uncheckedVariance] 16 | type WrappedRepr[+O] = Source[AckTup[O], Mat @uncheckedVariance] 17 | type Repr[+O] = AckedSource[O, Mat @uncheckedVariance] 18 | 19 | type UnwrappedReprMat[+O, +M] = Source[O, M] 20 | type WrappedReprMat[+O, +M] = Source[AckTup[O], M] 21 | type ReprMat[+O, +M] = AckedSource[O, M] 22 | 23 | lazy val shape = new AckedSourceShape(wrappedRepr.shape) 24 | val akkaGraph = wrappedRepr 25 | /** 26 | * Connect this [[akka.stream.scaladsl.Source]] to a [[akka.stream.scaladsl.Sink]], 27 | * concatenating the processing steps of both. 28 | */ 29 | def runAckMat[Mat2](combine: (Mat, Future[Done]) ⇒ Mat2)(implicit materializer: Materializer): Mat2 = 30 | wrappedRepr.toMat(AckedSink.ack.akkaSink)(combine).run 31 | 32 | def runAck(implicit materializer: Materializer) = runAckMat(Keep.right) 33 | 34 | def runWith[Mat2](sink: AckedSink[Out, Mat2])(implicit materializer: Materializer): Mat2 = 35 | wrappedRepr.runWith(sink.akkaSink) 36 | 37 | def runForeach(f: (Out) ⇒ Unit)(implicit materializer: Materializer): Future[Done] = 38 | runWith(AckedSink.foreach(f)) 39 | 40 | def to[Mat2](sink: AckedSink[Out, Mat2]): RunnableGraph[Mat] = 41 | wrappedRepr.to(sink.akkaSink) 42 | 43 | def toMat[Mat2, Mat3](sink: AckedSink[Out, Mat2])(combine: (Mat, Mat2) ⇒ Mat3): RunnableGraph[Mat3] = 44 | wrappedRepr.toMat(sink.akkaSink)(combine) 45 | 46 | /** 47 | See Source.via in akka-stream 48 | */ 49 | def via[T, Mat2](flow: AckedGraph[AckedFlowShape[Out, T], Mat2]): AckedSource[T, Mat] = 50 | andThen(wrappedRepr.via(flow.akkaGraph)) 51 | 52 | /** 53 | See Source.viaMat in akka-stream 54 | */ 55 | def viaMat[T, Mat2, Mat3](flow: AckedGraph[AckedFlowShape[Out, T], Mat2])(combine: (Mat, Mat2) ⇒ Mat3): AckedSource[T, Mat3] = 56 | andThenMat(wrappedRepr.viaMat(flow.akkaGraph)(combine)) 57 | 58 | /** 59 | Transform the materialized value of this AckedSource, leaving all other properties as they were. 60 | */ 61 | def mapMaterializedValue[Mat2](f: (Mat) ⇒ Mat2): ReprMat[Out, Mat2] = 62 | andThenMat(wrappedRepr.mapMaterializedValue(f)) 63 | 64 | protected def andThen[U](next: WrappedRepr[U] @uncheckedVariance): Repr[U] = { 65 | new AckedSource(next) 66 | } 67 | 68 | protected def andThenMat[U, Mat2](next: WrappedReprMat[U, Mat2]): ReprMat[U, Mat2] = { 69 | new AckedSource(next) 70 | } 71 | 72 | override def withAttributes(attr: Attributes): Repr[Out] = andThen { 73 | wrappedRepr.withAttributes(attr) 74 | } 75 | 76 | override def addAttributes(attr: Attributes): Repr[Out] = andThen { 77 | wrappedRepr.addAttributes(attr) 78 | } 79 | } 80 | 81 | object AckedSource { 82 | type OUTPUT[T] = AckTup[T] 83 | 84 | def apply[T](magnet: AckedSourceMagnet) = magnet.apply 85 | } 86 | 87 | trait AckedSourceMagnet { 88 | type Out 89 | def apply: Out 90 | } 91 | object AckedSourceMagnet extends LowerPriorityAckedSourceMagnet { 92 | implicit def fromPromiseIterable[T](iterable: scala.collection.immutable.Iterable[AckTup[T]]) = new AckedSourceMagnet { 93 | type Out = AckedSource[T, NotUsed] 94 | def apply = new AckedSource(Source(iterable)) 95 | } 96 | 97 | implicit def fromPromiseSource[T, M](source: Source[AckTup[T], M]) = new AckedSourceMagnet { 98 | type Out = AckedSource[T, M] 99 | def apply = new AckedSource(source) 100 | } 101 | } 102 | 103 | private[acked] abstract class LowerPriorityAckedSourceMagnet { 104 | implicit def fromIterable[T](iterable: scala.collection.immutable.Iterable[T]) = new AckedSourceMagnet { 105 | type Out = AckedSource[T, NotUsed] 106 | def apply = new AckedSource(Source(Stream.continually(Promise[Unit]()) zip iterable)) 107 | } 108 | 109 | implicit def fromSource[T, M](source: Source[T, M]) = new AckedSourceMagnet { 110 | type Out = AckedSource[T, M] 111 | def apply = new AckedSource(source.map(d => (Promise[Unit](), d))) 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/Components.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.NotUsed 4 | import akka.stream._ 5 | import akka.stream.scaladsl._ 6 | import akka.stream.stage._ 7 | 8 | import scala.concurrent._ 9 | import scala.collection.mutable.{Buffer, LinkedHashMap} 10 | 11 | object Components { 12 | 13 | /** 14 | Request bundling buffer. 15 | 16 | Borrowed heavily from Akka-stream 2.0-M1 implementation. Works like a normal 17 | buffer; however, duplicate items in the buffer get bundled, rather than 18 | queued; when the item into which the duplicate item was bundled gets acked, 19 | the duplicate item (and all other cohort bundled items) are acked. 20 | 21 | FIFO, except when duplicate items are bundled into items later in the queue. 22 | 23 | In order for bundling to work, items MUST be comparable by value (IE case 24 | classes) and MUST be immutable (IE case classes that don't use var). 25 | Ultimately, the input item is used as a key in a hashmap. 26 | 27 | @param size The size of the buffer. Bundled items do not count against the 28 | size. 29 | @param overflowStrategy How should we handle buffer overflow? Note: items 30 | are failed with DroppedException. 31 | 32 | @return An AckedFlow which runs the bundling buffer component. 33 | **/ 34 | def bundlingBuffer[T]( 35 | size: Int, 36 | overflowStrategy: OverflowStrategy 37 | ): AckedFlow[T, T, NotUsed] = 38 | AckedFlow { 39 | Flow[(Promise[Unit], T)].via(BundlingBuffer(size, overflowStrategy)) 40 | } 41 | 42 | sealed abstract class BundlingBufferException(msg: String) 43 | extends RuntimeException(msg) 44 | case class BufferOverflowException(msg: String) 45 | extends BundlingBufferException(msg) 46 | case class DroppedException(msg: String) extends BundlingBufferException(msg) 47 | 48 | /* we have to pull these out again and make the capitals for 49 | * pattern matching. Akka is the ultimate hider of useful 50 | * types. */ 51 | private val DropHead = OverflowStrategy.dropHead 52 | private val DropTail = OverflowStrategy.dropTail 53 | private val DropBuffer = OverflowStrategy.dropBuffer 54 | private val DropNew = OverflowStrategy.dropNew 55 | private val Backpressure = OverflowStrategy.backpressure 56 | private val Fail = OverflowStrategy.fail 57 | 58 | case class BundlingBuffer[U](size: Int, overflowStrategy: OverflowStrategy) 59 | extends GraphStage[FlowShape[(Promise[Unit], U), (Promise[Unit], U)]] { 60 | type T = (Promise[Unit], U) 61 | 62 | val in = Inlet[T]("BundlingBuffer.in") 63 | val out = Outlet[T]("BundlingBuffer.out") 64 | 65 | override def shape: FlowShape[T, T] = FlowShape.of(in, out) 66 | override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = 67 | new GraphStageLogic(shape) { 68 | private val promises: LinkedHashMap[U, Promise[Unit]] = 69 | LinkedHashMap.empty 70 | private val buffer: Buffer[U] = Buffer.empty 71 | private def bufferIsFull: Boolean = buffer.length >= size 72 | 73 | private var isHoldingUpstream = false 74 | private var isHoldingDownstream = false 75 | 76 | private def dequeue(): T = { 77 | val v = buffer.remove(0) 78 | (promises.remove(v).get, v) 79 | } 80 | 81 | private def enqueue(v: T): Unit = { 82 | promises.get(v._2) match { 83 | case Some(p) => 84 | v._1.completeWith(p.future) 85 | case None => 86 | promises(v._2) = v._1 87 | buffer.append(v._2) 88 | } 89 | } 90 | 91 | private def dropped(values: U*): Unit = 92 | values.foreach { i => 93 | promises 94 | .remove(i) 95 | .map( 96 | _.tryFailure( 97 | DroppedException( 98 | s"message was dropped due to buffer overflow; size = $size" 99 | ) 100 | ) 101 | ) 102 | } 103 | 104 | private def emitAll(): Unit = { 105 | val vs = buffer.toSeq.map { v => 106 | (promises.remove(v).get, v) 107 | } 108 | if (vs.nonEmpty) emitMultiple(out, vs.iterator) 109 | buffer.clear() 110 | } 111 | 112 | private def grabAndPull() = { 113 | if (isAvailable(in)) enqueue(grab(in)) 114 | if (!hasBeenPulled(in)) pull(in) 115 | } 116 | 117 | private val inHandler: InHandler = 118 | new InHandler { 119 | override def onUpstreamFinish(): Unit = { 120 | emitAll() 121 | completeStage() 122 | } 123 | 124 | override def onPush(): Unit = { 125 | if (bufferIsFull) { 126 | overflowStrategy match { 127 | case DropHead => 128 | dropped(buffer.remove(0)) 129 | grabAndPull() 130 | case DropTail => 131 | dropped(buffer.remove(buffer.length - 1)) 132 | grabAndPull() 133 | case DropBuffer => 134 | dropped(buffer.toSeq: _*) 135 | buffer.clear() 136 | grabAndPull() 137 | case DropNew => 138 | grab(in)._1.tryFailure( 139 | DroppedException( 140 | s"message was dropped due to buffer overflow; size = $size" 141 | ) 142 | ) 143 | if (!hasBeenPulled(in)) pull(in) 144 | case Fail => 145 | dropped(buffer.toSeq: _*) 146 | buffer.clear() 147 | failStage( 148 | new BufferOverflowException( 149 | s"Buffer overflow (max capacity was: $size)!" 150 | ) 151 | ) 152 | case Backpressure => 153 | isHoldingUpstream = true 154 | } 155 | } else grabAndPull() 156 | if (isHoldingDownstream && isAvailable(out) && buffer.nonEmpty) { 157 | push(out, dequeue()) 158 | isHoldingUpstream = false 159 | isHoldingDownstream = false 160 | } 161 | } 162 | 163 | } 164 | 165 | private val outHandler: OutHandler = 166 | new OutHandler { 167 | override def onPull(): Unit = { 168 | if (isClosed(in)) completeStage() 169 | else if (buffer.isEmpty) isHoldingDownstream = true 170 | else { 171 | push(out, dequeue()) 172 | if (isHoldingUpstream) { 173 | isHoldingUpstream = false 174 | pull(in) 175 | } 176 | } 177 | } 178 | } 179 | 180 | override def preStart(): Unit = pull(in) 181 | 182 | setHandler(in, inHandler) 183 | setHandler(out, outHandler) 184 | } 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/test/scala/com/timcharper/acked/AckedSourceSpec.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.NotUsed 4 | import akka.actor._ 5 | import akka.pattern.ask 6 | import akka.stream.ActorMaterializer 7 | import akka.stream.ActorMaterializerSettings 8 | import akka.stream.Materializer 9 | import akka.stream.Supervision 10 | import akka.stream.scaladsl.Keep 11 | import akka.stream.scaladsl.{Sink, Source} 12 | import org.scalatest.{FunSpec, Matchers} 13 | 14 | import scala.concurrent.{ExecutionContext, Future, Promise} 15 | import scala.concurrent.duration._ 16 | import scala.util.Random 17 | import scala.util.{Failure, Success, Try} 18 | 19 | class AckedSourceSpec extends FunSpec with Matchers with ActorSystemTest { 20 | describe("AckedSource operations") { 21 | def runLeTest[T, U](input: scala.collection.immutable.Iterable[T] = 22 | Range(1, 20))( 23 | fn: AckedSource[T, NotUsed] => Future[U] 24 | )(implicit materializer: Materializer) = { 25 | val withPromise = (Stream.continually(Promise[Unit]) zip input).toList 26 | val promises = withPromise.map(_._1) 27 | implicit val ec = ExecutionContext.Implicits.global 28 | 29 | val results = (promises zip Range(0, Int.MaxValue)).map { 30 | case (p, i) => 31 | p.future 32 | .map { r => 33 | Success(r) 34 | } 35 | .recover { case e => Failure(e) } 36 | } 37 | 38 | val returnValue = await(fn(new AckedSource(Source(withPromise)))) 39 | (results.map { f => 40 | Try { await(f, duration = 100.milliseconds) }.toOption 41 | }, returnValue) 42 | } 43 | 44 | def asOptBool(s: Seq[Option[Try[Unit]]]) = 45 | s.map { 46 | case Some(Success(_)) => Some(true); 47 | case Some(Failure(_)) => Some(false); case _ => None 48 | } 49 | 50 | def assertAcked(completions: Seq[Option[Try[Unit]]]) = 51 | asOptBool(completions) should be( 52 | List.fill(completions.length)(Some(true)) 53 | ) 54 | 55 | def assertOperationCatches( 56 | fn: (Throwable, AckedSource[Int, NotUsed]) => AckedSource[_, NotUsed] 57 | ) = { 58 | case object LeException extends Exception("le fail") 59 | implicit val materializer = ActorMaterializer( 60 | ActorMaterializerSettings(actorSystem).withSupervisionStrategy( 61 | Supervision.resumingDecider: Supervision.Decider 62 | ) 63 | ) 64 | val (completions, result) = runLeTest(Range.inclusive(1, 20)) { s => 65 | fn(LeException, s).runAck 66 | } 67 | completions should be( 68 | List.fill(completions.length)(Some(Failure(LeException))) 69 | ) 70 | } 71 | 72 | describe("filter") { 73 | it("acks the promises that fail the filter") { 74 | implicit val materializer = ActorMaterializer() 75 | val (completions, result) = runLeTest(Range.inclusive(1, 20)) { 76 | _.filter { n => 77 | n % 2 == 0 78 | }.acked.runWith(Sink.fold(0)(_ + _)) 79 | } 80 | result should be(110) 81 | implicit val ec = SameThreadExecutionContext 82 | assertAcked(completions) 83 | } 84 | 85 | it("catches exceptions and propagates them to the promise") { 86 | assertOperationCatches { (e, source) => 87 | source.filter { n => 88 | throw e 89 | } 90 | } 91 | } 92 | } 93 | 94 | describe("map") { 95 | it("catches exceptions and propagates them to the promise") { 96 | assertOperationCatches { (e, source) => 97 | source.map { n => 98 | throw e 99 | } 100 | } 101 | } 102 | } 103 | 104 | describe("grouped") { 105 | it("acks all messages when the group is acked") { 106 | implicit val materializer = ActorMaterializer() 107 | val (completions, result) = runLeTest(Range.inclusive(1, 20)) { 108 | _.grouped(20).acked.runWith(Sink.fold(0) { (a, b) => 109 | a + 1 110 | }) 111 | } 112 | result should be(1) 113 | assertAcked(completions) 114 | } 115 | 116 | it("rejects all messages when the group fails") { 117 | assertOperationCatches { (e, source) => 118 | source.grouped(5).map(n => throw e) 119 | } 120 | } 121 | } 122 | 123 | describe("mapAsync") { 124 | it("catches exceptions and propagates them to the promise") { 125 | assertOperationCatches { (e, source) => 126 | source.mapAsync(4) { n => 127 | throw e 128 | } 129 | } 130 | assertOperationCatches { (e, source) => 131 | source.mapAsync(4) { n => 132 | Future.failed(e) 133 | } 134 | } 135 | } 136 | 137 | for { 138 | (thisCase, fn) <- Map[String, (Int) => Future[Int]]("successful" -> { 139 | n => 140 | Future.successful(n) 141 | }, "failed future" -> { n => 142 | Future.failed(new Exception(s"$n is not the number of the day!!!")) 143 | }, "exception thrown" -> { n => 144 | throw new Exception(s"$n is not the number of the day!!!") 145 | }) 146 | } { 147 | 148 | it( 149 | s"only calls the function once per element when ${thisCase} (regression test)" 150 | ) { 151 | var count = 0 152 | implicit val materializer = ActorMaterializer( 153 | ActorMaterializerSettings(actorSystem).withSupervisionStrategy( 154 | Supervision.resumingDecider: Supervision.Decider 155 | ) 156 | ) 157 | val (completions, result) = runLeTest(1 to 1) { 158 | _.mapAsync(4) { n => 159 | count = count + 1 160 | fn(n) 161 | }.runAck 162 | } 163 | 164 | count should be(1) 165 | } 166 | } 167 | } 168 | 169 | describe("mapAsyncUnordered") { 170 | it("catches exceptions and propagates them to the promise") { 171 | assertOperationCatches { (e, source) => 172 | source.mapAsync(4) { n => 173 | throw e 174 | } 175 | } 176 | assertOperationCatches { (e, source) => 177 | source.mapAsync(4) { n => 178 | Future.failed(e) 179 | } 180 | } 181 | } 182 | } 183 | 184 | describe("groupBy") { 185 | it("catches exceptions and propagates them to the promise") { 186 | assertOperationCatches { (e, source) => 187 | source 188 | .groupBy(1, { n => 189 | throw e 190 | }) 191 | .mergeSubstreams 192 | } 193 | } 194 | } 195 | 196 | describe("conflate") { 197 | it("catches exceptions and propagates them to the promise") { 198 | assertOperationCatches { (e, source) => 199 | source.conflateWithSeed(n => throw e) { (a: Int, b) => 200 | 5 201 | } 202 | } 203 | } 204 | } 205 | describe("log") { 206 | // TODO - it looks like log does not resume exceptions! Bug in akka-stream? 207 | // it("catches exceptions and propagates them to the promise") { 208 | // assertOperationCatches { (e, source) => source.log("hi", { n => throw e}) } 209 | // } 210 | } 211 | 212 | describe("mapConcat") { 213 | it("Acks messages that are filtered by returning List.empty") { 214 | implicit val materializer = ActorMaterializer() 215 | val (completions, result) = runLeTest(Range.inclusive(1, 20)) { 216 | _.mapConcat(n => List.empty[Int]).acked.runWith(Sink.fold(0)(_ + _)) 217 | } 218 | result should be(0) 219 | assertAcked(completions) 220 | } 221 | 222 | it("Acks messages that are split into multiple messages") { 223 | implicit val materializer = ActorMaterializer() 224 | val (completions, result) = runLeTest(Range.inclusive(1, 20)) { 225 | _.mapConcat(n => List(n, n)).acked.runWith(Sink.fold(0)(_ + _)) 226 | } 227 | result should be(420) 228 | assertAcked(completions) 229 | } 230 | 231 | it("catches exceptions and propagates them to the promise") { 232 | assertOperationCatches { (e, source) => 233 | source.mapConcat { n => 234 | throw e 235 | } 236 | } 237 | } 238 | } 239 | 240 | describe("alsoTo") { 241 | it("acknowledges elements after they've hit both sinks") { 242 | implicit val materializer = ActorMaterializer( 243 | ActorMaterializerSettings(actorSystem).withSupervisionStrategy( 244 | Supervision.resumingDecider: Supervision.Decider 245 | ) 246 | ) 247 | val oddFailure = new Exception("odd") 248 | val evenFailure = new Exception("even") 249 | val rejectOdds = AckedSink.foreach { n: Int => 250 | if ((n % 2) == 1) 251 | throw (oddFailure) 252 | } 253 | val rejectEvens = AckedSink.foreach { n: Int => 254 | if ((n % 2) == 0) 255 | throw (evenFailure) 256 | } 257 | val (completions, result) = runLeTest(List(1, 2)) { numbers => 258 | numbers.alsoTo(rejectEvens).runWith(rejectOdds) 259 | } 260 | completions shouldBe List( 261 | Some(Failure(oddFailure)), 262 | Some(Failure(evenFailure)) 263 | ) 264 | } 265 | } 266 | 267 | describe("alsoToMat") { 268 | it("acknowledges elements after they've hit both sinks") { 269 | implicit val materializer = ActorMaterializer( 270 | ActorMaterializerSettings(actorSystem).withSupervisionStrategy( 271 | Supervision.resumingDecider: Supervision.Decider 272 | ) 273 | ) 274 | val oddFailure = new Exception("odd") 275 | val evenFailure = new Exception("even") 276 | val rejectOdds = AckedSink.foreach { n: Int => 277 | if ((n % 2) == 1) 278 | throw (oddFailure) 279 | } 280 | val rejectEvens = AckedSink.foreach { n: Int => 281 | if ((n % 2) == 0) 282 | throw (evenFailure) 283 | } 284 | val (completions, result) = runLeTest(List(1, 2)) { numbers => 285 | numbers 286 | .alsoToMat(rejectEvens)(Keep.right) 287 | .toMat(rejectOdds) { (left, right) => 288 | left.flatMap(_ => right)(scala.concurrent.ExecutionContext.global) 289 | } 290 | .run() 291 | } 292 | completions shouldBe List( 293 | Some(Failure(oddFailure)), 294 | Some(Failure(evenFailure)) 295 | ) 296 | } 297 | } 298 | 299 | describe("splitWhen") { 300 | it( 301 | "routes each element to a new acknowledged stream when the predicate matches" 302 | ) { 303 | implicit val materializer = ActorMaterializer() 304 | val (completions, result) = runLeTest(1 to 20) { src => 305 | src 306 | .splitWhen(_ % 4 == 0) 307 | .fold(0) { 308 | case (r, i) => 309 | r + 1 310 | } 311 | .mergeSubstreams 312 | .fold(0) { case (r, _) => r + 1 } 313 | .runWith(AckedSink.head) 314 | } 315 | completions.distinct.shouldBe(List(Some(Success(())))) 316 | result shouldBe 6 317 | } 318 | } 319 | } 320 | 321 | it("stress test") { 322 | val ex = new Exception("lame") 323 | var unexpectedException = false 324 | (1 until 20) foreach { _ => 325 | val data = Stream.continually(Promise[Unit]) zip Range( 326 | 1, 327 | Math.max(40, (Random.nextInt(200))) 328 | ) 329 | val decider: Supervision.Decider = { (e: Throwable) => 330 | if (e != ex) { 331 | unexpectedException = true 332 | println(s"Stream error; message dropped. ${e.getMessage.toString}") 333 | } 334 | Supervision.Resume 335 | } 336 | implicit val materializer = ActorMaterializer( 337 | ActorMaterializerSettings(actorSystem).withSupervisionStrategy(decider) 338 | ) 339 | 340 | import scala.concurrent.ExecutionContext.Implicits.global 341 | await( 342 | AckedSource(data) 343 | .grouped(Math.max(1, Random.nextInt(11))) 344 | .mapConcat { 345 | _ filter { 346 | case 59 => throw ex 347 | case n if n % 2 == 0 => true 348 | case _ => false 349 | } 350 | } 351 | .mapAsync(8) { 352 | case 36 => 353 | Future { 354 | Thread.sleep(Math.abs(scala.util.Random.nextInt(10))) 355 | throw ex 356 | } 357 | case 12 => 358 | Thread.sleep(Math.abs(scala.util.Random.nextInt(10))) 359 | throw ex 360 | case x => 361 | Future { 362 | Thread.sleep(Math.abs(scala.util.Random.nextInt(10))) 363 | x 364 | } 365 | } 366 | .runAck 367 | ) 368 | // make sure every promise is fulfilled 369 | data.foreach { 370 | case (p, _) => 371 | Try(await(p.future)) match { 372 | case Failure(`ex`) => () 373 | case Success(()) => () 374 | case other => throw new Exception(s"${other} not expected") 375 | } 376 | } 377 | assert(!unexpectedException) 378 | } 379 | } 380 | } 381 | -------------------------------------------------------------------------------- /src/main/scala/com/timcharper/acked/AckedFlowOps.scala: -------------------------------------------------------------------------------- 1 | package com.timcharper.acked 2 | 3 | import akka.event.LoggingAdapter 4 | import akka.stream.Attributes 5 | import akka.stream.scaladsl.Keep 6 | import akka.stream.scaladsl.SubFlow 7 | import akka.stream.{Graph, Materializer, OverflowStrategy} 8 | import akka.stream.scaladsl.{Flow, Keep, RunnableGraph, Sink, Source} 9 | import scala.annotation.unchecked.uncheckedVariance 10 | import scala.collection.immutable 11 | import scala.concurrent.{Future, Promise} 12 | import scala.concurrent.duration._ 13 | import scala.inline 14 | import scala.language.higherKinds 15 | import scala.language.implicitConversions 16 | import scala.language.existentials 17 | 18 | abstract class AckedFlowOps[+Out, +Mat] extends AnyRef { self => 19 | type UnwrappedRepr[+O] <: akka.stream.scaladsl.FlowOps[O, Mat] 20 | type WrappedRepr[+O] <: akka.stream.scaladsl.FlowOps[AckTup[O], Mat] 21 | type Repr[+O] <: AckedFlowOps[O, Mat] 22 | import FlowHelpers.{propException, propFutureException} 23 | 24 | protected val wrappedRepr: WrappedRepr[Out] 25 | 26 | /** 27 | Concatenates this Flow with the given Source so the first element 28 | emitted by that source is emitted after the last element of this 29 | flow. 30 | 31 | See FlowOps.++ in akka-stream 32 | */ 33 | def ++[U >: Out, Mat2](that: AckedGraph[AckedSourceShape[U], Mat2]): Repr[U] = 34 | andThen { 35 | wrappedRepr.concat(that.akkaGraph) 36 | } 37 | 38 | /** 39 | Concatenates this Flow with the given Source so the first element 40 | emitted by that source is emitted after the last element of this 41 | flow. 42 | 43 | See FlowOps.concat in akka-stream 44 | */ 45 | def concat[U >: Out, Mat2]( 46 | that: AckedGraph[AckedSourceShape[U], Mat2] 47 | ): Repr[U] = 48 | andThen { 49 | wrappedRepr.concat(that.akkaGraph) 50 | } 51 | 52 | def alsoTo(that: AckedGraph[AckedSinkShape[Out], _]): Repr[Out] = { 53 | implicit val ec = SameThreadExecutionContext 54 | andThen { 55 | val forking = wrappedRepr.map { 56 | case (p, data) => 57 | val l = Promise[Unit] 58 | val r = Promise[Unit] 59 | p.completeWith(l.future.flatMap { _ => 60 | r.future 61 | }) 62 | ((l, r), data) 63 | // null 64 | } 65 | forking 66 | .alsoTo( 67 | Flow[((Promise[Unit], Promise[Unit]), Out)] 68 | .map { case ((_, p), data) => (p, data) } 69 | .to(that.akkaGraph) 70 | ) 71 | .map { case ((p, _), data) => (p, data) } 72 | .asInstanceOf[WrappedRepr[Out]] 73 | } 74 | } 75 | 76 | def completionTimeout(timeout: FiniteDuration): Repr[Out] = 77 | andThen(wrappedRepr.completionTimeout(timeout)) 78 | 79 | /** 80 | See FlowOps.collect in akka-stream 81 | 82 | A map and a filter. Elements for which the provided 83 | PartialFunction is not defined are acked. 84 | */ 85 | def collect[T](pf: PartialFunction[Out, T]): Repr[T] = 86 | andThen { 87 | wrappedRepr.mapConcat { 88 | case (p, data) => 89 | if (pf.isDefinedAt(data)) { 90 | List((p, propException(p)(pf(data)))) 91 | } else { 92 | p.success(()) 93 | List.empty 94 | } 95 | } 96 | } 97 | 98 | /** 99 | * This operation applies the given predicate to all incoming 100 | * elements and emits them to a stream of output streams, always 101 | * beginning a new one with the current element if the given 102 | * predicate returns true for it. This means that for the following 103 | * series of predicate values, three substreams will be produced 104 | * with lengths 1, 2, and 3: 105 | * 106 | * {{{ 107 | * false, // element goes into first substream 108 | * true, false, // elements go into second substream 109 | * true, false, false // elements go into third substream 110 | * }}} 111 | * 112 | * In case the *first* element of the stream matches the predicate, 113 | * the first substream emitted by splitWhen will start from that 114 | * element. For example: 115 | * 116 | * {{{ 117 | * true, false, false // first substream starts from the split-by element 118 | * true, false // subsequent substreams operate the same way 119 | * }}} 120 | * 121 | * If the split predicate `p` throws an exception and the 122 | * supervision decision is [[akka.stream.Supervision.Stop]] the 123 | * stream and substreams will be completed with failure. 124 | * 125 | * If the split predicate `p` throws an exception and the 126 | * supervision decision is [[akka.stream.Supervision.Resume]] or 127 | * [[akka.stream.Supervision.Restart]] the element is dropped and 128 | * the stream and substreams continue. 129 | * 130 | * Exceptions thrown in predicate will be propagated via the 131 | * acknowledgement channel 132 | * 133 | * '''Emits when''' an element for which the provided predicate is 134 | * true, opening and emitting a new substream for subsequent 135 | * element 136 | * 137 | * '''Backpressures when''' there is an element pending for the 138 | * next substream, but the previous is not fully consumed yet, or 139 | * the substream backpressures 140 | * 141 | * '''Completes when''' upstream completes 142 | * 143 | * '''Cancels when''' downstream cancels and substreams cancel 144 | * 145 | */ 146 | def splitWhen[U >: Out]( 147 | predicate: (Out) ⇒ Boolean 148 | ): AckedSubFlow[Out, Mat, Repr] = 149 | andThenSubFlow { 150 | wrappedRepr.splitWhen { 151 | case (promise, d) => propException(promise) { predicate(d) } 152 | } 153 | } 154 | 155 | // /** 156 | // * This operation applies the given predicate to all incoming elements and 157 | // * emits them to a stream of output streams. It *ends* the current substream when the 158 | // * predicate is true. This means that for the following series of predicate values, 159 | // * three substreams will be produced with lengths 2, 2, and 3: 160 | // * 161 | // * {{{ 162 | // * false, true, // elements go into first substream 163 | // * false, true, // elements go into second substream 164 | // * false, false, true // elements go into third substream 165 | // * }}} 166 | // * 167 | // * If the split predicate `p` throws an exception and the supervision decision 168 | // * is [[akka.stream.Supervision.Stop]] the stream and substreams will be completed 169 | // * with failure. 170 | // * 171 | // * If the split predicate `p` throws an exception and the supervision decision 172 | // * is [[akka.stream.Supervision.Resume]] or [[akka.stream.Supervision.Restart]] 173 | // * the element is dropped and the stream and substreams continue. 174 | // * 175 | // * '''Emits when''' an element passes through. When the provided predicate is true it emitts the element 176 | // * and opens a new substream for subsequent element 177 | // * 178 | // * '''Backpressures when''' there is an element pending for the next substream, but the previous 179 | // * is not fully consumed yet, or the substream backpressures 180 | // * 181 | // * '''Completes when''' upstream completes 182 | // * 183 | // * '''Cancels when''' downstream cancels and substreams cancel 184 | // * 185 | // * See also [[FlowOps.splitWhen]]. 186 | // */ 187 | def splitAfter[U >: Out]( 188 | predicate: (Out) ⇒ Boolean 189 | ): AckedSubFlow[Out, Mat, Repr] = 190 | andThenSubFlow { 191 | wrappedRepr.splitAfter { 192 | case (promise, d) => propException(promise) { predicate(d) } 193 | } 194 | } 195 | 196 | def andThenSubFlow[U >: Out, Mat2 >: Mat]( 197 | wrappedSubFlow: SubFlow[AckTup[U], Mat2, wrappedRepr.Repr, _] 198 | ): AckedSubFlow[U, Mat2, Repr] = { 199 | new AckedSubFlow.Impl[U, Mat2, Repr, wrappedRepr.Repr]( 200 | wrappedSubFlow, 201 | new AckedSubFlow.Converter[wrappedRepr.Repr, Repr] { 202 | def apply[T](from: wrappedRepr.Repr[AckTup[T]]): Repr[T] = 203 | self.andThen(from) 204 | } 205 | ) 206 | } 207 | 208 | /** 209 | See FlowOps.groupedWithin in akka-stream 210 | 211 | Downstream acknowledgement applies to the resulting group (IE: if 212 | it yields a group of 100, then downstream you can only either ack 213 | or nack the entire group). 214 | */ 215 | def groupedWithin(n: Int, d: FiniteDuration): Repr[immutable.Seq[Out]] = { 216 | andThenCombine { wrappedRepr.groupedWithin(n, d) } 217 | } 218 | 219 | /** 220 | See FlowOps.buffer in akka-stream 221 | 222 | Does not accept an OverflowStrategy because only backpressure and 223 | fail are supported. 224 | */ 225 | def buffer(size: Int, failOnOverflow: Boolean = false): Repr[Out] = andThen { 226 | wrappedRepr.buffer( 227 | size, 228 | if (failOnOverflow) OverflowStrategy.fail 229 | else OverflowStrategy.backpressure 230 | ) 231 | } 232 | 233 | /** 234 | See FlowOps.grouped in akka-stream 235 | 236 | Downstream acknowledgement applies to the resulting group (IE: if 237 | it yields a group of 100, then downstream you can only either ack 238 | or nack the entire group). 239 | */ 240 | def grouped(n: Int): Repr[immutable.Seq[Out]] = { 241 | andThenCombine { wrappedRepr.grouped(n) } 242 | } 243 | 244 | /** 245 | See FlowOps.mapConcat in akka-stream 246 | 247 | Splits a single element into 0 or more items. 248 | 249 | If 0 items, then signal completion of this element. Otherwise, 250 | signal completion of this element after all resulting elements are 251 | signaled for completion. 252 | */ 253 | def mapConcat[T](f: Out ⇒ immutable.Iterable[T]): Repr[T] = andThen { 254 | wrappedRepr.mapConcat { 255 | case (p, data) => 256 | val items = Stream.continually(Promise[Unit]) zip propException(p)( 257 | f(data) 258 | ) 259 | if (items.length == 0) { 260 | p.success(()) // effectively a filter. We're done with this message. 261 | items 262 | } else { 263 | implicit val ec = SameThreadExecutionContext 264 | p.completeWith(Future.sequence(items.map(_._1.future)).map(_ => ())) 265 | items 266 | } 267 | } 268 | } 269 | 270 | /** 271 | Yields an Unwrapped Repr with only the data; after this point, message are acked. 272 | */ 273 | def acked = wrappedRepr.map { 274 | case (p, data) => 275 | p.success(()) 276 | data 277 | } 278 | 279 | /** 280 | Yields an unacked Repr with the promise and the data. Note, this 281 | is inherently unsafe, as the method says. There is no timeout for 282 | the acknowledgement promises. Failing to complete the promises 283 | will cause a consumer with a non-infinite QoS to eventually stall. 284 | */ 285 | def unsafe = wrappedRepr 286 | 287 | /** 288 | Yields a non-acked flow/source of AckedSource, keyed by the return 289 | value of the provided function. 290 | 291 | See FlowOps.groupBy in akka-stream 292 | */ 293 | def groupBy[K, U >: Out](maxSubstreams: Int, f: (Out) ⇒ K) = 294 | andThenSubFlow { 295 | wrappedRepr.groupBy(maxSubstreams, { 296 | case (p, o) => propException(p) { f(o) } 297 | }) 298 | } 299 | 300 | /** 301 | Filters elements from the stream for which the predicate returns 302 | false. Filtered items are acked. 303 | 304 | See FlowOps.filter in akka-stream 305 | */ 306 | def filter(predicate: (Out) ⇒ Boolean): Repr[Out] = andThen { 307 | wrappedRepr.filter { 308 | case (p, data) => 309 | val result = (propException(p)(predicate(data))) 310 | if (!result) p.success(()) 311 | result 312 | } 313 | } 314 | 315 | /** 316 | Filters elements from the stream for which the predicate returns 317 | true. Filtered items are acked. 318 | 319 | See FlowOps.filterNot in akka-stream 320 | */ 321 | def filterNot(predicate: (Out) => Boolean): Repr[Out] = andThen { 322 | wrappedRepr.filterNot { 323 | case (p, data) => 324 | val result = (propException(p)(predicate(data))) 325 | if (result) p.success(()) 326 | result 327 | } 328 | } 329 | 330 | /** 331 | * Similar to `scan` but only emits its result when the upstream completes, 332 | * after which it also completes. Applies the given function towards its current and next value, 333 | * yielding the next current value. 334 | * 335 | * If the function `f` throws an exception and the supervision decision is 336 | * [[akka.stream.Supervision.Restart]] current value starts at `zero` again 337 | * the stream will continue. 338 | * 339 | * '''Emits when''' upstream completes 340 | * 341 | * '''Backpressures when''' downstream backpressures 342 | * 343 | * '''Completes when''' upstream completes 344 | * 345 | * '''Cancels when''' downstream cancels 346 | */ 347 | def fold[T](zero: T)(f: (T, Out) ⇒ T): Repr[T] = andThen { 348 | wrappedRepr.fold((Promise[Unit], zero)) { 349 | case ((accP, accElem), (p, elem)) => 350 | accP.completeWith(p.future) 351 | (p, propException(p: Promise[Unit])(f(accElem, elem))) 352 | } 353 | } 354 | 355 | /** 356 | If the time between two processed elements exceed the provided 357 | timeout, the stream is failed with a 358 | scala.concurrent.TimeoutException. 359 | */ 360 | def idleTimeout(timeout: FiniteDuration): Repr[Out] = 361 | andThen(wrappedRepr.idleTimeout(timeout)) 362 | 363 | /** 364 | Delays the initial element by the specified duration. 365 | */ 366 | def initialDelay(delay: FiniteDuration): Repr[Out] = 367 | andThen(wrappedRepr.initialDelay(delay)) 368 | 369 | /** 370 | If the first element has not passed through this stage before the 371 | provided timeout, the stream is failed with a 372 | scala.concurrent.TimeoutException. 373 | */ 374 | def initialTimeout(timeout: FiniteDuration): Repr[Out] = 375 | andThen(wrappedRepr.initialTimeout(timeout)) 376 | 377 | /** 378 | Intersperses stream with provided element, similar to how 379 | scala.collection.immutable.List.mkString injects a separator 380 | between a List's elements. 381 | */ 382 | def intersperse[T >: Out](inject: T): Repr[T] = 383 | andThen(wrappedRepr.intersperse((DummyPromise, inject))) 384 | 385 | /** 386 | Intersperses stream with provided element, similar to how 387 | scala.collection.immutable.List.mkString injects a separator 388 | between a List's elements. 389 | */ 390 | def intersperse[T >: Out](start: T, inject: T, end: T): Repr[T] = { 391 | andThen( 392 | wrappedRepr.intersperse( 393 | (DummyPromise, start), 394 | (DummyPromise, inject), 395 | (DummyPromise, end) 396 | ) 397 | ) 398 | } 399 | 400 | /** 401 | Injects additional elements if the upstream does not emit for a 402 | configured amount of time. 403 | */ 404 | def keepAlive[U >: Out](maxIdle: FiniteDuration, 405 | injectedElem: () ⇒ U): Repr[U] = 406 | andThen { 407 | wrappedRepr.keepAlive(maxIdle, () => (DummyPromise, injectedElem())) 408 | } 409 | 410 | /** 411 | See FlowOps.log in akka-stream 412 | */ 413 | def log(name: String, extract: (Out) ⇒ Any = identity)( 414 | implicit log: LoggingAdapter = null 415 | ): Repr[Out] = andThen { 416 | wrappedRepr.log(name, { case (p, d) => propException(p) { extract(d) } }) 417 | } 418 | 419 | /** 420 | See FlowOps.map in akka-stream 421 | */ 422 | def map[T](f: Out ⇒ T): Repr[T] = andThen { 423 | wrappedRepr.map { 424 | case (p, d) => 425 | implicit val ec = SameThreadExecutionContext 426 | (p, propException(p)(f(d))) 427 | } 428 | } 429 | 430 | /** 431 | Merge the given Source to this Flow, taking elements as they arrive from input streams, picking randomly when several elements ready. 432 | 433 | Emits when one of the inputs has an element available 434 | 435 | Backpressures when downstream backpressures 436 | 437 | Completes when all upstreams complete 438 | 439 | Cancels when downstream cancels 440 | */ 441 | def merge[U >: Out](that: AckedGraph[AckedSourceShape[U], _]): Repr[U] = 442 | andThen { 443 | wrappedRepr.merge(that.akkaGraph) 444 | } 445 | 446 | /** 447 | See FlowOps.mapAsync in akka-stream 448 | */ 449 | def mapAsync[T](parallelism: Int)(f: Out ⇒ Future[T]): Repr[T] = andThen { 450 | wrappedRepr.mapAsync(parallelism) { 451 | case (p, d) => 452 | implicit val ec = SameThreadExecutionContext 453 | propFutureException(p)(f(d)) map { r => 454 | (p, r) 455 | } 456 | } 457 | } 458 | 459 | /** 460 | See FlowOps.mapAsyncUnordered in akka-stream 461 | */ 462 | def mapAsyncUnordered[T](parallelism: Int)(f: Out ⇒ Future[T]): Repr[T] = 463 | andThen { 464 | wrappedRepr.mapAsyncUnordered(parallelism) { 465 | case (p, d) => 466 | implicit val ec = SameThreadExecutionContext 467 | propFutureException(p)(f(d)) map { r => 468 | (p, r) 469 | } 470 | } 471 | } 472 | 473 | /** 474 | See FlowOps.conflateWithSeed in akka-stream 475 | 476 | Conflated items are grouped together into a single message, the 477 | acknowledgement of which acknowledges every message that went into 478 | the group. 479 | */ 480 | def conflateWithSeed[S](seed: (Out) ⇒ S)(aggregate: (S, Out) ⇒ S): Repr[S] = 481 | andThen { 482 | wrappedRepr.conflateWithSeed({ 483 | case (p, data) => (p, propException(p)(seed(data))) 484 | }) { 485 | case ((seedPromise, seedData), (p, element)) => 486 | seedPromise.completeWith(p.future) 487 | (p, propException(p)(aggregate(seedData, element))) 488 | } 489 | } 490 | 491 | /** 492 | See FlowOps.take in akka-stream 493 | */ 494 | def take(n: Long): Repr[Out] = andThen { 495 | wrappedRepr.take(n) 496 | } 497 | 498 | /** 499 | See FlowOps.takeWhile in akka-stream 500 | */ 501 | def takeWhile(predicate: (Out) ⇒ Boolean): Repr[Out] = andThen { 502 | wrappedRepr.takeWhile { 503 | case (p, out) => 504 | propException(p)(predicate(out)) 505 | } 506 | } 507 | 508 | /** 509 | Combine the elements of current flow and the given Source into a 510 | stream of tuples. 511 | */ 512 | def zip[U](that: AckedGraph[AckedSourceShape[U], _]): Repr[(Out, U)] = 513 | andThen { 514 | wrappedRepr.zip(that.akkaGraph).map { 515 | case ((p1, d1), (p2, d2)) => 516 | p2.completeWith(p1.future) 517 | (p1, (d1, d2)) 518 | } 519 | } 520 | 521 | /** 522 | Put together the elements of current flow and the given Source 523 | into a stream of combined elements using a combiner function. 524 | */ 525 | def zipWith[Out2, Out3]( 526 | that: AckedGraph[AckedSourceShape[Out2], _] 527 | )(combine: (Out, Out2) ⇒ Out3): Repr[Out3] = 528 | andThen { 529 | wrappedRepr.zipWith(that.akkaGraph)({ 530 | case ((p1, d1), (p2, d2)) => 531 | p2.completeWith(p1.future) 532 | (p1, propException(p1)(combine(d1, d2))) 533 | }) 534 | } 535 | 536 | /** 537 | See FlowOps.takeWithin in akka-stream 538 | */ 539 | def takeWithin(d: FiniteDuration): Repr[Out] = 540 | andThen { 541 | wrappedRepr.takeWithin(d) 542 | } 543 | 544 | protected def andThen[U](next: WrappedRepr[U]): Repr[U] 545 | 546 | // The compiler needs a little bit of help to know that this conversion is possible 547 | @inline 548 | implicit def collapse1to0[U, Mat2]( 549 | next: wrappedRepr.Repr[AckTup[U]] 550 | ): WrappedRepr[U] = next.asInstanceOf[WrappedRepr[U]] 551 | 552 | // Combine all promises into one, such that the fulfillment of that promise fulfills the entire group 553 | private def andThenCombine[U, Mat2 >: Mat]( 554 | next: wrappedRepr.Repr[immutable.Seq[AckTup[U]]] 555 | ): Repr[immutable.Seq[U]] = 556 | andThen { 557 | next.map { data => 558 | (data.map(_._1).reduce { (p1, p2) => 559 | p1.completeWith(p2.future); p2 560 | }, data.map(_._2)) 561 | } 562 | } 563 | } 564 | 565 | class AckedFlow[-In, +Out, +Mat]( 566 | val wrappedRepr: Flow[AckTup[In], AckTup[Out], Mat] 567 | ) extends AckedFlowOpsMat[Out, Mat] 568 | with AckedGraph[AckedFlowShape[In, Out], Mat] { 569 | type UnwrappedRepr[+O] = 570 | Flow[In @uncheckedVariance, O, Mat @uncheckedVariance] 571 | type WrappedRepr[+O] = 572 | Flow[AckTup[In] @uncheckedVariance, AckTup[O], Mat @uncheckedVariance] 573 | 574 | type UnwrappedReprMat[+O, +M] = Flow[In @uncheckedVariance, O, M] 575 | type WrappedReprMat[+O, +M] = 576 | Flow[AckTup[In] @uncheckedVariance, AckTup[O], M] 577 | 578 | type Repr[+O] = AckedFlow[In @uncheckedVariance, O, Mat @uncheckedVariance] 579 | type ReprMat[+O, +M] = AckedFlow[In @uncheckedVariance, O, M] 580 | 581 | lazy val shape = new AckedFlowShape(wrappedRepr.shape) 582 | val akkaGraph = wrappedRepr 583 | 584 | def to[Mat2](sink: AckedSink[Out, Mat2]): AckedSink[In, Mat] = 585 | AckedSink(wrappedRepr.to(sink.akkaSink)) 586 | 587 | def toMat[Mat2, Mat3]( 588 | sink: AckedSink[Out, Mat2] 589 | )(combine: (Mat, Mat2) ⇒ Mat3): AckedSink[In, Mat3] = 590 | AckedSink(wrappedRepr.toMat(sink.akkaSink)(combine)) 591 | 592 | protected def andThen[U](next: WrappedRepr[U] @uncheckedVariance): Repr[U] = { 593 | new AckedFlow(next) 594 | } 595 | 596 | protected def andThenMat[U, Mat2]( 597 | next: WrappedReprMat[U, Mat2] @uncheckedVariance 598 | ): ReprMat[U, Mat2] = { 599 | new AckedFlow(next) 600 | } 601 | 602 | /** 603 | See Flow.via in akka-stream 604 | */ 605 | def via[T, Mat2]( 606 | flow: AckedGraph[AckedFlowShape[Out, T], Mat2] 607 | ): AckedFlow[In, T, Mat] = 608 | andThen(wrappedRepr.via(flow.akkaGraph)) 609 | 610 | /** 611 | See Flow.viaMat in akka-stream 612 | */ 613 | def viaMat[T, Mat2, Mat3]( 614 | flow: AckedGraph[AckedFlowShape[Out, T], Mat2] 615 | )(combine: (Mat, Mat2) ⇒ Mat3): AckedFlow[In, T, Mat3] = 616 | andThenMat(wrappedRepr.viaMat(flow.akkaGraph)(combine)) 617 | 618 | /** 619 | Transform the materialized value of this AckedFlow, leaving all other properties as they were. 620 | */ 621 | def mapMaterializedValue[Mat2](f: (Mat) ⇒ Mat2): ReprMat[Out, Mat2] = 622 | andThenMat(wrappedRepr.mapMaterializedValue(f)) 623 | 624 | override def withAttributes(attr: Attributes): Repr[Out] = andThen { 625 | wrappedRepr.withAttributes(attr) 626 | } 627 | 628 | override def addAttributes(attr: Attributes): Repr[Out] = andThen { 629 | wrappedRepr.addAttributes(attr) 630 | } 631 | } 632 | 633 | abstract class AckedFlowOpsMat[+Out, +Mat] extends AckedFlowOps[Out, Mat] { 634 | import FlowHelpers.{propException, propFutureException} 635 | 636 | type UnwrappedRepr[+O] <: akka.stream.scaladsl.FlowOpsMat[O, Mat] 637 | type WrappedRepr[+O] <: akka.stream.scaladsl.FlowOpsMat[AckTup[O], Mat] 638 | 639 | type UnwrappedReprMat[+O, +M] <: akka.stream.scaladsl.FlowOpsMat[O, M] 640 | type WrappedReprMat[+O, +M] <: akka.stream.scaladsl.FlowOpsMat[AckTup[O], M] 641 | 642 | type Repr[+O] <: AckedFlowOpsMat[O, Mat @uncheckedVariance] 643 | type ReprMat[+O, +M] <: AckedFlowOpsMat[O, M] 644 | 645 | def alsoToMat[Mat2, Mat3]( 646 | that: AckedGraph[AckedSinkShape[Out], Mat2] 647 | )(matF: (Mat, Mat2) => Mat3): ReprMat[Out, Mat3] = { 648 | implicit val ec = SameThreadExecutionContext 649 | andThenMat { 650 | val forking = wrappedRepr.map { 651 | case (p, data) => 652 | val l = Promise[Unit] 653 | val r = Promise[Unit] 654 | p.completeWith(l.future.flatMap { _ => 655 | r.future 656 | }) 657 | ((l, r), data) 658 | // null 659 | } 660 | // HACK! Work around https://github.com/akka/akka/issues/19336 661 | val elevated = forking 662 | .asInstanceOf[UnwrappedRepr[((Promise[Unit], Promise[Unit]), Out)]] 663 | 664 | elevated 665 | .alsoToMat( 666 | Flow[((Promise[Unit], Promise[Unit]), Out)] 667 | .map { case ((_, p), data) => (p, data) } 668 | .toMat(that.akkaGraph)(Keep.right) 669 | )(matF) 670 | .map { case ((p, _), data) => (p, data) } 671 | .asInstanceOf[WrappedReprMat[Out, Mat3]] 672 | } 673 | } 674 | 675 | // /** 676 | // Put together the elements of current flow and the given Source 677 | // into a stream of combined elements using a combiner function. 678 | // */ 679 | def zipWithMat[Out2, Out3, Mat2, Mat3]( 680 | that: AckedGraph[AckedSourceShape[Out2], Mat2] 681 | )( 682 | combine: (Out, Out2) ⇒ Out3 683 | )(matF: (Mat, Mat2) ⇒ Mat3): ReprMat[Out3, Mat3] = { 684 | andThenMat { 685 | wrappedRepr.zipWithMat(that.akkaGraph)({ 686 | case ((p1, d1), (p2, d2)) => 687 | p2.completeWith(p1.future) 688 | (p1, propException(p1)(combine(d1, d2))) 689 | })(matF) 690 | } 691 | } 692 | 693 | @inline 694 | protected implicit def collapse2to0Mat[U, Mat2]( 695 | next: wrappedRepr.ReprMat[_, _]#ReprMat[AckTup[U], Mat2] 696 | ): WrappedReprMat[U, Mat2] = next.asInstanceOf[WrappedReprMat[U, Mat2]] 697 | 698 | // /** 699 | // Combine the elements of current flow and the given Source into a 700 | // stream of tuples. 701 | // */ 702 | def zipMat[U, Mat2, Mat3]( 703 | that: AckedGraph[AckedSourceShape[U], Mat2] 704 | )(matF: (Mat, Mat2) ⇒ Mat3): ReprMat[(Out, U), Mat3] = { 705 | andThenMat { 706 | wrappedRepr.zipMat(that.akkaGraph)(matF).map { 707 | case ((p1, d1), (p2, d2)) => 708 | p2.completeWith(p1.future) 709 | (p1, (d1, d2)) 710 | } 711 | } 712 | } 713 | 714 | // /** 715 | // Merge the given Source to this Flow, taking elements as they 716 | // arrive from input streams, picking randomly when several elements 717 | // ready. 718 | // */ 719 | def mergeMat[U >: Out, Mat2, Mat3]( 720 | that: AckedGraph[AckedSourceShape[U], Mat2] 721 | )(matF: (Mat, Mat2) ⇒ Mat3): ReprMat[U, Mat3] = 722 | andThenMat { 723 | wrappedRepr.mergeMat(that.akkaGraph)(matF) 724 | } 725 | 726 | protected def andThenMat[U, Mat2]( 727 | next: WrappedReprMat[U, Mat2] 728 | ): ReprMat[U, Mat2] 729 | } 730 | 731 | object AckedFlow { 732 | def apply[T] = new AckedFlow(Flow.apply[AckTup[T]]) 733 | 734 | def apply[In, Out, Mat](wrappedFlow: Flow[AckTup[In], AckTup[Out], Mat]) = 735 | new AckedFlow(wrappedFlow) 736 | } 737 | --------------------------------------------------------------------------------