├── project └── build.properties ├── README.md ├── src └── main │ └── scala │ └── akka │ └── stream │ └── alpakka │ └── nats │ ├── javadsl │ ├── NatsStreamingSimpleSource.scala │ ├── NatsStreamingSimpleSink.scala │ ├── NatsStreamingSourceWithAck.scala │ └── NatsStreamingSinkWithCompletion.scala │ ├── scaladsl │ ├── NatsStreamingSimpleSource.scala │ ├── NatsStreamingSimpleSink.scala │ ├── NatsStreamingSourceWithAck.scala │ └── NatsStreamingSinkWithCompletion.scala │ ├── StreamingConnectionProvider.scala │ ├── NatsStreamingMessage.scala │ ├── NatsStreamingSinkStageLogic.scala │ ├── NatsStreamingSourceStageLogic.scala │ └── NatsStreamingSettings.scala ├── LICENSE └── .gitignore /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.1.6 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Akka Nats Streaming 2 | 3 | [Akka Streams](https://doc.akka.io/docs/akka/current/stream/index.html) connectors (Sources & Sinks) for [Nats Streaming](https://www.google.com/url?sa=t&rct=j&q=&esrc=s&source=web&cd=1&ved=0ahUKEwi1946h09DbAhXEbZoKHU2EDzcQFggpMAA&url=https%3A%2F%2Fnats.io%2Fdocumentation%2Fstreaming%2Fnats-streaming-intro%2F&usg=AOvVaw0SsD41lHVd91SD6PqmVcqG). 4 | 5 | Missing: 6 | 7 | - [ ] Tests 8 | - [ ] Documentation 9 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/javadsl/NatsStreamingSimpleSource.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.javadsl 2 | 3 | import akka.NotUsed 4 | import akka.stream.alpakka.nats.{IncomingMessage, NatsStreamingSimpleSourceStage, SimpleSubscriptionSettings} 5 | import akka.stream.javadsl.Source 6 | import com.typesafe.config.Config 7 | 8 | object NatsStreamingSimpleSource { 9 | def create(settings: SimpleSubscriptionSettings): Source[IncomingMessage[Array[Byte]], NotUsed] = 10 | Source.fromGraph(new NatsStreamingSimpleSourceStage(settings)) 11 | 12 | def create(config: Config): Source[IncomingMessage[Array[Byte]], NotUsed] = 13 | create(SimpleSubscriptionSettings.fromConfig(config)) 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/scaladsl/NatsStreamingSimpleSource.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.scaladsl 2 | 3 | import akka.NotUsed 4 | import akka.stream.alpakka.nats.{IncomingMessage, NatsStreamingSimpleSourceStage, SimpleSubscriptionSettings} 5 | import akka.stream.scaladsl.Source 6 | import com.typesafe.config.Config 7 | 8 | object NatsStreamingSimpleSource { 9 | def apply(settings: SimpleSubscriptionSettings): Source[IncomingMessage[Array[Byte]], NotUsed] = 10 | Source.fromGraph(new NatsStreamingSimpleSourceStage(settings)) 11 | 12 | def apply(config: Config): Source[IncomingMessage[Array[Byte]], NotUsed] = 13 | apply(SimpleSubscriptionSettings.fromConfig(config)) 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/javadsl/NatsStreamingSimpleSink.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.javadsl 2 | 3 | import akka.Done 4 | import akka.stream.alpakka.nats.{NatsStreamingSimpleSinkStage, OutgoingMessage, PublishingSettings} 5 | import akka.stream.javadsl.Sink 6 | import com.typesafe.config.Config 7 | 8 | import scala.concurrent.Future 9 | 10 | object NatsStreamingSimpleSink { 11 | def create(settings: PublishingSettings): Sink[OutgoingMessage[Array[Byte]], Future[Done]] = 12 | Sink.fromGraph(new NatsStreamingSimpleSinkStage(settings)) 13 | 14 | def create(config: Config): Sink[OutgoingMessage[Array[Byte]], Future[Done]] = 15 | create(PublishingSettings.fromConfig(config)) 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/scaladsl/NatsStreamingSimpleSink.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.scaladsl 2 | 3 | import akka.Done 4 | import akka.stream.alpakka.nats.{NatsStreamingSimpleSinkStage, OutgoingMessage, PublishingSettings} 5 | import akka.stream.scaladsl.Sink 6 | import com.typesafe.config.Config 7 | 8 | import scala.concurrent.Future 9 | 10 | object NatsStreamingSimpleSink { 11 | def apply(settings: PublishingSettings): Sink[OutgoingMessage[Array[Byte]], Future[Done]] = 12 | Sink.fromGraph(new NatsStreamingSimpleSinkStage(settings)) 13 | 14 | def apply(config: Config): Sink[OutgoingMessage[Array[Byte]], Future[Done]] = 15 | apply(PublishingSettings.fromConfig(config)) 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/javadsl/NatsStreamingSourceWithAck.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.javadsl 2 | 3 | import akka.NotUsed 4 | import akka.stream.alpakka.nats.{IncomingMessageWithAck, NatsStreamingSourceWithAckStage, SubscriptionWithAckSettings} 5 | import akka.stream.javadsl.Source 6 | import com.typesafe.config.Config 7 | 8 | object NatsStreamingSourceWithAck { 9 | def create(settings: SubscriptionWithAckSettings): Source[IncomingMessageWithAck[Array[Byte]], NotUsed] = 10 | Source.fromGraph(new NatsStreamingSourceWithAckStage(settings)) 11 | 12 | def create(config: Config): Source[IncomingMessageWithAck[Array[Byte]], NotUsed] = 13 | create(SubscriptionWithAckSettings.fromConfig(config)) 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/scaladsl/NatsStreamingSourceWithAck.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.scaladsl 2 | 3 | import akka.NotUsed 4 | import akka.stream.alpakka.nats.{IncomingMessageWithAck, NatsStreamingSourceWithAckStage, SubscriptionWithAckSettings} 5 | import akka.stream.scaladsl.Source 6 | import com.typesafe.config.Config 7 | 8 | object NatsStreamingSourceWithAck { 9 | def apply(settings: SubscriptionWithAckSettings): Source[IncomingMessageWithAck[Array[Byte]], NotUsed] = 10 | Source.fromGraph(new NatsStreamingSourceWithAckStage(settings)) 11 | 12 | def apply(config: Config): Source[IncomingMessageWithAck[Array[Byte]], NotUsed] = 13 | apply(SubscriptionWithAckSettings.fromConfig(config)) 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/javadsl/NatsStreamingSinkWithCompletion.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.javadsl 2 | 3 | import akka.Done 4 | import akka.stream.alpakka.nats.{NatsStreamingSinkWithCompletionStage, OutgoingMessageWithCompletion, PublishingSettings} 5 | import akka.stream.javadsl.Sink 6 | import com.typesafe.config.Config 7 | 8 | import scala.concurrent.Future 9 | 10 | object NatsStreamingSinkWithCompletion { 11 | def create(settings: PublishingSettings): Sink[OutgoingMessageWithCompletion[Array[Byte]], Future[Done]] = 12 | Sink.fromGraph(new NatsStreamingSinkWithCompletionStage(settings)) 13 | 14 | def create(config: Config): Sink[OutgoingMessageWithCompletion[Array[Byte]], Future[Done]] = 15 | create(PublishingSettings.fromConfig(config)) 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/scaladsl/NatsStreamingSinkWithCompletion.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats.scaladsl 2 | 3 | import akka.Done 4 | import akka.stream.alpakka.nats.{NatsStreamingSinkWithCompletionStage, OutgoingMessageWithCompletion, PublishingSettings} 5 | import akka.stream.scaladsl.Sink 6 | import com.typesafe.config.Config 7 | 8 | import scala.concurrent.Future 9 | 10 | object NatsStreamingSinkWithCompletion { 11 | def apply(settings: PublishingSettings): Sink[OutgoingMessageWithCompletion[Array[Byte]], Future[Done]] = 12 | Sink.fromGraph(new NatsStreamingSinkWithCompletionStage(settings)) 13 | 14 | def apply(config: Config): Sink[OutgoingMessageWithCompletion[Array[Byte]], Future[Done]] = 15 | apply(PublishingSettings.fromConfig(config)) 16 | } 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/StreamingConnectionProvider.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats 2 | 3 | import com.typesafe.config.Config 4 | import io.nats.streaming.{NatsStreaming, Options, StreamingConnection} 5 | 6 | trait StreamingConnectionProvider { 7 | def connection: StreamingConnection 8 | } 9 | 10 | final case class NatsStreamingConnectionBuilder(clusterId: String, clientId: String, options: Options) extends StreamingConnectionProvider{ 11 | def connection: StreamingConnection = NatsStreaming.connect(clusterId, clientId, options) 12 | } 13 | 14 | object NatsStreamingConnectionBuilder{ 15 | def fromConfig(config: Config): NatsStreamingConnectionBuilder = fromSettings(NatsStreamingConnectionSettings.fromConfig(config)) 16 | def fromSettings(settings: NatsStreamingConnectionSettings): NatsStreamingConnectionBuilder = { 17 | val b = new Options.Builder().natsUrl(settings.url) 18 | val bConT = settings.connectionTimeout.map(b.connectWait).getOrElse(b) 19 | val bPubAckT = settings.publishAckTimeout.map(bConT.pubAckWait).getOrElse(bConT) 20 | val bMaxInF = settings.publishMaxInFlight.map(bPubAckT.maxPubAcksInFlight).getOrElse(bPubAckT) 21 | val options = settings.discoverPrefix.map(bMaxInF.discoverPrefix).getOrElse(bMaxInF).build() 22 | NatsStreamingConnectionBuilder(settings.clusterId, settings.clientId, options) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/NatsStreamingMessage.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats 2 | 3 | import akka.Done 4 | 5 | import scala.concurrent.{Future, Promise} 6 | import scala.util.{Failure, Success, Try} 7 | 8 | sealed trait NatsStreamingMessage[T]{ 9 | def data: T 10 | def transform[T2](f: T => T2): NatsStreamingMessage[T2] 11 | def subject: Option[String] 12 | } 13 | 14 | sealed trait NatsStreamingIncoming[T] extends NatsStreamingMessage[T] 15 | sealed trait NatsStreamingOutgoing[T] extends NatsStreamingMessage[T] 16 | 17 | case class IncomingMessage[T](data: T, subject: Option[String] = None) extends NatsStreamingIncoming[T]{ 18 | def transform[T2](f: T => T2): IncomingMessage[T2] = 19 | IncomingMessage(f(data), subject) 20 | } 21 | 22 | case class IncomingMessageWithAck[T](data: T, subject: Option[String] = None) extends NatsStreamingIncoming[T]{ 23 | private[nats] val promise = 24 | Promise[Done] 25 | def transform[T2](f: T => T2): IncomingMessageWithAck[T2] = 26 | IncomingMessageWithAck(f(data), subject, promise) 27 | def ack: Try[T] = 28 | if(promise.trySuccess(Done)) Success(data) else Failure(new Exception("Already completed")) 29 | } 30 | 31 | object IncomingMessageWithAck{ 32 | private[nats] def apply[T](data: T, subject: Option[String], p: Promise[Done]): IncomingMessageWithAck[T] = 33 | new IncomingMessageWithAck(data, subject){ 34 | override val promise: Promise[Done] = p 35 | } 36 | } 37 | 38 | case class OutgoingMessage[T](data: T, subject: Option[String] = None) extends NatsStreamingOutgoing[T]{ 39 | def transform[T2](f: T => T2): OutgoingMessage[T2] = 40 | OutgoingMessage(f(data), subject) 41 | } 42 | 43 | case class OutgoingMessageWithCompletion[T](data: T, subject: Option[String] = None) extends NatsStreamingOutgoing[T]{ 44 | private[nats] val promise = Promise[Done] 45 | def transform[T2](f: T => T2): OutgoingMessageWithCompletion[T2] = 46 | OutgoingMessageWithCompletion(f(data), subject, promise) 47 | def completion: Future[Done] = 48 | promise.future 49 | } 50 | 51 | object OutgoingMessageWithCompletion{ 52 | private[nats] def apply[T](data: T, subject: Option[String], p: Promise[Done]): OutgoingMessageWithCompletion[T] = 53 | new OutgoingMessageWithCompletion(data, subject){ 54 | override val promise: Promise[Done] = p 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### OSX ### 2 | .DS_Store 3 | .AppleDouble 4 | .LSOverride 5 | 6 | # Icon must end with two \r 7 | Icon 8 | 9 | 10 | # Thumbnails 11 | ._* 12 | 13 | # Files that might appear in the root of a volume 14 | .DocumentRevisions-V100 15 | .fseventsd 16 | .Spotlight-V100 17 | .TemporaryItems 18 | .Trashes 19 | .VolumeIcon.icns 20 | 21 | # Directories potentially created on remote AFP share 22 | .AppleDB 23 | .AppleDesktop 24 | Network Trash Folder 25 | Temporary Items 26 | .apdisk 27 | 28 | 29 | ### Intellij ### 30 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 31 | 32 | *.iml 33 | 34 | ## Directory-based project format: 35 | .idea/ 36 | # if you remove the above rule, at least ignore the following: 37 | 38 | # User-specific stuff: 39 | # .idea/workspace.xml 40 | # .idea/tasks.xml 41 | # .idea/dictionaries 42 | # .idea/shelf 43 | 44 | # Sensitive or high-churn files: 45 | # .idea/dataSources.ids 46 | # .idea/dataSources.xml 47 | # .idea/sqlDataSources.xml 48 | # .idea/dynamic.xml 49 | # .idea/uiDesigner.xml 50 | 51 | # Gradle: 52 | # .idea/gradle.xml 53 | # .idea/libraries 54 | 55 | # Mongo Explorer plugin: 56 | # .idea/mongoSettings.xml 57 | 58 | ## File-based project format: 59 | *.ipr 60 | *.iws 61 | 62 | ## Plugin-specific files: 63 | 64 | # IntelliJ 65 | /out/ 66 | 67 | # mpeltonen/sbt-idea plugin 68 | .idea_modules/ 69 | 70 | # JIRA plugin 71 | atlassian-ide-plugin.xml 72 | 73 | # Crashlytics plugin (for Android Studio and IntelliJ) 74 | com_crashlytics_export_strings.xml 75 | crashlytics.properties 76 | crashlytics-build.properties 77 | fabric.properties 78 | 79 | 80 | ### Eclipse ### 81 | 82 | .metadata 83 | bin/ 84 | tmp/ 85 | *.tmp 86 | *.bak 87 | *.swp 88 | *~.nib 89 | local.properties 90 | .settings/ 91 | .loadpath 92 | 93 | # Eclipse Core 94 | .project 95 | 96 | # External tool builders 97 | .externalToolBuilders/ 98 | 99 | # Locally stored "Eclipse launch configurations" 100 | *.launch 101 | 102 | # PyDev specific (Python IDE for Eclipse) 103 | *.pydevproject 104 | 105 | # CDT-specific (C/C++ Development Tooling) 106 | .cproject 107 | 108 | # JDT-specific (Eclipse Java Development Tools) 109 | .classpath 110 | 111 | # Java annotation processor (APT) 112 | .factorypath 113 | 114 | # PDT-specific (PHP Development Tools) 115 | .buildpath 116 | 117 | # sbteclipse plugin 118 | .target 119 | 120 | # TeXlipse plugin 121 | .texlipse 122 | 123 | # STS (Spring Tool Suite) 124 | .springBeans 125 | 126 | 127 | ### Scala ### 128 | *.class 129 | *.log 130 | 131 | # sbt specific 132 | .cache 133 | .history 134 | .lib/ 135 | dist/* 136 | target/ 137 | lib_managed/ 138 | src_managed/ 139 | project/boot/ 140 | project/plugins/project/ 141 | 142 | # Scala-IDE specific 143 | .scala_dependencies 144 | .worksheet 145 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/NatsStreamingSinkStageLogic.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats 2 | 3 | import akka.Done 4 | import akka.stream.stage._ 5 | import akka.stream.{Attributes, Inlet, SinkShape} 6 | import io.nats.client.{ConnectionEvent, NATSException} 7 | import io.nats.streaming.{AckHandler, StreamingConnection} 8 | 9 | import scala.concurrent.{Future, Promise} 10 | import scala.util.control.NonFatal 11 | 12 | private[nats] abstract class NatsStreamingSinkStageLogic[T <: NatsStreamingOutgoing[Array[Byte]]]( 13 | settings: PublishingSettings, 14 | promise: Promise[Done], 15 | shape: SinkShape[T], 16 | in: Inlet[T] 17 | ) extends GraphStageLogic(shape) with StageLogging{ 18 | protected val successCallback: AsyncCallback[String] = getAsyncCallback(handleSuccess) 19 | protected val failureCallback: AsyncCallback[Throwable] = getAsyncCallback(handleFailure) 20 | private var connection: StreamingConnection = _ 21 | def ah(m: T): AckHandler 22 | 23 | override def preStart(): Unit = 24 | try{ 25 | connection = settings.cp.connection 26 | val natsConnection = connection.getNatsConnection 27 | natsConnection.setClosedCallback((_: ConnectionEvent) => failureCallback.invoke(new Exception("Connection closed"))) 28 | natsConnection.setDisconnectedCallback((_: ConnectionEvent) => failureCallback.invoke(new Exception("Disconnected"))) 29 | natsConnection.setExceptionHandler((ex: NATSException) => failureCallback.invoke(ex)) 30 | pull(in) 31 | super.preStart() 32 | } catch { 33 | case NonFatal(e) => 34 | failureCallback.invoke(e) 35 | } 36 | 37 | override def postStop(): Unit = { 38 | if(settings.closeConnectionAfterStop){ 39 | try{ 40 | connection.close() 41 | } catch { 42 | case NonFatal(e) => 43 | log.error(e, "Exception during nats connection close") 44 | } 45 | } 46 | promise.tryFailure(new RuntimeException("stage stopped unexpectedly")) 47 | super.postStop() 48 | } 49 | 50 | def handleFailure(ex: Throwable): Unit = { 51 | log.error(ex, "Caught Exception. Failing stage...") 52 | promise.tryFailure(ex) 53 | failStage(ex) 54 | } 55 | 56 | def handleSuccess(nuid: String): Unit = { 57 | log.debug("Successfully pushed {}", nuid) 58 | if(settings.parallel) () else pull(in) 59 | } 60 | 61 | setHandler(in, new InHandler { 62 | override def onUpstreamFailure(ex: Throwable): Unit = { 63 | promise.tryFailure(ex) 64 | super.onUpstreamFailure(ex) 65 | } 66 | 67 | override def onUpstreamFinish(): Unit = { 68 | promise.trySuccess(Done) 69 | super.onUpstreamFinish() 70 | } 71 | override def onPush(): Unit = { 72 | val m = grab(in) 73 | connection.publish(m.subject.getOrElse(settings.defaultSubject), m.data, ah(m)) 74 | if(settings.parallel) pull(in) else () 75 | } 76 | }) 77 | } 78 | 79 | private[nats] class NatsStreamingSimpleSinkStageLogic( 80 | settings: PublishingSettings, 81 | promise: Promise[Done], 82 | shape: SinkShape[OutgoingMessage[Array[Byte]]], 83 | in: Inlet[OutgoingMessage[Array[Byte]]] 84 | ) extends NatsStreamingSinkStageLogic(settings, promise, shape, in){ 85 | def ah(m: OutgoingMessage[Array[Byte]]): AckHandler = 86 | (nuid: String, ex: Exception) => if (Option(ex).isDefined) failureCallback.invoke(ex) else successCallback.invoke(nuid) 87 | } 88 | 89 | private[nats] class NatsStreamingSinkWithCompletionStageLogic( 90 | settings: PublishingSettings, 91 | promise: Promise[Done], 92 | shape: SinkShape[OutgoingMessageWithCompletion[Array[Byte]]], 93 | in: Inlet[OutgoingMessageWithCompletion[Array[Byte]]] 94 | ) extends NatsStreamingSinkStageLogic(settings, promise, shape, in){ 95 | def ah(m: OutgoingMessageWithCompletion[Array[Byte]]): AckHandler = (nuid: String, ex: Exception) => if (Option(ex).isDefined) { 96 | m.promise.tryFailure(ex) 97 | failureCallback.invoke(ex) 98 | } else { 99 | m.promise.trySuccess(Done) 100 | successCallback.invoke(nuid) 101 | } 102 | } 103 | 104 | private[nats] class NatsStreamingSimpleSinkStage(settings: PublishingSettings) 105 | extends GraphStageWithMaterializedValue[SinkShape[OutgoingMessage[Array[Byte]]], Future[Done]]{ 106 | val in: Inlet[OutgoingMessage[Array[Byte]]] = Inlet("NatsStreamingSimpleSink.in") 107 | val shape: SinkShape[OutgoingMessage[Array[Byte]]] = SinkShape(in) 108 | def createLogicAndMaterializedValue(inheritedAttributes: Attributes): (GraphStageLogic, Future[Done]) = { 109 | val promise = Promise[Done] 110 | val logic = new NatsStreamingSimpleSinkStageLogic(settings, promise, shape, in) 111 | (logic, promise.future) 112 | } 113 | } 114 | 115 | private[nats] class NatsStreamingSinkWithCompletionStage(settings: PublishingSettings) 116 | extends GraphStageWithMaterializedValue[SinkShape[OutgoingMessageWithCompletion[Array[Byte]]], Future[Done]]{ 117 | val in: Inlet[OutgoingMessageWithCompletion[Array[Byte]]] = Inlet("NatsStreamingSinkWithComplete.in") 118 | val shape: SinkShape[OutgoingMessageWithCompletion[Array[Byte]]] = SinkShape(in) 119 | def createLogicAndMaterializedValue(inheritedAttributes: Attributes): (GraphStageLogic, Future[Done]) = { 120 | val promise = Promise[Done] 121 | val logic = new NatsStreamingSinkWithCompletionStageLogic(settings, promise, shape, in) 122 | (logic, promise.future) 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/NatsStreamingSourceStageLogic.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import akka.Done 6 | import akka.stream.impl.Buffer 7 | import akka.stream.stage._ 8 | import akka.stream.{Attributes, Outlet, SourceShape} 9 | import io.nats.client.{ConnectionEvent, NATSException} 10 | import io.nats.streaming.{Message, MessageHandler, StreamingConnection} 11 | 12 | import scala.concurrent.Promise 13 | import scala.concurrent.duration.FiniteDuration 14 | import scala.util.control.NonFatal 15 | 16 | private[nats] abstract class NatsStreamingSourceStageLogic[T1 <: NatsStreamingSubscriptionSettings, T2 <: NatsStreamingIncoming[Array[Byte]]]( 17 | settings: T1, 18 | shape: SourceShape[T2], 19 | out: Outlet[T2] 20 | ) extends GraphStageLogic(shape) with OutHandler with StageLogging{ 21 | private final var downstreamWaiting = false 22 | private final var closed = false 23 | private final var subscriptions: Seq[io.nats.streaming.Subscription] = Seq.empty 24 | private final var connection: StreamingConnection = _ 25 | private final var failureLogic: AsyncCallback[Throwable] = _ 26 | protected final var buffer: Buffer[T2] = _ 27 | protected final var processingLogic: AsyncCallback[Unit] = _ 28 | protected final val scheduled = new java.util.concurrent.atomic.AtomicBoolean(false) 29 | protected val messageHandler: MessageHandler 30 | 31 | private final def cleanup(): Unit = if(settings.closeConnectionAfterStop){ 32 | if(null != connection && !closed) try{ 33 | connection.close() 34 | closed = true 35 | } catch { 36 | case NonFatal(e) => 37 | log.error(e, "Failed to close nats connection") 38 | cleanup() 39 | } 40 | } else { 41 | subscriptions.foreach(_.close()) 42 | } 43 | 44 | private final def handleFailure(e: Throwable): Unit = { 45 | log.error(e, "Caught Exception. Failing stage...") 46 | failStage(e) 47 | } 48 | 49 | private final def process(u: Unit): Unit = { 50 | if (!scheduled.compareAndSet(true, false)) throw new IllegalStateException("Code should never reach here") 51 | if (downstreamWaiting && (!buffer.isEmpty)) { 52 | val e = buffer.dequeue() 53 | if(null != e){ 54 | downstreamWaiting = false 55 | push(out, e) 56 | } 57 | } 58 | u 59 | } 60 | 61 | override def preStart(): Unit = try{ 62 | buffer = Buffer[T2](settings.bufferSize, materializer) 63 | failureLogic = getAsyncCallback(handleFailure) 64 | processingLogic = getAsyncCallback(process) 65 | connection = settings.cp.connection 66 | subscriptions = settings.subjects.map{s => 67 | connection.subscribe(s, settings.subscriptionQueue, messageHandler, settings.subscriptionOptions) 68 | } 69 | val natsConnection = connection.getNatsConnection 70 | natsConnection.setClosedCallback((_: ConnectionEvent) => failureLogic.invoke(new Exception("Connection closed"))) 71 | natsConnection.setDisconnectedCallback((_: ConnectionEvent) => failureLogic.invoke(new Exception("Disconnected"))) 72 | natsConnection.setExceptionHandler((e: NATSException) => failureLogic.invoke(e)) 73 | if (scheduled.compareAndSet(false, true)) processingLogic.invoke(()) 74 | log.debug("Nats connection initiated") 75 | super.preStart() 76 | } catch{ 77 | case NonFatal(e) => 78 | handleFailure(e) 79 | } 80 | 81 | override def postStop(): Unit = { 82 | try{ 83 | cleanup() 84 | } catch { 85 | case NonFatal(e) => 86 | log.error(e, "Exception during cleanup") 87 | } 88 | super.postStop() 89 | } 90 | 91 | override def onPull(): Unit = if (buffer.isEmpty) { 92 | downstreamWaiting = true 93 | } else { 94 | val e = buffer.dequeue() 95 | if(null == e) { 96 | downstreamWaiting = true 97 | } else { 98 | push(out, e) 99 | } 100 | } 101 | setHandler(out, this) 102 | } 103 | 104 | private[nats] class NatsStreamingSimpleSourceStageLogic( 105 | settings: SimpleSubscriptionSettings, 106 | shape: SourceShape[IncomingMessage[Array[Byte]]], 107 | out: Outlet[IncomingMessage[Array[Byte]]] 108 | ) extends NatsStreamingSourceStageLogic(settings, shape, out){ 109 | val messageHandler: MessageHandler = (msg: Message) => { 110 | buffer.enqueue(IncomingMessage(msg.getData, Option(msg.getSubject))) 111 | if (settings.manualAcks) msg.ack() 112 | if (scheduled.compareAndSet(false, true)) processingLogic.invoke(()) 113 | } 114 | } 115 | 116 | private[nats] class NatsStreamingSourceWithAckStageLogic( 117 | settings: SubscriptionWithAckSettings, 118 | shape: SourceShape[IncomingMessageWithAck[Array[Byte]]], 119 | out: Outlet[IncomingMessageWithAck[Array[Byte]]] 120 | ) extends NatsStreamingSourceStageLogic(settings, shape, out){ 121 | val messageHandler: MessageHandler = (msg: Message) => { 122 | val promise = Promise[Done]() 123 | buffer.enqueue(IncomingMessageWithAck(msg.getData, Option(msg.getSubject), promise)) 124 | if (scheduled.compareAndSet(false, true)) processingLogic.invoke(()) 125 | val cancelable = materializer.scheduleOnce(FiniteDuration(settings.manualAckTimeout.toNanos, TimeUnit.NANOSECONDS), () => { 126 | promise.tryFailure(new Exception(s"Didn't process message during ${settings.manualAckTimeout}")) 127 | () 128 | }) 129 | promise.future.foreach { _ => 130 | msg.ack() 131 | cancelable.cancel() 132 | }(materializer.executionContext) 133 | } 134 | } 135 | 136 | class NatsStreamingSimpleSourceStage(settings: SimpleSubscriptionSettings) 137 | extends GraphStage[SourceShape[IncomingMessage[Array[Byte]]]]{ 138 | val out: Outlet[IncomingMessage[Array[Byte]]] = Outlet("NatsStreamingSimpleSource.out") 139 | val shape: SourceShape[IncomingMessage[Array[Byte]]] = SourceShape(out) 140 | def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new NatsStreamingSimpleSourceStageLogic(settings, shape, out) 141 | } 142 | 143 | class NatsStreamingSourceWithAckStage(settings: SubscriptionWithAckSettings) 144 | extends GraphStage[SourceShape[IncomingMessageWithAck[Array[Byte]]]] 145 | { 146 | require(settings.manualAckTimeout.compareTo(settings.autoRequeueTimeout.get) <= 0) 147 | val out: Outlet[IncomingMessageWithAck[Array[Byte]]] = Outlet("NatsStreamingSourceWithAck.out") 148 | val shape: SourceShape[IncomingMessageWithAck[Array[Byte]]] = SourceShape(out) 149 | def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new NatsStreamingSourceWithAckStageLogic(settings, shape, out) 150 | } 151 | -------------------------------------------------------------------------------- /src/main/scala/akka/stream/alpakka/nats/NatsStreamingSettings.scala: -------------------------------------------------------------------------------- 1 | package akka.stream.alpakka.nats 2 | 3 | import java.time.{Duration, Instant} 4 | 5 | import com.typesafe.config.Config 6 | import io.nats.streaming.SubscriptionOptions 7 | 8 | import scala.collection.JavaConverters._ 9 | import scala.util.{Failure, Success, Try} 10 | 11 | sealed trait DeliveryStartPosition 12 | 13 | object DeliveryStartPosition{ 14 | case object OnlyNew extends DeliveryStartPosition 15 | case object AllAvailable extends DeliveryStartPosition 16 | case object LastReceived extends DeliveryStartPosition 17 | case class AfterSequenceNumber(seq: Long) extends DeliveryStartPosition 18 | case class AfterTime(time: Instant) extends DeliveryStartPosition 19 | final def fromConfig(config: Config): DeliveryStartPosition = Try(config.getLong("deliver-after-sequence-number")) match { 20 | case Success(seq) => AfterSequenceNumber(seq) 21 | case Failure(_) => 22 | Try(Instant.ofEpochMilli(config.getLong("deliver-after-epoch-millis"))) 23 | .orElse(Try(Instant.parse(config.getString("deliver-after-instance")))) match { 24 | case Success(time) => AfterTime(time) 25 | case Failure(_) => config.getString("deliver").toLowerCase match { 26 | case "only-new" => OnlyNew 27 | case "all-available" => AllAvailable 28 | case "last-received" => LastReceived 29 | } 30 | } 31 | } 32 | } 33 | 34 | final case class NatsStreamingConnectionSettings( 35 | clusterId: String, 36 | clientId: String, 37 | url: String, 38 | connectionTimeout: Option[Duration], 39 | publishAckTimeout: Option[Duration], 40 | publishMaxInFlight: Option[Int], 41 | discoverPrefix: Option[String] 42 | ) 43 | 44 | case object NatsStreamingConnectionSettings{ 45 | private final def url(host: String, port: Int): String = "nats://" + host + ":" + port.toString 46 | final def fromConfig(config: Config): NatsStreamingConnectionSettings = 47 | NatsStreamingConnectionSettings( 48 | clusterId = config.getString("cluster-id"), 49 | clientId = config.getString("client-id"), 50 | url = Try(config.getString("url")).getOrElse(url(config.getString("host"), config.getInt("port"))), 51 | connectionTimeout = Try(config.getDuration("connection-timeout")).toOption, 52 | publishAckTimeout = Try(config.getDuration("pub-ack-timeout")).toOption, 53 | publishMaxInFlight = Try(config.getInt("pub-max-in-flight")).toOption, 54 | discoverPrefix = Try(config.getString("discover-prefix")).toOption 55 | ) 56 | } 57 | 58 | sealed trait NatsStreamingSubscriptionSettings{ 59 | def cp: StreamingConnectionProvider 60 | def subjects: Seq[String] 61 | def subscriptionQueue: String 62 | def durableSubscriptionName: Option[String] 63 | def startPosition: DeliveryStartPosition 64 | def subMaxInFlight: Option[Int] 65 | def bufferSize: Int 66 | def autoRequeueTimeout: Option[Duration] 67 | def manualAcks: Boolean 68 | def closeConnectionAfterStop: Boolean 69 | def subscriptionOptions: SubscriptionOptions = { 70 | val b = new SubscriptionOptions.Builder() 71 | val bMaxInFlight = subMaxInFlight.map(b.maxInFlight).getOrElse(b) 72 | val bAckWait = autoRequeueTimeout.map(bMaxInFlight.ackWait).getOrElse(bMaxInFlight) 73 | val bDurable = durableSubscriptionName.map(bAckWait.durableName).getOrElse(bAckWait) 74 | val builder = if(manualAcks) bDurable.manualAcks() else bDurable 75 | startPosition match { 76 | case DeliveryStartPosition.OnlyNew => builder 77 | case DeliveryStartPosition.AllAvailable => builder.deliverAllAvailable 78 | case DeliveryStartPosition.LastReceived => builder.startWithLastReceived 79 | case DeliveryStartPosition.AfterSequenceNumber(seq) => builder.startAtSequence(seq) 80 | case DeliveryStartPosition.AfterTime(time) => builder.startAtTime(time) 81 | } 82 | }.build() 83 | } 84 | 85 | case object NatsStreamingSubscriptionSettings{ 86 | val defaultBufferSize = 1024 87 | } 88 | 89 | final case class SimpleSubscriptionSettings( 90 | cp: StreamingConnectionProvider, 91 | subjects: Seq[String], 92 | subscriptionQueue: String, 93 | durableSubscriptionName: Option[String], 94 | startPosition: DeliveryStartPosition, 95 | subMaxInFlight: Option[Int], 96 | bufferSize: Int, 97 | autoRequeueTimeout: Option[Duration], 98 | manualAcks: Boolean, 99 | closeConnectionAfterStop: Boolean 100 | ) extends NatsStreamingSubscriptionSettings 101 | 102 | case object SimpleSubscriptionSettings{ 103 | def fromConfig(config: Config, connectionProvider: StreamingConnectionProvider): SimpleSubscriptionSettings = 104 | SimpleSubscriptionSettings( 105 | cp = connectionProvider, 106 | subjects = Try(config.getStringList("subjects").asScala) 107 | .orElse(Try(config.getString("subjects").split(',').map(_.trim).toSeq)) 108 | .orElse(Try(config.getStringList("subject").asScala)) 109 | .getOrElse(config.getString("subject").split(',').map(_.trim).toSeq), 110 | subscriptionQueue = config.getString("subscription-queue"), 111 | durableSubscriptionName = Try(config.getString("durable-subscription-name")).toOption, 112 | startPosition = Try(DeliveryStartPosition.fromConfig(config)).getOrElse(DeliveryStartPosition.OnlyNew), 113 | subMaxInFlight = Try(config.getInt("subscription-max-in-flight")).toOption, 114 | bufferSize = Try(config.getInt("buffer-size")).getOrElse(NatsStreamingSubscriptionSettings.defaultBufferSize), 115 | autoRequeueTimeout = Try(config.getDuration("auto-requeue-timeout")).toOption, 116 | manualAcks = Try(config.getBoolean("manual-acks")).getOrElse(true), 117 | closeConnectionAfterStop = Try(config.getBoolean("close-connection-after-stop")).getOrElse(true) 118 | ) 119 | def fromConfig(config: Config): SimpleSubscriptionSettings = fromConfig(config, NatsStreamingConnectionBuilder.fromConfig(config)) 120 | } 121 | 122 | final case class SubscriptionWithAckSettings( 123 | cp: StreamingConnectionProvider, 124 | subjects: Seq[String], 125 | subscriptionQueue: String, 126 | durableSubscriptionName: Option[String], 127 | startPosition: DeliveryStartPosition, 128 | subMaxInFlight: Option[Int], 129 | manualAckTimeout: Duration, 130 | autoRequeueTimeout: Option[Duration], 131 | bufferSize: Int, 132 | manualAcks: Boolean, 133 | closeConnectionAfterStop: Boolean 134 | ) extends NatsStreamingSubscriptionSettings 135 | 136 | case object SubscriptionWithAckSettings{ 137 | def fromConfig(config: Config, connectionProvider: StreamingConnectionProvider): SubscriptionWithAckSettings = { 138 | val simple = SimpleSubscriptionSettings.fromConfig(config, connectionProvider) 139 | SubscriptionWithAckSettings( 140 | simple.cp, 141 | simple.subjects, 142 | simple.subscriptionQueue, 143 | simple.durableSubscriptionName, 144 | simple.startPosition, 145 | simple.subMaxInFlight, 146 | config.getDuration("manual-ack-timeout"), 147 | Some(config.getDuration("auto-requeue-timeout")), 148 | simple.bufferSize, 149 | manualAcks = true, 150 | simple.closeConnectionAfterStop 151 | ) 152 | } 153 | def fromConfig(config: Config): SubscriptionWithAckSettings = 154 | fromConfig(config, NatsStreamingConnectionBuilder.fromConfig(config)) 155 | } 156 | 157 | final case class PublishingSettings( 158 | cp: StreamingConnectionProvider, 159 | defaultSubject: String, 160 | parallel: Boolean, 161 | closeConnectionAfterStop: Boolean 162 | ) 163 | 164 | case object PublishingSettings{ 165 | def fromConfig(config: Config): PublishingSettings = 166 | PublishingSettings( 167 | cp = NatsStreamingConnectionBuilder.fromConfig(config), 168 | defaultSubject = config.getString("default-subject"), 169 | parallel = config.getBoolean("parallel"), 170 | closeConnectionAfterStop = Try(config.getBoolean("close-connection-after-stop")).getOrElse(true) 171 | ) 172 | } 173 | --------------------------------------------------------------------------------