├── project ├── plugins.sbt └── build.properties ├── src └── main │ ├── scala │ └── com │ │ └── experiments │ │ └── integration │ │ ├── domain │ │ └── JokeEvent.scala │ │ ├── serialization │ │ ├── JokeEventJsonProtocol.scala │ │ └── JokeJsonProtocol.scala │ │ ├── ServerMain.scala │ │ ├── rest │ │ └── Routes.scala │ │ └── actors │ │ ├── JokeFetcher.scala │ │ └── JokePublisher.scala │ └── resources │ ├── application.conf │ └── logback.groovy ├── .gitignore └── README.md /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 0.13.12 -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/domain/JokeEvent.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.domain 2 | 3 | case class JokeEvent(id: Int, message: String) 4 | 5 | 6 | -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | loglevel = INFO 3 | stdout-loglevel = INFO 4 | loggers = ["akka.event.slf4j.Slf4jLogger"] 5 | } 6 | 7 | app { 8 | port = 9000 9 | // use environment variable provided it's there 10 | port = ${?PORT} 11 | 12 | host = "localhost" 13 | host = ${?HOST} 14 | } -------------------------------------------------------------------------------- /src/main/resources/logback.groovy: -------------------------------------------------------------------------------- 1 | import ch.qos.logback.core.*; 2 | import ch.qos.logback.classic.encoder.PatternLayoutEncoder; 3 | 4 | appender(name="CONSOLE", clazz=ConsoleAppender) { 5 | encoder(PatternLayoutEncoder) { 6 | pattern = "date=[%date{ISO8601}] level=[%level] [%X{akkaSource}]: message=%msg\n" 7 | } 8 | } 9 | 10 | root(level=DEBUG, appenderNames=["CONSOLE"]) -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/serialization/JokeEventJsonProtocol.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.serialization 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import com.experiments.integration.domain.JokeEvent 5 | import spray.json.DefaultJsonProtocol 6 | 7 | object JokeEventJsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { 8 | implicit val jokeEventFormat = jsonFormat2(JokeEvent) 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/serialization/JokeJsonProtocol.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.serialization 2 | 3 | import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport 4 | import com.experiments.integration.actors.JokeFetcher.{Joke, Result} 5 | import spray.json.DefaultJsonProtocol 6 | 7 | object JokeJsonProtocol extends SprayJsonSupport with DefaultJsonProtocol { 8 | implicit val jokeFormat = jsonFormat3(Joke) 9 | // remapping JSON field names to Scala (note: only JSON fields can be seen here) 10 | // JSON -> Scala 11 | // type -> response 12 | // value -> joke 13 | implicit val resultFormat = jsonFormat(Result, "type", "value") 14 | } 15 | -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/ServerMain.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration 2 | 3 | import akka.actor.ActorSystem 4 | import akka.http.scaladsl.Http 5 | import akka.stream.ActorMaterializer 6 | import com.experiments.integration.actors.JokeFetcher 7 | import com.experiments.integration.rest.Routes 8 | import com.typesafe.config.ConfigFactory 9 | 10 | object ServerMain extends App with Routes { 11 | val config = ConfigFactory.load() 12 | implicit val actorSystem = ActorSystem(name = "jokes-actor-system", config) 13 | implicit val streamMaterializer = ActorMaterializer() 14 | implicit val executionContext = actorSystem.dispatcher 15 | val log = actorSystem.log 16 | 17 | val host = config.getString("app.host") 18 | val port = config.getInt("app.port") 19 | 20 | actorSystem.actorOf(JokeFetcher.props, "joke-fetcher") 21 | 22 | val bindingFuture = Http().bindAndHandle(routes, host, port) 23 | bindingFuture.map(_.localAddress).map(addr => s"Bound to $addr").foreach(log.info) 24 | } 25 | -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/rest/Routes.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.rest 2 | 3 | import akka.http.scaladsl.server.Directives._ 4 | import akka.stream.scaladsl.Source 5 | import com.experiments.integration.actors.JokePublisher 6 | import com.experiments.integration.domain.JokeEvent 7 | import com.experiments.integration.serialization.JokeEventJsonProtocol._ 8 | import de.heikoseeberger.akkasse.ServerSentEvent 9 | import spray.json.JsonWriter 10 | import de.heikoseeberger.akkasse.EventStreamMarshalling._ // Needed for marshalling 11 | import ch.megard.akka.http.cors.CorsDirectives._ // Needed for CORS 12 | 13 | trait Routes { 14 | val routes = cors() { 15 | streamingJokes 16 | } 17 | 18 | private def wrapWithServerSentEvent[T](element: T)(implicit writer: JsonWriter[T]): ServerSentEvent = 19 | ServerSentEvent(data = writer.write(element).compactPrint, eventType = "jsonJoke") 20 | 21 | def streamingJokes = 22 | path("streaming-jokes") { 23 | get { 24 | val source = Source.actorPublisher[JokeEvent](JokePublisher.props) 25 | // long notation is used to pass in implicit JSON marshaller 26 | .map(j => wrapWithServerSentEvent(j)) 27 | 28 | complete(source) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/intellij,scala 3 | 4 | ### Intellij ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff: 9 | .idea/workspace.xml 10 | .idea/tasks.xml 11 | .idea/dictionaries 12 | .idea/vcs.xml 13 | .idea/jsLibraryMappings.xml 14 | 15 | # Sensitive or high-churn files: 16 | .idea/dataSources.ids 17 | .idea/dataSources.xml 18 | .idea/dataSources.local.xml 19 | .idea/sqlDataSources.xml 20 | .idea/dynamic.xml 21 | .idea/uiDesigner.xml 22 | 23 | # Gradle: 24 | .idea/gradle.xml 25 | .idea/libraries 26 | 27 | # Mongo Explorer plugin: 28 | .idea/mongoSettings.xml 29 | 30 | ## File-based project format: 31 | *.iws 32 | 33 | ## Plugin-specific files: 34 | 35 | # IntelliJ 36 | /out/ 37 | 38 | # mpeltonen/sbt-idea plugin 39 | .idea_modules/ 40 | 41 | # JIRA plugin 42 | atlassian-ide-plugin.xml 43 | 44 | # Crashlytics plugin (for Android Studio and IntelliJ) 45 | com_crashlytics_export_strings.xml 46 | crashlytics.properties 47 | crashlytics-build.properties 48 | fabric.properties 49 | 50 | ### Intellij Patch ### 51 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 52 | 53 | # *.iml 54 | # modules.xml 55 | # .idea/misc.xml 56 | # *.ipr 57 | 58 | 59 | ### Scala ### 60 | *.class 61 | *.log 62 | 63 | # sbt specific 64 | .cache 65 | .history 66 | .lib/ 67 | dist/* 68 | target/ 69 | lib_managed/ 70 | src_managed/ 71 | project/boot/ 72 | project/plugins/project/ 73 | 74 | # Scala-IDE specific 75 | .scala_dependencies 76 | .worksheet 77 | 78 | # ENSIME specific 79 | .ensime_cache/ 80 | .ensime -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/actors/JokeFetcher.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.actors 2 | 3 | import akka.actor.{Actor, ActorLogging, Cancellable, Props, Status} 4 | import akka.http.scaladsl.Http 5 | import akka.http.scaladsl.model.HttpMethods._ 6 | import akka.http.scaladsl.model.HttpRequest 7 | import akka.http.scaladsl.model.StatusCodes._ 8 | import akka.http.scaladsl.unmarshalling.Unmarshal 9 | import akka.pattern.pipe 10 | import akka.stream.ActorMaterializer 11 | import akka.util.Timeout 12 | import com.experiments.integration.actors.JokeFetcher.{FetchJoke, Result} 13 | import com.experiments.integration.domain.JokeEvent 14 | import com.experiments.integration.serialization.JokeJsonProtocol._ 15 | 16 | import scala.concurrent.Future 17 | import scala.concurrent.duration._ 18 | import scala.language.postfixOps 19 | 20 | class JokeFetcher extends Actor with ActorLogging { 21 | implicit val timeout = Timeout(5 seconds) 22 | implicit val materializer = ActorMaterializer()(context.system) 23 | implicit val ec = context.system.dispatcher 24 | 25 | val http = Http(context.system) 26 | val url = "http://api.icndb.com/jokes/random?escape=javascript" 27 | var cancellable: Option[Cancellable] = None 28 | 29 | override def preStart(): Unit = { 30 | log.debug("Starting up Joke Fetcher") 31 | cancellable = Some( 32 | context.system.scheduler.schedule(initialDelay = 1 second, interval = 1 second, receiver = self, FetchJoke) 33 | ) 34 | } 35 | 36 | override def postStop(): Unit = 37 | cancellable.foreach(c => c.cancel()) 38 | 39 | override def receive: Receive = { 40 | case FetchJoke => 41 | val futureResponse = http.singleRequest(HttpRequest(GET, url)) flatMap { 42 | httpResponse => 43 | httpResponse.status match { 44 | case OK => 45 | val futureResult = Unmarshal(httpResponse).to[Result] 46 | futureResult.map(x => Some(x)) 47 | case _ => 48 | log.error("Non 200 OK response code, error obtaining jokes") 49 | Future.successful(None) 50 | } 51 | } 52 | futureResponse pipeTo self 53 | 54 | case Status.Failure(throwable) => 55 | log.error("Could not obtain jokes", throwable) 56 | 57 | case Some(Result(_, joke)) => 58 | context.system.eventStream.publish(JokeEvent(joke.id, joke.joke)) 59 | } 60 | } 61 | 62 | object JokeFetcher { 63 | 64 | sealed trait Command 65 | 66 | case object FetchJoke extends Command 67 | 68 | // Type safe HTTP response 69 | case class Joke(id: Int, joke: String, categories: List[String]) 70 | 71 | case class Result(response: String, joke: Joke) 72 | 73 | def props = Props[JokeFetcher] 74 | } 75 | -------------------------------------------------------------------------------- /src/main/scala/com/experiments/integration/actors/JokePublisher.scala: -------------------------------------------------------------------------------- 1 | package com.experiments.integration.actors 2 | 3 | import akka.actor.{ActorLogging, Props} 4 | import akka.stream.actor.ActorPublisher 5 | import akka.stream.actor.ActorPublisherMessage._ 6 | import com.experiments.integration.domain.JokeEvent 7 | 8 | import scala.annotation.tailrec 9 | 10 | /** 11 | * This class is the integration point between Akka Actors and Akka Streams 12 | * This actor is instantiated for every request that requires a Streaming response 13 | * This lives for the duration of the Stream lifetime and is terminated after 14 | * 15 | * This actor subscribes to the Event Stream to obtain Joke Events and publishes those 16 | * Joke Events into the Stream 17 | */ 18 | class JokePublisher extends ActorPublisher[JokeEvent] with ActorLogging { 19 | val MaxBufferSize = 100 20 | var buffer = Vector.empty[JokeEvent] 21 | 22 | override def preStart(): Unit = { 23 | log.debug("Joke Publisher created, subscribing to Event Stream") 24 | context.system.eventStream.subscribe(self, classOf[JokeEvent]) 25 | } 26 | 27 | override def postStop(): Unit = { 28 | log.debug("Joke Publisher stopping, un-subscribing to Event Stream") 29 | context.system.eventStream.unsubscribe(self) 30 | } 31 | 32 | @tailrec 33 | private def deliverBuffer(): Unit = 34 | if (totalDemand > 0 && isActive) { 35 | // You are allowed to send as many elements as have been requested by the stream subscriber 36 | // total demand is a Long and can be larger than what the buffer has 37 | if (totalDemand <= Int.MaxValue) { 38 | val (sendDownstream, holdOn) = buffer.splitAt(totalDemand.toInt) 39 | buffer = holdOn 40 | // send the stuff downstream 41 | sendDownstream.foreach(onNext) 42 | } else { 43 | val (sendDownStream, holdOn) = buffer.splitAt(Int.MaxValue) 44 | buffer = holdOn 45 | sendDownStream.foreach(onNext) 46 | // recursive call checks whether is more demand before repeating the process 47 | deliverBuffer() 48 | } 49 | } 50 | 51 | override def receive: Receive = { 52 | case j: JokeEvent if buffer.size == MaxBufferSize => 53 | log.warning("Buffer is full, ignoring incoming JokeEvent") 54 | 55 | case j: JokeEvent => 56 | // send elements to the stream immediately since there is demand from downstream and we 57 | // have not buffered anything so why bother buffering, send immediately 58 | // You send elements to the stream by calling onNext 59 | if (buffer.isEmpty && totalDemand > 0L && isActive) onNext(j) 60 | // there is no demand from downstream so we will store the result in our buffer 61 | // Note that :+= means add to end of Vector 62 | // this allows us to respect backpressure 63 | else buffer :+= j 64 | 65 | case Cancel => 66 | log.info("Stream cancelled") 67 | // Note: postStop is automatically invoked 68 | context.stop(self) 69 | 70 | // A request from downstream to send more data 71 | // When the stream subscriber requests more elements the ActorPublisherMessage.Request message is 72 | // delivered to this actor, and you can act on that event. The totalDemand is updated automatically. 73 | case Request(_) => 74 | deliverBuffer() 75 | 76 | case other => 77 | log.warning(s"Unknown message $other received") 78 | } 79 | } 80 | 81 | object JokePublisher { 82 | def props = Props[JokePublisher] 83 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Akka HTTP + Akka Streams + Akka Actors Integration Example 2 | This application is able to respond with a Streaming JSON response of 3 | Chuck Norris jokes taken from [here](http://www.icndb.com/api/) 4 | 5 | ### Purpose 6 | The purpose of this application was to help familiarize myself (and 7 | anyone interested) with how Akka Streams can integrate with Akka Actors 8 | and Akka HTTP. Also, I was bored and looking for something fun 9 | to do :smile:. 10 | 11 | ### Problem Statement 12 | The Internet Chuck Norris Database returns a discrete JSON response for 13 | every HTTP request sent out. Whilst this is okay for most cases, it 14 | becomes a bit cumbersome if you want to look at Chuck Norris jokes 15 | as you have to keep hitting the endpoint yourself. The goal of this 16 | project is to turn those discretized JSON responses into a continuous 17 | JSON stream. This is done with the help of Akka HTTP, Akka Actors and 18 | Akka Streams. 19 | 20 | ### Overview 21 | #### The JokeFetcher Actor 22 | The `JokeFetcher` Akka Actor is used to continuously poll the ICNDB 23 | endpoint behind the scenes and publish each discrete JSON response onto 24 | the Event Stream as a JokeEvent. 25 | 26 | #### The JokePublisher Actor (the integration point with Akka Streams) 27 | The `JokePublisher` Akka Actor is a point of integration between Akka 28 | Actors and Akka Streams. The `JokePublisher` is created whenever a 29 | request to the `/streaming-jokes` endpoint which in turn creates an Akka 30 | Stream to stream the response back to the requester. It lives for the 31 | duration of that Stream and gets terminated once the Stream completes 32 | (does not happen in this case because it is an infinite Stream) or when 33 | the Stream is cancelled (happens when the requester does not want 34 | anymore data). As you can see the JokePublisher is tied to the duration 35 | of each Stream. This means if 6 users come in and request data from 36 | `/streaming-jokes` then 6 actors of JokePublisher will be created, each 37 | of them will be responsible for publishing data into that specific 38 | user's streaming response. The `JokePublisher` on creation, will 39 | subscribe to the Akka EventStream and listen for `JokeEvent`s and send 40 | them downstream to provide streaming responses. As you can see, 41 | publishing messages to the `JokePublisher` Actor will end up in the 42 | Akka Stream which is why this is a point of integration between Akka 43 | Actors and Akka Streams when it comes to publishing data into the Akka 44 | Stream. This actor respects backpressure. It will only send information 45 | as fast as the downstream consumer can consume. It will drop messages if 46 | they are coming in too quickly. 47 | 48 | #### Endpoints 49 | Users can hit the `/streaming-jokes` route in order to get back an [SSE](http://www.html5rocks.com/en/tutorials/eventsource/basics/) 50 | streaming JSON response of Chuck Norris jokes in JSON format. 51 | 52 | ### Pre-requisites: 53 | - Scala 2.11.8 54 | - [SBT](http://www.scala-sbt.org/) 55 | 56 | ### Instructions: 57 | - `sbt run` to start the application 58 | - Visit `localhost:9000/streaming-jokes` to see the Streaming response 59 | 60 | ### Consumption of Streaming SSE JSON with a JavaScript client 61 | Here is an example of how to consume the Stream using JavaScript 62 | ```javascript 63 | 64 | 65 | 66 | 67 | Consuming Streaming JSON 68 | 69 | 70 | 71 | 79 | 80 | ``` 81 | The above example will print out the SSE Events received from the server 82 | in the browser. You may need to use the [CORS Chrome Addon](https://chrome.google.com/webstore/detail/allow-control-allow-origi/nlfbmbojpeacfghkpbjhddihlkkiljbi/related) 83 | 84 | ##### Preview: 85 | ![streaming](https://cloud.githubusercontent.com/assets/14280155/18819716/ba70748c-8363-11e6-9d17-68c17999d068.gif) 86 | 87 | ## Credits: 88 | - [Akka](http://akka.io) 89 | - Heiko Seeberger's [Akka SSE extension](https://github.com/hseeberger/akka-sse) 90 | - Lomig Megard's [Akka HTTP CORS extension](https://github.com/lomigmegard/akka-http-cors) 91 | --------------------------------------------------------------------------------