├── .bsp └── sbt.json ├── .gitignore ├── .idea ├── codeStyles │ ├── Project.xml │ └── codeStyleConfig.xml ├── hydra.xml ├── misc.xml ├── modules.xml ├── modules │ ├── akka-streams-build.iml │ └── akka-streams.iml ├── sbt.xml ├── scala_compiler.xml └── vcs.xml ├── README.md ├── build.sbt ├── project └── build.properties └── src └── main ├── resources └── application.conf └── scala ├── part1_recap ├── AkkaRecap.scala └── ScalaRecap.scala ├── part2_primer ├── BackpressureBasics.scala ├── FirstPrinciples.scala ├── MaterializingStreams.scala └── OperatorFusion.scala ├── part3_graphs ├── BidirectionalFlows.scala ├── GraphBasics.scala ├── GraphCycles.scala ├── GraphMaterializedValues.scala ├── MoreOpenGraphs.scala └── OpenGraphs.scala ├── part4_techniques ├── AdvancedBackpressure.scala ├── FaultTolerance.scala ├── IntegratingWithActors.scala ├── IntegratingWithExternalServices.scala └── TestingStreamsSpec.scala ├── part5_advanced ├── CustomGraphShapes.scala ├── CustomOperators.scala ├── DynamicStreamHandling.scala └── Substreams.scala └── playground └── Playground.scala /.bsp/sbt.json: -------------------------------------------------------------------------------- 1 | {"name":"sbt","version":"1.9.4","bspVersion":"2.1.0-M1","languages":["scala"],"argv":["/Users/daniel/Library/Java/JavaVirtualMachines/temurin-17.0.9/Contents/Home/bin/java","-Xms100m","-Xmx100m","-classpath","/Users/daniel/Library/Application Support/JetBrains/IdeaIC2023.3/plugins/Scala/launcher/sbt-launch.jar","-Dsbt.script=/Users/daniel/Library/Application%20Support/Coursier/bin/sbt","xsbt.boot.Boot","-bsp"]} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.gitignore.io/api/sbt,java,scala,intellij 3 | # Edit at https://www.gitignore.io/?templates=sbt,java,scala,intellij 4 | 5 | ### Intellij ### 6 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 7 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 8 | 9 | # User-specific stuff 10 | .idea/**/workspace.xml 11 | .idea/**/tasks.xml 12 | .idea/**/usage.statistics.xml 13 | .idea/**/dictionaries 14 | .idea/**/shelf 15 | 16 | # Generated files 17 | .idea/**/contentModel.xml 18 | 19 | # Sensitive or high-churn files 20 | .idea/**/dataSources/ 21 | .idea/**/dataSources.ids 22 | .idea/**/dataSources.local.xml 23 | .idea/**/sqlDataSources.xml 24 | .idea/**/dynamic.xml 25 | .idea/**/uiDesigner.xml 26 | .idea/**/dbnavigator.xml 27 | 28 | # Gradle 29 | .idea/**/gradle.xml 30 | .idea/**/libraries 31 | 32 | # Gradle and Maven with auto-import 33 | # When using Gradle or Maven with auto-import, you should exclude module files, 34 | # since they will be recreated, and may cause churn. Uncomment if using 35 | # auto-import. 36 | # .idea/modules.xml 37 | # .idea/*.iml 38 | # .idea/modules 39 | 40 | # CMake 41 | cmake-build-*/ 42 | 43 | # Mongo Explorer plugin 44 | .idea/**/mongoSettings.xml 45 | 46 | # File-based project format 47 | *.iws 48 | 49 | # IntelliJ 50 | out/ 51 | 52 | # mpeltonen/sbt-idea plugin 53 | .idea_modules/ 54 | 55 | # JIRA plugin 56 | atlassian-ide-plugin.xml 57 | 58 | # Cursive Clojure plugin 59 | .idea/replstate.xml 60 | 61 | # Crashlytics plugin (for Android Studio and IntelliJ) 62 | com_crashlytics_export_strings.xml 63 | crashlytics.properties 64 | crashlytics-build.properties 65 | fabric.properties 66 | 67 | # Editor-based Rest Client 68 | .idea/httpRequests 69 | 70 | # Android studio 3.1+ serialized cache file 71 | .idea/caches/build_file_checksums.ser 72 | 73 | ### Intellij Patch ### 74 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 75 | 76 | # *.iml 77 | # modules.xml 78 | # .idea/misc.xml 79 | # *.ipr 80 | 81 | # Sonarlint plugin 82 | .idea/sonarlint 83 | 84 | ### Java ### 85 | # Compiled class file 86 | *.class 87 | 88 | # Log file 89 | *.log 90 | 91 | # BlueJ files 92 | *.ctxt 93 | 94 | # Mobile Tools for Java (J2ME) 95 | .mtj.tmp/ 96 | 97 | # Package Files # 98 | *.jar 99 | *.war 100 | *.nar 101 | *.ear 102 | *.zip 103 | *.tar.gz 104 | *.rar 105 | 106 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 107 | hs_err_pid* 108 | 109 | ### SBT ### 110 | # Simple Build Tool 111 | # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control 112 | 113 | dist/* 114 | target/ 115 | lib_managed/ 116 | src_managed/ 117 | project/boot/ 118 | project/plugins/project/ 119 | .history 120 | .cache 121 | .lib/ 122 | 123 | ### Scala ### 124 | 125 | # End of https://www.gitignore.io/api/sbt,java,scala,intellij 126 | -------------------------------------------------------------------------------- /.idea/codeStyles/Project.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | 10 | 11 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /.idea/codeStyles/codeStyleConfig.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | -------------------------------------------------------------------------------- /.idea/hydra.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 9 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/modules/akka-streams-build.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 115 | -------------------------------------------------------------------------------- /.idea/modules/akka-streams.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /.idea/sbt.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 18 | 19 | -------------------------------------------------------------------------------- /.idea/scala_compiler.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # The official repository for the Rock the JVM Akka Streams with Scala course 2 | 3 | **(for the Udemy version, click [here](https://github.com/rockthejvm/udemy-akka-streams))** 4 | 5 | This repository contains the code we wrote during [Rock the JVM's Akka Streams with Scala](https://rockthejvm.com/course/akka-streams) course. Unless explicitly mentioned, the code in this repository is exactly what was caught on camera. 6 | 7 | ### How to install 8 | - either clone the repo or download as zip 9 | - open with IntelliJ as an SBT project 10 | 11 | No need to do anything else, as the IDE will take care to download and apply the appropriate library dependencies. 12 | 13 | ### For questions or suggestions 14 | 15 | If you have changes to suggest to this repo, either 16 | - submit a GitHub issue 17 | - tell me in the course Q/A forum 18 | - submit a pull request! 19 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "akka-streams" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.13.14" 6 | 7 | lazy val akkaVersion = "2.6.19" 8 | lazy val scalaTestVersion = "3.2.7" 9 | 10 | libraryDependencies ++= Seq( 11 | "com.typesafe.akka" %% "akka-stream" % akkaVersion, 12 | "com.typesafe.akka" %% "akka-stream-testkit" % akkaVersion, 13 | "com.typesafe.akka" %% "akka-testkit" % akkaVersion, 14 | "org.scalatest" %% "scalatest" % scalaTestVersion 15 | ) 16 | 17 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.9.9 2 | -------------------------------------------------------------------------------- /src/main/resources/application.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | loglevel = "DEBUG" 3 | } 4 | 5 | dedicated-dispatcher { 6 | type = Dispatcher 7 | executor = "thread-pool-executor" 8 | thread-pool-executor { 9 | fixed-pool-size = 5 10 | } 11 | } -------------------------------------------------------------------------------- /src/main/scala/part1_recap/AkkaRecap.scala: -------------------------------------------------------------------------------- 1 | package part1_recap 2 | 3 | import akka.actor.SupervisorStrategy.{Restart, Stop} 4 | import akka.actor.{Actor, ActorLogging, ActorSystem, OneForOneStrategy, PoisonPill, Props, Stash, SupervisorStrategy} 5 | import akka.util.Timeout 6 | 7 | object AkkaRecap extends App { 8 | 9 | class SimpleActor extends Actor with ActorLogging with Stash { 10 | override def receive: Receive = { 11 | case "createChild" => 12 | val childActor = context.actorOf(Props[SimpleActor], "myChild") 13 | childActor ! "hello" 14 | case "stashThis" => 15 | stash() 16 | case "change handler NOW" => 17 | unstashAll() 18 | context.become(anotherHandler) 19 | 20 | case "change" => context.become(anotherHandler) 21 | case message => println(s"I received: $message") 22 | } 23 | 24 | def anotherHandler: Receive = { 25 | case message => println(s"In another receive handler: $message") 26 | } 27 | 28 | override def preStart(): Unit = { 29 | log.info("I'm starting") 30 | } 31 | 32 | override def supervisorStrategy: SupervisorStrategy = OneForOneStrategy() { 33 | case _: RuntimeException => Restart 34 | case _ => Stop 35 | } 36 | } 37 | 38 | // actor encapsulation 39 | val system = ActorSystem("AkkaRecap") 40 | // #1: you can only instantiate an actor through the actor system 41 | val actor = system.actorOf(Props[SimpleActor], "simpleActor") 42 | // #2: sending messages 43 | actor ! "hello" 44 | /* 45 | - messages are sent asynchronously 46 | - many actors (in the millions) can share a few dozen threads 47 | - each message is processed/handled ATOMICALLY 48 | - no need for locks 49 | */ 50 | 51 | // changing actor behavior + stashing 52 | // actors can spawn other actors 53 | // guardians: /system, /user, / = root guardian 54 | 55 | // actors have a defined lifecycle: they can be started, stopped, suspended, resumed, restarted 56 | 57 | // stopping actors - context.stop 58 | actor ! PoisonPill 59 | 60 | // logging 61 | // supervision 62 | 63 | // configure Akka infrastructure: dispatchers, routers, mailboxes 64 | 65 | // schedulers 66 | import scala.concurrent.duration._ 67 | import system.dispatcher 68 | system.scheduler.scheduleOnce(2.seconds) { 69 | actor ! "delayed happy birthday!" 70 | } 71 | 72 | // Akka patterns including FSM + ask pattern 73 | import akka.pattern.ask 74 | implicit val timeout: Timeout = Timeout(3.seconds) 75 | 76 | val future = actor ? "question" 77 | 78 | // the pipe pattern 79 | import akka.pattern.pipe 80 | val anotherActor = system.actorOf(Props[SimpleActor], "anotherSimpleActor") 81 | future.mapTo[String].pipeTo(anotherActor) 82 | } 83 | -------------------------------------------------------------------------------- /src/main/scala/part1_recap/ScalaRecap.scala: -------------------------------------------------------------------------------- 1 | package part1_recap 2 | 3 | import scala.concurrent.Future 4 | import scala.util.{Failure, Success} 5 | 6 | object ScalaRecap extends App { 7 | 8 | val aCondition: Boolean = false 9 | def myFunction(x: Int) = { 10 | // code 11 | if (x > 4) 42 else 65 12 | } 13 | // instructions vs expressions 14 | // types + type inference 15 | 16 | // OO features of Scala 17 | class Animal 18 | trait Carnivore { 19 | def eat(a: Animal): Unit 20 | } 21 | 22 | object Carnivore 23 | 24 | // generics 25 | abstract class MyList[+A] 26 | 27 | // method notations 28 | 1 + 2 // infix notation 29 | 1.+(2) 30 | 31 | // FP 32 | val anIncrementer: Int => Int = (x: Int) => x + 1 33 | anIncrementer(1) 34 | 35 | List(1,2,3).map(anIncrementer) 36 | // HOF: flatMap, filter 37 | // for-comprehensions 38 | 39 | // Monads: Option, Try 40 | 41 | // Pattern matching! 42 | val unknown: Any = 2 43 | val order = unknown match { 44 | case 1 => "first" 45 | case 2 => "second" 46 | case _ => "unknown" 47 | } 48 | 49 | try { 50 | // code that can throw an exception 51 | throw new RuntimeException 52 | } catch { 53 | case e: Exception => println("I caught one!") 54 | } 55 | 56 | /** 57 | * Scala advanced 58 | */ 59 | 60 | // multithreading 61 | 62 | import scala.concurrent.ExecutionContext.Implicits.global 63 | val future = Future { 64 | // long computation here 65 | // executed on SOME other thread 66 | 42 67 | } 68 | // map, flatMap, filter + other niceties e.g. recover/recoverWith 69 | 70 | future.onComplete { 71 | case Success(value) => println(s"I found the meaning of life: $value") 72 | case Failure(exception) => println(s"I found $exception while searching for the meaning of life!") 73 | } // on SOME thread 74 | 75 | val partialFunction: PartialFunction[Int, Int] = { 76 | case 1 => 42 77 | case 2 => 65 78 | case _ => 999 79 | } 80 | // based on pattern matching! 81 | 82 | // type aliases 83 | type AkkaReceive = PartialFunction[Any, Unit] 84 | def receive: AkkaReceive = { 85 | case 1 => println("hello!") 86 | case _ => println("confused...") 87 | } 88 | 89 | // Implicits! 90 | implicit val timeout = 3000 91 | def setTimeout(f: () => Unit)(implicit timeout: Int) = f() 92 | 93 | setTimeout(() => println("timeout"))// other arg list injected by the compiler 94 | 95 | // conversions 96 | // 1) implicit methods 97 | case class Person(name: String) { 98 | def greet: String = s"Hi, my name is $name" 99 | } 100 | 101 | implicit def fromStringToPerson(name: String) = Person(name) 102 | "Peter".greet 103 | // fromStringToPerson("Peter").greet 104 | 105 | // 2) implicit classes 106 | implicit class Dog(name: String) { 107 | def bark = println("Bark!") 108 | } 109 | "Lassie".bark 110 | // new Dog("Lassie").bark 111 | 112 | // implicit organizations 113 | // local scope 114 | implicit val numberOrdering: Ordering[Int] = Ordering.fromLessThan(_ > _) 115 | List(1,2,3).sorted //(numberOrdering) => List(3,2,1) 116 | 117 | // imported scope 118 | 119 | // companion objects of the types involved in the call 120 | object Person { 121 | implicit val personOrdering: Ordering[Person] = Ordering.fromLessThan((a, b) => a.name.compareTo(b.name) < 0) 122 | } 123 | 124 | List(Person("Bob"), Person("Alice")).sorted // (Person.personOrdering) 125 | // => List(Person("Alice"), Person("Bob")) 126 | 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/main/scala/part2_primer/BackpressureBasics.scala: -------------------------------------------------------------------------------- 1 | package part2_primer 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.{ActorMaterializer, OverflowStrategy} 5 | import akka.stream.scaladsl.{Flow, Sink, Source} 6 | 7 | object BackpressureBasics extends App { 8 | 9 | implicit val system = ActorSystem("BackpressureBasics") 10 | // this line needs to be here for Akka < 2.6 11 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 12 | 13 | val fastSource = Source(1 to 1000) 14 | val slowSink = Sink.foreach[Int] { x => 15 | // simulate a long processing 16 | Thread.sleep(1000) 17 | println(s"Sink: $x") 18 | } 19 | 20 | // fastSource.to(slowSink).run() // fusing?! 21 | // not backpressure 22 | 23 | // fastSource.async.to(slowSink).run() 24 | // backpressure 25 | 26 | val simpleFlow = Flow[Int].map { x => 27 | println(s"Incoming: $x") 28 | x + 1 29 | } 30 | 31 | fastSource.async 32 | .via(simpleFlow).async 33 | .to(slowSink) 34 | // .run() 35 | 36 | /* 37 | reactions to backpressure (in order): 38 | - try to slow down if possible 39 | - buffer elements until there's more demand 40 | - drop down elements from the buffer if it overflows 41 | - tear down/kill the whole stream (failure) 42 | */ 43 | 44 | val bufferedFlow = simpleFlow.buffer(10, overflowStrategy = OverflowStrategy.dropHead) 45 | fastSource.async 46 | .via(bufferedFlow).async 47 | .to(slowSink) 48 | .run() 49 | 50 | /* 51 | 1-16: nobody is backpressured 52 | 17-26: flow will buffer, flow will start dropping at the next element 53 | 26-1000: flow will always drop the oldest element 54 | => 991-1000 => 992 - 1001 => sink 55 | */ 56 | 57 | /* 58 | overflow strategies: 59 | - drop head = oldest 60 | - drop tail = newest 61 | - drop new = exact element to be added = keeps the buffer 62 | - drop the entire buffer 63 | - backpressure signal 64 | - fail 65 | */ 66 | 67 | // throttling 68 | import scala.concurrent.duration._ 69 | fastSource.throttle(10, 1.second).runWith(Sink.foreach(println)) 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/part2_primer/FirstPrinciples.scala: -------------------------------------------------------------------------------- 1 | package part2_primer 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.ActorMaterializer 5 | import akka.stream.scaladsl.{Flow, Sink, Source} 6 | 7 | import scala.concurrent.Future 8 | 9 | object FirstPrinciples extends App { 10 | 11 | implicit val system = ActorSystem("FirstPrinciples") 12 | // this line needs to be here for Akka < 2.6 13 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 14 | 15 | // sources 16 | val source = Source(1 to 10) 17 | // sinks 18 | val sink = Sink.foreach[Int](println) 19 | 20 | val graph = source.to(sink) 21 | // graph.run() 22 | 23 | // flows transform elements 24 | val flow = Flow[Int].map(x => x + 1) 25 | val sourceWithFlow = source.via(flow) 26 | val flowWithSink = flow.to(sink) 27 | 28 | // sourceWithFlow.to(sink).run() 29 | // source.to(flowWithSink).run() 30 | // source.via(flow).to(sink).run() 31 | 32 | // nulls are NOT allowed 33 | // val illegalSource = Source.single[String](null) 34 | // illegalSource.to(Sink.foreach(println)).run() 35 | // use Options instead 36 | 37 | // various kinds of sources 38 | val finiteSource = Source.single(1) 39 | val anotherFiniteSource = Source(List(1, 2, 3)) 40 | val emptySource = Source.empty[Int] 41 | val infiniteSource = Source(Stream.from(1)) // do not confuse an Akka stream with a "collection" Stream 42 | import scala.concurrent.ExecutionContext.Implicits.global 43 | val futureSource = Source.fromFuture(Future(42)) 44 | 45 | // sinks 46 | val theMostBoringSink = Sink.ignore 47 | val foreachSink = Sink.foreach[String](println) 48 | val headSink = Sink.head[Int] // retrieves head and then closes the stream 49 | val foldSink = Sink.fold[Int, Int](0)((a, b) => a + b) 50 | 51 | // flows - usually mapped to collection operators 52 | val mapFlow = Flow[Int].map(x => 2 * x) 53 | val takeFlow = Flow[Int].take(5) 54 | // drop, filter 55 | // NOT have flatMap 56 | 57 | // source -> flow -> flow -> ... -> sink 58 | val doubleFlowGraph = source.via(mapFlow).via(takeFlow).to(sink) 59 | // doubleFlowGraph.run() 60 | 61 | // syntactic sugars 62 | val mapSource = Source(1 to 10).map(x => x * 2) // Source(1 to 10).via(Flow[Int].map(x => x * 2)) 63 | // run streams directly 64 | // mapSource.runForeach(println) // mapSource.to(Sink.foreach[Int](println)).run() 65 | 66 | // OPERATORS = components 67 | 68 | /** 69 | * Exercise: create a stream that takes the names of persons, then you will keep the first 2 names with length > 5 characters. 70 | * 71 | */ 72 | val names = List("Alice", "Bob", "Charlie", "David", "Martin", "AkkaStreams") 73 | val nameSource = Source(names) 74 | val longNameFlow = Flow[String].filter(name => name.length > 5) 75 | val limitFlow = Flow[String].take(2) 76 | val nameSink = Sink.foreach[String](println) 77 | 78 | nameSource.via(longNameFlow).via(limitFlow).to(nameSink).run() 79 | nameSource.filter(_.length > 5).take(2).runForeach(println) 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/scala/part2_primer/MaterializingStreams.scala: -------------------------------------------------------------------------------- 1 | package part2_primer 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.ActorMaterializer 5 | import akka.stream.scaladsl.{Flow, Keep, Sink, Source} 6 | 7 | import scala.util.{Failure, Success} 8 | 9 | object MaterializingStreams extends App { 10 | 11 | implicit val system = ActorSystem("MaterializingStreams") 12 | // this line needs to be here for Akka < 2.6 13 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 14 | import system.dispatcher 15 | 16 | val simpleGraph = Source(1 to 10).to(Sink.foreach(println)) 17 | // val simpleMaterializedValue = simpleGraph.run() 18 | 19 | val source = Source(1 to 10) 20 | val sink = Sink.reduce[Int]((a, b) => a + b) 21 | // val sumFuture = source.runWith(sink) 22 | // sumFuture.onComplete { 23 | // case Success(value) => println(s"The sum of all elements is :$value") 24 | // case Failure(ex) => println(s"The sum of the elements could not be computed: $ex") 25 | // } 26 | 27 | // choosing materialized values 28 | val simpleSource = Source(1 to 10) 29 | val simpleFlow = Flow[Int].map(x => x + 1) 30 | val simpleSink = Sink.foreach[Int](println) 31 | val graph = simpleSource.viaMat(simpleFlow)(Keep.right).toMat(simpleSink)(Keep.right) 32 | graph.run().onComplete { 33 | case Success(_) => println("Stream processing finished.") 34 | case Failure(exception) => println(s"Stream processing failed with: $exception") 35 | } 36 | 37 | // sugars 38 | Source(1 to 10).runWith(Sink.reduce[Int](_ + _)) // source.to(Sink.reduce)(Keep.right) 39 | Source(1 to 10).runReduce[Int](_ + _) // same 40 | 41 | // backwards 42 | Sink.foreach[Int](println).runWith(Source.single(42)) // source(..).to(sink...).run() 43 | // both ways 44 | Flow[Int].map(x => 2 * x).runWith(simpleSource, simpleSink) 45 | 46 | /** 47 | * - return the last element out of a source (use Sink.last) 48 | * - compute the total word count out of a stream of sentences 49 | * - map, fold, reduce 50 | */ 51 | val f1 = Source(1 to 10).toMat(Sink.last)(Keep.right).run() 52 | val f2 = Source(1 to 10).runWith(Sink.last) 53 | 54 | val sentenceSource = Source(List( 55 | "Akka is awesome", 56 | "I love streams", 57 | "Materialized values are killing me" 58 | )) 59 | val wordCountSink = Sink.fold[Int, String](0)((currentWords, newSentence) => currentWords + newSentence.split(" ").length) 60 | val g1 = sentenceSource.toMat(wordCountSink)(Keep.right).run() 61 | val g2 = sentenceSource.runWith(wordCountSink) 62 | val g3 = sentenceSource.runFold(0)((currentWords, newSentence) => currentWords + newSentence.split(" ").length) 63 | 64 | val wordCountFlow = Flow[String].fold[Int](0)((currentWords, newSentence) => currentWords + newSentence.split(" ").length) 65 | val g4 = sentenceSource.via(wordCountFlow).toMat(Sink.head)(Keep.right).run() 66 | val g5 = sentenceSource.viaMat(wordCountFlow)(Keep.left).toMat(Sink.head)(Keep.right).run() 67 | val g6 = sentenceSource.via(wordCountFlow).runWith(Sink.head) 68 | val g7 = wordCountFlow.runWith(sentenceSource, Sink.head)._2 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/scala/part2_primer/OperatorFusion.scala: -------------------------------------------------------------------------------- 1 | package part2_primer 2 | 3 | import akka.actor.{Actor, ActorSystem, Props} 4 | import akka.stream.ActorMaterializer 5 | import akka.stream.scaladsl.{Flow, Sink, Source} 6 | 7 | object OperatorFusion extends App { 8 | 9 | implicit val system = ActorSystem("OperatorFusion") 10 | // this line needs to be here for Akka < 2.6 11 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 12 | 13 | val simpleSource = Source(1 to 1000) 14 | val simpleFlow = Flow[Int].map(_ + 1) 15 | val simpleFlow2 = Flow[Int].map(_ * 10) 16 | val simpleSink = Sink.foreach[Int](println) 17 | 18 | // this runs on the SAME ACTOR 19 | // simpleSource.via(simpleFlow).via(simpleFlow2).to(simpleSink).run() 20 | // operator/component FUSION 21 | 22 | // "equivalent" behavior 23 | class SimpleActor extends Actor { 24 | override def receive: Receive = { 25 | case x: Int => 26 | // flow operations 27 | val x2 = x + 1 28 | val y = x2 * 10 29 | // sink operation 30 | println(y) 31 | } 32 | } 33 | val simpleActor = system.actorOf(Props[SimpleActor]) 34 | // (1 to 1000).foreach(simpleActor ! _) 35 | 36 | // complex flows: 37 | val complexFlow = Flow[Int].map { x => 38 | // simulating a long computation 39 | Thread.sleep(1000) 40 | x + 1 41 | } 42 | val complexFlow2 = Flow[Int].map { x => 43 | // simulating a long computation 44 | Thread.sleep(1000) 45 | x * 10 46 | } 47 | 48 | // simpleSource.via(complexFlow).via(complexFlow2).to(simpleSink).run() 49 | 50 | // async boundary 51 | // simpleSource.via(complexFlow).async // runs on one actor 52 | // .via(complexFlow2).async // runs on another actor 53 | // .to(simpleSink) // runs on a third actor 54 | // .run() 55 | 56 | // ordering guarantees 57 | Source(1 to 3) 58 | .map(element => { println(s"Flow A: $element"); element }).async 59 | .map(element => { println(s"Flow B: $element"); element }).async 60 | .map(element => { println(s"Flow C: $element"); element }).async 61 | .runWith(Sink.ignore) 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/BidirectionalFlows.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.{ActorMaterializer, BidiShape, ClosedShape} 5 | import akka.stream.scaladsl.{Flow, GraphDSL, RunnableGraph, Sink, Source} 6 | 7 | object BidirectionalFlows extends App { 8 | 9 | implicit val system = ActorSystem("BidirectionalFlows") 10 | // this line needs to be here for Akka < 2.6 11 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 12 | 13 | /* 14 | Example: cryptography 15 | */ 16 | def encrypt(n: Int)(string: String) = string.map(c => (c + n).toChar) 17 | def decrypt(n: Int)(string: String) = string.map(c => (c - n).toChar) 18 | 19 | // bidiFlow 20 | val bidiCryptoStaticGraph = GraphDSL.create() { implicit builder => 21 | val encryptionFlowShape = builder.add(Flow[String].map(encrypt(3))) 22 | val decryptionFlowShape = builder.add(Flow[String].map(decrypt(3))) 23 | 24 | // BidiShape(encryptionFlowShape.in, encryptionFlowShape.out, decryptionFlowShape.in, decryptionFlowShape.out) 25 | BidiShape.fromFlows(encryptionFlowShape, decryptionFlowShape) 26 | } 27 | 28 | val unencryptedStrings = List("akka", "is", "awesome", "testing", "bidirectional", "flows") 29 | val unencryptedSource = Source(unencryptedStrings) 30 | val encryptedSource = Source(unencryptedStrings.map(encrypt(3))) 31 | 32 | val cryptoBidiGraph = RunnableGraph.fromGraph( 33 | GraphDSL.create() { implicit builder => 34 | import GraphDSL.Implicits._ 35 | 36 | val unencryptedSourceShape = builder.add(unencryptedSource) 37 | val encryptedSourceShape = builder.add(encryptedSource) 38 | val bidi = builder.add(bidiCryptoStaticGraph) 39 | val encryptedSinkShape = builder.add(Sink.foreach[String](string => println(s"Encrypted: $string"))) 40 | val decryptedSinkShape = builder.add(Sink.foreach[String](string => println(s"Decrypted: $string"))) 41 | 42 | unencryptedSourceShape ~> bidi.in1 ; bidi.out1 ~> encryptedSinkShape 43 | decryptedSinkShape <~ bidi.out2 ; bidi.in2 <~ encryptedSourceShape 44 | 45 | ClosedShape 46 | } 47 | ) 48 | 49 | cryptoBidiGraph.run() 50 | 51 | /* 52 | - encrypting/decrypting 53 | - encoding/decoding 54 | - serializing/deserializing 55 | */ 56 | } 57 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/GraphBasics.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import akka.NotUsed 4 | import akka.actor.ActorSystem 5 | import akka.stream.{ActorMaterializer, ClosedShape} 6 | import akka.stream.scaladsl.{Balance, Broadcast, Flow, GraphDSL, Merge, RunnableGraph, Sink, Source, Zip} 7 | 8 | object GraphBasics extends App { 9 | 10 | implicit val system = ActorSystem("GraphBasics") 11 | // this line needs to be here for Akka < 2.6 12 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 13 | 14 | val input = Source(1 to 1000) 15 | val incrementer = Flow[Int].map(x => x + 1) // hard computation 16 | val multiplier = Flow[Int].map(x => x * 10) // hard computation 17 | val output = Sink.foreach[(Int, Int)](println) 18 | 19 | // step 1 - setting up the fundamentals for the graph 20 | val graph = RunnableGraph.fromGraph( 21 | GraphDSL.create() { implicit builder: GraphDSL.Builder[NotUsed] => // builder = MUTABLE data structure 22 | import GraphDSL.Implicits._ // brings some nice operators into scope 23 | 24 | // step 2 - add the necessary components of this graph 25 | val broadcast = builder.add(Broadcast[Int](2)) // fan-out operator 26 | val zip = builder.add(Zip[Int, Int]) // fan-in operator 27 | 28 | // step 3 - tying up the components 29 | input ~> broadcast 30 | 31 | broadcast.out(0) ~> incrementer ~> zip.in0 32 | broadcast.out(1) ~> multiplier ~> zip.in1 33 | 34 | zip.out ~> output 35 | 36 | // step 4 - return a closed shape 37 | ClosedShape // FREEZE the builder's shape 38 | // shape 39 | } // graph 40 | ) // runnable graph 41 | 42 | // graph.run() // run the graph and materialize it 43 | 44 | /** 45 | * exercise 1: feed a source into 2 sinks at the same time (hint: use a broadcast) 46 | */ 47 | 48 | val firstSink = Sink.foreach[Int](x => println(s"First sink: $x")) 49 | val secondSink = Sink.foreach[Int](x => println(s"Second sink: $x")) 50 | 51 | // step 1 52 | val sourceToTwoSinksGraph = RunnableGraph.fromGraph( 53 | GraphDSL.create() { implicit builder => 54 | import GraphDSL.Implicits._ 55 | 56 | // step 2 - declaring the components 57 | val broadcast = builder.add(Broadcast[Int](2)) 58 | 59 | // step 3 - tying up the components 60 | input ~> broadcast ~> firstSink // implicit port numbering 61 | broadcast ~> secondSink 62 | // broadcast.out(0) ~> firstSink 63 | // broadcast.out(1) ~> secondSink 64 | 65 | // step 4 66 | ClosedShape 67 | } 68 | ) 69 | 70 | /** 71 | * exercise 2: balance 72 | */ 73 | 74 | import scala.concurrent.duration._ 75 | val fastSource = input.throttle(5, 1.second) 76 | val slowSource = input.throttle(2, 1.second) 77 | 78 | val sink1 = Sink.fold[Int, Int](0)((count, _) => { 79 | println(s"Sink 1 number of elements: $count") 80 | count + 1 81 | }) 82 | 83 | val sink2 = Sink.fold[Int, Int](0)((count, _) => { 84 | println(s"Sink 2 number of elements: $count") 85 | count + 1 86 | }) 87 | 88 | // step 1 89 | val balanceGraph = RunnableGraph.fromGraph( 90 | GraphDSL.create() { implicit builder => 91 | import GraphDSL.Implicits._ 92 | 93 | 94 | // step 2 -- declare components 95 | val merge = builder.add(Merge[Int](2)) 96 | val balance = builder.add(Balance[Int](2)) 97 | 98 | 99 | // step 3 -- tie them up 100 | fastSource ~> merge ~> balance ~> sink1 101 | slowSource ~> merge 102 | balance ~> sink2 103 | 104 | // step 4 105 | ClosedShape 106 | } 107 | ) 108 | 109 | balanceGraph.run() 110 | 111 | 112 | } 113 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/GraphCycles.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Merge, MergePreferred, RunnableGraph, Sink, Source, Zip, ZipWith} 5 | import akka.stream.{ActorMaterializer, ClosedShape, OverflowStrategy, UniformFanInShape} 6 | 7 | object GraphCycles extends App { 8 | 9 | implicit val system = ActorSystem("GraphCycles") 10 | // this line needs to be here for Akka < 2.6 11 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 12 | 13 | val accelerator = GraphDSL.create() { implicit builder => 14 | import GraphDSL.Implicits._ 15 | 16 | val sourceShape = builder.add(Source(1 to 100)) 17 | val mergeShape = builder.add(Merge[Int](2)) 18 | val incrementerShape = builder.add(Flow[Int].map { x => 19 | println(s"Accelerating $x") 20 | x + 1 21 | }) 22 | 23 | sourceShape ~> mergeShape ~> incrementerShape 24 | mergeShape <~ incrementerShape 25 | 26 | ClosedShape 27 | } 28 | 29 | // RunnableGraph.fromGraph(accelerator).run() 30 | // graph cycle deadlock! 31 | 32 | /* 33 | Solution 1: MergePreferred 34 | */ 35 | val actualAccelerator = GraphDSL.create() { implicit builder => 36 | import GraphDSL.Implicits._ 37 | 38 | val sourceShape = builder.add(Source(1 to 100)) 39 | val mergeShape = builder.add(MergePreferred[Int](1)) 40 | val incrementerShape = builder.add(Flow[Int].map { x => 41 | println(s"Accelerating $x") 42 | x + 1 43 | }) 44 | 45 | sourceShape ~> mergeShape ~> incrementerShape 46 | mergeShape.preferred <~ incrementerShape 47 | 48 | ClosedShape 49 | } 50 | 51 | // RunnableGraph.fromGraph(actualAccelerator).run() 52 | 53 | 54 | /* 55 | Solution 2: buffers 56 | */ 57 | val bufferedRepeater = GraphDSL.create() { implicit builder => 58 | import GraphDSL.Implicits._ 59 | 60 | val sourceShape = builder.add(Source(1 to 100)) 61 | val mergeShape = builder.add(Merge[Int](2)) 62 | val repeaterShape = builder.add(Flow[Int].buffer(10, OverflowStrategy.dropHead).map { x => 63 | println(s"Accelerating $x") 64 | Thread.sleep(100) 65 | x 66 | }) 67 | 68 | sourceShape ~> mergeShape ~> repeaterShape 69 | mergeShape <~ repeaterShape 70 | 71 | ClosedShape 72 | } 73 | 74 | // RunnableGraph.fromGraph(bufferedRepeater).run() 75 | 76 | /* 77 | cycles risk deadlocking 78 | - add bounds to the number of elements in the cycle 79 | 80 | boundedness vs liveness 81 | */ 82 | 83 | /** 84 | * Challenge: create a fan-in shape 85 | * - two inputs which will be fed with EXACTLY ONE number (1 and 1) 86 | * - output will emit an INFINITE FIBONACCI SEQUENCE based off those 2 numbers 87 | * 1, 2, 3, 5, 8 ... 88 | * 89 | * Hint: Use ZipWith and cycles, MergePreferred 90 | */ 91 | 92 | val fibonacciGenerator = GraphDSL.create() { implicit builder => 93 | import GraphDSL.Implicits._ 94 | 95 | val zip = builder.add(Zip[BigInt, BigInt]) 96 | val mergePreferred = builder.add(MergePreferred[(BigInt, BigInt)](1)) 97 | val fiboLogic = builder.add(Flow[(BigInt, BigInt)].map { pair => 98 | val last = pair._1 99 | val previous = pair._2 100 | 101 | Thread.sleep(100) 102 | 103 | (last + previous, last) 104 | }) 105 | val broadcast = builder.add(Broadcast[(BigInt, BigInt)](2)) 106 | val extractLast = builder.add(Flow[(BigInt, BigInt)].map(_._1)) 107 | 108 | zip.out ~> mergePreferred ~> fiboLogic ~> broadcast ~> extractLast 109 | mergePreferred.preferred <~ broadcast 110 | 111 | UniformFanInShape(extractLast.out, zip.in0, zip.in1) 112 | } 113 | 114 | // val fiboGraph = RunnableGraph.fromGraph( 115 | // GraphDSL.create() { implicit builder => 116 | // import GraphDSL.Implicits._ 117 | // 118 | // val source1 = builder.add(Source.single[BigInt](1)) 119 | // val source2 = builder.add(Source.single[BigInt](1)) 120 | // val sink = builder.add(Sink.foreach[BigInt](println)) 121 | // val fibo = builder.add(fibonacciGenerator) 122 | // 123 | // source1 ~> fibo.in(0) 124 | // source2 ~> fibo.in(1) 125 | // fibo.out ~> sink 126 | // 127 | // ClosedShape 128 | // } 129 | // ) 130 | // 131 | // fiboGraph.run() 132 | 133 | /** 134 | * ******************** 135 | * Old solution (more complicated) 136 | * ******************** 137 | */ 138 | 139 | val complicatedFibonacciGenerator = GraphDSL.create() { implicit builder => 140 | import GraphDSL.Implicits._ 141 | 142 | // two big feeds: one with the "last" number, and one with the "next-to-last" (previous) Fibonacci number 143 | val lastFeed = builder.add(MergePreferred[BigInt](1)) 144 | val previousFeed = builder.add(MergePreferred[BigInt](1)) 145 | 146 | /* 147 | The "last" feed will be split into 3: 148 | - the final output of the shape 149 | - a zip to sum with the previousFeed 150 | - a feedback loop to the previousFeed (the current "last" will become the next "previous") 151 | */ 152 | val broadcastLast = builder.add(Broadcast[BigInt](3)) 153 | 154 | // the actual Fibonacci logic 155 | val fiboLogic = builder.add(ZipWith((last: BigInt, previous: BigInt) => { 156 | Thread.sleep(100) // so you can actually see the result growing 157 | last + previous 158 | })) 159 | 160 | // hopefully connections are traceable on paper 161 | 162 | broadcastLast ~> previousFeed.preferred // feedback loop: current "last" becomes next "previous" 163 | lastFeed ~> broadcastLast ~> fiboLogic.in0 164 | previousFeed ~> fiboLogic.in1 165 | lastFeed.preferred <~ fiboLogic.out // feedback loop: next "last" is the sum of current "last" and "previous" 166 | 167 | UniformFanInShape( 168 | broadcastLast.out(2), // the unconnected output 169 | lastFeed.in(0), 170 | previousFeed.in(0) // and the regular ports of the MergePreferred components 171 | ) 172 | 173 | // So as you can see, quite involved. But it gives the same output! 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/GraphMaterializedValues.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.{ActorMaterializer, FlowShape, SinkShape} 5 | import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, Keep, Sink, Source} 6 | 7 | import scala.concurrent.Future 8 | import scala.util.{Failure, Success} 9 | 10 | object GraphMaterializedValues extends App { 11 | 12 | implicit val system = ActorSystem("GraphMaterializedValues") 13 | // this line needs to be here for Akka < 2.6 14 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 15 | 16 | val wordSource = Source(List("Akka", "is", "awesome", "rock", "the", "jvm")) 17 | val printer = Sink.foreach[String](println) 18 | val counter = Sink.fold[Int, String](0)((count, _) => count + 1) 19 | 20 | /* 21 | A composite component (sink) 22 | - prints out all strings which are lowercase 23 | - COUNTS the strings that are short (< 5 chars) 24 | */ 25 | 26 | // step 1 27 | val complexWordSink = Sink.fromGraph( 28 | GraphDSL.create(printer, counter)((printerMatValue, counterMatValue) => counterMatValue) { implicit builder => (printerShape, counterShape) => 29 | import GraphDSL.Implicits._ 30 | 31 | // step 2 - SHAPES 32 | val broadcast = builder.add(Broadcast[String](2)) 33 | val lowercaseFilter = builder.add(Flow[String].filter(word => word == word.toLowerCase)) 34 | val shortStringFilter = builder.add(Flow[String].filter(_.length < 5)) 35 | 36 | // step 3 - connections 37 | broadcast ~> lowercaseFilter ~> printerShape 38 | broadcast ~> shortStringFilter ~> counterShape 39 | 40 | // step 4 - the shape 41 | SinkShape(broadcast.in) 42 | } 43 | ) 44 | 45 | import system.dispatcher 46 | val shortStringsCountFuture = wordSource.toMat(complexWordSink)(Keep.right).run() 47 | shortStringsCountFuture.onComplete { 48 | case Success(count) => println(s"The total number of short strings is: $count") 49 | case Failure(exception) => println(s"The count of short strings failed: $exception") 50 | } 51 | 52 | /** 53 | * Exercise - enhance a flow to return a materialized value 54 | */ 55 | def enhanceFlow[A, B](flow: Flow[A, B, _]): Flow[A, B, Future[Int]] = { 56 | val counterSink = Sink.fold[Int, B](0)((count, _) => count + 1) 57 | 58 | Flow.fromGraph( 59 | GraphDSL.create(counterSink) { implicit builder => counterSinkShape => 60 | import GraphDSL.Implicits._ 61 | 62 | val broadcast = builder.add(Broadcast[B](2)) 63 | val originalFlowShape = builder.add(flow) 64 | 65 | originalFlowShape ~> broadcast ~> counterSinkShape 66 | 67 | FlowShape(originalFlowShape.in, broadcast.out(1)) 68 | } 69 | ) 70 | } 71 | 72 | val simpleSource = Source(1 to 42) 73 | val simpleFlow = Flow[Int].map(x => x) 74 | val simpleSink = Sink.ignore 75 | 76 | val enhancedFlowCountFuture = simpleSource.viaMat(enhanceFlow(simpleFlow))(Keep.right).toMat(simpleSink)(Keep.left).run() 77 | enhancedFlowCountFuture.onComplete { 78 | case Success(count) => println(s"$count elements went through the enhanced flow") 79 | case _ => println("Something failed") 80 | } 81 | 82 | 83 | /* 84 | Hint: use a broadcast and a Sink.fold 85 | */ 86 | } 87 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/MoreOpenGraphs.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import java.util.Date 4 | 5 | import akka.actor.ActorSystem 6 | import akka.stream.{ActorMaterializer, ClosedShape, FanOutShape2, UniformFanInShape} 7 | import akka.stream.scaladsl.{Broadcast, Flow, GraphDSL, RunnableGraph, Sink, Source, ZipWith} 8 | 9 | object MoreOpenGraphs extends App { 10 | 11 | implicit val system = ActorSystem("MoreOpenGraphs") 12 | // this line needs to be here for Akka < 2.6 13 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 14 | 15 | /* 16 | Example: Max3 operator 17 | - 3 inputs of type int 18 | - the maximum of the 3 19 | */ 20 | 21 | // step 1 22 | val max3StaticGraph = GraphDSL.create() { implicit builder => 23 | import GraphDSL.Implicits._ 24 | 25 | // step 2 - define aux SHAPES 26 | val max1 = builder.add(ZipWith[Int, Int, Int]((a, b) => Math.max(a, b))) 27 | val max2 = builder.add(ZipWith[Int, Int, Int]((a, b) => Math.max(a, b))) 28 | 29 | // step 3 30 | max1.out ~> max2.in0 31 | 32 | // step 4 33 | 34 | UniformFanInShape(max2.out, max1.in0, max1.in1, max2.in1) 35 | } 36 | 37 | val source1 = Source(1 to 10) 38 | val source2 = Source((1 to 10).map(_ => 5)) 39 | val source3 = Source((1 to 10).reverse) 40 | 41 | val maxSink = Sink.foreach[Int](x => println(s"Max is: $x")) 42 | 43 | // step 1 44 | val max3RunnableGraph = RunnableGraph.fromGraph( 45 | GraphDSL.create() { implicit builder => 46 | import GraphDSL.Implicits._ 47 | 48 | // step 2 - declare SHAPES 49 | val max3Shape = builder.add(max3StaticGraph) 50 | 51 | // step 3 - tie 52 | source1 ~> max3Shape.in(0) 53 | source2 ~> max3Shape.in(1) 54 | source3 ~> max3Shape.in(2) 55 | max3Shape.out ~> maxSink 56 | 57 | // step 4 58 | ClosedShape 59 | } 60 | ) 61 | 62 | // max3RunnableGraph.run() 63 | 64 | // same for UniformFanOutShape 65 | 66 | /* 67 | Non-uniform fan out shape 68 | 69 | Processing bank transactions 70 | Txn suspicious if amount > 10000 71 | 72 | Streams component for txns 73 | - output1: let the transaction go through 74 | - output2: suspicious txn ids 75 | */ 76 | 77 | case class Transaction(id: String, source: String, recipient: String, amount: Int, date: Date) 78 | 79 | val transactionSource = Source(List( 80 | Transaction("5273890572", "Paul", "Jim", 100, new Date), 81 | Transaction("3578902532", "Daniel", "Jim", 100000, new Date), 82 | Transaction("5489036033", "Jim", "Alice", 7000, new Date) 83 | )) 84 | 85 | val bankProcessor = Sink.foreach[Transaction](println) 86 | val suspiciousAnalysisService = Sink.foreach[String](txnId => println(s"Suspicious transaction ID: $txnId")) 87 | 88 | // step 1 89 | val suspiciousTxnStaticGraph = GraphDSL.create() { implicit builder => 90 | import GraphDSL.Implicits._ 91 | 92 | // step 2 - define SHAPES 93 | val broadcast = builder.add(Broadcast[Transaction](2)) 94 | val suspiciousTxnFilter = builder.add(Flow[Transaction].filter(txn => txn.amount > 10000)) 95 | val txnIdExtractor = builder.add(Flow[Transaction].map[String](txn => txn.id)) 96 | 97 | // step 3 - tie SHAPES 98 | broadcast.out(0) ~> suspiciousTxnFilter ~> txnIdExtractor 99 | 100 | // step 4 101 | new FanOutShape2(broadcast.in, broadcast.out(1), txnIdExtractor.out) 102 | } 103 | 104 | // step 1 105 | val suspiciousTxnRunnableGraph = RunnableGraph.fromGraph( 106 | GraphDSL.create() { implicit builder => 107 | import GraphDSL.Implicits._ 108 | 109 | // step 2 110 | val suspiciousTxnShape = builder.add(suspiciousTxnStaticGraph) 111 | 112 | // step 3 113 | transactionSource ~> suspiciousTxnShape.in 114 | suspiciousTxnShape.out0 ~> bankProcessor 115 | suspiciousTxnShape.out1 ~> suspiciousAnalysisService 116 | 117 | // step 4 118 | ClosedShape 119 | } 120 | ) 121 | 122 | suspiciousTxnRunnableGraph.run() 123 | } 124 | -------------------------------------------------------------------------------- /src/main/scala/part3_graphs/OpenGraphs.scala: -------------------------------------------------------------------------------- 1 | package part3_graphs 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream._ 5 | import akka.stream.scaladsl.{Broadcast, Concat, Flow, GraphDSL, RunnableGraph, Sink, Source} 6 | 7 | object OpenGraphs extends App { 8 | 9 | implicit val system = ActorSystem("OpenGraphs") 10 | // this line needs to be here for Akka < 2.6 11 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 12 | 13 | 14 | /* 15 | A composite source that concatenates 2 sources 16 | - emits ALL the elements from the first source 17 | - then ALL the elements from the second 18 | */ 19 | 20 | val firstSource = Source(1 to 10) 21 | val secondSource = Source(42 to 1000) 22 | 23 | // step 1 24 | val sourceGraph = Source.fromGraph( 25 | GraphDSL.create() { implicit builder => 26 | import GraphDSL.Implicits._ 27 | 28 | // step 2: declaring components 29 | val concat = builder.add(Concat[Int](2)) 30 | 31 | // step 3: tying them together 32 | firstSource ~> concat 33 | secondSource ~> concat 34 | 35 | // step 4 36 | SourceShape(concat.out) 37 | } 38 | ) 39 | 40 | // sourceGraph.to(Sink.foreach(println)).run() 41 | 42 | 43 | /* 44 | Complex sink 45 | */ 46 | val sink1 = Sink.foreach[Int](x => println(s"Meaningful thing 1: $x")) 47 | val sink2 = Sink.foreach[Int](x => println(s"Meaningful thing 2: $x")) 48 | 49 | // step 1 50 | val sinkGraph = Sink.fromGraph( 51 | GraphDSL.create() { implicit builder => 52 | import GraphDSL.Implicits._ 53 | 54 | // step 2 - add a broadcast 55 | val broadcast = builder.add(Broadcast[Int](2)) 56 | 57 | // step 3 - tie components together 58 | broadcast ~> sink1 59 | broadcast ~> sink2 60 | 61 | // step 4 - return the shape 62 | SinkShape(broadcast.in) 63 | } 64 | ) 65 | 66 | // firstSource.to(sinkGraph).run() 67 | 68 | /** 69 | * Challenge - complex flow? 70 | * Write your own flow that's composed of two other flows 71 | * - one that adds 1 to a number 72 | * - one that does number * 10 73 | */ 74 | val incrementer = Flow[Int].map(_ + 1) 75 | val multiplier = Flow[Int].map(_ * 10) 76 | 77 | // step 1 78 | val flowGraph = Flow.fromGraph( 79 | GraphDSL.create() { implicit builder => 80 | import GraphDSL.Implicits._ 81 | 82 | // everything operates on SHAPES 83 | 84 | // step 2 - define auxiliary SHAPES 85 | val incrementerShape = builder.add(incrementer) 86 | val multiplierShape = builder.add(multiplier) 87 | 88 | // step 3 - connect the SHAPES 89 | incrementerShape ~> multiplierShape 90 | 91 | FlowShape(incrementerShape.in, multiplierShape.out) // SHAPE 92 | } // static graph 93 | ) // component 94 | 95 | firstSource.via(flowGraph).to(Sink.foreach(println)).run() 96 | 97 | /** 98 | Exercise: flow from a sink and a source? 99 | */ 100 | def fromSinkAndSource[A, B](sink: Sink[A, _], source: Source[B, _]): Flow[A, B, _] = 101 | // step 1 102 | Flow.fromGraph( 103 | GraphDSL.create() { implicit builder => 104 | // step 2: declare the SHAPES 105 | val sourceShape = builder.add(source) 106 | val sinkShape = builder.add(sink) 107 | 108 | // step 3 109 | // step 4 - return the shape 110 | FlowShape(sinkShape.in, sourceShape.out) 111 | } 112 | ) 113 | 114 | val f = Flow.fromSinkAndSourceCoupled(Sink.foreach[String](println), Source(1 to 10)) 115 | 116 | } 117 | -------------------------------------------------------------------------------- /src/main/scala/part4_techniques/AdvancedBackpressure.scala: -------------------------------------------------------------------------------- 1 | package part4_techniques 2 | 3 | import java.util.Date 4 | 5 | import akka.actor.ActorSystem 6 | import akka.stream.{ActorMaterializer, OverflowStrategy} 7 | import akka.stream.scaladsl.{Flow, Sink, Source} 8 | 9 | object AdvancedBackpressure extends App { 10 | 11 | implicit val system = ActorSystem("AdvancedBackpressure") 12 | // this line needs to be here for Akka < 2.6 13 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 14 | 15 | // control backpressure 16 | val controlledFlow = Flow[Int].map(_ * 2).buffer(10, OverflowStrategy.dropHead) 17 | 18 | case class PagerEvent(description: String, date: Date, nInstances: Int = 1) 19 | case class Notification(email: String, pagerEvent: PagerEvent) 20 | 21 | val events = List( 22 | PagerEvent("Service discovery failed", new Date), 23 | PagerEvent("Illegal elements in the data pipeline", new Date), 24 | PagerEvent("Number of HTTP 500 spiked", new Date), 25 | PagerEvent("A service stopped responding", new Date) 26 | ) 27 | val eventSource = Source(events) 28 | 29 | val oncallEngineer = "daniel@rockthejvm.com" // a fast service for fetching oncall emails 30 | 31 | def sendEmail(notification: Notification) = 32 | println(s"Dear ${notification.email}, you have an event: ${notification.pagerEvent}") // actually send an email 33 | 34 | val notificationSink = Flow[PagerEvent].map(event => Notification(oncallEngineer, event)) 35 | .to(Sink.foreach[Notification](sendEmail)) 36 | 37 | // standard 38 | // eventSource.to(notificationSink).run() 39 | 40 | /* 41 | un-backpressurable source 42 | */ 43 | 44 | def sendEmailSlow(notification: Notification) = { 45 | Thread.sleep(1000) 46 | println(s"Dear ${notification.email}, you have an event: ${notification.pagerEvent}") // actually send an email 47 | } 48 | 49 | val aggregateNotificationFlow = Flow[PagerEvent] 50 | .conflate((event1, event2) =>{ 51 | val nInstances = event1.nInstances + event2.nInstances 52 | PagerEvent(s"You have $nInstances events that require your attention", new Date, nInstances) 53 | }) 54 | .map(resultingEvent => Notification(oncallEngineer, resultingEvent)) 55 | 56 | // eventSource.via(aggregateNotificationFlow).async.to(Sink.foreach[Notification](sendEmailSlow)).run() 57 | // alternative to backpressure 58 | 59 | /* 60 | Slow producers: extrapolate/expand 61 | */ 62 | import scala.concurrent.duration._ 63 | val slowCounter = Source(Stream.from(1)).throttle(1, 1.second) 64 | val hungrySink = Sink.foreach[Int](println) 65 | 66 | val extrapolator = Flow[Int].extrapolate(element => Iterator.from(element)) 67 | val repeater = Flow[Int].extrapolate(element => Iterator.continually(element)) 68 | 69 | slowCounter.via(repeater).to(hungrySink).run() 70 | 71 | val expander = Flow[Int].expand(element => Iterator.from(element)) 72 | } 73 | -------------------------------------------------------------------------------- /src/main/scala/part4_techniques/FaultTolerance.scala: -------------------------------------------------------------------------------- 1 | package part4_techniques 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.Supervision.{Resume, Stop} 5 | import akka.stream.{ActorAttributes, ActorMaterializer} 6 | import akka.stream.scaladsl.{RestartSource, Sink, Source} 7 | 8 | import scala.concurrent.duration._ 9 | import scala.util.Random 10 | 11 | object FaultTolerance extends App { 12 | 13 | implicit val system = ActorSystem("FaultTolerance") 14 | // this line needs to be here for Akka < 2.6 15 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 16 | 17 | // 1 - logging 18 | val faultySource = Source(1 to 10).map(e => if (e == 6) throw new RuntimeException else e) 19 | faultySource.log("trackingElements").to(Sink.ignore) 20 | // .run() 21 | 22 | // 2 - gracefully terminating a stream 23 | faultySource.recover { 24 | case _: RuntimeException => Int.MinValue 25 | } .log("gracefulSource") 26 | .to(Sink.ignore) 27 | // .run() 28 | 29 | // 3 - recover with another stream 30 | faultySource.recoverWithRetries(3, { 31 | case _: RuntimeException => Source(90 to 99) 32 | }) 33 | .log("recoverWithRetries") 34 | .to(Sink.ignore) 35 | // .run() 36 | 37 | 38 | 39 | // 4 - backoff supervision 40 | val restartSource = RestartSource.onFailuresWithBackoff( 41 | minBackoff = 1.seconds, 42 | maxBackoff = 30.seconds, 43 | randomFactor = 0.2, 44 | )(() => { 45 | val randomNumber = new Random().nextInt(20) 46 | Source(1 to 10).map(elem => if (elem == randomNumber) throw new RuntimeException else elem) 47 | }) 48 | 49 | restartSource 50 | .log("restartBackoff") 51 | .to(Sink.ignore) 52 | // .run() 53 | 54 | 55 | // 5 - supervision strategy 56 | val numbers = Source(1 to 20).map(n => if (n == 13) throw new RuntimeException("bad luck") else n).log("supervision") 57 | val supervisedNumbers = numbers.withAttributes(ActorAttributes.supervisionStrategy { 58 | /* 59 | Resume = skips the faulty element 60 | Stop = stop the stream 61 | Restart = resume + clears internal state 62 | */ 63 | case _: RuntimeException => Resume 64 | case _ => Stop 65 | }) 66 | 67 | supervisedNumbers.to(Sink.ignore).run() 68 | } 69 | -------------------------------------------------------------------------------- /src/main/scala/part4_techniques/IntegratingWithActors.scala: -------------------------------------------------------------------------------- 1 | package part4_techniques 2 | 3 | import akka.actor.{Actor, ActorLogging, ActorSystem, Props} 4 | import akka.stream.{ActorMaterializer, OverflowStrategy} 5 | import akka.stream.scaladsl.{Flow, Sink, Source} 6 | import akka.util.Timeout 7 | 8 | import scala.concurrent.duration._ 9 | 10 | object IntegratingWithActors extends App { 11 | 12 | implicit val system = ActorSystem("IntegratingWithActors") 13 | // this line needs to be here for Akka < 2.6 14 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 15 | 16 | class SimpleActor extends Actor with ActorLogging { 17 | override def receive: Receive = { 18 | case s: String => 19 | log.info(s"Just received a string: $s") 20 | sender() ! s"$s$s" 21 | case n: Int => 22 | log.info(s"Just received a number: $n") 23 | sender() ! (2 * n) 24 | case _ => 25 | } 26 | } 27 | 28 | val simpleActor = system.actorOf(Props[SimpleActor], "simpleActor") 29 | 30 | val numbersSource = Source(1 to 10) 31 | 32 | // actor as a flow 33 | implicit val timeout = Timeout(2.seconds) 34 | val actorBasedFlow = Flow[Int].ask[Int](parallelism = 4)(simpleActor) 35 | 36 | // numbersSource.via(actorBasedFlow).to(Sink.ignore).run() 37 | // numbersSource.ask[Int](parallelism = 4)(simpleActor).to(Sink.ignore).run() // equivalent 38 | 39 | /* 40 | Actor as a source 41 | */ 42 | val actorPoweredSource = Source.actorRef[Int](bufferSize = 10, overflowStrategy = OverflowStrategy.dropHead) 43 | val materializedActorRef = actorPoweredSource.to(Sink.foreach[Int](number => println(s"Actor powered flow got number: $number"))).run() 44 | materializedActorRef ! 10 45 | // terminating the stream 46 | materializedActorRef ! akka.actor.Status.Success("complete") 47 | 48 | /* 49 | Actor as a destination/sink 50 | - an init message 51 | - an ack message to confirm the reception 52 | - a complete message 53 | - a function to generate a message in case the stream throws an exception 54 | */ 55 | 56 | case object StreamInit 57 | case object StreamAck 58 | case object StreamComplete 59 | case class StreamFail(ex: Throwable) 60 | 61 | class DestinationActor extends Actor with ActorLogging { 62 | override def receive: Receive = { 63 | case StreamInit => 64 | log.info("Stream initialized") 65 | sender() ! StreamAck 66 | case StreamComplete => 67 | log.info("Stream complete") 68 | context.stop(self) 69 | case StreamFail(ex) => 70 | log.warning(s"Stream failed: $ex") 71 | case message => 72 | log.info(s"Message $message has come to its final resting point.") 73 | sender() ! StreamAck 74 | } 75 | } 76 | val destinationActor = system.actorOf(Props[DestinationActor], "destinationActor") 77 | 78 | val actorPoweredSink = Sink.actorRefWithAck[Int]( 79 | destinationActor, 80 | onInitMessage = StreamInit, 81 | onCompleteMessage = StreamComplete, 82 | ackMessage = StreamAck, 83 | onFailureMessage = throwable => StreamFail(throwable) // optional 84 | ) 85 | 86 | Source(1 to 10).to(actorPoweredSink).run() 87 | 88 | // Sink.actorRef() not recommended, unable to backpressure 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/scala/part4_techniques/IntegratingWithExternalServices.scala: -------------------------------------------------------------------------------- 1 | package part4_techniques 2 | 3 | import java.util.Date 4 | 5 | import akka.actor.{Actor, ActorLogging, ActorSystem, Props} 6 | import akka.stream.ActorMaterializer 7 | import akka.stream.scaladsl.{Sink, Source} 8 | import akka.util.Timeout 9 | 10 | import scala.concurrent.Future 11 | 12 | object IntegratingWithExternalServices extends App { 13 | 14 | implicit val system = ActorSystem("IntegratingWithExternalServices") 15 | // this line needs to be here for Akka < 2.6 16 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 17 | // import system.dispatcher // not recommended in practice for mapAsync 18 | implicit val dispatcher = system.dispatchers.lookup("dedicated-dispatcher") 19 | 20 | 21 | def genericExtService[A, B](element: A): Future[B] = ??? 22 | 23 | // example: simplified PagerDuty 24 | case class PagerEvent(application: String, description: String, date: Date) 25 | 26 | val eventSource = Source(List( 27 | PagerEvent("AkkaInfra", "Infrastructure broke", new Date), 28 | PagerEvent("FastDataPipeline", "Illegal elements in the data pipeline", new Date), 29 | PagerEvent("AkkaInfra", "A service stopped responding", new Date), 30 | PagerEvent("SuperFrontend", "A button doesn't work", new Date) 31 | )) 32 | 33 | object PagerService { 34 | private val engineers = List("Daniel", "John", "Lady Gaga") 35 | private val emails = Map( 36 | "Daniel" -> "daniel@rockthejvm.com", 37 | "John" -> "john@rockthejvm.com", 38 | "Lady Gaga" -> "ladygaga@rtjvm.com" 39 | ) 40 | 41 | def processEvent(pagerEvent: PagerEvent) = Future { 42 | val engineerIndex = (pagerEvent.date.toInstant.getEpochSecond / (24 * 3600)) % engineers.length 43 | val engineer = engineers(engineerIndex.toInt) 44 | val engineerEmail = emails(engineer) 45 | 46 | // page the engineer 47 | println(s"Sending engineer $engineerEmail a high priority notification: $pagerEvent") 48 | Thread.sleep(1000) 49 | 50 | // return the email that was paged 51 | engineerEmail 52 | } 53 | } 54 | 55 | val infraEvents = eventSource.filter(_.application == "AkkaInfra") 56 | val pagedEngineerEmails = infraEvents.mapAsync(parallelism = 1)(event => PagerService.processEvent(event)) 57 | // guarantees the relative order of elements 58 | val pagedEmailsSink = Sink.foreach[String](email => println(s"Successfully sent notification to $email")) 59 | // pagedEngineerEmails.to(pagedEmailsSink).run() 60 | 61 | class PagerActor extends Actor with ActorLogging { 62 | private val engineers = List("Daniel", "John", "Lady Gaga") 63 | private val emails = Map( 64 | "Daniel" -> "daniel@rockthejvm.com", 65 | "John" -> "john@rockthejvm.com", 66 | "Lady Gaga" -> "ladygaga@rtjvm.com" 67 | ) 68 | 69 | private def processEvent(pagerEvent: PagerEvent) = { 70 | val engineerIndex = (pagerEvent.date.toInstant.getEpochSecond / (24 * 3600)) % engineers.length 71 | val engineer = engineers(engineerIndex.toInt) 72 | val engineerEmail = emails(engineer) 73 | 74 | // page the engineer 75 | log.info(s"Sending engineer $engineerEmail a high priority notification: $pagerEvent") 76 | Thread.sleep(1000) 77 | 78 | // return the email that was paged 79 | engineerEmail 80 | } 81 | 82 | override def receive: Receive = { 83 | case pagerEvent: PagerEvent => 84 | sender() ! processEvent(pagerEvent) 85 | } 86 | } 87 | 88 | import akka.pattern.ask 89 | import scala.concurrent.duration._ 90 | implicit val timeout = Timeout(3.seconds) 91 | val pagerActor = system.actorOf(Props[PagerActor], "pagerActor") 92 | val alternativePagedEngineerEmails = infraEvents.mapAsync(parallelism = 4)(event => (pagerActor ? event).mapTo[String]) 93 | alternativePagedEngineerEmails.to(pagedEmailsSink).run() 94 | 95 | // do not confuse mapAsync with async (ASYNC boundary) 96 | } 97 | -------------------------------------------------------------------------------- /src/main/scala/part4_techniques/TestingStreamsSpec.scala: -------------------------------------------------------------------------------- 1 | package part4_techniques 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.ActorMaterializer 5 | import akka.stream.scaladsl.{Flow, Keep, Sink, Source} 6 | import akka.stream.testkit.scaladsl.{TestSink, TestSource} 7 | import akka.testkit.{TestKit, TestProbe} 8 | 9 | import scala.concurrent.Await 10 | import scala.concurrent.duration._ 11 | import scala.util.{Failure, Success} 12 | 13 | import org.scalatest.BeforeAndAfterAll 14 | import org.scalatest.wordspec.AnyWordSpecLike 15 | 16 | class TestingStreamsSpec extends TestKit(ActorSystem("TestingAkkaStreams")) 17 | with AnyWordSpecLike 18 | with BeforeAndAfterAll { 19 | 20 | // this line needs to be here for Akka < 2.6 21 | // // this line needs to be here for Akka < 2.6 22 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 23 | 24 | override def afterAll(): Unit = TestKit.shutdownActorSystem(system) 25 | 26 | "A simple stream" should { 27 | "satisfy basic assertions" in { 28 | // describe our test 29 | 30 | val simpleSource = Source(1 to 10) 31 | val simpleSink = Sink.fold(0)((a: Int, b: Int) => a + b) 32 | 33 | val sumFuture = simpleSource.toMat(simpleSink)(Keep.right).run() 34 | val sum = Await.result(sumFuture, 2.seconds) 35 | assert(sum == 55) 36 | } 37 | 38 | "integrate with test actors via materialized values" in { 39 | import akka.pattern.pipe 40 | import system.dispatcher 41 | 42 | val simpleSource = Source(1 to 10) 43 | val simpleSink = Sink.fold(0)((a: Int, b: Int) => a + b) 44 | 45 | val probe = TestProbe() 46 | 47 | simpleSource.toMat(simpleSink)(Keep.right).run().pipeTo(probe.ref) 48 | 49 | probe.expectMsg(55) 50 | } 51 | 52 | "integrate with a test-actor-based sink" in { 53 | val simpleSource = Source(1 to 5) 54 | val flow = Flow[Int].scan[Int](0)(_ + _) // 0, 1, 3, 6, 10, 15 55 | val streamUnderTest = simpleSource.via(flow) 56 | 57 | val probe = TestProbe() 58 | val probeSink = Sink.actorRef(probe.ref, "completionMessage") 59 | 60 | streamUnderTest.to(probeSink).run() 61 | probe.expectMsgAllOf(0, 1, 3, 6, 10, 15) 62 | } 63 | 64 | "integrate with Streams TestKit Sink" in { 65 | val sourceUnderTest = Source(1 to 5).map(_ * 2) 66 | 67 | val testSink = TestSink.probe[Int] 68 | val materializedTestValue = sourceUnderTest.runWith(testSink) 69 | 70 | materializedTestValue 71 | .request(5) 72 | .expectNext(2, 4, 6, 8, 10) 73 | .expectComplete() 74 | } 75 | 76 | "integrate with Streams TestKit Source" in { 77 | import system.dispatcher 78 | 79 | val sinkUnderTest = Sink.foreach[Int] { 80 | case 13 => throw new RuntimeException("bad luck!") 81 | case _ => 82 | } 83 | 84 | val testSource = TestSource.probe[Int] 85 | val materialized = testSource.toMat(sinkUnderTest)(Keep.both).run() 86 | val (testPublisher, resultFuture) = materialized 87 | 88 | testPublisher 89 | .sendNext(1) 90 | .sendNext(5) 91 | .sendNext(13) 92 | .sendComplete() 93 | 94 | resultFuture.onComplete { 95 | case Success(_) => fail("the sink under test should have thrown an exception on 13") 96 | case Failure(_) => // ok 97 | } 98 | } 99 | 100 | "test flows with a test source AND a test sink" in { 101 | val flowUnderTest = Flow[Int].map(_ * 2) 102 | 103 | val testSource = TestSource.probe[Int] 104 | val testSink = TestSink.probe[Int] 105 | 106 | val materialized = testSource.via(flowUnderTest).toMat(testSink)(Keep.both).run() 107 | val (publisher, subscriber) = materialized 108 | 109 | publisher 110 | .sendNext(1) 111 | .sendNext(5) 112 | .sendNext(42) 113 | .sendNext(99) 114 | .sendComplete() 115 | 116 | subscriber 117 | .request(4) // don't forget this! 118 | .expectNext(2, 10, 84, 198) 119 | .expectComplete() 120 | 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/main/scala/part5_advanced/CustomGraphShapes.scala: -------------------------------------------------------------------------------- 1 | package part5_advanced 2 | 3 | import akka.NotUsed 4 | import akka.actor.ActorSystem 5 | import akka.stream.scaladsl.{Balance, GraphDSL, Merge, RunnableGraph, Sink, Source} 6 | import akka.stream._ 7 | 8 | import scala.collection.immutable 9 | import scala.concurrent.duration._ 10 | 11 | object CustomGraphShapes extends App { 12 | 13 | implicit val system = ActorSystem("CustomGraphShapes") 14 | // this line needs to be here for Akka < 2.6 15 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 16 | 17 | // balance 2x3 shape 18 | case class Balance2x3 ( 19 | in0: Inlet[Int], 20 | in1: Inlet[Int], 21 | out0: Outlet[Int], 22 | out1: Outlet[Int], 23 | out2: Outlet[Int] 24 | ) extends Shape { 25 | 26 | // Inlet[T], Outlet[T] 27 | override val inlets: immutable.Seq[Inlet[_]] = List(in0, in1) 28 | override val outlets: immutable.Seq[Outlet[_]] = List(out0, out1, out2) 29 | 30 | override def deepCopy(): Shape = Balance2x3( 31 | in0.carbonCopy(), 32 | in1.carbonCopy(), 33 | out0.carbonCopy(), 34 | out1.carbonCopy(), 35 | out2.carbonCopy() 36 | ) 37 | } 38 | 39 | val balance2x3Impl = GraphDSL.create() { implicit builder => 40 | import GraphDSL.Implicits._ 41 | 42 | val merge = builder.add(Merge[Int](2)) 43 | val balance = builder.add(Balance[Int](3)) 44 | 45 | merge ~> balance 46 | 47 | Balance2x3( 48 | merge.in(0), 49 | merge.in(1), 50 | balance.out(0), 51 | balance.out(1), 52 | balance.out(2) 53 | ) 54 | } 55 | 56 | val balance2x3Graph = RunnableGraph.fromGraph( 57 | GraphDSL.create() { implicit builder => 58 | import GraphDSL.Implicits._ 59 | 60 | val slowSource = Source(Stream.from(1)).throttle(1, 1.second) 61 | val fastSource = Source(Stream.from(1)).throttle(2, 1.second) 62 | 63 | def createSink(index: Int) = Sink.fold(0)((count: Int, element: Int) => { 64 | println(s"[sink $index] Received $element, current count is $count") 65 | count + 1 66 | }) 67 | 68 | val sink1 = builder.add(createSink(1)) 69 | val sink2 = builder.add(createSink(2)) 70 | val sink3 = builder.add(createSink(3)) 71 | 72 | val balance2x3 = builder.add(balance2x3Impl) 73 | 74 | slowSource ~> balance2x3.in0 75 | fastSource ~> balance2x3.in1 76 | 77 | balance2x3.out0 ~> sink1 78 | balance2x3.out1 ~> sink2 79 | balance2x3.out2 ~> sink3 80 | 81 | ClosedShape 82 | } 83 | ) 84 | 85 | // balance2x3Graph.run() 86 | 87 | /** 88 | * Exercise: generalize the balance component, make it M x N 89 | */ 90 | case class BalanceMxN[T](override val inlets: List[Inlet[T]], override val outlets: List[Outlet[T]]) extends Shape { 91 | override def deepCopy(): Shape = BalanceMxN(inlets.map(_.carbonCopy()), outlets.map(_.carbonCopy())) 92 | } 93 | 94 | object BalanceMxN { 95 | def apply[T](inputCount: Int, outputCount: Int): Graph[BalanceMxN[T], NotUsed] = 96 | GraphDSL.create() { implicit builder => 97 | import GraphDSL.Implicits._ 98 | 99 | val merge = builder.add(Merge[T](inputCount)) 100 | val balance = builder.add(Balance[T](outputCount)) 101 | 102 | merge ~> balance 103 | 104 | BalanceMxN(merge.inlets.toList, balance.outlets.toList) 105 | } 106 | } 107 | 108 | val balanceMxNGraph = RunnableGraph.fromGraph( 109 | GraphDSL.create() { implicit builder => 110 | import GraphDSL.Implicits._ 111 | 112 | val slowSource = Source(Stream.from(1)).throttle(1, 1.second) 113 | val fastSource = Source(Stream.from(1)).throttle(2, 1.second) 114 | 115 | def createSink(index: Int) = Sink.fold(0)((count: Int, element: Int) => { 116 | println(s"[sink $index] Received $element, current count is $count") 117 | count + 1 118 | }) 119 | 120 | val sink1 = builder.add(createSink(1)) 121 | val sink2 = builder.add(createSink(2)) 122 | val sink3 = builder.add(createSink(3)) 123 | 124 | val balance2x3 = builder.add(BalanceMxN[Int](2, 3)) 125 | 126 | slowSource ~> balance2x3.inlets(0) 127 | fastSource ~> balance2x3.inlets(1) 128 | 129 | balance2x3.outlets(0) ~> sink1 130 | balance2x3.outlets(1) ~> sink2 131 | balance2x3.outlets(2) ~> sink3 132 | 133 | ClosedShape 134 | } 135 | ) 136 | 137 | balanceMxNGraph.run() 138 | 139 | } 140 | -------------------------------------------------------------------------------- /src/main/scala/part5_advanced/CustomOperators.scala: -------------------------------------------------------------------------------- 1 | package part5_advanced 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.scaladsl.{Flow, Keep, Sink, Source} 5 | import akka.stream._ 6 | import akka.stream.stage._ 7 | 8 | import scala.collection.mutable 9 | import scala.concurrent.{Future, Promise} 10 | import scala.util.{Failure, Random, Success} 11 | 12 | object CustomOperators extends App { 13 | 14 | implicit val system = ActorSystem("CustomOperators") 15 | // this line needs to be here for Akka < 2.6 16 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 17 | 18 | // 1 - a custom source which emits random numbers until canceled 19 | 20 | class RandomNumberGenerator(max: Int) extends GraphStage[/*step 0: define the shape*/SourceShape[Int]] { 21 | 22 | // step 1: define the ports and the component-specific members 23 | val outPort = Outlet[Int]("randomGenerator") 24 | val random = new Random() 25 | 26 | // step 2: construct a new shape 27 | override def shape: SourceShape[Int] = SourceShape(outPort) 28 | 29 | // step 3: create the logic 30 | override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { 31 | // step 4: 32 | // define mutable state 33 | // implement my logic here 34 | 35 | setHandler(outPort, new OutHandler { 36 | // when there is demand from downstream 37 | override def onPull(): Unit = { 38 | // emit a new element 39 | val nextNumber = random.nextInt(max) 40 | // push it out of the outPort 41 | push(outPort, nextNumber) 42 | } 43 | }) 44 | } 45 | } 46 | 47 | val randomGeneratorSource = Source.fromGraph(new RandomNumberGenerator(100)) 48 | // randomGeneratorSource.runWith(Sink.foreach(println)) 49 | 50 | // 2 - a custom sink that prints elements in batches of a given size 51 | 52 | class Batcher(batchSize: Int) extends GraphStage[SinkShape[Int]] { 53 | 54 | val inPort = Inlet[Int]("batcher") 55 | 56 | override def shape: SinkShape[Int] = SinkShape[Int](inPort) 57 | 58 | override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { 59 | 60 | override def preStart(): Unit = { 61 | pull(inPort) 62 | } 63 | 64 | // mutable state 65 | val batch = new mutable.Queue[Int] 66 | 67 | setHandler(inPort, new InHandler { 68 | // when the upstream wants to send me an element 69 | override def onPush(): Unit = { 70 | val nextElement = grab(inPort) 71 | batch.enqueue(nextElement) 72 | 73 | // assume some complex computation 74 | Thread.sleep(100) 75 | 76 | if(batch.size >= batchSize) { 77 | println("New batch: " + batch.dequeueAll(_ => true).mkString("[", ", ", "]")) 78 | } 79 | 80 | pull(inPort) // send demand upstream 81 | } 82 | 83 | override def onUpstreamFinish(): Unit = { 84 | if (batch.nonEmpty) { 85 | println("New batch: " + batch.dequeueAll(_ => true).mkString("[", ", ", "]")) 86 | println("Stream finished.") 87 | } 88 | } 89 | }) 90 | } 91 | } 92 | 93 | val batcherSink = Sink.fromGraph(new Batcher(10)) 94 | // randomGeneratorSource.to(batcherSink).run() 95 | 96 | /** 97 | * Exercise: a custom flow - a simple filter flow 98 | * - 2 ports: an input port and an output port 99 | */ 100 | 101 | class SimpleFilter[T](predicate: T => Boolean) extends GraphStage[FlowShape[T, T]] { 102 | 103 | val inPort = Inlet[T]("filterIn") 104 | val outPort = Outlet[T]("filterOut") 105 | 106 | override def shape: FlowShape[T, T] = FlowShape(inPort, outPort) 107 | 108 | override def createLogic(inheritedAttributes: Attributes): GraphStageLogic = new GraphStageLogic(shape) { 109 | setHandler(outPort, new OutHandler { 110 | override def onPull(): Unit = pull(inPort) 111 | }) 112 | 113 | setHandler(inPort, new InHandler { 114 | override def onPush(): Unit = { 115 | try { 116 | val nextElement = grab(inPort) 117 | 118 | if (predicate(nextElement)) { 119 | push(outPort, nextElement) // pass it on 120 | } else { 121 | pull(inPort) // ask for another element 122 | } 123 | } catch { 124 | case e: Throwable => failStage(e) 125 | } 126 | } 127 | }) 128 | } 129 | } 130 | 131 | val myFilter = Flow.fromGraph(new SimpleFilter[Int](_ > 50)) 132 | // randomGeneratorSource.via(myFilter).to(batcherSink).run() 133 | // backpressure OOTB!!!!! 134 | 135 | /** 136 | * Materialized values in graph stages 137 | */ 138 | 139 | // 3 - a flow that counts the number of elements that go through it 140 | class CounterFlow[T] extends GraphStageWithMaterializedValue[FlowShape[T, T], Future[Int]] { 141 | 142 | val inPort = Inlet[T]("counterInt") 143 | val outPort = Outlet[T]("counterOut") 144 | 145 | override val shape = FlowShape(inPort, outPort) 146 | 147 | override def createLogicAndMaterializedValue(inheritedAttributes: Attributes): (GraphStageLogic, Future[Int]) = { 148 | 149 | val promise = Promise[Int] 150 | val logic = new GraphStageLogic(shape) { 151 | // setting mutable state 152 | var counter = 0 153 | 154 | setHandler(outPort, new OutHandler { 155 | override def onPull(): Unit = pull(inPort) 156 | 157 | override def onDownstreamFinish(): Unit = { 158 | promise.success(counter) 159 | super.onDownstreamFinish() 160 | } 161 | }) 162 | 163 | setHandler(inPort, new InHandler { 164 | override def onPush(): Unit = { 165 | // extract the element 166 | val nextElement = grab(inPort) 167 | counter += 1 168 | // pass it on 169 | push(outPort, nextElement) 170 | } 171 | 172 | override def onUpstreamFinish(): Unit = { 173 | promise.success(counter) 174 | super.onUpstreamFinish() 175 | } 176 | 177 | override def onUpstreamFailure(ex: Throwable): Unit = { 178 | promise.failure(ex) 179 | super.onUpstreamFailure(ex) 180 | } 181 | }) 182 | } 183 | 184 | (logic, promise.future) 185 | } 186 | } 187 | 188 | val counterFlow = Flow.fromGraph(new CounterFlow[Int]) 189 | val countFuture = Source(1 to 10) 190 | // .map(x => if (x == 7) throw new RuntimeException("gotcha!") else x) 191 | .viaMat(counterFlow)(Keep.right) 192 | .to(Sink.foreach(x => if (x == 7) throw new RuntimeException("gotcha, sink!") else println(x))) 193 | // .to(Sink.foreach[Int](println)) 194 | .run() 195 | 196 | import system.dispatcher 197 | countFuture.onComplete { 198 | case Success(count) => println(s"The number of elements passed: $count") 199 | case Failure(ex) => println(s"Counting the elements failed: $ex") 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /src/main/scala/part5_advanced/DynamicStreamHandling.scala: -------------------------------------------------------------------------------- 1 | package part5_advanced 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.scaladsl.{BroadcastHub, Keep, MergeHub, Sink, Source} 5 | import akka.stream.{ActorMaterializer, KillSwitches} 6 | 7 | import scala.concurrent.duration._ 8 | 9 | object DynamicStreamHandling extends App { 10 | 11 | implicit val system = ActorSystem("DynamicStreamHandling") 12 | // this line needs to be here for Akka < 2.6 13 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 14 | import system.dispatcher 15 | 16 | // #1: Kill Switch 17 | 18 | val killSwitchFlow = KillSwitches.single[Int] 19 | val counter = Source(Stream.from(1)).throttle(1, 1.second).log("counter") 20 | val sink = Sink.ignore 21 | 22 | // val killSwitch = counter 23 | // .viaMat(killSwitchFlow)(Keep.right) 24 | // .to(sink) 25 | // .run() 26 | // 27 | // system.scheduler.scheduleOnce(3 seconds) { 28 | // killSwitch.shutdown() 29 | // } 30 | 31 | 32 | // shared kill switch 33 | val anotherCounter = Source(Stream.from(1)).throttle(2, 1.second).log("anotherCounter") 34 | val sharedKillSwitch = KillSwitches.shared("oneButtonToRuleThemAll") 35 | 36 | // counter.via(sharedKillSwitch.flow).runWith(Sink.ignore) 37 | // anotherCounter.via(sharedKillSwitch.flow).runWith(Sink.ignore) 38 | // 39 | // system.scheduler.scheduleOnce(3 seconds) { 40 | // sharedKillSwitch.shutdown() 41 | // } 42 | 43 | // MergeHub 44 | 45 | val dynamicMerge = MergeHub.source[Int] 46 | // val materializedSink = dynamicMerge.to(Sink.foreach[Int](println)).run() 47 | 48 | // use this sink any time we like 49 | // Source(1 to 10).runWith(materializedSink) 50 | // counter.runWith(materializedSink) 51 | 52 | // BroadcastHub 53 | 54 | val dynamicBroadcast = BroadcastHub.sink[Int] 55 | // val materializedSource = Source(1 to 100).runWith(dynamicBroadcast) 56 | 57 | // materializedSource.runWith(Sink.ignore) 58 | // materializedSource.runWith(Sink.foreach[Int](println)) 59 | 60 | /** 61 | * Challenge - combine a mergeHub and a broadcastHub. 62 | * 63 | * A publisher-subscriber component 64 | */ 65 | val merge = MergeHub.source[String] 66 | val bcast = BroadcastHub.sink[String] 67 | val (publisherPort, subscriberPort) = merge.toMat(bcast)(Keep.both).run() 68 | 69 | subscriberPort.runWith(Sink.foreach(e => println(s"I received: $e"))) 70 | subscriberPort.map(string => string.length).runWith(Sink.foreach(n => println(s"I got a number: $n"))) 71 | 72 | Source(List("Akka", "is", "amazing")).runWith(publisherPort) 73 | Source(List("I", "love", "Scala")).runWith(publisherPort) 74 | Source.single("STREEEEEEAMS").runWith(publisherPort) 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/scala/part5_advanced/Substreams.scala: -------------------------------------------------------------------------------- 1 | package part5_advanced 2 | 3 | import akka.actor.ActorSystem 4 | import akka.stream.ActorMaterializer 5 | import akka.stream.scaladsl.{Keep, Sink, Source} 6 | 7 | import scala.util.{Failure, Success} 8 | 9 | object Substreams extends App { 10 | 11 | implicit val system = ActorSystem("Substreams") 12 | implicit val materialized = ActorMaterializer() 13 | import system.dispatcher 14 | 15 | // 1 - grouping a stream by a certain function 16 | val wordsSource = Source(List("Akka", "is", "amazing", "learning", "substreams")) 17 | val groups = wordsSource.filter(_.nonEmpty).groupBy(30, _.toLowerCase().charAt(0)) 18 | 19 | groups.to(Sink.fold(0)((count, word) => { 20 | val newCount = count + 1 21 | println(s"I just received $word, count is $newCount") 22 | newCount 23 | })) 24 | .run() 25 | 26 | // 2 - merge substreams back 27 | val textSource = Source(List( 28 | "I love Akka Streams", 29 | "this is amazing", 30 | "learning from Rock the JVM" 31 | )) 32 | 33 | val totalCharCountFuture = textSource 34 | .groupBy(2, string => string.length % 2) 35 | .map(_.length) // do your expensive computation here 36 | .mergeSubstreams//WithParallelism(2) 37 | .toMat(Sink.reduce[Int](_ + _))(Keep.right) 38 | .run() 39 | 40 | totalCharCountFuture.onComplete { 41 | case Success(value) => println(s"Total char count: $value") 42 | case Failure(ex) => println(s"Char computation failed: $ex") 43 | } 44 | 45 | // 3 - splitting a stream into substreams, when a condition is met 46 | 47 | val text = 48 | "I love Akka Streams\n" + 49 | "this is amazing\n" + 50 | "learning from Rock the JVM\n" 51 | 52 | val anotherCharCountFuture = Source(text.toList) 53 | .splitWhen(c => c == '\n') 54 | .filter(_ != '\n') 55 | .map(_ => 1) 56 | .mergeSubstreams 57 | .toMat(Sink.reduce[Int](_ + _))(Keep.right) 58 | .run() 59 | 60 | anotherCharCountFuture.onComplete { 61 | case Success(value) => println(s"Total char count alternative: $value") 62 | case Failure(ex) => println(s"Char computation failed: $ex") 63 | } 64 | 65 | // 4 - flattening 66 | val simpleSource = Source(1 to 5) 67 | simpleSource.flatMapConcat(x => Source(x to (3 * x))).runWith(Sink.foreach(println)) 68 | simpleSource.flatMapMerge(2, x => Source(x to (3 * x))).runWith(Sink.foreach(println)) 69 | } 70 | -------------------------------------------------------------------------------- /src/main/scala/playground/Playground.scala: -------------------------------------------------------------------------------- 1 | package playground 2 | 3 | import akka.NotUsed 4 | import akka.actor.ActorSystem 5 | import akka.stream.ActorAttributes.supervisionStrategy 6 | import akka.stream.Supervision.resumingDecider 7 | import akka.stream.{ActorMaterializer, Attributes, FlowShape, Inlet, Outlet} 8 | import akka.stream.scaladsl.{Flow, Keep, RunnableGraph, Sink, Source} 9 | import akka.stream.stage.{GraphStage, GraphStageLogic, GraphStageWithMaterializedValue, InHandler, OutHandler} 10 | import akka.testkit.TestKit 11 | 12 | import scala.concurrent.{Future, Promise} 13 | 14 | object Playground extends App { 15 | 16 | implicit val system = ActorSystem("AkkaStreamsDemo") 17 | // this line needs to be here for Akka < 2.6 18 | // implicit val materializer: ActorMaterializer = ActorMaterializer() 19 | import system.dispatcher 20 | 21 | val source = Source(1 to 10) 22 | val flow = Flow[Int].map(x => { println(x); x }) 23 | val sink = Sink.fold[Int, Int](0)(_ + _) 24 | 25 | // connect the Source to the Sink, obtaining a RunnableGraph 26 | val runnable: RunnableGraph[Future[Int]] = source.via(flow).toMat(sink)(Keep.right) 27 | 28 | // materialize the flow and get the value of the FoldSink 29 | val sum: Future[Int] = runnable.run() 30 | sum.onComplete(x => println(s"Sum: $x")) 31 | 32 | val independentFlow = Flow[String].map(_.reverse) 33 | import akka.stream.scaladsl.FlowWithContext 34 | 35 | val independentFlowWithContext: FlowWithContext[String, Int, String, Int, NotUsed] = 36 | independentFlow.asFlowWithContext[String, Int, Int]((string, ctx) => string)(string => 0) 37 | 38 | val flowWithContext: FlowWithContext[String, Int, String, Int, NotUsed] = ??? 39 | val mapped = flowWithContext.map(_.reverse) 40 | val mappedVia = flowWithContext.via(independentFlowWithContext) 41 | } --------------------------------------------------------------------------------