├── pipeline-model.png ├── .gitignore ├── src ├── test │ └── scala │ │ └── com │ │ └── cosmin │ │ └── pipeline │ │ ├── UnitSpec.scala │ │ ├── StageSpec.scala │ │ └── PipelineSpec.scala └── main │ └── scala │ └── com │ └── cosmin │ ├── pipeline │ ├── Filter.scala │ ├── executor │ │ ├── PipelineExecutor.scala │ │ ├── AkkaExecutor.scala │ │ ├── actor │ │ │ ├── Worker.scala │ │ │ └── Supervisor.scala │ │ ├── SynchronouslyExecutor.scala │ │ └── AsyncExecutor.scala │ ├── Stage.scala │ └── Pipeline.scala │ └── examples │ ├── wordcount │ ├── Count.scala │ ├── Grep.scala │ ├── Cat.scala │ └── WordCount.scala │ ├── TypeConversions.scala │ └── HelloWorld.scala ├── LICENSE ├── myText.txt └── README.md /pipeline-model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cosminseceleanu/scala-pipeline/HEAD/pipeline-model.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | target 4 | 5 | ### IntelliJ IDEA ### 6 | *.iml 7 | *.ipr 8 | *.iws 9 | .idea 10 | out 11 | project 12 | 13 | .DS_Store 14 | -------------------------------------------------------------------------------- /src/test/scala/com/cosmin/pipeline/UnitSpec.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | import org.scalatest._ 4 | 5 | abstract class UnitSpec extends FunSuite with Matchers with 6 | OptionValues with Inside with Inspectors 7 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/Filter.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | trait Filter[In, Out] { 4 | def execute: In => Out 5 | } 6 | 7 | object Filter { 8 | def apply[In, Out](f: In => Out): Filter[In, Out] = new Filter[In, Out] { 9 | override def execute: In => Out = f 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/PipelineExecutor.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor 2 | 3 | import com.cosmin.pipeline.Stage 4 | 5 | import scala.util.Try 6 | 7 | trait PipelineExecutor[In, Out] { 8 | def execute(in: In, stages: List[Stage]) (onComplete: Try[Out] => Unit) : Unit 9 | } 10 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/wordcount/Count.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples.wordcount 2 | 3 | import com.cosmin.pipeline.Filter 4 | 5 | class Count extends Filter[Seq[String], Int] { 6 | override def execute: Seq[String] => Int = lines => lines.count(_ => true) 7 | } 8 | 9 | object Count { 10 | def apply(): Count = new Count() 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/wordcount/Grep.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples.wordcount 2 | 3 | import com.cosmin.pipeline.Filter 4 | 5 | class Grep(word: String) extends Filter[Seq[String], Seq[String]] { 6 | override def execute: Seq[String] => Seq[String] = lines => lines.filter(_.contains(word)) 7 | } 8 | 9 | object Grep { 10 | def apply(word: String): Grep = new Grep(word) 11 | } 12 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/wordcount/Cat.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples.wordcount 2 | 3 | import com.cosmin.pipeline.Filter 4 | 5 | import scala.io.Source 6 | 7 | class Cat() extends Filter[String, Seq[String]]{ 8 | override def execute: String => Seq[String] = file => Source.fromFile(file).getLines.toSeq 9 | } 10 | 11 | object Cat { 12 | def apply(): Cat = new Cat() 13 | } 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/Stage.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | trait Stage { 4 | type In 5 | type Out 6 | 7 | val filter: Filter[In, Out] 8 | 9 | def execute(in: In): Out = filter.execute(in) 10 | } 11 | 12 | object Stage { 13 | def apply[I, O](f: Filter[I, O]): Stage = new Stage() { 14 | override type In = I 15 | override type Out = O 16 | override val filter: Filter[In, Out] = f 17 | } 18 | } -------------------------------------------------------------------------------- /src/test/scala/com/cosmin/pipeline/StageSpec.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | class StageSpec extends UnitSpec { 4 | test("Text execute stage filter") { 5 | val intToStringFilter: Filter[Int, String] = new Filter[Int, String] { 6 | override def execute = i => i.toString 7 | } 8 | val stage = Stage(intToStringFilter) 9 | 10 | assertResult("1") (stage.execute(1.asInstanceOf[stage.In])) 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/TypeConversions.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples 2 | 3 | import com.cosmin.pipeline.{Filter, Pipeline} 4 | 5 | import scala.util.Success 6 | 7 | /** 8 | * Pipeline who receive a random int --> calculate sqrt --> create a message with sqrt result 9 | * Int -> Double -> String 10 | */ 11 | object TypeConversions { 12 | def main(args: Array[String]): Unit = { 13 | val pipeline = Pipeline[Int, Int]() | new Sqrt | (sqrt => s"Sqrt: $sqrt!") 14 | 15 | pipeline.execute(4) { 16 | case Success(output) => println(output) 17 | } 18 | } 19 | } 20 | 21 | class Sqrt extends Filter[Int, Double] { 22 | override def execute: Int => Double = input => Math.sqrt(input) 23 | } 24 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/AkkaExecutor.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor 2 | import akka.actor.ActorSystem 3 | import com.cosmin.pipeline.Stage 4 | import com.cosmin.pipeline.executor.actor.Supervisor 5 | import com.cosmin.pipeline.executor.actor.Supervisor.Start 6 | 7 | import scala.util.Try 8 | 9 | class AkkaExecutor[In, Out](system: ActorSystem) extends PipelineExecutor[In, Out] { 10 | override def execute(in: In, stages: List[Stage])(onComplete: Try[Out] => Unit): Unit = { 11 | val supervisor = system.actorOf(Supervisor.props[Out](stages, onComplete)) 12 | 13 | supervisor ! Start[In](in) 14 | } 15 | } 16 | 17 | object AkkaExecutor { 18 | def apply[In, Out]: AkkaExecutor[In, Out] = new AkkaExecutor[In, Out](ActorSystem("pipeline")) 19 | } 20 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/actor/Worker.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor.actor 2 | 3 | import akka.actor.{Actor, Props} 4 | import com.cosmin.pipeline.Stage 5 | import com.cosmin.pipeline.executor.actor.Supervisor.{StageCompleted, StageFailed} 6 | import com.cosmin.pipeline.executor.actor.Worker.Execute 7 | 8 | import scala.util.{Failure, Success, Try} 9 | 10 | object Worker { 11 | def props(stage: Stage): Props = Props(new Worker(stage)) 12 | 13 | final case class Execute[In](in: In) 14 | } 15 | 16 | class Worker(stage: Stage) extends Actor { 17 | override def receive: Receive = { 18 | case Execute(input) => 19 | executeStage(input) 20 | } 21 | 22 | private def executeStage(input: Any): Unit = { 23 | Try[stage.Out](stage.execute(input.asInstanceOf[stage.In])) match { 24 | case Success(result) => sender() ! StageCompleted[stage.Out](stage, result.asInstanceOf[stage.Out]) 25 | case Failure(e) => sender() ! StageFailed(stage, e) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/HelloWorld.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples 2 | 3 | import com.cosmin.pipeline.Pipeline 4 | import com.cosmin.pipeline.executor.AsyncExecutor 5 | 6 | import scala.util.Success 7 | 8 | /** 9 | * Pipeline who build 'Hello World!' message using anonymous filters for 'World' and '!' 10 | */ 11 | object HelloWorld { 12 | def main(args: Array[String]): Unit = { 13 | //build pipeline 14 | val pipeline = Pipeline[String, String]() | (i => s"$i World") | (i => s"$i!") 15 | 16 | //execute filters synchronous 17 | pipeline.execute("Hello") { 18 | case Success(out) => println(s"Sync --> $out") 19 | } 20 | 21 | executeAsync(pipeline) 22 | 23 | println("end") 24 | } 25 | 26 | def executeAsync(pipeline: Pipeline[String, String]): Unit = { 27 | implicit val executor: AsyncExecutor[String, String] = AsyncExecutor[String, String] 28 | pipeline.execute("Hey Async") { 29 | case Success(out) => println(s"Async --> $out") 30 | } 31 | 32 | Thread.sleep(2000) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/SynchronouslyExecutor.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor 2 | 3 | import com.cosmin.pipeline.Stage 4 | 5 | import scala.util.Try 6 | 7 | class SynchronouslyExecutor[In, Out] extends PipelineExecutor[In, Out] { 8 | override def execute(in: In, stages: List[Stage])(onComplete: Try[Out] => Unit): Unit = { 9 | val result: Try[Out] = Try[Out](doExecute[In, Out](stages, in)) 10 | 11 | onComplete(result) 12 | } 13 | 14 | def doExecute[I, O](stages: List[Stage], input: I): O = stages match { 15 | case Nil => input.asInstanceOf[O] 16 | case stage :: nextStage :: tail => 17 | val stageOutput = stage.execute(input.asInstanceOf[stage.In]) 18 | 19 | doExecute[stage.Out, nextStage.Out](nextStage :: tail, stageOutput).asInstanceOf[O] 20 | case stage :: tail => 21 | val stageOutput = stage.execute(input.asInstanceOf[stage.In]) 22 | 23 | doExecute[stage.Out, Out](tail, stageOutput).asInstanceOf[O] 24 | } 25 | } 26 | 27 | object SynchronouslyExecutor { 28 | def apply[In, Out](): SynchronouslyExecutor[In, Out] = new SynchronouslyExecutor() 29 | } 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Cosmin Seceleanu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/Pipeline.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | import com.cosmin.pipeline.executor.{PipelineExecutor, SynchronouslyExecutor} 4 | 5 | import scala.util.Try 6 | 7 | trait Pipeline[In, Out] { 8 | val stages: List[Stage] 9 | 10 | def | [X](f: Out => X): Pipeline[In, X] = pipe(f) 11 | 12 | def pipe[X](f: Out => X): Pipeline[In, X] = { 13 | val stage = Stage[Out, X](Filter[Out, X](f)) 14 | 15 | Pipeline[In, X](stage :: stages) 16 | } 17 | 18 | def | [X](filter: Filter[Out, X]): Pipeline[In, X] = pipe(filter) 19 | 20 | def pipe[X](filter: Filter[Out, X]): Pipeline[In, X] = { 21 | val stage = Stage[Out, X](filter) 22 | 23 | Pipeline[In, X](stage :: stages) 24 | } 25 | 26 | def execute(in: In)(onComplete: Try[Out] => Unit)(implicit pipelineExecutor: PipelineExecutor[In, Out] = SynchronouslyExecutor()): Unit = { 27 | pipelineExecutor.execute(in, stages.reverse) (onComplete) 28 | } 29 | } 30 | 31 | object Pipeline { 32 | def apply[In, Out](stagesList: List[Stage]): Pipeline[In, Out] = new Pipeline[In, Out] { 33 | override val stages: List[Stage] = stagesList 34 | } 35 | 36 | def apply[In, Out](): Pipeline[In, Out] = Pipeline[In, Out](List()) 37 | } -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/AsyncExecutor.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor 2 | import com.cosmin.pipeline.Stage 3 | 4 | import scala.concurrent.{ExecutionContext, Future} 5 | import scala.util.Try 6 | 7 | class AsyncExecutor[In, Out] (implicit executor: ExecutionContext) extends PipelineExecutor[In, Out] { 8 | override def execute(in: In, stages: List[Stage])(onComplete: Try[Out] => Unit): Unit = { 9 | val out: Future[Out] = doExecute[In, Out](stages, Future.successful[In](in)) 10 | 11 | out.onComplete(onComplete) 12 | } 13 | 14 | def doExecute[I, O](stages: List[Stage], input: Future[I]): Future[O] = stages match { 15 | case Nil => input.asInstanceOf[Future[O]] 16 | case stage :: nextStage :: tail => 17 | val stageOutput = input.map[stage.Out](i => stage.execute(i.asInstanceOf[stage.In])) 18 | 19 | doExecute[stage.Out, nextStage.Out](nextStage :: tail, stageOutput).asInstanceOf[Future[O]] 20 | case stage :: tail => 21 | val stageOutput = input.map[stage.Out](i => stage.execute(i.asInstanceOf[stage.In])) 22 | 23 | doExecute[stage.Out, Out](tail, stageOutput).asInstanceOf[Future[O]] 24 | } 25 | } 26 | 27 | object AsyncExecutor { 28 | def apply[In, Out]: AsyncExecutor[In, Out] = new AsyncExecutor() (scala.concurrent.ExecutionContext.Implicits.global) 29 | } 30 | -------------------------------------------------------------------------------- /src/test/scala/com/cosmin/pipeline/PipelineSpec.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline 2 | 3 | import scala.util.{Failure, Success} 4 | 5 | class PipelineSpec extends UnitSpec { 6 | test("Test building pipeline") { 7 | val pipeline: Pipeline[String, String] = createDefaultPipeline 8 | 9 | assert(2 == pipeline.stages.size) 10 | assertResult("input->f2") { 11 | val stage = pipeline.stages.head 12 | stage.execute("input".asInstanceOf[stage.In]) 13 | } 14 | } 15 | 16 | test("Test successful execution of a pipeline") { 17 | val pipeline: Pipeline[String, String] = createDefaultPipeline 18 | 19 | pipeline.execute("msg") { 20 | case Success(out) => assert("msg->f1->f2".equals(out)) 21 | case Failure(_) => fail("Pipeline execution expected be successful") 22 | } 23 | } 24 | 25 | test("Test failed execution of a pipeline") { 26 | val pipeline: Pipeline[String, String] = createDefaultPipeline | (s => throw new RuntimeException) 27 | 28 | pipeline.execute("msg") { 29 | case Success(out) => fail("Pipeline execution expected fail") 30 | case Failure(e) => assert(e.isInstanceOf[RuntimeException]) 31 | } 32 | } 33 | 34 | 35 | private def createDefaultPipeline = { 36 | val firstFilter: String => String = s => s"$s->f1" 37 | val secondFilter: String => String = s => s"$s->f2" 38 | 39 | Pipeline[String, String]() | firstFilter | secondFilter 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/pipeline/executor/actor/Supervisor.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.pipeline.executor.actor 2 | 3 | import akka.actor.{Actor, ActorLogging, ActorRef, Props} 4 | import com.cosmin.pipeline.Stage 5 | import com.cosmin.pipeline.executor.actor.Supervisor.{StageCompleted, StageFailed, Start} 6 | import com.cosmin.pipeline.executor.actor.Worker.Execute 7 | 8 | import scala.util.{Failure, Try} 9 | 10 | object Supervisor { 11 | def props[Out](stages: List[Stage], onComplete: Try[Out] => Unit): Props = Props(new Supervisor(stages, onComplete)) 12 | 13 | final case class Start[In](in: In) 14 | final case class StageCompleted[Out](stage: Stage, result: Out) 15 | final case class StageFailed(stage: Stage, e: Throwable) 16 | } 17 | 18 | class Supervisor[Out](stages: List[Stage], onComplete: Try[Out] => Unit) extends Actor with ActorLogging { 19 | private var remainingStages = stages 20 | private var stageToActor: Map[Stage, ActorRef] = Map.empty 21 | 22 | override def preStart(): Unit = stageToActor = stages.map(stage => (stage, context.actorOf(Worker.props(stage)))).toMap 23 | 24 | override def receive: Receive = { 25 | case Start(in) => executeStage(in) 26 | case StageCompleted(stage, result) => onStageCompleted(result) 27 | case StageFailed(stage, e) => 28 | onComplete(Failure(e)) 29 | context.stop(self) 30 | } 31 | 32 | private def onStageCompleted(result: Any): Unit = { 33 | remainingStages match { 34 | case Nil => 35 | onComplete(Try(result.asInstanceOf[Out])) 36 | context.stop(self) 37 | case _ => executeStage(result) 38 | } 39 | } 40 | 41 | private def executeStage(in: Any): Unit = { 42 | val stage = remainingStages.head 43 | stageToActor(stage) ! Execute[stage.In](in.asInstanceOf[stage.In]) 44 | remainingStages = remainingStages.tail 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/scala/com/cosmin/examples/wordcount/WordCount.scala: -------------------------------------------------------------------------------- 1 | package com.cosmin.examples.wordcount 2 | 3 | import com.cosmin.pipeline.Pipeline 4 | import com.cosmin.pipeline.executor.{AkkaExecutor, AsyncExecutor, PipelineExecutor, SynchronouslyExecutor} 5 | 6 | import scala.util.{Failure, Success} 7 | 8 | /** 9 | * Implementation of the following UNIX command 'cat "myText.txt" | grep "hello" | wc -l' 10 | */ 11 | object WordCount { 12 | def main(args: Array[String]): Unit = { 13 | val wordToFind = "hello" 14 | val pipeline = Pipeline[String, String]() | Cat() | Grep(wordToFind) | Count() 15 | 16 | doExecute(pipeline, SynchronouslyExecutor(), "Sync", wordToFind) 17 | doExecute(pipeline, SynchronouslyExecutor(), "Sync Failed", wordToFind) ("notFound.txt") 18 | 19 | executeAsync(pipeline, wordToFind) 20 | executeAsyncUsingAkka(pipeline, wordToFind) 21 | } 22 | 23 | private def executeAsync(pipeline: Pipeline[String, Int], wordToFind: String): Unit = { 24 | val asyncExecutor: AsyncExecutor[String, Int] = AsyncExecutor[String, Int] 25 | doExecute(pipeline, asyncExecutor, "Async", wordToFind) 26 | doExecute(pipeline, asyncExecutor, "Async Failed", wordToFind) ("notFound.txt") 27 | } 28 | 29 | private def executeAsyncUsingAkka(pipeline: Pipeline[String, Int], wordToFind: String): Unit = { 30 | val akkaExecutor: AkkaExecutor[String, Int] = AkkaExecutor[String, Int] 31 | doExecute(pipeline, akkaExecutor, "Akka Async", wordToFind) 32 | doExecute(pipeline, akkaExecutor, "Akka Async Failed", wordToFind) ("notFound.txt") 33 | } 34 | 35 | private def doExecute(pipe: Pipeline[String, Int], exec: PipelineExecutor[String, Int], prefix: String, word: String) (implicit file: String = "myText.txt") : Unit = { 36 | pipe.execute(file) { 37 | case Success(output) => println(s"$prefix ---> word '$word' was found on $output lines") 38 | case Failure(e) => println(s"$prefix ---> error: $e") 39 | } (exec) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /myText.txt: -------------------------------------------------------------------------------- 1 | Lorem ipsum dolor sit amet, consectetur adipiscing elit. Pellentesque sodales dapibus nulla. Nam ac sapien vitae massa iaculis sodales eget ac turpis. Nunc vitae diam et mi gravida lacinia sit amet a mauris. Aliquam pellentesque, tortor in iaculis posuere, lacus nibh dictum nulla, sed sollicitudin mauris tortor id leo. Phasellus semper eros varius sodales imperdiet. Praesent semper, leo at dignissim auctor, est orci eleifend ligula, quis iaculis lacus odio id felis. Nulla scelerisque est posuere nulla gravida feugiat. Nam eros velit, molestie ac tristique ac, rhoncus eu orci. Fusce dignissim ipsum sed augue semper, in ullamcorper ante accumsan. Nulla facilisi. Integer gravida ante eget hendrerit tristique. Aenean sed ante vel magna laoreet hendrerit. In suscipit consequat malesuada. 2 | 3 | Proin gravida nec urna eu consequat. In viverra metus at porttitor facilisis. Nam ac elementum leo, id venenatis metus. Curabitur posuere, augue at placerat faucibus, tortor leo interdum tellus, ac convallis eros ipsum vel tortor. Pellentesque ut enim pharetra, tristique felis eu, vestibulum sapien. Etiam hello felis sapien, varius ut semper hendrerit, dictum dictum hello justo. In sed sagittis turpis. Suspendisse potenti. Mauris ut leo sed nisi commodo efficitur varius vitae nisl. Cras congue libero non dolor vulputate, viverra sodales erat fringilla. Vestibulum in ante eu ipsum interdum venenatis id id ante. Nunc felis magna, lobortis eget urna vel, rhoncus suscipit lectus. Aliquam ex magna, porta eu egestas ut, dapibus non massa. 4 | 5 | Nam laoreet erat at lobortis fermentum. Proin auctor sit amet dui ac sagittis. Donec ut lorem eget enim commodo auctor. Aenean ornare vel quam non cursus. Aenean dui purus, semper sit amet maximus at, malesuada sed libero. Quisque fringilla, nunc lobortis bibendum gravida, nibh orci blandit eros, vitae scelerisque ex massa id dolor. Fusce eu aliquet nisl, quis facilisis risus. Praesent sed placerat orci. Fusce felis ante, rhoncus ut euismod vitae, aliquet id elit. Integer ac ipsum tempus, convallis nunc at, vestibulum ante. Sed congue rhoncus dolor, nec dictum nulla ultrices tempus. Curabitur fringilla, tortor nec facilisis aliquam, quam purus sodales metus, non sagittis neque tellus non velit. Duis mattis convallis cursus. Ut ac laoreet nisi, id consequat nibh. Praesent mattis enim risus, ut aliquam erat condimentum eu. 6 | 7 | Vivamus imperdiet vulputate dui in porta. Duis eu risus id mauris aliquet tincidunt. Integer ac mattis lectus. Morbi laoreet, erat a egestas facilisis, turpis massa porttitor augue, at hendrerit felis lorem ut elit. Curabitur ac ipsum sed odio tincidunt blandit sed quis lacus. Aenean vitae metus ut mauris gravida volutpat. Vestibulum ante ipsum primis in faucibus helloorci luctus et ultrices posuere cubilia Curae; Nulla ullamcorper euismod pharetra. Etiam at metus vulputate ex imperdiet consequat ut hendrerit leo. Nulla sed molestie lorem. Nulla ut lectus elit. Cras lobortis, velit ac posuere malesuada, quam ex ullamcorper metus, et luctus libero mi vitae ipsum. 8 | 9 | Fusce a ipsum vitae tortor molestie vehicula ac quis ipsum. Nam consectetur iaculis tellus, feugiat ornare turpis lobortis et. Pellentesque ut nisl congue, pharetra libero faucibus, imperdiet mi. Integer pulvinar orci vitae eros tempor, quis scelerisque elit dictum. Mauris non mi luctus, volutpat ipsum ac, fringilla metus. Ut venenatis nec magna eget euismod. Sed tempus porta quam eget scelerisque. 10 | hello -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pipeline Pattern 2 | In software engineering, a pipeline consists of a chain of processing elements (processes, threads, coroutines, functions, etc.), arranged so that the output of each element is the input of the next; the name is by analogy to a physical pipeline. 3 | 4 | ![alt text](https://github.com/cosminseceleanu/scala-pipeline/blob/master/pipeline-model.png) 5 | 6 | The filter transforms or filters the data it receives via the pipes with which it is connected. A filter can have any number of input pipes and any number of output pipes. 7 | 8 | The pipe is the connector that passes data from one filter to the next. It is a directional stream of data, that is usually implemented by a data buffer to store all data, until the next filter has time to process it. 9 | 10 | ## Introduction 11 | This project allows you to implement the pipeline pattern while creating reusable pipelines in your Scala applications. You can create pipelines consisting of one or more filters and then process them synchronous or asynchronous. Pipeline processing is initiated by some payload(input) and this payload will be passed and transformed from filter to filter in order to complete the required process. 12 | 13 | ## Usage 14 | In order to create a pipeline you must specify type of initial payload. 15 | ```scala 16 | import com.cosmin.pipeline.Pipeline 17 | 18 | val pipeline = Pipeline[Int, Int]() 19 | 20 | ``` 21 | 22 | Operations in a pipeline i.e. filters can be classes that extends Filter trait or pure functions that receive a input and returns processed input. 23 | In the below example we add Sqrt filter and an anonymus filter who creates a string message with the sqrt value of the input. 24 | 25 | ```scala 26 | import com.cosmin.pipeline.{Filter, Pipeline} 27 | 28 | class Sqrt extends Filter[Int, Double] { 29 | override def execute: Int => Double = input => Math.sqrt(input) 30 | } 31 | 32 | val pipeline = Pipeline[Int, Int]() | new Sqrt | (sqrt => s"Sqrt: $sqrt!") 33 | 34 | ``` 35 | After pipeline filters was added we can process them using execute method of pipeline who take as parameter initial payload and a callback(onComplete) as curried parameter to be called when pipeline was processed. Callback receive as input parameter a Try object with the value of the last filter output in case of success. 36 | 37 | ```scala 38 | pipeline.execute(4) { 39 | case Success(output) => println(output)// print to console: Sqrt: 2.0! 40 | } 41 | ``` 42 | 43 | ### Word Count Example 44 | 45 | ##### Objective: count appearances of a word in text file 46 | ```UNIX 47 | cat "myText.txt" | grep "hello" | wc -l 48 | ``` 49 | 50 | ##### Define filters 51 | ```scala 52 | import com.cosmin.pipeline.{Filter, Pipeline} 53 | import scala.io.Source 54 | 55 | class Cat() extends Filter[String, Seq[String]]{ 56 | override def execute: String => Seq[String] = file => Source.fromFile(file).getLines.toSeq 57 | } 58 | 59 | object Cat { 60 | def apply(): Cat = new Cat() 61 | } 62 | 63 | class Grep(word: String) extends Filter[Seq[String], Seq[String]] { 64 | override def execute: Seq[String] => Seq[String] = lines => lines.filter(_.contains(word)) 65 | } 66 | 67 | object Grep { 68 | def apply(word: String): Grep = new Grep(word) 69 | } 70 | 71 | class Count extends Filter[Seq[String], Int] { 72 | override def execute: Seq[String] => Int = lines => lines.count(_ => true) 73 | } 74 | 75 | object Count { 76 | def apply(): Count = new Count() 77 | } 78 | ``` 79 | 80 | ##### Building the pipeline 81 | ```scala 82 | import com.cosmin.pipeline.{Filter, Pipeline} 83 | val pipeline = Pipeline[String, String]() | Cat() | Grep("hello") | Count() 84 | 85 | ``` 86 | 87 | ##### Execute the pipeline 88 | ```scala 89 | pipeline.execute("myText.txt") { 90 | case Success(output) => println(s"word 'hello' was found on $output lines") 91 | } 92 | ``` 93 | Code above print to console 'word 'hello' was found on 3 lines' for the file https://github.com/cosminseceleanu/scala-pipeline/blob/master/myText.txt 94 | 95 | ## ToDo 96 | * async examples 97 | * executor using akka actors 98 | * tests 99 | 100 | --------------------------------------------------------------------------------