├── .gitignore ├── Chapter01 ├── build.sbt ├── project │ ├── build.properties │ └── plugins.sbt └── src │ ├── main │ └── scala │ │ ├── Main.scala │ │ └── Person.scala │ └── test │ └── scala │ └── MainSpec.scala ├── Chapter02 └── retirement-calculator │ ├── .gitignore │ ├── build.sbt │ ├── project │ ├── assembly.sbt │ └── build.properties │ └── src │ ├── main │ ├── resources │ │ ├── cpi.tsv │ │ └── sp500.tsv │ └── scala │ │ └── retcalc │ │ ├── EquityData.scala │ │ ├── InflationData.scala │ │ ├── RetCalc.scala │ │ ├── Returns.scala │ │ └── SimulatePlanApp.scala │ └── test │ ├── resources │ ├── .chapter3_validated.txt.swp │ ├── chapter3_validated.txt │ ├── cpi_2017.tsv │ └── sp500_2017.tsv │ └── scala │ └── retcalc │ ├── EquityDataSpec.scala │ ├── InflationDataSpec.scala │ ├── RetCalcIT.scala │ ├── RetCalcSpec.scala │ ├── ReturnsSpec.scala │ └── SimulatePlanAppIT.scala ├── Chapter03 ├── retirement-calculator │ ├── build.sbt │ ├── project │ │ ├── assembly.sbt │ │ └── build.properties │ └── src │ │ ├── main │ │ ├── resources │ │ │ ├── cpi.tsv │ │ │ └── sp500.tsv │ │ └── scala │ │ │ └── retcalc │ │ │ ├── EquityData.scala │ │ │ ├── InflationData.scala │ │ │ ├── RetCalc.scala │ │ │ ├── RetCalcError.scala │ │ │ ├── Returns.scala │ │ │ └── SimulatePlanApp.scala │ │ └── test │ │ ├── resources │ │ ├── cpi_2017.tsv │ │ └── sp500_2017.tsv │ │ └── scala │ │ └── retcalc │ │ ├── EquityDataSpec.scala │ │ ├── InflationDataSpec.scala │ │ ├── RetCalcIT.scala │ │ ├── RetCalcSpec.scala │ │ ├── ReturnsSpec.scala │ │ └── SimulatePlanAppIT.scala └── worksheets │ ├── build.sbt │ ├── project │ └── build.properties │ └── src │ └── test │ └── scala │ ├── either.sc │ ├── exceptions.sc │ ├── nonemptylist.sc │ ├── option.sc │ ├── referential_transparency.sc │ └── validated.sc ├── Chapter04 ├── build.sbt ├── project │ └── build.properties └── src │ └── main │ └── scala │ ├── currying.sc │ ├── implicits_appContext.sc │ ├── implicits_conversion.sc │ ├── implicits_future.sc │ ├── implicits_parameter.sc │ ├── implicits_resolution.sc │ ├── implicits_sdk.sc │ ├── lazyness.sc │ └── variance.sc ├── Chapter05 ├── build.sbt ├── project │ └── build.properties └── src │ ├── main │ └── scala │ │ ├── typeclasses.sc │ │ ├── typeclasses_applicative.sc │ │ ├── typeclasses_apply.sc │ │ ├── typeclasses_functor.sc │ │ ├── typeclasses_monad.sc │ │ ├── typeclasses_monoid.sc │ │ ├── typeclasses_monoid_monad.sc │ │ ├── typeclasses_ordering.sc │ │ └── typeclasses_semigroup.sc │ └── test │ └── scala │ └── EqualitySpec.scala ├── Chapter06-09 └── online-shoppping-cart │ ├── .gitignore │ ├── Future.sc │ ├── Procfile │ ├── README.md │ ├── build.sbt │ ├── client │ └── src │ │ └── main │ │ └── scala │ │ └── io │ │ └── fscala │ │ └── shopping │ │ └── client │ │ ├── CartDiv.scala │ │ ├── NotifyJS.scala │ │ ├── ProductDiv.scala │ │ └── UIManager.scala │ ├── project │ ├── build.properties │ └── plugins.sbt │ ├── server │ ├── JSON.sc │ ├── app │ │ ├── actors │ │ │ ├── BrowserActor.scala │ │ │ ├── BrowserManagerActor.scala │ │ │ └── UserActor.scala │ │ ├── controllers │ │ │ ├── Application.scala │ │ │ ├── WebServices.scala │ │ │ └── WebSockets.scala │ │ ├── dao │ │ │ └── ProductDao.scala │ │ └── views │ │ │ └── index.scala.html │ ├── conf │ │ ├── application.conf │ │ ├── evolutions │ │ │ └── default │ │ │ │ └── 1.sql │ │ ├── heroku.conf │ │ ├── logback.xml │ │ └── routes │ ├── public │ │ └── images │ │ │ └── favicon.png │ └── test │ │ ├── APISpec.scala │ │ ├── ApplicationSpec.scala │ │ ├── CartDaoSpec.scala │ │ ├── IntegrationSpec.scala │ │ └── ProductDaoSpec.scala │ └── shared │ └── src │ └── main │ └── scala │ └── io │ └── fscala │ └── shopping │ └── shared │ ├── Models.scala │ └── SharedMessages.scala ├── Chapter10-11 └── bitcoin-analyser │ ├── INSTRUCTIONS.md │ ├── build.sbt │ ├── data │ └── transactions │ │ ├── date=2018-09-09 │ │ ├── .part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet.crc │ │ ├── .part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet.crc │ │ ├── .part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet.crc │ │ ├── .part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet.crc │ │ ├── .part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet.crc │ │ ├── .part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet.crc │ │ ├── part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet │ │ ├── part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet │ │ ├── part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet │ │ ├── part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet │ │ ├── part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet │ │ └── part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet │ │ └── date=2018-09-10 │ │ ├── .part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet.crc │ │ ├── .part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet.crc │ │ ├── .part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet.crc │ │ ├── part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet │ │ ├── part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet │ │ └── part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet │ ├── project │ ├── assembly.sbt │ └── build.properties │ └── src │ ├── main │ ├── resources │ │ └── log4j.properties │ └── scala │ │ └── coinyser │ │ ├── AppConfig.scala │ │ ├── BatchProducer.scala │ │ ├── BatchProducerApp.scala │ │ ├── HttpTransaction.scala │ │ ├── KafkaConfig.scala │ │ ├── StreamingConsumer.scala │ │ ├── StreamingProducer.scala │ │ ├── Transaction.scala │ │ ├── WebsocketTransaction.scala │ │ ├── notebook-batch.sc │ │ └── notebook-streaming.sc │ └── test │ └── scala │ └── coinyser │ ├── BatchProducerAppIntelliJ.scala │ ├── BatchProducerIT.scala │ ├── BatchProducerSpec.scala │ ├── FakePusher.scala │ ├── StreamingConsumerApp.scala │ ├── StreamingConsumerSpec.scala │ ├── StreamingProducerApp.scala │ ├── StreamingProducerSpec.scala │ └── pusher-subscribe.sc ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .idea 3 | -------------------------------------------------------------------------------- /Chapter01/build.sbt: -------------------------------------------------------------------------------- 1 | name := "Examples" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.4" 6 | 7 | resolvers += "Artima Maven Repository" at "http://repo.artima.com/releases" 8 | 9 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" -------------------------------------------------------------------------------- /Chapter01/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.0.4 -------------------------------------------------------------------------------- /Chapter01/project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter01/project/plugins.sbt -------------------------------------------------------------------------------- /Chapter01/src/main/scala/Main.scala: -------------------------------------------------------------------------------- 1 | object Main extends App { 2 | val persons = List( 3 | Person(firstName = "Akira", lastName = "Sakura", age = 12), 4 | Person(firstName = "Peter", lastName = "Müller", age = 34), 5 | Person(firstName = "Nick", lastName = "Tagart", age = 52)) 6 | val adults = Person.filterAdult(persons) 7 | val descriptions = adults.map(p => p.description).mkString("\n\t") 8 | println(s"The adults are \n\t$descriptions") 9 | } 10 | -------------------------------------------------------------------------------- /Chapter01/src/main/scala/Person.scala: -------------------------------------------------------------------------------- 1 | case class Person(firstName: String, lastName: String, age: Int) { 2 | def description = s"$firstName $lastName is $age ${if (age <= 1) "year" else "years"} old" 3 | } 4 | 5 | object Person { 6 | def filterAdult(persons: List[Person]) : List[Person] = { 7 | persons.filter(_.age >= 18) 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /Chapter01/src/test/scala/MainSpec.scala: -------------------------------------------------------------------------------- 1 | import org.scalatest.{Matchers, WordSpec} 2 | 3 | class MainSpec extends WordSpec with Matchers { 4 | "A Person" should { 5 | "be instantiated with a age and name" in { 6 | val john = Person(firstName = "John", lastName = "Smith", 42) 7 | john.firstName should be("John") 8 | john.lastName should be("Smith") 9 | john.age should be(42) 10 | } 11 | "Get a human readable representation of the person" in { 12 | val paul = Person(firstName = "Paul", lastName = "Smith", age = 24) 13 | paul.description should be("Paul Smith is 24 years old") 14 | } 15 | } 16 | "The Person companion object" should { 17 | val (akira, peter, nick) = ( 18 | Person(firstName = "Akira", lastName = "Sakura", age = 12), 19 | Person(firstName = "Peter", lastName = "Müller", age = 34), 20 | Person(firstName = "Nick", lastName = "Tagart", age = 52) 21 | ) 22 | "return a list of adult person" in { 23 | val ref = List(akira, peter, nick) 24 | Person.filterAdult(ref) should be(List(peter, nick)) 25 | } 26 | "return an empty list if no adult in the list" in { 27 | val ref = List(akira) 28 | Person.filterAdult(ref) should be(List.empty[Person]) 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/.gitignore: -------------------------------------------------------------------------------- 1 | *.odt 2 | *.tex 3 | *.log 4 | target/ 5 | .ensime 6 | .ensime_cache 7 | .~lock* 8 | .idea 9 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/build.sbt: -------------------------------------------------------------------------------- 1 | name := "retirement_calculator" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.4" 6 | 7 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" 8 | 9 | mainClass in Compile := Some("retcalc.SimulatePlanApp") -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.0.4 -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/main/scala/retcalc/EquityData.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import scala.io.Source 4 | 5 | case class EquityData(monthId: String, value: Double, annualDividend: Double) { 6 | val monthlyDividend: Double = annualDividend / 12 7 | } 8 | 9 | object EquityData { 10 | def fromResource(resource: String): Vector[EquityData] = 11 | Source.fromResource(resource).getLines().drop(1).map { line => 12 | val fields = line.split("\t") 13 | EquityData( 14 | monthId = fields(0), 15 | value = fields(1).toDouble, 16 | annualDividend = fields(2).toDouble) 17 | }.toVector 18 | } 19 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/main/scala/retcalc/InflationData.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import scala.io.Source 4 | 5 | case class InflationData(monthId: String, value: Double) 6 | 7 | object InflationData { 8 | def fromResource(resource: String): Vector[InflationData] = 9 | Source.fromResource(resource).getLines().drop(1).map { line => 10 | val fields = line.split("\t") 11 | InflationData(monthId = fields(0), value = fields(1).toDouble) 12 | }.toVector 13 | 14 | } 15 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/main/scala/retcalc/RetCalc.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import scala.annotation.tailrec 4 | 5 | case class RetCalcParams(nbOfMonthsInRetirement: Int, 6 | netIncome: Int, 7 | currentExpenses: Int, 8 | initialCapital: Double) 9 | 10 | 11 | object RetCalc { 12 | 13 | def simulatePlan(returns: Returns, params: RetCalcParams, nbOfMonthsSavings: Int, 14 | monthOffset: Int = 0): (Double, Double) = { 15 | import params._ 16 | val capitalAtRetirement = futureCapital( 17 | returns = OffsetReturns(returns, monthOffset), 18 | nbOfMonths = nbOfMonthsSavings, netIncome = netIncome, currentExpenses = currentExpenses, 19 | initialCapital = initialCapital) 20 | 21 | val capitalAfterDeath = futureCapital( 22 | returns = OffsetReturns(returns, monthOffset + nbOfMonthsSavings), 23 | nbOfMonths = nbOfMonthsInRetirement, 24 | netIncome = 0, currentExpenses = currentExpenses, 25 | initialCapital = capitalAtRetirement) 26 | 27 | (capitalAtRetirement, capitalAfterDeath) 28 | } 29 | 30 | 31 | def nbOfMonthsSaving(params: RetCalcParams, returns: Returns): Int = { 32 | import params._ 33 | @tailrec 34 | def loop(months: Int): Int = { 35 | val (capitalAtRetirement, capitalAfterDeath) = simulatePlan(returns, params, months) 36 | 37 | if (capitalAfterDeath > 0.0) 38 | months 39 | else 40 | loop(months + 1) 41 | } 42 | 43 | if (netIncome > currentExpenses) 44 | loop(0) 45 | else 46 | Int.MaxValue 47 | } 48 | 49 | def futureCapital(returns: Returns, nbOfMonths: Int, netIncome: Int, currentExpenses: Int, 50 | initialCapital: Double): Double = { 51 | val monthlySavings = netIncome - currentExpenses 52 | (0 until nbOfMonths).foldLeft(initialCapital) { 53 | case (accumulated, month) => 54 | accumulated * (1 + Returns.monthlyRate(returns, month)) + monthlySavings 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/main/scala/retcalc/Returns.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | 4 | sealed trait Returns 5 | 6 | object Returns { 7 | def fromEquityAndInflationData(equities: Vector[EquityData], 8 | inflations: Vector[InflationData]): VariableReturns = { 9 | VariableReturns(returns = equities.zip(inflations).sliding(2).collect { 10 | case (prevEquity, prevInflation) +: (equity, inflation) +: Vector() => 11 | val inflationRate = inflation.value / prevInflation.value 12 | val totalReturn = (equity.value + equity.monthlyDividend) / prevEquity.value 13 | val realTotalReturn = totalReturn - inflationRate 14 | 15 | VariableReturn(equity.monthId, realTotalReturn) 16 | }.toVector) 17 | } 18 | 19 | 20 | def monthlyRate(returns: Returns, month: Int): Double = returns match { 21 | case FixedReturns(r) => r / 12 22 | case VariableReturns(rs) => rs(month % rs.length).monthlyRate 23 | case OffsetReturns(rs, offset) => monthlyRate(rs, month + offset) 24 | } 25 | 26 | def annualizedTotalReturn(returns: Returns): Double = returns match { 27 | case FixedReturns(r) => r 28 | case VariableReturns(rs) => 29 | val product = rs.foldLeft(1.0) { case (acc, ret) => 30 | acc * (1 + ret.monthlyRate) 31 | } 32 | (scala.math.pow(product, 1.0 / rs.size) - 1) * 12 33 | case OffsetReturns(rs, _) => annualizedTotalReturn(rs) 34 | } 35 | 36 | } 37 | 38 | case class FixedReturns(annualRate: Double) extends Returns 39 | 40 | case class VariableReturns(returns: Vector[VariableReturn]) extends Returns { 41 | def fromUntil(monthIdFrom: String, monthIdUntil: String): VariableReturns = 42 | VariableReturns( 43 | returns 44 | .dropWhile(_.monthId != monthIdFrom) 45 | .takeWhile(_.monthId != monthIdUntil)) 46 | } 47 | 48 | case class VariableReturn(monthId: String, monthlyRate: Double) 49 | case class OffsetReturns(orig: Returns, offset: Int) extends Returns 50 | 51 | 52 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/main/scala/retcalc/SimulatePlanApp.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | object SimulatePlanApp extends App { 4 | println(strMain(args)) 5 | 6 | def strMain(args: Array[String]): String = { 7 | val (from +: until +: Nil) = args(0).split(",").toList 8 | val nbOfYearsSaving = args(1).toInt 9 | val nbOfYearsInRetirement = args(2).toInt 10 | 11 | val allReturns = Returns.fromEquityAndInflationData( 12 | equities = EquityData.fromResource("sp500.tsv"), 13 | inflations = InflationData.fromResource("cpi.tsv")) 14 | val (capitalAtRetirement, capitalAfterDeath) = RetCalc.simulatePlan( 15 | returns = allReturns.fromUntil(from, until), 16 | params = RetCalcParams( 17 | nbOfMonthsInRetirement = nbOfYearsInRetirement * 12, 18 | netIncome = args(3).toInt, 19 | currentExpenses = args(4).toInt, 20 | initialCapital = args(5).toInt), 21 | nbOfMonthsSavings = nbOfYearsSaving * 12) 22 | 23 | s""" 24 | |Capital after $nbOfYearsSaving years of savings: ${capitalAtRetirement.round} 25 | |Capital after $nbOfYearsInRetirement years in retirement: ${capitalAfterDeath.round} 26 | """.stripMargin 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/resources/.chapter3_validated.txt.swp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter02/retirement-calculator/src/test/resources/.chapter3_validated.txt.swp -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/resources/chapter3_validated.txt: -------------------------------------------------------------------------------- 1 | We are now ... further. We are going to improve the SimulatePlanApp to give the users more information if one or many arguments passed to the program are wrong. 2 | When many arguments are wrong, for instance if the user passes some text instead of an Int, we want to report one error for every bad argument. 3 | 4 | Firstly, we need to change the test associated with SimulatePlanApp. Open SimulatePlanAppIT.scala and change as follows: 5 | 6 | 7 | (bullet points) 8 | The first two tests do not change much. We just changed the expectation to be a Valid(expectedResult). We are indeed going to change the return type of SimulatePlanApp.strMain: instead of returning a String, we are going to change it to return a Validated[String, String]. We expect strMain to return a Valid value containing the result if all arguments are correct. If some arguments are incorrect, it should return an Invalid value containing a String explaining what are the incorrect arguments(s). 9 | The third test is a new test. If we do not pass the right number of arguments, we expect strMain to return an Invalid value containng an usage example. 10 | The fourth test checks that one error for every bad argument is reported. 11 | 12 | The next step is to change strMain to validate the arguments. For this, we start by writing a smaller function that parses one String argument to produce an Int: 13 | 14 | We call the method catchOnly on Validated, which executes a block of code (in our case value.toInt) and catches a specific type of exception. If the block does not throw any exception, catchOnly returns a Valid value with the result. If the block throws the exception type passed as argument, catchOnly returns an Invalid value containing the exception. In our function we obtain a Validated[NumberFormatException, Int]. However our function parseInt must return a ValidatedNel[RetCalcError, Int]. In order to transform the error or "left" type, we call leftMap to produce a NonEmptyList[RetCalcError]. 15 | 16 | 17 | 18 | 19 | TODO talk about best way to read the book: 20 | 1. Read without trying any of the examples 21 | 2. Retype the examples. Better: get familiar with the environment, see how the autocomplete works, learn by making a few mistakes 22 | 3. Remake the examples without looking at the book 23 | 4. Implement extra features 24 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/resources/cpi_2017.tsv: -------------------------------------------------------------------------------- 1 | month cpi 2 | 2016.09 241.428 3 | 2016.10 241.729 4 | 2016.11 241.353 5 | 2016.12 241.432 6 | 2017.01 242.839 7 | 2017.02 243.603 8 | 2017.03 243.801 9 | 2017.04 244.524 10 | 2017.05 244.733 11 | 2017.06 244.955 12 | 2017.07 244.786 13 | 2017.08 245.519 14 | 2017.09 246.819 -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/resources/sp500_2017.tsv: -------------------------------------------------------------------------------- 1 | month SP500 dividend 2 | 2016.09 2157.69 45.03 3 | 2016.10 2143.02 45.25 4 | 2016.11 2164.99 45.48 5 | 2016.12 2246.63 45.7 6 | 2017.01 2275.12 45.93 7 | 2017.02 2329.91 46.15 8 | 2017.03 2366.82 46.38 9 | 2017.04 2359.31 46.66 10 | 2017.05 2395.35 46.94 11 | 2017.06 2433.99 47.22 12 | 2017.07 2454.10 47.54 13 | 2017.08 2456.22 47.85 14 | 2017.09 2492.84 48.17 15 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/EquityDataSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalatest.{Matchers, WordSpec} 4 | 5 | class EquityDataSpec extends WordSpec with Matchers { 6 | "EquityData.fromResource" should { 7 | "load market data from a tsv file" in { 8 | val data = EquityData.fromResource("sp500_2017.tsv") 9 | data should ===(Vector( 10 | EquityData("2016.09", 2157.69, 45.03), 11 | EquityData("2016.10", 2143.02, 45.25), 12 | EquityData("2016.11", 2164.99, 45.48), 13 | EquityData("2016.12", 2246.63, 45.7), 14 | EquityData("2017.01", 2275.12, 45.93), 15 | EquityData("2017.02", 2329.91, 46.15), 16 | EquityData("2017.03", 2366.82, 46.38), 17 | EquityData("2017.04", 2359.31, 46.66), 18 | EquityData("2017.05", 2395.35, 46.94), 19 | EquityData("2017.06", 2433.99, 47.22), 20 | EquityData("2017.07", 2454.10, 47.54), 21 | EquityData("2017.08", 2456.22, 47.85), 22 | EquityData("2017.09", 2492.84, 48.17) 23 | )) 24 | } 25 | } 26 | 27 | "EquityData.monthlyDividend" should { 28 | "return a monthly dividend" in { 29 | EquityData("2016.09", 2157.69, 45.03).monthlyDividend should === (45.03 / 12) 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/InflationDataSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalatest.{Matchers, WordSpec} 4 | 5 | class InflationDataSpec extends WordSpec with Matchers { 6 | "InflationData.fromResource" should { 7 | "load CPI data from a tsv file" in { 8 | val data = InflationData.fromResource("cpi_2017.tsv") 9 | data should ===(Vector( 10 | InflationData("2016.09", 241.428), 11 | InflationData("2016.10", 241.729), 12 | InflationData("2016.11", 241.353), 13 | InflationData("2016.12", 241.432), 14 | InflationData("2017.01", 242.839), 15 | InflationData("2017.02", 243.603), 16 | InflationData("2017.03", 243.801), 17 | InflationData("2017.04", 244.524), 18 | InflationData("2017.05", 244.733), 19 | InflationData("2017.06", 244.955), 20 | InflationData("2017.07", 244.786), 21 | InflationData("2017.08", 245.519), 22 | InflationData("2017.09", 246.819) 23 | )) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/RetCalcIT.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class RetCalcIT extends WordSpec with Matchers with TypeCheckedTripleEquals { 7 | implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001) 8 | 9 | val params = RetCalcParams( 10 | nbOfMonthsInRetirement = 40 * 12, 11 | netIncome = 3000, 12 | currentExpenses = 2000, 13 | initialCapital = 10000) 14 | 15 | 16 | "simulate a retirement plan with real market data" in { 17 | val returns = Returns.fromEquityAndInflationData( 18 | equities = EquityData.fromResource("sp500.tsv"), 19 | inflations = InflationData.fromResource("cpi.tsv")).fromUntil("1952.09", "2017.10") 20 | 21 | val (capitalAtRetirement, capitalAfterDeath) = 22 | RetCalc.simulatePlan(returns, params = params, nbOfMonthsSavings = 25 * 12) 23 | capitalAtRetirement should ===(468924.5522) 24 | capitalAfterDeath should ===(2958841.7675) 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/RetCalcSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class RetCalcSpec extends WordSpec with Matchers with TypeCheckedTripleEquals { 7 | 8 | implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001) 9 | 10 | "RetCalc.futureCapital" should { 11 | "calculate the amount of savings I will have in n months" in { 12 | // Excel =-FV(0.04/12,25*12,1000,10000,0) 13 | val actual = RetCalc.futureCapital(FixedReturns(0.04), nbOfMonths = 25 * 12, 14 | netIncome = 3000, currentExpenses = 2000, initialCapital = 10000) 15 | val expected = 541267.1990 16 | actual should ===(expected) 17 | } 18 | 19 | "calculate how much savings will be left after having taken a pension for n months" in { 20 | val actual = RetCalc.futureCapital(FixedReturns(0.04), nbOfMonths = 40 * 12, 21 | netIncome = 0, currentExpenses = 2000, initialCapital = 541267.198962) 22 | val expected = 309867.5316 23 | actual should ===(expected) 24 | } 25 | } 26 | 27 | val params = RetCalcParams( 28 | nbOfMonthsInRetirement = 40 * 12, 29 | netIncome = 3000, 30 | currentExpenses = 2000, 31 | initialCapital = 10000) 32 | 33 | "RetCalc.simulatePlan" should { 34 | "calculate the capital at retirement and the capital after death" in { 35 | val (capitalAtRetirement, capitalAfterDeath) = RetCalc.simulatePlan( 36 | returns = FixedReturns(0.04), params, nbOfMonthsSavings = 25 * 12) 37 | 38 | capitalAtRetirement should ===(541267.1990) 39 | capitalAfterDeath should ===(309867.5316) 40 | } 41 | 42 | "use different returns for capitalisation and drawdown" in { 43 | val nbOfMonthsSavings = 25 * 12 44 | val returns = VariableReturns( 45 | Vector.tabulate(nbOfMonthsSavings + params.nbOfMonthsInRetirement)(i => 46 | if (i < nbOfMonthsSavings) 47 | VariableReturn(i.toString, 0.04 / 12) 48 | else 49 | VariableReturn(i.toString, 0.03 / 12))) 50 | val (capitalAtRetirement, capitalAfterDeath) = 51 | RetCalc.simulatePlan(returns, params, nbOfMonthsSavings) 52 | // Excel: =-FV(0.04/12, 25*12, 1000, 10000) 53 | capitalAtRetirement should ===(541267.1990) 54 | // Excel: =-FV(0.03/12, 40*12, -2000, 541267.20) 55 | capitalAfterDeath should ===(-57737.7227) 56 | } 57 | 58 | } 59 | 60 | 61 | "RetCalc.nbOfMonthsSaving" should { 62 | "calculate how long I need to save before I can retire" in { 63 | val actual = RetCalc.nbOfMonthsSaving(params, FixedReturns(0.04)) 64 | val expected = 23 * 12 + 1 65 | actual should ===(expected) 66 | } 67 | 68 | "not crash if the resulting nbOfMonths is very high" in { 69 | val actual = RetCalc.nbOfMonthsSaving( 70 | params = RetCalcParams( 71 | nbOfMonthsInRetirement = 40 * 12, 72 | netIncome = 3000, currentExpenses = 2999, initialCapital = 0), 73 | returns = FixedReturns(0.01)) 74 | val expected = 8280 75 | actual should ===(expected) 76 | } 77 | 78 | "not loop forever if I enter bad parameters" in { 79 | val actual = RetCalc.nbOfMonthsSaving(params.copy(netIncome = 1000), FixedReturns(0.04)) 80 | actual should ===(Int.MaxValue) 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/ReturnsSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class ReturnsSpec extends WordSpec with Matchers with TypeCheckedTripleEquals { 7 | 8 | implicit val doubleEquality: Equality[Double] = 9 | TolerantNumerics.tolerantDoubleEquality(0.0001) 10 | 11 | "Returns.monthlyReturn" should { 12 | "return a fixed rate for a FixedReturn" in { 13 | Returns.monthlyRate(FixedReturns(0.04), 0) should ===(0.04 / 12) 14 | Returns.monthlyRate(FixedReturns(0.04), 10) should ===(0.04 / 12) 15 | } 16 | 17 | val variableReturns = VariableReturns( 18 | Vector(VariableReturn("2000.01", 0.1), VariableReturn("2000.02", 0.2))) 19 | "return the nth rate for VariableReturn" in { 20 | Returns.monthlyRate(variableReturns, 0) should ===(0.1) 21 | Returns.monthlyRate(variableReturns, 1) should ===(0.2) 22 | } 23 | 24 | "roll over from the first rate if n > length" in { 25 | Returns.monthlyRate(variableReturns, 2) should ===(0.1) 26 | Returns.monthlyRate(variableReturns, 3) should ===(0.2) 27 | Returns.monthlyRate(variableReturns, 4) should ===(0.1) 28 | } 29 | 30 | "return the n+offset th rate for OffsetReturn" in { 31 | val returns = OffsetReturns(variableReturns, 1) 32 | Returns.monthlyRate(returns, 0) should ===(0.2) 33 | Returns.monthlyRate(returns, 1) should ===(0.1) 34 | } 35 | } 36 | 37 | 38 | "Returns.fromEquityAndInflationData" should { 39 | "compute real total returns from equity and inflation data" in { 40 | val equities = Vector( 41 | EquityData("2117.01", 100.0, 10.0), 42 | EquityData("2117.02", 101.0, 12.0), 43 | EquityData("2117.03", 102.0, 12.0)) 44 | 45 | val inflations = Vector( 46 | InflationData("2117.01", 100.0), 47 | InflationData("2117.02", 102.0), 48 | InflationData("2117.03", 102.0)) 49 | 50 | val returns = Returns.fromEquityAndInflationData(equities, inflations) 51 | returns should ===(VariableReturns(Vector( 52 | VariableReturn("2117.02", (101.0 + 12.0 / 12) / 100.0 - 102.0 / 100.0), 53 | VariableReturn("2117.03", (102.0 + 12.0 / 12) / 101.0 - 102.0 / 102.0)))) 54 | } 55 | } 56 | 57 | "VariableReturns.fromUntil" should { 58 | "keep only a window of the returns" in { 59 | val variableReturns = VariableReturns(Vector.tabulate(12) { i => 60 | val d = (i + 1).toDouble 61 | VariableReturn(f"2017.$d%02.0f", d) 62 | }) 63 | 64 | variableReturns.fromUntil("2017.07", "2017.09").returns should ===(Vector( 65 | VariableReturn("2017.07", 7.0), 66 | VariableReturn("2017.08", 8.0) 67 | )) 68 | 69 | variableReturns.fromUntil("2017.10", "2018.01").returns should ===(Vector( 70 | VariableReturn("2017.10", 10.0), 71 | VariableReturn("2017.11", 11.0), 72 | VariableReturn("2017.12", 12.0) 73 | )) 74 | } 75 | } 76 | 77 | "Returns.annualizedTotalReturn" should { 78 | val returns = VariableReturns(Vector.tabulate(12)(i => VariableReturn(i.toString, i.toDouble / 100 / 12))) 79 | val avg = Returns.annualizedTotalReturn(returns) 80 | "compute a geometric mean of the returns" in { 81 | // Excel: GEOMEAN (see geomean.ods) 82 | avg should ===(0.0549505735) 83 | } 84 | 85 | "compute an average that can be used to calculate a futureCapital instead of using variable returns" in { 86 | // This calculation only works if the capital does not change over time 87 | // otherwise, the capital fluctuates as well as the interest rates, and we cannot use the mean 88 | val futCapVar = RetCalc.futureCapital(returns, 12, 0, 0, 500000) 89 | val futCapFix = RetCalc.futureCapital(FixedReturns(avg), 12, 0, 0, 500000) 90 | futCapVar should ===(futCapFix) 91 | } 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /Chapter02/retirement-calculator/src/test/scala/retcalc/SimulatePlanAppIT.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.TypeCheckedTripleEquals 4 | import org.scalatest.{Matchers, WordSpec} 5 | 6 | class SimulatePlanAppIT extends WordSpec with Matchers with TypeCheckedTripleEquals { 7 | "SimulatePlanApp.strMain" should { 8 | "simulate a retirement plan using market returns" in { 9 | val actualResult = SimulatePlanApp.strMain( 10 | Array("1952.09,2017.09", "25", "40", "3000", "2000", "10000")) 11 | 12 | val expectedResult = 13 | s""" 14 | |Capital after 25 years of savings: 468925 15 | |Capital after 40 years in retirement: 2958842 16 | """.stripMargin 17 | actualResult should === (expectedResult) 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/build.sbt: -------------------------------------------------------------------------------- 1 | name := "retirement-calculator" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.4" 6 | 7 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" 8 | libraryDependencies += "org.typelevel" %% "cats-core" % "1.0.1" 9 | scalacOptions += "-Ypartial-unification" 10 | 11 | mainClass in Compile := Some("retcalc.SimulatePlanApp") -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.6") -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.0.4 -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/EquityData.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import scala.io.Source 4 | 5 | case class EquityData(monthId: String, value: Double, annualDividend: Double) { 6 | val monthlyDividend: Double = annualDividend / 12 7 | } 8 | 9 | object EquityData { 10 | def fromResource(resource: String): Vector[EquityData] = 11 | Source.fromResource(resource).getLines().drop(1).map { line => 12 | val fields = line.split("\t") 13 | EquityData( 14 | monthId = fields(0), 15 | value = fields(1).toDouble, 16 | annualDividend = fields(2).toDouble) 17 | }.toVector 18 | } 19 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/InflationData.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import scala.io.Source 4 | 5 | case class InflationData(monthId: String, value: Double) 6 | 7 | object InflationData { 8 | def fromResource(resource: String): Vector[InflationData] = 9 | Source.fromResource(resource).getLines().drop(1).map { line => 10 | val fields = line.split("\t") 11 | InflationData(monthId = fields(0), value = fields(1).toDouble) 12 | }.toVector 13 | 14 | } 15 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/RetCalc.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import retcalc.RetCalcError.MoreExpensesThanIncome 4 | 5 | import scala.annotation.tailrec 6 | 7 | case class RetCalcParams(nbOfMonthsInRetirement: Int, 8 | netIncome: Int, 9 | currentExpenses: Int, 10 | initialCapital: Double) 11 | 12 | 13 | case class MultiSimResults(successCount: Int, 14 | simCount: Int, 15 | minCapitalAfterDeath: Double, 16 | maxCapitalAfterDeath: Double) { 17 | def successProbability: Double = successCount.toDouble / simCount 18 | } 19 | 20 | 21 | object RetCalc { 22 | 23 | def simulatePlan(returns: Returns, params: RetCalcParams, nbOfMonthsSavings: Int, 24 | monthOffset: Int = 0): Either[RetCalcError, (Double, Double)] = { 25 | import params._ 26 | 27 | for { 28 | capitalAtRetirement <- futureCapital( 29 | returns = OffsetReturns(returns, monthOffset), 30 | nbOfMonths = nbOfMonthsSavings, netIncome = netIncome, currentExpenses = currentExpenses, 31 | initialCapital = initialCapital) 32 | 33 | capitalAfterDeath <- futureCapital( 34 | returns = OffsetReturns(returns, monthOffset + nbOfMonthsSavings), 35 | nbOfMonths = nbOfMonthsInRetirement, 36 | netIncome = 0, currentExpenses = currentExpenses, 37 | initialCapital = capitalAtRetirement) 38 | } yield (capitalAtRetirement, capitalAfterDeath) 39 | } 40 | 41 | 42 | def nbOfMonthsSaving(params: RetCalcParams, returns: Returns): Either[RetCalcError, Int] = { 43 | import params._ 44 | @tailrec 45 | def loop(months: Int): Either[RetCalcError, Int] = { 46 | simulatePlan(returns, params, months) match { 47 | case Right((capitalAtRetirement, capitalAfterDeath)) => 48 | if (capitalAfterDeath > 0.0) 49 | Right(months) 50 | else 51 | loop(months + 1) 52 | 53 | case Left(err) => Left(err) 54 | } 55 | } 56 | 57 | if (netIncome > currentExpenses) 58 | loop(0) 59 | else 60 | Left(MoreExpensesThanIncome(netIncome, currentExpenses)) 61 | } 62 | 63 | 64 | def futureCapital(returns: Returns, nbOfMonths: Int, netIncome: Int, currentExpenses: Int, 65 | initialCapital: Double): Either[RetCalcError, Double] = { 66 | val monthlySavings = netIncome - currentExpenses 67 | (0 until nbOfMonths).foldLeft[Either[RetCalcError, Double]](Right(initialCapital)) { 68 | case (accumulated, month) => 69 | for { 70 | acc <- accumulated 71 | monthlyRate <- Returns.monthlyRate(returns, month) 72 | } yield acc * (1 + monthlyRate) + monthlySavings 73 | } 74 | } 75 | 76 | def multiSim(params: RetCalcParams, nbOfMonthsSavings: Int, variableReturns: VariableReturns): MultiSimResults = { 77 | variableReturns.returns.indices.foldLeft(MultiSimResults(0, 0, Double.PositiveInfinity, Double.NegativeInfinity)) { 78 | case (acc, i) => 79 | simulatePlan(variableReturns, params, nbOfMonthsSavings, i) match { 80 | case Right((capitalAtRetirement, capitalAfterDeath)) => 81 | MultiSimResults( 82 | successCount = if (capitalAfterDeath > 0) acc.successCount + 1 else acc.successCount, 83 | simCount = i + 1, 84 | minCapitalAfterDeath = if (capitalAfterDeath < acc.minCapitalAfterDeath) capitalAfterDeath else acc.minCapitalAfterDeath, 85 | maxCapitalAfterDeath = if (capitalAfterDeath > acc.maxCapitalAfterDeath) capitalAfterDeath else acc.maxCapitalAfterDeath) 86 | 87 | // Could have a more clever rule which would reduce params.nbOfMonthsInRetirement. 88 | // say If the capital after nbOfMonthsInRetirement/2 is > 2*capitalAtRetirement, 89 | // it is very likely that we can count it as a success, even if we cannot run 90 | // the simulation until the end 91 | case Left(err) => acc 92 | } 93 | } 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/RetCalcError.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import cats.data.ValidatedNel 4 | 5 | sealed abstract class RetCalcError(val message: String) 6 | 7 | object RetCalcError { 8 | type RetCalcResult[A] = ValidatedNel[RetCalcError, A] 9 | 10 | case class MoreExpensesThanIncome(income: Double, expenses: Double) extends RetCalcError( 11 | s"Expenses: $expenses >= $income. You will never be able to save enough to retire !") 12 | 13 | case class ReturnMonthOutOfBounds(month: Int, maximum: Int) extends RetCalcError( 14 | s"Cannot get the return for month $month. Accepted range: 0 to $maximum") 15 | 16 | case class InvalidNumber(name: String, value: String) extends RetCalcError( 17 | s"Invalid number for $name: $value") 18 | 19 | case class InvalidArgument(name: String, value: String, expectedFormat: String) extends RetCalcError( 20 | s"Invalid format for $name. Expected: $expectedFormat, actual: $value") 21 | 22 | } 23 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/Returns.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | 4 | sealed trait Returns 5 | 6 | object Returns { 7 | def fromEquityAndInflationData(equities: Vector[EquityData], 8 | inflations: Vector[InflationData]) 9 | : VariableReturns = { 10 | VariableReturns(equities.zip(inflations).sliding(2).collect { 11 | case (prevEquity, prevInflation) +: (equity, inflation) +: 12 | Vector() => 13 | val inflationRate = inflation.value / prevInflation.value 14 | val totalReturn = 15 | (equity.value + equity.monthlyDividend) / prevEquity.value 16 | val realTotalReturn = totalReturn - inflationRate 17 | 18 | VariableReturn(equity.monthId, realTotalReturn) 19 | }.toVector) 20 | } 21 | 22 | 23 | def monthlyRate(returns: Returns, month: Int): Either[RetCalcError, Double] = returns match { 24 | case FixedReturns(r) => Right(r / 12) 25 | 26 | case VariableReturns(rs) => 27 | if (rs.isDefinedAt(month)) 28 | Right(rs(month).monthlyRate) 29 | else 30 | Left(RetCalcError.ReturnMonthOutOfBounds(month, rs.size - 1)) 31 | 32 | case OffsetReturns(rs, offset) => monthlyRate(rs, month + offset) 33 | } 34 | 35 | def annualizedTotalReturn(returns: Returns): Double = returns match { 36 | case FixedReturns(r) => r 37 | case VariableReturns(rs) => 38 | val product = rs.foldLeft(1.0) { case (acc, ret) => 39 | acc * (1 + ret.monthlyRate) 40 | } 41 | (scala.math.pow(product, 1.0 / rs.size) - 1) * 12 42 | case OffsetReturns(rs, _) => annualizedTotalReturn(rs) 43 | } 44 | 45 | } 46 | 47 | case class FixedReturns(annualRate: Double) extends Returns 48 | 49 | case class VariableReturns(returns: Vector[VariableReturn]) extends Returns { 50 | def fromUntil(monthIdFrom: String, monthIdUntil: String): VariableReturns = 51 | VariableReturns( 52 | returns 53 | .dropWhile(_.monthId != monthIdFrom) 54 | .takeWhile(_.monthId != monthIdUntil)) 55 | } 56 | 57 | case class VariableReturn(monthId: String, monthlyRate: Double) 58 | 59 | case class OffsetReturns(orig: Returns, offset: Int) extends Returns 60 | 61 | 62 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/main/scala/retcalc/SimulatePlanApp.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import cats.data.Validated._ 4 | import cats.data.{NonEmptyList, Validated, ValidatedNel} 5 | import cats.implicits._ 6 | import retcalc.RetCalcError.{InvalidArgument, InvalidNumber, RetCalcResult} 7 | 8 | object SimulatePlanApp extends App { 9 | strMain(args) match { 10 | case Invalid(err) => 11 | println(err) 12 | sys.exit(1) 13 | 14 | case Valid(result) => 15 | println(result) 16 | sys.exit(0) 17 | } 18 | 19 | def parseInt(name: String, value: String): RetCalcResult[Int] = 20 | Validated 21 | .catchOnly[NumberFormatException](value.toInt) 22 | .leftMap(_ => NonEmptyList.of(InvalidNumber(name, value))) 23 | 24 | def parseParams(args: Array[String]): RetCalcResult[RetCalcParams] = 25 | ( 26 | parseInt("nbOfYearsRetired", args(2)), 27 | parseInt("netIncome", args(3)), 28 | parseInt("currentExpenses", args(4)), 29 | parseInt("initialCapital", args(5)) 30 | ).mapN { case (nbOfYearsRetired, netIncome, currentExpenses, initialCapital) => 31 | RetCalcParams( 32 | nbOfMonthsInRetirement = nbOfYearsRetired * 12, 33 | netIncome = netIncome, 34 | currentExpenses = currentExpenses, 35 | initialCapital = initialCapital) 36 | } 37 | 38 | def parseFromUntil(fromUntil: String): RetCalcResult[(String, String)] = { 39 | val array = fromUntil.split(",") 40 | if (array.length != 2) 41 | InvalidArgument(name = "fromUntil", value = fromUntil, expectedFormat = "from,until" 42 | ).invalidNel 43 | else 44 | (array(0), array(1)).validNel 45 | } 46 | 47 | def strSimulatePlan(returns: Returns, nbOfYearsSaving: Int, params: RetCalcParams) 48 | : RetCalcResult[String] = { 49 | RetCalc.simulatePlan( 50 | returns = returns, 51 | params = params, 52 | nbOfMonthsSavings = nbOfYearsSaving * 12 53 | ).map { 54 | case (capitalAtRetirement, capitalAfterDeath) => 55 | val nbOfYearsInRetirement = params.nbOfMonthsInRetirement / 12 56 | s""" 57 | |Capital after $nbOfYearsSaving years of savings: ${capitalAtRetirement.round} 58 | |Capital after $nbOfYearsInRetirement years in retirement: ${capitalAfterDeath.round} 59 | |""".stripMargin 60 | }.toValidatedNel 61 | } 62 | 63 | 64 | def strMain(args: Array[String]): Validated[String, String] = { 65 | if (args.length != 6) 66 | """Usage: 67 | |simulatePlan from,until nbOfYearsSaving nbOfYearsRetired netIncome currentExpenses initialCapital 68 | | 69 | |Example: 70 | |simulatePlan 1952.09,2017.09 25 40 3000 2000 10000 71 | |""".stripMargin.invalid 72 | else { 73 | val allReturns = Returns.fromEquityAndInflationData( 74 | equities = EquityData.fromResource("sp500.tsv"), 75 | inflations = InflationData.fromResource("cpi.tsv")) 76 | 77 | val vFromUntil = parseFromUntil(args(0)) 78 | val vNbOfYearsSaving = parseInt("nbOfYearsSaving", args(1)) 79 | val vParams = parseParams(args) 80 | 81 | (vFromUntil, vNbOfYearsSaving, vParams) 82 | .tupled 83 | .andThen { case ((from, until), nbOfYearsSaving, params) => 84 | strSimulatePlan(allReturns.fromUntil(from, until), nbOfYearsSaving, params) 85 | } 86 | .leftMap(nel => nel.map(_.message).toList.mkString("\n")) 87 | } 88 | } 89 | } 90 | 91 | case class SimulatePlanArgs(fromMonth: String, 92 | untilMonth: String, 93 | retCalcParams: RetCalcParams, 94 | nbOfMonthsSavings: Int) 95 | 96 | 97 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/resources/cpi_2017.tsv: -------------------------------------------------------------------------------- 1 | month cpi 2 | 2016.09 241.428 3 | 2016.10 241.729 4 | 2016.11 241.353 5 | 2016.12 241.432 6 | 2017.01 242.839 7 | 2017.02 243.603 8 | 2017.03 243.801 9 | 2017.04 244.524 10 | 2017.05 244.733 11 | 2017.06 244.955 12 | 2017.07 244.786 13 | 2017.08 245.519 14 | 2017.09 246.819 -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/resources/sp500_2017.tsv: -------------------------------------------------------------------------------- 1 | month SP500 dividend 2 | 2016.09 2157.69 45.03 3 | 2016.10 2143.02 45.25 4 | 2016.11 2164.99 45.48 5 | 2016.12 2246.63 45.7 6 | 2017.01 2275.12 45.93 7 | 2017.02 2329.91 46.15 8 | 2017.03 2366.82 46.38 9 | 2017.04 2359.31 46.66 10 | 2017.05 2395.35 46.94 11 | 2017.06 2433.99 47.22 12 | 2017.07 2454.10 47.54 13 | 2017.08 2456.22 47.85 14 | 2017.09 2492.84 48.17 15 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/EquityDataSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalatest.{Matchers, WordSpec} 4 | 5 | class EquityDataSpec extends WordSpec with Matchers { 6 | "EquityData.fromResource" should { 7 | "load market data from a tsv file" in { 8 | val data = EquityData.fromResource("sp500_2017.tsv") 9 | data should ===(Vector( 10 | EquityData("2016.09", 2157.69, 45.03), 11 | EquityData("2016.10", 2143.02, 45.25), 12 | EquityData("2016.11", 2164.99, 45.48), 13 | EquityData("2016.12", 2246.63, 45.7), 14 | EquityData("2017.01", 2275.12, 45.93), 15 | EquityData("2017.02", 2329.91, 46.15), 16 | EquityData("2017.03", 2366.82, 46.38), 17 | EquityData("2017.04", 2359.31, 46.66), 18 | EquityData("2017.05", 2395.35, 46.94), 19 | EquityData("2017.06", 2433.99, 47.22), 20 | EquityData("2017.07", 2454.10, 47.54), 21 | EquityData("2017.08", 2456.22, 47.85), 22 | EquityData("2017.09", 2492.84, 48.17) 23 | )) 24 | } 25 | } 26 | 27 | "EquityData.monthlyDividend" should { 28 | "return a monthly dividend" in { 29 | EquityData("2016.09", 2157.69, 45.03).monthlyDividend should === (45.03 / 12) 30 | } 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/InflationDataSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalatest.{Matchers, WordSpec} 4 | 5 | class InflationDataSpec extends WordSpec with Matchers { 6 | "InflationData.fromResource" should { 7 | "load CPI data from a tsv file" in { 8 | val data = InflationData.fromResource("cpi_2017.tsv") 9 | data should ===(Vector( 10 | InflationData("2016.09", 241.428), 11 | InflationData("2016.10", 241.729), 12 | InflationData("2016.11", 241.353), 13 | InflationData("2016.12", 241.432), 14 | InflationData("2017.01", 242.839), 15 | InflationData("2017.02", 243.603), 16 | InflationData("2017.03", 243.801), 17 | InflationData("2017.04", 244.524), 18 | InflationData("2017.05", 244.733), 19 | InflationData("2017.06", 244.955), 20 | InflationData("2017.07", 244.786), 21 | InflationData("2017.08", 245.519), 22 | InflationData("2017.09", 246.819) 23 | )) 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/RetCalcIT.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{EitherValues, Matchers, WordSpec} 5 | 6 | class RetCalcIT extends WordSpec with Matchers with TypeCheckedTripleEquals with EitherValues { 7 | implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001) 8 | 9 | val params = RetCalcParams( 10 | nbOfMonthsInRetirement = 40 * 12, 11 | netIncome = 3000, 12 | currentExpenses = 2000, 13 | initialCapital = 10000) 14 | 15 | 16 | "RetCalc.simulatePlan" should { 17 | "simulate a retirement plan with real market data" in { 18 | val returns = Returns.fromEquityAndInflationData( 19 | equities = EquityData.fromResource("sp500.tsv"), 20 | inflations = InflationData.fromResource("cpi.tsv")).fromUntil("1952.09", "2017.10") 21 | 22 | val (capitalAtRetirement, capitalAfterDeath) = 23 | RetCalc.simulatePlan(returns, params = params, nbOfMonthsSavings = 25 * 12).right.value 24 | capitalAtRetirement should ===(468924.5522) 25 | capitalAfterDeath should ===(2958841.7675) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/RetCalcSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{EitherValues, Matchers, WordSpec} 5 | 6 | class RetCalcSpec extends WordSpec with Matchers with TypeCheckedTripleEquals 7 | with EitherValues { 8 | 9 | implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001) 10 | 11 | "RetCalc.futureCapital" should { 12 | "calculate the amount of savings I will have in n months" in { 13 | // Excel =-FV(0.04/12,25*12,1000,10000,0) 14 | val actual = RetCalc.futureCapital(FixedReturns(0.04), 15 | nbOfMonths = 25 * 12, netIncome = 3000, 16 | currentExpenses = 2000, initialCapital = 10000).right.value 17 | val expected = 541267.1990 18 | actual should ===(expected) 19 | } 20 | 21 | "calculate how much savings will be left after having taken a pension for n months" in { 22 | val actual = RetCalc.futureCapital(FixedReturns(0.04), 23 | nbOfMonths = 40 * 12, netIncome = 0, currentExpenses = 2000, 24 | initialCapital = 541267.198962).right.value 25 | val expected = 309867.5316 26 | actual should ===(expected) 27 | } 28 | } 29 | 30 | val params = RetCalcParams( 31 | nbOfMonthsInRetirement = 40 * 12, 32 | netIncome = 3000, 33 | currentExpenses = 2000, 34 | initialCapital = 10000) 35 | 36 | "RetCalc.simulatePlan" should { 37 | "calculate the capital at retirement and the capital after death" in { 38 | val (capitalAtRetirement, capitalAfterDeath) = RetCalc.simulatePlan( 39 | returns = FixedReturns(0.04), params, nbOfMonthsSavings = 25 * 12).right.value 40 | 41 | capitalAtRetirement should ===(541267.1990) 42 | capitalAfterDeath should ===(309867.5316) 43 | } 44 | 45 | "use different returns for capitalisation and drawdown" in { 46 | val nbOfMonthsSavings = 25 * 12 47 | val returns = VariableReturns( 48 | Vector.tabulate(nbOfMonthsSavings + params.nbOfMonthsInRetirement)(i => 49 | if (i < nbOfMonthsSavings) 50 | VariableReturn(i.toString, 0.04 / 12) 51 | else 52 | VariableReturn(i.toString, 0.03 / 12))) 53 | val (capitalAtRetirement, capitalAfterDeath) = 54 | RetCalc.simulatePlan(returns, params, nbOfMonthsSavings).right.value 55 | // Excel: =-FV(0.04/12, 25*12, 1000, 10000) 56 | capitalAtRetirement should ===(541267.1990) 57 | // Excel: =-FV(0.03/12, 40*12, -2000, 541267.20) 58 | capitalAfterDeath should ===(-57737.7227) 59 | } 60 | 61 | } 62 | 63 | 64 | "RetCalc.nbOfMonthsSaving" should { 65 | "calculate how long I need to save before I can retire" in { 66 | val actual = RetCalc.nbOfMonthsSaving(params, FixedReturns(0.04)).right.value 67 | val expected = 23 * 12 + 1 68 | actual should ===(expected) 69 | } 70 | 71 | "not crash if the resulting nbOfMonths is very high" in { 72 | val actual = RetCalc.nbOfMonthsSaving( 73 | params = RetCalcParams( 74 | nbOfMonthsInRetirement = 40 * 12, 75 | netIncome = 3000, currentExpenses = 2999, initialCapital = 0), 76 | returns = FixedReturns(0.01)).right.value 77 | val expected = 8280 78 | actual should ===(expected) 79 | } 80 | 81 | "not loop forever if I enter bad parameters" in { 82 | val actual = RetCalc.nbOfMonthsSaving( 83 | params.copy(netIncome = 1000), FixedReturns(0.04)).left.value 84 | actual should ===(RetCalcError.MoreExpensesThanIncome(1000, 2000)) 85 | } 86 | } 87 | 88 | 89 | "RetCalc.multiSim" should { 90 | "try different starting months to calculate a probability of success" in { 91 | val nbOfMonthsSavings = 25 * 12 92 | val returns = VariableReturns(Vector.tabulate(80*12)(i => 93 | if (i <= 30*12) 94 | VariableReturn(i.toString, 0.04 / 12) 95 | else 96 | VariableReturn(i.toString, 0.03 / 12) 97 | )) 98 | 99 | val results = RetCalc.multiSim(params, nbOfMonthsSavings, returns) 100 | results should ===(MultiSimResults(21, 181, -205606.47053674585, 25810.339841347573)) 101 | // TODO assert 181 = (80 - 65) * 12 +1 102 | results.successProbability should ===(21.0 / 181) 103 | } 104 | 105 | "return the same results as a simple simulation when the rate does not change" in { 106 | val nbOfMonthsSavings = 25 * 12 107 | val returns = VariableReturns(Vector.fill(65*12 + 9)(VariableReturn("a", 0.04 / 12))) 108 | 109 | val (expectedCapitalAtRetirement, expectedCapitalAfterDeath) = 110 | RetCalc.simulatePlan(returns, params = params, nbOfMonthsSavings = 25 * 12).right.value 111 | val expected = MultiSimResults( 112 | successCount = 10, simCount = 10, 113 | minCapitalAfterDeath = expectedCapitalAfterDeath, 114 | maxCapitalAfterDeath = expectedCapitalAfterDeath) 115 | 116 | val actual = RetCalc.multiSim(params, nbOfMonthsSavings, returns) 117 | 118 | actual should ===(expected) 119 | } 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/ReturnsSpec.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 4 | import org.scalatest.{EitherValues, Matchers, WordSpec} 5 | 6 | class ReturnsSpec extends WordSpec with Matchers with TypeCheckedTripleEquals with EitherValues { 7 | 8 | implicit val doubleEquality: Equality[Double] = 9 | TolerantNumerics.tolerantDoubleEquality(0.0001) 10 | 11 | "Returns.monthlyReturn" should { 12 | "return a fixed rate for a FixedReturn" in { 13 | Returns.monthlyRate(FixedReturns(0.04), 0).right.value should ===(0.04 / 12) 14 | Returns.monthlyRate(FixedReturns(0.04), 10).right.value should ===(0.04 / 12) 15 | } 16 | 17 | val variableReturns = VariableReturns(Vector( 18 | VariableReturn("2000.01", 0.1), 19 | VariableReturn("2000.02", 0.2))) 20 | 21 | "return the nth rate for VariableReturn" in { 22 | Returns.monthlyRate(variableReturns, 0).right.value should ===(0.1) 23 | Returns.monthlyRate(variableReturns, 1).right.value should ===(0.2) 24 | } 25 | 26 | "return an error if n > length" in { 27 | Returns.monthlyRate(variableReturns, 2).left.value should ===( 28 | RetCalcError.ReturnMonthOutOfBounds(2, 1)) 29 | Returns.monthlyRate(variableReturns, 3).left.value should ===( 30 | RetCalcError.ReturnMonthOutOfBounds(3, 1)) 31 | } 32 | 33 | "return the n+offset th rate for OffsetReturn" in { 34 | val returns = OffsetReturns(variableReturns, 1) 35 | Returns.monthlyRate(returns, 0).right.value should ===(0.2) 36 | } 37 | } 38 | 39 | 40 | "Returns.fromEquityAndInflationData" should { 41 | "compute real total returns from equity and inflation data" in { 42 | val equities = Vector( 43 | EquityData("2117.01", 100.0, 10.0), 44 | EquityData("2117.02", 101.0, 12.0), 45 | EquityData("2117.03", 102.0, 12.0)) 46 | 47 | val inflations = Vector( 48 | InflationData("2117.01", 100.0), 49 | InflationData("2117.02", 102.0), 50 | InflationData("2117.03", 102.0)) 51 | 52 | val returns = Returns.fromEquityAndInflationData(equities, inflations) 53 | returns should ===(VariableReturns(Vector( 54 | VariableReturn("2117.02", (101.0 + 12.0 / 12) / 100.0 - 102.0 / 100.0), 55 | VariableReturn("2117.03", (102.0 + 12.0 / 12) / 101.0 - 102.0 / 102.0)))) 56 | } 57 | } 58 | 59 | "VariableReturns.fromUntil" should { 60 | "keep only a window of the returns" in { 61 | val variableReturns = VariableReturns(Vector.tabulate(12) { i => 62 | val d = (i + 1).toDouble 63 | VariableReturn(f"2017.$d%02.0f", d) 64 | }) 65 | 66 | variableReturns.fromUntil("2017.07", "2017.09").returns should ===(Vector( 67 | VariableReturn("2017.07", 7.0), 68 | VariableReturn("2017.08", 8.0) 69 | )) 70 | 71 | variableReturns.fromUntil("2017.10", "2018.01").returns should ===(Vector( 72 | VariableReturn("2017.10", 10.0), 73 | VariableReturn("2017.11", 11.0), 74 | VariableReturn("2017.12", 12.0) 75 | )) 76 | } 77 | } 78 | 79 | "Returns.annualizedTotalReturn" should { 80 | val returns = VariableReturns(Vector.tabulate(12)(i => VariableReturn(i.toString, i.toDouble / 100 / 12))) 81 | val avg = Returns.annualizedTotalReturn(returns) 82 | "compute a geometric mean of the returns" in { 83 | // Excel: GEOMEAN (see geomean.ods) 84 | avg should ===(0.0549505735) 85 | } 86 | 87 | "compute an average that can be used to calculate a futureCapital instead of using variable returns" in { 88 | // This calculation only works if the capital does not change over time 89 | // otherwise, the capital fluctuates as well as the interest rates, and we cannot use the mean 90 | val futCapVar = RetCalc.futureCapital(returns, 12, 0, 0, 500000).right.value 91 | val futCapFix = RetCalc.futureCapital(FixedReturns(avg), 12, 0, 0, 500000).right.value 92 | futCapVar should ===(futCapFix) 93 | } 94 | } 95 | 96 | } 97 | -------------------------------------------------------------------------------- /Chapter03/retirement-calculator/src/test/scala/retcalc/SimulatePlanAppIT.scala: -------------------------------------------------------------------------------- 1 | package retcalc 2 | 3 | import cats.data.Validated.{Invalid, Valid} 4 | import org.scalactic.TypeCheckedTripleEquals 5 | import org.scalatest.{Matchers, WordSpec} 6 | 7 | class SimulatePlanAppIT extends WordSpec with Matchers with TypeCheckedTripleEquals { 8 | "SimulatePlanApp.strMain" should { 9 | "simulate a retirement plan using market returns" in { 10 | val actualResult = SimulatePlanApp.strMain( 11 | Array("1952.09,2017.09", "25", "40", "3000", "2000", "10000")) 12 | 13 | val expectedResult = 14 | s""" 15 | |Capital after 25 years of savings: 468925 16 | |Capital after 40 years in retirement: 2958842 17 | |""".stripMargin 18 | actualResult should ===(Valid(expectedResult)) 19 | } 20 | 21 | "return an error when the period exceeds the returns bounds" in { 22 | val actualResult = SimulatePlanApp.strMain( 23 | Array("1952.09,2017.09", "25", "60", "3000", "2000", "10000")) 24 | val expectedResult = "Cannot get the return for month 780. Accepted range: 0 to 779" 25 | actualResult should ===(Invalid(expectedResult)) 26 | } 27 | 28 | "return an usage example when the number of arguments is incorrect" in { 29 | val result = SimulatePlanApp.strMain( 30 | Array("1952.09:2017.09", "25.0", "60", "3'000", "2000.0")) 31 | result should ===(Invalid( 32 | """Usage: 33 | |simulatePlan from,until nbOfYearsSaving nbOfYearsRetired netIncome currentExpenses initialCapital 34 | | 35 | |Example: 36 | |simulatePlan 1952.09,2017.09 25 40 3000 2000 10000 37 | |""".stripMargin)) 38 | } 39 | 40 | "return several errors when several arguments are invalid" in { 41 | val result = SimulatePlanApp.strMain( 42 | Array("1952.09:2017.09", "25.0", "60", "3'000", "2000.0", "10000")) 43 | result should ===(Invalid( 44 | """Invalid format for fromUntil. Expected: from,until, actual: 1952.09:2017.09 45 | |Invalid number for nbOfYearsSaving: 25.0 46 | |Invalid number for netIncome: 3'000 47 | |Invalid number for currentExpenses: 2000.0""".stripMargin)) 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Chapter03/worksheets/build.sbt: -------------------------------------------------------------------------------- 1 | name := "chapter3" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.4" 6 | 7 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" 8 | libraryDependencies += "org.typelevel" %% "cats-core" % "1.0.1" 9 | scalacOptions += "-Ypartial-unification" -------------------------------------------------------------------------------- /Chapter03/worksheets/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.0.4 -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/either.sc: -------------------------------------------------------------------------------- 1 | def divide(x: Double, y: Double): Either[String, Double] = 2 | if (y == 0) 3 | Left(s"$x cannot be divided by zero") 4 | else 5 | Right(x / y) 6 | 7 | divide(6, 3) 8 | // res0: Either[String,Double] = Right(2.0) 9 | divide(6, 0) 10 | // res1: Either[String,Double] = Left(6.0 cannot be divided by zero) 11 | 12 | def getPersonAge(name: String, db: Map[String, Int]): Either[String, Int] = 13 | db.get(name).toRight(s"$name is not present in db") 14 | 15 | def personDescription(name: String, db: Map[String, Int]): String = 16 | getPersonAge(name, db) match { 17 | case Right(age) => s"$name is $age years old" 18 | case Left(error) => error 19 | } 20 | 21 | val db = Map("John" -> 25, "Rob" -> 40) 22 | personDescription("John", db) 23 | // res4: String = John is 25 years old 24 | personDescription("Michael", db) 25 | // res5: String = Michael is not present in db 26 | 27 | def averageAge(name1: String, name2: String, db: Map[String, Int]): Either[String, Double] = 28 | getPersonAge(name1, db).flatMap(age1 => 29 | getPersonAge(name2, db).map(age2 => 30 | (age1 + age2).toDouble / 2)) 31 | 32 | averageAge("John", "Rob", db) 33 | // res4: Either[String,Double] = Right(32.5) 34 | averageAge("John", "Michael", db) 35 | // res5: Either[String,Double] = Left(Michael is not present in db) 36 | 37 | getPersonAge("bob", db).left.map(err => s"The error was: $err") 38 | // res6: scala.util.Either[String,Int] = Left(The error was: bob is not present in db) 39 | 40 | def averageAge2(name1: String, name2: String, db: Map[String, Int]): Either[String, Double] = 41 | for { 42 | age1 <- getPersonAge(name1, db) 43 | age2 <- getPersonAge(name2, db) 44 | } yield (age1 + age2).toDouble / 2 -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/exceptions.sc: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | case class Person(name: String, age: Int) 5 | 6 | case class AgeNegativeException(message: String) extends Exception(message) 7 | 8 | def createPerson(description: String): Person = { 9 | val split = description.split(" ") 10 | val age = split(1).toInt 11 | if (age < 0) 12 | throw AgeNegativeException(s"age: $age should be > 0") 13 | else 14 | Person(split(0), age) 15 | } 16 | 17 | //createPerson("John -10") 18 | //createPerson("John 1000000000000000000000000000") 19 | //createPerson("John 10.45") 20 | 21 | def averageAge(descriptions: Vector[String]): Double = { 22 | val total = descriptions.map(createPerson).map(_.age).sum 23 | total / descriptions.length 24 | } 25 | 26 | import scala.util.control.NonFatal 27 | 28 | def personsSummary(personsInput: String): String = { 29 | val descriptions = personsInput.split("\n").toVector 30 | val avg = try { 31 | averageAge(descriptions) 32 | } catch { 33 | case e: AgeNegativeException => 34 | println(s"one of the persons has a negative age: $e") 35 | 0 36 | case NonFatal(e) => 37 | println(s"something was wrong in the input: $e") 38 | 0 39 | } 40 | s"${descriptions.length} persons with an average age of $avg" 41 | } 42 | 43 | personsSummary( 44 | """John 25 45 | |Sharleen 45""".stripMargin) 46 | 47 | 48 | import java.io.IOException 49 | import java.net.URL 50 | import scala.annotation.tailrec 51 | 52 | val stream = new URL("https://www.packtpub.com/").openStream() 53 | val htmlPage: String = 54 | try { 55 | @tailrec 56 | def loop(builder: StringBuilder): String = { 57 | val i = stream.read() 58 | if (i != -1) 59 | loop(builder.append(i.toChar)) 60 | else 61 | builder.toString() 62 | } 63 | loop(StringBuilder.newBuilder) 64 | } catch { 65 | case e: IOException => s"cannot read URL: $e" 66 | } 67 | finally { 68 | stream.close() 69 | } 70 | 71 | 72 | val htmlPage2 = scala.io.Source.fromURL("https://www.packtpub.com/").mkString 73 | -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/nonemptylist.sc: -------------------------------------------------------------------------------- 1 | import cats.data.NonEmptyList 2 | 3 | 4 | NonEmptyList(1, List(2, 3)) 5 | // res0: cats.data.NonEmptyList[Int] = NonEmptyList(1, 2, 3) 6 | NonEmptyList.fromList(List(1, 2, 3)) 7 | // res3: Option[cats.data.NonEmptyList[Int]] = Some(NonEmptyList(1, 2, 3)) 8 | NonEmptyList.fromList(List.empty[Int]) 9 | // res4: Option[cats.data.NonEmptyList[Int]] = None 10 | val nel = NonEmptyList.of(1, 2, 3) 11 | // nel: cats.data.NonEmptyList[Int] = NonEmptyList(1, 2, 3) 12 | 13 | nel.head 14 | // res0: Int = 1 15 | nel.tail 16 | // res1: List[Int] = List(2, 3) 17 | nel.map(_ + 1) 18 | // res2: cats.data.NonEmptyList[Int] = NonEmptyList(2, 3, 4) 19 | 20 | -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/option.sc: -------------------------------------------------------------------------------- 1 | val opt0: Option[Int] = None 2 | val opt1: Option[Int] = Some(1) 3 | 4 | val list0 = List.empty[String] 5 | list0.headOption 6 | list0.lastOption 7 | val list3 = List("Hello", "World") 8 | list3.headOption 9 | list3.lastOption 10 | 11 | 12 | def personDescription(name: String, db: Map[String, Int]): String = 13 | db.get(name) match { 14 | case Some(age) => s"$name is $age years old" 15 | case None => s"$name is not present in db" 16 | } 17 | 18 | val db = Map("John" -> 25, "Rob" -> 40) 19 | personDescription("John", db) 20 | personDescription("Michael", db) 21 | 22 | 23 | 24 | def personDesc(name: String, db: Map[String, Int]): String = { 25 | val optString: Option[String] = db.get(name).map(age => s"$name is $age years old") 26 | optString.getOrElse(s"$name is not present in db") 27 | } 28 | 29 | def averageAgeA(name1: String, name2: String, db: Map[String, Int]): Option[Double] = { 30 | val optOptAvg: Option[Option[Double]] = 31 | db.get(name1).map(age1 => 32 | db.get(name2).map(age2 => 33 | (age1 + age2).toDouble / 2)) 34 | optOptAvg.flatten 35 | } 36 | 37 | def averageAgeB(name1: String, name2: String, db: Map[String, Int]): Option[Double] = 38 | db.get(name1).flatMap(age1 => 39 | db.get(name2).map(age2 => 40 | (age1 + age2).toDouble / 2)) 41 | 42 | def averageAgeC(name1: String, name2: String, db: Map[String, Int]): Option[Double] = 43 | for { 44 | age1 <- db.get(name1) 45 | age2 <- db.get(name2) 46 | } yield (age1 + age2).toDouble / 2 47 | 48 | 49 | 50 | averageAgeA("John", "Rob", db) 51 | // res6: Option[Double] = Some(32.5) 52 | averageAgeA("John", "Michael", db) 53 | // res7: Option[Double] = None 54 | 55 | for { 56 | i <- Vector("one", "two") 57 | j <- Vector(1, 2, 3) 58 | } yield (i, j) 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/referential_transparency.sc: -------------------------------------------------------------------------------- 1 | def pureSquare(x: Int): Int = x * x 2 | val pureExpr = pureSquare(4) + pureSquare(3) 3 | // pureExpr: Int = 25 4 | 5 | val pureExpr2 = 16 + 9 6 | // pureExpr2: Int = 25 7 | 8 | var globalState = 1 9 | def impure(x: Int): Int = { 10 | globalState = globalState + x 11 | globalState 12 | } 13 | val impureExpr = impure(3) 14 | val impureExpr2 = 4 15 | 16 | 17 | import scala.util.Random 18 | def impureRand(): Int = Random.nextInt() 19 | impureRand() 20 | //res0: Int = -528134321 21 | val impureExprRand = impureRand() + impureRand() 22 | //impureExprRand: Int = 681209667 23 | val impureExprRand2 = -528134321 + -528134321 24 | 25 | def pureRand(seed: Int): Int = new Random(seed).nextInt() 26 | pureRand(10) 27 | //res1: Int = -1157793070 28 | val pureExprRand = pureRand(10) + pureRand(10) 29 | //pureExprRand: Int = 1979381156 30 | val pureExprRand2 = -1157793070 + -1157793070 31 | //pureExprRand2: Int = 1979381156 32 | 33 | 34 | def area(width: Double, height: Double): Double = { 35 | if (width > 5 || height > 5) 36 | throw new IllegalArgumentException("too big") 37 | else 38 | width * height 39 | } 40 | 41 | val total = try { 42 | area(6, 2) + area(4, 2) 43 | } catch { 44 | case e: IllegalArgumentException => 0 45 | } 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /Chapter03/worksheets/src/test/scala/validated.sc: -------------------------------------------------------------------------------- 1 | import cats.data._ 2 | import cats.data.Validated._ 3 | import cats.implicits._ 4 | 5 | val valid1: Validated[NonEmptyList[String], Int] = Valid(1) 6 | // valid1: cats.data.Validated[cats.data.NonEmptyList[String],Int] = Valid(1) 7 | val valid2: ValidatedNel[String, Int] = 2.validNel 8 | // valid2: cats.data.ValidatedNel[String,Int] = Valid(2) 9 | valid1.map2(valid2) { case (i1, i2) => i1 + i2 } 10 | // res0: cats.data.Validated[cats.data.NonEmptyList[String],Int] = Valid(3) 11 | (valid1, valid2).mapN { case (i1, i2) => i1 + i2 } 12 | // res1: cats.data.ValidatedNel[String,Int] = Valid(3) 13 | 14 | 15 | val invalid3: ValidatedNel[String, Int] = Invalid(NonEmptyList.of("error")) 16 | val invalid4 = "another error".invalidNel[Int] 17 | (valid1, valid2, invalid3, invalid4).mapN { case (i1, i2, i3, i4) => i1 + i2 + i3 + i4 } 18 | // res2: cats.data.ValidatedNel[String,Int] = Invalid(NonEmptyList(error, another error)) -------------------------------------------------------------------------------- /Chapter04/build.sbt: -------------------------------------------------------------------------------- 1 | name := "advanced-features" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.6" 6 | 7 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" 8 | libraryDependencies += "org.typelevel" %% "cats-core" % "1.1.0" 9 | libraryDependencies += "org.typelevel" %% "cats-laws" % "1.1.0" 10 | scalacOptions += "-Ypartial-unification" 11 | -------------------------------------------------------------------------------- /Chapter04/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.3 2 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/currying.sc: -------------------------------------------------------------------------------- 1 | def multiply(x: Int, y: Int): Int = x * y 2 | // multiply: multiply[](val x: Int,val y: Int) => Int 3 | val multiplyVal = (x: Int, y: Int) => x * y 4 | // multiplyVal: (Int, Int) => Int = $Lambda$1611/314131783@6a67fdf3 5 | val multiplyVal2 = multiply _ 6 | // multiplyVal2: (Int, Int) => Int = $Lambda$1619/657050655@322b60f 7 | 8 | multiply(2, 3) 9 | multiplyVal(2, 3) 10 | multiplyVal2(2, 3) 11 | 12 | val multiplyCurried = multiplyVal.curried 13 | // multiplyCurried: Int => (Int => Int) = ... 14 | 15 | multiplyVal(2, 3) 16 | // res3: Int = 6 17 | multiplyCurried(2) 18 | // res4: Int => Int = ... 19 | multiplyCurried(2)(3) 20 | // res5: Int = 6 21 | 22 | case class Item(description: String, price: Double) 23 | 24 | def discount(percentage: Double)(item: Item): Item = 25 | item.copy(price = item.price * (1 - percentage / 100)) 26 | 27 | discount(10)(Item("Monitor", 500)) 28 | // res6: Item = Item(Monitor,450.0) 29 | 30 | val discount10 = discount(10) _ 31 | // discount10: Item => Item = ... 32 | discount10(Item("Monitor", 500)) 33 | // res7: Item = Item(Monitor,450.0) 34 | 35 | 36 | val items = Vector(Item("Monitor", 500), Item("Laptop", 700)) 37 | items.map(discount(10)) 38 | // res8: Vector[Item] = Vector(Item(Monitor,450.0), Item(Laptop,630.0)) 39 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_appContext.sc: -------------------------------------------------------------------------------- 1 | import cats.data.ValidatedNel 2 | 3 | case class Timeout(millis: Int) 4 | 5 | trait PriceService { 6 | def getPrice(productName: String)(implicit timeout: Timeout): ValidatedNel[String, Double] 7 | } 8 | 9 | 10 | case class Product(name: String, price: Double) 11 | 12 | trait DataService { 13 | def getProduct(name: String): ValidatedNel[String, Product] 14 | 15 | def saveProduct(product: Product): ValidatedNel[String, Unit] 16 | } 17 | 18 | 19 | class AppContext(implicit val defaultTimeout: Timeout, 20 | val priceService: PriceService, 21 | val dataService: DataService) 22 | 23 | import cats.implicits._ 24 | def updatePrice(productName: String)(implicit appContext: AppContext): ValidatedNel[String, Double] = { 25 | import appContext._ 26 | (dataService.getProduct(productName), priceService.getPrice(productName)).tupled.andThen { 27 | case (product, newPrice) => 28 | dataService.saveProduct(product.copy(price = newPrice)).map(_ => 29 | newPrice 30 | ) 31 | } 32 | } 33 | 34 | 35 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_conversion.sc: -------------------------------------------------------------------------------- 1 | import java.time.LocalDate 2 | import java.time.temporal.ChronoUnit.DAYS 3 | 4 | import scala.collection.immutable.StringOps 5 | 6 | 7 | implicit def stringToLocalDate(s: String): LocalDate = LocalDate.parse(s) 8 | "2018-09-01".getDayOfWeek 9 | "2018-09-01".getYear 10 | DAYS.between("2018-09-01", "2018-10-10") 11 | 12 | //"2018".getMonth 13 | 14 | class IntOps(val i: Int) { 15 | def square: Int = i * i 16 | } 17 | implicit def intToIntOps(i: Int): IntOps = new IntOps(i) 18 | 19 | 5.square 20 | 21 | //implicit class IntOps2(val i: Int) extends AnyVal { 22 | // def square: Int = i * i 23 | //} 24 | // 25 | //5.square 26 | 27 | "abcd".reverse 28 | val abcd: StringOps = Predef.augmentString("abcd") 29 | abcd.reverse 30 | 31 | case class Person(name: String, age: Int) 32 | object Person { 33 | implicit val ordering: Ordering[Person] = Ordering.by(_.age) 34 | } 35 | 36 | List(Person("Omer", 40), Person("Bart", 10)).sorted 37 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_future.sc: -------------------------------------------------------------------------------- 1 | import scala.concurrent.Future 2 | import scala.concurrent.ExecutionContext.Implicits.global 3 | 4 | Future(println(Thread.currentThread().getName)) -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_parameter.sc: -------------------------------------------------------------------------------- 1 | import cats.data.{Validated, ValidatedNel} 2 | 3 | case class AppContext(message: String) 4 | implicit val myAppCtx: AppContext = AppContext("implicit world") 5 | 6 | def greeting(prefix: String)(implicit appCtx: AppContext): String = 7 | prefix + appCtx.message 8 | 9 | greeting("hello ") 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_resolution.sc: -------------------------------------------------------------------------------- 1 | class A(val n: Int) { 2 | def +(other: A) = new A(n + other.n) 3 | } 4 | object A { 5 | implicit def fromInt(n: Int) = new A(n) 6 | } 7 | 8 | 1 + new A(1) -------------------------------------------------------------------------------- /Chapter04/src/main/scala/implicits_sdk.sc: -------------------------------------------------------------------------------- 1 | import scala.collection.breakOut 2 | val map: Map[String, Int] = Vector("hello", "world").map(s => s -> s.length)(breakOut) 3 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/lazyness.sc: -------------------------------------------------------------------------------- 1 | class StrictDemo { 2 | val strictVal = { 3 | println("Evaluating strictVal") 4 | "Hello" 5 | } 6 | } 7 | 8 | val strictDemo = new StrictDemo 9 | 10 | //Evaluating strictVal 11 | //strictDemo: StrictDemo = StrictDemo@32fac009 12 | 13 | class LazyDemo { 14 | lazy val lazyVal = { 15 | println("Evaluating lazyVal") 16 | "Hello" 17 | } 18 | } 19 | 20 | val lazyDemo = new LazyDemo 21 | // lazyDemo: LazyDemo = LazyDemo@13ca84d5 22 | lazyDemo.lazyVal + " World" 23 | 24 | // Evaluating lazyVal 25 | // res0: String = Hello World 26 | 27 | class LazyChain { 28 | lazy val val1 = { 29 | println("Evaluating val1") 30 | "Hello" 31 | } 32 | lazy val val2 = { 33 | println("Evaluating val2") 34 | val1 + " lazy" 35 | } 36 | lazy val val3 = { 37 | println("Evaluating val3") 38 | val2 + " chain" 39 | } 40 | } 41 | 42 | val lazyChain = new LazyChain 43 | // lazyChain: LazyChain = LazyChain@4ca51fa 44 | lazyChain.val3 45 | 46 | // Evaluating val3 47 | // Evaluating val2 48 | // Evaluating val1 49 | // res1: String = Hello lazy chain 50 | 51 | object AppConfig { 52 | lazy val greeting: String = { 53 | println("Loading greeting") 54 | "Hello " 55 | } 56 | } 57 | 58 | def greet(name: String, greeting: => String): String = { 59 | if (name == "Mikael") 60 | greeting + name 61 | else 62 | s"I don't know you $name" 63 | } 64 | greet("Bob", AppConfig.greeting) 65 | // res2: String = I don't know you Bob 66 | greet("Mikael", AppConfig.greeting) 67 | // Loading greeting 68 | // res3: String = Hello Mikael 69 | 70 | // Lazy data structures 71 | def evenPlusOne(xs: Vector[Int]): Vector[Int] = 72 | xs.filter { x => println(s"filter $x"); x % 2 == 0 } 73 | .map { x => println(s"map $x"); x + 1 } 74 | 75 | evenPlusOne(Vector(0, 1, 2)) 76 | 77 | 78 | def lazyEvenPlusOne(xs: Vector[Int]): Vector[Int] = 79 | xs.withFilter { x => println(s"filter $x"); x % 2 == 0 } 80 | .map { x => println(s"map $x") ; x + 1 } 81 | 82 | lazyEvenPlusOne(Vector(0, 1, 2)) 83 | 84 | def lazyEvenPlusTwo(xs: Vector[Int]): Vector[Int] = 85 | xs.withFilter { x => println(s"filter $x"); x % 2 == 0 } 86 | .map { x => println(s"map $x") ; x + 1 } 87 | .map { x => println(s"map2 $x") ; x + 1 } 88 | 89 | lazyEvenPlusTwo(Vector(0, 1, 2)) 90 | 91 | 92 | def lazyEvenPlusTwoStream(xs: Stream[Int]): Stream[Int] = 93 | xs.filter { x => println(s"filter $x"); x % 2 == 0 } 94 | .map { x => println(s"map $x") ; x + 1 } 95 | .map { x => println(s"map2 $x") ; x + 1 } 96 | 97 | lazyEvenPlusTwoStream(Stream(0, 1, 2)).toVector 98 | 99 | 100 | // Infinite Stream 101 | val evenInts: Stream[Int] = 0 #:: 2 #:: evenInts.tail.map(_ + 2) 102 | evenInts.take(10).toVector 103 | // res8: Vector[Int] = Vector(0, 2, 4, 6, 8, 10, 12, 14, 16, 18) 104 | 105 | -------------------------------------------------------------------------------- /Chapter04/src/main/scala/variance.sc: -------------------------------------------------------------------------------- 1 | trait Animal 2 | case class Cat(name: String) extends Animal 3 | case class Dog(name: String) extends Animal 4 | 5 | val animal1: Animal = Cat("Max") 6 | val animal2: Animal = Dog("Dolly") 7 | implicitly[Dog <:< Animal] 8 | 9 | 10 | // Decoder 11 | trait InvariantDecoder[A] { 12 | def decode(s: String): Option[A] 13 | } 14 | object InvariantCatDecoder extends InvariantDecoder[Cat] { 15 | val CatRegex = """Cat\((\w+\))""".r 16 | def decode(s: String): Option[Cat] = s match { 17 | case CatRegex(name) => Some(Cat(name)) 18 | case _ => None 19 | } 20 | } 21 | // does not compile: 22 | // val invariantAnimalDecoder: InvariantDecoder[Animal] = InvariantCatDecoder 23 | 24 | trait CovariantDecoder[+A] { 25 | def decode(s: String): Option[A] 26 | } 27 | implicitly[CovariantDecoder[Cat] <:< CovariantDecoder[Animal]] 28 | 29 | object CovariantCatDecoder extends CovariantDecoder[Cat] { 30 | val CatRegex = """Cat\((\w+\))""".r 31 | def decode(s: String): Option[Cat] = s match { 32 | case CatRegex(name) => Some(Cat(name)) 33 | case _ => None 34 | } 35 | } 36 | 37 | val covariantAnimalDecoder: CovariantDecoder[Animal] = CovariantCatDecoder 38 | covariantAnimalDecoder.decode("Cat(Ulysse)") 39 | // res0: Option[Animal] = Some(Cat(Ulysse))) 40 | implicitly[CovariantDecoder[Cat] <:< CovariantDecoder[Animal]] 41 | 42 | 43 | 44 | object CovariantCatAndDogDecoder extends CovariantDecoder[Animal] { 45 | val CatRegex = """Cat\((\w+\))""".r 46 | val DogRegex = """Dog\((\w+\))""".r 47 | def decode(s: String): Option[Animal] = s match { 48 | case CatRegex(name) => Some(Cat(name)) 49 | case DogRegex(name) => Some(Dog(name)) 50 | case _ => None 51 | } 52 | } 53 | 54 | val covariantCatsAndDogsDecoder = CovariantCatAndDogDecoder 55 | 56 | covariantCatsAndDogsDecoder.decode("Cat(Garfield)") 57 | covariantCatsAndDogsDecoder.decode("Dog(Aiko)") 58 | 59 | 60 | // Contravariant Encoder 61 | trait Encoder[-A] { 62 | def encode(a: A): String 63 | } 64 | object AnimalEncoder extends Encoder[Animal] { 65 | def encode(a: Animal): String = a.toString 66 | } 67 | val catEncoder: Encoder[Cat] = AnimalEncoder 68 | catEncoder.encode(Cat("Luna")) 69 | // res1: String = Cat(Luna) 70 | 71 | 72 | /* 73 | trait Codec[+A] { 74 | def encode(a: A): String 75 | def decode(s: String): Option[A] 76 | } 77 | Error:(55, 15) covariant type A occurs in contravariant position in type A of value a 78 | def encode(a: A): String 79 | ^ 80 | */ 81 | trait Codec[A] { 82 | def encode(a: A): String 83 | def decode(s: String): Option[A] 84 | } 85 | 86 | object CatAndDogCodec extends Codec[Animal] { 87 | val CatRegex = """Cat\((\w+\))""".r 88 | val DogRegex = """Dog\((\w+\))""".r 89 | 90 | override def encode(a: Animal) = a.toString 91 | 92 | override def decode(s: String): Option[Animal] = s match { 93 | case CatRegex(name) => Some(Cat(name)) 94 | case DogRegex(name) => Some(Dog(name)) 95 | case _ => None 96 | } 97 | } 98 | 99 | val cat = CatAndDogCodec.decode("Cat(Garfield)") 100 | 101 | CatAndDogCodec.encode(cat.get) 102 | 103 | 104 | // Covariance in collections 105 | val cats: Vector[Cat] = Vector(Cat("Max")) 106 | val animals: Vector[Animal] = cats 107 | 108 | val catsAndDogs = cats :+ Dog("Medor") 109 | // catsAndDogs: Vector[Product with Serializable with Animal] = Vector(Cat(Max), Dog(Medor)) 110 | 111 | val serializables = catsAndDogs :+ "string" 112 | // serializables: Vector[Serializable] = Vector(Cat(Max), Dog(Medor), string) 113 | val anys = serializables :+ 1 114 | // anys: Vector[Any] = Vector(Cat(Max), Dog(Medor), string, 1) 115 | 116 | 117 | -------------------------------------------------------------------------------- /Chapter05/build.sbt: -------------------------------------------------------------------------------- 1 | name := "type-classes" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.12.6" 6 | 7 | libraryDependencies += "org.scalatest" %% "scalatest" % "3.0.4" % "test" 8 | libraryDependencies += "org.typelevel" %% "cats-core" % "1.1.0" 9 | libraryDependencies += "org.typelevel" %% "cats-laws" % "1.1.0" 10 | scalacOptions += "-Ypartial-unification" 11 | -------------------------------------------------------------------------------- /Chapter05/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.3 2 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses.sc: -------------------------------------------------------------------------------- 1 | trait Combine[A] { 2 | def combine(x: A, y: A): A 3 | } 4 | 5 | object Combine { 6 | def apply[A](implicit combineA: Combine[A]): Combine[A] = combineA 7 | 8 | implicit val combineInt: Combine[Int] = new Combine[Int] { 9 | override def combine(x: Int, y: Int): Int = x + y 10 | } 11 | 12 | implicit val combineString: Combine[String] = new Combine[String] { 13 | override def combine(x: String, y: String) = x + y 14 | } 15 | 16 | implicit def combineOption[A: Combine]: Combine[Option[A]] = new Combine[Option[A]] { 17 | override def combine(optX: Option[A], optY: Option[A]): Option[A] = 18 | for { 19 | x <- optX 20 | y <- optY 21 | } yield Combine[A].combine(x, y) 22 | } 23 | 24 | 25 | 26 | implicit class CombineOps[A](val x: A)(implicit combineA: Combine[A]) { 27 | def combine(y: A): A = combineA.combine(x, y) 28 | } 29 | 30 | } 31 | 32 | 33 | Combine[Int].combine(1, 2) 34 | // res0: Int = 3 35 | Combine[String].combine("Hello", " type class") 36 | // res1: String = Hello type class 37 | 38 | import Combine.CombineOps 39 | 2.combine(3) 40 | // res2: Int = 5 41 | "abc".combine("def") 42 | // res3: String = abcdef 43 | 44 | Option(3).combine(Option(4)) 45 | // res4: Option[Int] = Some(7) 46 | Option(3) combine Option.empty 47 | // res5: Option[Int] = None 48 | Option("Hello ") combine Option(" world") 49 | // res6: Option[String] = Some(Hello world) -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_applicative.sc: -------------------------------------------------------------------------------- 1 | import cats.Applicative 2 | import cats.data.{Validated, ValidatedNel} 3 | import cats.implicits._ 4 | 5 | Applicative[Option].pure(1) 6 | // res0: Option[Int] = Some(1) 7 | 3.pure[Option] 8 | // res1: Option[Int] = Some(3) 9 | 10 | type Result[A] = ValidatedNel[Throwable, A] 11 | Applicative[Result].pure("hi pure") 12 | // res2: Result[String] = Valid(hi pure) 13 | "hi pure".pure[Result] 14 | // res3: Result[String] = Valid(hi pure) 15 | 16 | // Laws: applicative identity 17 | // pure id <*> v = v 18 | val fa = Option(1) 19 | type A = Int 20 | type F[X] = Option[X] 21 | ((identity[A] _).pure[F] <*> fa) == fa 22 | // Another way to define this law 23 | val id = (identity[Int] _) 24 | Applicative[F].pure(id) <*> fa == fa 25 | 26 | // Laws: Composition 27 | type B = String 28 | type C = Double 29 | // pure (.) <*> u <*> v <*> w = u <*> (v <*> w) 30 | val fab: F[A => B] = Option(_.toString) 31 | val fbc: F[B => C] = Option(_.toDouble / 2) 32 | (fbc <*> (fab <*> fa)) == ((fbc.map(_.compose[A] _) <*> fab) <*> fa) 33 | 34 | // Laws: Homomorphism 35 | // pure f <*> pure x = pure (f x) 36 | val a = 1 37 | type AB = (A) => B 38 | val f : AB = _.toString 39 | 40 | Applicative[F].pure(f) <*> Applicative[F].pure(a) == Applicative[F].pure(f(a)) 41 | 42 | // Laws: Interchange 43 | // u <*> pure y = pure ($ y) <*> u 44 | fab <*> Applicative[F].pure(a) == Applicative[F].pure((f: A => B) => f(a)) <*> fab 45 | 46 | // Traverse 47 | def parseIntO(s: String): Option[Int] = Either.catchNonFatal(s.toInt).toOption 48 | Vector("1", "2" , "3").traverse(parseIntO) 49 | // res5: Option[Vector[Int]] = Some(Vector(1, 2, 3)) 50 | Vector("1", "boom" , "3").traverse(parseIntO) 51 | // res6: Option[Vector[Int]] = None 52 | 53 | def parseIntV(s: String): ValidatedNel[Throwable, Int] = Validated.catchNonFatal(s.toInt).toValidatedNel 54 | Vector("1", "2" , "3").traverse(parseIntV) 55 | // res7: ValidatedNel[Throwable, Vector[Int]] = Valid(Vector(1, 2, 3)) 56 | Vector("1", "boom" , "crash").traverse(parseIntV) 57 | // res8: ValidatedNel[Throwable, Vector[Int]] = 58 | // Invalid(NonEmptyList( 59 | // NumberFormatException: For input string: "boom", 60 | // NumberFormatException: For input string: "crash")) 61 | 62 | val vecOpt: Vector[Option[Int]] = Vector(Option(1), Option(2), Option(3)) 63 | val optVec: Option[Vector[Int]] = vecOpt.sequence 64 | // optVec: Option[Vector[Int]] = Some(Vector(1, 2, 3)) 65 | 66 | import scala.concurrent._ 67 | import ExecutionContext.Implicits.global 68 | import duration.Duration 69 | 70 | val vecFut: Vector[Future[Int]] = Vector(Future(1), Future(2), Future(3)) 71 | val futVec: Future[Vector[Int]] = vecFut.sequence 72 | 73 | Await.result(futVec, Duration.Inf) 74 | // res9: Vector[Int] = Vector(1, 2, 3) 75 | 76 | 77 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_apply.sc: -------------------------------------------------------------------------------- 1 | 2 | import cats.data.Validated 3 | import cats.implicits._ 4 | 5 | 6 | Option[String => String]("Hello " + _).ap(Some("Apply")) 7 | // res0: Option[String] = Some(Hello Apply) 8 | Option[String => String]("Hello " + _) <*> None 9 | // res1: Option[String] = None 10 | Option.empty[String => String] <*> Some("Apply") 11 | // res2: Option[String] = None 12 | 13 | val addOne: Int => Int = _ + 1 14 | val multByTwo: Int => Int = _ * 2 15 | Vector(addOne, multByTwo) <*> Vector(1, 2, 3) 16 | // res3: Vector[Int] = Vector(2, 3, 4, 2, 4, 6) 17 | 18 | def parseIntO(s: String): Option[Int] = Either.catchNonFatal(s.toInt).toOption 19 | parseIntO("6").map2(parseIntO("2"))(_ / _) 20 | // res4: Option[Int] = Some(3) 21 | parseIntO("abc").map2(parseIntO("def"))(_ / _) 22 | // res5: Option[Int] = None 23 | 24 | def parseIntE(s: String): Either[Throwable, Int] = Either.catchNonFatal(s.toInt) 25 | parseIntE("6").map2(parseIntE("2"))(_ / _) 26 | // res6: Either[Throwable,Int] = Right(3) 27 | parseIntE("abc").map2(parseIntE("3"))(_ / _) 28 | // res7: Either[Throwable,Int] = Left(java.lang.NumberFormatException: For input string: "abc") 29 | 30 | (parseIntE("1"), parseIntE("2"), parseIntE("3")).mapN( (a,b,c) => a + b + c) 31 | // res8: Either[Throwable,Int] = Right(6) 32 | 33 | import cats.data.ValidatedNel 34 | def parseIntV(s: String): ValidatedNel[Throwable, Int] = Validated.catchNonFatal(s.toInt).toValidatedNel 35 | (parseIntV("abc"), parseIntV("def"), parseIntV("3")).mapN( (a,b,c) => a + b + c) 36 | // res9: ValidatedNel[Throwable,Int] = Invalid(NonEmptyList( 37 | // java.lang.NumberFormatException: For input string: "abc", 38 | // java.lang.NumberFormatException: For input string: "def")) 39 | 40 | // Exercise: associativity law 41 | val fa = Option(1) 42 | val fb = Option(2) 43 | val fc = Option(3) 44 | (fa product (fb product fc)) == ((fa product fb) product fc).map { case ((a, b), c) => (a, (b, c))} 45 | 46 | // Exercise: ap function composition 47 | val fab: Option[Int => String] = Option(_.toString) 48 | val fbc: Option[String => Double] = Option(_.toDouble / 2) 49 | (fbc <*> (fab <*> fa)) == ((fbc.map(_.compose[Int] _) <*> fab) <*> fa) 50 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_functor.sc: -------------------------------------------------------------------------------- 1 | import cats.Functor 2 | import cats.implicits._ 3 | 4 | def addOne[F[_] : Functor](fa: F[Int]): F[Int] = fa.map(_ + 1) 5 | addOne(Vector(1, 2, 3)) 6 | // res0: Vector[Int] = Vector(2, 3, 4) 7 | addOne(Option(1)) 8 | // res1: Option[Int] = Some(2) 9 | addOne(1.asRight) 10 | // res2: Either[Nothing,Int] = Right(2) 11 | addOne[cats.Id](1) 12 | 13 | // Exercise 14 | val brokenFunctor: Functor[Vector] = new Functor[Vector] { 15 | override def map[A, B](fa: Vector[A])(f: A => B): Vector[B] = { 16 | fa.map(f).reverse 17 | } 18 | } 19 | 20 | val fa = Vector(1, 2, 3) 21 | val f: Int => Int = _ + 1 22 | val g: Int => Int = _ * 2 23 | brokenFunctor.map(fa)(identity) == fa 24 | brokenFunctor.map(brokenFunctor.map(fa)(f))(g) == brokenFunctor.map(fa)(f andThen g) 25 | 26 | 27 | def square(x: Double): Double = x * x 28 | def squareVector: Vector[Double] => Vector[Double] = 29 | Functor[Vector].lift(square) 30 | squareVector(Vector(1, 2, 3)) 31 | // res0: Vector[Double] = Vector(1.0, 4.0, 9.0) 32 | 33 | def squareOption: Option[Double] => Option[Double] = 34 | Functor[Option].lift(square) 35 | squareOption(Some(3)) 36 | // res1: Option[Double] = Some(9.0) 37 | 38 | Vector("Functors", "are", "great").fproduct(_.length).toMap 39 | // res2: Map[String,Int] = Map(Functors -> 8, are -> 3, great -> 5) 40 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_monad.sc: -------------------------------------------------------------------------------- 1 | import cats.{Id, Monad} 2 | import cats.implicits._ 3 | 4 | 5 | val fa: Vector[Int] = Vector(1) 6 | val f: Int => Vector[String] = i => Vector(i.toString) 7 | val g: String => Vector[Double] = s => Vector(s.toDouble) 8 | type F[X] = Vector[X] 9 | // Law: FlatMap associativity 10 | ((fa flatMap f) flatMap g) == (fa flatMap(f(_) flatMap g)) 11 | 12 | // Law: Left/Right identity 13 | val a = 2 14 | Monad[F].pure(a).flatMap(f) == f(a) 15 | fa.flatMap(Monad[F].pure) == fa 16 | 17 | case class Item(id: Int, label: String, price: Double, category: String) 18 | 19 | 20 | trait ItemApi[F[_]] { 21 | def findAllItems: F[Vector[Item]] 22 | 23 | def saveItem(item: Item): F[Unit] 24 | } 25 | 26 | 27 | def startSalesSeason[F[_] : Monad](api: ItemApi[F]): F[Unit] = { 28 | for { 29 | items <- api.findAllItems 30 | _ <- items.traverse { item => 31 | val discount = if (item.category == "shoes") 0.80 else 0.70 32 | val discountedItem = item.copy(price = item.price * discount) 33 | api.saveItem(discountedItem) 34 | } 35 | } yield () 36 | } 37 | 38 | // Exercise: implement Api using the Id monad 39 | object IdItemApi extends ItemApi[Id] { 40 | var allItems = Map[Int, Item]( 41 | 1 -> Item(1, "Nick Air Superjump size 10", 50.0, "shoes"), 42 | 2 -> Item(2, "Luis Boutton handbag", 300.0, "handbags") 43 | ) 44 | 45 | def findAllItems: Vector[Item] = allItems.values.toVector 46 | 47 | def saveItem(item: Item): Unit = { 48 | allItems = allItems + (item.id -> item) 49 | } 50 | } 51 | 52 | startSalesSeason(IdItemApi) 53 | IdItemApi.findAllItems.foreach(println) 54 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_monoid.sc: -------------------------------------------------------------------------------- 1 | import cats.implicits._ 2 | import cats.kernel.Monoid 3 | 4 | 5 | Monoid[Int].empty 6 | // res0: Int = 0 7 | Monoid[String].empty 8 | // res1: String = 9 | Monoid[Option[Double]].empty 10 | // res2: Option[Double] = None 11 | Monoid[Vector[Int]].empty 12 | // res2: Vector[Int] = Vector() 13 | Monoid[Either[String, Int]].empty 14 | // res4: Either[String,Int] = Right(0) 15 | 16 | (3 |+| Monoid[Int].empty) == 3 17 | ("Hello identity" |+| Monoid[String].empty) == "Hello identity" 18 | (Option(3) |+| Monoid[Option[Int]].empty) == Option(3) 19 | 20 | Vector(1, 2, 3).combineAll 21 | // res8: Int = 6 22 | 23 | Vector(1, 2, 3).foldLeft(0) { case (acc, i) => acc + i } 24 | 25 | Vector("1", "2", "3").foldMap(s => (s, s.toInt)) 26 | // res10: (String, Int) = (123,6) 27 | 28 | // Exercise 29 | val monoidMultInt: Monoid[Int] = new Monoid[Int] { 30 | override def empty: Int = 1 31 | 32 | override def combine(x: Int, y: Int): Int = x * y 33 | } 34 | Vector(1, 2, 3, 4).combineAll(monoidMultInt) 35 | 36 | // Exercise 37 | val (count, sum) = Vector(10, 12, 14, 8).foldMap(i => (1, i)) 38 | val average = sum.toDouble / count 39 | 40 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_monoid_monad.sc: -------------------------------------------------------------------------------- 1 | import cats.kernel.Monoid 2 | import cats.implicits._ 3 | 4 | def monoidMonad[A] = new Monoid[Vector[A] => Vector[A]] { 5 | override def empty = identity 6 | 7 | override def combine(f: Vector[A] => Vector[A], g: Vector[A] => Vector[A]) 8 | : Vector[A] => Vector[A] = 9 | xs => f(xs) flatMap(y => g(Vector(y))) 10 | } 11 | 12 | implicit val monoidMonadInt: Monoid[Vector[Int] => Vector[Int]] = monoidMonad[Int] 13 | 14 | val fn = ((xs: Vector[Int]) => xs.map(_+1)) |+| ((xs: Vector[Int]) => xs.map(_*2)) 15 | fn(Vector(1,2,3)) 16 | Vector(1,2,3) flatMap (i => Vector(i * 2)) 17 | 18 | // TODO try KleisliMonoid 19 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_ordering.sc: -------------------------------------------------------------------------------- 1 | 2 | Vector(1,3,2).sorted 3 | 4 | import java.time.LocalDate 5 | implicit val dateOrdering: Ordering[LocalDate] = Ordering.fromLessThan[LocalDate](_ isBefore _) 6 | import Ordering.Implicits._ 7 | 8 | LocalDate.of(2018, 5, 18) < LocalDate.of(2017, 1, 1) 9 | // res1: Boolean = false 10 | Vector(LocalDate.of(2018, 5, 18), LocalDate.of(2018, 6, 1)).sorted(dateOrdering.reverse) 11 | // res2: Vector[LocalDate] = Vector(2018-06-01, 2018-05-18) 12 | -------------------------------------------------------------------------------- /Chapter05/src/main/scala/typeclasses_semigroup.sc: -------------------------------------------------------------------------------- 1 | import cats.implicits._ 2 | 3 | import scala.concurrent.duration.Duration 4 | import scala.concurrent.{Await, Future} 5 | 6 | 1 |+| 2 7 | // res0: Int = 3 8 | "Hello " |+| "World !" 9 | // res1: String = Hello World ! 10 | (1, 2, "Hello ") |+| (2, 4, "World !") 11 | // res2: (Int, Int, String) = (3,6,Hello World !) 12 | 13 | Vector(1, 2) |+| Vector(3, 4) 14 | // res3: Vector[Int] = Vector(1, 2, 3, 4) 15 | Option(1) |+| Option(2) 16 | // res4: Option[Int] = Some(3) 17 | Option(1) |+| None |+| Option(2) 18 | // res5: Option[Int] = Some(3) 19 | 20 | 21 | 1.asRight |+| 2.asRight 22 | // res6: Either[B,Int] = Right(3) 23 | 1.asRight[String] |+| 2.asRight |+| "error".asLeft 24 | // res7: Either[String,Int] = Left(error) 25 | "error1".asLeft[Int] |+| "error2".asLeft 26 | // res8: Either[String,Int] = Left(error1) 27 | 28 | // Exercise 29 | 1.validNel[String] |+| 2.validNel 30 | // res9: Validated[NonEmptyList[String],Int] = Valid(3) 31 | 1.validNel[String] |+| "error1".invalidNel |+| "error2".invalidNel 32 | // res10: Validated[NonEmptyList[String],Int] = Invalid(NonEmptyList(error1, error2)) 33 | 34 | 35 | -------------------------------------------------------------------------------- /Chapter05/src/test/scala/EqualitySpec.scala: -------------------------------------------------------------------------------- 1 | import org.scalactic.{Equality, TolerantNumerics, TypeCheckedTripleEquals} 2 | import org.scalatest.{Matchers, WordSpec} 3 | 4 | class EqualitySpec extends WordSpec with Matchers with TypeCheckedTripleEquals{ 5 | implicit val doubleEquality: Equality[Double] = TolerantNumerics.tolerantDoubleEquality(0.0001) 6 | 7 | implicit def vectorEquality[A](implicit eqA: Equality[A]): Equality[Vector[A]] = new Equality[Vector[A]] { 8 | override def areEqual(v1: Vector[A], b: Any): Boolean = b match { 9 | case v2: Vector[_] => 10 | v1.size == v2.size && 11 | v1.zip(v2).forall { case ((x, y)) => eqA.areEqual(x, y)} 12 | case _ => false 13 | } 14 | } 15 | 16 | "Equality" should { 17 | "allow to compare two Double with a tolerance" in { 18 | 1.6 + 1.8 should ===(3.4) 19 | } 20 | 21 | "allow to compare two Vector[Double] with a tolerance" in { 22 | Vector(1.6 + 1.8, 0.0051) should === (Vector(3.4, 0.0052)) 23 | } 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### SBT template 3 | # Simple Build Tool 4 | # http://www.scala-sbt.org/release/docs/Getting-Started/Directories.html#configuring-version-control 5 | 6 | dist/* 7 | target/ 8 | lib_managed/ 9 | src_managed/ 10 | project/boot/ 11 | project/plugins/project/ 12 | .history 13 | .cache 14 | .lib/ 15 | ### JetBrains template 16 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 17 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 18 | 19 | # User-specific stuff: 20 | # Remove everything coming from Idea 21 | out/ 22 | .idea 23 | 24 | # CMake 25 | cmake-build-debug/ 26 | cmake-build-release/ 27 | 28 | ## File-based project format: 29 | *.iws 30 | 31 | ## Plugin-specific files: 32 | 33 | # mpeltonen/sbt-idea plugin 34 | .idea_modules/ 35 | 36 | # JIRA plugin 37 | atlassian-ide-plugin.xml 38 | 39 | # Cursive Clojure plugin 40 | .idea/replstate.xml 41 | 42 | # Crashlytics plugin (for Android Studio and IntelliJ) 43 | com_crashlytics_export_strings.xml 44 | crashlytics.properties 45 | crashlytics-build.properties 46 | fabric.properties 47 | ### Scala template 48 | *.class 49 | *.log 50 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/Future.sc: -------------------------------------------------------------------------------- 1 | import scala.concurrent.ExecutionContext.Implicits.global 2 | import scala.concurrent.duration._ 3 | import scala.concurrent.{Await, Future} 4 | import scala.util.{Failure, Success} 5 | 6 | val f1 = Future {1} 7 | val f2 = Future {2} 8 | val f3 = Future {3} 9 | 10 | 11 | val s = for { 12 | v1 <- f1 13 | v2 <- f2 14 | v3 <- f3 15 | } yield (v1 + v2 + v3) 16 | 17 | val fma = f1.flatMap { v1 => 18 | f2.map(v2 => 19 | v1 + v2 20 | ) 21 | } 22 | 23 | 24 | 25 | def sum(v: Int*) = { 26 | v.sum 27 | } 28 | 29 | val minExpected = 7 30 | val res = for { 31 | v1 <- f1 32 | v2 <- f2 33 | v3 <- f3 34 | if (sum(v1, v2, v3) > minExpected) 35 | } yield (v1, v2, v3) 36 | 37 | 38 | 39 | res.onComplete { 40 | case Success(result) => println(s"The result is $result") 41 | case Failure(e) => println("The sum is not big enough") 42 | } 43 | 44 | Await.ready(res, 1 second) 45 | 46 | 47 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/Procfile: -------------------------------------------------------------------------------- 1 | web: server/target/universal/stage/bin/server -Dhttp.port=$PORT -Dconfig.file=server/conf/heroku.conf 2 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/README.md: -------------------------------------------------------------------------------- 1 | # shopping 2 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/build.sbt: -------------------------------------------------------------------------------- 1 | import sbtcrossproject.{crossProject, CrossType} 2 | 3 | lazy val server = (project in file("server")).settings(commonSettings) 4 | .settings( 5 | scalaJSProjects := Seq(client), 6 | pipelineStages in Assets := Seq(scalaJSPipeline), 7 | pipelineStages := Seq(digest, gzip), 8 | // triggers scalaJSPipeline when using compile or continuous compilation 9 | compile in Compile := ((compile in Compile) dependsOn scalaJSPipeline).value, 10 | libraryDependencies ++= Seq( 11 | "com.vmunier" %% "scalajs-scripts" % "1.1.1", 12 | "com.typesafe.play" %% "play-slick" % "3.0.0", 13 | "com.typesafe.play" %% "play-slick-evolutions" % "3.0.0", 14 | "com.h2database" % "h2" % "1.4.196", 15 | "com.dripower" %% "play-circe" % "2609.0", 16 | "io.swagger" %% "swagger-play2" % "1.6.0", 17 | "org.webjars" % "swagger-ui" % "3.10.0", 18 | guice, 19 | "org.scalatestplus.play" %% "scalatestplus-play" % "3.1.2" % "test" 20 | ), 21 | ).enablePlugins(PlayScala) 22 | // .disablePlugins(PlayLayoutPlugin) 23 | .disablePlugins(PlayFilters) 24 | .dependsOn(sharedJvm) 25 | 26 | lazy val client = (project in file("client")).settings(commonSettings).settings( 27 | scalaJSUseMainModuleInitializer := true, 28 | libraryDependencies ++= Seq( 29 | "org.scala-js" %%% "scalajs-dom" % "0.9.5", 30 | "com.lihaoyi" %%% "scalatags" % "0.6.7", 31 | "org.querki" %%% "jquery-facade" % "1.2", 32 | "io.circe" %%% "circe-core" % "0.9.3", 33 | "io.circe" %%% "circe-generic" % "0.9.3", 34 | "io.circe" %%% "circe-parser" % "0.9.3" 35 | ), 36 | jsDependencies ++= Seq( 37 | "org.webjars" % "jquery" % "2.2.1" / "jquery.js" 38 | minified "jquery.min.js", 39 | "org.webjars" % "notifyjs" % "0.4.2" / "notify.js") 40 | ).enablePlugins(ScalaJSWeb) 41 | .dependsOn(sharedJs) 42 | 43 | lazy val shared = crossProject(JSPlatform, JVMPlatform) 44 | .crossType(CrossType.Pure) 45 | .in(file("shared")) 46 | .settings(commonSettings) 47 | 48 | lazy val sharedJvm = shared.jvm 49 | lazy val sharedJs = shared.js 50 | 51 | lazy val commonSettings = Seq( 52 | scalaVersion := "2.12.4", 53 | organization := "io.fscala" 54 | ) 55 | 56 | // loads the server project at sbt startup 57 | onLoad in Global := (onLoad in Global).value andThen { s: State => "project server" :: s } 58 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/client/src/main/scala/io/fscala/shopping/client/CartDiv.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.client 2 | 3 | import org.scalajs.dom.html.Div 4 | import scalatags.JsDom.all._ 5 | import io.fscala.shopping.shared.Product 6 | 7 | case class CartDiv(lines: Set[CartLine]) { 8 | def content: Div = lines.foldLeft(div.render) { (a, b) => 9 | a.appendChild(b.content).render 10 | a 11 | } 12 | 13 | def addProduct(line: CartLine): CartDiv = { 14 | CartDiv(this.lines + line) 15 | } 16 | 17 | } 18 | 19 | case class CartLine(qty: Int, product: Product) { 20 | def content: Div = div(`class` := "row", id := s"cart-${product.code}-row")( 21 | div(`class` := "col-1")(getDeleteButton), 22 | div(`class` := "col-2")(getQuantityInput), 23 | div(`class` := "col-6")(getProductLabel), 24 | div(`class` := "col")(getPriceLabel) 25 | ).render 26 | 27 | private def getQuantityInput = input(id := s"cart-${product.code}-qty", onchange := changeQty, value := qty.toString, `type` := "text", style := "width: 100%;").render 28 | 29 | private def getProductLabel = label(product.name).render 30 | 31 | private def getPriceLabel = label(product.price * qty).render 32 | 33 | private def getDeleteButton = button(`type` := "button", onclick := removeFromCart)("X").render 34 | 35 | private def changeQty() = () => UIManager.updateProduct(product) 36 | 37 | private def removeFromCart() = () => UIManager.deleteProduct(product) 38 | } -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/client/src/main/scala/io/fscala/shopping/client/NotifyJS.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.client 2 | 3 | import scala.scalajs.js 4 | import scala.scalajs.js.annotation.{JSGlobal, ScalaJSDefined} 5 | 6 | @js.native 7 | @JSGlobal("$") 8 | object NotifyJS extends js.Object { 9 | def notify(msg: String, option: Options): String = js.native 10 | } 11 | 12 | @ScalaJSDefined 13 | trait Options extends js.Object { 14 | // whether to hide the notification on click 15 | var clickToHide: js.UndefOr[Boolean] = js.undefined 16 | // whether to auto-hide the notification 17 | var autoHide: js.UndefOr[Boolean] = js.undefined 18 | // if autoHide, hide after milliseconds 19 | var autoHideDelay: js.UndefOr[Int] = js.undefined 20 | // show the arrow pointing at the element 21 | var arrowShow: js.UndefOr[Boolean] = js.undefined 22 | // arrow size in pixels 23 | var arrowSize: js.UndefOr[Int] = js.undefined 24 | // position defines the notification position though uses the defaults below 25 | var position: js.UndefOr[String] = js.undefined 26 | // default positions 27 | var elementPosition: js.UndefOr[String] = js.undefined 28 | var globalPosition: js.UndefOr[String] = js.undefined 29 | // default style 30 | var style: js.UndefOr[String] = js.undefined 31 | // default class (string or [string]) 32 | var className: js.UndefOr[String] = js.undefined 33 | // show animation 34 | var showAnimation: js.UndefOr[String] = js.undefined 35 | // show animation duration 36 | var showDuration: js.UndefOr[Int] = js.undefined 37 | // hide animation 38 | var hideAnimation: js.UndefOr[String] = js.undefined 39 | // hide animation duration 40 | var hideDuration: js.UndefOr[Int] = js.undefined 41 | // padding between element and notification 42 | var gap: js.UndefOr[Int] = js.undefined 43 | } 44 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/client/src/main/scala/io/fscala/shopping/client/ProductDiv.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.client 2 | 3 | import io.fscala.shopping.shared.Product 4 | import org.scalajs.dom.html.Div 5 | import scalatags.JsDom.all._ 6 | 7 | 8 | case class ProductDiv(product: Product) { 9 | def content: Div = div(`class` := "col")(getProductDescription, getButton).render 10 | 11 | private def getProductDescription = 12 | div( 13 | p(product.name), 14 | p(product.description), 15 | p(product.price)) 16 | 17 | 18 | private def getButton = button(`type` := "button", onclick := addToCart)("Add to Cart") 19 | 20 | private def addToCart() = () => UIManager.addOneProduct(product) 21 | } 22 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/client/src/main/scala/io/fscala/shopping/client/UIManager.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.client 2 | 3 | 4 | import io.circe.generic.auto._ 5 | import io.circe.parser._ 6 | import io.circe.syntax._ 7 | import io.fscala.shopping.shared._ 8 | import org.querki.jquery._ 9 | import org.scalajs.dom 10 | import org.scalajs.dom.html.Document 11 | import org.scalajs.dom.raw.{CloseEvent, Event, MessageEvent, WebSocket} 12 | 13 | import scala.scalajs.js.UndefOr 14 | import scala.util.{Random, Try} 15 | 16 | 17 | object UIManager { 18 | 19 | val origin: UndefOr[String] = dom.document.location.origin 20 | val cart: CartDiv = CartDiv(Set.empty[CartLine]) 21 | 22 | val webSocket: WebSocket = getWebSocket 23 | 24 | val dummyUserName = s"user-${Random.nextInt(1000)}" 25 | 26 | 27 | def main(args: Array[String]): Unit = { 28 | val settings = JQueryAjaxSettings.url(s"$origin/v1/login").data(dummyUserName).contentType("text/plain") 29 | $.post(settings._result).done((_: String) => { 30 | initUI(origin) 31 | }) 32 | } 33 | 34 | private def initUI(origin: UndefOr[String]) = { 35 | 36 | $.get(url = s"$origin/v1/products", dataType = "text") 37 | .done((answers: String) => { 38 | val products = decode[Seq[Product]](answers) 39 | products.right.map { seq => 40 | seq.foreach(p => { 41 | $("#products").append(ProductDiv(p).content) 42 | }) 43 | initCartUI(origin, seq) 44 | } 45 | }) 46 | .fail((xhr: JQueryXHR, textStatus: String, textError: String) => 47 | println(s"call failed: $textStatus with status code: ${xhr.status} $textError") 48 | ) 49 | } 50 | 51 | private def initCartUI(origin: UndefOr[String], products: Seq[Product]) = { 52 | $.get(url = s"$origin/v1/cart/products", dataType = "text") 53 | .done((answers: String) => { 54 | val carts = decode[Seq[Cart]](answers) 55 | carts.right.map { cartLines => 56 | cartLines.foreach { cartDao => 57 | val product = products.find(_.code == cartDao.productCode) 58 | product match { 59 | case Some(p) => 60 | val cartLine = CartLine(cartDao.quantity, p) 61 | val cartContent = UIManager.cart.addProduct(cartLine).content 62 | $("#cartPanel").append(cartContent) 63 | case None => 64 | println(s"product code ${cartDao.productCode} doesn't exists in the catalog") 65 | } 66 | } 67 | } 68 | }) 69 | .fail((xhr: JQueryXHR, textStatus: String, textError: String) => 70 | println(s"call failed: $textStatus with status code: ${xhr.status} $textError") 71 | ) 72 | } 73 | 74 | def addOneProduct(product: Product): JQueryDeferred = { 75 | val quantity = 1 76 | 77 | def onDone = () => { 78 | val cartContent = cart.addProduct(CartLine(quantity, product)).content 79 | $("#cartPanel").append(cartContent) 80 | println(s"Product $product added in the cart") 81 | webSocket.send(CartEvent(dummyUserName, product, Add).asJson.noSpaces) 82 | } 83 | 84 | postInCart(product.code, quantity, onDone) 85 | } 86 | 87 | def updateProduct(product: Product): JQueryDeferred = { 88 | putInCart(product.code, quantity(product.code)) 89 | } 90 | 91 | def deleteProduct(product: Product): JQueryDeferred = { 92 | def onDone = () => { 93 | val cartContent = $(s"#cart-${product.code}-row") 94 | cartContent.remove() 95 | webSocket.send(CartEvent(dummyUserName, product, Remove).asJson.noSpaces) 96 | println(s"Product ${product.code} removed from the cart") 97 | } 98 | 99 | deletefromCart(product.code, onDone) 100 | } 101 | 102 | private def getWebSocket: WebSocket = { 103 | val ws = new WebSocket(getWebsocketUri(dom.document, "v1/cart/events")) 104 | ws.onopen = { (event: Event) ⇒ 105 | println(s"webSocket.onOpen '${event.`type`}'") 106 | event.preventDefault() 107 | } 108 | 109 | ws.onerror = { (event: Event) => 110 | System.err.println(s"webSocket.onError '${event.getClass}'") 111 | } 112 | 113 | ws.onmessage = { (event: MessageEvent) => 114 | println(s"[webSocket.onMessage] '${event.data.toString}'...") 115 | val msg = decode[Alarm](event.data.toString) 116 | msg match { 117 | case Right(alarm) => 118 | println(s"[webSocket.onMessage] Got alarm event : $alarm)") 119 | notify(alarm) 120 | case Left(e) => 121 | println(s"[webSocket.onMessage] Got a unknown event : $msg)") 122 | } 123 | } 124 | 125 | ws.onclose = { (event: CloseEvent) ⇒ 126 | println(s"webSocket.onClose '${event.`type`}'") 127 | } 128 | ws 129 | } 130 | 131 | private def notify(alarm: Alarm): Unit = { 132 | val notifyClass = if (alarm.action == Add) "info" else "warn" 133 | NotifyJS.notify(alarm.message, new Options { 134 | className = notifyClass 135 | globalPosition = "right bottom" 136 | }) 137 | } 138 | 139 | private def quantity(productCode: String) = Try { 140 | val inputText = $(s"#cart-$productCode-qty") 141 | if (inputText.length != 0) 142 | Integer.parseInt(inputText.`val`().asInstanceOf[String]) 143 | else 1 144 | }.getOrElse(1) 145 | 146 | private def postInCart(productCode: String, quantity: Int, onDone: () => Unit) = { 147 | val url = s"${UIManager.origin}/v1/cart/products/$productCode/quantity/$quantity" 148 | $.post(JQueryAjaxSettings.url(url)._result) 149 | .done(onDone) 150 | .fail(() => println("cannot add a product twice")) 151 | } 152 | 153 | private def putInCart(productCode: String, updatedQuantity: Int) = { 154 | val url = s"${UIManager.origin}/v1/cart/products/$productCode/quantity/$updatedQuantity" 155 | $.ajax(JQueryAjaxSettings.url(url).method("PUT")._result) 156 | .done() 157 | } 158 | 159 | private def deletefromCart(productCode: String, onDone: () => Unit) = { 160 | val url = s"${UIManager.origin}/v1/cart/products/$productCode" 161 | $.ajax(JQueryAjaxSettings.url(url).method("DELETE")._result) 162 | .done(onDone) 163 | } 164 | 165 | private def getWebsocketUri(document: Document, context: String): String = { 166 | val wsProtocol = if (dom.document.location.protocol == "https:") "wss" else "ws" 167 | 168 | s"$wsProtocol://${ 169 | dom.document.location.host 170 | }/$context" 171 | } 172 | 173 | } 174 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.0.3 2 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // Comment to get more information during initialization 2 | logLevel := Level.Warn 3 | 4 | // Resolvers 5 | resolvers += "Typesafe repository" at "https://repo.typesafe.com/typesafe/releases/" 6 | 7 | // Sbt plugins 8 | 9 | // Use Scala.js 1.x 10 | addSbtPlugin("com.vmunier" % "sbt-web-scalajs" % "1.0.8-0.6") 11 | addSbtPlugin("org.scala-js" % "sbt-scalajs" % "0.6.24") 12 | addSbtPlugin("com.typesafe.play" % "sbt-plugin" % "2.6.15") 13 | addSbtPlugin("org.portable-scala" % "sbt-scalajs-crossproject" % "0.5.0") 14 | addSbtPlugin("com.typesafe.sbt" % "sbt-gzip" % "1.0.2") 15 | addSbtPlugin("com.typesafe.sbt" % "sbt-digest" % "1.1.4") 16 | addSbtPlugin("com.typesafe.sbteclipse" % "sbteclipse-plugin" % "5.2.4") 17 | 18 | addSbtPlugin("ch.epfl.scala" % "sbt-web-scalajs-bundler" % "0.13.0") -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/JSON.sc: -------------------------------------------------------------------------------- 1 | import io.circe.generic.auto._ 2 | import io.circe.syntax._ 3 | import io.circe.parser._ 4 | import io.fscala.shopping.shared.Product 5 | 6 | val newProduct = Product("NewOne","New","The brand new product", 100.0) 7 | 8 | newProduct.asJson 9 | 10 | val json = """{ 11 | | "name" : "NewOne", 12 | | "code" : "New", 13 | | "description" : "The brand new product", 14 | | "price" : 100.0 15 | |}""".stripMargin 16 | 17 | decode[Product](json) -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/actors/BrowserActor.scala: -------------------------------------------------------------------------------- 1 | package actors 2 | 3 | 4 | import akka.actor._ 5 | import io.circe.generic.auto._ 6 | import io.circe.parser.decode 7 | import io.fscala.shopping.shared.CartEvent 8 | 9 | object BrowserActor { 10 | def props(browserManager: ActorRef) = Props(new BrowserActor(browserManager)) 11 | } 12 | 13 | class BrowserActor(browserManager: ActorRef) extends Actor with ActorLogging { 14 | def receive = { 15 | case msg: String => 16 | log.info("Received JSON message: {}", msg) 17 | decode[CartEvent](msg) match { 18 | case Right(cartEvent) => 19 | log.info("Got {} message", cartEvent) 20 | browserManager forward cartEvent 21 | case Left(error) => log.info("Unhandled message : {}", error) 22 | } 23 | 24 | } 25 | } -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/actors/BrowserManagerActor.scala: -------------------------------------------------------------------------------- 1 | package actors 2 | 3 | import actors.BrowserManagerActor.AddBrowser 4 | import akka.actor.{Actor, ActorLogging, ActorRef, Props, Terminated} 5 | import io.circe.generic.auto._ 6 | import io.circe.syntax._ 7 | import io.fscala.shopping.shared._ 8 | 9 | import scala.collection.mutable.ListBuffer 10 | 11 | object BrowserManagerActor { 12 | def props() = Props(new BrowserManagerActor()) 13 | 14 | case class AddBrowser(browser: ActorRef) 15 | 16 | } 17 | 18 | private class BrowserManagerActor() extends Actor with ActorLogging { 19 | 20 | val browsers: ListBuffer[ActorRef] = ListBuffer.empty[ActorRef] 21 | 22 | def receive: Receive = { 23 | 24 | case AddBrowser(b) => 25 | context.watch(b) 26 | browsers += b 27 | log.info("websocket {} added", b.path) 28 | 29 | case CartEvent(user, product, action) => 30 | val messageText = s"The user '$user' ${action.toString} ${product.name}" 31 | log.info("Sending alarm to all the browser with '{}' action: {}", messageText, action) 32 | browsers.foreach(_ ! Alarm(messageText, action).asJson.noSpaces) 33 | 34 | case Terminated(b) => 35 | browsers -= b 36 | log.info("websocket {} removed", b.path) 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/actors/UserActor.scala: -------------------------------------------------------------------------------- 1 | package actors 2 | 3 | 4 | import akka.actor._ 5 | import io.circe.generic.auto._ 6 | import io.circe.parser.decode 7 | import io.circe.syntax._ 8 | import io.fscala.shopping.shared.{CartEvent, WebsocketMessage} 9 | 10 | object UserActor { 11 | def props(browser: ActorRef, browserManager :ActorRef) = Props(new UserActor(browser, browserManager)) 12 | } 13 | 14 | class UserActor(browser: ActorRef, browserManager :ActorRef) extends Actor with ActorLogging { 15 | def receive = { 16 | case msg: String => 17 | log.info("Received JSON message: {}",msg) 18 | decode[CartEvent](msg) match { 19 | case Right(cartEvent) => 20 | log.info("Got {} message", cartEvent) 21 | browserManager forward cartEvent 22 | case Left(error) => log.info("Unhandled message : {}",error) 23 | } 24 | 25 | } 26 | } 27 | 28 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/controllers/Application.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import javax.inject._ 4 | import play.api.libs.circe.Circe 5 | import play.api.mvc._ 6 | 7 | 8 | @Singleton 9 | class Application @Inject()(cc: ControllerComponents) extends AbstractController(cc) with Circe { 10 | 11 | def index = Action { 12 | Ok(views.html.index("Shopping Page")) 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/controllers/WebServices.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import dao.{CartDao, ProductDao} 4 | import io.circe.generic.auto._ 5 | import io.circe.parser._ 6 | import io.circe.syntax._ 7 | import io.fscala.shopping.shared.{Cart, Product, ProductInCart} 8 | import io.swagger.annotations._ 9 | import javax.inject.{Inject, Singleton} 10 | import play.api.Logger 11 | import play.api.libs.circe.Circe 12 | import play.api.mvc._ 13 | 14 | import scala.concurrent.ExecutionContext.Implicits.global 15 | import scala.concurrent.Future 16 | 17 | 18 | @Singleton 19 | @Api(value = "Product and Cart API") 20 | class WebServices @Inject()(cc: ControllerComponents, productDao: ProductDao, cartsDao: CartDao) extends AbstractController(cc) with Circe { 21 | 22 | 23 | val recoverError: PartialFunction[Throwable, Result] = { 24 | case e: org.h2.jdbc.JdbcSQLException => 25 | Logger.error("Inserting duplicate in the database", e) 26 | BadRequest("Cannot insert duplicates in the database") 27 | case e: Throwable => 28 | Logger.error("Error while writing in the database", e) 29 | InternalServerError("Cannot write in the database") 30 | } 31 | 32 | // *********** User Controler ******** // 33 | @ApiOperation(value = "Login to the service", consumes = "text/plain") 34 | @ApiImplicitParams(Array( 35 | new ApiImplicitParam( 36 | value = "Create a session for this user", 37 | required = true, 38 | dataType = "java.lang.String", // complete path 39 | paramType = "body" 40 | ) 41 | )) 42 | @ApiResponses(Array(new ApiResponse(code = 200, message = "login success"), new ApiResponse(code = 400, message = "Invalid user name supplied"))) 43 | def login() = Action { request => 44 | request.body.asText match { 45 | case None => BadRequest 46 | case Some(user) => Ok.withSession("user" -> user) 47 | } 48 | } 49 | 50 | // *********** CART Controler ******** // 51 | @ApiOperation(value = "List the product in the cart", consumes = "text/plain") 52 | @ApiResponses(Array(new ApiResponse(code = 200, message = "Product added"), 53 | new ApiResponse(code = 401, message = "unauthorized, please login before to proceed"), 54 | new ApiResponse(code = 500, message = "Internal server error, database error"))) 55 | def listCartProducts(): Action[AnyContent] = Action.async { request => 56 | val userOption = request.session.get("user") 57 | userOption match { 58 | case Some(user) => 59 | Logger.info(s"User '$user' is asking for the list of product in the cart") 60 | val futureInsert = cartsDao.cart4(user) 61 | 62 | futureInsert.map(products => Ok(products.asJson)).recover(recoverError) 63 | case None => Future.successful(Unauthorized) 64 | } 65 | } 66 | 67 | @ApiOperation(value = "Delete a product from the cart", consumes = "text/plain") 68 | @ApiResponses(Array(new ApiResponse(code = 200, message = "Product delete from the cart"), 69 | new ApiResponse(code = 401, message = "unauthorized, please login before to proceed"), 70 | new ApiResponse(code = 500, message = "Internal server error, database error"))) 71 | def deleteCartProduct(@ApiParam(name = "id", value = "The product code", required = true) id: String): Action[AnyContent] = Action.async { request => 72 | val userOption = request.session.get("user") 73 | userOption match { 74 | case Some(user) => 75 | Logger.info(s"User '$user' is asking to delete the product '$id' from the cart") 76 | val futureInsert = cartsDao.remove(ProductInCart(user, id)) 77 | futureInsert.map(_ => Ok).recover(recoverError) 78 | case None => Future.successful(Unauthorized) 79 | } 80 | } 81 | 82 | @ApiOperation(value = "Add a product in the cart", consumes = "text/plain") 83 | @ApiResponses(Array(new ApiResponse(code = 200, message = "Product added in the cart"), 84 | new ApiResponse(code = 400, message = "Cannot insert duplicates in the database"), 85 | new ApiResponse(code = 401, message = "unauthorized, please login before to proceed"), 86 | new ApiResponse(code = 500, message = "Internal server error, database error"))) 87 | def addCartProduct(@ApiParam(name = "id", value = "The product code", required = true) id: String, @ApiParam(name = "quantity", value = "The quantity to add", required = true) quantity: String): Action[AnyContent] = Action.async { request => 88 | val userOption = request.session.get("user") 89 | userOption match { 90 | case Some(user) => 91 | Logger.info(s"User '$user' is adding $quantity times the product'$id' in it's cart") 92 | val futureInsert = cartsDao.insert(Cart(user, id, quantity.toInt)) 93 | futureInsert.map(_ => Ok).recover(recoverError) 94 | case None => Future.successful(Unauthorized) 95 | } 96 | } 97 | 98 | @ApiOperation(value = "Update a product quantity in the cart", consumes = "text/plain") 99 | @ApiResponses(Array(new ApiResponse(code = 200, message = "Product updated in the cart"), 100 | new ApiResponse(code = 401, message = "unauthorized, please login before to proceed"), 101 | new ApiResponse(code = 500, message = "Internal server error, database error"))) 102 | def updateCartProduct(@ApiParam(name = "id", value = "The product code", required = true, example = "ALD1") id: String, @ApiParam(name = "quantity", value = "The quantity to update", required = true) quantity: String): Action[AnyContent] = Action.async { request => 103 | val userOption = request.session.get("user") 104 | userOption match { 105 | case Some(user) => 106 | Logger.info(s"User '$user' is updating the product'$id' in it's cart with a quantity of $quantity") 107 | val futureInsert = cartsDao.update(Cart(user, id, quantity.toInt)) 108 | futureInsert.map(_ => Ok).recover(recoverError) 109 | case None => Future.successful(Unauthorized) 110 | } 111 | } 112 | 113 | // *********** Product Controler ******** // 114 | @ApiOperation(value = "List all the products") 115 | @ApiResponses(Array(new ApiResponse(code = 200, message = "The list of all the product"))) 116 | def listProduct(): Action[AnyContent] = Action.async { _ => 117 | val futureProducts = productDao.all() 118 | for ( 119 | products <- futureProducts 120 | ) yield Ok(products.asJson) 121 | } 122 | 123 | @ApiOperation(value = "Add a product", consumes = "text/plain") 124 | @ApiImplicitParams(Array( 125 | new ApiImplicitParam( 126 | value = "The product to add", 127 | required = true, 128 | dataType = "io.fscala.shopping.shared.Product", // complete path 129 | paramType = "body" 130 | ) 131 | )) 132 | @ApiResponses(Array(new ApiResponse(code = 200, message = "Product added"), 133 | new ApiResponse(code = 400, message = "Invalid body supplied"), 134 | new ApiResponse(code = 500, message = "Internal server error, database error"))) 135 | def addProduct(): Action[AnyContent] = Action.async { request => 136 | val productOrNot = decode[Product](request.body.asText.getOrElse("")) 137 | productOrNot match { 138 | case Right(product) => 139 | val futureInsert = productDao.insert(product) 140 | futureInsert.map(_ => Ok).recover(recoverError) 141 | case Left(error) => 142 | Logger.error("Error while adding a product", error) 143 | Future.successful(BadRequest) 144 | } 145 | } 146 | 147 | } 148 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/controllers/WebSockets.scala: -------------------------------------------------------------------------------- 1 | package controllers 2 | 3 | import actors.{BrowserManagerActor, BrowserActor} 4 | import akka.actor.{ActorRef, ActorSystem} 5 | import akka.stream.Materializer 6 | import javax.inject._ 7 | import play.api.Logger 8 | import play.api.libs.streams.ActorFlow 9 | import play.api.mvc.{AbstractController, ControllerComponents, WebSocket} 10 | 11 | @Singleton 12 | class WebSockets @Inject()( 13 | implicit actorSystem: ActorSystem, 14 | materializer: Materializer, 15 | cc: ControllerComponents) extends AbstractController(cc) { 16 | 17 | val managerActor: ActorRef = actorSystem.actorOf(BrowserManagerActor.props(), "manager-actor") 18 | 19 | def cartEventWS: WebSocket = WebSocket.accept[String, String] { 20 | implicit request => 21 | ActorFlow.actorRef { out => 22 | Logger.info(s"Got a new websocket connection from ${request.host}") 23 | managerActor ! BrowserManagerActor.AddBrowser(out) 24 | BrowserActor.props(managerActor) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/dao/ProductDao.scala: -------------------------------------------------------------------------------- 1 | package dao 2 | 3 | import io.fscala.shopping.shared 4 | import io.fscala.shopping.shared.{Cart, CartKey, Product, ProductInCart} 5 | import javax.inject.Inject 6 | import play.api.db.slick.{DatabaseConfigProvider, HasDatabaseConfigProvider} 7 | import slick.jdbc.JdbcProfile 8 | 9 | import scala.concurrent.{ExecutionContext, Future} 10 | 11 | 12 | class ProductDao @Inject()(protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[JdbcProfile] { 13 | 14 | import profile.api._ 15 | 16 | def all(): Future[Seq[shared.Product]] = db.run(products.result) 17 | 18 | def insert(product: shared.Product): Future[Unit] = db.run(products insertOrUpdate product).map { _ => () } 19 | 20 | private class ProductTable(tag: Tag) extends Table[shared.Product](tag, "PRODUCT") { 21 | def name = column[String]("NAME") 22 | 23 | def code = column[String]("CODE") 24 | 25 | def description = column[String]("DESCRIPTION") 26 | 27 | def price = column[Double]("PRICE") 28 | 29 | override def * = (name, code, description, price) <> (Product.tupled, Product.unapply) 30 | } 31 | 32 | private val products = TableQuery[ProductTable] 33 | } 34 | 35 | class CartDao @Inject()(protected val dbConfigProvider: DatabaseConfigProvider)(implicit executionContext: ExecutionContext) extends HasDatabaseConfigProvider[JdbcProfile] { 36 | 37 | import profile.api._ 38 | 39 | 40 | def cart4(usr: String): Future[Seq[Cart]] = db.run(carts.filter(_.user === usr).result) 41 | 42 | def insert(cart: Cart): Future[_] = db.run(carts += cart) 43 | 44 | def remove(cart: ProductInCart): Future[Int] = db.run(carts.filter(c => matchKey(c, cart)).delete) 45 | 46 | def update(cart: Cart): Future[Int] = { 47 | val q = for { 48 | c <- carts if matchKey(c, cart) 49 | } yield c.quantity 50 | db.run(q.update(cart.quantity)) 51 | } 52 | 53 | private def matchKey(c: CartTable, cart: CartKey): Rep[Boolean] = { 54 | c.user === cart.user && c.productCode === cart.productCode 55 | } 56 | 57 | def all(): Future[Seq[Cart]] = db.run(carts.result) 58 | 59 | private class CartTable(tag: Tag) extends Table[Cart](tag, "CART") { 60 | 61 | def user = column[String]("USER") 62 | 63 | def productCode = column[String]("CODE") 64 | 65 | def quantity = column[Int]("QTY") 66 | 67 | override def * = (user, productCode, quantity) <> (Cart.tupled, Cart.unapply) 68 | } 69 | 70 | private val carts = TableQuery[CartTable] 71 | 72 | } 73 | 74 | 75 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/app/views/index.scala.html: -------------------------------------------------------------------------------- 1 | @(title: String) 2 | 3 | 4 | 5 | 6 | @title 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 |
15 |
16 |
17 |
18 |
19 |
20 | 21 |
22 |
23 |
24 | 25 | @scalajs.html.scripts("client", routes.Assets.versioned(_).toString, name => getClass.getResource(s"/public/$name") != null) 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/conf/application.conf: -------------------------------------------------------------------------------- 1 | play.i18n.langs = ["en"] 2 | 3 | play.server.http.idleTimeout = 10 seconds 4 | 5 | play.assets { 6 | path = "/public" 7 | urlPrefix = "/assets" 8 | } 9 | 10 | # Default database configuration 11 | slick.dbs.default.profile = "slick.jdbc.H2Profile$" 12 | slick.dbs.default.db.driver = "org.h2.Driver" 13 | slick.dbs.default.db.url = "jdbc:h2:mem:shopping" 14 | 15 | play.modules.enabled += "play.modules.swagger.SwaggerModule" 16 | 17 | play.evolutions.enabled = true 18 | play.evolutions.db.default.autoApply = true 19 | 20 | play.filters.hosts { 21 | # Allow requests from heroku and the temparary domain and localhost. 22 | allowed = ["shopping-fs.herokuapp.com", "localhost"] 23 | } 24 | play.filters.headers.contentSecurityPolicy = "default-src * 'self' 'unsafe-inline' data:" 25 | 26 | play.filters { 27 | cors { 28 | # Filter paths by a whitelist of path prefixes 29 | pathPrefixes = ["/"] 30 | 31 | # The allowed origins. If null, all origins are allowed. 32 | allowedOrigins = null 33 | 34 | # The allowed HTTP methods. If null, all methods are allowed 35 | allowedHttpMethods = null 36 | 37 | preflightMaxAge = 3 days 38 | } 39 | } 40 | #Remove all the security filter 41 | play.http.filters = play.api.http.NoHttpFilters 42 | 43 | api.version = "1.0.0" 44 | swagger.api.info = { 45 | description: "API for the online shopping example", 46 | title: "Online Shopping" 47 | } -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/conf/evolutions/default/1.sql: -------------------------------------------------------------------------------- 1 | # --- !Ups 2 | CREATE TABLE IF NOT EXISTS PUBLIC.PRODUCT ( 3 | name VARCHAR(100) NOT NULL, 4 | code VARCHAR(255) NOT NULL, 5 | description VARCHAR(1000) NOT NULL, 6 | price DOUBLE NOT NULL, 7 | PRIMARY KEY(code) 8 | ); 9 | INSERT INTO PUBLIC.PRODUCT (name,code, description, price) VALUES ('NAO','ALD1','NAO is an humanoid robot.', 3500); 10 | INSERT INTO PUBLIC.PRODUCT (name,code, description, price) VALUES ('PEPPER','ALD2','PEPPER is a robot moving with wheels and with a screen as human interaction',7000); 11 | INSERT INTO PUBLIC.PRODUCT (name,code, description, price) VALUES ('BEOBOT','BEO1','Beobot is a multipurpose robot.',159); 12 | 13 | 14 | CREATE TABLE IF NOT EXISTS PUBLIC.CART ( 15 | id IDENTITY AUTO_INCREMENT, 16 | user VARCHAR(255) NOT NULL, 17 | code VARCHAR(255) NOT NULL, 18 | qty INT NOT NULL, 19 | PRIMARY KEY(id), 20 | CONSTRAINT UC_CART UNIQUE (user,code) 21 | ); 22 | 23 | # --- !Downs 24 | DROP TABLE PRODUCT; 25 | DROP TABLE CART; -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/conf/heroku.conf: -------------------------------------------------------------------------------- 1 | include "application" 2 | 3 | play.http.secret.key="kUNSMzxg/ 2 | 3 | 4 | log/app.log 5 | 6 | 7 | log/app.%d{dd-MM-yyyy}.log 8 | 60 9 | 10 | 11 | 12 | %d [%thread] [%class] %5p - %m%n 13 | 14 | 15 | 16 | 17 | 19 | 20 | %-5level %X{akkaTimestamp} %-4relative [%thread] %logger{35} - %msg %n 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/conf/routes: -------------------------------------------------------------------------------- 1 | # Routes 2 | # This file defines all application routes (Higher priority routes first) 3 | # ~~~~ 4 | 5 | # Home page 6 | GET / controllers.Application.index 7 | 8 | # Login 9 | POST /v1/login controllers.WebServices.login 10 | 11 | # Product API 12 | GET /v1/products controllers.WebServices.listProduct 13 | POST /v1/products/add controllers.WebServices.addProduct 14 | 15 | # Cart API 16 | POST /v1/cart/products/:id/quantity/:quantity controllers.WebServices.addCartProduct(id,quantity) 17 | GET /v1/cart/products controllers.WebServices.listCartProducts() 18 | DELETE /v1/cart/products/:id controllers.WebServices.deleteCartProduct(id) 19 | PUT /v1/cart/products/:id/quantity/:quantity controllers.WebServices.updateCartProduct(id,quantity) 20 | 21 | GET /assets/*file controllers.Assets.versioned(path="/public", file: Asset) 22 | 23 | GET /v1/swagger.json controllers.ApiHelpController.getResources 24 | 25 | GET /docs/swagger-ui/*file controllers.Assets.at(path:String="/public/lib/swagger-ui", file:String) 26 | 27 | # Websocket 28 | 29 | GET /v1/cart/events controllers.WebSockets.cartEventWS -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/public/images/favicon.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter06-09/online-shoppping-cart/server/public/images/favicon.png -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/test/APISpec.scala: -------------------------------------------------------------------------------- 1 | import io.circe.generic.auto._ 2 | import io.circe.parser._ 3 | import io.fscala.shopping.shared.Cart 4 | import org.scalatest.concurrent.ScalaFutures 5 | import org.scalatest.time.{Millis, Seconds, Span} 6 | import org.scalatestplus.play.PlaySpec 7 | import org.scalatestplus.play.guice.GuiceOneServerPerSuite 8 | import play.api.libs.ws.{DefaultWSCookie, WSClient} 9 | import play.api.test.Helpers._ 10 | 11 | import scala.concurrent.Await 12 | import scala.concurrent.ExecutionContext.Implicits.global 13 | import scala.concurrent.duration.DurationInt 14 | 15 | class APISpec extends PlaySpec with ScalaFutures with GuiceOneServerPerSuite { 16 | 17 | implicit val defaultPatience = 18 | PatienceConfig(timeout = Span(20, Seconds), interval = Span(100, Millis)) 19 | 20 | val baseURL = s"localhost:$port/v1" 21 | val productsURL = s"http://$baseURL/products" 22 | val addProductsURL = s"http://$baseURL/products/add" 23 | val productsInCartURL = s"http://$baseURL/cart/products" 24 | 25 | def deleteProductInCartURL(productID: String) = s"http://$baseURL/cart/products/$productID" 26 | 27 | def actionProductInCartURL(productID: String, quantity: Int) = s"http://$baseURL/cart/products/$productID/quantity/$quantity" 28 | 29 | val login = s"http://$baseURL/login" 30 | 31 | 32 | "The API" should { 33 | val wsClient = app.injector.instanceOf[WSClient] 34 | 35 | 36 | "list all the product" in { 37 | 38 | val response = wsClient.url(productsURL).get().futureValue 39 | println(response.body) 40 | response.status mustBe OK 41 | response.body must include("PEPPER") 42 | response.body must include("NAO") 43 | response.body must include("BEOBOT") 44 | 45 | } 46 | 47 | "add a product" in { 48 | 49 | val newProduct = 50 | """ 51 | { 52 | "name" : "NewOne", 53 | "code" : "New", 54 | "description" : "The brand new product", 55 | "price" : 100.0 56 | } 57 | """ 58 | 59 | val posted = wsClient.url(addProductsURL).post(newProduct).futureValue 60 | posted.status mustBe OK 61 | 62 | val response = wsClient.url(productsURL).get().futureValue 63 | println(response.body) 64 | response.body must include("NewOne") 65 | } 66 | 67 | lazy val defaultCookie = { 68 | val loginCookies = Await.result(wsClient.url(login).post("me") 69 | .map(p => p.headers.get("Set-Cookie").map( 70 | _.head.split(";").head)), 1 seconds) 71 | val play_session = loginCookies.get.split("=").tail.mkString("") 72 | 73 | DefaultWSCookie("PLAY_SESSION", play_session) 74 | } 75 | 76 | "list all the products in a cart" in { 77 | val response = wsClient.url(productsInCartURL) 78 | .addCookies(defaultCookie).get().futureValue 79 | println(response) 80 | response.status mustBe OK 81 | 82 | val listOfProduct = decode[Seq[Cart]](response.body) 83 | listOfProduct.right.get mustBe empty 84 | } 85 | 86 | "add a product in the cart" in { 87 | val productID = "ALD1" 88 | val quantity = 1 89 | val posted = wsClient.url(actionProductInCartURL(productID, quantity)) 90 | .addCookies(defaultCookie).post("").futureValue 91 | posted.status mustBe OK 92 | 93 | val response = wsClient.url(productsInCartURL) 94 | .addCookies(defaultCookie).get().futureValue 95 | println(response) 96 | response.status mustBe OK 97 | response.body must include("ALD1") 98 | } 99 | 100 | "delete a product from the cart" in { 101 | val productID = "ALD1" 102 | val posted = wsClient.url(deleteProductInCartURL(productID)) 103 | .addCookies(defaultCookie).delete().futureValue 104 | posted.status mustBe OK 105 | 106 | val response = wsClient.url(productsInCartURL) 107 | .addCookies(defaultCookie).get().futureValue 108 | println(response) 109 | response.status mustBe OK 110 | response.body mustNot include("ALD1") 111 | } 112 | 113 | "update a product quantity in the cart" in { 114 | val productID = "ALD1" 115 | val quantity = 1 116 | val posted = wsClient.url(actionProductInCartURL(productID, quantity)) 117 | .addCookies(defaultCookie).post("").futureValue 118 | posted.status mustBe OK 119 | 120 | val newQuantity = 99 121 | val update = wsClient.url(actionProductInCartURL(productID, newQuantity)) 122 | .addCookies(defaultCookie).put("").futureValue 123 | update.status mustBe OK 124 | 125 | val response = wsClient.url(productsInCartURL) 126 | .addCookies(defaultCookie).get().futureValue 127 | println(response) 128 | response.status mustBe OK 129 | response.body must include(productID) 130 | response.body must include(newQuantity.toString) 131 | } 132 | 133 | "return a cookie when a user logins" in { 134 | val cookieFuture = wsClient.url(login).post("myID").map { 135 | response => 136 | response.headers.get("Set-Cookie").map( 137 | header => header.head.split(";") 138 | .filter(_.startsWith("PLAY_SESSION")).head) 139 | } 140 | 141 | val play_session_Key = cookieFuture.futureValue.get.split("=").head 142 | play_session_Key must equal("PLAY_SESSION") 143 | } 144 | } 145 | 146 | } 147 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/test/ApplicationSpec.scala: -------------------------------------------------------------------------------- 1 | import org.scalatest.Matchers._ 2 | import org.scalatest.concurrent.PatienceConfiguration.Timeout 3 | import org.scalatest.concurrent.ScalaFutures 4 | import org.scalatest.time.{Seconds, Span} 5 | import org.scalatestplus.play.PlaySpec 6 | import org.scalatestplus.play.guice.GuiceOneServerPerSuite 7 | import play.api.libs.ws.WSClient 8 | import play.api.test.Helpers._ 9 | import scala.concurrent.duration._ 10 | 11 | 12 | 13 | class ApplicationSpec extends PlaySpec with ScalaFutures with GuiceOneServerPerSuite { 14 | "Application" should { 15 | val wsClient = app.injector.instanceOf[WSClient] 16 | val myPublicAddress = s"localhost:$port" 17 | 18 | "send 404 on a bad request" in { 19 | val testURL = s"http://$myPublicAddress/boom" 20 | 21 | whenReady(wsClient.url(testURL).get(), Timeout(1 second)) { response => 22 | response.status mustBe NOT_FOUND 23 | } 24 | } 25 | 26 | "render the index page" in { 27 | val testURL = s"http://$myPublicAddress/" 28 | 29 | whenReady(wsClient.url(testURL).get(), Timeout(1 second)) { response => 30 | response.status mustBe OK 31 | response.contentType should include("text/html") 32 | } 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/test/CartDaoSpec.scala: -------------------------------------------------------------------------------- 1 | import dao.CartDao 2 | import io.fscala.shopping.shared.{Cart, ProductInCart} 3 | import org.scalatest.Matchers._ 4 | import org.scalatest.concurrent.ScalaFutures 5 | import org.scalatest.RecoverMethods._ 6 | import org.scalatestplus.play._ 7 | import org.scalatestplus.play.guice.GuiceOneAppPerSuite 8 | import play.api.Application 9 | 10 | import scala.concurrent.ExecutionContext.Implicits.global 11 | import scala.concurrent.Future 12 | 13 | class CartDaoSpec extends PlaySpec with ScalaFutures with GuiceOneAppPerSuite { 14 | 15 | "CartDao" should { 16 | val app2dao = Application.instanceCache[CartDao] 17 | "be empty on database creation" in { 18 | val dao: CartDao = app2dao(app) 19 | 20 | dao.all().futureValue shouldBe empty 21 | } 22 | 23 | "accept to add new cart" in { 24 | val dao: CartDao = app2dao(app) 25 | val user = "userAdd" 26 | 27 | val expected = Set( 28 | Cart(user, "ALD1", 1), 29 | Cart(user, "BEO1", 5) 30 | ) 31 | val noise = Set( 32 | Cart("userNoise", "ALD2", 10) 33 | ) 34 | val allCarts = expected ++ noise 35 | 36 | val insertFutures = allCarts.map(dao.insert) 37 | 38 | whenReady(Future.sequence(insertFutures)) { _ => 39 | dao.cart4(user).futureValue should contain theSameElementsAs expected 40 | dao.all().futureValue.size should equal(allCarts.size) 41 | } 42 | } 43 | "error thrown when adding a cart with same user and productCode" in { 44 | val dao: CartDao = app2dao(app) 45 | val user = "userAdd" 46 | 47 | val expected = Set( 48 | Cart(user, "ALD1", 1), 49 | Cart(user, "BEO1", 5) 50 | ) 51 | val noise = Set( 52 | Cart(user, "ALD1", 10) 53 | ) 54 | val allCarts = expected ++ noise 55 | 56 | 57 | val insertFutures = allCarts.map(dao.insert) 58 | 59 | recoverToSucceededIf[org.h2.jdbc.JdbcSQLException]{ 60 | Future.sequence(insertFutures) 61 | } 62 | } 63 | 64 | "accept to remove a product in a cart" in { 65 | val dao: CartDao = app2dao(app) 66 | val user = "userRmv" 67 | val initial = Vector( 68 | Cart(user, "ALD1", 1), 69 | Cart(user, "BEO1", 5) 70 | ) 71 | val expected = Vector(Cart(user, "ALD1", 1)) 72 | 73 | whenReady(Future.sequence(initial.map(dao.insert(_)))) { _ => 74 | dao.remove(ProductInCart(user, "BEO1")).futureValue 75 | dao.cart4(user).futureValue should contain theSameElementsAs (expected) 76 | } 77 | } 78 | 79 | "accept to update quantities of an item in a cart" in { 80 | val dao: CartDao = app2dao(app) 81 | val user = "userUpd" 82 | val initial = Vector(Cart(user, "ALD1", 1)) 83 | val expected = Vector(Cart(user, "ALD1", 5)) 84 | 85 | whenReady(Future.sequence(initial.map(dao.insert(_)))) { _ => 86 | dao.update(Cart(user, "ALD1", 5)).futureValue 87 | dao.cart4(user).futureValue should contain theSameElementsAs (expected) 88 | } 89 | } 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/test/IntegrationSpec.scala: -------------------------------------------------------------------------------- 1 | import org.scalatestplus.play.PlaySpec 2 | import org.scalatestplus.play.guice.GuiceOneServerPerSuite 3 | import play.api.libs.ws.WSClient 4 | 5 | import scala.concurrent.duration.DurationInt 6 | import scala.concurrent.Await 7 | import org.scalatest._ 8 | import Matchers._ 9 | 10 | class IntegrationSpec extends PlaySpec with GuiceOneServerPerSuite { 11 | "Application" should { 12 | val wsClient = app.injector.instanceOf[WSClient] 13 | val myPublicAddress = s"localhost:$port" 14 | "work from within a browser" in { 15 | 16 | val testURL = s"http://$myPublicAddress/" 17 | 18 | val response = Await.result(wsClient.url(testURL).get(), 1 seconds) 19 | 20 | response.body should include ("Shopping Page") 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/server/test/ProductDaoSpec.scala: -------------------------------------------------------------------------------- 1 | 2 | 3 | import dao.ProductDao 4 | import io.fscala.shopping.shared.Product 5 | import org.scalatest.Matchers._ 6 | import org.scalatest.concurrent.ScalaFutures 7 | import org.scalatestplus.play._ 8 | import org.scalatestplus.play.guice.GuiceOneAppPerSuite 9 | import play.api.Application 10 | 11 | 12 | class ProductDaoSpec extends PlaySpec with ScalaFutures with GuiceOneAppPerSuite { 13 | "ProductDao" should { 14 | "Have default rows on database creation" in { 15 | val app2dao = Application.instanceCache[ProductDao] 16 | val dao: ProductDao = app2dao(app) 17 | 18 | val expected = Set( 19 | Product("PEPPER", "ALD2", "PEPPER is a robot moving with wheels and with a screen as human interaction", 7000), 20 | Product("NAO", "ALD1", "NAO is an humanoid robot.", 3500), 21 | Product("BEOBOT", "BEO1", "Beobot is a multipurpose robot.", 159.0) 22 | ) 23 | 24 | dao.all().futureValue should contain theSameElementsAs (expected) 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/shared/src/main/scala/io/fscala/shopping/shared/Models.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.shared 2 | 3 | case class Product(name: String, code : String, description : String, price: Double) 4 | 5 | abstract class CartKey { 6 | def user: String 7 | def productCode: String 8 | } 9 | 10 | case class ProductInCart(user:String, productCode: String) extends CartKey 11 | 12 | case class Cart(user:String, productCode: String, quantity: Int) extends CartKey 13 | 14 | case class User(sessionID: String) 15 | -------------------------------------------------------------------------------- /Chapter06-09/online-shoppping-cart/shared/src/main/scala/io/fscala/shopping/shared/SharedMessages.scala: -------------------------------------------------------------------------------- 1 | package io.fscala.shopping.shared 2 | 3 | object SharedMessages { 4 | def itWorks = "It works!" 5 | } 6 | 7 | 8 | sealed trait ActionOnCart 9 | 10 | case object Add extends ActionOnCart 11 | 12 | case object Remove extends ActionOnCart 13 | 14 | sealed trait WebsocketMessage 15 | 16 | case class CartEvent(user: String, product: Product, action: ActionOnCart) extends WebsocketMessage 17 | 18 | case class Alarm(message: String, action: ActionOnCart) -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/INSTRUCTIONS.md: -------------------------------------------------------------------------------- 1 | # Installation 2 | ## JDK 3 | Download and install the latest JDK 1.8 from https://www.oracle.com/technetwork/java/javase/downloads/jdk8-downloads-2133151.html 4 | 5 | ## SBT 6 | Follow the instructions here: 7 | https://www.scala-sbt.org/1.0/docs/Setup.html 8 | 9 | ## Zeppelin 10 | - download Zeppelin: http://www.apache.org/dyn/closer.cgi/zeppelin/zeppelin-0.7.3/zeppelin-0.7.3-bin-all.tgz 11 | 12 | ### Windows 13 | We will install all tools in C:\opt . Feel free to use a different folder if you prefer. 14 | - Create a directory C:\opt 15 | - Install 7Zip to be able to decompress .tgz files: https://www.7-zip.org/download.html 16 | 17 | Zeppelin depends on Spark which depends on Hadoop. On Windows, Hadoop requires additional binary utilities: 18 | - Download https://github.com/steveloughran/winutils/archive/master.zip 19 | - Extract the directory hadoop-2.6.0 in master.zip to C:\opt\hadoop-2.6.0 20 | - Extract zeppelin-0.7.3-bin-all.tgz to C:\opt\zeppelin-0.7.3-bin-all 21 | - Edit C:\opt\zeppelin-0.7.3-bin-all\conf, and add the following line at the bottom: 22 | set HADOOP_HOME=C:/opt/hadoop-2.6.0 23 | - We need to fix some permissioning issue with the Hive folder: 24 | ``` 25 | cd C:\tmp\ 26 | mkdir hive 27 | C:\opt\hadoop-2.6.0\bin\winutils.exe chmod 777 hive 28 | ``` 29 | 30 | Run Zeppelin: 31 | ``` 32 | cd C:\opt\zeppelin-0.7.3-bin-all\bin 33 | zeppelin.cmd 34 | ``` 35 | 36 | ### Linux / MacOs 37 | We will install all tools in /opt. Feel free to use a different folder if you prefer. 38 | ```bash 39 | sudo mkdir /opt && sudo chown $USER /opt 40 | cd /opt 41 | tar xfz ~/Downloads/zeppelin-0.7.3-bin-all.tgz 42 | cd /opt/zeppelin-0.7.3-bin-all 43 | bin/zeppelin-daemon.sh start 44 | ``` 45 | 46 | ### Post-installation steps (all platforms) 47 | With your browser, go to http://localhost:8080 then click on the top right button -> Interpreter -> Spark -> Edit -> Dependencies 48 | 49 | Add the following dependencies: 50 | - org.apache.spark:spark-streaming-kafka-0-10_2.11:jar:2.1.0 51 | - org.apache.kafka:kafka-clients:jar:0.10.2.2 52 | 53 | 54 | 55 | ## Kafka 56 | - Download Kafka 1.1.1 https://www.apache.org/dyn/closer.cgi?path=/kafka/1.1.1/kafka_2.11-1.1.1.tgz 57 | 58 | ### Windows 59 | - Extract kafka_2.11-1.1.1.tgz to C:\opt\kafka_2.11-1.1.1 60 | - Start a Zookeeper in a command prompt: 61 | ``` 62 | cd C:\opt\kafka_2.11-1.1.1\bin\windows 63 | zookeeper-server-start.bat ..\..\config\zookeeper.properties 64 | ``` 65 | - Start Kafka in another command prompt: 66 | ``` 67 | cd C:\opt\kafka_2.11-1.1.1\bin\windows 68 | kafka-server-start.bat ..\..\config\server.properties 69 | ``` 70 | - Start a console consumer in another command prompt: 71 | ``` 72 | cd C:\opt\kafka_2.11-1.1.1\bin\windows 73 | kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic transactions --from-beginning 74 | ``` 75 | 76 | ### Linux / MacOs 77 | - Extract kafka_2.11-1.1.1.tgz to /opt/kafka_2.11-1.1.1 78 | - Start Zookeeper and Kafka in a terminal: 79 | ``` 80 | cd /opt/kafka_2.11-1.1.1 81 | bin/zookeeper-server-start.sh config/zookeeper.properties & 82 | bin/kafka-server-start.sh config/server.properties & 83 | ``` 84 | - Start a console consumer in another terminal: 85 | ``` 86 | cd /opt/kafka_2.11-1.1.1 87 | bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic transactions --from-beginning 88 | ``` 89 | 90 | # Running the Batch Producer 91 | This will write the last day of transactions from midnight to the data directory, then write new transactions every hour. 92 | 93 | Go to the project directory and run BatchProducerAppIntelliJ: 94 | ``` 95 | cd Scala-Programming-Projects/Chapter10-11/bitcoin-analyser 96 | sbt 97 | test:runMain coinyser.BatchProducerAppIntelliJ 98 | ``` 99 | 100 | # Running the Streaming Producer 101 | This will send live transactions to a kafka topic "transactions". 102 | ``` 103 | cd Scala-Programming-Projects/Chapter10-11/bitcoin-analyser 104 | sbt 105 | test:runMain coinyser.StreamingProducerApp 106 | ``` 107 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/build.sbt: -------------------------------------------------------------------------------- 1 | name := "bitcoin-analyser" 2 | 3 | version := "0.1" 4 | 5 | scalaVersion := "2.11.11" 6 | val sparkVersion = "2.3.1" 7 | 8 | libraryDependencies ++= Seq( 9 | "org.apache.spark" %% "spark-core" % sparkVersion % Provided, 10 | "org.apache.spark" %% "spark-core" % sparkVersion % Test classifier "tests", 11 | "org.apache.spark" %% "spark-core" % sparkVersion % Test classifier "test-sources", 12 | "org.apache.spark" %% "spark-sql" % sparkVersion % Provided, 13 | "org.apache.spark" %% "spark-sql" % sparkVersion % Test classifier "tests", 14 | "org.apache.spark" %% "spark-sql" % sparkVersion % Test classifier "test-sources", 15 | "org.apache.spark" %% "spark-catalyst" % sparkVersion % Test classifier "tests", 16 | "org.apache.spark" %% "spark-catalyst" % sparkVersion % Test classifier "test-sources", 17 | "com.typesafe.scala-logging" %% "scala-logging" % "3.9.0", 18 | "org.scalatest" %% "scalatest" % "3.0.4" % "test", 19 | "org.typelevel" %% "cats-core" % "1.1.0", 20 | "org.typelevel" %% "cats-effect" % "1.0.0-RC2", 21 | "org.apache.spark" %% "spark-streaming" % sparkVersion % Provided, 22 | "org.apache.spark" %% "spark-sql-kafka-0-10" % sparkVersion % Provided exclude ("net.jpountz.lz4", "lz4"), 23 | "com.pusher" % "pusher-java-client" % "1.8.0") 24 | 25 | scalacOptions += "-Ypartial-unification" 26 | 27 | // Avoids SI-3623 28 | target := file("/tmp/sbt/bitcoin-analyser") 29 | 30 | assemblyOption in assembly := (assemblyOption in assembly).value.copy(includeScala = false) 31 | test in assembly := {} 32 | mainClass in assembly := Some("coinyser.BatchProducerAppSpark") 33 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/.part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-30a257ae-4520-470a-9420-ffa01e20168d.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-7f73adb4-dfcb-40db-aa98-c204ad92f9d2.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-a943c9fe-fca2-4800-8cf5-a70e01eee675.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-b37c5aa1-30bd-4802-8457-75cad399e126.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-bf9386d0-bdab-496e-a0cc-1d3a76efb128.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-09/part-00000-d29d29dc-5109-46b9-a93e-d82a8e81b023.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/.part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet.crc -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-4c1b87e4-d129-4a65-97e0-d6d9b6a79442.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-626a9f68-9c4a-43f6-93bf-23022ab2255f.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PacktPublishing/Scala-Programming-Projects/9a3e0905d384500f7c6879d62aad289a83438276/Chapter10-11/bitcoin-analyser/data/transactions/date=2018-09-10/part-00000-67f22d39-b93c-42c0-9197-9890604b4ea4.c000.snappy.parquet -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/project/assembly.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.7") -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version = 1.1.6 -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Licensed to the Apache Software Foundation (ASF) under one or more 3 | # contributor license agreements. See the NOTICE file distributed with 4 | # this work for additional information regarding copyright ownership. 5 | # The ASF licenses this file to You under the Apache License, Version 2.0 6 | # (the "License"); you may not use this file except in compliance with 7 | # the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | # Set everything to be logged to the console 19 | log4j.rootCategory=INFO, console 20 | log4j.appender.console=org.apache.log4j.ConsoleAppender 21 | log4j.appender.console.target=System.err 22 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 23 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 24 | 25 | # Set the default spark-shell log level to WARN. When running the spark-shell, the 26 | # log level for this class is used to overwrite the root logger's log level, so that 27 | # the user can have different defaults for the shell and regular Spark apps. 28 | log4j.logger.org.apache.spark.repl.Main=WARN 29 | 30 | # Settings to quiet third party logs that are too verbose 31 | log4j.logger.org.spark_project.jetty=WARN 32 | log4j.logger.org.spark_project.jetty.util.component.AbstractLifeCycle=ERROR 33 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 34 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 35 | 36 | # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support 37 | log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL 38 | log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR 39 | 40 | # Parquet related logging 41 | log4j.logger.org.apache.parquet=WARN 42 | log4j.logger.org.apache.parquet.CorruptStatistics=ERROR 43 | log4j.logger.parquet.CorruptStatistics=ERROR 44 | 45 | log4j.logger.org.apache.kafka=WARN 46 | log4j.logger.org.apache.spark.sql.execution.streaming=WARN 47 | log4j.logger.org.apache.spark=WARN 48 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/AppConfig.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.net.URI 4 | 5 | import scala.concurrent.duration.FiniteDuration 6 | 7 | case class AppConfig(transactionStorePath: URI) 8 | 9 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/BatchProducer.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.net.URI 4 | import java.time.Instant 5 | import java.util.Scanner 6 | import java.util.concurrent.TimeUnit 7 | 8 | import cats.Monad 9 | import cats.effect.{IO, Timer} 10 | import cats.implicits._ 11 | import org.apache.spark.sql.functions.{explode, from_json, lit} 12 | import org.apache.spark.sql.types._ 13 | import org.apache.spark.sql.{Dataset, SaveMode, SparkSession} 14 | 15 | import scala.concurrent.duration._ 16 | 17 | class AppContext(val transactionStorePath: URI) 18 | (implicit val spark: SparkSession, 19 | implicit val timer: Timer[IO]) 20 | 21 | object BatchProducer { 22 | val WaitTime: FiniteDuration = 59.minute 23 | /** Number of seconds required by the API to make a transaction visible */ 24 | val ApiLag: FiniteDuration = 5.seconds 25 | 26 | 27 | def processRepeatedly(initialJsonTxs: IO[Dataset[Transaction]], jsonTxs: IO[Dataset[Transaction]]) 28 | (implicit appContext: AppContext): IO[Unit] = { 29 | import appContext._ 30 | 31 | for { 32 | beforeRead <- currentInstant 33 | firstEnd = beforeRead.minusSeconds(ApiLag.toSeconds) 34 | firstTxs <- initialJsonTxs 35 | firstStart = truncateInstant(firstEnd, 1.day) 36 | _ <- Monad[IO].tailRecM((firstTxs, firstStart, firstEnd)) { 37 | case (txs, start, instant) => 38 | processOneBatch(jsonTxs, txs, start, instant).map(_.asLeft) 39 | } 40 | } yield () 41 | } 42 | 43 | def processOneBatch(fetchNextTransactions: IO[Dataset[Transaction]], 44 | transactions: Dataset[Transaction], 45 | saveStart: Instant, 46 | saveEnd: Instant)(implicit appCtx: AppContext) 47 | : IO[(Dataset[Transaction], Instant, Instant)] = { 48 | import appCtx._ 49 | 50 | val transactionsToSave = filterTxs(transactions, saveStart, saveEnd) 51 | for { 52 | _ <- BatchProducer.save(transactionsToSave, appCtx.transactionStorePath) 53 | _ <- IO.sleep(WaitTime) 54 | 55 | beforeRead <- currentInstant 56 | // We are sure that lastTransactions contain all transactions until end 57 | end = beforeRead.minusSeconds(ApiLag.toSeconds) 58 | nextTransactions <- fetchNextTransactions 59 | } yield (nextTransactions, saveEnd, end) 60 | } 61 | 62 | 63 | def currentInstant(implicit timer: Timer[IO]): IO[Instant] = 64 | timer.clockRealTime(TimeUnit.SECONDS) map Instant.ofEpochSecond 65 | 66 | // Truncates to the start of interval 67 | def truncateInstant(instant: Instant, interval: FiniteDuration): Instant = { 68 | Instant.ofEpochSecond(instant.getEpochSecond / interval.toSeconds * interval.toSeconds) 69 | 70 | } 71 | 72 | def jsonToHttpTransactions(json: String)(implicit spark: SparkSession): Dataset[HttpTransaction] = { 73 | import spark.implicits._ 74 | val ds: Dataset[String] = Seq(json).toDS() 75 | val txSchema: StructType = spark.emptyDataset[HttpTransaction].schema 76 | val schema = ArrayType(txSchema) 77 | val arrayColumn = from_json($"value", schema) 78 | ds.select(explode(arrayColumn).alias("v")) 79 | .select("v.*") 80 | .as[HttpTransaction] 81 | } 82 | 83 | def httpToDomainTransactions(ds: Dataset[HttpTransaction]): Dataset[Transaction] = { 84 | import ds.sparkSession.implicits._ 85 | ds.select( 86 | $"date".cast(LongType).cast(TimestampType).as("timestamp"), 87 | $"date".cast(LongType).cast(TimestampType).cast(DateType).as("date"), 88 | $"tid".cast(IntegerType), 89 | $"price".cast(DoubleType), 90 | $"type".cast(BooleanType).as("sell"), 91 | $"amount".cast(DoubleType)) 92 | .as[Transaction] 93 | } 94 | 95 | def readTransactions(jsonTxs: IO[String])(implicit spark: SparkSession): IO[Dataset[Transaction]] = { 96 | jsonTxs.map(json => httpToDomainTransactions(jsonToHttpTransactions(json))) 97 | } 98 | 99 | 100 | 101 | 102 | def filterTxs(transactions: Dataset[Transaction], fromInstant: Instant, untilInstant: Instant) 103 | : Dataset[Transaction] = { 104 | import transactions.sparkSession.implicits._ 105 | transactions.filter( 106 | ($"timestamp" >= lit(fromInstant.getEpochSecond).cast(TimestampType)) && 107 | ($"timestamp" < lit(untilInstant.getEpochSecond).cast(TimestampType))) 108 | } 109 | 110 | def unsafeSave(transactions: Dataset[Transaction], path: URI): Unit = 111 | transactions 112 | .write 113 | .mode(SaveMode.Append) 114 | .partitionBy("date") 115 | .parquet(path.toString) 116 | 117 | 118 | def save(transactions: Dataset[Transaction], path: URI): IO[Unit] = 119 | IO(unsafeSave(transactions, path)) 120 | 121 | 122 | } 123 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/BatchProducerApp.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.net.{URI, URL} 4 | 5 | import cats.effect.{ExitCode, IO, IOApp} 6 | import coinyser.BatchProducer.{httpToDomainTransactions, jsonToHttpTransactions} 7 | import com.typesafe.scalalogging.StrictLogging 8 | import org.apache.spark.sql.{Dataset, SparkSession} 9 | 10 | import scala.io.Source 11 | 12 | class BatchProducerApp extends IOApp with StrictLogging { 13 | 14 | implicit val spark: SparkSession = SparkSession.builder.master("local[*]").getOrCreate() 15 | implicit val appContext: AppContext = new AppContext(new URI("./data/transactions")) 16 | 17 | def bitstampUrl(timeParam: String): URL = 18 | new URL("https://www.bitstamp.net/api/v2/transactions/btcusd?time=" + timeParam) 19 | 20 | def transactionsIO(timeParam: String): IO[Dataset[Transaction]] = { 21 | val url = bitstampUrl(timeParam) 22 | val jsonIO = IO { 23 | logger.info(s"calling $url") 24 | Source.fromURL(url).mkString 25 | } 26 | jsonIO.map(json => httpToDomainTransactions(jsonToHttpTransactions(json))) 27 | } 28 | 29 | val initialJsonTxs: IO[Dataset[Transaction]] = transactionsIO("day") 30 | val nextJsonTxs: IO[Dataset[Transaction]] = transactionsIO("hour") 31 | 32 | def run(args: List[String]): IO[ExitCode] = 33 | BatchProducer.processRepeatedly(initialJsonTxs, nextJsonTxs).map(_ => ExitCode.Success) 34 | 35 | } 36 | 37 | object BatchProducerAppSpark extends BatchProducerApp 38 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/HttpTransaction.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | case class HttpTransaction(date: String, 4 | tid: String, 5 | price: String, 6 | `type`: String, 7 | amount: String) 8 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/KafkaConfig.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | case class KafkaConfig(bootStrapServers: String, 4 | transactionsTopic: String) 5 | 6 | 7 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/StreamingConsumer.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import org.apache.spark.sql.{DataFrame, Dataset, SparkSession} 4 | import org.apache.spark.sql.functions._ 5 | 6 | object StreamingConsumer { 7 | def fromJson(df: DataFrame): Dataset[Transaction] = { 8 | import df.sparkSession.implicits._ 9 | val schema = Seq.empty[Transaction].toDS().schema 10 | df.select(from_json(col("value").cast("string"), schema).alias("v")) 11 | .select("v.*").as[Transaction] 12 | } 13 | 14 | def transactionStream(implicit spark: SparkSession, config: KafkaConfig): Dataset[Transaction] = 15 | fromJson(spark.readStream.format("kafka") 16 | .option("kafka.bootstrap.servers", config.bootStrapServers) 17 | .option("startingoffsets", "earliest") 18 | .option("subscribe", config.transactionsTopic) 19 | .load() 20 | ) 21 | 22 | } 23 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/StreamingProducer.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.sql.Timestamp 4 | import java.text.SimpleDateFormat 5 | import java.util.TimeZone 6 | 7 | import cats.effect.IO 8 | import com.fasterxml.jackson.databind.ObjectMapper 9 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 10 | import com.pusher.client.Client 11 | import com.pusher.client.channel.SubscriptionEventListener 12 | import com.typesafe.scalalogging.StrictLogging 13 | 14 | object StreamingProducer extends StrictLogging { 15 | 16 | def subscribe(pusher: Client)(onTradeReceived: String => Unit): IO[Unit] = 17 | for { 18 | _ <- IO(pusher.connect()) 19 | channel <- IO(pusher.subscribe("live_trades")) 20 | 21 | _ <- IO(channel.bind("trade", new SubscriptionEventListener() { 22 | override def onEvent(channel: String, event: String, data: String): Unit = { 23 | logger.info(s"Received event: $event with data: $data") 24 | onTradeReceived(data) 25 | } 26 | })) 27 | } yield () 28 | 29 | 30 | val mapper: ObjectMapper = { 31 | val m = new ObjectMapper() 32 | m.registerModule(DefaultScalaModule) 33 | val sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss") 34 | // Very important: the storage must be in UTC 35 | sdf.setTimeZone(TimeZone.getTimeZone("UTC")) 36 | m.setDateFormat(sdf) 37 | } 38 | 39 | def deserializeWebsocketTransaction(s: String): WebsocketTransaction = 40 | mapper.readValue(s, classOf[WebsocketTransaction]) 41 | 42 | def convertWsTransaction(wsTx: WebsocketTransaction): Transaction = 43 | Transaction( 44 | timestamp = new Timestamp(wsTx.timestamp.toLong * 1000), tid = wsTx.id, 45 | price = wsTx.price, sell = wsTx.`type` == 1, amount = wsTx.amount) 46 | 47 | def serializeTransaction(tx: Transaction): String = 48 | mapper.writeValueAsString(tx) 49 | 50 | } 51 | 52 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/Transaction.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.sql.{Date, Timestamp} 4 | import java.time.ZoneOffset 5 | 6 | case class Transaction(timestamp: Timestamp, 7 | date: Date, 8 | tid: Int, 9 | price: Double, 10 | sell: Boolean, 11 | amount: Double) 12 | 13 | 14 | object Transaction { 15 | def apply(timestamp: Timestamp, 16 | tid: Int, 17 | price: Double, 18 | sell: Boolean, 19 | amount: Double) = 20 | new Transaction( 21 | timestamp = timestamp, 22 | date = Date.valueOf( 23 | timestamp.toInstant.atOffset(ZoneOffset.UTC).toLocalDate), 24 | tid = tid, 25 | price = price, 26 | sell = sell, 27 | amount = amount) 28 | } 29 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/WebsocketTransaction.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | case class WebsocketTransaction(amount: Double, 4 | buy_order_id: Long, 5 | sell_order_id: Long, 6 | amount_str: String, 7 | price_str: String, 8 | timestamp: String, 9 | price: Double, 10 | `type`: Int, 11 | id: Int) 12 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/notebook-batch.sc: -------------------------------------------------------------------------------- 1 | // The code in this worksheet is meant to be run in Zeppelin. 2 | // However it can also can be run in IntelliJ: just paste it in a REPL session. 3 | 4 | // These definitions are provided in Zeppelin, do not paste in paragraphs 5 | import org.apache.spark.sql.{Dataset, SparkSession} 6 | implicit val spark: SparkSession = SparkSession.builder.master("local[*]").appName("coinyser").getOrCreate() 7 | import spark.implicits._ 8 | import org.apache.spark.sql.functions._ 9 | val z = new { 10 | def show[A](ds: Dataset[A]): Unit = ds.show(false) 11 | } 12 | 13 | val transactions = spark.read.parquet("/home/mikael/projects/Scala-Programming-Projects/bitcoin-analyser/data/transactions") 14 | z.show(transactions.sort($"timestamp")) 15 | 16 | val group = transactions.groupBy(window($"timestamp", "20 minutes")) 17 | 18 | val tmpAgg = group.agg( 19 | count("tid").as("count"), 20 | avg("price").as("avgPrice"), 21 | stddev("price").as("stddevPrice"), 22 | last("price").as("lastPrice"), 23 | sum("amount").as("sumAmount")) 24 | 25 | val aggregate = tmpAgg.select("window.start", "count", "avgPrice", "lastPrice", "stddevPrice", "sumAmount").sort("start").cache() 26 | 27 | z.show(aggregate) -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/main/scala/coinyser/notebook-streaming.sc: -------------------------------------------------------------------------------- 1 | // The code in this worksheet is meant to be run in Zeppelin. 2 | // However it can also can be run in IntelliJ: just paste it in a REPL session. 3 | 4 | // These definitions are provided in Zeppelin, do not paste in paragraphs 5 | import org.apache.spark.sql.{Dataset, SparkSession} 6 | implicit val spark: SparkSession = SparkSession.builder.master("local[*]").appName("coinyser").getOrCreate() 7 | import spark.implicits._ 8 | import org.apache.spark.sql.functions._ 9 | val z = new { 10 | def show[A](ds: Dataset[A]): Unit = ds.show(false) 11 | } 12 | 13 | // Paste the following code in the Notebook 14 | case class Transaction(timestamp: java.sql.Timestamp, 15 | date: String, 16 | tid: Int, 17 | price: Double, 18 | sell: Boolean, 19 | amount: Double) 20 | val schema = Seq.empty[Transaction].toDS().schema 21 | 22 | val dfStream = { 23 | spark.readStream.format("kafka") 24 | .option("kafka.bootstrap.servers", "localhost:9092") 25 | .option("startingoffsets", "latest") 26 | .option("subscribe", "transactions") 27 | .load() 28 | .select( 29 | from_json(col("value").cast("string"), schema) 30 | .alias("v")).select("v.*").as[Transaction] 31 | } 32 | 33 | val query = { 34 | dfStream 35 | .writeStream 36 | .format("memory") 37 | .queryName("transactionsStream") 38 | .outputMode("append") 39 | .start() 40 | } 41 | 42 | z.show(spark.table("transactionsStream").sort("timestamp")) 43 | 44 | 45 | val aggDfStream = { 46 | dfStream 47 | .withWatermark("timestamp", "1 second") 48 | .groupBy(window($"timestamp", "10 seconds").as("window")) 49 | .agg( 50 | count($"tid").as("count"), 51 | avg("price").as("avgPrice"), 52 | stddev("price").as("stddevPrice"), 53 | last("price").as("lastPrice"), 54 | sum("amount").as("sumAmount") 55 | ) 56 | .select("window.start", "count", "avgPrice", "lastPrice", "stddevPrice", "sumAmount") 57 | } 58 | 59 | val aggQuery = { 60 | aggDfStream 61 | .writeStream 62 | .format("memory") 63 | .queryName("aggregateStream") 64 | .outputMode("append") 65 | .start() 66 | } 67 | 68 | z.show(spark.table("aggregateStream").sort("start")) -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/BatchProducerAppIntelliJ.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | object BatchProducerAppIntelliJ extends BatchProducerApp 4 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/BatchProducerIT.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.sql.Timestamp 4 | import java.time.Instant 5 | import java.util.concurrent.TimeUnit 6 | 7 | import cats.effect.{IO, Timer} 8 | import org.apache.spark.sql.test.SharedSparkSession 9 | import org.scalatest.{Matchers, WordSpec} 10 | 11 | import scala.concurrent.duration.FiniteDuration 12 | 13 | 14 | class BatchProducerIT extends WordSpec with Matchers with SharedSparkSession { 15 | 16 | import testImplicits._ 17 | 18 | "BatchProducer.save" should { 19 | "save a Dataset[Transaction] to parquet" in withTempDir { tmpDir => 20 | val transaction1 = Transaction(timestamp = new Timestamp(1532365695000L), tid = 70683282, price = 7740.00, sell = false, amount = 0.10041719) 21 | val transaction2 = Transaction(timestamp = new Timestamp(1532365693000L), tid = 70683281, price = 7739.99, sell = false, amount = 0.00148564) 22 | val sourceDS = Seq(transaction1, transaction2).toDS() 23 | 24 | val uri = tmpDir.toURI 25 | BatchProducer.save(sourceDS, uri).unsafeRunSync() 26 | tmpDir.list() should contain("date=2018-07-23") 27 | val readDS = spark.read.parquet(uri.toString).as[Transaction] 28 | spark.read.parquet(uri + "/date=2018-07-23").show() 29 | sourceDS.collect() should contain theSameElementsAs readDS.collect() 30 | } 31 | } 32 | 33 | "BatchProducer.processOneBatch" should { 34 | "filter and save a batch of transaction, wait 59 mn, fetch the next batch" in withTempDir { tmpDir => 35 | implicit object FakeTimer extends Timer[IO] { 36 | private var clockRealTimeInMillis: Long = Instant.parse("2018-08-02T01:00:00Z").toEpochMilli 37 | 38 | def clockRealTime(unit: TimeUnit): IO[Long] = 39 | IO(unit.convert(clockRealTimeInMillis, TimeUnit.MILLISECONDS)) 40 | 41 | def sleep(duration: FiniteDuration): IO[Unit] = IO { 42 | clockRealTimeInMillis = clockRealTimeInMillis + duration.toMillis 43 | } 44 | 45 | def shift: IO[Unit] = ??? 46 | 47 | def clockMonotonic(unit: TimeUnit): IO[Long] = ??? 48 | } 49 | implicit val appContext: AppContext = new AppContext(transactionStorePath = tmpDir.toURI) 50 | 51 | implicit def toTimestamp(str: String): Timestamp = Timestamp.from(Instant.parse(str)) 52 | val tx1 = Transaction("2018-08-01T23:00:00Z", 1, 7657.58, true, 0.021762) 53 | val tx2 = Transaction("2018-08-02T01:00:00Z", 2, 7663.85, false, 0.01385517) 54 | val tx3 = Transaction("2018-08-02T01:58:30Z", 3, 7663.85, false, 0.03782426) 55 | val tx4 = Transaction("2018-08-02T01:58:59Z", 4, 7663.86, false, 0.15750809) 56 | val tx5 = Transaction("2018-08-02T02:30:00Z", 5, 7661.49, true, 0.1) 57 | 58 | // Start at 01:00, tx 2 ignored (too soon) 59 | val txs0 = Seq(tx1) 60 | // Fetch at 01:59, get nb 2 and 3, but will miss nb 4 because of Api lag 61 | val txs1 = Seq(tx2, tx3) 62 | // Fetch at 02:58, get nb 3, 4, 5 63 | val txs2 = Seq(tx3, tx4, tx5) 64 | // Fetch at 03:57, get nothing 65 | val txs3 = Seq.empty[Transaction] 66 | 67 | val start0 = Instant.parse("2018-08-02T00:00:00Z") 68 | val end0 = Instant.parse("2018-08-02T00:59:55Z") 69 | val threeBatchesIO = 70 | for { 71 | tuple1 <- BatchProducer.processOneBatch(IO(txs1.toDS()), txs0.toDS(), start0, end0) // end - Api lag 72 | (ds1, start1, end1) = tuple1 73 | 74 | tuple2 <- BatchProducer.processOneBatch(IO(txs2.toDS()), ds1, start1, end1) 75 | (ds2, start2, end2) = tuple2 76 | 77 | _ <- BatchProducer.processOneBatch(IO(txs3.toDS()), ds2, start2, end2) 78 | } yield (ds1, start1, end1, ds2, start2, end2) 79 | 80 | val (ds1, start1, end1, ds2, start2, end2) = threeBatchesIO.unsafeRunSync() 81 | ds1.collect() should contain theSameElementsAs txs1 82 | start1 should ===(end0) 83 | end1 should ===(Instant.parse("2018-08-02T01:58:55Z")) // initialClock + 1mn - 15s - 5s 84 | 85 | ds2.collect() should contain theSameElementsAs txs2 86 | start2 should ===(end1) 87 | end2 should ===(Instant.parse("2018-08-02T02:57:55Z")) // initialClock + 1mn -15s + 1mn -15s -5s = end1 + 45s 88 | 89 | val lastClock = Instant.ofEpochMilli( 90 | FakeTimer.clockRealTime(TimeUnit.MILLISECONDS).unsafeRunSync()) 91 | lastClock should === (Instant.parse("2018-08-02T03:57:00Z")) 92 | 93 | val savedTransactions = spark.read.parquet(tmpDir.toString).as[Transaction].collect() 94 | val expectedTxs = Seq(tx2, tx3, tx4, tx5) 95 | savedTransactions should contain theSameElementsAs expectedTxs 96 | } 97 | } 98 | 99 | 100 | } 101 | 102 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/BatchProducerSpec.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.io.{BufferedOutputStream, StringReader} 4 | import java.nio.CharBuffer 5 | import java.sql.Timestamp 6 | 7 | import cats.effect.IO 8 | import org.apache.spark.sql._ 9 | import org.apache.spark.sql.test.SharedSparkSession 10 | import org.scalatest.{Matchers, WordSpec} 11 | 12 | 13 | class BatchProducerSpec extends WordSpec with Matchers with SharedSparkSession { 14 | 15 | val httpTransaction1 = HttpTransaction("1532365695", "70683282", "7740.00", "0", "0.10041719") 16 | val httpTransaction2 = HttpTransaction("1532365693", "70683281", "7739.99", "0", "0.00148564") 17 | 18 | "BatchProducer.jsonToHttpTransaction" should { 19 | "create a Dataset[HttpTransaction] from a Json string" in { 20 | val json = 21 | """[{"date": "1532365695", "tid": "70683282", "price": "7740.00", "type": "0", "amount": "0.10041719"}, 22 | |{"date": "1532365693", "tid": "70683281", "price": "7739.99", "type": "0", "amount": "0.00148564"}]""".stripMargin 23 | 24 | val ds: Dataset[HttpTransaction] = BatchProducer.jsonToHttpTransactions(json) 25 | ds.collect() should contain theSameElementsAs Seq(httpTransaction1, httpTransaction2) 26 | } 27 | } 28 | 29 | "BatchProducer.httpToDomainTransactions" should { 30 | "transform a Dataset[HttpTransaction] into a Dataset[Transaction]" in { 31 | import testImplicits._ 32 | val source: Dataset[HttpTransaction] = Seq(httpTransaction1, httpTransaction2).toDS() 33 | val target: Dataset[Transaction] = BatchProducer.httpToDomainTransactions(source) 34 | val transaction1 = Transaction(timestamp = new Timestamp(1532365695000L), tid = 70683282, price = 7740.00, sell = false, amount = 0.10041719) 35 | val transaction2 = Transaction(timestamp = new Timestamp(1532365693000L), tid = 70683281, price = 7739.99, sell = false, amount = 0.00148564) 36 | 37 | target.collect() should contain theSameElementsAs Seq(transaction1, transaction2) 38 | } 39 | } 40 | 41 | } 42 | 43 | 44 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/FakePusher.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import com.pusher.client.Client 4 | import com.pusher.client.channel._ 5 | import com.pusher.client.connection.{Connection, ConnectionEventListener, ConnectionState} 6 | 7 | class FakePusher(val fakeTrades: Vector[String]) extends Client { 8 | var connected = false 9 | 10 | 11 | def subscribe(channelName: String): Channel = new FakeChannel(channelName, fakeTrades) 12 | 13 | def connect(): Unit = { 14 | connected = true 15 | } 16 | 17 | def disconnect(): Unit = ??? 18 | 19 | 20 | def subscribe(channelName: String, listener: ChannelEventListener, eventNames: String*): Channel = ??? 21 | 22 | def getPrivateChannel(channelName: String): PrivateChannel = ??? 23 | 24 | def subscribePrivate(channelName: String): PrivateChannel = ??? 25 | 26 | def subscribePrivate(channelName: String, listener: PrivateChannelEventListener, eventNames: String*): PrivateChannel = ??? 27 | 28 | def subscribePresence(channelName: String): PresenceChannel = ??? 29 | 30 | def subscribePresence(channelName: String, listener: PresenceChannelEventListener, eventNames: String*): PresenceChannel = ??? 31 | 32 | def getPresenceChannel(channelName: String): PresenceChannel = ??? 33 | 34 | def getConnection: Connection = ??? 35 | 36 | def getChannel(channelName: String): Channel = ??? 37 | 38 | def unsubscribe(channelName: String): Unit = ??? 39 | 40 | 41 | def connect(eventListener: ConnectionEventListener, connectionStates: ConnectionState*): Unit = ??? 42 | } 43 | 44 | class FakeChannel(val channelName: String, fakeTrades: Vector[String]) extends Channel { 45 | def getName: String = ??? 46 | 47 | def isSubscribed: Boolean = ??? 48 | 49 | def bind(eventName: String, listener: SubscriptionEventListener): Unit = { 50 | fakeTrades foreach { t => 51 | listener.onEvent(channelName, eventName, t) 52 | } 53 | } 54 | 55 | def unbind(eventName: String, listener: SubscriptionEventListener): Unit = ??? 56 | } 57 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/StreamingConsumerApp.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import org.apache.spark.sql.{Dataset, SparkSession} 4 | import org.apache.spark.sql.functions._ 5 | 6 | object StreamingConsumerApp extends App { 7 | 8 | implicit val spark: SparkSession = SparkSession 9 | .builder 10 | .master("local[*]") 11 | .appName("StreamingConsumerApp") 12 | .getOrCreate() 13 | 14 | implicit val config: KafkaConfig = KafkaConfig( 15 | bootStrapServers = "localhost:9092", 16 | transactionsTopic = "transactions_draft3" 17 | ) 18 | 19 | val txStream: Dataset[Transaction] = StreamingConsumer.transactionStream 20 | 21 | import spark.implicits._ 22 | 23 | // TODO move that to a Query class between batch and streaming 24 | val groupedStream = txStream 25 | .withWatermark("date", "1 second") 26 | .groupBy(window($"date", "1 minutes").as("window")) 27 | .agg( 28 | count($"tid").as("count"), 29 | avg("price").as("avgPrice"), 30 | stddev("price").as("stddevPrice"), 31 | last("price").as("lastPrice"), 32 | sum("amount").as("sumAmount") 33 | ) 34 | .select("window.start", "count", "avgPrice", "lastPrice", "stddevPrice", "sumAmount") 35 | 36 | groupedStream 37 | .writeStream 38 | .format("console") 39 | .queryName("groupedTx") 40 | .outputMode("append") 41 | .start() 42 | 43 | 44 | Thread.sleep(Long.MaxValue) 45 | 46 | } 47 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/StreamingConsumerSpec.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.util.TimeZone 4 | 5 | import coinyser.StreamingProducerSpec.SampleJsonTransaction 6 | import org.apache.spark.sql.{Dataset, SparkSession} 7 | import org.scalactic.TypeCheckedTripleEquals 8 | import org.scalatest.{BeforeAndAfterAll, EitherValues, Matchers, WordSpec} 9 | import org.scalatest.concurrent.Eventually 10 | 11 | class StreamingConsumerSpec extends WordSpec with Matchers with BeforeAndAfterAll with TypeCheckedTripleEquals with Eventually with EitherValues { 12 | TimeZone.setDefault(TimeZone.getTimeZone("UTC")) 13 | 14 | implicit val spark: SparkSession = SparkSession 15 | .builder 16 | .master("local[*]") 17 | .appName("coinyser") 18 | .getOrCreate() 19 | 20 | import spark.implicits._ 21 | 22 | "StreamingConsumer.fromJson" should { 23 | "convert a DataFrame of Json values to a Dataset[Transaction]" in { 24 | val df = Seq(SampleJsonTransaction).toDF("value") 25 | val ds: Dataset[Transaction] = StreamingConsumer.fromJson(df) 26 | ds.collect().toSeq should === (Seq(StreamingProducerSpec.SampleTransaction)) 27 | 28 | 29 | } 30 | 31 | } 32 | 33 | // TODO test transactionStream 34 | 35 | } 36 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/StreamingProducerApp.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import cats.effect.{ExitCode, IO, IOApp} 4 | import com.pusher.client.Pusher 5 | import StreamingProducer._ 6 | import org.apache.kafka.clients.producer.{KafkaProducer, ProducerRecord} 7 | import scala.collection.JavaConversions._ 8 | 9 | object StreamingProducerApp extends IOApp { 10 | val topic = "transactions" 11 | 12 | val pusher = new Pusher("de504dc5763aeef9ff52") 13 | 14 | val props = Map( 15 | "bootstrap.servers" -> "localhost:9092", 16 | "key.serializer" -> "org.apache.kafka.common.serialization.IntegerSerializer", 17 | "value.serializer" -> "org.apache.kafka.common.serialization.StringSerializer") 18 | 19 | def run(args: List[String]): IO[ExitCode] = { 20 | val kafkaProducer = new KafkaProducer[Int, String](props) 21 | 22 | subscribe(pusher) { wsTx => 23 | val tx = convertWsTransaction(deserializeWebsocketTransaction(wsTx)) 24 | val jsonTx = serializeTransaction(tx) 25 | kafkaProducer.send(new ProducerRecord(topic, tx.tid, jsonTx)) 26 | }.flatMap(_ => IO.never) 27 | } 28 | } 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/StreamingProducerSpec.scala: -------------------------------------------------------------------------------- 1 | package coinyser 2 | 3 | import java.sql.Timestamp 4 | 5 | import coinyser.StreamingProducerSpec._ 6 | import org.scalactic.TypeCheckedTripleEquals 7 | import org.scalatest.{Matchers, WordSpec} 8 | 9 | class StreamingProducerSpec extends WordSpec with Matchers with TypeCheckedTripleEquals { 10 | 11 | "StreamingProducer.deserializeWebsocketTransaction" should { 12 | "deserialize a valid String to a WebsocketTransaction" in { 13 | val str = 14 | """{"amount": 0.045318270000000001, "buy_order_id": 1969499130, 15 | |"sell_order_id": 1969495276, "amount_str": "0.04531827", 16 | |"price_str": "6339.73", "timestamp": "1533797395", 17 | |"price": 6339.7299999999996, "type": 0, "id": 71826763}""".stripMargin 18 | StreamingProducer.deserializeWebsocketTransaction(str) should 19 | ===(SampleWebsocketTransaction) 20 | } 21 | } 22 | 23 | "StreamingProducer.convertWsTransaction" should { 24 | "convert a WebSocketTransaction to a Transaction" in { 25 | StreamingProducer.convertWsTransaction(SampleWebsocketTransaction) should 26 | ===(SampleTransaction) 27 | } 28 | } 29 | 30 | "StreamingProducer.serializeTransaction" should { 31 | "serialize a Transaction to a String" in { 32 | StreamingProducer.serializeTransaction(SampleTransaction) should 33 | ===(SampleJsonTransaction) 34 | } 35 | } 36 | 37 | "StreamingProducer.subscribe" should { 38 | "register a callback that receives live trades" in { 39 | val pusher = new FakePusher(Vector("a", "b", "c")) 40 | var receivedTrades = Vector.empty[String] 41 | val io = StreamingProducer.subscribe(pusher) { trade => receivedTrades = receivedTrades :+ trade } 42 | io.unsafeRunSync() 43 | receivedTrades should ===(Vector("a", "b", "c")) 44 | } 45 | } 46 | } 47 | 48 | object StreamingProducerSpec { 49 | val SampleWebsocketTransaction = WebsocketTransaction( 50 | amount = 0.04531827, buy_order_id = 1969499130, sell_order_id = 1969495276, 51 | amount_str = "0.04531827", price_str = "6339.73", timestamp = "1533797395", 52 | price = 6339.73, `type` = 0, id = 71826763) 53 | 54 | val SampleTransaction = Transaction( 55 | timestamp = new Timestamp(1533797395000L), tid = 71826763, 56 | price = 6339.73, sell = false, amount = 0.04531827) 57 | 58 | val SampleJsonTransaction = 59 | """{"timestamp":"2018-08-09 06:49:55", 60 | |"date":"2018-08-09","tid":71826763,"price":6339.73,"sell":false, 61 | |"amount":0.04531827}""".stripMargin 62 | 63 | } 64 | -------------------------------------------------------------------------------- /Chapter10-11/bitcoin-analyser/src/test/scala/coinyser/pusher-subscribe.sc: -------------------------------------------------------------------------------- 1 | import com.pusher.client.Pusher 2 | import com.pusher.client.channel.SubscriptionEventListener 3 | 4 | 5 | val pusher = new Pusher("de504dc5763aeef9ff52") 6 | pusher.connect() 7 | val channel = pusher.subscribe("live_trades") 8 | 9 | channel.bind("trade", new SubscriptionEventListener() { 10 | override def onEvent(channel: String, event: String, data: String): Unit = { 11 | println(s"Received event: $event with data: $data") 12 | } 13 | }) 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Packt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | # Scala Programming Projects 5 | 6 | Scala Programming Projects 7 | 8 | This is the code repository for [Scala Programming Projects](https://www.packtpub.com/application-development/scala-programming-projects?utm_source=github&utm_medium=repository&utm_campaign=9781788397643), published by Packt. 9 | 10 | **Build real world projects using popular Scala frameworks like Play, Akka, and Spark** 11 | 12 | ## What is this book about? 13 | Scala is a type-safe JVM language that incorporates object-oriented and functional programming (OOP and FP) aspects. This book gets you started with essentials of software development by guiding you through various aspects of Scala programming, helping you bridge the gap between learning and implementing. You will learn about the unique features of Scala through diverse applications and experience simple yet powerful approaches for software development. 14 | 15 | This book covers the following exciting features: 16 | * Build, test, and package code using Scala Build Tool 17 | * Decompose code into functions, classes, and packages for maintainability 18 | * Implement the functional programming capabilities of Scala 19 | * Develop a simple CRUD REST API using the Play framework 20 | * Access a relational database using Slick 21 | 22 | If you feel this book is for you, get your [copy](https://www.amazon.com/dp/1788397649) today! 23 | 24 | https://www.packtpub.com/ 26 | 27 | ## Instructions and Navigations 28 | All of the code is organized into folders. For example, Chapter02. 29 | 30 | The code will look like the following: 31 | ``` 32 | class LazyDemo { 33 | lazy val lazyVal = { 34 | println("Evaluating lazyVal") 35 | ``` 36 | 37 | **Following is what you need for this book:** 38 | If you are an amateur programmer who wishes to learn how to use Scala, this book is for you. Knowledge of Java will be beneficial, but not necessary, to understand the concepts covered in this book. 39 | 40 | With the following software and hardware list you can run all code files present in the book (Chapter 1-11). 41 | ### Software and Hardware List 42 | | Chapter | Software required | Hardware required | 43 | | -------- | ------------------------------------ | ----------------------------------- | 44 | | 1-11 | IntelliJ Idea 2018.2.4 | 2 GB RAM minimum, 4 GB RAM recommended, 1.5 GB hard disk space + at least 1 GB for caches, 1024x768 minimum screen resolution| 45 | |6-9|Heroku CLI 7.16.0|A system with 4 GB RAM| 46 | |10-11|Spark 2.3.1|A system with 4 GB RAM| 47 | |11|Zeppelin 0.8.0|A system with 4 GB RAM| 48 | |11|Kafka 1.1.0|A system with 4 GB RAM| 49 | 50 | We also provide a PDF file that has color images of the screenshots/diagrams used in this book. [Click here to download it](https://www.packtpub.com/sites/default/files/downloads/9781788397643_ColorImages.pdf). 51 | 52 | ### Related products 53 | * Professional Scala [[Packt]](https://www.packtpub.com/web-development/professional-scala?utm_source=github&utm_medium=repository&utm_campaign=9781789533835) [[Amazon]](https://www.amazon.com/dp/B07G49XFYJ) 54 | 55 | * Scala Design Patterns - Second Edition [[Packt]](https://www.packtpub.com/application-development/scala-design-patterns-second-edition?utm_source=github&utm_medium=repository&utm_campaign=9781788471305) [[Amazon]](https://www.amazon.com/dp/B075Z2CMRX) 56 | 57 | ## Get to Know the Author 58 | **Mikaël Valot** 59 | is Principal Software Engineer at IHS Markit in London, UK. He is the lead developer of a strategic market risk solution for banking regulation. 60 | 61 | He has over 15 years of experience in the financial industry of the UK, Switzerland, and France. He has a Diplôme d'Ingénieur in Computing (equivalent to an M.Sc.) from Telecom Nancy, France. 62 | 63 | After years of working with Java, he started developing professionally with Scala in 2010, and never looked back. He was a speaker at Scala Exchange 2015. 64 | 65 | When he is not coding in Scala, Mikaël likes to dabble with Haskell, the Robotic Operating System, and deep learning. 66 | 67 | **Nicolas Jorand** 68 | is a senior developer. He worked for the finance industry for about 15 years before switching to the energy industry. He is a freelancer enjoying a partial time at Romande Energy, a Swiss utility company providing exclusively green electricity. Nicolas is a full-stack developer, playing with microcontrollers, developing standard web user and 3D interfaces on Unity, developing software to animate a humanoid robot (Nao) and finally, working with Scala on integration and backend software. All these projects are done with the same leitmotif; "In the dev process, get the issues as early as possible." 69 | 70 | ### Suggestions and Feedback 71 | [Click here](https://docs.google.com/forms/d/e/1FAIpQLSdy7dATC6QmEL81FIUuymZ0Wy9vH1jHkvpY57OiMeKGqib_Ow/viewform) if you have any feedback or suggestions. 72 | 73 | 74 | ### Download a free PDF 75 | 76 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF. 77 |

https://packt.link/free-ebook/9781788397643

--------------------------------------------------------------------------------