├── .gitignore ├── EffectfulFunctionalProgramming_VisualIntuition.png ├── LICENSE ├── README.md ├── build.sbt ├── exerciseClassic ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseConcurrent ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseCustom ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseError ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseOptics ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseReader ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseState ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseTask ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseWriter ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── project ├── build.properties └── plugins.sbt └── solutions ├── exercise2io ├── README.md └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseClassic └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseConcurrent └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseCustom └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseError └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseOptics └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseReader └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseState └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala ├── exerciseTask └── src │ ├── main │ └── scala │ │ └── scan │ │ └── Scanner.scala │ └── test │ └── scala │ └── scan │ └── ScannerSpec.scala └── exerciseWriter └── src ├── main └── scala │ └── scan │ └── Scanner.scala └── test └── scala └── scan └── ScannerSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | *.log 3 | 4 | # sbt specific 5 | .cache 6 | .history 7 | .lib/ 8 | dist/* 9 | target/ 10 | lib_managed/ 11 | src_managed/ 12 | project/boot/ 13 | project/plugins/project/ 14 | 15 | # Scala-IDE specific 16 | .scala_dependencies 17 | .worksheet 18 | .idea/ -------------------------------------------------------------------------------- /EffectfulFunctionalProgramming_VisualIntuition.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhutchison/GettingWorkDoneWithExtensibleEffects/7756d435f2e902869ddf9eadd1f7f0f292758cb1/EffectfulFunctionalProgramming_VisualIntuition.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Getting Work Done With Effectful Functional Programming 2 | 3 | A workshop on programming with *Effectful Functional Programming*. This can be described as a style of pure functional programming 4 | emphasizing the use of *effects*. It readily handles the complex or messy programming problems that are often encountered in industry. 5 | 6 | It has a close connection to the [Monad](https://typelevel.org/cats/typeclasses/monad.html) and 7 | [Applicative](https://typelevel.org/cats/typeclasses/applicative.html) type classes, as they are how we build large effectful 8 | programs out of small effectful programs. 9 | 10 | ### What are Effects? 11 | 12 | So what's a "effect" then? Like many abstract concepts, it is best grasped by seeing many examples, as we'll do in this workshop. 13 | But here's a brief overview: 14 | 15 | - Recall that functional programs can only act upon the world through the values they compute. 16 | - Effectful function programs compute *effectful functional values*, which have the form `F[A]`. 17 | - The type `A` in `F[A]` represents the pure payload value the program computes. 18 | - The type that wraps around it, `F[_]`, represents the effects that the program must resolve in order to, or additional to, yielding 19 | the `A` payload. 20 | 21 | It turns out that a huge variety and complexity of different program behaviours can be represented as an effectful value `F[A]`. 22 | 23 |
24 | Expand diagram: Effectful Functional Programming - a visual intuition 25 |

26 | 27 | ![diagram](EffectfulFunctionalProgramming_VisualIntuition.png) 28 | 29 |

30 |
31 | 32 | 33 | ### Libraries 34 | 35 | The workshop consists of a series of practical exercises using the following open source libraries: 36 | - [Cats](https://typelevel.org/cats/) 37 | - [Eff](https://github.com/atnos-org/eff) 38 | - [Monix](https://monix.io/) 39 | - [Monocle](http://julien-truffaut.github.io/Monocle/) 40 | - [Cats Effect](https://github.com/typelevel/cats-effect) 41 | 42 | ### Use Case 43 | 44 | Each exercise is an alternate implementation of the same use case: 45 | 46 | *Ever had a full disk? Where does the space go? Implement a program that can find the largest N files in a directory tree* 47 | 48 | 49 | ## Setup 50 | 51 | - Wifi/Internet required. 52 | 53 | - You will need Java 8+ and Simple Build Tool (`sbt`) [installed](http://www.scala-sbt.org/release/docs/Setup.html). 54 | 55 | - While SBT will download Scala and the Eff libraries on-demand, this can be a slow process. Before the workshop, it is recommended 56 | to run `sbt update` in the base directory to pre-download the required libraries. This may take a few minutes up to 1 hour, 57 | depending what you have cached locally in `~/.ivy2/cache`. 58 | 59 | - Import the base SBT project into your IDE: [Intellij](https://www.jetbrains.com/help/idea/2016.1/creating-and-running-your-scala-application.html), 60 | [Eclipse ScalaIDE](http://scala-ide.org/) or [Ensime](http://ensime.org/). 61 | 62 | - Or work with any editor and the SBT command line if you prefer. 63 | 64 | *Be warned that IDE presentation compilers don't correctly handle some Eff code*, and may 65 | flag valid code as invalid. Try your code with the full Scala compiler via SBT command line before concluding there is a problem. 66 | 67 | ## Exercises 68 | 69 | The SBT base project contains nine exercise projects, each with a README with instructions to attempt. Each of them contains 70 | a different implementation of a file scanner program. 71 | 72 | It is suggested to do the exercises in this order. The instruction pages are best viewed in a browser; reach them here: 73 | - [Classic](exerciseClassic/README.md) - File Scanning in a classic Scala style 74 | - [Task effect](exerciseTask/README.md) - Using Monix task effect to defer execution 75 | - [Reader effect](exerciseReader/README.md) - Using Reader effect for dependency injection and abstracting the environment 76 | - [Error effect](exerciseError/README.md) - Using Either effect for error handling 77 | - [Writer effect](exerciseWriter/README.md) - Using Writer effect for logging 78 | - [State effect](exerciseState/README.md) - Using State effect to keep track of Symlinks encountered 79 | - [Concurrency](exerciseConcurrency/README.md) - Scanning directories in parallel with applicative traversal 80 | - [Optics](exerciseOptics/README.md) - Using Optics to change the focus of a Reader effect 81 | - [Custom Effects](exerciseCustom/README.md) - Using a custom Filesystem effect 82 | 83 | 84 | There are three types of tasks you'll encounter 85 | - :mag: _Study Code_ Study existing application and test code 86 | - :pencil: _Write Code_ Adding missing code or changing existing code at an indicated line or method. 87 | - :arrow_forward: _Run Code_ Run the file scanner (eg `exercise1/run`) or the unit tests (eg `exercise1/test`) from SBT prompt. 88 | 89 | Each project can be compiled, run or tested separately; errors in one project won't affect the others. 90 | 91 | *Initially, most exercises will not compile and/or run, until you complete the specified tasks. To try running the code, 92 | go to the corresponding `solutions` project. * 93 | 94 | ## Solutions 95 | 96 | There is a [solutions](solutions/) subfolder containing corresponding solution subprojects. 97 | 98 | There is learning value in attempting a hard problem, getting stuck, then reviewing the solution. 99 | Use the solutions if you get blocked! 100 | 101 | ## Using SBT 102 | 103 | Start SBT in the base directory and then operate from the SBT prompt. Invoking each 104 | SBT command from the shell (eg `sbt exercise1/compile`) is slower due to JVM startup costs. 105 | ``` 106 | /Users/ben_hutchison/projects/GettingWorkDoneWithExtensibleEffects $ sbt 107 | Getting org.scala-sbt sbt 0.13.13 ... 108 | ..further sbt loading omitted.. 109 | > 110 | ``` 111 | 112 | To list all exercise- and solution- subproject names: 113 | ``` 114 | > projects 115 | ``` 116 | 117 | Try running the file scanner (ie `main` method) of subproject `solutionExerciseClassic` on the current directory. 118 | ``` 119 | > solutionExerciseClassic/run . 120 | ``` 121 | 122 | To compile sources in subproject `exercise1`: 123 | ``` 124 | > exerciseClassic/compile 125 | ``` 126 | 127 | To run any unit tests (in `src/test/scala/*`) under subproject `exerciseClassic` 128 | ``` 129 | > exerciseClassic/test 130 | ``` 131 | 132 | 133 | *SBT commands should be scoped to a subproject (eg `exerciseClassic/test`). Running eg `test` at the top level will load 134 | 10 copies of the classes into the SBT JVM, potentially leading to `OutOfMemoryError: Metaspace`* 135 | 136 | 137 | ## "Learn by Doing" 138 | 139 | This project teaches Extensible Effects in practice; what it feels like to code with the Eff framework. 140 | 141 | It doesn't make any attempt to cover 142 | the complex, subtle theory behind Eff, a refinement of 25 years experience of programming with monads, and isn't a complete picture of Eff 143 | by any means. At the time of writing however, there are more resources available covering the theory, than practice, of Eff, including: 144 | 145 | - The original paper [Extensible effects: an alternative to monad transformers](https://www.cs.indiana.edu/~sabry/papers/exteff.pdf) 146 | in Haskell and followup refinement [Freer Monads, More Extensible Effects](http://okmij.org/ftp/Haskell/extensible/more.pdf). 147 | 148 | - [Video presentation](https://www.youtube.com/watch?v=3Ltgkjpme-Y) of the above material by Oleg Kiselyov 149 | 150 | - [The Eff monad, one monad to rule them all](https://www.youtube.com/watch?v=KGJLeHhsZBo) by Eff library creator Eric Torreborre 151 | 152 | - My own video [Getting Work Done with the Eff Monad in Scala](https://www.youtube.com/watch?v=LhGq4HlozV4) 153 | 154 | ## Workshop History 155 | 156 | April 2017 157 | 158 | * Initial version based on Eff 4.3.1, cats 0.9.0 and Monix 2.2.4. Includes 5 exercises introducing 159 | `Reader`, `Either`, `Task` and `Writer` effects. 160 | 161 | * Presented at [Melbourne Scala meetup](https://www.meetup.com/en-AU/Melbourne-Scala-User-Group/events/240544821/) 162 | 163 | May 2017 164 | 165 | * Presented at for [YOW Lambdajam 2017, Sydney](http://lambdajam.yowconference.com.au/archive-2017/ben-hutchison-3/) 166 | 167 | April 2018 168 | 169 | * Upgrade libraries to Eff 5.2, Monic 3.0, cats 1.1, sbt 1.1 and introduce Cats Effect 0.10.1 library to use IO effect 170 | rather than Task, and Monocle 1.5 optics library. 171 | 172 | * Rewrite existing exercises 1 - 5 to reflect updated libraries, slightly changed emphasis. Add three new exercises covering 173 | State, Optics and Custom effects 174 | 175 | May 2018 176 | 177 | * Presented at [Melbourne Scala meetup](https://www.meetup.com/en-AU/Melbourne-Scala-User-Group/) 178 | 179 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | name := "GettingWorkDoneWithExtensibleEffects" 2 | 3 | version := "0.1" 4 | 5 | ThisBuild / scalaVersion := "2.12.5" 6 | 7 | val commonSettings = Seq( 8 | libraryDependencies ++= Seq( 9 | "org.scala-lang.modules" %% "scala-java8-compat" % "0.8.0", 10 | "org.typelevel" %% "cats-core" % "1.1.0", 11 | "org.typelevel" %% "mouse" % "0.17", 12 | "io.monix" %% "monix-eval" % "3.0.0-RC1", 13 | "org.atnos" %% "eff" % "5.2.0", 14 | "org.atnos" %% "eff-monix" % "5.2.0", 15 | "org.atnos" %% "eff-cats-effect" % "5.2.0", 16 | "com.github.julien-truffaut" %% "monocle-core" % "1.5.1-cats", 17 | "com.github.julien-truffaut" %% "monocle-generic" % "1.5.1-cats", 18 | "com.github.julien-truffaut" %% "monocle-macro" % "1.5.1-cats", 19 | "org.specs2" %% "specs2-core" % "4.0.3" % "test" 20 | ), 21 | // to write types like Reader[String, ?] 22 | addCompilerPlugin("org.spire-math" %% "kind-projector" % "0.9.6"), 23 | //to allow tuple extraction and type ascription in for expressions 24 | addCompilerPlugin("com.olegpy" %% "better-monadic-for" % "0.2.0"), 25 | // to get types like Reader[String, ?] (with more than one type parameter) correctly inferred for scala 2.12.x 26 | scalacOptions += "-Ypartial-unification", 27 | scalacOptions in Test += "-Yrangepos" 28 | ) 29 | 30 | 31 | 32 | lazy val exerciseClassic = (project in file("exerciseClassic")).settings(commonSettings) 33 | 34 | lazy val exerciseTask = (project in file("exerciseTask")).settings(commonSettings) 35 | 36 | lazy val exerciseReader = (project in file("exerciseReader")).settings(commonSettings) 37 | 38 | lazy val exerciseError = (project in file("exerciseError")).settings(commonSettings) 39 | 40 | lazy val exerciseWriter = (project in file("exerciseWriter")).settings(commonSettings) 41 | 42 | lazy val exerciseConcurrent = (project in file("exerciseConcurrent")).settings(commonSettings) 43 | 44 | lazy val exerciseState = (project in file("exerciseState")).settings(commonSettings) 45 | 46 | lazy val exerciseOptics = (project in file("exerciseOptics")).settings(commonSettings) 47 | 48 | lazy val exerciseCustom = (project in file("exerciseCustom")).settings(commonSettings) 49 | 50 | 51 | 52 | lazy val solutionExerciseClassic = (project in file("solutions/exerciseClassic")).settings(commonSettings) 53 | 54 | lazy val solutionExerciseTask = (project in file("solutions/exerciseTask")).settings(commonSettings) 55 | 56 | lazy val solutionExerciseReader = (project in file("solutions/exerciseReader")).settings(commonSettings) 57 | 58 | lazy val solutionExerciseError = (project in file("solutions/exerciseError")).settings(commonSettings) 59 | 60 | lazy val solutionExerciseWriter = (project in file("solutions/exerciseWriter")).settings(commonSettings) 61 | 62 | lazy val solutionExerciseConcurrent = (project in file("solutions/exerciseConcurrent")).settings(commonSettings) 63 | 64 | lazy val solutionExerciseState = (project in file("solutions/exerciseState")).settings(commonSettings) 65 | 66 | lazy val solutionExerciseOptics = (project in file("solutions/exerciseOptics")).settings(commonSettings) 67 | 68 | lazy val solutionExerciseCustom = (project in file("solutions/exerciseCustom")).settings(commonSettings) 69 | 70 | 71 | lazy val exercise1 = (project in file("exercise1")).settings(commonSettings) 72 | 73 | lazy val exercise2 = (project in file("exercise2")).settings(commonSettings) 74 | 75 | lazy val exercise3 = (project in file("exercise3")).settings(commonSettings) 76 | 77 | lazy val exercise4 = (project in file("exercise4")).settings(commonSettings) 78 | 79 | lazy val exercise5 = (project in file("exercise5")).settings(commonSettings) 80 | 81 | lazy val solutionExercise1 = (project in file("solutions/exercise1")).settings(commonSettings) 82 | 83 | lazy val solutionExercise2 = (project in file("solutions/exercise2")).settings(commonSettings) 84 | 85 | lazy val solutionExercise2io = (project in file("solutions/exercise2io")).settings(commonSettings) 86 | 87 | lazy val solutionExercise3 = (project in file("solutions/exercise3")).settings(commonSettings) 88 | 89 | lazy val solutionExercise4 = (project in file("solutions/exercise4")).settings(commonSettings) 90 | 91 | lazy val solutionExercise5 = (project in file("solutions/exercise5")).settings(commonSettings) 92 | 93 | val testSolutions = TaskKey[Unit]("testSolutions", "Run all solution tests") 94 | testSolutions := Seq( 95 | solutionExerciseClassic / Test / test, 96 | solutionExerciseTask / Test / test, 97 | solutionExerciseReader / Test / test, 98 | solutionExerciseError / Test / test, 99 | solutionExerciseWriter / Test / test, 100 | solutionExerciseState / Test / test, 101 | solutionExerciseConcurrent / Test / test, 102 | solutionExerciseOptics / Test / test, 103 | solutionExerciseCustom / Test / test, 104 | ).dependOn.value 105 | 106 | -------------------------------------------------------------------------------- /exerciseClassic/README.md: -------------------------------------------------------------------------------- 1 | # Classic File Scanner without Eff 2 | 3 | In all the exercises, we will look at variants of a File Scanner. The scanner finds and reports on the largest 10 files 4 | under a directory specified by the user, as well as collecting some stats about how many total files and total bytes are found. 5 | 6 | This first exercise uses regular Scala features without Eff: 7 | 8 | - File system operations are done directly inline, primarily using the `java.nio.file.Files` API provided in Java 8. This 9 | is preferable to the operations offered through the older `java.io.File` API because error conditions are signalled by an 10 | exception rather than simply returning an uninformative `null`. 11 | 12 | - Error handling is by throwing exceptions which will bubble to the top. 13 | 14 | ## Tasks 15 | 16 | ### :arrow_forward: _Run Code_ 17 | 18 | Run the scanner on current directory with `solutionExerciseClassic/run .` Does the result seem correct? 19 | 20 | Try it on some larger directory of your computer. 21 | 22 | ### :mag: _Study Code_ 23 | 24 | Study the algorithm used by the scanner. The key data structure is a `PathScan`, which contains a sorted list of the 25 | largest N files and their sizes in bytes, plus a count of total files scanned and total bytes across all files. 26 | 27 | Note how the scanner must combine `PathScan`s of differing subdirectories together to yield 28 | a single `PathScan` that summarizes both the *top N* files and *total* files visited. This combine operation has a 29 | [Monoid](http://typelevel.org/cats/typeclasses/monoid.html) structure. 30 | 31 | ### :pencil: _Write Code_ 32 | 33 | Complete the Monoid instance for PathScan. 34 | 35 | ### :arrow_forward: _Run Code_ 36 | 37 | Run the unit tests with `exerciseClassic/test` to verify your Monoid implementation. 38 | 39 | ### :mag: _Study Code_ 40 | 41 | Examine the [ScannerSpec](src/test/scala/scan/ScannerSpec.scala). 42 | 43 | Note how much of the test is involved with creating- and cleaning up- actual files. It would be nice 44 | is we could separate testing the algorithm from the filesystem. How should this be done? 45 | -------------------------------------------------------------------------------- /exerciseClassic/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.implicits._ 10 | 11 | 12 | object Scanner { 13 | 14 | def main(args: Array[String]): Unit = { 15 | println(scanReport(Paths.get(args(0)), 10)) 16 | } 17 | 18 | def scanReport(base: Path, topN: Int): String = { 19 | val scan = pathScan(FilePath(base), topN) 20 | 21 | ReportFormat.largeFilesReport(scan, base.toString) 22 | } 23 | 24 | def pathScan(filePath: FilePath, topN: Int): PathScan = filePath match { 25 | case File(path) => 26 | val fs = FileSize.ofFile(Paths.get(path)) 27 | PathScan(SortedSet(fs), fs.size, 1) 28 | case Directory(path) => 29 | val files = { 30 | val jstream = Files.list(Paths.get(path)) 31 | try jstream.toScala[List] 32 | finally jstream.close() 33 | } 34 | val subscans = files.map(subpath => pathScan(FilePath(subpath), topN)) 35 | subscans.combineAll(PathScan.topNMonoid(topN)) 36 | case Other(_) => 37 | PathScan.empty 38 | } 39 | 40 | } 41 | 42 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 43 | 44 | object PathScan { 45 | 46 | def empty = PathScan(SortedSet.empty, 0, 0) 47 | 48 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 49 | def empty: PathScan = PathScan.empty 50 | 51 | def combine(p1: PathScan, p2: PathScan): PathScan = ??? 52 | } 53 | 54 | } 55 | 56 | case class FileSize(path: Path, size: Long) 57 | 58 | object FileSize { 59 | 60 | def ofFile(file: Path) = { 61 | FileSize(file, Files.size(file)) 62 | } 63 | 64 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long ](_.size).reverse 65 | 66 | } 67 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 68 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 69 | //can overlap 70 | sealed trait FilePath { 71 | def path: String 72 | } 73 | object FilePath { 74 | 75 | def apply(path: Path): FilePath = 76 | if (Files.isRegularFile(path)) 77 | File(path.toString) 78 | else if (Files.isDirectory(path)) 79 | Directory(path.toString) 80 | else 81 | Other(path.toString) 82 | } 83 | case class File(path: String) extends FilePath 84 | case class Directory(path: String) extends FilePath 85 | case class Other(path: String) extends FilePath 86 | 87 | 88 | //Common pure code that is unaffected by the migration to Eff 89 | object ReportFormat { 90 | 91 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 92 | if (scan.largestFiles.nonEmpty) { 93 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 94 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.path}").mkString("", "\n", "\n") + 95 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 96 | } 97 | else 98 | s"No files found under path: $rootDir" 99 | } 100 | 101 | def formatByteString(bytes: Long): String = { 102 | if (bytes < 1000) 103 | s"${bytes} B" 104 | else { 105 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 106 | val pre = "KMGTPE".charAt(exp - 1) 107 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /exerciseClassic/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.PrintWriter 4 | import java.nio.file._ 5 | 6 | import org.specs2._ 7 | 8 | import scala.collection.immutable.SortedSet 9 | 10 | class ScannerSpec extends mutable.Specification { 11 | 12 | "Report Format" ! { 13 | val base = deletedOnExit(Files.createTempDirectory("exerciseClassic")) 14 | val base1 = deletedOnExit(fillFile(base, 1)) 15 | val base2 = deletedOnExit(fillFile(base, 2)) 16 | val subdir = deletedOnExit(Files.createTempDirectory(base, "subdir")) 17 | val sub1 = deletedOnExit(fillFile(subdir, 1)) 18 | val sub3 = deletedOnExit(fillFile(subdir, 3)) 19 | 20 | val actual = Scanner.pathScan(FilePath(base), 2) 21 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 22 | 23 | actual.mustEqual(expected) 24 | } 25 | 26 | def fillFile(dir: Path, size: Int) = { 27 | val path = dir.resolve(s"$size.txt") 28 | val w = new PrintWriter(path.toFile) 29 | try w.write("a" * size) 30 | finally w.close 31 | path 32 | } 33 | 34 | def deletedOnExit(p: Path) = { 35 | p.toFile.deleteOnExit() 36 | p 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /exerciseConcurrent/README.md: -------------------------------------------------------------------------------- 1 | # Concurrent Scanning 2 | 3 | The scanning of a directory tree can be done in parallel by processing each subdirectory in separate tasks. Because we have 4 | lifted our program into `Task`s, it is easy to enable concurrent scanning. 5 | 6 | A good general principle for effectful programming is to declare the dependencies between computations (including their effects) 7 | using monad flatMaps, then the runtime can execute as much in parallel near automagically. 8 | 9 | ### :pencil: _Write Code_ 10 | 11 | - In `pathScan` we currently use the `traverse` combinator to walk through the subdirectories and recursively invoke 12 | `pathScan` on each of them. Traverse implies that we want to process them strictly in order, but there is a variant 13 | `traverseA` (short for "traverse applicative") which says we can visit them in any order, completing when they have all 14 | been processed. Replace `traverse` with `traverseA` and the tasks can execute concurrently. Nothing more needed! 15 | 16 | ### :arrow_forward: _Run Code_ 17 | 18 | Run the tests to verify your task based implementation still gives the correct output. 19 | 20 | Run the scanner on a large directory tree. Do it several times as the results will likely include noise. 21 | Do you see a speed-up from concurrent scanning? 22 | 23 | You may see no improvement, or only a small % improvement (as I did). This may be because your hard drive or SSD has limited capability 24 | to serve requests in parallel. 25 | 26 | 27 | ### :arrow_forward: Run Code_ 28 | 29 | By default Monix batches the execution of a series of Tasks serially in the same thread to avoid thread context switches. 30 | The `BatchedExecution(32)` configuration in the Scanner specifies that 32 tasks should be executed by a thread before 31 | releasing control and returning to the configured Monix `Scheduler`. 32 | 33 | When tasks are structurally independent of each other, as is expressed by using `traverseA`, the batch size affects the 34 | degree of concurrency that will be enabled when the program executes. Too much, and a threads can context switch wastefully. 35 | (This is what happens with Scala `Future` and is the reason why Monix tasks typically run faster). Too little, and the available 36 | parallelism in the underlying work may not be achieved. 37 | 38 | Monix defaults to a batch size 1024. For tasks which do IO such as the Scanner, this may be too high. The value 32 was 39 | derived experimentally running the program against large directory scans on a Macbook SSD drive. 40 | 41 | - Run the Scanner on a directory tree with 100s of files, big enough that it takes approx 10secs to complete. 42 | Try varying the batch size parameter, by doubling or halving it progressively. 43 | Take multiple timings at each batch size. Is 32 the optimal value on your hardware or something else? 44 | 45 | - Also, if you have a multicore machine, observe the CPU usage as you change the batch size. You may see it rise with 46 | a smaller batch size, even if overall performance worsens. This is because it's doing more work in parallel but wasting 47 | effort on switching work between threads. 48 | -------------------------------------------------------------------------------- /exerciseConcurrent/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask[R, Filesystem] 57 | 58 | start <- taskDelay(System.currentTimeMillis()) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 61 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 62 | 63 | finish <- taskDelay(System.currentTimeMillis()) 64 | 65 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 66 | 67 | } yield ReportFormat.largeFilesReport(scan, base.toString) 68 | 69 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 70 | 71 | case f: File => 72 | for { 73 | fs <- FileSize.ofFile(f) 74 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 75 | } yield PathScan(SortedSet(fs), fs.size, 1) 76 | 77 | case dir: Directory => 78 | for { 79 | filesystem <- ask[R, Filesystem] 80 | topN <- takeTopN 81 | fileList <- taskDelay(filesystem.listFiles(dir)) 82 | childScans <- fileList.traverse(pathScan[R](_)) 83 | _ <- { 84 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 85 | val fileCount = fileList.count(_.isInstanceOf[File]) 86 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 87 | } 88 | } yield childScans.combineAll(topN) 89 | 90 | case Other(_) => 91 | PathScan.empty.pureEff[R] 92 | } 93 | 94 | 95 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 96 | scanConfig <- ask 97 | } yield new Monoid[PathScan] { 98 | def empty: PathScan = PathScan.empty 99 | 100 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 101 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 102 | p1.totalSize + p2.totalSize, 103 | p1.totalCount + p2.totalCount 104 | ) 105 | } 106 | 107 | } 108 | 109 | trait Filesystem { 110 | 111 | def filePath(path: String): FilePath 112 | 113 | def length(file: File): Long 114 | 115 | def listFiles(directory: Directory): List[FilePath] 116 | 117 | } 118 | case object DefaultFilesystem extends Filesystem { 119 | 120 | def filePath(path: String): FilePath = 121 | if (Files.isRegularFile(Paths.get(path))) 122 | File(path.toString) 123 | else if (Files.isDirectory(Paths.get(path))) 124 | Directory(path) 125 | else 126 | Other(path) 127 | 128 | def length(file: File) = Files.size(Paths.get(file.path)) 129 | 130 | def listFiles(directory: Directory) = { 131 | val files = Files.list(Paths.get(directory.path)) 132 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 133 | case Directory(path) => List(Directory(path)) 134 | case File(path) => List(File(path)) 135 | case Other(path) => List.empty 136 | }) 137 | finally files.close() 138 | } 139 | 140 | } 141 | 142 | case class ScanConfig(topN: Int) 143 | 144 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 145 | 146 | object PathScan { 147 | 148 | def empty = PathScan(SortedSet.empty, 0, 0) 149 | 150 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 151 | def empty: PathScan = PathScan.empty 152 | 153 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 154 | p1.largestFiles.union(p2.largestFiles).take(n), 155 | p1.totalSize + p2.totalSize, 156 | p1.totalCount + p2.totalCount 157 | ) 158 | } 159 | 160 | } 161 | 162 | case class FileSize(file: File, size: Long) 163 | 164 | object FileSize { 165 | 166 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 167 | fs <- ask 168 | } yield FileSize(file, fs.length(file)) 169 | 170 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 171 | 172 | } 173 | 174 | object EffTypes { 175 | 176 | type _filesystem[R] = Reader[Filesystem, ?] <= R 177 | type _config[R] = Reader[ScanConfig, ?] <= R 178 | type _err[R] = Either[String, ?] <= R 179 | type _log[R] = Writer[Log, ?] <= R 180 | } 181 | 182 | sealed trait Log {def msg: String} 183 | object Log { 184 | def error: String => Log = Error(_) 185 | def info: String => Log = Info(_) 186 | def debug: String => Log = Debug(_) 187 | } 188 | case class Error(msg: String) extends Log 189 | case class Info(msg: String) extends Log 190 | case class Debug(msg: String) extends Log 191 | 192 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 193 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 194 | //can overlap 195 | sealed trait FilePath { 196 | def path: String 197 | } 198 | 199 | case class File(path: String) extends FilePath 200 | case class Directory(path: String) extends FilePath 201 | case class Other(path: String) extends FilePath 202 | 203 | //Common pure code that is unaffected by the migration to Eff 204 | object ReportFormat { 205 | 206 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 207 | if (scan.largestFiles.nonEmpty) { 208 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 209 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 210 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 211 | } 212 | else 213 | s"No files found under path: $rootDir" 214 | } 215 | 216 | def formatByteString(bytes: Long): String = { 217 | if (bytes < 1000) 218 | s"${bytes} B" 219 | else { 220 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 221 | val pre = "KMGTPE".charAt(exp - 1) 222 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /exerciseConcurrent/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base), fs) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | logs.forall(expectedLogs.contains) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /exerciseCustom/README.md: -------------------------------------------------------------------------------- 1 | # Custom Effects 2 | 3 | In previous examples, we've been using `Task` to wrap code that is side-effecting, deferring its effects until the the 4 | Eff program is interpreted, following the style of Haskell's `IO`. 5 | 6 | This style, whereby all IO is wrapped in a single catch-all type, is coming under increasing challenge. It ought to be 7 | possible to be more descriptive about what type of external effects a program has, beyond simply acknowledging that 8 | they exist. 9 | 10 | In this example, we'll look at how custom effects can be used for this goal. We'll take the filesystem related operations 11 | that the program uses and factor them into a filesystem effect. We'll leave the interactions with the console (via `println`) 12 | and the clock (via `System.currentTimeMillis`) wrapped in Task, showing that custom IO effects can be introduced gradually 13 | into a program. 14 | 15 | 16 | ### :mag: _Study Code_ 17 | 18 | - The sealed trait `FilesystemCmd[A]` defines the operations supported by the custom effect. There are 3 subclasses, 19 | one for each operation. The generic type `A` represents the value returned when this operation is run. 20 | 21 | - The `FilesystemCmd` companion object also defines three combinators that introduce filesystem effects into an Eff 22 | program. Notice that the all work similarly, creating an instance of a FilesystemCmd, and then calling `Eff.send` 23 | to add the instance into the Eff stack of effects to be resolved at interpret-time. 24 | 25 | - As well as introducing effects, we need a way to resolve them. The abstract `Filesystem` class includes an interpreter 26 | that resolves filesystem effects out of the stack. This has been left incomplete as an exercise for you. 27 | 28 | - Note the presence of a `_filesystem` member constraint in the main `pathScan` method. 29 | 30 | ### :pencil: _Write Code_ 31 | 32 | - In `Filesystem`, complete the `match` clause inside `runFilesystemCmds`. You'll need to define logic for interpreting 33 | each type of `FilesystemCmd`. 34 | 35 | 36 | ### :arrow_forward: _Run Code_ 37 | 38 | Run tests to verify your implementation works correctly. 39 | 40 | ### :mag: _Study Code_ 41 | 42 | - Examine the unit tests. How does the use of a custom test change the testing approach? 43 | 44 | - What other effect is *not* needed in this version as a result? 45 | 46 | -------------------------------------------------------------------------------- /exerciseCustom/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, FilesystemCmd, Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runFilesystemCmds(DefaultFilesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | start <- taskDelay(System.currentTimeMillis()) 57 | 58 | base <- FilesystemCmd.filePath(base) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]](base).runReader[ScanConfig](ScanConfig(topNValid)) 61 | 62 | finish <- taskDelay(System.currentTimeMillis()) 63 | 64 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 65 | 66 | } yield ReportFormat.largeFilesReport(scan, base.toString) 67 | 68 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 69 | 70 | case f: File => 71 | for { 72 | fs <- FileSize.ofFile(f) 73 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 74 | } yield PathScan(SortedSet(fs), fs.size, 1) 75 | 76 | case dir: Directory => 77 | for { 78 | topN <- takeTopN 79 | fileList <- FilesystemCmd.listFiles(dir) 80 | childScans <- fileList.traverse(pathScan[R](_)) 81 | _ <- { 82 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 83 | val fileCount = fileList.count(_.isInstanceOf[File]) 84 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 85 | } 86 | } yield childScans.combineAll(topN) 87 | 88 | case Other(_) => 89 | PathScan.empty.pureEff[R] 90 | } 91 | 92 | 93 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 94 | scanConfig <- ask 95 | } yield new Monoid[PathScan] { 96 | def empty: PathScan = PathScan.empty 97 | 98 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 99 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 100 | p1.totalSize + p2.totalSize, 101 | p1.totalCount + p2.totalCount 102 | ) 103 | } 104 | 105 | } 106 | 107 | sealed trait FilesystemCmd[+A] 108 | 109 | object FilesystemCmd { 110 | 111 | implicit class EffFilesystemCmdOps[R, A](e: Eff[R, A]) { 112 | 113 | def runFilesystemCmds[U](fs: Filesystem)(implicit m: Member.Aux[FilesystemCmd, R, U]): Eff[U, A] = fs.runFilesystemCmds(e) 114 | } 115 | 116 | def filePath[R: _filesystem](path: String): Eff[R, FilePath] = Eff.send[FilesystemCmd, R, FilePath](MkFilePath(path)) 117 | 118 | def length[R: _filesystem](file: File): Eff[R, Long] = Eff.send[FilesystemCmd, R, Long](Length(file)) 119 | 120 | def listFiles[R: _filesystem](directory: Directory): Eff[R, List[FilePath]] = Eff.send[FilesystemCmd, R, List[FilePath]](ListFiles(directory)) 121 | 122 | } 123 | 124 | case class MkFilePath(path: String) extends FilesystemCmd[FilePath] 125 | case class Length(file: File) extends FilesystemCmd[Long] 126 | case class ListFiles(directory: Directory) extends FilesystemCmd[List[FilePath]] 127 | 128 | trait Filesystem { 129 | 130 | def runFilesystemCmds[R, A, U](effects: Eff[R, A])(implicit m: Member.Aux[FilesystemCmd, R, U]): Eff[U, A] = { 131 | 132 | val sideEffect = new SideEffect[FilesystemCmd] { 133 | def apply[X](fsc: FilesystemCmd[X]): X = 134 | (fsc match { 135 | 136 | //replace this handler with your FilesystemCmd cases here 137 | case _ => ??? 138 | 139 | }).asInstanceOf[X] 140 | 141 | def applicative[X, Tr[_] : Traverse](ms: Tr[FilesystemCmd[X]]): Tr[X] = 142 | ms.map(apply) 143 | } 144 | Interpret.interpretUnsafe(effects)(sideEffect)(m) 145 | } 146 | 147 | protected def filePath(path: String): FilePath 148 | 149 | protected def length(file: File): Long 150 | 151 | protected def listFiles(directory: Directory): List[FilePath] 152 | 153 | } 154 | object DefaultFilesystem extends Filesystem { 155 | 156 | protected def filePath(path: String): FilePath = 157 | if (Files.isRegularFile(Paths.get(path))) 158 | File(path.toString) 159 | else if (Files.isDirectory(Paths.get(path))) 160 | Directory(path) 161 | else 162 | Other(path) 163 | 164 | protected def length(file: File): Long = Files.size(Paths.get(file.path)) 165 | 166 | protected def listFiles(directory: Directory) = { 167 | val files = Files.list(Paths.get(directory.path)) 168 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 169 | case Directory(path) => List(Directory(path)) 170 | case File(path) => List(File(path)) 171 | case Other(path) => List.empty 172 | }) 173 | finally files.close() 174 | } 175 | 176 | } 177 | 178 | case class ScanConfig(topN: Int) 179 | 180 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 181 | 182 | object PathScan { 183 | 184 | def empty = PathScan(SortedSet.empty, 0, 0) 185 | 186 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 187 | def empty: PathScan = PathScan.empty 188 | 189 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 190 | p1.largestFiles.union(p2.largestFiles).take(n), 191 | p1.totalSize + p2.totalSize, 192 | p1.totalCount + p2.totalCount 193 | ) 194 | } 195 | 196 | } 197 | 198 | case class FileSize(file: File, size: Long) 199 | 200 | object FileSize { 201 | 202 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = FilesystemCmd.length(file).map(FileSize(file, _)) 203 | 204 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 205 | 206 | } 207 | 208 | object EffTypes { 209 | 210 | type _filesystem[R] = FilesystemCmd |= R 211 | type _config[R] = Reader[ScanConfig, ?] <= R 212 | type _err[R] = Either[String, ?] <= R 213 | type _log[R] = Writer[Log, ?] <= R 214 | } 215 | 216 | sealed trait Log {def msg: String} 217 | object Log { 218 | def error: String => Log = Error(_) 219 | def info: String => Log = Info(_) 220 | def debug: String => Log = Debug(_) 221 | } 222 | case class Error(msg: String) extends Log 223 | case class Info(msg: String) extends Log 224 | case class Debug(msg: String) extends Log 225 | 226 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 227 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 228 | //can overlap 229 | sealed trait FilePath { 230 | def path: String 231 | } 232 | 233 | case class File(path: String) extends FilePath 234 | case class Directory(path: String) extends FilePath 235 | case class Other(path: String) extends FilePath 236 | 237 | //Common pure code that is unaffected by the migration to Eff 238 | object ReportFormat { 239 | 240 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 241 | if (scan.largestFiles.nonEmpty) { 242 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 243 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 244 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 245 | } 246 | else 247 | s"No files found under path: $rootDir" 248 | } 249 | 250 | def formatByteString(bytes: Long): String = { 251 | if (bytes < 1000) 252 | s"${bytes} B" 253 | else { 254 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 255 | val pre = "KMGTPE".charAt(exp - 1) 256 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 257 | } 258 | } 259 | } 260 | -------------------------------------------------------------------------------- /exerciseCustom/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, FilesystemCmd, Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T]) = 56 | program.runReader(ScanConfig(2)).runFilesystemCmds(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base)) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | logs.forall(expectedLogs.contains) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /exerciseError/README.md: -------------------------------------------------------------------------------- 1 | # Exercise 3: Error handling without exceptions 2 | 3 | Exceptions are "considered harmful" in functional programming. The core of the objection is that they 4 | require a special execution mode from the runtime, that has very different behaviour to normal execution. 5 | But using effects, it's possible to implement exception-like behavior using pure functions and with no special runtime 6 | support, that can readily handle almost all error scenarios. 7 | 8 | In this version, there are two sources of errors: 9 | 10 | - Invalid user input 11 | 12 | - The `Filesystem` API abstracted in the reader exercise throws IOExceptions to signal errors in file system operations, such as listing 13 | a directory that doesnt exist. 14 | 15 | ## Tasks 16 | 17 | ### :mag: _Study Code_ 18 | 19 | - The set of Effects used at the top level of the program now includes a `Either[String, ?]` effect. The `?` here in the type represents a payload type which will be filled in when 20 | a computation using this effect occurs (a type like this containing unfilled parameters, or holes, is called *higher-kinded*) 21 | This effect is used to model errors resulting from validating user input. 22 | 23 | - Exceptions thrown by existing API methods are automatically caught and stored by the Task effect that is already present in the stack. 24 | A program that might throw an exception can be represented by the effect `Either[Throwable, ?]`. We'll deal with any 25 | exceptions the task effect might have stored in the exercise below. 26 | 27 | - Note how the interpretation of the Eff program now includes `runEither` to resolve the error effect. Note how `main` 28 | has to deal with the possiblity of error, because the interpretation result becomes an `Either`. 29 | 30 | - Despite wrapping an `Either` around the scan result, note how the return type of `scanReport` 31 | remains unchanged from the previous version, as `Eff[R, PathScan]`. This is a notable feature of the Eff-style of programming; 32 | the Eff expression just specifies the stack type (`R`) and the payload type (`PathScan` here). To understand fully what 33 | effects are going on, it's necessary to look at what `Member` typeclasses are declared on the `R` type. 34 | 35 | Which leads to the next task, adding a Member for the error effect.. 36 | 37 | ### :pencil: _Write Code_ 38 | 39 | - The `scanReport` method is doing validation that will raise errors, so we need to declare that an `Either[String, ?]` effect 40 | must be present. Do this by adding `_err` to the context bounds on the effect stack `R`. Where is `_err` defined? 41 | 42 | - In `scanReport`, fill in the `???` by validating the `topN` Int value is >= 0. The eff library defines a combinator for raising an error 43 | called `left`, it is already available via `import org.atnos.eff.all._`. An invalid int value should result in a message 44 | like "Invalid number of files -1". A valid value will need to be lifted into an Eff expression using `.pureEff[R]` 45 | 46 | - Exceptions thrown from the Filesystem and trapped by the Task effect will be rethrown when we call `runSyncUnsafe`. 47 | Lets instead convert them to an `Either[String, ?]` and combine them with the validation Either. 48 | 49 | Use the `attempt` combinator on `Task`, adding it after `runAsync`. This materializes any trapped exceptions and returns 50 | a `Task[Either[Throwable, T]]` that won't throw exceptions when run. 51 | 52 | But there's still a problem. We end up with two different types of errors in our result payload, `Throwable` and `String`. 53 | Add `.leftMap(_.toString).flatten` to the end of the interpretation to convert the Throwables to Strings and unnest 54 | the `Either`s. 55 | 56 | ### :mag: _Study Code_ 57 | 58 | - Whats going on in `pathScan` with this code: 59 | ``` 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]](fs.filePath(base)). 61 | runReader[ScanConfig](ScanConfig(topNValid)) 62 | ``` 63 | This is an example of extending the effect stack `R` (with `Fx.prepend[Reader[ScanConfig, ?], R]`) in a part of the program. 64 | We then interpret the effect out of the stack part-way through the program, rather than at the end, with 65 | `runReader[ScanConfig](ScanConfig(topNValid))`. 66 | 67 | ### :pencil: _Write Code_ 68 | 69 | Examine the test code in `ScannerSpec` and note the new test for a Filesystem exception "Error from Filesystem". 70 | 71 | Work out what value is expected when invoked on the mock filesystem provided, and replace the `???` to get the test working. 72 | 73 | ### :arrow_forward: _Run Code_ 74 | 75 | Run both tests to verify both happy and sad paths. 76 | -------------------------------------------------------------------------------- /exerciseError/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Either[String, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runAsync.runSyncUnsafe(1.minute) match { 41 | case Right(report) => println(report) 42 | case Left(msg) => println(s"Scan failed: $msg") 43 | } 44 | } 45 | 46 | def scanReport[R: _task: _filesystem](args: Array[String]): Eff[R, String] = for { 47 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 48 | 49 | topN <- { 50 | val n = args.lift(1).getOrElse("10") 51 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 52 | } 53 | topNValid <- ??? 54 | 55 | fs <- ask[R, Filesystem] 56 | 57 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]](fs.filePath(base)). 58 | runReader[ScanConfig](ScanConfig(topNValid)) 59 | 60 | } yield ReportFormat.largeFilesReport(scan, base.toString) 61 | 62 | def pathScan[R: _task: _filesystem: _config](path: FilePath): Eff[R, PathScan] = path match { 63 | case f: File => 64 | for { 65 | fs <- FileSize.ofFile(f) 66 | } yield PathScan(SortedSet(fs), fs.size, 1) 67 | case dir: Directory => 68 | for { 69 | filesystem <- ask[R, Filesystem] 70 | topN <- takeTopN 71 | fileList <- taskDelay(filesystem.listFiles(dir)) 72 | childScans <- fileList.traverse(pathScan[R](_)) 73 | } yield childScans.combineAll(topN) 74 | case Other(_) => 75 | PathScan.empty.pureEff[R] 76 | } 77 | 78 | 79 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 80 | scanConfig <- ask 81 | } yield new Monoid[PathScan] { 82 | def empty: PathScan = PathScan.empty 83 | 84 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 85 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 86 | p1.totalSize + p2.totalSize, 87 | p1.totalCount + p2.totalCount 88 | ) 89 | } 90 | 91 | } 92 | 93 | trait Filesystem { 94 | 95 | def filePath(path: String): FilePath 96 | 97 | def length(file: File): Long 98 | 99 | def listFiles(directory: Directory): List[FilePath] 100 | 101 | } 102 | case object DefaultFilesystem extends Filesystem { 103 | 104 | def filePath(path: String): FilePath = 105 | if (Files.isRegularFile(Paths.get(path))) 106 | File(path.toString) 107 | else if (Files.isDirectory(Paths.get(path))) 108 | Directory(path) 109 | else 110 | Other(path) 111 | 112 | def length(file: File) = Files.size(Paths.get(file.path)) 113 | 114 | def listFiles(directory: Directory) = { 115 | val files = Files.list(Paths.get(directory.path)) 116 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 117 | case Directory(path) => List(Directory(path)) 118 | case File(path) => List(File(path)) 119 | case Other(path) => List.empty 120 | }) 121 | finally files.close() 122 | } 123 | 124 | } 125 | 126 | case class ScanConfig(topN: Int) 127 | 128 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 129 | 130 | object PathScan { 131 | 132 | def empty = PathScan(SortedSet.empty, 0, 0) 133 | 134 | } 135 | 136 | case class FileSize(file: File, size: Long) 137 | 138 | object FileSize { 139 | 140 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 141 | fs <- ask 142 | } yield FileSize(file, fs.length(file)) 143 | 144 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 145 | 146 | } 147 | 148 | object EffTypes { 149 | 150 | type _filesystem[R] = Reader[Filesystem, ?] <= R 151 | type _config[R] = Reader[ScanConfig, ?] <= R 152 | type _err[R] = Either[String, ?] <= R 153 | } 154 | 155 | 156 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 157 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 158 | //can overlap 159 | sealed trait FilePath { 160 | def path: String 161 | } 162 | 163 | case class File(path: String) extends FilePath 164 | case class Directory(path: String) extends FilePath 165 | case class Other(path: String) extends FilePath 166 | 167 | //Common pure code that is unaffected by the migration to Eff 168 | object ReportFormat { 169 | 170 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 171 | if (scan.largestFiles.nonEmpty) { 172 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 173 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 174 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 175 | } 176 | else 177 | s"No files found under path: $rootDir" 178 | } 179 | 180 | def formatByteString(bytes: Long): String = { 181 | if (bytes < 1000) 182 | s"${bytes} B" 183 | else { 184 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 185 | val pre = "KMGTPE".charAt(exp - 1) 186 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /exerciseError/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).runAsync.attempt.runSyncUnsafe(3.seconds) 57 | 58 | "file scan" ! { 59 | val actual = run(Scanner.pathScan(base), fs) 60 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 61 | 62 | actual.mustEqual(expected) 63 | } 64 | 65 | "Error from Filesystem" ! { 66 | val emptyFs: Filesystem = MockFilesystem(directories, Map.empty) 67 | 68 | val actual = runE(Scanner.scanReport(Array("base", "10")), emptyFs) 69 | val expected = ??? 70 | 71 | actual.mustEqual(expected) 72 | } 73 | 74 | type E = Fx.fx3[Task, Reader[Filesystem, ?], Either[String, ?]] 75 | def runE[T](program: Eff[E, T], fs: Filesystem) = 76 | //there are two nested Either in the stack, one from Exceptions and one from errors raised by the program 77 | //we convert to a common error type String then flatten 78 | program.runReader(fs).runEither.runAsync.attempt.runSyncUnsafe(3.seconds).leftMap(_.toString).flatten 79 | 80 | "Error - Report with non-numeric input" ! { 81 | val actual = runE(Scanner.scanReport(Array("base", "not a number")), fs) 82 | val expected = Left("Number of files must be numeric: not a number") 83 | 84 | actual.mustEqual(expected) 85 | } 86 | 87 | "Error - Report with non-positive input" ! { 88 | val actual = runE(Scanner.scanReport(Array("base", "-1")), fs) 89 | val expected = Left("Invalid number of files -1") 90 | 91 | actual.mustEqual(expected) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /exerciseOptics/README.md: -------------------------------------------------------------------------------- 1 | # Transforming Effects with Functions & Optics 2 | 3 | In this exercise we'll look at how we can transform within effect "families" using functions and Monocle optics. 4 | 5 | We have seen several examples of effects which have a second type parameter in addition to the payload: 6 | 7 | - `Either[E, ?]` has the type of the error `E` 8 | - `Reader[A, ?]` has the type of value to be provided/injected `A` 9 | - `State[S, ?]` has the type of the state `S` 10 | 11 | We can think of these effects as being families of related effects, sharing the same basic mechanism but with a particular 12 | type focus. But, eg `Reader[A, ?]` and `Reader[B, ?]`, are different effects in the Eff effect stack and can't be combined. 13 | 14 | The recommended approach is where possible, to work in terms of one effect from an effect family across your program 15 | (ie one `Reader`, one `State`). This tends to be the most robust and reliable solution from a type inference perspective. 16 | However, it's sometimes necessary or convenient to be able to combine effects from the same family, but with different foci, 17 | together. You may need to integrate modules written by different authors that both focus on types from their own modules, 18 | for example. So we'll look at how you can do that: 19 | 20 | - To transform a `Either[E, ?]` into `Either[E1, ?]`, we simply need a function `E => E1` and we can `leftMap` it (`Either` 21 | is a *covariant* functor). `Writer` works similarly. 22 | 23 | - To transform a `Reader[A, ?]` into `Reader[B, ?]`, we need a function `B => A` to transform the input while we read it 24 | (`Reader` is a *contravariant* functor). 25 | 26 | - `State[S, ?]` consumes and emits its `S` value, so a single function won't do. We need a bidirectional transform between 27 | `S` and `T` to produce a `State[T, ?]` effect. This is what a `Lens[S, T]` is. 28 | 29 | In this exercise, we'll look at transforming two `Reader` effects to a common shared type, but the approach should extend 30 | to any effect that has a focus type. 31 | 32 | ### :mag: _Study Code_ 33 | 34 | - You should have a conceptual idea of how [Monocle lenses](http://julien-truffaut.github.io/Monocle/optics/lens.html) 35 | work to follow this exercise. 36 | 37 | 38 | - Recall in previous examples we had two distinct readers for two different type of configuration, the `Filesystem` and 39 | the number, `topN`, of the largest files that the scanner would keep track of. Now, there is a new class `AppConfig` that 40 | models the applications config, which has these two config items as fields. We want to write the overall program purely in 41 | terms of `AppConfig` and transform its `Reader` effect into readers of the other two subtypes. 42 | 43 | Find `AppConfig` and its companion. Note that it has an implicit `Lens` declared between `AppConfig` and `Filesystem`. 44 | 45 | 46 | - Note that `pathScan` has a single reader effect `_appconfig`, but calls methods like `takeTopN` and `FileSize.ofFile` 47 | that declare different reader effects `_config` and `_filesystem` respectively. So effect transformation is occurring 48 | here implicitly, driven by the type system. 49 | 50 | 51 | - The mechanism used to transform effects is `EffOptics.readerLens`. We wants to `transform` the membership typeclasses, 52 | yielding a typeclass that certifies membership for the transformed focus. The `~>` denotes a *natural transformation*, 53 | which is a "higher kinded function" from an `A[_]` to a `B[_]`. 54 | 55 | Note that `EffOptics.readerLens` is marked implicit, so it is wired in by the compiler whenever a method is searching 56 | for an effect member typeclass. However, `EffOptics.readerLens` itself trails an implicit dependency upon a `Lens[S, T]` 57 | to be in implicit scope. 58 | 59 | 60 | ### :pencil: _Write Code_ 61 | 62 | - Complete the implementation of the transform `apply` method in `EffOptics.readerLens`; ie how can you use a `Lens[S, T]` 63 | to build a `Reader[T]` from a `Reader[S, ?]`? 64 | 65 | - Add a second lens in `AppConfig` to `ScanConfig`, using Monocle's `GenLens` macro. Ensure it's marked `implicit` so it 66 | can passed by the compiler when effects need to be transformed. 67 | 68 | 69 | ### :arrow_forward: Run Code_ 70 | 71 | Run the unit tests to check that your implementation is correct. 72 | 73 | 74 | 75 | ### :mag: _Study Code_ 76 | 77 | - Why did we use `Lens` and not just `A => B` to transform our `Reader` effects? A function would have worked, as the 78 | bidrectional nature of `Lens` is only required for `State` effects where the change of focus is two-way. 79 | 80 | The reason we avoided function was because we didn't want to depend upon any function `A => B` implicitly. It's a very 81 | generic type and likely to lead to ambiguity. Lens is a less common type so we could be more confident that there wouldn't be random 82 | lenses already in implicit scope. 83 | -------------------------------------------------------------------------------- /exerciseOptics/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection._ 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import monix.eval._ 17 | import monix.execution._ 18 | import monocle._ 19 | import monocle.macros._ 20 | 21 | import scala.concurrent.duration._ 22 | 23 | import EffTypes._ 24 | import EffOptics._ 25 | 26 | 27 | object Scanner { 28 | 29 | type R = Fx.fx2[Task, Reader[AppConfig, ?]] 30 | 31 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 32 | 33 | def main(args: Array[String]): Unit = { 34 | val program = scanReport[R](args(0)).map(println) 35 | 36 | program.runReader(AppConfig(ScanConfig(10), DefaultFilesystem)).runAsync.runSyncUnsafe(1.minute) 37 | } 38 | 39 | def scanReport[R: _task: _appconfig](base: String): Eff[R, String] = for { 40 | fs <- ask[R, Filesystem] 41 | scan <- pathScan(fs.filePath(base)) 42 | } yield ReportFormat.largeFilesReport(scan, base.toString) 43 | 44 | 45 | def pathScan[R: _task: _appconfig](path: FilePath): Eff[R, PathScan] = path match { 46 | case f: File => 47 | for { 48 | fs <- FileSize.ofFile[R](f) 49 | } yield PathScan(SortedSet(fs), fs.size, 1) 50 | case dir: Directory => 51 | for { 52 | filesystem <- ask[R, Filesystem] 53 | topN <- takeTopN[R] 54 | childScans <- filesystem.listFiles(dir).traverse(pathScan(_)) 55 | } yield childScans.combineAll(topN) 56 | case Other(_) => 57 | PathScan.empty.pureEff[R] 58 | } 59 | 60 | 61 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 62 | scanConfig <- ask[R, ScanConfig] 63 | } yield new Monoid[PathScan] { 64 | def empty: PathScan = PathScan.empty 65 | 66 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 67 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 68 | p1.totalSize + p2.totalSize, 69 | p1.totalCount + p2.totalCount 70 | ) 71 | } 72 | } 73 | 74 | object EffOptics { 75 | 76 | // "If I have a Reader of S effect, and a Lens from S to T, then I have a Reader of T effect" 77 | implicit def readerLens[R, S, T](implicit m: MemberIn[Reader[S, ?], R], l: Lens[S, T]): MemberIn[Reader[T, ?], R] = 78 | m.transform(new (Reader[T, ?] ~> Reader[S, ?]) { 79 | def apply[X](f: Reader[T, X]) = ??? 80 | }) 81 | 82 | } 83 | 84 | trait Filesystem { 85 | 86 | def filePath(path: String): FilePath 87 | 88 | def length(file: File): Long 89 | 90 | def listFiles(directory: Directory): List[FilePath] 91 | 92 | } 93 | case object DefaultFilesystem extends Filesystem { 94 | 95 | def filePath(path: String): FilePath = 96 | if (Files.isRegularFile(Paths.get(path))) 97 | File(path.toString) 98 | else if (Files.isDirectory(Paths.get(path))) 99 | Directory(path) 100 | else 101 | Other(path) 102 | 103 | def length(file: File) = Files.size(Paths.get(file.path)) 104 | 105 | def listFiles(directory: Directory) = { 106 | val files = Files.list(Paths.get(directory.path)) 107 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 108 | case Directory(path) => List(Directory(path)) 109 | case File(path) => List(File(path)) 110 | case Other(path) => List.empty 111 | }) 112 | finally files.close() 113 | } 114 | 115 | } 116 | 117 | case class AppConfig(scanConfig: ScanConfig, filesystem: Filesystem) 118 | object AppConfig { 119 | 120 | implicit val _filesystem: Lens[AppConfig, Filesystem] = GenLens[AppConfig](_.filesystem) 121 | } 122 | 123 | case class ScanConfig(topN: Int) 124 | 125 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 126 | 127 | object PathScan { 128 | 129 | def empty = PathScan(SortedSet.empty, 0, 0) 130 | 131 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 132 | def empty: PathScan = PathScan.empty 133 | 134 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 135 | p1.largestFiles.union(p2.largestFiles).take(n), 136 | p1.totalSize + p2.totalSize, 137 | p1.totalCount + p2.totalCount 138 | ) 139 | } 140 | 141 | } 142 | 143 | case class FileSize(file: File, size: Long) 144 | 145 | object FileSize { 146 | 147 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 148 | fs <- ask[R, Filesystem] 149 | } yield FileSize(file, fs.length(file)) 150 | 151 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 152 | 153 | } 154 | 155 | object EffTypes { 156 | 157 | type _appconfig[R] = Reader[AppConfig, ?] |= R 158 | type _filesystem[R] = Reader[Filesystem, ?] |= R 159 | type _config[R] = Reader[ScanConfig, ?] |= R 160 | } 161 | 162 | 163 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 164 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 165 | //can overlap 166 | sealed trait FilePath { 167 | def path: String 168 | } 169 | 170 | case class File(path: String) extends FilePath 171 | case class Directory(path: String) extends FilePath 172 | case class Other(path: String) extends FilePath 173 | 174 | //Common pure code that is unaffected by the migration to Eff 175 | object ReportFormat { 176 | 177 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 178 | if (scan.largestFiles.nonEmpty) { 179 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 180 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 181 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 182 | } 183 | else 184 | s"No files found under path: $rootDir" 185 | } 186 | 187 | def formatByteString(bytes: Long): String = { 188 | if (bytes < 1000) 189 | s"${bytes} B" 190 | else { 191 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 192 | val pre = "KMGTPE".charAt(exp - 1) 193 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /exerciseOptics/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io._ 4 | import java.io._ 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | 11 | import org.atnos.eff._ 12 | import org.atnos.eff.all._ 13 | import org.atnos.eff.syntax.all._ 14 | 15 | import org.atnos.eff.addon.monix._ 16 | import org.atnos.eff.addon.monix.task._ 17 | import org.atnos.eff.syntax.addon.monix.task._ 18 | 19 | import org.specs2._ 20 | 21 | import scala.collection.immutable.SortedSet 22 | 23 | import scala.concurrent.duration._ 24 | 25 | import monix.execution.Scheduler.Implicits.global 26 | 27 | class ScannerSpec extends mutable.Specification { 28 | 29 | import EffOptics._ 30 | 31 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 32 | 33 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 34 | 35 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 36 | 37 | def filePath(path: String): FilePath = 38 | if (directories.keySet.contains(Directory(path))) 39 | Directory(path) 40 | else if (fileSizes.keySet.contains(File(path))) 41 | File(path) 42 | else 43 | throw new FileNotFoundException(path) 44 | } 45 | 46 | "file scan" ! { 47 | val base = Directory("base") 48 | val base1 = File(s"${base.path}/1.txt") 49 | val base2 = File(s"${base.path}/2.txt") 50 | val subdir = Directory(s"${base.path}/subdir") 51 | val sub1 = File(s"${subdir.path}/1.txt") 52 | val sub3 = File(s"${subdir.path}/3.txt") 53 | val fs: Filesystem = MockFilesystem( 54 | Map( 55 | base -> List(subdir, base1, base2), 56 | subdir -> List(sub1, sub3) 57 | ), 58 | Map(base1 -> 1, base2 -> 2, sub1 -> 1, sub3 -> 3) 59 | ) 60 | 61 | val program = Scanner.pathScan[Scanner.R](base) 62 | val actual = program.runReader(AppConfig(ScanConfig(2), fs)).runAsync.runSyncUnsafe(3.seconds) 63 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 64 | 65 | actual.mustEqual(expected) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /exerciseReader/README.md: -------------------------------------------------------------------------------- 1 | # File Scanner with Reader configuration injection 2 | 3 | Hopefully the previous exercises demonstrated that there are downsides and costs to direct coupling against external APIs like a filesystem. 4 | 5 | ## Two Implementations of an Interface: Production vs Test 6 | 7 | One of the classic ways to decouple from an external API is to abstract it into an interface implemented by the API. 8 | In "production" the real API is used while during testing a mock version is substituted. 9 | 10 | This version of the scanner uses this strategy, creating a `Filesystem` trait that defines the operations 11 | the scanner needs from a real filesystem: 12 | 13 | - classifying a file path as a file or a directory 14 | - listing directory contents 15 | - querying the length of files 16 | 17 | ## Using Reader effects to inject dependencies 18 | 19 | The scanner then needs to be parameterized on a `Filesystem` implementation, which requires passing it to all the code sites where 20 | filesystem operations are used. The [Reader monad is an elegant way to inject dependencies into 21 | code](http://functionaltalks.org/2013/06/17/runar-oli-bjarnason-dead-simple-dependency-injection/) and well supported by 22 | Eff framework. 23 | 24 | In fact, the scanner depends upon two external dependencies; not the filesystem but also *topN*, the number of largest files 25 | to report on. So two distinct reader "effects" are combined. 26 | 27 | ## Using Eff to stack Reader and Task effects 28 | 29 | We'll need to modify the Task-based version of the program to also support these new Reader effect. Eff lets us stack multiple effects 30 | in a single monad type. 31 | 32 | ## Tasks 33 | 34 | ### :mag: _Study Code_ Declaring Effects 35 | 36 | This version introduces the [Eff](https://github.com/atnos-org/eff) framework. 37 | Examine the changes to `PathScan.scan` and identify: 38 | 39 | - The return type is now an *Eff expression* `Eff[R, PathScan]`. This is read as *a program that when run has effects described 40 | by the effect set `R` and yields a `PathScan` result*. 41 | 42 | - The effect set `R` passed a type parameter. 43 | 44 | - The *member typeclasses* `_filesystem`, `_config` and `_task` that denote the effects that `R` must include for this function to 45 | operate (`R` can also include other effects not used in this function). 46 | 47 | - The `Task` type used in the previous task exercise has been replaced by a *task effect*, which can be combined with 48 | reader effects despite the two have different behaviors. The `_task` member tells us that the task effect is present. 49 | 50 | - The `for {..} yield` expression in the function body. Eff programs are built by flatMapping over a sequence of sub-steps. 51 | Note the `ask` step that yields `fs` (the current `Filesystem`). Where does the `ask` method come from? 52 | 53 | ### :mag: _Study Code_ Interpreting Effects 54 | 55 | Examine `main`. Note how we build the Eff program first, then *interpret* (run) it. To run the program, both 56 | `Reader` effects need their dependency provided. Once these effects are resolved, the call to `runAsync` "gets rid" of the 57 | `Eff` wrapper and leaves us with a `Task`, which we can run using `runSyncUnsafe(1.minute)` just as in the task exercise.. 58 | 59 | - What happens if you re-order the two calls to `runReader`? 60 | - What happens if you remove one call to `runReader`? 61 | - Can you reorder the call to `runAsync` relative to the `runReader` calls? 62 | 63 | ### :pencil: _Write Code_ 64 | 65 | Extend the use of the Reader effect to `takeTopN` and `FileSize.ofFile`. Both of these methods should be converted 66 | to: 67 | 68 | - Accept a type parameter `R` and one of the member typeclasses (`_filesystem` and `_config`) denoting the dependency they 69 | need. 70 | 71 | - Return their result in an Eff expression 72 | 73 | - Use a for-expression internally to `ask` for their dependency, and then `yield` their result. 74 | 75 | If you've made the changes correctly, there shouldn't be any manual passing of the `Filesystem` or `ScanConfig` parameters. 76 | 77 | 78 | ### :arrow_forward: _Run Code_ 79 | 80 | Run the test to verify your changes are working correctly 81 | 82 | ### :mag: _Study Code_ Easy testing 83 | 84 | Note how the [ScannerSpec](src/test/scala/scan/ScannerSpec.scala) tests most of the program's logic using Plain Old Scala Objects 85 | and without doing IO 86 | 87 | ### :mag: :question: _Optional Study_ 88 | 89 | Examine the `DefaultFilesystem.listFiles` method and note the try/finally construct. The reason for this is 90 | that `listFiles` returns a `Stream` of the directory contents, which holds open a file handle until the stream is cleaned up. 91 | 92 | -------------------------------------------------------------------------------- /exerciseReader/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import org.atnos.eff._ 13 | import org.atnos.eff.all._ 14 | import org.atnos.eff.syntax.all._ 15 | 16 | import org.atnos.eff.addon.monix._ 17 | import org.atnos.eff.addon.monix.task._ 18 | import org.atnos.eff.syntax.addon.monix.task._ 19 | 20 | import monix.eval._ 21 | import monix.execution._ 22 | 23 | import EffTypes._ 24 | 25 | import scala.concurrent.duration._ 26 | 27 | 28 | object Scanner { 29 | 30 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?]] 31 | 32 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 33 | 34 | def main(args: Array[String]): Unit = { 35 | val program = scanReport[R](args(0)).map(println) 36 | 37 | program.runReader(ScanConfig(10)).runReader(DefaultFilesystem: Filesystem).runAsync.runSyncUnsafe(1.minute) 38 | } 39 | 40 | def scanReport[R: _task: _filesystem: _config](base: String): Eff[R, String] = for { 41 | fs <- ask[R, Filesystem] 42 | scan <- pathScan(fs.filePath(base)) 43 | } yield ReportFormat.largeFilesReport(scan, base.toString) 44 | 45 | def pathScan[R: _task: _filesystem: _config](path: FilePath): Eff[R, PathScan] = path match { 46 | case f: File => 47 | for { 48 | fs <- ask[R, Filesystem] 49 | filesize = FileSize.ofFile(f, fs) 50 | } yield PathScan(SortedSet(filesize), filesize.size, 1) 51 | case dir: Directory => 52 | for { 53 | config <- ask[R, ScanConfig] 54 | topN = takeTopN(config) 55 | filesystem <- ask[R, Filesystem] 56 | childScans <- filesystem.listFiles(dir).traverse(pathScan[R](_)) 57 | } yield childScans.combineAll(topN) 58 | case Other(_) => 59 | PathScan.empty.pureEff[R] 60 | } 61 | 62 | 63 | def takeTopN(config: ScanConfig): Monoid[PathScan] = 64 | new Monoid[PathScan] { 65 | def empty: PathScan = PathScan.empty 66 | 67 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 68 | p1.largestFiles.union(p2.largestFiles).take(config.topN), 69 | p1.totalSize + p2.totalSize, 70 | p1.totalCount + p2.totalCount 71 | ) 72 | } 73 | 74 | } 75 | 76 | trait Filesystem { 77 | 78 | def filePath(path: String): FilePath 79 | 80 | def length(file: File): Long 81 | 82 | def listFiles(directory: Directory): List[FilePath] 83 | 84 | } 85 | case object DefaultFilesystem extends Filesystem { 86 | 87 | def filePath(path: String): FilePath = 88 | if (Files.isRegularFile(Paths.get(path))) 89 | File(path.toString) 90 | else if (Files.isDirectory(Paths.get(path))) 91 | Directory(path) 92 | else 93 | Other(path) 94 | 95 | def length(file: File) = Files.size(Paths.get(file.path)) 96 | 97 | def listFiles(directory: Directory) = { 98 | val files = Files.list(Paths.get(directory.path)) 99 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 100 | case Directory(path) => List(Directory(path)) 101 | case File(path) => List(File(path)) 102 | case Other(path) => List.empty 103 | }) 104 | finally files.close() 105 | } 106 | 107 | } 108 | 109 | case class ScanConfig(topN: Int) 110 | 111 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 112 | 113 | object PathScan { 114 | 115 | def empty = PathScan(SortedSet.empty, 0, 0) 116 | 117 | } 118 | 119 | case class FileSize(file: File, size: Long) 120 | 121 | object FileSize { 122 | 123 | def ofFile(file: File, fs: Filesystem) = FileSize(file, fs.length(file)) 124 | 125 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 126 | 127 | } 128 | 129 | object EffTypes { 130 | 131 | type _filesystem[R] = Reader[Filesystem, ?] <= R 132 | type _config[R] = Reader[ScanConfig, ?] <= R 133 | } 134 | 135 | 136 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 137 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 138 | //can overlap 139 | sealed trait FilePath { 140 | def path: String 141 | } 142 | 143 | case class File(path: String) extends FilePath 144 | case class Directory(path: String) extends FilePath 145 | case class Other(path: String) extends FilePath 146 | 147 | //Common pure code that is unaffected by the migration to Eff 148 | object ReportFormat { 149 | 150 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 151 | if (scan.largestFiles.nonEmpty) { 152 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 153 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 154 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 155 | } 156 | else 157 | s"No files found under path: $rootDir" 158 | } 159 | 160 | def formatByteString(bytes: Long): String = { 161 | if (bytes < 1000) 162 | s"${bytes} B" 163 | else { 164 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 165 | val pre = "KMGTPE".charAt(exp - 1) 166 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /exerciseReader/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io._ 4 | import java.io._ 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | 11 | import org.atnos.eff._ 12 | import org.atnos.eff.all._ 13 | import org.atnos.eff.syntax.all._ 14 | 15 | import org.atnos.eff.addon.monix._ 16 | import org.atnos.eff.addon.monix.task._ 17 | import org.atnos.eff.syntax.addon.monix.task._ 18 | 19 | import org.specs2._ 20 | 21 | import scala.collection.immutable.SortedSet 22 | 23 | import scala.concurrent.duration._ 24 | 25 | import monix.execution.Scheduler.Implicits.global 26 | 27 | class ScannerSpec extends mutable.Specification { 28 | 29 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 30 | 31 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 32 | 33 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 34 | 35 | def filePath(path: String): FilePath = 36 | if (directories.keySet.contains(Directory(path))) 37 | Directory(path) 38 | else if (fileSizes.keySet.contains(File(path))) 39 | File(path) 40 | else 41 | throw new FileNotFoundException(path) 42 | } 43 | 44 | "file scan" ! { 45 | val base = Directory("base") 46 | val base1 = File(s"${base.path}/1.txt") 47 | val base2 = File(s"${base.path}/2.txt") 48 | val subdir = Directory(s"${base.path}/subdir") 49 | val sub1 = File(s"${subdir.path}/1.txt") 50 | val sub3 = File(s"${subdir.path}/3.txt") 51 | val fs: Filesystem = MockFilesystem( 52 | Map( 53 | base -> List(subdir, base1, base2), 54 | subdir -> List(sub1, sub3) 55 | ), 56 | Map(base1 -> 1, base2 -> 2, sub1 -> 1, sub3 -> 3) 57 | ) 58 | 59 | val program = Scanner.pathScan[Scanner.R](base) 60 | val actual = program.runReader(ScanConfig(2)).runReader(fs).runAsync.runSyncUnsafe(3.seconds) 61 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 62 | 63 | actual.mustEqual(expected) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /exerciseState/README.md: -------------------------------------------------------------------------------- 1 | # Stafeful computation with the State effect 2 | 3 | The `State[S, ?]` models a stateful computation that receives a current state `S`, and emits a new state `S` along with 4 | the payload. 5 | 6 | We will use `State` to try to account for the effect of symlinks in our file traversal. If we traverse a directory tree 7 | that contains multiple symlinks to the same target file, we won't count the size of the target file more than once. 8 | 9 | To do this we'll need to keep track if what files we've seen before. This is what requires the use of a State effect. 10 | In this case, the state we'll be tracking (ie type `S`) will be `Set[FilePath]` 11 | 12 | 13 | ## Tasks 14 | 15 | ### :mag: _Study Code_ 16 | 17 | - Note that `FilePath` now includes a new case `Symlink(path: String, linkTo: FilePath)` and the `DefaultFilesystem` uses the 18 | File API to distinguish symlinks. Previously they were automatically followed. 19 | 20 | 21 | ### :pencil: _Write Code_ 22 | 23 | - Define an alias `_sym[R]` in `EffTypes` to indicate that `State[Set[FilePath], ?]` is a member of effect stack `R` 24 | 25 | - Add the member constraint to `pathScan` and `scanReport` 26 | 27 | - We need to interpret the state effect in `main`. We'll use the 28 | [`evalStateZero[Set[FilePath]]`](https://github.com/atnos-org/eff/blob/4d289be/shared/src/main/scala/org/atnos/eff/syntax/state.scala#L28) 29 | combinator. Eval here means 30 | that the final state isn't returned, just the computed payload. The zero suffix indicates that the zero value of a 31 | `Monoid[Set[FilePath]]` in scope should be used as the initial state. That's coming in from `imports cats.implicits._`. 32 | The zero value of a set is `Set.empty`. 33 | 34 | - In `pathScan`, you'll should add a case to handle `Symlink`. To read/write the current state use the `get`/`put` combinators, 35 | and/or `modify(f: S => S)` to run a state modification function. Your logic should check if the target of a symlink has been 36 | visited; if it has, then return an empty pathscan, while if it hasn't, add it to the visited set and invoke `pathScan` on the target. 37 | 38 | This is a tricky task. Remember to look at the [solution](../solutionExerciseState/src/main/scala/scan/Scanner.scala) 39 | for guidance if you get stuck. 40 | 41 | ### :arrow_forward: _Run Code_ 42 | 43 | - Run the test to check that symlinks are being handled. 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /exerciseState/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx5[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?], State[Set[FilePath], ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).evalStateZero[Set[FilePath]].runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask 57 | 58 | start <- taskDelay(System.currentTimeMillis()) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 61 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 62 | 63 | finish <- taskDelay(System.currentTimeMillis()) 64 | 65 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 66 | 67 | } yield ReportFormat.largeFilesReport(scan, base.toString) 68 | 69 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 70 | 71 | case f: File => 72 | for { 73 | fs <- FileSize.ofFile(f) 74 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 75 | } yield PathScan(SortedSet(fs), fs.size, 1) 76 | 77 | case dir: Directory => 78 | for { 79 | filesystem <- ask[R, Filesystem] 80 | topN <- takeTopN 81 | fileList <- taskDelay(filesystem.listFiles(dir)) 82 | childScans <- fileList.traverse(pathScan(_)) 83 | _ <- { 84 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 85 | val fileCount = fileList.count(_.isInstanceOf[File]) 86 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 87 | } 88 | } yield childScans.combineAll(topN) 89 | 90 | case Other(_) => 91 | PathScan.empty.pureEff 92 | } 93 | 94 | 95 | 96 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 97 | scanConfig <- ask 98 | } yield new Monoid[PathScan] { 99 | def empty: PathScan = PathScan.empty 100 | 101 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 102 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 103 | p1.totalSize + p2.totalSize, 104 | p1.totalCount + p2.totalCount 105 | ) 106 | } 107 | 108 | } 109 | 110 | trait Filesystem { 111 | 112 | def filePath(path: String): FilePath 113 | 114 | def length(file: File): Long 115 | 116 | def listFiles(directory: Directory): List[FilePath] 117 | 118 | } 119 | case object DefaultFilesystem extends Filesystem { 120 | 121 | def filePath(pathStr: String): FilePath = { 122 | val path = Paths.get(pathStr) 123 | if (Files.isSymbolicLink(path)) 124 | Symlink(pathStr, filePath(Files.readSymbolicLink(path).toString)) 125 | else if (Files.isRegularFile(path)) 126 | File(path.toString) 127 | else if (Files.isDirectory(path)) 128 | Directory(pathStr) 129 | else 130 | Other(pathStr) 131 | } 132 | 133 | def length(file: File) = Files.size(Paths.get(file.path)) 134 | 135 | def listFiles(directory: Directory) = { 136 | val files = Files.list(Paths.get(directory.path)) 137 | try files.toScala[List].flatMap(path => listFilePath(path.toString)) 138 | finally files.close() 139 | } 140 | 141 | private def listFilePath(path: String): List[FilePath] = { 142 | filePath(path) match { 143 | case Directory(path) => List(Directory(path)) 144 | case File(path) => List(File(path)) 145 | case Symlink(path, to) => listFilePath(to.path) 146 | case Other(path) => List.empty 147 | } 148 | } 149 | } 150 | 151 | case class ScanConfig(topN: Int) 152 | 153 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 154 | 155 | object PathScan { 156 | 157 | def empty = PathScan(SortedSet.empty, 0, 0) 158 | 159 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 160 | def empty: PathScan = PathScan.empty 161 | 162 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 163 | p1.largestFiles.union(p2.largestFiles).take(n), 164 | p1.totalSize + p2.totalSize, 165 | p1.totalCount + p2.totalCount 166 | ) 167 | } 168 | 169 | } 170 | 171 | case class FileSize(file: File, size: Long) 172 | 173 | object FileSize { 174 | 175 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 176 | fs <- ask 177 | } yield FileSize(file, fs.length(file)) 178 | 179 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 180 | 181 | } 182 | 183 | object EffTypes { 184 | 185 | type _filesystem[R] = Reader[Filesystem, ?] <= R 186 | type _config[R] = Reader[ScanConfig, ?] <= R 187 | type _err[R] = Either[String, ?] <= R 188 | type _log[R] = Writer[Log, ?] <= R 189 | } 190 | 191 | sealed trait Log {def msg: String} 192 | object Log { 193 | def error: String => Log = Error(_) 194 | def info: String => Log = Info(_) 195 | def debug: String => Log = Debug(_) 196 | } 197 | case class Error(msg: String) extends Log 198 | case class Info(msg: String) extends Log 199 | case class Debug(msg: String) extends Log 200 | 201 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 202 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 203 | //can overlap 204 | sealed trait FilePath { 205 | def path: String 206 | } 207 | 208 | case class File(path: String) extends FilePath 209 | case class Directory(path: String) extends FilePath 210 | case class Symlink(path: String, linkTo: FilePath) extends FilePath 211 | case class Other(path: String) extends FilePath 212 | 213 | //Common pure code that is unaffected by the migration to Eff 214 | object ReportFormat { 215 | 216 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 217 | if (scan.largestFiles.nonEmpty) { 218 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 219 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 220 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 221 | } 222 | else 223 | s"No files found under path: $rootDir" 224 | } 225 | 226 | def formatByteString(bytes: Long): String = { 227 | if (bytes < 1000) 228 | s"${bytes} B" 229 | else { 230 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 231 | val pre = "KMGTPE".charAt(exp - 1) 232 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 233 | } 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /exerciseState/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val linkTarget = File(s"/somewhere/else/7.txt") 42 | val base1 = File(s"${base.path}/1.txt") 43 | val baseLink = Symlink(s"${base.path}/7.txt", linkTarget) 44 | val subdir = Directory(s"${base.path}/subdir") 45 | val sub2 = File(s"${subdir.path}/2.txt") 46 | val subLink = Symlink(s"${subdir.path}/7.txt", linkTarget) 47 | val directories = Map( 48 | base -> List(subdir, base1, baseLink), 49 | subdir -> List(sub2, subLink) 50 | ) 51 | val fileSizes = Map(base1 -> 1L, sub2 -> 2L, linkTarget -> 7L) 52 | val fs = MockFilesystem(directories, fileSizes) 53 | 54 | type R = Fx.fx5[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?], State[Set[FilePath], ?]] 55 | 56 | def run[T](program: Eff[R, T], fs: Filesystem) = 57 | program.runReader(ScanConfig(2)).runReader(fs).evalStateZero[Set[FilePath]].taskAttempt.runWriter[Log].runAsync.runSyncUnsafe(3.seconds) 58 | 59 | val expected = Right(new PathScan(SortedSet(FileSize(linkTarget, 7), FileSize(sub2, 2)), 10, 3)) 60 | 61 | val (actual, logs) = run(Scanner.pathScan[R](base), fs) 62 | 63 | "Report Format" ! {actual.mustEqual(expected)} 64 | 65 | } 66 | -------------------------------------------------------------------------------- /exerciseTask/README.md: -------------------------------------------------------------------------------- 1 | # Asynchronous Tasks 2 | 3 | 4 | A `Task[T]` represent a chunk of work that yields a value of type `T` on completion. 5 | 6 | Tasks allow fine control over what threads run the task and with what level of concurrency. They also allow the separation 7 | of the definition of work (when tasks are created and chained together) from its execution (when tasks are run and any 8 | side-effects occur). In this way, they play a similar role to the `IO` wrappers of Haskell, Cats-Effect or Scalaz. 9 | 10 | ## The Monix Task Library 11 | 12 | We will use [Monix 3.x Tasks](https://monix.io/docs/3x/eval/task.html). 13 | 14 | We choose Monix over [Cats Effect IO](https://typelevel.org/cats-effect/datatypes/io.html) because Monix offers a better 15 | default [Execution Model](https://monix.io/docs/3x/execution/scheduler.html#execution-model) for Tasks and more options 16 | than Cats-Effect does, and is relatively similar in other aspects. 17 | 18 | We choose Monix tasks over Scala's default Futures because of Monix's 19 | [Execution Model](https://monix.io/docs/3x/execution/scheduler.html#execution-model) has more control and much more performant 20 | defaults than Scala Futures. 21 | 22 | ## Tasks 23 | 24 | ### :mag: _Study Code_ 25 | 26 | - In the classic version, method `scanReport` returned `String`. What is its return type now? What about method `pathScan`? 27 | 28 | ### :pencil: _Write Code_ 29 | 30 | - The top level `main` still returns `Unit`. It builds a program, a chain of `Task`s, but they'll have no effect until they're run. 31 | Use `runSyncUnsafe(1.minute)` to run the tasks. The word Unsafe in its name indicates that it will cause side-effects. 32 | The idea with Tasks is to run them at only one at the top of your top program. 33 | 34 | - There is a compile error in `pathScan`. The `map` method no longer has the correct type, because we want to traverse a list 35 | of subdirectories, converting each into a `Task` yielding their scan, and then roll the traversal into a single overall Task. 36 | Replace `map` with `traverse`, which is "like map, but for mapping to effectful values" 37 | 38 | - One more will appear problem. Tasks can be run asynchronously, so they need an implicit `Scheduler` available to dispatch them to. 39 | Add `implicit val s = Scheduler(ExecutionModel.BatchedExecution(32))` at the top of `Scanner` object. The 40 | `BatchedExecution(32)` means that a chain of up to 32 Tasks will be run batched together in one thread before any context switch. 41 | 42 | 43 | ### :arrow_forward: _Run Code_ 44 | 45 | Run the tests to verify your task based implementation still gives the correct output. 46 | 47 | ### :mag: _Study Code_ 48 | 49 | - Examine the tests for the Task version. Notice that the introduction of Task, which makes this version *purely functional* 50 | hasn't made the tests any easier to write. Still lots of messy files and directories created and cleaned-up. Why is this 51 | and what could you do about it? 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /exerciseTask/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.implicits._ 10 | 11 | import monix.eval._ 12 | import monix.execution._ 13 | 14 | import scala.concurrent.duration._ 15 | 16 | 17 | object Scanner { 18 | 19 | def main(args: Array[String]): Unit = { 20 | val program = scanReport(Paths.get(args(0)), 10).map(println) 21 | } 22 | 23 | def scanReport(base: Path, topN: Int): Task[String] = for { 24 | scan <- pathScan(FilePath(base), topN) 25 | } yield ReportFormat.largeFilesReport(scan, base.toString) 26 | 27 | def pathScan(filePath: FilePath, topN: Int): Task[PathScan] = filePath match { 28 | case File(path) => 29 | Task { 30 | val fs = FileSize.ofFile(Paths.get(path)) 31 | PathScan(SortedSet(fs), fs.size, 1) 32 | } 33 | case Directory(path) => 34 | for { 35 | files <- Task { 36 | val jstream = Files.list(Paths.get(path)) 37 | try jstream.toScala[List] 38 | finally jstream.close() 39 | } 40 | scans <- files.map(subpath => pathScan(FilePath(subpath), topN)) 41 | } yield scans.combineAll(PathScan.topNMonoid(topN)) 42 | case Other(_) => 43 | Task(PathScan.empty) 44 | } 45 | 46 | } 47 | 48 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 49 | 50 | object PathScan { 51 | 52 | def empty = PathScan(SortedSet.empty, 0, 0) 53 | 54 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 55 | def empty: PathScan = PathScan.empty 56 | 57 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 58 | p1.largestFiles.union(p2.largestFiles).take(n), 59 | p1.totalSize + p2.totalSize, 60 | p1.totalCount + p2.totalCount 61 | ) 62 | } 63 | 64 | } 65 | 66 | case class FileSize(path: Path, size: Long) 67 | 68 | object FileSize { 69 | 70 | def ofFile(file: Path) = { 71 | FileSize(file, Files.size(file)) 72 | } 73 | 74 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long ](_.size).reverse 75 | 76 | } 77 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 78 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 79 | //can overlap 80 | sealed trait FilePath { 81 | def path: String 82 | } 83 | object FilePath { 84 | 85 | def apply(path: Path): FilePath = 86 | if (Files.isRegularFile(path)) 87 | File(path.toString) 88 | else if (Files.isDirectory(path)) 89 | Directory(path.toString) 90 | else 91 | Other(path.toString) 92 | } 93 | case class File(path: String) extends FilePath 94 | case class Directory(path: String) extends FilePath 95 | case class Other(path: String) extends FilePath 96 | 97 | //Common pure code that is unaffected by the migration to Eff 98 | object ReportFormat { 99 | 100 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 101 | if (scan.largestFiles.nonEmpty) { 102 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 103 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.path}").mkString("", "\n", "\n") + 104 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 105 | } 106 | else 107 | s"No files found under path: $rootDir" 108 | } 109 | 110 | def formatByteString(bytes: Long): String = { 111 | if (bytes < 1000) 112 | s"${bytes} B" 113 | else { 114 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 115 | val pre = "KMGTPE".charAt(exp - 1) 116 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 117 | } 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /exerciseTask/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.PrintWriter 4 | import java.nio.file._ 5 | 6 | import org.specs2._ 7 | 8 | import scala.collection.immutable.SortedSet 9 | 10 | import scala.concurrent.duration._ 11 | 12 | import monix.execution.Scheduler.Implicits.global 13 | 14 | class ScannerSpec extends mutable.Specification { 15 | 16 | "Report Format" ! { 17 | val base = deletedOnExit(Files.createTempDirectory("exerciseTask")) 18 | val base1 = deletedOnExit(fillFile(base, 1)) 19 | val base2 = deletedOnExit(fillFile(base, 2)) 20 | val subdir = deletedOnExit(Files.createTempDirectory(base, "subdir")) 21 | val sub1 = deletedOnExit(fillFile(subdir, 1)) 22 | val sub3 = deletedOnExit(fillFile(subdir, 3)) 23 | 24 | val actual = Scanner.pathScan(FilePath(base), 2).runSyncUnsafe(3.seconds) 25 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 26 | 27 | actual.mustEqual(expected) 28 | } 29 | 30 | def fillFile(dir: Path, size: Int) = { 31 | val path = dir.resolve(s"$size.txt") 32 | val w = new PrintWriter(path.toFile) 33 | try w.write("a" * size) 34 | finally w.close 35 | path 36 | } 37 | 38 | def deletedOnExit(p: Path) = { 39 | p.toFile.deleteOnExit() 40 | p 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /exerciseWriter/README.md: -------------------------------------------------------------------------------- 1 | # Logging with the Writer effect 2 | 3 | The `Writer[L, ?]` effect lets the computation emit additional values of type `L` as a side-effect of computation. It is commonly 4 | used to functional logging, but you could also view the log value as an appendix or supplimentary information about the 5 | computation. 6 | 7 | This exercise is a use-case of functional logging and how "logs-as-values" lets us easily write unit tests around the log output. 8 | 9 | 10 | ## Tasks 11 | 12 | ### :mag: _Study Code_ 13 | 14 | - The effect stack `R` includes `Writer[Log]`, where `Log` is a sealed hierarchy of Log events at different levels. 15 | 16 | - In `PathScan.scan`, the `tell` operator is used to emit `Log` values. Because `tell` just emits a log, it has type 17 | `Eff[R, Unit]`. So an underscore is used on the lefthand-side of the for (eg `_ <- tell(x)`). 18 | 19 | - In `main`, the interpretation of the Eff program now includes a `runWriterUnsafe` step. This is one of several 20 | approaches to logging offerred by Eff, where we send a side-effecting handler function (eg `println`) 21 | into the interpreter, that logs each event as it is emitted, rather than returning an accumulation. It's not pure, 22 | but it has the advantage of not accumulating data in memory during execution. 23 | 24 | ### :arrow_forward: _Run Code_ 25 | 26 | Run the tests. They now verify not just the program output but the logs. They should fail because they expect log output 27 | for each file visited by the scan. 28 | 29 | ### :pencil: _Write Code_ 30 | 31 | Make the test pass by adding the appropriate `tell` statement to the `File` case in `pathScan`. 32 | 33 | ### :arrow_forward: _Run Code_ 34 | 35 | `run` the scanner on a real directory tree and check the logging works as expected. 36 | 37 | ### :pencil: _Write Code_ 38 | 39 | - Measure the time the overall scan took. Take start and end times in `scanReport`, and use a `tell` to log the elapsed 40 | millis. Ensure your calls to the clock are wrapped in `taskDelay` effects to ensure the clock timings are taken when the program *runs*, and not when it is *created*. 41 | 42 | - Try changing the interpretation in main to use the plain `runWriter`. What does the program return now? How can you 43 | print the logs? 44 | 45 | 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /exerciseWriter/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask[R, Filesystem] 57 | 58 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 59 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 60 | 61 | } yield ReportFormat.largeFilesReport(scan, base.toString) 62 | 63 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 64 | 65 | case f: File => 66 | for { 67 | fs <- FileSize.ofFile(f) 68 | } yield PathScan(SortedSet(fs), fs.size, 1) 69 | 70 | case dir: Directory => 71 | for { 72 | filesystem <- ask[R, Filesystem] 73 | topN <- takeTopN 74 | fileList <- taskDelay(filesystem.listFiles(dir)) 75 | childScans <- fileList.traverse(pathScan[R](_)) 76 | _ <- { 77 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 78 | val fileCount = fileList.count(_.isInstanceOf[File]) 79 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 80 | } 81 | } yield childScans.combineAll(topN) 82 | 83 | case Other(_) => 84 | PathScan.empty.pureEff[R] 85 | } 86 | 87 | 88 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 89 | scanConfig <- ask 90 | } yield new Monoid[PathScan] { 91 | def empty: PathScan = PathScan.empty 92 | 93 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 94 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 95 | p1.totalSize + p2.totalSize, 96 | p1.totalCount + p2.totalCount 97 | ) 98 | } 99 | 100 | } 101 | 102 | trait Filesystem { 103 | 104 | def filePath(path: String): FilePath 105 | 106 | def length(file: File): Long 107 | 108 | def listFiles(directory: Directory): List[FilePath] 109 | 110 | } 111 | case object DefaultFilesystem extends Filesystem { 112 | 113 | def filePath(path: String): FilePath = 114 | if (Files.isRegularFile(Paths.get(path))) 115 | File(path.toString) 116 | else if (Files.isDirectory(Paths.get(path))) 117 | Directory(path) 118 | else 119 | Other(path) 120 | 121 | def length(file: File) = Files.size(Paths.get(file.path)) 122 | 123 | def listFiles(directory: Directory) = { 124 | val files = Files.list(Paths.get(directory.path)) 125 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 126 | case Directory(path) => List(Directory(path)) 127 | case File(path) => List(File(path)) 128 | case Other(path) => List.empty 129 | }) 130 | finally files.close() 131 | } 132 | 133 | } 134 | 135 | case class ScanConfig(topN: Int) 136 | 137 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 138 | 139 | object PathScan { 140 | 141 | def empty = PathScan(SortedSet.empty, 0, 0) 142 | 143 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 144 | def empty: PathScan = PathScan.empty 145 | 146 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 147 | p1.largestFiles.union(p2.largestFiles).take(n), 148 | p1.totalSize + p2.totalSize, 149 | p1.totalCount + p2.totalCount 150 | ) 151 | } 152 | 153 | } 154 | 155 | case class FileSize(file: File, size: Long) 156 | 157 | object FileSize { 158 | 159 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 160 | fs <- ask 161 | } yield FileSize(file, fs.length(file)) 162 | 163 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 164 | 165 | } 166 | 167 | object EffTypes { 168 | 169 | type _filesystem[R] = Reader[Filesystem, ?] <= R 170 | type _config[R] = Reader[ScanConfig, ?] <= R 171 | type _err[R] = Either[String, ?] <= R 172 | type _log[R] = Writer[Log, ?] <= R 173 | } 174 | 175 | sealed trait Log {def msg: String} 176 | object Log { 177 | def error: String => Log = Error(_) 178 | def info: String => Log = Info(_) 179 | def debug: String => Log = Debug(_) 180 | } 181 | case class Error(msg: String) extends Log 182 | case class Info(msg: String) extends Log 183 | case class Debug(msg: String) extends Log 184 | 185 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 186 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 187 | //can overlap 188 | sealed trait FilePath { 189 | def path: String 190 | } 191 | 192 | case class File(path: String) extends FilePath 193 | case class Directory(path: String) extends FilePath 194 | case class Other(path: String) extends FilePath 195 | 196 | //Common pure code that is unaffected by the migration to Eff 197 | object ReportFormat { 198 | 199 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 200 | if (scan.largestFiles.nonEmpty) { 201 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 202 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 203 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 204 | } 205 | else 206 | s"No files found under path: $rootDir" 207 | } 208 | 209 | def formatByteString(bytes: Long): String = { 210 | if (bytes < 1000) 211 | s"${bytes} B" 212 | else { 213 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 214 | val pre = "KMGTPE".charAt(exp - 1) 215 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 216 | } 217 | } 218 | } 219 | -------------------------------------------------------------------------------- /exerciseWriter/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base), fs) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | expectedLogs.forall(logs.contains) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.1.1 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/benhutchison/GettingWorkDoneWithExtensibleEffects/7756d435f2e902869ddf9eadd1f7f0f292758cb1/project/plugins.sbt -------------------------------------------------------------------------------- /solutions/exercise2io/README.md: -------------------------------------------------------------------------------- 1 | # Exercise 2: Separating Declaration and Execution with the IO effect 2 | 3 | In the first step, we will transform our scanner into a purely functional program by introducing `IO` as our first Eff effect. 4 | IO "suspends" effectful actions like reading the filesystem, so that they aren't run when they're declared - while letting 5 | us talk about and depend upon the results of such actions during program construction. 6 | 7 | We're going to use the IO effect from [cats.effect](https://typelevel.org/cats-effect/). While it can be used standalone, 8 | we'll bundle it inside an Eff container to make it easy to add additional effects in later exercises. 9 | 10 | ## Tasks 11 | 12 | ### :mag: _Study Code_ Declaring Effects 13 | 14 | Examine the changes to `PathScan.scan` in this version and identify: 15 | 16 | - The return type is now an *Eff expression* `Eff[R, PathScan]`. This is read as *a program that when run has effects described 17 | by the effect set `R` and yields a `PathScan` result*. 18 | 19 | - The effect set `R` passed a type parameter. 20 | 21 | - The *member typeclasses* `_filesystem` and `_config` that denote the effects that `R` must include for this function to 22 | operate (`R` can also include other effects not used in this function). 23 | 24 | - The `for {..} yield` expression in the function body. Eff programs are built by flatMapping over a sequence of sub-steps. 25 | Note the `ask` step that yields `fs` (the current `Filesystem`). Where does the `ask` method come from? 26 | 27 | ### :mag: _Study Code_ Interpreting Effects 28 | 29 | Examine `Scanner.pathScan`. Note how we build the Eff program first, then *interpret* (run) it. To run the program, both 30 | `Reader` effects need their dependency provided. Once these effects are resolved, the final call to `run` completes the 31 | program and yields the final result. 32 | 33 | - What happens if you re-order the two calls to `runReader`? 34 | - What happens if you remove one call to `runReader`? 35 | 36 | ### :pencil: _Write Code_ 37 | 38 | Extend the use of the Reader effect to `PathScan.takeTopN` and `FileSize.ofFile`. Both of these methods should be converted 39 | to: 40 | 41 | - Accept a type parameter `R` and one of the member typeclasses (`_filesystem` and `_config`) denoting the dependency they 42 | need. 43 | 44 | - Return their result in an Eff expression 45 | 46 | - Use a for-expression internally to `ask` for their dependency, and then `yield` their result. 47 | 48 | If you've made the changes correctly, there shouldn't be any manual passing of the `Filesystem` or `ScanaConfig` parameters. 49 | 50 | 51 | ### :arrow_forward: _Run Code_ 52 | 53 | Run the test to verify your changes are working correctly 54 | 55 | ### :mag: _Study Code_ Easy testing 56 | 57 | Note how the [ScannerSpec](src/test/scala/scan/ScannerSpec.scala) tests most of the program's logic using Plain Old Scala Objects 58 | and without doing IO 59 | 60 | ### :mag: :question: _Optional Study_ 61 | 62 | Examine the `DefaultFilesystem.listFiles` method and note the try/finally construct. The reason for this is 63 | that `listFiles` returns a `Stream` of the directory contents, which holds open a file handle until the stream is cleaned up. 64 | 65 | -------------------------------------------------------------------------------- /solutions/exercise2io/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | 9 | import cats._ 10 | import cats.data._ 11 | import cats.implicits._ 12 | 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import cats.effect._ 19 | 20 | import org.atnos.eff.addon.cats.effect.IOEffect._ 21 | import org.atnos.eff.syntax.addon.cats.effect._ 22 | 23 | object Scanner { 24 | 25 | type R = Fx1[IO] 26 | 27 | def main(args: Array[String]): Unit = run[R](args).unsafeRunSync 28 | 29 | def run[R: _io](args: Array[String]): Eff[R, Unit] = for { 30 | r <- scanReport(Paths.get(args(0)), 10) 31 | } yield println(r) 32 | 33 | def scanReport[R: _io](base: Path, topN: Int): Eff[R, String] = for { 34 | scan <- pathScan(base, topN) 35 | } yield ReportFormat.largeFilesReport(scan, base.toString) 36 | 37 | def pathScan[R: _io](path: Path, topN: Int): Eff[R, PathScan] = for { 38 | fp <- FilePath(path) 39 | 40 | scan <- fp match { 41 | 42 | case File(_) => for { 43 | fs <- FileSize.ofFile(path) 44 | } yield PathScan(SortedSet(fs), fs.size, 1) 45 | 46 | case Directory(_) => for { 47 | files <- ioDelay { 48 | val jstream = Files.list(path) 49 | try jstream.toScala[List] 50 | finally jstream.close() 51 | } 52 | subScans <- files.traverse(pathScan(_, topN)) 53 | } yield subScans.combineAll(PathScan.topNMonoid(topN)) 54 | 55 | case Other(_) => 56 | PathScan.empty.pureEff[R] 57 | } 58 | } yield scan 59 | 60 | } 61 | 62 | 63 | sealed trait FilePath { 64 | def path: String 65 | } 66 | object FilePath { 67 | 68 | def apply[R: _io](path: Path): Eff[R, FilePath] = ioDelay( 69 | if (Files.isRegularFile(path)) 70 | File(path.toString) 71 | else if (Files.isDirectory(path)) 72 | Directory(path.toString) 73 | else 74 | Other(path.toString) 75 | ) 76 | } 77 | case class File(path: String) extends FilePath 78 | case class Directory(path: String) extends FilePath 79 | case class Other(path: String) extends FilePath 80 | 81 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 82 | 83 | object PathScan { 84 | 85 | def empty = PathScan(SortedSet.empty, 0, 0) 86 | 87 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 88 | def empty: PathScan = PathScan.empty 89 | 90 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 91 | p1.largestFiles.union(p2.largestFiles).take(n), 92 | p1.totalSize + p2.totalSize, 93 | p1.totalCount + p2.totalCount 94 | ) 95 | } 96 | 97 | } 98 | 99 | case class FileSize(path: Path, size: Long) 100 | 101 | object FileSize { 102 | 103 | def ofFile[R: _io](file: Path) = ioDelay(FileSize(file, Files.size(file))) 104 | 105 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 106 | 107 | } 108 | object ReportFormat { 109 | 110 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 111 | if (scan.largestFiles.nonEmpty) { 112 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 113 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.path}").mkString("", "\n", "\n") + 114 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 115 | } 116 | else 117 | s"No files found under path: $rootDir" 118 | } 119 | 120 | def formatByteString(bytes: Long): String = { 121 | if (bytes < 1000) 122 | s"${bytes} B" 123 | else { 124 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 125 | val pre = "KMGTPE".charAt(exp - 1) 126 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /solutions/exercise2io/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.PrintWriter 4 | import java.nio.file._ 5 | 6 | import org.specs2._ 7 | 8 | import scala.collection.immutable.SortedSet 9 | 10 | import cats.effect._ 11 | 12 | import org.atnos.eff.addon.cats.effect.IOEffect._ 13 | import org.atnos.eff.syntax.addon.cats.effect._ 14 | 15 | class ScannerSpec extends mutable.Specification { 16 | 17 | "Report Format" ! { 18 | val base = deletedOnExit(Files.createTempDirectory("exercise1")) 19 | val base1 = deletedOnExit(fillFile(base, 1)) 20 | val base2 = deletedOnExit(fillFile(base, 2)) 21 | val subdir = deletedOnExit(Files.createTempDirectory(base, "subdir")) 22 | val sub1 = deletedOnExit(fillFile(subdir, 1)) 23 | val sub3 = deletedOnExit(fillFile(subdir, 3)) 24 | 25 | val scanProgram = Scanner.pathScan(base, 2) 26 | val actual = scanProgram.unsafeRunSync 27 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 28 | 29 | actual.mustEqual(expected) 30 | } 31 | 32 | def fillFile(dir: Path, size: Int) = { 33 | val path = dir.resolve(s"$size.txt") 34 | val w = new PrintWriter(path.toFile) 35 | try w.write("a" * size) 36 | finally w.close 37 | path 38 | } 39 | 40 | def deletedOnExit(p: Path) = { 41 | p.toFile.deleteOnExit() 42 | p 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /solutions/exerciseClassic/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.implicits._ 10 | 11 | 12 | object Scanner { 13 | 14 | def main(args: Array[String]): Unit = { 15 | println(scanReport(Paths.get(args(0)), 10)) 16 | } 17 | 18 | def scanReport(base: Path, topN: Int): String = { 19 | val scan = pathScan(FilePath(base), topN) 20 | 21 | ReportFormat.largeFilesReport(scan, base.toString) 22 | } 23 | 24 | def pathScan(filePath: FilePath, topN: Int): PathScan = filePath match { 25 | case File(path) => 26 | val fs = FileSize.ofFile(Paths.get(path)) 27 | PathScan(SortedSet(fs), fs.size, 1) 28 | case Directory(path) => 29 | val files = { 30 | val jstream = Files.list(Paths.get(path)) 31 | try jstream.toScala[List] 32 | finally jstream.close() 33 | } 34 | val subscans = files.map(subpath => pathScan(FilePath(subpath), topN)) 35 | subscans.combineAll(PathScan.topNMonoid(topN)) 36 | case Other(_) => 37 | PathScan.empty 38 | } 39 | 40 | } 41 | 42 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 43 | 44 | object PathScan { 45 | 46 | def empty = PathScan(SortedSet.empty, 0, 0) 47 | 48 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 49 | def empty: PathScan = PathScan.empty 50 | 51 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 52 | p1.largestFiles.union(p2.largestFiles).take(n), 53 | p1.totalSize + p2.totalSize, 54 | p1.totalCount + p2.totalCount 55 | ) 56 | } 57 | 58 | } 59 | 60 | case class FileSize(path: Path, size: Long) 61 | 62 | object FileSize { 63 | 64 | def ofFile(file: Path) = { 65 | FileSize(file, Files.size(file)) 66 | } 67 | 68 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long ](_.size).reverse 69 | 70 | } 71 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 72 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 73 | //can overlap 74 | sealed trait FilePath { 75 | def path: String 76 | } 77 | object FilePath { 78 | 79 | def apply(path: Path): FilePath = 80 | if (Files.isRegularFile(path)) 81 | File(path.toString) 82 | else if (Files.isDirectory(path)) 83 | Directory(path.toString) 84 | else 85 | Other(path.toString) 86 | } 87 | case class File(path: String) extends FilePath 88 | case class Directory(path: String) extends FilePath 89 | case class Other(path: String) extends FilePath 90 | 91 | 92 | //Common pure code that is unaffected by the migration to Eff 93 | object ReportFormat { 94 | 95 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 96 | if (scan.largestFiles.nonEmpty) { 97 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 98 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.path}").mkString("", "\n", "\n") + 99 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 100 | } 101 | else 102 | s"No files found under path: $rootDir" 103 | } 104 | 105 | def formatByteString(bytes: Long): String = { 106 | if (bytes < 1000) 107 | s"${bytes} B" 108 | else { 109 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 110 | val pre = "KMGTPE".charAt(exp - 1) 111 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 112 | } 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /solutions/exerciseClassic/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.PrintWriter 4 | import java.nio.file._ 5 | 6 | import org.specs2._ 7 | 8 | import scala.collection.immutable.SortedSet 9 | 10 | class ScannerSpec extends mutable.Specification { 11 | 12 | "Report Format" ! { 13 | val base = deletedOnExit(Files.createTempDirectory("exerciseClassic")) 14 | val base1 = deletedOnExit(fillFile(base, 1)) 15 | val base2 = deletedOnExit(fillFile(base, 2)) 16 | val subdir = deletedOnExit(Files.createTempDirectory(base, "subdir")) 17 | val sub1 = deletedOnExit(fillFile(subdir, 1)) 18 | val sub3 = deletedOnExit(fillFile(subdir, 3)) 19 | 20 | val actual = Scanner.pathScan(FilePath(base), 2) 21 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 22 | 23 | actual.mustEqual(expected) 24 | } 25 | 26 | def fillFile(dir: Path, size: Int) = { 27 | val path = dir.resolve(s"$size.txt") 28 | val w = new PrintWriter(path.toFile) 29 | try w.write("a" * size) 30 | finally w.close 31 | path 32 | } 33 | 34 | def deletedOnExit(p: Path) = { 35 | p.toFile.deleteOnExit() 36 | p 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /solutions/exerciseConcurrent/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask[R, Filesystem] 57 | 58 | start <- taskDelay(System.currentTimeMillis()) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 61 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 62 | 63 | finish <- taskDelay(System.currentTimeMillis()) 64 | 65 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 66 | 67 | } yield ReportFormat.largeFilesReport(scan, base.toString) 68 | 69 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 70 | 71 | case f: File => 72 | for { 73 | fs <- FileSize.ofFile(f) 74 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 75 | } yield PathScan(SortedSet(fs), fs.size, 1) 76 | 77 | case dir: Directory => 78 | for { 79 | filesystem <- ask[R, Filesystem] 80 | topN <- takeTopN 81 | fileList <- taskDelay(filesystem.listFiles(dir)) 82 | childScans <- fileList.traverseA(pathScan[R](_)) 83 | _ <- { 84 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 85 | val fileCount = fileList.count(_.isInstanceOf[File]) 86 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 87 | } 88 | } yield childScans.combineAll(topN) 89 | 90 | case Other(_) => 91 | PathScan.empty.pureEff[R] 92 | } 93 | 94 | 95 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 96 | scanConfig <- ask 97 | } yield new Monoid[PathScan] { 98 | def empty: PathScan = PathScan.empty 99 | 100 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 101 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 102 | p1.totalSize + p2.totalSize, 103 | p1.totalCount + p2.totalCount 104 | ) 105 | } 106 | 107 | } 108 | 109 | trait Filesystem { 110 | 111 | def filePath(path: String): FilePath 112 | 113 | def length(file: File): Long 114 | 115 | def listFiles(directory: Directory): List[FilePath] 116 | 117 | } 118 | case object DefaultFilesystem extends Filesystem { 119 | 120 | def filePath(path: String): FilePath = 121 | if (Files.isRegularFile(Paths.get(path))) 122 | File(path.toString) 123 | else if (Files.isDirectory(Paths.get(path))) 124 | Directory(path) 125 | else 126 | Other(path) 127 | 128 | def length(file: File) = Files.size(Paths.get(file.path)) 129 | 130 | def listFiles(directory: Directory) = { 131 | val files = Files.list(Paths.get(directory.path)) 132 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 133 | case Directory(path) => List(Directory(path)) 134 | case File(path) => List(File(path)) 135 | case Other(path) => List.empty 136 | }) 137 | finally files.close() 138 | } 139 | 140 | } 141 | 142 | case class ScanConfig(topN: Int) 143 | 144 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 145 | 146 | object PathScan { 147 | 148 | def empty = PathScan(SortedSet.empty, 0, 0) 149 | 150 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 151 | def empty: PathScan = PathScan.empty 152 | 153 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 154 | p1.largestFiles.union(p2.largestFiles).take(n), 155 | p1.totalSize + p2.totalSize, 156 | p1.totalCount + p2.totalCount 157 | ) 158 | } 159 | 160 | } 161 | 162 | case class FileSize(file: File, size: Long) 163 | 164 | object FileSize { 165 | 166 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 167 | fs <- ask 168 | } yield FileSize(file, fs.length(file)) 169 | 170 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 171 | 172 | } 173 | 174 | object EffTypes { 175 | 176 | type _filesystem[R] = Reader[Filesystem, ?] <= R 177 | type _config[R] = Reader[ScanConfig, ?] <= R 178 | type _err[R] = Either[String, ?] <= R 179 | type _log[R] = Writer[Log, ?] <= R 180 | } 181 | 182 | sealed trait Log {def msg: String} 183 | object Log { 184 | def error: String => Log = Error(_) 185 | def info: String => Log = Info(_) 186 | def debug: String => Log = Debug(_) 187 | } 188 | case class Error(msg: String) extends Log 189 | case class Info(msg: String) extends Log 190 | case class Debug(msg: String) extends Log 191 | 192 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 193 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 194 | //can overlap 195 | sealed trait FilePath { 196 | def path: String 197 | } 198 | 199 | case class File(path: String) extends FilePath 200 | case class Directory(path: String) extends FilePath 201 | case class Other(path: String) extends FilePath 202 | 203 | //Common pure code that is unaffected by the migration to Eff 204 | object ReportFormat { 205 | 206 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 207 | if (scan.largestFiles.nonEmpty) { 208 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 209 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 210 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 211 | } 212 | else 213 | s"No files found under path: $rootDir" 214 | } 215 | 216 | def formatByteString(bytes: Long): String = { 217 | if (bytes < 1000) 218 | s"${bytes} B" 219 | else { 220 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 221 | val pre = "KMGTPE".charAt(exp - 1) 222 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /solutions/exerciseConcurrent/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base), fs) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | logs.forall(expectedLogs.contains) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /solutions/exerciseCustom/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, FilesystemCmd, Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runFilesystemCmds(DefaultFilesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | start <- taskDelay(System.currentTimeMillis()) 57 | 58 | base <- FilesystemCmd.filePath(base) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]](base).runReader[ScanConfig](ScanConfig(topNValid)) 61 | 62 | finish <- taskDelay(System.currentTimeMillis()) 63 | 64 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 65 | 66 | } yield ReportFormat.largeFilesReport(scan, base.toString) 67 | 68 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 69 | 70 | case f: File => 71 | for { 72 | fs <- FileSize.ofFile(f) 73 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 74 | } yield PathScan(SortedSet(fs), fs.size, 1) 75 | 76 | case dir: Directory => 77 | for { 78 | topN <- takeTopN 79 | fileList <- FilesystemCmd.listFiles(dir) 80 | childScans <- fileList.traverse(pathScan[R](_)) 81 | _ <- { 82 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 83 | val fileCount = fileList.count(_.isInstanceOf[File]) 84 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 85 | } 86 | } yield childScans.combineAll(topN) 87 | 88 | case Other(_) => 89 | PathScan.empty.pureEff[R] 90 | } 91 | 92 | 93 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 94 | scanConfig <- ask 95 | } yield new Monoid[PathScan] { 96 | def empty: PathScan = PathScan.empty 97 | 98 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 99 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 100 | p1.totalSize + p2.totalSize, 101 | p1.totalCount + p2.totalCount 102 | ) 103 | } 104 | 105 | } 106 | 107 | sealed trait FilesystemCmd[+A] 108 | 109 | object FilesystemCmd { 110 | 111 | implicit class EffFilesystemCmdOps[R, A](e: Eff[R, A]) { 112 | 113 | def runFilesystemCmds[U](fs: Filesystem)(implicit m: Member.Aux[FilesystemCmd, R, U]): Eff[U, A] = fs.runFilesystemCmds(e) 114 | } 115 | 116 | def filePath[R: _filesystem](path: String): Eff[R, FilePath] = Eff.send[FilesystemCmd, R, FilePath](MkFilePath(path)) 117 | 118 | def length[R: _filesystem](file: File): Eff[R, Long] = Eff.send[FilesystemCmd, R, Long](Length(file)) 119 | 120 | def listFiles[R: _filesystem](directory: Directory): Eff[R, List[FilePath]] = Eff.send[FilesystemCmd, R, List[FilePath]](ListFiles(directory)) 121 | 122 | } 123 | 124 | case class MkFilePath(path: String) extends FilesystemCmd[FilePath] 125 | case class Length(file: File) extends FilesystemCmd[Long] 126 | case class ListFiles(directory: Directory) extends FilesystemCmd[List[FilePath]] 127 | 128 | trait Filesystem { 129 | 130 | def runFilesystemCmds[R, A, U](effects: Eff[R, A])(implicit m: Member.Aux[FilesystemCmd, R, U]): Eff[U, A] = { 131 | 132 | val sideEffect = new SideEffect[FilesystemCmd] { 133 | def apply[X](fsc: FilesystemCmd[X]): X = 134 | (fsc match { 135 | case MkFilePath(path) => filePath(path) 136 | 137 | case Length(file) => length(file) 138 | 139 | case ListFiles(directory) => listFiles(directory) 140 | }) .asInstanceOf[X] 141 | 142 | def applicative[X, Tr[_] : Traverse](ms: Tr[FilesystemCmd[X]]): Tr[X] = 143 | ms.map(apply) 144 | } 145 | Interpret.interpretUnsafe(effects)(sideEffect)(m) 146 | } 147 | 148 | protected def filePath(path: String): FilePath 149 | 150 | protected def length(file: File): Long 151 | 152 | protected def listFiles(directory: Directory): List[FilePath] 153 | 154 | } 155 | object DefaultFilesystem extends Filesystem { 156 | 157 | protected def filePath(path: String): FilePath = 158 | if (Files.isRegularFile(Paths.get(path))) 159 | File(path.toString) 160 | else if (Files.isDirectory(Paths.get(path))) 161 | Directory(path) 162 | else 163 | Other(path) 164 | 165 | protected def length(file: File): Long = Files.size(Paths.get(file.path)) 166 | 167 | protected def listFiles(directory: Directory) = { 168 | val files = Files.list(Paths.get(directory.path)) 169 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 170 | case Directory(path) => List(Directory(path)) 171 | case File(path) => List(File(path)) 172 | case Other(path) => List.empty 173 | }) 174 | finally files.close() 175 | } 176 | 177 | } 178 | 179 | case class ScanConfig(topN: Int) 180 | 181 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 182 | 183 | object PathScan { 184 | 185 | def empty = PathScan(SortedSet.empty, 0, 0) 186 | 187 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 188 | def empty: PathScan = PathScan.empty 189 | 190 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 191 | p1.largestFiles.union(p2.largestFiles).take(n), 192 | p1.totalSize + p2.totalSize, 193 | p1.totalCount + p2.totalCount 194 | ) 195 | } 196 | 197 | } 198 | 199 | case class FileSize(file: File, size: Long) 200 | 201 | object FileSize { 202 | 203 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = FilesystemCmd.length(file).map(FileSize(file, _)) 204 | 205 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 206 | 207 | } 208 | 209 | object EffTypes { 210 | 211 | type _filesystem[R] = FilesystemCmd |= R 212 | type _config[R] = Reader[ScanConfig, ?] <= R 213 | type _err[R] = Either[String, ?] <= R 214 | type _log[R] = Writer[Log, ?] <= R 215 | } 216 | 217 | sealed trait Log {def msg: String} 218 | object Log { 219 | def error: String => Log = Error(_) 220 | def info: String => Log = Info(_) 221 | def debug: String => Log = Debug(_) 222 | } 223 | case class Error(msg: String) extends Log 224 | case class Info(msg: String) extends Log 225 | case class Debug(msg: String) extends Log 226 | 227 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 228 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 229 | //can overlap 230 | sealed trait FilePath { 231 | def path: String 232 | } 233 | 234 | case class File(path: String) extends FilePath 235 | case class Directory(path: String) extends FilePath 236 | case class Other(path: String) extends FilePath 237 | 238 | //Common pure code that is unaffected by the migration to Eff 239 | object ReportFormat { 240 | 241 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 242 | if (scan.largestFiles.nonEmpty) { 243 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 244 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 245 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 246 | } 247 | else 248 | s"No files found under path: $rootDir" 249 | } 250 | 251 | def formatByteString(bytes: Long): String = { 252 | if (bytes < 1000) 253 | s"${bytes} B" 254 | else { 255 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 256 | val pre = "KMGTPE".charAt(exp - 1) 257 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 258 | } 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /solutions/exerciseCustom/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, FilesystemCmd, Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T]) = 56 | program.runReader(ScanConfig(2)).runFilesystemCmds(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base)) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | logs.forall(expectedLogs.contains) 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /solutions/exerciseError/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Either[String, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runAsync.attempt. 41 | runSyncUnsafe(1.minute).leftMap(_.toString).flatten match { 42 | case Right(report) => println(report) 43 | case Left(msg) => println(s"Scan failed: $msg") 44 | } 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask[R, Filesystem] 57 | 58 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 59 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 60 | 61 | } yield ReportFormat.largeFilesReport(scan, base.toString) 62 | 63 | def pathScan[R: _task: _filesystem: _config](path: FilePath): Eff[R, PathScan] = path match { 64 | case f: File => 65 | for { 66 | fs <- FileSize.ofFile(f) 67 | } yield PathScan(SortedSet(fs), fs.size, 1) 68 | case dir: Directory => 69 | for { 70 | filesystem <- ask[R, Filesystem] 71 | topN <- takeTopN 72 | fileList <- taskDelay(filesystem.listFiles(dir)) 73 | childScans <- fileList.traverse(pathScan[R](_)) 74 | } yield childScans.combineAll(topN) 75 | case Other(_) => 76 | PathScan.empty.pureEff[R] 77 | } 78 | 79 | 80 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 81 | scanConfig <- ask 82 | } yield new Monoid[PathScan] { 83 | def empty: PathScan = PathScan.empty 84 | 85 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 86 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 87 | p1.totalSize + p2.totalSize, 88 | p1.totalCount + p2.totalCount 89 | ) 90 | } 91 | 92 | } 93 | 94 | trait Filesystem { 95 | 96 | def filePath(path: String): FilePath 97 | 98 | def length(file: File): Long 99 | 100 | def listFiles(directory: Directory): List[FilePath] 101 | 102 | } 103 | case object DefaultFilesystem extends Filesystem { 104 | 105 | def filePath(path: String): FilePath = 106 | if (Files.isRegularFile(Paths.get(path))) 107 | File(path.toString) 108 | else if (Files.isDirectory(Paths.get(path))) 109 | Directory(path) 110 | else 111 | Other(path) 112 | 113 | def length(file: File) = Files.size(Paths.get(file.path)) 114 | 115 | def listFiles(directory: Directory) = { 116 | val files = Files.list(Paths.get(directory.path)) 117 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 118 | case Directory(path) => List(Directory(path)) 119 | case File(path) => List(File(path)) 120 | case Other(path) => List.empty 121 | }) 122 | finally files.close() 123 | } 124 | 125 | } 126 | 127 | case class ScanConfig(topN: Int) 128 | 129 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 130 | 131 | object PathScan { 132 | 133 | def empty = PathScan(SortedSet.empty, 0, 0) 134 | 135 | } 136 | 137 | case class FileSize(file: File, size: Long) 138 | 139 | object FileSize { 140 | 141 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 142 | fs <- ask 143 | } yield FileSize(file, fs.length(file)) 144 | 145 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 146 | 147 | } 148 | 149 | object EffTypes { 150 | 151 | type _filesystem[R] = Reader[Filesystem, ?] <= R 152 | type _config[R] = Reader[ScanConfig, ?] <= R 153 | type _err[R] = Either[String, ?] <= R 154 | } 155 | 156 | 157 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 158 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 159 | //can overlap 160 | sealed trait FilePath { 161 | def path: String 162 | } 163 | 164 | case class File(path: String) extends FilePath 165 | case class Directory(path: String) extends FilePath 166 | case class Other(path: String) extends FilePath 167 | 168 | //Common pure code that is unaffected by the migration to Eff 169 | object ReportFormat { 170 | 171 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 172 | if (scan.largestFiles.nonEmpty) { 173 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 174 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 175 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 176 | } 177 | else 178 | s"No files found under path: $rootDir" 179 | } 180 | 181 | def formatByteString(bytes: Long): String = { 182 | if (bytes < 1000) 183 | s"${bytes} B" 184 | else { 185 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 186 | val pre = "KMGTPE".charAt(exp - 1) 187 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 188 | } 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /solutions/exerciseError/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).runAsync.attempt.runSyncUnsafe(3.seconds) 57 | 58 | "file scan" ! { 59 | val actual = run(Scanner.pathScan(base), fs) 60 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 61 | 62 | actual.mustEqual(expected) 63 | } 64 | 65 | "Error from Filesystem" ! { 66 | val emptyFs: Filesystem = MockFilesystem(directories, Map.empty) 67 | 68 | val actual = runE(Scanner.scanReport(Array("base", "10")), emptyFs) 69 | val expected = Left(new IOException().toString) 70 | 71 | actual.mustEqual(expected) 72 | } 73 | 74 | type E = Fx.fx3[Task, Reader[Filesystem, ?], Either[String, ?]] 75 | def runE[T](program: Eff[E, T], fs: Filesystem) = 76 | //there are two nested Either in the stack, one from Exceptions and one from errors raised by the program 77 | //we convert to a common error type String then flatten 78 | program.runReader(fs).runEither.runAsync.attempt.runSyncUnsafe(3.seconds).leftMap(_.toString).flatten 79 | 80 | "Error - Report with non-numeric input" ! { 81 | val actual = runE(Scanner.scanReport(Array("base", "not a number")), fs) 82 | val expected = Left("Number of files must be numeric: not a number") 83 | 84 | actual.mustEqual(expected) 85 | } 86 | 87 | "Error - Report with non-positive input" ! { 88 | val actual = runE(Scanner.scanReport(Array("base", "-1")), fs) 89 | val expected = Left("Invalid number of files -1") 90 | 91 | actual.mustEqual(expected) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /solutions/exerciseOptics/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection._ 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import monix.eval._ 17 | import monix.execution._ 18 | import monocle._ 19 | import monocle.macros._ 20 | 21 | import scala.concurrent.duration._ 22 | 23 | import EffTypes._ 24 | import EffOptics._ 25 | 26 | 27 | object Scanner { 28 | 29 | type R = Fx.fx2[Task, Reader[AppConfig, ?]] 30 | 31 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 32 | 33 | def main(args: Array[String]): Unit = { 34 | val program = scanReport[R](args(0)).map(println) 35 | 36 | program.runReader(AppConfig(ScanConfig(10), DefaultFilesystem)).runAsync.runSyncUnsafe(1.minute) 37 | } 38 | 39 | def scanReport[R: _task: _appconfig](base: String): Eff[R, String] = for { 40 | fs <- ask[R, Filesystem] 41 | scan <- pathScan(fs.filePath(base)) 42 | } yield ReportFormat.largeFilesReport(scan, base.toString) 43 | 44 | 45 | def pathScan[R: _task: _appconfig](path: FilePath): Eff[R, PathScan] = path match { 46 | case f: File => 47 | for { 48 | fs <- FileSize.ofFile[R](f) 49 | } yield PathScan(SortedSet(fs), fs.size, 1) 50 | case dir: Directory => 51 | for { 52 | filesystem <- ask[R, Filesystem] 53 | topN <- takeTopN[R] 54 | childScans <- filesystem.listFiles(dir).traverse(pathScan(_)) 55 | } yield childScans.combineAll(topN) 56 | case Other(_) => 57 | PathScan.empty.pureEff[R] 58 | } 59 | 60 | 61 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 62 | scanConfig <- ask[R, ScanConfig] 63 | } yield new Monoid[PathScan] { 64 | def empty: PathScan = PathScan.empty 65 | 66 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 67 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 68 | p1.totalSize + p2.totalSize, 69 | p1.totalCount + p2.totalCount 70 | ) 71 | } 72 | } 73 | 74 | object EffOptics { 75 | 76 | // "If I have a Reader of S effect, and a Lens from S to T, then I have a Reader of T effect" 77 | implicit def readerLens[R, S, T](implicit m: MemberIn[Reader[S, ?], R], l: Lens[S, T]): MemberIn[Reader[T, ?], R] = 78 | m.transform(new (Reader[T, ?] ~> Reader[S, ?]) { 79 | def apply[X](f: Reader[T, X]) = Reader[S, X](s => f(l.get(s))) 80 | }) 81 | 82 | } 83 | 84 | trait Filesystem { 85 | 86 | def filePath(path: String): FilePath 87 | 88 | def length(file: File): Long 89 | 90 | def listFiles(directory: Directory): List[FilePath] 91 | 92 | } 93 | case object DefaultFilesystem extends Filesystem { 94 | 95 | def filePath(path: String): FilePath = 96 | if (Files.isRegularFile(Paths.get(path))) 97 | File(path.toString) 98 | else if (Files.isDirectory(Paths.get(path))) 99 | Directory(path) 100 | else 101 | Other(path) 102 | 103 | def length(file: File) = Files.size(Paths.get(file.path)) 104 | 105 | def listFiles(directory: Directory) = { 106 | val files = Files.list(Paths.get(directory.path)) 107 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 108 | case Directory(path) => List(Directory(path)) 109 | case File(path) => List(File(path)) 110 | case Other(path) => List.empty 111 | }) 112 | finally files.close() 113 | } 114 | 115 | } 116 | 117 | case class AppConfig(scanConfig: ScanConfig, filesystem: Filesystem) 118 | object AppConfig { 119 | 120 | implicit val _scanConfig: Lens[AppConfig, ScanConfig] = GenLens[AppConfig](_.scanConfig) 121 | implicit val _filesystem: Lens[AppConfig, Filesystem] = GenLens[AppConfig](_.filesystem) 122 | } 123 | 124 | case class ScanConfig(topN: Int) 125 | 126 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 127 | 128 | object PathScan { 129 | 130 | def empty = PathScan(SortedSet.empty, 0, 0) 131 | 132 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 133 | def empty: PathScan = PathScan.empty 134 | 135 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 136 | p1.largestFiles.union(p2.largestFiles).take(n), 137 | p1.totalSize + p2.totalSize, 138 | p1.totalCount + p2.totalCount 139 | ) 140 | } 141 | 142 | } 143 | 144 | case class FileSize(file: File, size: Long) 145 | 146 | object FileSize { 147 | 148 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 149 | fs <- ask[R, Filesystem] 150 | } yield FileSize(file, fs.length(file)) 151 | 152 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 153 | 154 | } 155 | 156 | object EffTypes { 157 | 158 | type _appconfig[R] = Reader[AppConfig, ?] |= R 159 | type _filesystem[R] = Reader[Filesystem, ?] |= R 160 | type _config[R] = Reader[ScanConfig, ?] |= R 161 | } 162 | 163 | 164 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 165 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 166 | //can overlap 167 | sealed trait FilePath { 168 | def path: String 169 | } 170 | 171 | case class File(path: String) extends FilePath 172 | case class Directory(path: String) extends FilePath 173 | case class Other(path: String) extends FilePath 174 | 175 | //Common pure code that is unaffected by the migration to Eff 176 | object ReportFormat { 177 | 178 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 179 | if (scan.largestFiles.nonEmpty) { 180 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 181 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 182 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 183 | } 184 | else 185 | s"No files found under path: $rootDir" 186 | } 187 | 188 | def formatByteString(bytes: Long): String = { 189 | if (bytes < 1000) 190 | s"${bytes} B" 191 | else { 192 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 193 | val pre = "KMGTPE".charAt(exp - 1) 194 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /solutions/exerciseOptics/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io._ 4 | import java.io._ 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | 11 | import org.atnos.eff._ 12 | import org.atnos.eff.all._ 13 | import org.atnos.eff.syntax.all._ 14 | 15 | import org.atnos.eff.addon.monix._ 16 | import org.atnos.eff.addon.monix.task._ 17 | import org.atnos.eff.syntax.addon.monix.task._ 18 | 19 | import org.specs2._ 20 | 21 | import scala.collection.immutable.SortedSet 22 | 23 | import scala.concurrent.duration._ 24 | 25 | import monix.execution.Scheduler.Implicits.global 26 | 27 | class ScannerSpec extends mutable.Specification { 28 | 29 | import EffOptics._ 30 | 31 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 32 | 33 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 34 | 35 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 36 | 37 | def filePath(path: String): FilePath = 38 | if (directories.keySet.contains(Directory(path))) 39 | Directory(path) 40 | else if (fileSizes.keySet.contains(File(path))) 41 | File(path) 42 | else 43 | throw new FileNotFoundException(path) 44 | } 45 | 46 | "file scan" ! { 47 | val base = Directory("base") 48 | val base1 = File(s"${base.path}/1.txt") 49 | val base2 = File(s"${base.path}/2.txt") 50 | val subdir = Directory(s"${base.path}/subdir") 51 | val sub1 = File(s"${subdir.path}/1.txt") 52 | val sub3 = File(s"${subdir.path}/3.txt") 53 | val fs: Filesystem = MockFilesystem( 54 | Map( 55 | base -> List(subdir, base1, base2), 56 | subdir -> List(sub1, sub3) 57 | ), 58 | Map(base1 -> 1, base2 -> 2, sub1 -> 1, sub3 -> 3) 59 | ) 60 | 61 | val program = Scanner.pathScan[Scanner.R](base) 62 | val actual = program.runReader(AppConfig(ScanConfig(2), fs)).runAsync.runSyncUnsafe(3.seconds) 63 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 64 | 65 | actual.mustEqual(expected) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /solutions/exerciseReader/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import org.atnos.eff._ 13 | import org.atnos.eff.all._ 14 | import org.atnos.eff.syntax.all._ 15 | 16 | import org.atnos.eff.addon.monix._ 17 | import org.atnos.eff.addon.monix.task._ 18 | import org.atnos.eff.syntax.addon.monix.task._ 19 | 20 | import monix.eval._ 21 | import monix.execution._ 22 | 23 | import EffTypes._ 24 | 25 | import scala.concurrent.duration._ 26 | 27 | 28 | object Scanner { 29 | 30 | type R = Fx.fx3[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?]] 31 | 32 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 33 | 34 | def main(args: Array[String]): Unit = { 35 | val program = scanReport[R](args(0)).map(println) 36 | 37 | program.runReader(ScanConfig(10)).runReader(DefaultFilesystem: Filesystem).runAsync.runSyncUnsafe(1.minute) 38 | } 39 | 40 | def scanReport[R: _task: _filesystem: _config](base: String): Eff[R, String] = for { 41 | fs <- ask[R, Filesystem] 42 | scan <- pathScan(fs.filePath(base)) 43 | } yield ReportFormat.largeFilesReport(scan, base.toString) 44 | 45 | def pathScan[R: _task: _filesystem: _config](path: FilePath): Eff[R, PathScan] = path match { 46 | case f: File => 47 | for { 48 | fs <- FileSize.ofFile(f) 49 | } yield PathScan(SortedSet(fs), fs.size, 1) 50 | case dir: Directory => 51 | for { 52 | filesystem <- ask[R, Filesystem] 53 | topN <- takeTopN 54 | childScans <- filesystem.listFiles(dir).traverse(pathScan[R](_)) 55 | } yield childScans.combineAll(topN) 56 | case Other(_) => 57 | PathScan.empty.pureEff[R] 58 | } 59 | 60 | 61 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 62 | scanConfig <- ask 63 | } yield new Monoid[PathScan] { 64 | def empty: PathScan = PathScan.empty 65 | 66 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 67 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 68 | p1.totalSize + p2.totalSize, 69 | p1.totalCount + p2.totalCount 70 | ) 71 | } 72 | 73 | } 74 | 75 | trait Filesystem { 76 | 77 | def filePath(path: String): FilePath 78 | 79 | def length(file: File): Long 80 | 81 | def listFiles(directory: Directory): List[FilePath] 82 | 83 | } 84 | case object DefaultFilesystem extends Filesystem { 85 | 86 | def filePath(path: String): FilePath = 87 | if (Files.isRegularFile(Paths.get(path))) 88 | File(path.toString) 89 | else if (Files.isDirectory(Paths.get(path))) 90 | Directory(path) 91 | else 92 | Other(path) 93 | 94 | def length(file: File) = Files.size(Paths.get(file.path)) 95 | 96 | def listFiles(directory: Directory) = { 97 | val files = Files.list(Paths.get(directory.path)) 98 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 99 | case Directory(path) => List(Directory(path)) 100 | case File(path) => List(File(path)) 101 | case Other(path) => List.empty 102 | }) 103 | finally files.close() 104 | } 105 | 106 | } 107 | 108 | case class ScanConfig(topN: Int) 109 | 110 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 111 | 112 | object PathScan { 113 | 114 | def empty = PathScan(SortedSet.empty, 0, 0) 115 | 116 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 117 | def empty: PathScan = PathScan.empty 118 | 119 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 120 | p1.largestFiles.union(p2.largestFiles).take(n), 121 | p1.totalSize + p2.totalSize, 122 | p1.totalCount + p2.totalCount 123 | ) 124 | } 125 | 126 | } 127 | 128 | case class FileSize(file: File, size: Long) 129 | 130 | object FileSize { 131 | 132 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 133 | fs <- ask 134 | } yield FileSize(file, fs.length(file)) 135 | 136 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 137 | 138 | } 139 | 140 | object EffTypes { 141 | 142 | type _filesystem[R] = Reader[Filesystem, ?] <= R 143 | type _config[R] = Reader[ScanConfig, ?] <= R 144 | } 145 | 146 | 147 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 148 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 149 | //can overlap 150 | sealed trait FilePath { 151 | def path: String 152 | } 153 | 154 | case class File(path: String) extends FilePath 155 | case class Directory(path: String) extends FilePath 156 | case class Other(path: String) extends FilePath 157 | 158 | //Common pure code that is unaffected by the migration to Eff 159 | object ReportFormat { 160 | 161 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 162 | if (scan.largestFiles.nonEmpty) { 163 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 164 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 165 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 166 | } 167 | else 168 | s"No files found under path: $rootDir" 169 | } 170 | 171 | def formatByteString(bytes: Long): String = { 172 | if (bytes < 1000) 173 | s"${bytes} B" 174 | else { 175 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 176 | val pre = "KMGTPE".charAt(exp - 1) 177 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /solutions/exerciseReader/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io._ 4 | import java.io._ 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | 11 | import org.atnos.eff._ 12 | import org.atnos.eff.all._ 13 | import org.atnos.eff.syntax.all._ 14 | 15 | import org.atnos.eff.addon.monix._ 16 | import org.atnos.eff.addon.monix.task._ 17 | import org.atnos.eff.syntax.addon.monix.task._ 18 | 19 | import org.specs2._ 20 | 21 | import scala.collection.immutable.SortedSet 22 | 23 | import scala.concurrent.duration._ 24 | 25 | import monix.execution.Scheduler.Implicits.global 26 | 27 | class ScannerSpec extends mutable.Specification { 28 | 29 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 30 | 31 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 32 | 33 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 34 | 35 | def filePath(path: String): FilePath = 36 | if (directories.keySet.contains(Directory(path))) 37 | Directory(path) 38 | else if (fileSizes.keySet.contains(File(path))) 39 | File(path) 40 | else 41 | throw new FileNotFoundException(path) 42 | } 43 | 44 | "file scan" ! { 45 | val base = Directory("base") 46 | val base1 = File(s"${base.path}/1.txt") 47 | val base2 = File(s"${base.path}/2.txt") 48 | val subdir = Directory(s"${base.path}/subdir") 49 | val sub1 = File(s"${subdir.path}/1.txt") 50 | val sub3 = File(s"${subdir.path}/3.txt") 51 | val fs: Filesystem = MockFilesystem( 52 | Map( 53 | base -> List(subdir, base1, base2), 54 | subdir -> List(sub1, sub3) 55 | ), 56 | Map(base1 -> 1, base2 -> 2, sub1 -> 1, sub3 -> 3) 57 | ) 58 | 59 | val program = Scanner.pathScan[Scanner.R](base) 60 | val actual = program.runReader(ScanConfig(2)).runReader(fs).runAsync.runSyncUnsafe(3.seconds) 61 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 62 | 63 | actual.mustEqual(expected) 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /solutions/exerciseState/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx5[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?], State[Set[FilePath], ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).evalStateZero[Set[FilePath]].runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log: _sym](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask 57 | 58 | start <- taskDelay(System.currentTimeMillis()) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 61 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 62 | 63 | finish <- taskDelay(System.currentTimeMillis()) 64 | 65 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 66 | 67 | } yield ReportFormat.largeFilesReport(scan, base.toString) 68 | 69 | def pathScan[R: _task: _filesystem: _config: _log: _sym](path: FilePath): Eff[R, PathScan] = path match { 70 | 71 | case f: File => 72 | for { 73 | fs <- FileSize.ofFile(f) 74 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 75 | } yield PathScan(SortedSet(fs), fs.size, 1) 76 | 77 | case dir: Directory => 78 | for { 79 | filesystem <- ask[R, Filesystem] 80 | topN <- takeTopN 81 | fileList <- taskDelay(filesystem.listFiles(dir)) 82 | childScans <- fileList.traverse(pathScan(_)) 83 | _ <- { 84 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 85 | val fileCount = fileList.count(_.isInstanceOf[File]) 86 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 87 | } 88 | } yield childScans.combineAll(topN) 89 | 90 | case Symlink(_, to) => 91 | for { 92 | linksVisited <- get 93 | scan <- if (linksVisited.contains(to)) 94 | PathScan.empty.pureEff[R] 95 | else 96 | modify((set: Set[FilePath]) => set + to) >> pathScan(to) 97 | } yield scan 98 | 99 | case Other(_) => 100 | PathScan.empty.pureEff 101 | } 102 | 103 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 104 | scanConfig <- ask 105 | } yield new Monoid[PathScan] { 106 | def empty: PathScan = PathScan.empty 107 | 108 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 109 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 110 | p1.totalSize + p2.totalSize, 111 | p1.totalCount + p2.totalCount 112 | ) 113 | } 114 | 115 | } 116 | 117 | trait Filesystem { 118 | 119 | def filePath(path: String): FilePath 120 | 121 | def length(file: File): Long 122 | 123 | def listFiles(directory: Directory): List[FilePath] 124 | 125 | } 126 | case object DefaultFilesystem extends Filesystem { 127 | 128 | def filePath(pathStr: String): FilePath = { 129 | val path = Paths.get(pathStr) 130 | if (Files.isSymbolicLink(path)) 131 | Symlink(pathStr, filePath(Files.readSymbolicLink(path).toString)) 132 | else if (Files.isRegularFile(path)) 133 | File(path.toString) 134 | else if (Files.isDirectory(path)) 135 | Directory(pathStr) 136 | else 137 | Other(pathStr) 138 | } 139 | 140 | def length(file: File) = Files.size(Paths.get(file.path)) 141 | 142 | def listFiles(directory: Directory) = { 143 | val files = Files.list(Paths.get(directory.path)) 144 | try files.toScala[List].flatMap(path => listFilePath(path.toString)) 145 | finally files.close() 146 | } 147 | 148 | private def listFilePath(path: String): List[FilePath] = { 149 | filePath(path) match { 150 | case Directory(path) => List(Directory(path)) 151 | case File(path) => List(File(path)) 152 | case Symlink(path, to) => listFilePath(to.path) 153 | case Other(path) => List.empty 154 | } 155 | } 156 | } 157 | 158 | case class ScanConfig(topN: Int) 159 | 160 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 161 | 162 | object PathScan { 163 | 164 | def empty = PathScan(SortedSet.empty, 0, 0) 165 | 166 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 167 | def empty: PathScan = PathScan.empty 168 | 169 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 170 | p1.largestFiles.union(p2.largestFiles).take(n), 171 | p1.totalSize + p2.totalSize, 172 | p1.totalCount + p2.totalCount 173 | ) 174 | } 175 | 176 | } 177 | 178 | case class FileSize(file: File, size: Long) 179 | 180 | object FileSize { 181 | 182 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 183 | fs <- ask 184 | } yield FileSize(file, fs.length(file)) 185 | 186 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 187 | 188 | } 189 | 190 | object EffTypes { 191 | 192 | type _filesystem[R] = Reader[Filesystem, ?] <= R 193 | type _config[R] = Reader[ScanConfig, ?] <= R 194 | type _err[R] = Either[String, ?] <= R 195 | type _log[R] = Writer[Log, ?] <= R 196 | type _sym[R] = State[Set[FilePath], ?] <= R 197 | } 198 | 199 | sealed trait Log {def msg: String} 200 | object Log { 201 | def error: String => Log = Error(_) 202 | def info: String => Log = Info(_) 203 | def debug: String => Log = Debug(_) 204 | } 205 | case class Error(msg: String) extends Log 206 | case class Info(msg: String) extends Log 207 | case class Debug(msg: String) extends Log 208 | 209 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 210 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 211 | //can overlap 212 | sealed trait FilePath { 213 | def path: String 214 | } 215 | 216 | case class File(path: String) extends FilePath 217 | case class Directory(path: String) extends FilePath 218 | case class Symlink(path: String, linkTo: FilePath) extends FilePath 219 | case class Other(path: String) extends FilePath 220 | 221 | //Common pure code that is unaffected by the migration to Eff 222 | object ReportFormat { 223 | 224 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 225 | if (scan.largestFiles.nonEmpty) { 226 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 227 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 228 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 229 | } 230 | else 231 | s"No files found under path: $rootDir" 232 | } 233 | 234 | def formatByteString(bytes: Long): String = { 235 | if (bytes < 1000) 236 | s"${bytes} B" 237 | else { 238 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 239 | val pre = "KMGTPE".charAt(exp - 1) 240 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 241 | } 242 | } 243 | } 244 | -------------------------------------------------------------------------------- /solutions/exerciseState/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val linkTarget = File(s"/somewhere/else/7.txt") 42 | val base1 = File(s"${base.path}/1.txt") 43 | val baseLink = Symlink(s"${base.path}/7.txt", linkTarget) 44 | val subdir = Directory(s"${base.path}/subdir") 45 | val sub2 = File(s"${subdir.path}/2.txt") 46 | val subLink = Symlink(s"${subdir.path}/7.txt", linkTarget) 47 | val directories = Map( 48 | base -> List(subdir, base1, baseLink), 49 | subdir -> List(sub2, subLink) 50 | ) 51 | val fileSizes = Map(base1 -> 1L, sub2 -> 2L, linkTarget -> 7L) 52 | val fs = MockFilesystem(directories, fileSizes) 53 | 54 | type R = Fx.fx5[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?], State[Set[FilePath], ?]] 55 | 56 | def run[T](program: Eff[R, T], fs: Filesystem) = 57 | program.runReader(ScanConfig(2)).runReader(fs).evalStateZero[Set[FilePath]].taskAttempt.runWriter[Log].runAsync.runSyncUnsafe(3.seconds) 58 | 59 | val expected = Right(new PathScan(SortedSet(FileSize(linkTarget, 7), FileSize(sub2, 2)), 10, 3)) 60 | 61 | val (actual, logs) = run(Scanner.pathScan[R](base), fs) 62 | 63 | "Report Format" ! {actual.mustEqual(expected)} 64 | 65 | } 66 | -------------------------------------------------------------------------------- /solutions/exerciseTask/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.implicits._ 10 | 11 | import monix.eval._ 12 | import monix.execution._ 13 | 14 | import scala.concurrent.duration._ 15 | 16 | 17 | object Scanner { 18 | 19 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 20 | 21 | def main(args: Array[String]): Unit = { 22 | val program = scanReport(Paths.get(args(0)), 10).map(println) 23 | 24 | program.runSyncUnsafe(1.minute) 25 | } 26 | 27 | def scanReport(base: Path, topN: Int): Task[String] = for { 28 | scan <- pathScan(FilePath(base), topN) 29 | } yield ReportFormat.largeFilesReport(scan, base.toString) 30 | 31 | def pathScan(filePath: FilePath, topN: Int): Task[PathScan] = filePath match { 32 | case File(path) => 33 | Task { 34 | val fs = FileSize.ofFile(Paths.get(path)) 35 | PathScan(SortedSet(fs), fs.size, 1) 36 | } 37 | case Directory(path) => 38 | for { 39 | files <- Task { 40 | val jstream = Files.list(Paths.get(path)) 41 | try jstream.toScala[List] 42 | finally jstream.close() 43 | } 44 | scans <- files.traverse(subpath => pathScan(FilePath(subpath), topN)) 45 | } yield scans.combineAll(PathScan.topNMonoid(topN)) 46 | case Other(_) => 47 | Task(PathScan.empty) 48 | } 49 | 50 | } 51 | 52 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 53 | 54 | object PathScan { 55 | 56 | def empty = PathScan(SortedSet.empty, 0, 0) 57 | 58 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 59 | def empty: PathScan = PathScan.empty 60 | 61 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 62 | p1.largestFiles.union(p2.largestFiles).take(n), 63 | p1.totalSize + p2.totalSize, 64 | p1.totalCount + p2.totalCount 65 | ) 66 | } 67 | 68 | } 69 | 70 | case class FileSize(path: Path, size: Long) 71 | 72 | object FileSize { 73 | 74 | def ofFile(file: Path) = { 75 | FileSize(file, Files.size(file)) 76 | } 77 | 78 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long ](_.size).reverse 79 | 80 | } 81 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 82 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 83 | //can overlap 84 | sealed trait FilePath { 85 | def path: String 86 | } 87 | object FilePath { 88 | 89 | def apply(path: Path): FilePath = 90 | if (Files.isRegularFile(path)) 91 | File(path.toString) 92 | else if (Files.isDirectory(path)) 93 | Directory(path.toString) 94 | else 95 | Other(path.toString) 96 | } 97 | case class File(path: String) extends FilePath 98 | case class Directory(path: String) extends FilePath 99 | case class Other(path: String) extends FilePath 100 | 101 | //Common pure code that is unaffected by the migration to Eff 102 | object ReportFormat { 103 | 104 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 105 | if (scan.largestFiles.nonEmpty) { 106 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 107 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.path}").mkString("", "\n", "\n") + 108 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 109 | } 110 | else 111 | s"No files found under path: $rootDir" 112 | } 113 | 114 | def formatByteString(bytes: Long): String = { 115 | if (bytes < 1000) 116 | s"${bytes} B" 117 | else { 118 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 119 | val pre = "KMGTPE".charAt(exp - 1) 120 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 121 | } 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /solutions/exerciseTask/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.PrintWriter 4 | import java.nio.file._ 5 | 6 | import org.specs2._ 7 | 8 | import scala.collection.immutable.SortedSet 9 | 10 | import scala.concurrent.duration._ 11 | 12 | import monix.execution.Scheduler.Implicits.global 13 | 14 | class ScannerSpec extends mutable.Specification { 15 | 16 | "Report Format" ! { 17 | val base = deletedOnExit(Files.createTempDirectory("exerciseTask")) 18 | val base1 = deletedOnExit(fillFile(base, 1)) 19 | val base2 = deletedOnExit(fillFile(base, 2)) 20 | val subdir = deletedOnExit(Files.createTempDirectory(base, "subdir")) 21 | val sub1 = deletedOnExit(fillFile(subdir, 1)) 22 | val sub3 = deletedOnExit(fillFile(subdir, 3)) 23 | 24 | val actual = Scanner.pathScan(FilePath(base), 2).runSyncUnsafe(3.seconds) 25 | val expected = new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4) 26 | 27 | actual.mustEqual(expected) 28 | } 29 | 30 | def fillFile(dir: Path, size: Int) = { 31 | val path = dir.resolve(s"$size.txt") 32 | val w = new PrintWriter(path.toFile) 33 | try w.write("a" * size) 34 | finally w.close 35 | path 36 | } 37 | 38 | def deletedOnExit(p: Path) = { 39 | p.toFile.deleteOnExit() 40 | p 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /solutions/exerciseWriter/src/main/scala/scan/Scanner.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.nio.file._ 4 | 5 | import scala.compat.java8.StreamConverters._ 6 | import scala.collection.SortedSet 7 | 8 | import cats._ 9 | import cats.data._ 10 | import cats.implicits._ 11 | 12 | import mouse.all._ 13 | 14 | import org.atnos.eff._ 15 | import org.atnos.eff.all._ 16 | import org.atnos.eff.syntax.all._ 17 | 18 | import org.atnos.eff.addon.monix._ 19 | import org.atnos.eff.addon.monix.task._ 20 | import org.atnos.eff.syntax.addon.monix.task._ 21 | 22 | import monix.eval._ 23 | import monix.execution._ 24 | 25 | import EffTypes._ 26 | 27 | import scala.concurrent.duration._ 28 | 29 | 30 | object Scanner { 31 | val Usage = "Scanner [number of largest files to track]" 32 | 33 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Either[String, ?], Writer[Log, ?]] 34 | 35 | implicit val s = Scheduler(ExecutionModel.BatchedExecution(32)) 36 | 37 | def main(args: Array[String]): Unit = { 38 | val program = scanReport[R](args).map(println) 39 | 40 | program.runReader(DefaultFilesystem: Filesystem).runEither.runWriterUnsafe[Log]{ 41 | case Error(msg) => System.err.println(msg) 42 | case Info(msg) => System.out.println(msg) 43 | case _ => () 44 | }.runAsync.runSyncUnsafe(1.minute) 45 | } 46 | 47 | def scanReport[R: _task: _filesystem: _err: _log](args: Array[String]): Eff[R, String] = for { 48 | base <- optionEither(args.lift(0), s"Path to scan must be specified.\n$Usage") 49 | 50 | topN <- { 51 | val n = args.lift(1).getOrElse("10") 52 | fromEither(n.parseInt.leftMap(_ => s"Number of files must be numeric: $n")) 53 | } 54 | topNValid <- if (topN < 0) left[R, String, Int](s"Invalid number of files $topN") else topN.pureEff[R] 55 | 56 | fs <- ask[R, Filesystem] 57 | 58 | start <- taskDelay(System.currentTimeMillis()) 59 | 60 | scan <- pathScan[Fx.prepend[Reader[ScanConfig, ?], R]]( 61 | fs.filePath(base)).runReader[ScanConfig](ScanConfig(topNValid)) 62 | 63 | finish <- taskDelay(System.currentTimeMillis()) 64 | 65 | _ <- tell(Log.info(s"Scan of $base completed in ${finish - start}ms")) 66 | 67 | } yield ReportFormat.largeFilesReport(scan, base.toString) 68 | 69 | def pathScan[R: _task: _filesystem: _config: _log](path: FilePath): Eff[R, PathScan] = path match { 70 | 71 | case f: File => 72 | for { 73 | fs <- FileSize.ofFile(f) 74 | _ <- tell(Log.debug(s"File ${fs.file.path} Size ${ReportFormat.formatByteString(fs.size)}")) 75 | } yield PathScan(SortedSet(fs), fs.size, 1) 76 | 77 | case dir: Directory => 78 | for { 79 | filesystem <- ask[R, Filesystem] 80 | topN <- takeTopN 81 | fileList <- taskDelay(filesystem.listFiles(dir)) 82 | childScans <- fileList.traverse(pathScan[R](_)) 83 | _ <- { 84 | val dirCount = fileList.count(_.isInstanceOf[Directory]) 85 | val fileCount = fileList.count(_.isInstanceOf[File]) 86 | tell(Log.debug(s"Scanning directory '$dir': $dirCount subdirectories and $fileCount files")) 87 | } 88 | } yield childScans.combineAll(topN) 89 | 90 | case Other(_) => 91 | PathScan.empty.pureEff[R] 92 | } 93 | 94 | 95 | def takeTopN[R: _config]: Eff[R, Monoid[PathScan]] = for { 96 | scanConfig <- ask 97 | } yield new Monoid[PathScan] { 98 | def empty: PathScan = PathScan.empty 99 | 100 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 101 | p1.largestFiles.union(p2.largestFiles).take(scanConfig.topN), 102 | p1.totalSize + p2.totalSize, 103 | p1.totalCount + p2.totalCount 104 | ) 105 | } 106 | 107 | } 108 | 109 | trait Filesystem { 110 | 111 | def filePath(path: String): FilePath 112 | 113 | def length(file: File): Long 114 | 115 | def listFiles(directory: Directory): List[FilePath] 116 | 117 | } 118 | case object DefaultFilesystem extends Filesystem { 119 | 120 | def filePath(path: String): FilePath = 121 | if (Files.isRegularFile(Paths.get(path))) 122 | File(path.toString) 123 | else if (Files.isDirectory(Paths.get(path))) 124 | Directory(path) 125 | else 126 | Other(path) 127 | 128 | def length(file: File) = Files.size(Paths.get(file.path)) 129 | 130 | def listFiles(directory: Directory) = { 131 | val files = Files.list(Paths.get(directory.path)) 132 | try files.toScala[List].flatMap(path => filePath(path.toString) match { 133 | case Directory(path) => List(Directory(path)) 134 | case File(path) => List(File(path)) 135 | case Other(path) => List.empty 136 | }) 137 | finally files.close() 138 | } 139 | 140 | } 141 | 142 | case class ScanConfig(topN: Int) 143 | 144 | case class PathScan(largestFiles: SortedSet[FileSize], totalSize: Long, totalCount: Long) 145 | 146 | object PathScan { 147 | 148 | def empty = PathScan(SortedSet.empty, 0, 0) 149 | 150 | def topNMonoid(n: Int): Monoid[PathScan] = new Monoid[PathScan] { 151 | def empty: PathScan = PathScan.empty 152 | 153 | def combine(p1: PathScan, p2: PathScan): PathScan = PathScan( 154 | p1.largestFiles.union(p2.largestFiles).take(n), 155 | p1.totalSize + p2.totalSize, 156 | p1.totalCount + p2.totalCount 157 | ) 158 | } 159 | 160 | } 161 | 162 | case class FileSize(file: File, size: Long) 163 | 164 | object FileSize { 165 | 166 | def ofFile[R: _filesystem](file: File): Eff[R, FileSize] = for { 167 | fs <- ask 168 | } yield FileSize(file, fs.length(file)) 169 | 170 | implicit val ordering: Ordering[FileSize] = Ordering.by[FileSize, Long](_.size).reverse 171 | 172 | } 173 | 174 | object EffTypes { 175 | 176 | type _filesystem[R] = Reader[Filesystem, ?] <= R 177 | type _config[R] = Reader[ScanConfig, ?] <= R 178 | type _err[R] = Either[String, ?] <= R 179 | type _log[R] = Writer[Log, ?] <= R 180 | } 181 | 182 | sealed trait Log {def msg: String} 183 | object Log { 184 | def error: String => Log = Error(_) 185 | def info: String => Log = Info(_) 186 | def debug: String => Log = Debug(_) 187 | } 188 | case class Error(msg: String) extends Log 189 | case class Info(msg: String) extends Log 190 | case class Debug(msg: String) extends Log 191 | 192 | //I prefer an closed set of disjoint cases over a series of isX(): Boolean tests, as provided by the Java API 193 | //The problem with boolean test methods is they make it unclear what the complete set of possible states is, and which tests 194 | //can overlap 195 | sealed trait FilePath { 196 | def path: String 197 | } 198 | 199 | case class File(path: String) extends FilePath 200 | case class Directory(path: String) extends FilePath 201 | case class Other(path: String) extends FilePath 202 | 203 | //Common pure code that is unaffected by the migration to Eff 204 | object ReportFormat { 205 | 206 | def largeFilesReport(scan: PathScan, rootDir: String): String = { 207 | if (scan.largestFiles.nonEmpty) { 208 | s"Largest ${scan.largestFiles.size} file(s) found under path: $rootDir\n" + 209 | scan.largestFiles.map(fs => s"${(fs.size * 100)/scan.totalSize}% ${formatByteString(fs.size)} ${fs.file}").mkString("", "\n", "\n") + 210 | s"${scan.totalCount} total files found, having total size ${formatByteString(scan.totalSize)} bytes.\n" 211 | } 212 | else 213 | s"No files found under path: $rootDir" 214 | } 215 | 216 | def formatByteString(bytes: Long): String = { 217 | if (bytes < 1000) 218 | s"${bytes} B" 219 | else { 220 | val exp = (Math.log(bytes) / Math.log(1000)).toInt 221 | val pre = "KMGTPE".charAt(exp - 1) 222 | s"%.1f ${pre}B".format(bytes / Math.pow(1000, exp)) 223 | } 224 | } 225 | } 226 | -------------------------------------------------------------------------------- /solutions/exerciseWriter/src/test/scala/scan/ScannerSpec.scala: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import java.io.FileNotFoundException 4 | import java.io.IOException 5 | import java.nio.file._ 6 | 7 | import cats._ 8 | import cats.data._ 9 | import cats.implicits._ 10 | import org.atnos.eff._ 11 | import org.atnos.eff.all._ 12 | import org.atnos.eff.syntax.all._ 13 | import org.atnos.eff.addon.monix._ 14 | import org.atnos.eff.addon.monix.task._ 15 | import org.atnos.eff.syntax.addon.monix.task._ 16 | import org.specs2._ 17 | 18 | import scala.collection.immutable.SortedSet 19 | import scala.concurrent.duration._ 20 | import monix.eval._ 21 | import monix.execution.Scheduler.Implicits.global 22 | 23 | class ScannerSpec extends mutable.Specification { 24 | 25 | case class MockFilesystem(directories: Map[Directory, List[FilePath]], fileSizes: Map[File, Long]) extends Filesystem { 26 | 27 | def length(file: File) = fileSizes.getOrElse(file, throw new IOException()) 28 | 29 | def listFiles(directory: Directory) = directories.getOrElse(directory, throw new IOException()) 30 | 31 | def filePath(path: String): FilePath = 32 | if (directories.keySet.contains(Directory(path))) 33 | Directory(path) 34 | else if (fileSizes.keySet.contains(File(path))) 35 | File(path) 36 | else 37 | throw new FileNotFoundException(path) 38 | } 39 | 40 | val base = Directory("base") 41 | val base1 = File(s"${base.path}/1.txt") 42 | val base2 = File(s"${base.path}/2.txt") 43 | val subdir = Directory(s"${base.path}/subdir") 44 | val sub1 = File(s"${subdir.path}/1.txt") 45 | val sub3 = File(s"${subdir.path}/3.txt") 46 | val directories = Map( 47 | base -> List(subdir, base1, base2), 48 | subdir -> List(sub1, sub3) 49 | ) 50 | val fileSizes = Map(base1 -> 1L, base2 -> 2L, sub1 -> 1L, sub3 -> 3L) 51 | val fs = MockFilesystem(directories, fileSizes) 52 | 53 | type R = Fx.fx4[Task, Reader[Filesystem, ?], Reader[ScanConfig, ?], Writer[Log, ?]] 54 | 55 | def run[T](program: Eff[R, T], fs: Filesystem) = 56 | program.runReader(ScanConfig(2)).runReader(fs).taskAttempt.runWriter.runAsync.runSyncUnsafe(3.seconds) 57 | 58 | val expected = Right(new PathScan(SortedSet(FileSize(sub3, 3), FileSize(base2, 2)), 7, 4)) 59 | val expectedLogs = Set( 60 | Log.info("Scan started on Directory(base)"), 61 | Log.debug("Scanning directory 'Directory(base)': 1 subdirectories and 2 files"), 62 | Log.debug("File base/1.txt Size 1 B"), 63 | Log.debug("File base/2.txt Size 2 B"), 64 | Log.debug("Scanning directory 'Directory(base/subdir)': 0 subdirectories and 2 files"), 65 | Log.debug("File base/subdir/1.txt Size 1 B"), 66 | Log.debug("File base/subdir/3.txt Size 3 B") 67 | ) 68 | 69 | val (actual, logs) = run(Scanner.pathScan(base), fs) 70 | 71 | "Report Format" ! {actual.mustEqual(expected)} 72 | 73 | "Logs messages are emitted (ignores order due to non-determinstic concurrent execution)" ! { 74 | expectedLogs.forall(logs.contains) 75 | } 76 | } 77 | --------------------------------------------------------------------------------