├── README.md ├── TODO.md ├── logo.png ├── presentation.html ├── presentation.md ├── project ├── Build.scala ├── build.properties └── plugins.sbt ├── strucs-core └── src │ ├── main │ └── scala │ │ └── strucs │ │ ├── ComposeCodec.scala │ │ ├── FieldExtractor.scala │ │ ├── Nil.scala │ │ ├── Struct.scala │ │ ├── StructKey.scala │ │ ├── StructKeyProvider.scala │ │ ├── Wrapper.scala │ │ └── package.scala │ └── test │ └── scala │ └── strucs │ ├── EncoderSpec.scala │ ├── FieldExtractorSpec.scala │ └── StructSpec.scala ├── strucs-demo └── src │ └── main │ └── tut │ ├── README.md │ ├── logo.png │ ├── presentation.html │ └── presentation.md ├── strucs-fix └── src │ ├── main │ └── scala │ │ └── strucs │ │ └── fix │ │ ├── CodecFix.scala │ │ ├── FixElement.scala │ │ ├── FixEncodeException.scala │ │ ├── TagCodecFix.scala │ │ ├── ValueCodecFix.scala │ │ └── dict │ │ └── fix42 │ │ └── package.scala │ └── test │ └── scala │ └── strucs │ └── fix │ └── CodecFixSpec.scala ├── strucs-json └── src │ ├── main │ └── scala │ │ └── strucs │ │ └── json │ │ └── StrucsEncodeJson.scala │ └── test │ └── scala │ └── strucs │ └── json │ └── CodecJsonSpec.scala └── strucs-spark └── src ├── main └── scala │ └── strucs │ └── spark │ └── StructDataFrame.scala └── test ├── resources └── log4j.properties └── scala └── strucs └── spark ├── SparkApp.scala └── StructDataFrameSpec.scala /README.md: -------------------------------------------------------------------------------- 1 | # Strucs - Flexible data structures in Scala 2 | 3 | Strucs is a lightweight library that allows to manipulate, encode and decode flexible data structures while maintaining immutability and type safety. 4 | 5 | A Struct is analogous to a case class that can accept new fields dynamically. 6 | 7 | Using the strucs extensions, a single struc instance can be easily serialized/deserialized to various formats, such as JSON, [FIX protocol](https://en.wikipedia.org/wiki/Financial_Information_eXchange), Protobuf, ... 8 | 9 | [Slides for Scala eXchange 2015](https://rawgit.com/mikaelv/strucs/master/presentation.html#1) 10 | 11 | ## Quick start 12 | 13 | ### Create/Add/Update 14 | 15 | ```scala 16 | import strucs._ 17 | 18 | case class Ticker(v: String) extends AnyVal 19 | case class Quantity(v: BigDecimal) extends AnyVal 20 | case class Price(v: BigDecimal) extends AnyVal 21 | ``` 22 | ```scala 23 | scala> val order = Struct(Ticker("^FTSE")) 24 | order: strucs.Struct[Ticker with strucs.Nil] = Struct(Map(StructKey(class Ticker) -> Ticker(^FTSE))) 25 | 26 | scala> val order2 = order.add(Quantity(5)) 27 | order2: strucs.Struct[Ticker with strucs.Nil with Quantity] = Struct(Map(StructKey(class Ticker) -> Ticker(^FTSE), StructKey(class Quantity) -> Quantity(5))) 28 | 29 | scala> order2.get[Ticker] 30 | res1: Ticker = Ticker(^FTSE) 31 | 32 | scala> val order3 = order2.update(Ticker("^FCHI")) 33 | order3: strucs.Struct[Ticker with strucs.Nil with Quantity] = Struct(Map(StructKey(class Ticker) -> Ticker(^FCHI), StructKey(class Quantity) -> Quantity(5))) 34 | 35 | scala> order3.get[Ticker] 36 | res2: Ticker = Ticker(^FCHI) 37 | ``` 38 | order3 does not have a Price field. Any attempt to access it is rejected by the compiler. 39 | ```scala 40 | scala> order3.get[Price] 41 | :21: error: Cannot prove that Ticker with strucs.Nil with Quantity <:< Price. 42 | order3.get[Price] 43 | ^ 44 | ``` 45 | 46 | ### Structural typing 47 | *When I see a bird that walks like a duck and swims like a duck and quacks like a duck, I call that bird a duck.* 48 | 49 | Let's define a function that accepts any Struct that has two specific fields. 50 | ```scala 51 | scala> def totalPrice[T <: Quantity with Price](struct: Struct[T]): BigDecimal = { 52 | | struct.get[Quantity].v * struct.get[Price].v 53 | | } 54 | totalPrice: [T <: Quantity with Price](struct: strucs.Struct[T])BigDecimal 55 | ``` 56 | A call with an incompatible Struct is rejected by the compiler: 57 | ```scala 58 | scala> totalPrice(order3) 59 | :22: error: inferred type arguments [Ticker with strucs.Nil with Quantity] do not conform to method totalPrice's type parameter bounds [T <: Quantity with Price] 60 | totalPrice(order3) 61 | ^ 62 | :22: error: type mismatch; 63 | found : strucs.Struct[Ticker with strucs.Nil with Quantity] 64 | required: strucs.Struct[T] 65 | totalPrice(order3) 66 | ^ 67 | ``` 68 | But succeeds when we add the required field: 69 | ```scala 70 | scala> totalPrice(order3.add(Price(10))) 71 | res5: BigDecimal = 50 72 | ``` 73 | 74 | 75 | ### Encoding/Decoding 76 | Provided that the encoders/decoders for the fields are in scope, the same struct instance can be encoded/decoded to various formats: 77 | ```scala 78 | import strucs.json._ 79 | import strucs.fix._ 80 | import strucs.fix.dict.fix42._ // defines common FIX 4.2 tags with their codec 81 | import CodecFix._ 82 | import StrucsCodecJson._ 83 | import StrucsEncodeJson._ 84 | import StrucsDecodeJson._ 85 | import argonaut._ 86 | import Argonaut._ 87 | 88 | type MyOrder = Struct[OrderQty with Symbol with Nil] 89 | val order: MyOrder = Struct.empty + OrderQty(10) + Symbol("^FTSE") 90 | ``` 91 | The order can be encoded/decoded to/from FIX if we add the required tags BeginString and MsgType. 92 | ```scala 93 | scala> val fixOrder = order + BeginString.Fix42 + MsgType.OrderSingle 94 | fixOrder: strucs.Struct[strucs.fix.dict.fix42.OrderQty with strucs.fix.dict.fix42.Symbol with strucs.Nil with strucs.fix.dict.fix42.BeginString with strucs.fix.dict.fix42.MsgType] = Struct(Map(StructKey(class OrderQty) -> OrderQty(10), StructKey(class Symbol) -> Symbol(^FTSE), StructKey(class BeginString) -> BeginString(FIX.4.2), StructKey(class MsgType) -> MsgType(D))) 95 | 96 | scala> val fix = fixOrder.toFixMessageString 97 | fix: String = 8=FIX.4.2?9=20?35=D?38=10?55=^FTSE?10=036? 98 | 99 | scala> fix.toStruct[fixOrder.Mixin] 100 | res7: scala.util.Try[strucs.Struct[fixOrder.Mixin]] = Success(Struct(Map(StructKey(class MsgType) -> MsgType(D), StructKey(class BeginString) -> BeginString(FIX.4.2), StructKey(class Symbol) -> Symbol(^FTSE), StructKey(class OrderQty) -> OrderQty(10)))) 101 | ``` 102 | If we define the [Argonaut](http://argonaut.io/) Json codecs for Symbol and OrderQty, 103 | ```scala 104 | implicit val symbolCodecJson: CodecJson[Symbol] = StrucsCodecJson.fromWrapper[Symbol, String]("symbol") 105 | implicit val orderQtyCodecJson: CodecJson[OrderQty] = StrucsCodecJson.fromWrapper[OrderQty, BigDecimal]("quantity") 106 | ``` 107 | We can encode/decode our order to/from Json 108 | ```scala 109 | scala> val json = order.toJsonString 110 | json: String = {"quantity":10,"symbol":"^FTSE"} 111 | 112 | scala> json.decodeOption[MyOrder] 113 | res8: Option[MyOrder] = Some(Struct(Map(StructKey(class Symbol) -> Symbol(^FTSE), StructKey(class OrderQty) -> OrderQty(10)))) 114 | ``` 115 | 116 | ### More examples 117 | Please check out the unit tests for more usage examples. 118 | 119 | ## Motivation 120 | Consider a program which manages Orders. 121 | A common approach would be to use case classes with simple types for its fields: 122 | ```scala 123 | scala> case class SimpleOrder(symbol: String, quantity: BigDecimal, price: BigDecimal) 124 | defined class SimpleOrder 125 | ``` 126 | However, using simple types such as String, Int, BigDecimal, ... everywhere can rapidly make the code confusing and fragile. 127 | Imagine we have to extract the price and quantity of all the FTSE orders 128 | ```scala 129 | scala> def simpleFootsieOrders(orders: List[SimpleOrder]): List[(BigDecimal, BigDecimal)] = 130 | | orders collect { 131 | | case SimpleOrder(sym, q, p) if sym == "^FTSE" => (q, p) 132 | | } 133 | simpleFootsieOrders: (orders: List[SimpleOrder])List[(BigDecimal, BigDecimal)] 134 | ``` 135 | If I do not get the argument order right (or if it has been refactored), the code above will compile but will not do what I expect. 136 | Furthermore, the return type is List[(BigDecimal, BigDecimal)], which is unclear for the users of the function. 137 | 138 | 139 | We need stronger types to make our code clearer and safer. You you might want to use [value classes](http://docs.scala-lang.org/overviews/core/value-classes.html) as follows: 140 | ```scala 141 | case class Symbol(v: String) extends AnyVal 142 | val FTSE = Symbol("FTSE") 143 | case class Quantity(v: BigDecimal) extends AnyVal 144 | case class Price(v: BigDecimal) extends AnyVal 145 | 146 | case class TypedOrder(symbol: Symbol, quantity: Quantity, price: Price) 147 | ``` 148 | ```scala 149 | scala> def typedFootsieOrders(orders: List[TypedOrder]): List[(Quantity, Price)] = 150 | | orders.collect { 151 | | case TypedOrder(sym, q, p) if sym == FTSE => (q, p) 152 | | } 153 | typedFootsieOrders: (orders: List[TypedOrder])List[(Quantity, Price)] 154 | ``` 155 | Now the return type is much clearer and safer, and my matching expression is safer as well: 156 | I cannot inadvertently swap arguments without getting a compilation error. 157 | 158 | On the other hand, we now observe that the names of the attributes are redundant with their types. 159 | It would be nicer if we could declare them only once. 160 | Also, I cannot easily reuse a set of fields, such as symbol and quantity, in another case class. I need to redefine the class with all its fields: 161 | ```scala 162 | scala> case class StopPrice(v: BigDecimal) 163 | defined class StopPrice 164 | 165 | scala> case class StopOrder(symbol: Symbol, quantity: Quantity, price: StopPrice) 166 | defined class StopOrder 167 | ``` 168 | If I then want to define a function that accepts StopOrder or TypedOrder, I would typically define a common trait that these classes will extend. 169 | ```scala 170 | scala> trait Order { 171 | | def symbol: Symbol 172 | | } 173 | defined trait Order 174 | 175 | scala> def filterFootsie(orders: List[Order]): List[Order] = orders.filter(_.symbol == FTSE) 176 | filterFootsie: (orders: List[Order])List[Order] 177 | ``` 178 | This leads to some duplication, and it may not even be feasible if TypedOrder is defined in a third party library. 179 | 180 | With strucs, we can define the same as follows: 181 | ```scala 182 | type BaseOrderType = Symbol with Quantity with Nil 183 | type StructOrder = Struct[BaseOrderType with Price] 184 | type StructStopOrder = Struct[BaseOrderType with StopPrice] 185 | def filterFootsie[T <: Symbol](orders: List[Struct[T]]) = 186 | orders.filter(_.get[Symbol] == FTSE) 187 | ``` 188 | The different "order" types are now **composable**. I can define an abstraction BaseOrder, and reuse it to define other Order types. 189 | Also, I do not have to declare field names anymore, as I use only the types of the fields to access them. 190 | This composition capability also applies to instances: 191 | 192 | ```scala 193 | scala> val baseOrder = Struct.empty + FTSE + Quantity(100) 194 | baseOrder: strucs.Struct[strucs.Nil with Symbol with Quantity] = Struct(Map(StructKey(class Symbol) -> Symbol(FTSE), StructKey(class Quantity) -> Quantity(100))) 195 | 196 | scala> val order: StructOrder = baseOrder + Price(30) 197 | order: StructOrder = Struct(Map(StructKey(class Symbol) -> Symbol(FTSE), StructKey(class Quantity) -> Quantity(100), StructKey(class Price) -> Price(30))) 198 | 199 | scala> val stopOrder: StructStopOrder = baseOrder + StopPrice(20) 200 | stopOrder: StructStopOrder = Struct(Map(StructKey(class Symbol) -> Symbol(FTSE), StructKey(class Quantity) -> Quantity(100), StructKey(class StopPrice) -> StopPrice(20))) 201 | 202 | scala> filterFootsie(List(order, order.update(Symbol("CAC40")))) 203 | res10: List[strucs.Struct[Symbol with Quantity with strucs.Nil with Price]] = List(Struct(Map(StructKey(class Symbol) -> Symbol(FTSE), StructKey(class Quantity) -> Quantity(100), StructKey(class Price) -> Price(30)))) 204 | ``` 205 | 206 | -------------------------------------------------------------------------------- /TODO.md: -------------------------------------------------------------------------------- 1 | By Mid-November 2 | - documentation/test for get with options 3 | - improve imports structure (be inspired from argonaut) 4 | - Json example with adding common bits for mongo 5 | 6 | Easy 7 | 8 | ? - Example with pre-defined Struct type (say OrderSingle), verifies that it does not compile / that it compiles regardless of the insertion order 9 | 10 | 11 | Nice to have 12 | - use Monoid for Fix 13 | - example using disjunction (Either) 14 | - compare with Shapeless ? 15 | 16 | LONG-TERM - hard 17 | - look at the compiled byte-code for optimization: does the AnyVal have any effect ? 18 | - use packed arrays -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikaelv/strucs/61df544baa9989f06fb7c6147dde0b68f53a2f92/logo.png -------------------------------------------------------------------------------- /presentation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Title 5 | 6 | 32 | 33 | 34 | 35 | 37 | 43 | 44 | 45 | -------------------------------------------------------------------------------- /presentation.md: -------------------------------------------------------------------------------- 1 | class: center, middle 2 | # strucs 3 | ![logo](logo.png) 4 | 5 | Flexible data structures in scala 6 | 7 | https://github.com/mikaelv/strucs 8 | --- 9 | # Plan 10 | 1. Why ? 11 | 2. Adding fields 12 | 3. Getting fields 13 | 4. Composing Structs 14 | 5. Structural typing 15 | 6. Under the hood 16 | 7. Future 17 | --- 18 | # Why ? 19 | Case classes are not composable 20 | 21 | 22 | 23 | ```scala 24 | case class CreatePersonJsonPayload(name: String, age: Int) 25 | case class PersonModel(name: String, age: Int, address: Option[Address]) 26 | case class PersonDatabaseRow(id: String, name: String, age: Int, addressId: String) 27 | ``` 28 | * How can I define the common fields only once ? 29 | * Alternative: shapeless records 30 | * a Struct behaves more like a HSet 31 | * No FIX library for Scala 32 | ```scala 33 | val order = Struct.empty + BeginString.Fix42 + MsgType.OrderSingle + OrderQty(10) + Symbol("^FTSE") 34 | println(order.toFixMessageString) 35 | 8=FIX.4.2?9=20?35=D?38=10?55=^FTSE?10=036? 36 | ``` 37 | 38 | --- 39 | # Adding fields 40 | 41 | ```scala 42 | case class Name(v: String) extends AnyVal 43 | case class Age(v: Int) extends AnyVal 44 | ``` 45 | 46 | 47 | ```scala 48 | scala> val person = Struct.empty + Name("Mikael") + Age(39) 49 | person: strucs.Struct[strucs.Nil with Name with Age] = Struct(Map(StructKey(class Name) -> Name(Mikael), StructKey(class Age) -> Age(39))) 50 | 51 | scala> person.update(Name("Albert")) 52 | res0: strucs.Struct[strucs.Nil with Name with Age] = Struct(Map(StructKey(class Name) -> Name(Albert), StructKey(class Age) -> Age(39))) 53 | ``` 54 | ```scala 55 | scala> person + Name("Robert") 56 | :17: error: Cannot prove that strucs.Nil with Name with Age <:!< Name. 57 | person + Name("Robert") 58 | ^ 59 | ``` 60 | ??? 61 | Each field of the struct must have its own type. Referred to as Wrapper type. 62 | Inside a Struct, each field is uniquely identified by its type 63 | We will have a look at the internal structure later on 64 | --- 65 | # Getting fields 66 | ```scala 67 | scala> person.get[Name] 68 | res2: Name = Name(Mikael) 69 | ``` 70 | ```scala 71 | scala> person.get[Street] 72 | :17: error: not found: type Street 73 | person.get[Street] 74 | ^ 75 | ``` 76 | --- 77 | # Composing Structs 78 | ```scala 79 | type PersonData = Name with Age with Nil 80 | type Person = Struct[PersonData] 81 | val person: Person = Struct.empty + Name("Mikael") + Age(39) 82 | ``` 83 | 84 | 85 | ```scala 86 | type AddressData = Street with City with Nil 87 | type Address = Struct[AddressData] 88 | val address: Address = Struct(City("London")) + Street("52 Upper Street") 89 | ``` 90 | ```scala 91 | scala> type PersonAddress = Struct[PersonData with AddressData] 92 | defined type alias PersonAddress 93 | 94 | scala> val personAddress: PersonAddress = person ++ address 95 | personAddress: PersonAddress = Struct(Map(StructKey(class Name) -> Name(Mikael), StructKey(class Age) -> Age(39), StructKey(class City) -> City(London), StructKey(class Street) -> Street(52 Upper Street))) 96 | ``` 97 | --- 98 | # Structural typing 99 | ```scala 100 | scala> def adult[T <: Age with Name](struct: Struct[T]): String = { 101 | | struct.get[Name].v + 102 | | (if (struct.get[Age].v >= 18) " is an adult" else " is a child") 103 | | } 104 | adult: [T <: Age with Name](struct: strucs.Struct[T])String 105 | 106 | scala> adult(person) 107 | res4: String = Mikael is an adult 108 | ``` 109 | --- 110 | # Encoding/Decoding 111 | 112 | 113 | 114 | ```scala 115 | type MyOrder = Struct[OrderQty with Symbol with Nil] 116 | val json = """{"quantity":10,"symbol":"^FTSE"}""" 117 | ``` 118 | ```scala 119 | scala> val order = json.decodeOption[MyOrder] 120 | order: Option[MyOrder] = Some(Struct(Map(StructKey(class Symbol) -> Symbol(^FTSE), StructKey(class OrderQty) -> OrderQty(10)))) 121 | ``` 122 | ```scala 123 | scala> val fixOrder = order.get + BeginString.Fix42 + MsgType.OrderSingle 124 | fixOrder: strucs.Struct[strucs.fix.dict.fix42.OrderQty with strucs.fix.dict.fix42.Symbol with strucs.Nil with strucs.fix.dict.fix42.BeginString with strucs.fix.dict.fix42.MsgType] = Struct(Map(StructKey(class Symbol) -> Symbol(^FTSE), StructKey(class OrderQty) -> OrderQty(10), StructKey(class BeginString) -> BeginString(FIX.4.2), StructKey(class MsgType) -> MsgType(D))) 125 | 126 | scala> val fix = fixOrder.toFixMessageString 127 | fix: String = 8=FIX.4.2?9=20?35=D?38=10?55=^FTSE?10=036? 128 | ``` 129 | 130 | --- 131 | # Under the hood: Struct 132 | ```scala 133 | case class Struct[F](private val fields: Map[StructKey, Any]) { 134 | 135 | def +[T](value: T)(implicit k: StructKeyProvider[T], ev: F <:!< T ): 136 | Struct[F with T] = 137 | new Struct[F with T](fields + (k.key -> value)) 138 | 139 | def get[T](implicit k: StructKeyProvider[T], ev: F <:< T): T = 140 | fields(k.key).asInstanceOf[T] 141 | 142 | /** Get a subset of the fields */ 143 | def shrink[F2](implicit ev: F <:< F2): Struct[F2] = 144 | this.asInstanceOf[Struct[F2]] 145 | 146 | } 147 | 148 | object Struct { 149 | def empty: Struct[Nil] = new Struct[Nil](Map.empty) 150 | } 151 | ``` 152 | --- 153 | # Under the hood: CodecFix 154 | 155 | 156 | ```scala 157 | trait CodecFix[A] { 158 | def encode(a: A): FixElement 159 | def decode(fix: FixElement): Try[A] 160 | } 161 | ``` 162 | Sample implementations: 163 | ```scala 164 | case class OrderQty(v: BigDecimal) extends AnyVal 165 | object OrderQty { 166 | implicit val codec: CodecFix[OrderQty] = 167 | new TagCodecFix[OrderQty, BigDecimal](38) 168 | } 169 | 170 | case class Symbol(v: String) extends AnyVal 171 | object Symbol { 172 | implicit val codec: CodecFix[Symbol] = 173 | new TagCodecFix[Symbol, String](55) 174 | } 175 | ``` 176 | --- 177 | # Under the hood: ComposeCodec 178 | 179 | 180 | ```scala 181 | /** Defines how a Codec[Struct[_]] can be built using the codecs of its fields */ 182 | trait ComposeCodec[Codec[_]] { 183 | 184 | /** Build a Codec for an empty Struct */ 185 | def zero: Codec[Struct[Nil]] 186 | 187 | /** Build a Codec using a field codec a and a codec b for the rest */ 188 | def prepend[A : StructKeyProvider, B]( 189 | ca: Codec[A], 190 | cb: Codec[Struct[B]]): Codec[Struct[A with B]] 191 | } 192 | ``` 193 | ```scala 194 | def composeCodec: ComposeCodec[CodecFix] = ??? 195 | 196 | def codec1: CodecFix[Struct[Symbol with Nil]] = 197 | composeCodec.prepend[Symbol, Nil](Symbol.codec, composeCodec.zero) 198 | 199 | def codec2: CodecFix[Struct[OrderQty with Symbol with Nil]] = 200 | composeCodec.prepend[OrderQty, Symbol with Nil](OrderQty.codec, codec1) 201 | ``` 202 | 203 | --- 204 | ```scala 205 | object CodecFix { 206 | /** Automatically create a CodecFix for any Struct[A] 207 | * @tparam T mixin, each type M in the mixin 208 | must have an implicit CodecFix[M] in scope */ 209 | implicit def makeCodecFix[T]: CodecFix[Struct[T]] = 210 | macro ComposeCodec.macroImpl[CodecFix[_], T] 211 | 212 | 213 | implicit object ComposeCodecFix extends ComposeCodec[CodecFix] { 214 | /** Build a Codec for an empty Struct */ 215 | def zero: CodecFix[Struct[Nil]] = new CodecFix[Struct[Nil]] { 216 | override def encode(a: Struct[Nil]): FixElement = FixGroup.empty 217 | override def decode(fix: FixElement): Try[Struct[Nil]] = Success(Struct.empty) 218 | } 219 | 220 | /** Build a Codec using a field codec a and a codec b for the rest */ 221 | def prepend[A: StructKeyProvider, B]( 222 | ca: CodecFix[A], 223 | cb: CodecFix[Struct[B]]) = new CodecFix[Struct[A with B]] { 224 | def encode(a: Struct[A with B]): FixElement = { 225 | val bfix = cb.encode(a.shrink[B]) 226 | val afix = ca.encode(a.get[A]) 227 | afix + bfix 228 | } 229 | 230 | def decode(fix: FixElement): Try[Struct[A with B]] = { 231 | for { 232 | structb <- cb.decode(fix) 233 | a <- ca.decode(fix) 234 | } yield structb.+[A](a) 235 | } 236 | } 237 | } 238 | } 239 | 240 | ``` 241 | --- 242 | # Under the hood: argonaut.DecodeJson 243 | 244 | The implementation of ComposeCodec is very similar 245 | ```scala 246 | object StrucsDecodeJson { 247 | implicit def makeDecodeJson[T]: DecodeJson[Struct[T]] = 248 | macro ComposeCodec.macroImpl[DecodeJson[_], T] 249 | 250 | implicit object ComposeDecodeJson extends ComposeCodec[DecodeJson] { 251 | /** Build a Codec for an empty Struct */ 252 | def zero = new DecodeJson[Struct[Nil]] { 253 | def decode(c: HCursor): DecodeResult[Struct[Nil]] = 254 | DecodeResult.ok(Struct.empty) 255 | } 256 | 257 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 258 | def prepend[A: StructKeyProvider, B]( 259 | ca: DecodeJson[A], 260 | cb: DecodeJson[Struct[B]]) = new DecodeJson[Struct[A with B]] { 261 | def decode(c: HCursor): DecodeResult[Struct[A with B]] = { 262 | for { 263 | structb <- cb.decode(c) 264 | a <- ca.decode(c) 265 | } yield structb.+[A](a) 266 | } 267 | } 268 | } 269 | } 270 | ``` 271 | --- 272 | # Future developments 273 | * Benchmarks & Optimizations 274 | * Struct <==> case class 275 | * Struct <==> Avro 276 | * Struct <==> Protobuf 277 | * Typed Spark DataFrame ? 278 | 279 | ### Contributions are welcome ! 280 | --- 281 | class: center, middle 282 | # Questions ? 283 | 284 | 285 | Mikael Valot 286 | 287 | twitter: @leakimav 288 | 289 | 290 | https://github.com/mikaelv/strucs 291 | 292 | 293 | --- 294 | -------------------------------------------------------------------------------- /project/Build.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | import Keys._ 3 | 4 | object BuildSettings { 5 | val buildSettings = Defaults.defaultSettings ++ Seq( 6 | organization := "strucs", 7 | version := "1.0.0", 8 | scalaVersion := "2.11.6", 9 | resolvers += Resolver.sonatypeRepo("snapshots"), 10 | resolvers += Resolver.sonatypeRepo("releases"), 11 | resolvers += "Apache snapshot" at "https://repository.apache.org/content/groups/snapshots/", 12 | scalacOptions ++= Seq() 13 | ) 14 | } 15 | 16 | object StrucsBuild extends Build { 17 | import BuildSettings._ 18 | 19 | lazy val root: Project = Project( 20 | "strucs", 21 | file("."), 22 | settings = buildSettings 23 | ) aggregate(json, fix, core, demo) 24 | 25 | lazy val json: Project = Project( 26 | "strucs-json", 27 | file("strucs-json"), 28 | settings = buildSettings ++ Seq( 29 | libraryDependencies += "io.argonaut" %% "argonaut" % "6.1", 30 | libraryDependencies ++= scalaTest ++ joda 31 | ) 32 | ) dependsOn(core) 33 | 34 | lazy val fix: Project = Project( 35 | "strucs-fix", 36 | file("strucs-fix"), 37 | settings = buildSettings ++ Seq( 38 | libraryDependencies ++= scalaTest ++ joda 39 | ) 40 | ) dependsOn(core) 41 | 42 | 43 | lazy val spark: Project = Project( 44 | "strucs-spark", 45 | file("strucs-spark"), 46 | settings = buildSettings ++ Seq( 47 | libraryDependencies += "org.apache.spark" %% "spark-core" % "1.5.2", 48 | libraryDependencies += "org.apache.spark" %% "spark-sql" % "1.5.2", 49 | libraryDependencies ++= scalaTest ++ joda 50 | ) 51 | ) dependsOn(core) 52 | 53 | 54 | 55 | lazy val core: Project = Project( 56 | "strucs-core", 57 | file("strucs-core"), 58 | settings = buildSettings ++ Seq( 59 | libraryDependencies ++= scalaTest ++ Seq( 60 | "org.scala-lang" % "scala-reflect" % "2.11.6" 61 | )) 62 | ) 63 | 64 | lazy val demo: Project = Project("strucs-demo", file("strucs-demo")) 65 | .dependsOn(core, fix, json) 66 | .settings(buildSettings) 67 | .settings(tut.Plugin.tutSettings) 68 | 69 | lazy val scalaTest = Seq("org.scalatest" % "scalatest_2.11" % "2.2.4" % "test") 70 | 71 | lazy val joda = Seq("joda-time" % "joda-time" % "2.8") 72 | } -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.8 -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | logLevel := Level.Warn 2 | 3 | resolvers += Resolver.url( 4 | "tpolecat-sbt-plugin-releases", 5 | url("http://dl.bintray.com/content/tpolecat/sbt-plugin-releases"))( 6 | Resolver.ivyStylePatterns) 7 | 8 | addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.4.0") -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/ComposeCodec.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import scala.language.experimental.macros 4 | import scala.languageFeature.higherKinds 5 | import scala.reflect.macros.blackbox 6 | 7 | 8 | 9 | /** Defines how a Codec[Struct[_]] can be built using the codecs of its fields */ 10 | trait ComposeCodec[Codec[_]] { 11 | 12 | /** Build a Codec for an empty Struct */ 13 | def zero: Codec[Struct[Nil]] 14 | 15 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 16 | def prepend[A : StructKeyProvider, B](ca: Codec[A], cb: => Codec[Struct[B]]): Codec[Struct[A with B]] 17 | 18 | } 19 | 20 | trait Monoid[F] { 21 | def zero: F 22 | def prepend(a: F, b: F): F 23 | } 24 | 25 | 26 | 27 | object ComposeCodec { 28 | /** Conversion from/to a function A=>T to/from an Encode[A] */ 29 | trait ConvertEncode[Encode[_], T] { 30 | def fromFunc[A](encode: A => T): Encode[A] 31 | def toFunc[A](enc: Encode[A]): A => T 32 | } 33 | 34 | /** If we know how to compose the encoded values T (using Monoid[T]), 35 | * we can define how to compose an Encoder to T 36 | */ 37 | def makeComposeCodec[Enc[_], T](implicit tMonoid: Monoid[T], conv: ConvertEncode[Enc, T]) = new ComposeCodec[Enc] { 38 | 39 | /** Build a Codec for an empty Struct */ 40 | override def zero: Enc[Struct[Nil]] = conv.fromFunc {a: Struct[Nil] => tMonoid.zero} 41 | 42 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 43 | override def prepend[A: StructKeyProvider, B](ca: Enc[A], 44 | cb: => Enc[Struct[B]]): Enc[Struct[A with B]] = conv.fromFunc { s: Struct[A with B] => 45 | val bencoded = conv.toFunc[Struct[B]](cb)(s.shrink[B]) 46 | val aencoded = conv.toFunc[A](ca)(s.get[A]) 47 | tMonoid.prepend(aencoded, bencoded) 48 | } 49 | } 50 | 51 | 52 | /** Make a Codec for a Struct[T], by calling the ComposeCodec for each constituent of T */ 53 | def makeCodec[Codec[_], T]: Codec[Struct[T]] = macro macroImpl[Codec[_], T] 54 | 55 | 56 | 57 | def macroImpl[Codec: c.WeakTypeTag, T : c.WeakTypeTag](c: blackbox.Context) = { 58 | def info(msg: String) = c.info(c.enclosingPosition, "strucs.ComposeCodec - "+msg, true) 59 | 60 | // Extract constituents 61 | import c.universe._ 62 | val typeTag: c.universe.WeakTypeTag[T] = implicitly[WeakTypeTag[T]] 63 | //info("creating codec for type: "+typeTag.tpe.toString) 64 | 65 | val types = FieldExtractor.extractTypes(typeTag.tpe)(c.universe) 66 | //info("extracted types: "+types.mkString(", ")) 67 | 68 | val codecTypeTag = implicitly[WeakTypeTag[Codec]] 69 | val codecSymbol = codecTypeTag.tpe.typeSymbol 70 | 71 | def implicitCodec(tpe: Type): Tree = q"implicitly[$codecSymbol[$tpe]]" 72 | 73 | val composed = types.foldLeft[Tree](q"comp.zero"){ case (tree, tpe) => 74 | q"comp.prepend(${implicitCodec(tpe.asInstanceOf[Type])}, $tree)" 75 | } 76 | val codec = q"val comp = implicitly[strucs.ComposeCodec[$codecSymbol]]; $composed.asInstanceOf[$codecSymbol[Struct[${typeTag.tpe}]]]" 77 | //info("codec = "+codec.toString) 78 | codec 79 | } 80 | 81 | 82 | 83 | } 84 | 85 | 86 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/FieldExtractor.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | 4 | import scala.reflect.api.Universe 5 | 6 | /** Extracts the constituents of a Mixin: String with Int => List(String, Int) */ 7 | object FieldExtractor { 8 | 9 | def extractTypes(mixin: Universe#Type, acc: List[Universe#Type] = List.empty)(implicit universe: Universe): List[Universe#Type] = { 10 | import universe._ 11 | val nilSymbol = typeOf[Nil].typeSymbol 12 | mixin match { 13 | // Filter out Nil 14 | case t if t.typeSymbol == nilSymbol => acc 15 | // Constituents of a Mixin => recursive call 16 | case RefinedType(types, _) => types.foldLeft(acc) { (accTypes, tpe) => extractTypes(tpe, accTypes) } 17 | // type reference, such as myStrut.Mixin => we need to resolve the reference 18 | case TypeRef(pre @ SingleType(ppre, psym), sym, args) if !psym.isModule => extractTypes(sym.typeSignatureIn(pre), acc) 19 | case _ => mixin +: acc 20 | } 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/Nil.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | /** The only type in an empty Struct */ 4 | trait Nil 5 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/Struct.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | /** 4 | * Extensible data structure with type safety 5 | * @tparam F mixin of all the fields types 6 | */ 7 | case class Struct[F](private[strucs] val fields: Map[StructKey, Any]) { 8 | type Mixin = F 9 | 10 | def +[T](value: T)(implicit k: StructKeyProvider[T], ev: F <:!< T ) = add[T](value) 11 | 12 | /** Adds a field. Pass an Option[T] if the field is optional */ 13 | def add[T](value: T)(implicit k: StructKeyProvider[T], ev: F <:!< T ) : Struct[F with T] = new Struct[F with T](fields + (k.key -> value)) 14 | 15 | /** Updates an existing field */ 16 | def update[T >: F](value: T)(implicit k: StructKeyProvider[T]) : Struct[F] = new Struct[F](fields + (k.key -> value)) 17 | 18 | def ++[F2](rec: Struct[F2]): Struct[F with F2] = merge[F2](rec) 19 | 20 | /** Add all fields from another Struct */ 21 | def merge[F2](rec: Struct[F2]): Struct[F with F2] = new Struct[F with F2](fields ++ rec.fields) 22 | 23 | /** Get a field */ 24 | def get[T >: F](implicit k: StructKeyProvider[T]): T = fields(k.key).asInstanceOf[T] 25 | 26 | /** Get a subset of the fields */ 27 | def shrink[F2](implicit ev: F <:< F2): Struct[F2] = { 28 | // the internal Map is kept untouched, but the evidence parameters of all accessors guarantee that 29 | // removed fields will not be accessible anymore 30 | this.asInstanceOf[Struct[F2]] 31 | } 32 | 33 | } 34 | 35 | object Struct { 36 | def apply[T](t: T)(implicit k: StructKeyProvider[T]): Struct[T with Nil] = new Struct(Map(k.key -> t)) 37 | def empty: Struct[Nil] = new Struct[Nil](Map.empty) 38 | } 39 | 40 | 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/StructKey.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | /** 4 | * Key for retrieving a field in a Struct 5 | */ 6 | case class StructKey(value: String) extends AnyVal 7 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/StructKeyProvider.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import scala.reflect.runtime.universe._ 4 | 5 | 6 | /** Type class that gives a key of a given type, for usage in the Struct Map */ 7 | case class StructKeyProvider[T](key: StructKey) 8 | 9 | trait LowPriorityStructKeyProviderImplicits { 10 | /** Provides the implicit value for StructKey[Option[T]] when the implicit value for StructKey[T] is in scope */ 11 | implicit def convertToOption[T](implicit rk: StructKeyProvider[T]): StructKeyProvider[Option[T]] = new StructKeyProvider[Option[T]](StructKey(rk.key.value)) 12 | implicit def convertToSome[T](implicit rk: StructKeyProvider[T]): StructKeyProvider[Some[T]] = new StructKeyProvider[Some[T]](StructKey(rk.key.value)) 13 | 14 | 15 | /** Provides a key using the Class name. Beware of ADT types: use Option(x) instead of Some(x) */ 16 | implicit def simpleNameStructKeyProvider[A : TypeTag]: StructKeyProvider[A] = StructKeyProvider[A](StructKey(implicitly[TypeTag[A]].tpe.typeSymbol.toString)) 17 | // TODO make sure two fields with the same name in different packages do not clash 18 | 19 | } 20 | 21 | 22 | /** 23 | * 24 | */ 25 | object StructKeyProvider extends LowPriorityStructKeyProviderImplicits { 26 | 27 | } 28 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/Wrapper.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import scala.language.experimental.macros 4 | import scala.reflect.macros.blackbox 5 | 6 | /** typeclass for wrapper types, such as case class Name(v: String) 7 | * a wrapper can be built from a value, and we can extract a value from it 8 | * @tparam W type of the Wrapper 9 | * @tparam V type of the value 10 | */ 11 | trait Wrapper[W, V] extends Serializable { 12 | def make(v: V): Option[W] // TODO use Try or Either ? Look at Spray or akka-http for error handling 13 | def value(w: W): V 14 | } 15 | 16 | object Wrapper { 17 | def macroImpl[W: c.WeakTypeTag, V : c.WeakTypeTag](c: blackbox.Context) = { 18 | import c.universe._ 19 | val wsym = c.weakTypeOf[W].typeSymbol 20 | val vtype = c.weakTypeOf[V] 21 | 22 | if (!wsym.isClass || !wsym.asClass.isCaseClass) c.abort(c.enclosingPosition, s"$wsym is not a case class. Please define a Wrapper[$wsym, ?] manually") 23 | val fields = wsym.typeSignature.decls.toList.collect{ case x: TermSymbol if x.isVal && x.isCaseAccessor => x } 24 | if (fields.size != 1) c.abort(c.enclosingPosition, s"$wsym must have exactly one field. Please define a Wrapper[$wsym, ?] manually") 25 | val field = fields.head.getter 26 | 27 | val expr = q""" 28 | new strucs.Wrapper[$wsym, $vtype] { 29 | def make(v: $vtype): Option[$wsym] = Some(new $wsym(v)) 30 | def value(w: $wsym): $vtype = w.$field 31 | } 32 | """ 33 | //c.info(c.enclosingPosition, expr.toString, true) 34 | expr 35 | } 36 | 37 | /** Automatically creates a Wrapper for case classes */ 38 | implicit def materializeWrapper[W, V]: Wrapper[W, V] = macro macroImpl[W, V] 39 | 40 | // TODO can we get rid of this ? It simplifies declaration a Wrapper in companion objects, but it looks suspicious. Maybe wrapper could be a class ? 41 | // Basically I want to transform a function value into a method => google it ! 42 | /** Creates a new Wrapper implementation */ 43 | def apply[W, V](pMake: V => Option[W], pValue: W => V): Wrapper[W, V] = new Wrapper[W, V] { 44 | override def make(v: V): Option[W] = pMake(v) 45 | 46 | override def value(w: W): V = pValue(w) 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /strucs-core/src/main/scala/strucs/package.scala: -------------------------------------------------------------------------------- 1 | /** 2 | * Global definitions 3 | */ 4 | package object strucs { 5 | 6 | // Thanks to Regis Jean-Gilles in http://stackoverflow.com/questions/6909053/enforce-type-difference 7 | @annotation.implicitNotFound(msg = "Cannot prove that ${A} <:!< ${B}.") 8 | trait <:!<[A,B] 9 | object <:!< { 10 | class Impl[A, B] 11 | object Impl { 12 | implicit def nsub[A, B] : A Impl B = null 13 | implicit def nsubAmbig1[A, B>:A] : A Impl B = null 14 | implicit def nsubAmbig2[A, B>:A] : A Impl B = null 15 | } 16 | 17 | implicit def foo[A,B]( implicit e: A Impl B ): A <:!< B = null 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /strucs-core/src/test/scala/strucs/EncoderSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import org.scalactic.TypeCheckedTripleEquals 4 | import org.scalatest.{FlatSpec, Matchers} 5 | import strucs.ComposeCodec._ 6 | import strucs.EncoderSpec._ 7 | 8 | /** 9 | * 10 | */ 11 | class EncoderSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 12 | 13 | "An EncodeCommaSeparated[Struct[Name with Age with City]]" should "be created with a macro" in { 14 | // TODO add an enum 15 | val person = Struct.empty + Name("Bart") + Age(10) + City("Springfield") 16 | 17 | val encoder: EncodeCommaSeparated[Struct[Name with Age with City with Nil]] = ComposeCodec.makeCodec[EncodeCommaSeparated, Name with Age with City with Nil] 18 | encoder.encode(person) should === ("Bart, 10, Springfield") 19 | } 20 | } 21 | 22 | object EncoderSpec { 23 | /** 24 | * Dummy codec: outputs the content of each field, separated by commas 25 | */ 26 | trait EncodeCommaSeparated[A] { 27 | def encode(a: A): String 28 | } 29 | 30 | 31 | object EncodeCommaSeparated { 32 | 33 | implicit val monoid = new Monoid[String] { 34 | override def zero: String = "" 35 | 36 | override def prepend(a: String, b: String): String = 37 | if (a == "") b 38 | else if (b == "") a 39 | else a + ", " + b 40 | } 41 | 42 | implicit val trans = new ConvertEncode[EncodeCommaSeparated, String] { 43 | override def fromFunc[A](_encode: (A) => String) = new EncodeCommaSeparated[A] { 44 | override def encode(a: A): String = _encode(a) 45 | } 46 | 47 | override def toFunc[A](enc: EncodeCommaSeparated[A]) = enc.encode 48 | } 49 | implicit val composeEncode: ComposeCodec[EncodeCommaSeparated] = ComposeCodec.makeComposeCodec[EncodeCommaSeparated, String] 50 | 51 | } 52 | 53 | 54 | case class Name(v: String) extends AnyVal 55 | case class Age(v: Int) extends AnyVal 56 | case class City(v: String) extends AnyVal 57 | 58 | implicit val nameEncoder: EncodeCommaSeparated[Name] = new EncodeCommaSeparated[Name] { 59 | override def encode(t: Name): String = t.v 60 | } 61 | implicit val ageEncoder: EncodeCommaSeparated[Age] = new EncodeCommaSeparated[Age] { 62 | override def encode(t: Age): String = t.v.toString 63 | } 64 | implicit val cityEncoder: EncodeCommaSeparated[City] = new EncodeCommaSeparated[City] { 65 | override def encode(t: City): String = t.v.toString 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /strucs-core/src/test/scala/strucs/FieldExtractorSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import org.scalactic.TypeCheckedTripleEquals 4 | import org.scalatest.{FlatSpec, Matchers} 5 | import strucs.FieldExtractorSpec.Field1 6 | 7 | class FieldExtractorSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 8 | implicit val universe = scala.reflect.runtime.universe 9 | import universe._ 10 | 11 | 12 | "a FieldExtractor" should "extract the scalar fields from a mixin" in { 13 | val tpe = typeOf[Field1 with Int] 14 | val symbols = FieldExtractor.extractTypes(tpe) 15 | symbols shouldBe List(typeOf[Int], typeOf[Field1]) 16 | } 17 | 18 | it should "extract the scalar fields from a type reference" in { 19 | val struct = Struct.empty + "Hello" + 1 20 | val tpe = typeOf[struct.Mixin] 21 | val symbols = FieldExtractor.extractTypes(tpe) 22 | symbols shouldBe List(typeOf[Int], typeOf[java.lang.String]) 23 | 24 | } 25 | 26 | it should "extract the Struct fields from a mixin" in { 27 | val types = FieldExtractor.extractTypes(typeOf[String with Struct[String with Int]]) 28 | val expected = List(typeOf[Struct[String with Int]], typeOf[String]) 29 | types.map(_.toString) shouldBe expected.map(_.toString) 30 | } 31 | 32 | 33 | } 34 | 35 | object FieldExtractorSpec { 36 | case class Field1(v: String) extends AnyVal 37 | } 38 | -------------------------------------------------------------------------------- /strucs-core/src/test/scala/strucs/StructSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs 2 | 3 | import org.scalactic.TypeCheckedTripleEquals 4 | import org.scalatest.{FlatSpec, Matchers} 5 | 6 | import scala.reflect.runtime.universe._ 7 | 8 | /** 9 | * 10 | */ 11 | class StructSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 12 | 13 | case class Name(v: String) 14 | case class Age(v: Int) 15 | case class City(v: String) 16 | 17 | val name = Name("Omer") 18 | val city = City("Springfield") 19 | val name1 = Name("Marge") 20 | val age = Age(45) 21 | 22 | def baseStruct = Struct(name).add(age) 23 | 24 | "A Struct" should "add new fields" in { 25 | val s = baseStruct 26 | implicitly[s.type <:< Struct[Name with Age with Nil]] 27 | } 28 | 29 | it should "get an added field" in { 30 | val s = baseStruct 31 | s.get[Name] should ===(name) 32 | s.get[Age] should ===(age) 33 | } 34 | 35 | it should "forbid to add an existing field" in { 36 | val s = baseStruct 37 | assertTypeError("s.add(Age(10))") 38 | } 39 | 40 | it should "forbid to get or update a non-added field" in { 41 | val s = baseStruct 42 | assertTypeError("s.get[City]") 43 | assertTypeError("s.update(City(\"Paris\"))") 44 | } 45 | 46 | it should "update a field" in { 47 | val s = baseStruct.update(Age(10)) 48 | s.get[Age].v should ===(10) 49 | } 50 | 51 | it should "allow to write functions that accept a Struct with more Fields than the type parameter" in { 52 | def fn[T <: Name with Age](struct: Struct[T]): Name = { 53 | struct.get[Name] 54 | } 55 | 56 | fn(baseStruct.add(city)) should be(name) 57 | } 58 | 59 | it should "manage optional fields" ignore { 60 | // TODO 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /strucs-demo/src/main/tut/README.md: -------------------------------------------------------------------------------- 1 | # Strucs - Flexible data structures in Scala 2 | 3 | Strucs is a lightweight library that allows to manipulate, encode and decode flexible data structures while maintaining immutability and type safety. 4 | 5 | A Struct is analogous to a case class that can accept new fields dynamically. 6 | 7 | Using the strucs extensions, a single struc instance can be easily serialized/deserialized to various formats, such as JSON, [FIX protocol](https://en.wikipedia.org/wiki/Financial_Information_eXchange), Protobuf, ... 8 | 9 | [Slides for Scala eXchange 2015](https://rawgit.com/mikaelv/strucs/master/presentation.html#1) 10 | 11 | ## Quick start 12 | 13 | ### Create/Add/Update 14 | 15 | ```tut:silent 16 | import strucs._ 17 | 18 | case class Ticker(v: String) extends AnyVal 19 | case class Quantity(v: BigDecimal) extends AnyVal 20 | case class Price(v: BigDecimal) extends AnyVal 21 | ``` 22 | ```tut 23 | val order = Struct(Ticker("^FTSE")) 24 | val order2 = order.add(Quantity(5)) 25 | order2.get[Ticker] 26 | val order3 = order2.update(Ticker("^FCHI")) 27 | order3.get[Ticker] 28 | ``` 29 | order3 does not have a Price field. Any attempt to access it is rejected by the compiler. 30 | ```tut:fail 31 | order3.get[Price] 32 | ``` 33 | 34 | ### Structural typing 35 | *When I see a bird that walks like a duck and swims like a duck and quacks like a duck, I call that bird a duck.* 36 | 37 | Let's define a function that accepts any Struct that has two specific fields. 38 | ```tut 39 | def totalPrice[T <: Quantity with Price](struct: Struct[T]): BigDecimal = { 40 | struct.get[Quantity].v * struct.get[Price].v 41 | } 42 | ``` 43 | A call with an incompatible Struct is rejected by the compiler: 44 | ```tut:fail 45 | totalPrice(order3) 46 | ``` 47 | But succeeds when we add the required field: 48 | ```tut 49 | totalPrice(order3.add(Price(10))) 50 | ``` 51 | 52 | 53 | ### Encoding/Decoding 54 | Provided that the encoders/decoders for the fields are in scope, the same struct instance can be encoded/decoded to various formats: 55 | ```tut:silent 56 | import strucs.json._ 57 | import strucs.fix._ 58 | import strucs.fix.dict.fix42._ // defines common FIX 4.2 tags with their codec 59 | import CodecFix._ 60 | import StrucsCodecJson._ 61 | import StrucsEncodeJson._ 62 | import StrucsDecodeJson._ 63 | import argonaut._ 64 | import Argonaut._ 65 | 66 | type MyOrder = Struct[OrderQty with Symbol with Nil] 67 | val order: MyOrder = Struct.empty + OrderQty(10) + Symbol("^FTSE") 68 | ``` 69 | The order can be encoded/decoded to/from FIX if we add the required tags BeginString and MsgType. 70 | ```tut 71 | val fixOrder = order + BeginString.Fix42 + MsgType.OrderSingle 72 | val fix = fixOrder.toFixMessageString 73 | fix.toStruct[fixOrder.Mixin] 74 | ``` 75 | If we define the [Argonaut](http://argonaut.io/) Json codecs for Symbol and OrderQty, 76 | ```tut:silent 77 | implicit val symbolCodecJson: CodecJson[Symbol] = StrucsCodecJson.fromWrapper[Symbol, String]("symbol") 78 | implicit val orderQtyCodecJson: CodecJson[OrderQty] = StrucsCodecJson.fromWrapper[OrderQty, BigDecimal]("quantity") 79 | ``` 80 | We can encode/decode our order to/from Json 81 | ```tut 82 | val json = order.toJsonString 83 | json.decodeOption[MyOrder] 84 | ``` 85 | 86 | ### More examples 87 | Please check out the unit tests for more usage examples. 88 | 89 | ## Motivation 90 | Consider a program which manages Orders. 91 | A common approach would be to use case classes with simple types for its fields: 92 | ```tut 93 | case class SimpleOrder(symbol: String, quantity: BigDecimal, price: BigDecimal) 94 | ``` 95 | However, using simple types such as String, Int, BigDecimal, ... everywhere can rapidly make the code confusing and fragile. 96 | Imagine we have to extract the price and quantity of all the FTSE orders 97 | ```tut 98 | def simpleFootsieOrders(orders: List[SimpleOrder]): List[(BigDecimal, BigDecimal)] = 99 | orders collect { 100 | case SimpleOrder(sym, q, p) if sym == "^FTSE" => (q, p) 101 | } 102 | ``` 103 | If I do not get the argument order right (or if it has been refactored), the code above will compile but will not do what I expect. 104 | Furthermore, the return type is List[(BigDecimal, BigDecimal)], which is unclear for the users of the function. 105 | 106 | 107 | We need stronger types to make our code clearer and safer. You you might want to use [value classes](http://docs.scala-lang.org/overviews/core/value-classes.html) as follows: 108 | ```tut:silent 109 | case class Symbol(v: String) extends AnyVal 110 | val FTSE = Symbol("FTSE") 111 | case class Quantity(v: BigDecimal) extends AnyVal 112 | case class Price(v: BigDecimal) extends AnyVal 113 | 114 | case class TypedOrder(symbol: Symbol, quantity: Quantity, price: Price) 115 | ``` 116 | ```tut 117 | def typedFootsieOrders(orders: List[TypedOrder]): List[(Quantity, Price)] = 118 | orders.collect { 119 | case TypedOrder(sym, q, p) if sym == FTSE => (q, p) 120 | } 121 | ``` 122 | Now the return type is much clearer and safer, and my matching expression is safer as well: 123 | I cannot inadvertently swap arguments without getting a compilation error. 124 | 125 | On the other hand, we now observe that the names of the attributes are redundant with their types. 126 | It would be nicer if we could declare them only once. 127 | Also, I cannot easily reuse a set of fields, such as symbol and quantity, in another case class. I need to redefine the class with all its fields: 128 | ```tut 129 | case class StopPrice(v: BigDecimal) 130 | case class StopOrder(symbol: Symbol, quantity: Quantity, price: StopPrice) 131 | ``` 132 | If I then want to define a function that accepts StopOrder or TypedOrder, I would typically define a common trait that these classes will extend. 133 | ```tut 134 | trait Order { 135 | def symbol: Symbol 136 | } 137 | def filterFootsie(orders: List[Order]): List[Order] = orders.filter(_.symbol == FTSE) 138 | ``` 139 | This leads to some duplication, and it may not even be feasible if TypedOrder is defined in a third party library. 140 | 141 | With strucs, we can define the same as follows: 142 | ```tut:silent 143 | type BaseOrderType = Symbol with Quantity with Nil 144 | type StructOrder = Struct[BaseOrderType with Price] 145 | type StructStopOrder = Struct[BaseOrderType with StopPrice] 146 | def filterFootsie[T <: Symbol](orders: List[Struct[T]]) = 147 | orders.filter(_.get[Symbol] == FTSE) 148 | ``` 149 | The different "order" types are now **composable**. I can define an abstraction BaseOrder, and reuse it to define other Order types. 150 | Also, I do not have to declare field names anymore, as I use only the types of the fields to access them. 151 | This composition capability also applies to instances: 152 | 153 | ```tut 154 | val baseOrder = Struct.empty + FTSE + Quantity(100) 155 | val order: StructOrder = baseOrder + Price(30) 156 | val stopOrder: StructStopOrder = baseOrder + StopPrice(20) 157 | filterFootsie(List(order, order.update(Symbol("CAC40")))) 158 | 159 | ``` 160 | 161 | -------------------------------------------------------------------------------- /strucs-demo/src/main/tut/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mikaelv/strucs/61df544baa9989f06fb7c6147dde0b68f53a2f92/strucs-demo/src/main/tut/logo.png -------------------------------------------------------------------------------- /strucs-demo/src/main/tut/presentation.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Title 5 | 6 | 32 | 33 | 34 | 35 | 37 | 43 | 44 | -------------------------------------------------------------------------------- /strucs-demo/src/main/tut/presentation.md: -------------------------------------------------------------------------------- 1 | class: center, middle 2 | # strucs 3 | ![logo](logo.png) 4 | 5 | Flexible data structures in scala 6 | 7 | https://github.com/mikaelv/strucs 8 | --- 9 | # Plan 10 | 1. Why ? 11 | 2. Adding fields 12 | 3. Getting fields 13 | 4. Composing Structs 14 | 5. Structural typing 15 | 6. Under the hood 16 | 7. Future 17 | --- 18 | # Why ? 19 | Case classes are not composable 20 | 21 | ```tut:invisible 22 | case class Address(street: String) 23 | ``` 24 | ```tut:silent 25 | case class CreatePersonJsonPayload(name: String, age: Int) 26 | case class PersonModel(name: String, age: Int, address: Option[Address]) 27 | case class PersonDatabaseRow(id: String, name: String, age: Int, addressId: String) 28 | ``` 29 | * How can I define the common fields only once ? 30 | * Alternative: shapeless records 31 | * a Struct behaves more like a HSet 32 | * No FIX library for Scala 33 | ```scala 34 | val order = Struct.empty + BeginString.Fix42 + MsgType.OrderSingle + 35 | OrderQty(10) + Symbol("^FTSE") 36 | println(order.toFixMessageString) 37 | 8=FIX.4.2?9=20?35=D?38=10?55=^FTSE?10=036? 38 | ``` 39 | 40 | --- 41 | # Adding fields 42 | 43 | ```tut:silent 44 | case class Name(v: String) extends AnyVal 45 | case class Age(v: Int) extends AnyVal 46 | ``` 47 | ```tut:invisible 48 | import strucs._ 49 | ``` 50 | ```tut 51 | val person = Struct.empty + Name("Mikael") + Age(39) 52 | person.update(Name("Albert")) 53 | ``` 54 | ```tut:fail 55 | person + Name("Robert") 56 | ``` 57 | ??? 58 | Each field of the struct must have its own type. Referred to as Wrapper type. 59 | Inside a Struct, each field is uniquely identified by its type 60 | We will have a look at the internal structure later on 61 | --- 62 | # Getting fields 63 | ```tut 64 | person.get[Name] 65 | ``` 66 | ```tut:fail 67 | person.get[Street] 68 | ``` 69 | --- 70 | # Composing Structs 71 | ```tut:silent 72 | type PersonData = Name with Age with Nil 73 | type Person = Struct[PersonData] 74 | val person: Person = Struct.empty + Name("Mikael") + Age(39) 75 | ``` 76 | ```tut:invisible 77 | case class Street(v: String) extends AnyVal 78 | case class City(v: String) extends AnyVal 79 | ``` 80 | ```tut:silent 81 | type AddressData = Street with City with Nil 82 | type Address = Struct[AddressData] 83 | val address: Address = Struct(City("London")) + Street("52 Upper Street") 84 | ``` 85 | ```tut 86 | type PersonAddress = Struct[PersonData with AddressData] 87 | val personAddress: PersonAddress = person ++ address 88 | ``` 89 | --- 90 | # Structural typing 91 | ```tut 92 | def adult[T <: Age with Name](struct: Struct[T]): String = { 93 | struct.get[Name].v + 94 | (if (struct.get[Age].v >= 18) " is an adult" else " is a child") 95 | } 96 | adult(person) 97 | ``` 98 | --- 99 | # Encoding/Decoding 100 | 101 | ```tut:invisible 102 | import strucs.json._ 103 | import strucs.fix._ 104 | import strucs.fix.dict.fix42._ // defines common FIX 4.2 tags with their codec 105 | import CodecFix._ 106 | import StrucsCodecJson._ 107 | import StrucsEncodeJson._ 108 | import StrucsDecodeJson._ 109 | import argonaut._ 110 | import Argonaut._ 111 | implicit val symbolCodecJson: CodecJson[Symbol] = StrucsCodecJson.fromWrapper[Symbol, String]("symbol") 112 | implicit val orderQtyCodecJson: CodecJson[OrderQty] = StrucsCodecJson.fromWrapper[OrderQty, BigDecimal]("quantity") 113 | ``` 114 | ```tut:silent 115 | type MyOrder = Struct[OrderQty with Symbol with Nil] 116 | val json = """{"quantity":10,"symbol":"^FTSE"}""" 117 | ``` 118 | ```tut 119 | val order = json.decodeOption[MyOrder] 120 | ``` 121 | ```tut 122 | val fixOrder = order.get + BeginString.Fix42 + MsgType.OrderSingle 123 | val fix = fixOrder.toFixMessageString 124 | ``` 125 | 126 | --- 127 | # Under the hood: Struct 128 | ```tut:silent 129 | case class Struct[F](private val fields: Map[StructKey, Any]) { 130 | 131 | def +[T](value: T)(implicit k: StructKeyProvider[T], ev: F <:!< T ): 132 | Struct[F with T] = 133 | new Struct[F with T](fields + (k.key -> value)) 134 | 135 | def get[T](implicit k: StructKeyProvider[T], ev: F <:< T): T = 136 | fields(k.key).asInstanceOf[T] 137 | 138 | /** Get a subset of the fields */ 139 | def shrink[F2](implicit ev: F <:< F2): Struct[F2] = 140 | this.asInstanceOf[Struct[F2]] 141 | 142 | } 143 | 144 | object Struct { 145 | def empty: Struct[Nil] = new Struct[Nil](Map.empty) 146 | } 147 | ``` 148 | --- 149 | # Under the hood: CodecFix 150 | ```tut:invisible 151 | import scala.util.{Failure, Success, Try} 152 | import scala.language.experimental.macros 153 | import strucs.fix.{CodecFix => _} 154 | ``` 155 | ```scala 156 | trait CodecFix[A] { 157 | def encode(a: A): FixElement 158 | def decode(fix: FixElement): Try[A] 159 | } 160 | ``` 161 | Sample implementations: 162 | ```scala 163 | case class OrderQty(v: BigDecimal) extends AnyVal 164 | object OrderQty { 165 | implicit val codec: CodecFix[OrderQty] = 166 | new TagCodecFix[OrderQty, BigDecimal](38) 167 | } 168 | 169 | case class Symbol(v: String) extends AnyVal 170 | object Symbol { 171 | implicit val codec: CodecFix[Symbol] = 172 | new TagCodecFix[Symbol, String](55) 173 | } 174 | ``` 175 | --- 176 | # Under the hood: ComposeCodec 177 | ```tut:invisible 178 | import strucs.fix.CodecFix 179 | ``` 180 | ```tut:silent 181 | /** Defines how a Codec[Struct[_]] can be built using the codecs of its fields */ 182 | trait ComposeCodec[Codec[_]] { 183 | 184 | /** Build a Codec for an empty Struct */ 185 | def zero: Codec[Struct[Nil]] 186 | 187 | /** Build a Codec using a field codec a and a codec b for the rest */ 188 | def prepend[A : StructKeyProvider, B]( 189 | ca: Codec[A], 190 | cb: Codec[Struct[B]]): Codec[Struct[A with B]] 191 | } 192 | ``` 193 | ```tut:silent 194 | def composeCodec: ComposeCodec[CodecFix] = ??? 195 | 196 | def codec1: CodecFix[Struct[Symbol with Nil]] = 197 | composeCodec.prepend[Symbol, Nil](Symbol.codec, composeCodec.zero) 198 | 199 | def codec2: CodecFix[Struct[OrderQty with Symbol with Nil]] = 200 | composeCodec.prepend[OrderQty, Symbol with Nil](OrderQty.codec, codec1) 201 | ``` 202 | 203 | --- 204 | ```tut:silent 205 | object CodecFix { 206 | /** Automatically create a CodecFix for any Struct[A] 207 | * @tparam T mixin, each type M in the mixin 208 | must have an implicit CodecFix[M] in scope */ 209 | implicit def makeCodecFix[T]: CodecFix[Struct[T]] = 210 | macro ComposeCodec.macroImpl[CodecFix[_], T] 211 | 212 | 213 | implicit object ComposeCodecFix extends ComposeCodec[CodecFix] { 214 | /** Build a Codec for an empty Struct */ 215 | def zero: CodecFix[Struct[Nil]] = new CodecFix[Struct[Nil]] { 216 | override def encode(a: Struct[Nil]): FixElement = FixGroup.empty 217 | override def decode(fix: FixElement): Try[Struct[Nil]] = Success(Struct.empty) 218 | } 219 | 220 | /** Build a Codec using a field codec a and a codec b for the rest */ 221 | def prepend[A: StructKeyProvider, B]( 222 | ca: CodecFix[A], 223 | cb: CodecFix[Struct[B]]) = new CodecFix[Struct[A with B]] { 224 | def encode(a: Struct[A with B]): FixElement = { 225 | val bfix = cb.encode(a.shrink[B]) 226 | val afix = ca.encode(a.get[A]) 227 | afix + bfix 228 | } 229 | 230 | def decode(fix: FixElement): Try[Struct[A with B]] = { 231 | for { 232 | structb <- cb.decode(fix) 233 | a <- ca.decode(fix) 234 | } yield structb.+[A](a) 235 | } 236 | } 237 | } 238 | } 239 | 240 | ``` 241 | --- 242 | # Under the hood: argonaut.DecodeJson 243 | 244 | The implementation of ComposeCodec is very similar 245 | ```tut:silent 246 | object StrucsDecodeJson { 247 | implicit def makeDecodeJson[T]: DecodeJson[Struct[T]] = 248 | macro ComposeCodec.macroImpl[DecodeJson[_], T] 249 | 250 | implicit object ComposeDecodeJson extends ComposeCodec[DecodeJson] { 251 | /** Build a Codec for an empty Struct */ 252 | def zero = new DecodeJson[Struct[Nil]] { 253 | def decode(c: HCursor): DecodeResult[Struct[Nil]] = 254 | DecodeResult.ok(Struct.empty) 255 | } 256 | 257 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 258 | def prepend[A: StructKeyProvider, B]( 259 | ca: DecodeJson[A], 260 | cb: DecodeJson[Struct[B]]) = new DecodeJson[Struct[A with B]] { 261 | def decode(c: HCursor): DecodeResult[Struct[A with B]] = { 262 | for { 263 | structb <- cb.decode(c) 264 | a <- ca.decode(c) 265 | } yield structb.+[A](a) 266 | } 267 | } 268 | } 269 | } 270 | ``` 271 | --- 272 | # Future developments 273 | * Benchmarks & Optimizations 274 | * Struct <==> case class 275 | * Struct <==> Avro 276 | * Struct <==> Protobuf 277 | * Typed Spark DataFrame ? 278 | 279 | ### Contributions are welcome ! 280 | --- 281 | class: center, middle 282 | # Questions ? 283 | 284 | 285 | Mikael Valot 286 | 287 | twitter: @leakimav 288 | 289 | 290 | https://github.com/mikaelv/strucs 291 | 292 | 293 | --- -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/CodecFix.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | import strucs._ 4 | import strucs.fix.dict.fix42.{MsgType, BeginString} 5 | import scala.language.experimental.macros 6 | 7 | import scala.util.{Failure, Success, Try} 8 | 9 | /** 10 | * typeclass. Defines how a Struct can be encoded/decoded to/from FIX. 11 | */ 12 | trait CodecFix[A] { 13 | def encode(a: A): FixElement 14 | def decode(fix: FixElement): Try[A] 15 | } 16 | 17 | 18 | 19 | 20 | object CodecFix { 21 | /** Automatically create a CodecFix for any Struct[A] 22 | * @tparam T mixin, each type M in the mixin must have an implicit CodecFix[M] in scope */ 23 | implicit def makeCodecFix[T]: CodecFix[Struct[T]] = macro ComposeCodec.macroImpl[CodecFix[_], T] 24 | 25 | 26 | 27 | // TODO generalize with a codec that returns a B : Monoid ? 28 | implicit object ComposeCodecFix extends ComposeCodec[CodecFix] { 29 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 30 | override def prepend[A: StructKeyProvider, B](ca: CodecFix[A], cb: => CodecFix[Struct[B]]): CodecFix[Struct[A with B]] = new CodecFix[Struct[A with B]] { 31 | override def encode(a: Struct[A with B]): FixElement = { 32 | val bfix = cb.encode(a.shrink[B]) 33 | val afix = ca.encode(a.get[A]) 34 | afix + bfix 35 | } 36 | 37 | override def decode(fix: FixElement): Try[Struct[A with B]] = { 38 | for { 39 | structb <- cb.decode(fix) 40 | a <- ca.decode(fix) 41 | 42 | } yield structb.add[A](a) 43 | } 44 | } 45 | 46 | /** Build a Codec for an empty Struct */ 47 | override def zero: CodecFix[Struct[Nil]] = new CodecFix[Struct[Nil]] { 48 | override def encode(a: Struct[Nil]): FixElement = FixGroup.empty 49 | 50 | override def decode(fix: FixElement): Try[Struct[Nil]] = Success(Struct.empty) 51 | } 52 | 53 | } 54 | 55 | 56 | /** Pimp Struct with helpful methods */ 57 | implicit class EncodeFixOps[A <: MsgType with BeginString](struct: Struct[A])(implicit codec: CodecFix[Struct[A]]) { 58 | def toFixMessage: FixMessage = { 59 | FixMessage( 60 | BeginString.codec.encode(struct.get[BeginString]), 61 | MsgType.codec.encode(struct.get[MsgType]), 62 | codec.encode(struct).toGroup.remove(Set(MsgType.codec.tag, BeginString.codec.tag))) 63 | } 64 | 65 | def toFixMessageString: String = toFixMessage.toFixString 66 | } 67 | 68 | implicit class DecodeFixOps(fixString: String) { 69 | def toStruct[A](implicit codec: CodecFix[Struct[A]]): Try[Struct[A]] = { 70 | for { 71 | fixMsg <- FixMessage.decode(fixString) 72 | struct <- codec.decode(fixMsg) 73 | } yield struct 74 | } 75 | } 76 | 77 | } -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/FixElement.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | import strucs.fix.dict.fix42.MsgType 4 | 5 | import scala.util.{Failure, Success, Try} 6 | 7 | /** 8 | * Can be a tag/value pair or a Group of tag/value pair 9 | */ 10 | sealed trait FixElement { 11 | def +(other: FixElement): FixElement 12 | 13 | def toFixString: String 14 | 15 | def toGroup: FixGroup 16 | } 17 | 18 | 19 | 20 | /** Tag/Value Pair */ 21 | case class FixTagValue(tag: Int, value: String) extends FixElement { 22 | override def +(other: FixElement): FixElement = other match { 23 | case FixGroup(pairs) => FixGroup(this +: pairs) 24 | case t@FixTagValue(_, _) => FixGroup(Vector(this, t)) 25 | case m@FixMessage(_, _, FixGroup(pairs)) => m.copy(body = FixGroup(this +: pairs)) 26 | } 27 | 28 | def toFixString: String = s"$tag=$value" 29 | 30 | def toGroup: FixGroup = new FixGroup(Vector(this)) 31 | } 32 | 33 | object FixTagValue { 34 | def decode(tagValue: String): FixTagValue = { 35 | // TODO error handling 36 | val split = tagValue.split("=") 37 | FixTagValue(split(0).toInt, split(1)) 38 | } 39 | } 40 | 41 | 42 | 43 | /** Group of tags. Can be the header, body, or trailer */ 44 | case class FixGroup(pairs: Vector[FixTagValue]) extends FixElement { 45 | import FixGroup.SOH 46 | 47 | override def +(other: FixElement): FixElement = other match { 48 | case FixGroup(o) => FixGroup(pairs ++ o) 49 | case t@FixTagValue(_, _) => FixGroup(pairs :+ t) 50 | case m@FixMessage(_, _, FixGroup(o)) => m.copy(body = FixGroup(pairs ++ o)) 51 | } 52 | 53 | def toFixString: String = pairs.map(_.toFixString).mkString("", SOH, "") 54 | 55 | def checksum: Int = toFixString.sum 56 | 57 | def length: Int = toFixString.length 58 | 59 | def get(tag: Int): Option[FixTagValue] = pairs.find(_.tag == tag) 60 | 61 | def remove(tags: Set[Int]): FixGroup = FixGroup(pairs.filterNot(pair => tags.contains(pair.tag))) 62 | 63 | def toGroup: FixGroup = this 64 | } 65 | 66 | object FixGroup { 67 | def apply(pairs: (Int, String)*): FixGroup = FixGroup(pairs.map( kv => FixTagValue(kv._1, kv._2)).toVector) 68 | 69 | def empty: FixGroup = new FixGroup(Vector.empty) 70 | 71 | /** Separator between key value pairs */ 72 | val SOH = "\u0001" 73 | 74 | // TODO error handling 75 | def decode(fix: String): Try[FixGroup] = Try { 76 | new FixGroup(fix.split(SOH).map { 77 | FixTagValue.decode(_) 78 | }.toVector) 79 | } 80 | } 81 | 82 | 83 | 84 | 85 | import FixGroup.SOH 86 | 87 | /** Represents a Fix message, encodes the length and checksum when writing to a String */ 88 | case class FixMessage(beginString: FixTagValue, msgType: FixTagValue, body: FixGroup) extends FixElement { 89 | 90 | override def +(other: FixElement): FixElement = body + other 91 | 92 | override def toGroup: FixGroup = body 93 | 94 | 95 | def headerWithLength: FixGroup = { 96 | val bodyLength = (msgType.toGroup.length + 1 + body.length + 1).toString 97 | new FixGroup(Vector(beginString, FixTagValue(9, bodyLength), msgType)) 98 | } 99 | 100 | def trailer: FixGroup = FixGroup(10 -> ((headerWithLength.checksum + 1 + body.checksum + 1) % 256).formatted("%03d")) 101 | 102 | def toFixString: String = headerWithLength.toFixString + SOH + body.toFixString + SOH + trailer.toFixString + SOH 103 | } 104 | 105 | object FixMessage { 106 | def decode(fix: String): Try[FixMessage] = { 107 | FixGroup.decode(fix) flatMap { g => 108 | val optMsg = for { 109 | tag35 <- g.get(35) 110 | tag8 <- g.get(8) 111 | } yield FixMessage(tag8, tag35, g.remove(Set(8, 35, 9, 10))) 112 | optMsg match { 113 | case None => Failure(new FixDecodeException(s"Tags 8 or 35 not found in $fix")) 114 | case Some(msg) => Success(msg) 115 | } 116 | } 117 | } 118 | 119 | } -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/FixEncodeException.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | /** 4 | * 5 | */ 6 | class FixEncodeException(msg: String, optThrowable: Option[Throwable] = None) extends Exception(msg, optThrowable.orNull) { 7 | 8 | } 9 | 10 | /** 11 | * 12 | */ 13 | class FixDecodeException(msg: String, optThrowable: Option[Throwable] = None) extends Exception(msg, optThrowable.orNull) { 14 | 15 | } 16 | -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/TagCodecFix.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | import strucs.Wrapper 4 | import strucs.fix.dict.fix42.{MsgType, BeginString} 5 | 6 | import scala.util.{Failure, Success, Try} 7 | 8 | /** Encodes a single tag / value pair. 9 | * Uses the value codec and the wrapper to encode/decode the value */ 10 | class TagCodecFix[W, V](val tag: Int)(implicit wrapper: Wrapper[W, V], valueCodec: ValueCodecFix[V]) extends CodecFix[W] { 11 | override def encode(a: W): FixTagValue = FixTagValue(tag, valueCodec.encode(wrapper.value(a))) 12 | 13 | /** @param fix is always a FixGroup when called from outside */ 14 | override def decode(fix: FixElement): Try[W] = fix match { 15 | case FixTagValue(t, value) if t == tag => valueCodec.decode(value) flatMap { 16 | wrapper.make(_).map(Success(_)).getOrElse(Failure(new FixDecodeException(s"Wrapper: $wrapper cannot parse $value in tag $t"))) 17 | } 18 | 19 | // get the FixTagValue and call decode again 20 | case g@FixGroup(pairs) => g.get(tag).map(decode).getOrElse(Failure(new FixDecodeException(s"Cannot find tag $tag in $fix"))) 21 | case m@FixMessage(beginStr, msgType, body) => 22 | if (tag == MsgType.Tag) 23 | decode(msgType) 24 | else if (tag == BeginString.Tag) 25 | decode(beginStr) 26 | else 27 | body.get(tag).map(decode).getOrElse(Failure(new FixDecodeException(s"Cannot find tag $tag in $fix"))) 28 | 29 | case _ => Failure(new FixDecodeException(s"Cannot decode $fix")) 30 | } 31 | } -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/ValueCodecFix.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | import org.joda.time.{DateTimeZone, DateTime} 4 | import org.joda.time.format.{DateTimeFormat, DateTimeFormatterBuilder, DateTimeFormatter, ISODateTimeFormat} 5 | 6 | import scala.util.{Success, Try} 7 | 8 | /** typeclass. defines how basic types (String, Int, ...) can be encoded/decoded to/from FIX */ 9 | trait ValueCodecFix[A] { 10 | def encode(a: A): String 11 | def decode(s: String): Try[A] 12 | } 13 | 14 | object ValueCodecFix { 15 | implicit object StringValueCodec extends ValueCodecFix[String] { 16 | override def encode(a: String): String = a 17 | 18 | override def decode(s: String): Try[String] = Success(s) 19 | } 20 | 21 | implicit object IntValueCodec extends ValueCodecFix[Int] { 22 | override def encode(a: Int): String = a.toString 23 | 24 | override def decode(s: String): Try[Int] = Try { s.toInt } 25 | } 26 | 27 | 28 | implicit object BigDecimalValueCodec extends ValueCodecFix[BigDecimal] { 29 | override def encode(a: BigDecimal): String = a.toString() 30 | 31 | override def decode(s: String): Try[BigDecimal] = Try { BigDecimal(s) } 32 | } 33 | 34 | implicit object DateTimeValueCodec extends ValueCodecFix[DateTime] { 35 | private val formatter = DateTimeFormat.forPattern("yyyyMMdd-HH:mm:ss").withZoneUTC() 36 | override def encode(a: DateTime): String = formatter.print(a) 37 | 38 | override def decode(s: String): Try[DateTime] = Try { formatter.parseDateTime(s) } 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /strucs-fix/src/main/scala/strucs/fix/dict/fix42/package.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix.dict 2 | 3 | import strucs.fix.{CodecFix, TagCodecFix} 4 | import org.joda.time.DateTime 5 | import strucs.Wrapper 6 | import Wrapper.materializeWrapper 7 | import strucs.Wrapper 8 | 9 | /** 10 | * Tag names for FIX 4.2 11 | */ 12 | package object fix42 { 13 | 14 | case class BeginString(v: String) extends AnyVal 15 | object BeginString { 16 | val Fix42 = BeginString("FIX.4.2") 17 | val Tag = 8 18 | implicit val codec = new TagCodecFix[BeginString, String](Tag) 19 | 20 | val Fix42TV = codec.encode(Fix42) 21 | } 22 | 23 | case class BodyLength(v: Int) extends AnyVal 24 | object BodyLength { 25 | implicit val codec: CodecFix[BodyLength] = new TagCodecFix[BodyLength, Int](9) 26 | } 27 | 28 | case class CheckSum(v: Int) extends AnyVal 29 | object CheckSum { 30 | implicit val codec: CodecFix[CheckSum] = new TagCodecFix[CheckSum, Int](10) 31 | } 32 | 33 | case class ClOrdId(v: String) extends AnyVal 34 | object ClOrdId { 35 | implicit val codec: CodecFix[ClOrdId] = new TagCodecFix[ClOrdId, String](11) 36 | } 37 | 38 | 39 | case class MsgSeqNum(v: String) extends AnyVal 40 | object MsgSeqNum { 41 | implicit val codec: CodecFix[MsgSeqNum] = new TagCodecFix[MsgSeqNum, String](34) 42 | } 43 | 44 | case class MsgType(v: String) extends AnyVal 45 | object MsgType { 46 | val OrderSingle = MsgType("D") 47 | val Logon = MsgType("A") 48 | val Tag = 35 49 | implicit val codec = new TagCodecFix[MsgType, String](Tag) 50 | 51 | val OrderSingleTV = codec.encode(OrderSingle) 52 | val LogonTV = codec.encode(Logon) 53 | } 54 | 55 | case class OrderQty(v: BigDecimal) extends AnyVal 56 | object OrderQty { 57 | implicit val codec: CodecFix[OrderQty] = new TagCodecFix[OrderQty, BigDecimal](38) 58 | } 59 | 60 | 61 | 62 | 63 | case class SenderCompID(v: String) extends AnyVal 64 | object SenderCompID { 65 | implicit val codec: CodecFix[SenderCompID] = new TagCodecFix[SenderCompID, String](49) 66 | } 67 | 68 | case class SendingTime(v: DateTime) extends AnyVal 69 | object SendingTime { 70 | implicit val codec: CodecFix[SendingTime] = new TagCodecFix[SendingTime, DateTime](52) 71 | } 72 | 73 | 74 | /** Example of a lenient enumeration. 75 | * We just provide some common constants but the user of the library can add his own */ 76 | case class OrderCapacity(v: String) extends AnyVal 77 | object OrderCapacity { 78 | val AgencySingleOrder = OrderCapacity("A") 79 | val ShortExampleTransaction = OrderCapacity("B") 80 | 81 | implicit val codec: CodecFix[OrderCapacity] = new TagCodecFix[OrderCapacity, String](47) 82 | } 83 | 84 | /** Example of a more strict enumeration. 85 | * This can be pattern-matched with a compiler warning for missing patterns. 86 | * We still allow other values, but it is materialized with a "Other" type */ 87 | sealed abstract class OrdType(val v: String) 88 | object OrdType { 89 | case object Market extends OrdType("1") 90 | case object Limit extends OrdType("2") 91 | case object Stop extends OrdType("3") 92 | case class Other(_v: String) extends OrdType(_v) 93 | 94 | val all = Seq(Market, Limit, Stop) 95 | def make(fixValue: String): Option[OrdType] = all.find(_.v == fixValue).orElse(Some(Other(fixValue))) 96 | 97 | // decoding a 54=X if X is not declared in 'all' would fail 98 | implicit val wrapper: Wrapper[OrdType, String] = Wrapper(make, _.v) 99 | implicit val codec: CodecFix[OrdType] = new TagCodecFix[OrdType, String](40) 100 | } 101 | 102 | 103 | /** Example of a strict enumeration, the only values that can encoded/decoded are declared here */ 104 | sealed abstract class Side(val v: String) 105 | object Side { 106 | case object Buy extends Side("1") 107 | case object Sell extends Side("2") 108 | 109 | val all = Seq(Buy, Sell) 110 | def make(fixValue: String): Option[Side] = all.find(_.v == fixValue) 111 | 112 | // decoding a 54=X if X is not declared in 'all' would fail 113 | implicit val wrapper: Wrapper[Side, String] = Wrapper(make, _.v) 114 | implicit val codec: CodecFix[Side] = new TagCodecFix[Side, String](54) 115 | } 116 | 117 | sealed abstract class HandlInst(val v: String) 118 | object HandlInst { 119 | case object AutomatedPrivateNoBroker extends HandlInst("1") 120 | case object AutomatedPublicBrokerOk extends HandlInst("2") 121 | case object ManualBest extends HandlInst("3") 122 | 123 | val all = Seq(AutomatedPrivateNoBroker, AutomatedPublicBrokerOk, ManualBest) 124 | def make(fixValue: String): Option[HandlInst] = all.find(_.v == fixValue) 125 | 126 | implicit val wrapper: Wrapper[HandlInst, String] = Wrapper(make, _.v) 127 | implicit val codec: CodecFix[HandlInst] = new TagCodecFix[HandlInst, String](21) 128 | } 129 | 130 | 131 | case class Symbol(v: String) extends AnyVal 132 | object Symbol { 133 | implicit val codec: CodecFix[Symbol] = new TagCodecFix[Symbol, String](55) 134 | } 135 | 136 | case class TargetCompID(v: String) extends AnyVal 137 | object TargetCompID { 138 | implicit val codec: CodecFix[TargetCompID] = new TagCodecFix[TargetCompID, String](56) 139 | } 140 | 141 | case class TimeInForce(v: String) extends AnyVal 142 | object TimeInForce { 143 | implicit val codec: CodecFix[TimeInForce] = new TagCodecFix[TimeInForce, String](59) 144 | } 145 | 146 | case class TransactTime(v: DateTime) extends AnyVal 147 | object TransactTime { 148 | implicit val codec: CodecFix[TransactTime] = new TagCodecFix[TransactTime, DateTime](60) 149 | } 150 | 151 | 152 | case class OnBehalfOfCompID(v: String) extends AnyVal 153 | object OnBehalfOfCompID { 154 | implicit val codec: CodecFix[OnBehalfOfCompID] = new TagCodecFix[OnBehalfOfCompID, String](115) 155 | } 156 | 157 | case class SecurityExchange(v: String) extends AnyVal 158 | object SecurityExchange { 159 | val NYSE = SecurityExchange("N") 160 | implicit val codec: CodecFix[SecurityExchange] = new TagCodecFix[SecurityExchange, String](207) 161 | } 162 | 163 | 164 | 165 | } 166 | -------------------------------------------------------------------------------- /strucs-fix/src/test/scala/strucs/fix/CodecFixSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs.fix 2 | 3 | import org.joda.time.{DateTime, DateTimeZone} 4 | import org.scalactic.TypeCheckedTripleEquals 5 | import org.scalatest.{FlatSpec, Matchers} 6 | import strucs.Struct 7 | import strucs.fix.CodecFix._ 8 | import strucs.fix.FixGroup.SOH 9 | import strucs.fix.dict.fix42._ 10 | 11 | import scala.util.Success 12 | 13 | 14 | /** 15 | * More examples at http://fiximulator.org/FIXimulator_Thesis.pdf 16 | */ 17 | class CodecFixSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 18 | 19 | /** It's easier to use \n or ; for separating key/value pairs in testcases. */ 20 | private implicit class SemicolonToSOH(s: String) { 21 | def toSOH: String = s.stripMargin.replaceAll("[;\n]", SOH) 22 | } 23 | 24 | "a CodecFix" should "encode/decode a New Order Single" in { 25 | val struct = Struct.empty + 26 | BeginString.Fix42 + 27 | MsgType.OrderSingle + 28 | MsgSeqNum("4") + // TODO remove ?: protocol field 29 | SenderCompID("ABC_DEFG01") + 30 | SendingTime(new DateTime(2009,3,23,15,40,29, DateTimeZone.UTC)) + 31 | TargetCompID("CCG") + 32 | OnBehalfOfCompID("XYZ") + 33 | ClOrdId("NF 0542/03232009") + 34 | (Side.Buy: Side) + 35 | OrderQty(100) + 36 | Symbol("CVS") + 37 | (OrdType.Market: OrdType) + 38 | TimeInForce("0") + 39 | OrderCapacity.AgencySingleOrder + 40 | TransactTime(new DateTime(2009,3,23,15,40,29, DateTimeZone.UTC)) + 41 | (HandlInst.AutomatedPrivateNoBroker: HandlInst) + 42 | SecurityExchange.NYSE 43 | 44 | val fixString = 45 | """8=FIX.4.2 46 | |9=146 47 | |35=D 48 | |34=4 49 | |49=ABC_DEFG01 50 | |52=20090323-15:40:29 51 | |56=CCG 52 | |115=XYZ 53 | |11=NF 0542/03232009 54 | |54=1 55 | |38=100 56 | |55=CVS 57 | |40=1 58 | |59=0 59 | |47=A 60 | |60=20090323-15:40:29 61 | |21=1 62 | |207=N 63 | |10=195 64 | |""".toSOH 65 | 66 | 67 | val encodedFix = struct.toFixMessageString 68 | encodedFix should be (fixString) 69 | 70 | val decodedStruct = fixString.toStruct[struct.Mixin] 71 | decodedStruct.get should === (struct) 72 | } 73 | 74 | 75 | val fixMsgString = "8=FIX.4.2;9=65;35=A;49=SERVER;56=CLIENT;34=177;52=20090107-18:15:16;98=0;108=30;10=062;".toSOH 76 | 77 | "A FixMessage" should "encode with length and checksum" in { 78 | // Example from https://en.wikipedia.org/wiki/Financial_Information_eXchange#Body_length 79 | val msg = FixMessage(BeginString.Fix42TV, MsgType.LogonTV, FixGroup(49 -> "SERVER", 56 -> "CLIENT", 34 -> "177", 52 -> "20090107-18:15:16", 98 -> "0", 108 -> "30")) 80 | msg.toFixString should be (fixMsgString) 81 | } 82 | 83 | "A FixMessage" should "decode" in { 84 | FixMessage.decode(fixMsgString) should ===(Success(FixMessage(BeginString.Fix42TV, MsgType.LogonTV, FixGroup(49 -> "SERVER", 56 -> "CLIENT", 34 -> "177", 52 -> "20090107-18:15:16", 98 -> "0", 108 -> "30")))) 85 | } 86 | 87 | "A FixMessage" should "keep the same fix string after decode and encode" in { 88 | val fixStr = "8=FIX.4.2;9=137;35=D;34=8;49=BANZAI;52=20081005-14:35:46.672;56=FIXIMULATOR;11=1223217346597;21=1;38=5000;40=1;54=2;55=IBM;59=0;60=20081005-14:35:46.668;10=153;".toSOH 89 | FixMessage.decode(fixStr) map (_.toFixString) should === (Success(fixStr)) 90 | } 91 | 92 | 93 | } 94 | 95 | 96 | -------------------------------------------------------------------------------- /strucs-json/src/main/scala/strucs/json/StrucsEncodeJson.scala: -------------------------------------------------------------------------------- 1 | package strucs.json 2 | 3 | import _root_.argonaut._ 4 | import strucs.ComposeCodec.ConvertEncode 5 | import strucs._ 6 | import Argonaut._ 7 | import scala.language.experimental.macros 8 | 9 | /** Encodes a Struct using Argonaut's EncodeJson typeclass */ 10 | object StrucsEncodeJson { 11 | /** Build a field:value pair encoder */ 12 | def fromWrapper[W, V](fieldName: String)(implicit wrapper: Wrapper[W, V], valueEncode: EncodeJson[V]): EncodeJson[W] = new EncodeJson[W] { 13 | override def encode(w: W): Json = jSingleObject(fieldName, valueEncode.encode(wrapper.value(w))) 14 | } 15 | 16 | /** Automatically create an EncodeJson for any Struct[A] 17 | * @tparam T mixin, each type M in the mixin must have an implicit EncodeJson[M] in scope */ 18 | implicit def makeEncodeJson[T]: EncodeJson[Struct[T]] = macro ComposeCodec.macroImpl[EncodeJson[_], T] 19 | 20 | implicit val monoid = new Monoid[Json] { 21 | override def zero: Json = jObject(JsonObject.empty) 22 | 23 | override def prepend(a: Json, b: Json): Json = { 24 | a.assoc match { 25 | case Some(assoc :: Nil) => assoc ->: b 26 | case _ => sys.error(s"Cannot prepend $a to $b") 27 | } 28 | } 29 | } 30 | 31 | implicit val trans = new ConvertEncode[EncodeJson, Json] { 32 | override def fromFunc[A](_encode: (A) => Json) = EncodeJson[A](_encode) 33 | override def toFunc[A](enc: EncodeJson[A]) = enc.encode 34 | } 35 | implicit val composeEncode: ComposeCodec[EncodeJson] = ComposeCodec.makeComposeCodec[EncodeJson, Json] 36 | 37 | 38 | 39 | val nospacePreserveOrder = nospace.copy(preserveOrder = true) 40 | 41 | /** Pimp Struct with helpful methods */ 42 | implicit class EncodeJsonOps[A](struct: Struct[A])(implicit encode: EncodeJson[Struct[A]]) { 43 | def toJson: Json = encode.encode(struct) 44 | def toJsonString: String = nospacePreserveOrder.pretty(toJson) 45 | } 46 | 47 | } 48 | 49 | object StrucsDecodeJson { 50 | /** Build a field:value pair decoder */ 51 | def fromWrapper[W, V](fieldName: String)(implicit wrapper: Wrapper[W, V], valueDecode: DecodeJson[V]): DecodeJson[W] = new DecodeJson[W] { 52 | override def decode(c: HCursor): DecodeResult[W] = { 53 | val wrapperDecode = valueDecode.flatMap { value => 54 | new DecodeJson[W] { 55 | override def decode(c: HCursor): DecodeResult[W] = wrapper.make(value) match { 56 | case None => DecodeResult.fail("Invalid value " + value, c.history) 57 | case Some(w) => DecodeResult.ok(w) 58 | } 59 | } 60 | } 61 | 62 | c.get(fieldName)(wrapperDecode) 63 | } 64 | } 65 | 66 | implicit def makeDecodeJson[T]: DecodeJson[Struct[T]] = macro ComposeCodec.macroImpl[DecodeJson[_], T] 67 | 68 | implicit object ComposeDecodeJson extends ComposeCodec[DecodeJson] { 69 | /** Build a Codec for an empty Struct */ 70 | override def zero: DecodeJson[Struct[Nil]] = new DecodeJson[Struct[Nil]] { 71 | override def decode(c: HCursor): DecodeResult[Struct[Nil]] = DecodeResult.ok(Struct.empty) 72 | 73 | } 74 | 75 | /** Build a Codec using a field codec a and a codec b for the rest of the Struct */ 76 | override def prepend[A: StructKeyProvider, B](ca: DecodeJson[A], cb: => DecodeJson[Struct[B]]): DecodeJson[Struct[A with B]] = new DecodeJson[Struct[A with B]] { 77 | override def decode(c: HCursor): DecodeResult[Struct[A with B]] = { 78 | for { 79 | structb <- cb.decode(c) 80 | a <- ca.decode(c) 81 | } yield structb.add[A](a) 82 | } 83 | 84 | } 85 | } 86 | } 87 | 88 | /** Encode and Decode a Struct using Argonaut's CodecJson */ 89 | object StrucsCodecJson { 90 | def fromWrapper[W, V](fieldName: String)(implicit wrapper: Wrapper[W, V], valueEncode: EncodeJson[V], valueDecode: DecodeJson[V]): CodecJson[W] = CodecJson.derived[W]( 91 | StrucsEncodeJson.fromWrapper[W, V](fieldName), StrucsDecodeJson.fromWrapper[W, V](fieldName)) 92 | } 93 | -------------------------------------------------------------------------------- /strucs-json/src/test/scala/strucs/json/CodecJsonSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs.json 2 | 3 | import argonaut.Argonaut._ 4 | import argonaut.{CodecJson, DecodeJson, EncodeJson} 5 | import CodecJsonSpec.Gender.Male 6 | import CodecJsonSpec._ 7 | import StrucsDecodeJson._ 8 | import StrucsEncodeJson._ 9 | import org.scalactic.TypeCheckedTripleEquals 10 | import org.scalatest.{FlatSpec, Matchers} 11 | import strucs._ 12 | 13 | import scalaz.{-\/, \/-} 14 | 15 | /** 16 | */ 17 | class CodecJsonSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 18 | val person = Struct.empty + Name("Albert") + Age(76) + City("Princeton") + (Male: Gender) 19 | 20 | "an EncodeJson" should "encode a Person" in { 21 | val json = person.toJsonString 22 | json should === ("""{"name":"Albert","age":76,"city":"Princeton","gender":"M"}""") 23 | } 24 | 25 | "a DecodeJson" should "decode a json string into a Person" in { 26 | val json = """{"name":"Albert","age":76,"city":"Princeton","gender":"M"}""" 27 | val dperson = json.decode[Person] 28 | dperson shouldBe \/-(person) 29 | } 30 | 31 | "a DecodeJson" should "return an error when a field is missing" in { 32 | val json = """{"name":"Albert","age":76,"gender":"M"}""" 33 | val dperson = json.decodeEither[Person] 34 | dperson should === (-\/("Attempt to decode value on failed cursor.: [*.--\\(city)]")) 35 | } 36 | 37 | "a DecodeJson" should "return an error when an enumeration has an invalid value" in { 38 | val json = """{"name":"Albert","age":76,"city":"Princeton","gender":"Z"}""" 39 | val dperson = json.decodeEither[Person] 40 | dperson should === (-\/("Invalid value Z: [--\\(gender)]")) 41 | } 42 | 43 | "an EncodeJson" should "encode a Person with a nested Address" in { 44 | val address = Address(Struct.empty + Line1("52 Upper Street") + PostCode("N1 0QH")) 45 | val personAdr = person + address 46 | val json = personAdr.toJsonString 47 | 48 | json should === ("""{"name":"Albert","age":76,"city":"Princeton","gender":"M","address":{"line1":"52 Upper Street","postCode":"N1 0QH"}}""") 49 | } 50 | 51 | "a DecodeJson" should "decode a Person with a nested Address" in { 52 | val json = """{"address":{"line1":"52 Upper Street","postCode":"N1 0QH"}, "name":"Albert","age":76,"city":"Princeton","gender":"M"}""" 53 | val dperson = json.decodeEither[Struct[Name with Age with City with Gender with Address]] 54 | 55 | val address = Address(Struct.empty + Line1("52 Upper Street") + PostCode("N1 0QH")) 56 | val expected = person + address 57 | 58 | dperson shouldBe \/-(expected) 59 | } 60 | } 61 | 62 | object CodecJsonSpec { 63 | type Person = Struct[Name with Age with City with Gender] 64 | type AddressStruct = Struct[Line1 with PostCode with Nil] 65 | case class Address(v: AddressStruct) 66 | 67 | case class Name(v: String) extends AnyVal 68 | case class Age(v: Int) extends AnyVal 69 | case class City(v: String) extends AnyVal 70 | case class Line1(v: String) extends AnyVal 71 | case class PostCode(v: String) extends AnyVal 72 | 73 | sealed abstract class Gender(val v: String) 74 | object Gender { 75 | case object Male extends Gender("M") 76 | case object Female extends Gender("F") 77 | 78 | val all = Seq(Male, Female) 79 | def make(value: String): Option[Gender] = all.find(_.v == value) 80 | } 81 | 82 | 83 | 84 | 85 | // Defines how to encode/decode Name to/from a Struct 86 | implicit val nameCodec: CodecJson[Name] = StrucsCodecJson.fromWrapper[Name, String]("name") 87 | 88 | implicit val ageCodec: CodecJson[Age] = StrucsCodecJson.fromWrapper[Age, Int]("age") 89 | // We can also declare encode and decode separately 90 | implicit val cityEncode: EncodeJson[City] = StrucsEncodeJson.fromWrapper[City, String]("city") 91 | implicit val cityDecode: DecodeJson[City] = StrucsDecodeJson.fromWrapper[City, String]("city") 92 | 93 | implicit val genderWrapper: Wrapper[Gender, String] = Wrapper(Gender.make, _.v) 94 | implicit val genderCodec: CodecJson[Gender] = StrucsCodecJson.fromWrapper[Gender, String]("gender") 95 | 96 | implicit val line1Codec: CodecJson[Line1] = StrucsCodecJson.fromWrapper[Line1, String]("line1") 97 | implicit val postCodeCodec: CodecJson[PostCode] = StrucsCodecJson.fromWrapper[PostCode, String]("postCode") 98 | implicit val addressCodec: CodecJson[Address] = StrucsCodecJson.fromWrapper[Address, AddressStruct]("address") 99 | } 100 | -------------------------------------------------------------------------------- /strucs-spark/src/main/scala/strucs/spark/StructDataFrame.scala: -------------------------------------------------------------------------------- 1 | package strucs.spark 2 | 3 | import org.apache.spark.rdd.RDD 4 | import org.apache.spark.sql.Column 5 | import org.apache.spark.sql.{Row, Column, GroupedData, DataFrame} 6 | import strucs._ 7 | 8 | 9 | 10 | /** 11 | * Wraps a DataFrame to make all operations type safe 12 | * @param wrappers keep the wrapper for each field so that we can lazily call wrapper.make 13 | * when we want to extract single fields from a row (see RowMap) 14 | */ 15 | class StructDataFrame[F](val df: DataFrame, wrappers: Map[StructKey, Wrapper[_, _]]) extends Serializable { 16 | import StructDataFrame.WrapperAny 17 | 18 | def select(columns: Col[_]): StructDataFrame[columns.Mixin] = 19 | new StructDataFrame[columns.Mixin](df.select(columns.columns :_*), columns.wrappers) 20 | 21 | // These are not strictly necessary, but they make the API nicer 22 | def select[A >: F : WrapperAny : StructKeyProvider]: StructDataFrame[A] = select(Col[A]) 23 | def select[A >: F : WrapperAny : StructKeyProvider, 24 | B >: F : WrapperAny : StructKeyProvider ]: StructDataFrame[A with B] = select(Col[A] + Col[B]) 25 | 26 | 27 | def groupBy[G](columns: Col[G]): StructGroupedData[G, F] = // TODO use Col[_] as in select, to avoid inference to Nothing 28 | new StructGroupedData[G, F](df.groupBy(columns.columns: _*), columns) 29 | 30 | def groupBy[A >: F : WrapperAny : StructKeyProvider]: StructGroupedData[A, F] = groupBy(Col[A]) 31 | 32 | def groupBy[A >: F : WrapperAny : StructKeyProvider, 33 | B >: F : WrapperAny : StructKeyProvider]: StructGroupedData[A with B, F] = groupBy[A with B](Col[A] + Col[B]) 34 | 35 | 36 | 37 | def collect(): Array[Struct[F]] = rdd.collect() 38 | 39 | def show() = df.show() 40 | 41 | def rdd: RDD[Struct[F]] = df.rdd.map(row => Struct(RowMap(row, wrappers))) 42 | } 43 | 44 | 45 | /** Wraps a Row in order to access it as if it was a Map */ 46 | case class RowMap(row: Row, wrappers: Map[StructKey, Wrapper[_, _]]) extends Map[StructKey, Any] { 47 | // It is easier to rebuild a brand new map rather than fiddling with row.schema 48 | override def +[B1 >: Any](kv: (StructKey, B1)): Map[StructKey, B1] = Map(keyValues: _*) + kv 49 | 50 | override def get(key: StructKey): Option[Any] = wrappers(key) match { 51 | case wrapper:Wrapper[w, v] => wrapper.make(row.getAs[v](key.value)) 52 | } 53 | 54 | def keyValues: Seq[(StructKey, Any)] = (row.schema.fieldNames.map(StructKey.apply) zip row.toSeq).toSeq 55 | 56 | override def iterator: Iterator[(StructKey, Any)] = keyValues.iterator 57 | 58 | override def -(key: StructKey): Map[StructKey, Any] = ??? // not called from Struct 59 | } 60 | 61 | /** Transforms a set of Types into a set of Column that can be used in spark's api */ 62 | trait Col[A] { self => 63 | type Mixin = A 64 | def columns: Seq[Column] 65 | def wrappers: Map[StructKey, Wrapper[_, _]] 66 | 67 | // TODO type constraint B >: F ?? 68 | def +[B](colB: Col[B]): Col[A with B] = new Col[A with B] { 69 | def columns = self.columns ++ colB.columns 70 | def wrappers = self.wrappers ++ colB.wrappers 71 | } 72 | } 73 | 74 | object Col { 75 | // TODO create a ColumnProvider instead of StructKeyProvider? That would allow to manage expressions as well 76 | def apply[A](implicit k: StructKeyProvider[A], wa: Wrapper[A, _]): Col[A] = new Col[A] { 77 | 78 | def columns = Seq(new Column(k.key.value)) 79 | 80 | def wrappers = Map(k.key -> wa) 81 | } 82 | } 83 | 84 | 85 | object StructDataFrame { 86 | 87 | type WrapperAny[A] = Wrapper[A, _] 88 | 89 | implicit class DataFrameOps(df: DataFrame) { 90 | 91 | // TODO verify that df's schema is compatible with cols 92 | def toStructDF(cols: Col[_]): StructDataFrame[cols.Mixin] = new StructDataFrame[cols.Mixin](df, cols.wrappers) 93 | 94 | def toStructDF[A : StructKeyProvider : WrapperAny]: StructDataFrame[A] = toStructDF(Col[A]) 95 | 96 | def toStructDF[A : StructKeyProvider : WrapperAny, 97 | B : StructKeyProvider : WrapperAny]: StructDataFrame[A with B] = toStructDF(Col[A] + Col[B]) 98 | 99 | def toStructDF[A : StructKeyProvider : WrapperAny, 100 | B : StructKeyProvider : WrapperAny, 101 | C : StructKeyProvider : WrapperAny]: StructDataFrame[A with B with C] = toStructDF(Col[A] + Col[B] + Col[C]) 102 | 103 | def toStructDF[A : StructKeyProvider : WrapperAny, 104 | B : StructKeyProvider : WrapperAny, 105 | C : StructKeyProvider : WrapperAny, 106 | D : StructKeyProvider : WrapperAny]: StructDataFrame[A with B with C with D] = toStructDF(Col[A] + Col[B] + Col[C] + Col[D]) 107 | 108 | 109 | } 110 | 111 | } 112 | 113 | import StructDataFrame._ 114 | class StructGroupedData[G, F](g: GroupedData, cols: Col[G]) { 115 | /** Calls an aggregate function expr on column A. The aggregate function returns a new column B */ 116 | def agg[A >: F, B](expr: String => Column)(implicit ka: StructKeyProvider[A], kb: StructKeyProvider[B], wb: Wrapper[B, _]): StructDataFrame[G with B] = 117 | g.agg(expr(ka.key.value).as(kb.key.value)).toStructDF(cols + Col[B]) 118 | 119 | def agg[A >: F, B, C](exprB: String => Column, exprC: String => Column)(implicit ka: StructKeyProvider[A], kb: StructKeyProvider[B], kc: StructKeyProvider[C], wb: Wrapper[B, _], wc: Wrapper[C, _]): StructDataFrame[G with B with C] = { 120 | val a = ka.key.value 121 | g.agg(exprB(a).as(kb.key.value), exprC(a).as(kc.key.value)).toStructDF(cols + Col[B] + Col[C]) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /strucs-spark/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory=INFO, console 2 | log4j.appender.console=org.apache.log4j.ConsoleAppender 3 | log4j.appender.console.target=System.err 4 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 6 | -------------------------------------------------------------------------------- /strucs-spark/src/test/scala/strucs/spark/SparkApp.scala: -------------------------------------------------------------------------------- 1 | package strucs.spark 2 | 3 | import org.apache.spark.rdd.RDD 4 | import org.apache.spark.sql.SQLContext 5 | import org.apache.spark.{SparkConf, SparkContext} 6 | import strucs.spark.StructDataFrame._ 7 | import strucs._ 8 | 9 | abstract class KeyCompanion[T](key: String) { 10 | implicit val keyProvider: StructKeyProvider[T] = StructKeyProvider[T](StructKey(key)) 11 | } 12 | 13 | // TODO explore tagged types 14 | case class Id(v: Int) extends AnyVal 15 | object Id { 16 | implicit val keyProvider = StructKeyProvider[Id](StructKey("id")) 17 | implicit val wrapper = Wrapper.materializeWrapper[Id, Int] 18 | } 19 | 20 | case class City(v: String) extends AnyVal 21 | object City { 22 | implicit val keyProvider: StructKeyProvider[City] = StructKeyProvider[City](StructKey("city")) 23 | implicit val wrapper = Wrapper.materializeWrapper[City, String] 24 | } 25 | 26 | 27 | case class Name(v: String) extends AnyVal 28 | object Name { 29 | implicit val keyProvider: StructKeyProvider[Name] = StructKeyProvider[Name](StructKey("name")) 30 | implicit val wrapper = Wrapper.materializeWrapper[Name, String] 31 | } 32 | case class Age(v: Int) extends AnyVal 33 | object Age { 34 | implicit val keyProvider = StructKeyProvider[Age](StructKey("age")) 35 | implicit val wrapper = Wrapper.materializeWrapper[Age, Int] 36 | 37 | } 38 | case class AvgAge(v: Double) extends AnyVal 39 | object AvgAge extends KeyCompanion("avgAge") { 40 | implicit val wrapper = Wrapper.materializeWrapper[AvgAge, Double] 41 | } 42 | 43 | case class MaxAge(v: Int) extends AnyVal 44 | object MaxAge extends KeyCompanion("maxAge") { 45 | implicit val wrapper = Wrapper.materializeWrapper[MaxAge, Int] 46 | } 47 | 48 | 49 | /** 50 | */ 51 | object SparkApp extends App { 52 | val conf = new SparkConf().setAppName("Simple Application").setMaster("local[2]") 53 | val sc = new SparkContext(conf) 54 | val sqlc = new SQLContext(sc) 55 | import org.apache.spark.sql.functions._ 56 | import sqlc.implicits._ 57 | 58 | val df = sc.makeRDD(Seq( 59 | ("Albert", 72), 60 | ("Gerard", 55), 61 | ("Gerard", 65))).toDF( 62 | "name" , "age") 63 | // Standard DataFrame: runtime failure if these fields do not exist 64 | df.groupBy("name").agg(avg("age"), max("age")).show() 65 | 66 | // StructDataFrame: method calls are type safe after the initial conversion 67 | val sdf: StructDataFrame[Name with Age]= df.toStructDF[Name, Age] 68 | sdf.select[Name].show() 69 | val avgSdf = sdf.groupBy[Name].agg[Age, AvgAge](avg) 70 | avgSdf.show() 71 | avgSdf.select[AvgAge].show() 72 | 73 | sdf.groupBy[Name].agg[Age, AvgAge, MaxAge](avg, max).select[Name, MaxAge].show() // TODO it would be nice to verify that avg cannot be called on a non-numeric type 74 | 75 | // RDD style 76 | val rdd: RDD[Struct[Name with AvgAge with Nil]] = avgSdf.rdd.map(s => Struct.empty + s.get[Name] + s.get[AvgAge]) 77 | println(rdd.collect().mkString("\n")) 78 | 79 | } 80 | 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /strucs-spark/src/test/scala/strucs/spark/StructDataFrameSpec.scala: -------------------------------------------------------------------------------- 1 | package strucs.spark 2 | 3 | import org.apache.spark.sql.SQLContext 4 | import org.apache.spark.{SparkContext, SparkConf} 5 | import org.scalactic.TypeCheckedTripleEquals 6 | import org.scalatest.{Matchers, FlatSpec} 7 | import strucs.{Nil, Struct} 8 | 9 | /** 10 | */ 11 | class StructDataFrameSpec extends FlatSpec with Matchers with TypeCheckedTripleEquals { 12 | val conf = new SparkConf().setAppName("Simple Application").setMaster("local[2]") 13 | val sc = new SparkContext(conf) 14 | val sqlc = new SQLContext(sc) 15 | import org.apache.spark.sql.functions._ 16 | import sqlc.implicits._ 17 | import StructDataFrame._ 18 | 19 | val df = sc.makeRDD(Seq( 20 | (1, "Albert", 72, "Paris"), 21 | (2, "Gerard", 55, "London"), 22 | (3, "Gerard", 65, "London"))).toDF( 23 | "id", "name" , "age", "city") 24 | 25 | val sdf1: StructDataFrame[Name] = df.toStructDF[Name] 26 | val sdf2: StructDataFrame[Id with Name with Age with City] = df.toStructDF[Id, Name, Age, City] 27 | 28 | "a StructDataFrame" can "select 1 column by its type" in { 29 | val actual: Array[Name] = sdf1.select[Name].collect().map(_.get[Name]) 30 | val expected = Array(Name("Albert"), Name("Gerard"), Name("Gerard")) 31 | actual should === (expected) 32 | } 33 | 34 | it should "prevent from selecting a column that is not in the DataFrame" in { 35 | assertTypeError("sdf1.select[AvgAge]") 36 | } 37 | 38 | it can "select 2 columns by their types" in { 39 | val actual: Array[(Name, Age)] = sdf2.select[Name, Age].collect().map(s => (s.get[Name], s.get[Age])) 40 | val expected = Array( 41 | (Name("Albert"), Age(72)), 42 | (Name("Gerard"), Age(55)), 43 | (Name("Gerard"), Age(65))) 44 | actual should === (expected) 45 | } 46 | 47 | it can "group by 2 columns using their types" in { 48 | val grouped = sdf2.groupBy[City, Name].agg[Age, AvgAge, MaxAge](avg, max) 49 | grouped.show() 50 | val actual = grouped.collect().map(s => (s.get[Name].v, s.get[AvgAge].v, s.get[MaxAge].v)) 51 | val expected = Array( 52 | ("Gerard", 60D, 65), 53 | ("Albert", 72D, 72) 54 | ) 55 | actual should === (expected) 56 | } 57 | 58 | 59 | } 60 | --------------------------------------------------------------------------------