├── .gitignore ├── project ├── build.properties └── plugins.sbt ├── src ├── main │ └── scala │ │ └── com │ │ └── tapad │ │ └── druid │ │ └── client │ │ ├── Expression.scala │ │ ├── Granularity.scala │ │ ├── Time.scala │ │ ├── OrderBy.scala │ │ ├── Aggregation.scala │ │ ├── TimeSeriesQuery.scala │ │ ├── GroupByQuery.scala │ │ ├── PostAggregation.scala │ │ ├── DSL.scala │ │ ├── TopNQuery.scala │ │ ├── QueryFilter.scala │ │ ├── DruidClient.scala │ │ └── Grammar.scala └── test │ └── scala │ └── com │ └── tapad │ └── druid │ └── client │ ├── IntervalGrammarSpec.scala │ ├── SqlClientExample.scala │ ├── WikipediaExample.scala │ └── Grammarspec.scala └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | .idea 3 | target 4 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.9 2 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/Expression.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s.JsonAST.JValue 4 | 5 | trait Expression { 6 | def toJson : JValue 7 | } 8 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | credentials += Credentials(Path.userHome / ".ivy2" / ".credentials") 2 | 3 | resolvers += "Scala Tools Nexus" at "http://nexus.tapad.com:8080/nexus/content/groups/aggregate/" 4 | 5 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/Granularity.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | case class Granularity(name: String) 4 | object Granularity { 5 | val All = Granularity("all") 6 | val Second = Granularity("second") 7 | val Minute = Granularity("minute") 8 | val FifteenMinute = Granularity("fifteen_minute") 9 | val ThirtyMinute = Granularity("thirty_minute") 10 | val Day = Granularity("day") 11 | val Hour = Granularity("hour") 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/Time.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.joda.time.format.ISODateTimeFormat 4 | import org.joda.time.{DateTime, Interval} 5 | 6 | object Time { 7 | private final val DateTimeFormat = ISODateTimeFormat.dateTime().withOffsetParsed() 8 | def intervalToString(i: Interval) : String = 9 | "%s/%s".format(DateTimeFormat.print(i.getStart), DateTimeFormat.print(i.getEnd)) 10 | 11 | def parse(s: String) : DateTime = DateTimeFormat.parseDateTime(s) 12 | } 13 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/OrderBy.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s.JsonAST._ 4 | import org.json4s.JsonDSL._ 5 | 6 | case class ColumnOrder(columnName: String, direction: String) extends Expression { 7 | def toJson: JValue = JObject( 8 | "dimension" -> columnName, 9 | "direction" -> direction 10 | ) 11 | } 12 | case class OrderBy(cols: Seq[ColumnOrder], limit : Option[Int] = None) extends Expression { 13 | def toJson: JValue = JObject( 14 | "type" -> "default", 15 | "columns" -> cols.map(_.toJson), 16 | "limit" -> limit.map(i => JInt(BigInt(i))).getOrElse(JNull) 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/Aggregation.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s.JsonAST._ 4 | import org.json4s.JsonDSL._ 5 | 6 | case class Aggregation(typeName: String, fieldName: String, outputName: String) extends Expression { 7 | def toJson = JObject( 8 | "type" -> typeName, 9 | "name" -> outputName, 10 | "fieldName" -> fieldName 11 | ) 12 | 13 | def as(outputName: String) = copy(outputName = outputName) 14 | } 15 | 16 | case class MultiFieldAggregation(typeName: String, fieldNames: Seq[String], outputName: String) extends Expression { 17 | def toJson = JObject( 18 | "type" -> typeName, 19 | "name" -> outputName, 20 | "fieldNames" -> fieldNames 21 | ) 22 | 23 | def as(outputName: String) = copy(outputName = outputName) 24 | } 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /src/test/scala/com/tapad/druid/client/IntervalGrammarSpec.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.scalatest.{Matchers, FlatSpec} 4 | import org.joda.time.{DateTime, Interval} 5 | import org.joda.time.format.DateTimeFormat 6 | 7 | class IntervalGrammarSpec extends FlatSpec with Matchers { 8 | import Grammar._ 9 | def date(s: String) = DateTimeFormat.forPattern("YYYY-MM-dd").parseDateTime(s) 10 | 11 | "The interval parser" should "parse intervals" in { 12 | def interval(s: String) : Interval = parser.parseAll(parser.interval, s) match { 13 | case parser.Success(r, _) => r.asInstanceOf[Interval] 14 | case x => fail(x.toString) 15 | } 16 | val i = interval("between '2013-01-01' and now()") 17 | i.getStart should be(date("2013-01-01")) 18 | i.getEnd.withTimeAtStartOfDay should be(new DateTime().withTimeAtStartOfDay) 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/test/scala/com/tapad/druid/client/SqlClientExample.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.joda.time.{DateTime, Interval} 4 | 5 | import scala.concurrent.{Future, ExecutionContext} 6 | import scala.util.{Failure, Success} 7 | 8 | object SqlClientExample { 9 | 10 | def main(args: Array[String]) { 11 | implicit val executionContext = ExecutionContext.Implicits.global 12 | val client = DruidClient("http://druid01.prd.nj1.tapad.com:8082") 13 | 14 | import com.tapad.druid.client.DSL._ 15 | 16 | client.queryTopN( 17 | "daily between '2015-12-03' and '2015-12-04' " + 18 | "select top 10 action_id, longSum(count) from tap where " + 19 | "campaign_id = 4383 order by count" 20 | ).onComplete { 21 | case Success(data) => data.data.foreach { println } 22 | } 23 | 24 | // client.queryTimeSeries( 25 | // "hourly between '2015-11-18T00:00:00.000-05:00' and '2015-11-26' " + 26 | // "select longSum(count), hyperUnique(unique_device_count) from tap where " + 27 | // "tactic_id = 129910 and " + 28 | // "action_id = 'impression'" 29 | // ).onComplete { 30 | // case Success(data) => data.data.foreach { case (ts, values) => println(s"$ts ${values("unique_device_count")}") } 31 | // } 32 | 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Getting started (requires the Wikipedia example to be running): 2 | 3 | See also https://github.com/daggerrz/druid-scala-client/blob/master/src/test/scala/com/tapad/druid/client/WikipediaExample.scala 4 | 5 | ```scala 6 | import scala.concurrent.ExecutionContext 7 | import org.joda.time.{DateTime, Interval} 8 | import scala.util.{Failure, Success} 9 | 10 | implicit val executionContext = ExecutionContext.Implicits.global 11 | val client = DruidClient("http://localhost:8083") 12 | 13 | import com.tapad.druid.client.DSL._ 14 | val query = GroupByQuery( 15 | source = "wikipedia", 16 | interval = new Interval(new DateTime().minusDays(1), new DateTime()), 17 | dimensions = Seq("page"), 18 | granularity = Granularity.All, 19 | aggregate = Seq( 20 | sum("count") as "edits", 21 | sum("added") as "chars_added" 22 | ), 23 | postAggregate = Seq( 24 | "chars_added" / "edits" as "chars_per_edit" 25 | ), 26 | filter = "namespace" === "article" and "country" === "United States", 27 | orderBy = Seq( 28 | "chars_added" desc 29 | ), 30 | limit = Some(100) 31 | ) 32 | 33 | client(query).onComplete { 34 | case Success(resp) => 35 | resp.data.foreach { row => 36 | println("Page %s, %s edits, %s chars added, %s per edit".format( 37 | row("page"), row("edits"), row("chars_added"), row("chars_per_edit") 38 | )) 39 | } 40 | case Failure(ex) => 41 | ex.printStackTrace() 42 | } 43 | ``` 44 | -------------------------------------------------------------------------------- /src/test/scala/com/tapad/druid/client/WikipediaExample.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import scala.concurrent.ExecutionContext 4 | import org.joda.time.{DateTime, Interval} 5 | import scala.util.{Failure, Success} 6 | 7 | object WikipediaExample { 8 | 9 | def main(args: Array[String]) { 10 | implicit val executionContext = ExecutionContext.Implicits.global 11 | val client = DruidClient("http://druid01.prd.nj1.tapad.com:8083") 12 | 13 | import com.tapad.druid.client.DSL._ 14 | val query = GroupByQuery( 15 | source = "tap", 16 | interval = new Interval(new DateTime().minusMonths(12), new DateTime()), 17 | dimensions = Seq("user_agent"), 18 | granularity = Granularity.All, 19 | aggregate = Seq( 20 | sum("count") as "user_agent_count" 21 | ), 22 | postAggregate = Seq( 23 | // "chars_added" / "edits" as "chars_per_edit" 24 | ), 25 | // filter = "namespace" === "article" and "country" === "United States", 26 | // orderBy = Seq( 27 | // "chars_added" desc 28 | // ), 29 | limit = Some(100) 30 | ) 31 | 32 | client(query).onComplete { 33 | case Success(resp) => 34 | resp.data.foreach { row => 35 | println(row) 36 | // println("Page %s, %s edits, %s chars added, %s per edit".format( 37 | // row("page"), row("edits"), row("chars_added"), row("chars_per_edit") 38 | // )) 39 | } 40 | System.exit(0) 41 | case Failure(ex) => 42 | ex.printStackTrace() 43 | System.exit(0) 44 | } 45 | 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/TimeSeriesQuery.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.joda.time.{DateTime, Interval} 4 | import org.json4s.JsonAST._ 5 | import org.json4s.JsonDSL._ 6 | import org.json4s.DefaultFormats._ 7 | import org.joda.time.format.ISODateTimeFormat 8 | 9 | case class TimeSeriesQuery(source: String, 10 | interval: Interval, 11 | granularity: Granularity, 12 | aggregate: Seq[Aggregation], 13 | postAggregate: Seq[PostAggregation] = Nil, 14 | filter : QueryFilter = QueryFilter.All) { 15 | def toJson : JValue = { 16 | JObject( 17 | "queryType" -> "timeseries", 18 | "dataSource" -> source, 19 | "granularity" -> granularity.name, 20 | "aggregations" -> aggregate.map(_.toJson), 21 | "postAggregations" -> postAggregate.map(_.toJson), 22 | "intervals" -> Time.intervalToString(interval), 23 | "filter" -> filter.toJson 24 | ) 25 | } 26 | } 27 | 28 | case class TimeSeriesResponse(data: Seq[(DateTime, Map[String, Any])]) 29 | object TimeSeriesResponse { 30 | implicit val formats = org.json4s.DefaultFormats 31 | def parse(js: JValue) : TimeSeriesResponse = { 32 | js match { 33 | case JArray(results) => 34 | val data = results.map { r => 35 | val time = Time.parse((r \ "timestamp").extract[String]) 36 | val values = (r \ "result").asInstanceOf[JObject].values 37 | time -> values 38 | } 39 | TimeSeriesResponse(data) 40 | case err @ _ => 41 | throw new IllegalArgumentException("Invalid time series response: " + err) 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/GroupByQuery.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.joda.time.Interval 4 | import org.json4s.JsonAST._ 5 | import org.json4s.JsonDSL._ 6 | 7 | case class GroupByQuery(source: String, 8 | interval: Interval, 9 | granularity: Granularity, 10 | dimensions: Seq[String], 11 | aggregate: Seq[Aggregation], 12 | postAggregate: Seq[PostAggregation] = Nil, 13 | filter : QueryFilter = QueryFilter.All, 14 | orderBy: Seq[ColumnOrder] = Nil, 15 | limit: Option[Int] = None) { 16 | def toJson : JValue = { 17 | JObject( 18 | "queryType" -> "groupBy", 19 | "dataSource" -> source, 20 | "granularity" -> granularity.name, 21 | "dimensions" -> dimensions, 22 | "aggregations" -> aggregate.map(_.toJson), 23 | "postAggregations" -> postAggregate.map(_.toJson), 24 | "intervals" -> Time.intervalToString(interval), 25 | "filter" -> filter.toJson, 26 | "orderBy" -> OrderBy(orderBy, limit).toJson 27 | ) 28 | } 29 | } 30 | 31 | case class GroupByResponse(data: Seq[Map[String, Any]]) 32 | object GroupByResponse { 33 | implicit val formats = org.json4s.DefaultFormats 34 | def parse(js: JValue) : GroupByResponse = { 35 | js match { 36 | case JArray(results) => 37 | val data = results.map { r => 38 | (r \ "event").asInstanceOf[JObject].values 39 | } 40 | GroupByResponse(data) 41 | case err @ _ => 42 | throw new IllegalArgumentException("Invalid time series response: " + err) 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/PostAggregation.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s.JsonAST.{JObject, JValue} 4 | import org.json4s.JsonAST._ 5 | import org.json4s.JsonDSL._ 6 | 7 | trait PostAggregationFieldSpec extends Expression { 8 | private def arith(rhs: PostAggregationFieldSpec, fn: String) : PostAggregation = ArithmeticPostAggregation("n/a", fn, Seq(this, rhs)) 9 | 10 | def *(rhs: PostAggregationFieldSpec) = arith(rhs, "*") 11 | def /(rhs: PostAggregationFieldSpec) = arith(rhs, "/") 12 | def +(rhs: PostAggregationFieldSpec) = arith(rhs, "+") 13 | def -(rhs: PostAggregationFieldSpec) = arith(rhs, "-") 14 | } 15 | trait PostAggregation extends PostAggregationFieldSpec { 16 | def as(outputName: String) : PostAggregation 17 | } 18 | 19 | object PostAggregation { 20 | def constant(value: Double) = ConstantPostAggregation("constant", value) 21 | 22 | case class FieldAccess(fieldName: String) extends PostAggregationFieldSpec { 23 | def toJson: JValue = JObject( 24 | "type" -> "fieldAccess", 25 | "fieldName" -> fieldName 26 | ) 27 | } 28 | 29 | } 30 | 31 | case class ConstantPostAggregation(outputName: String, value: Double) extends PostAggregation { 32 | def toJson: JValue = JObject( 33 | "type" -> "constant", 34 | "name" -> outputName, 35 | "value" -> value 36 | ) 37 | 38 | def as(outputName: String): PostAggregation = copy(outputName = outputName) 39 | } 40 | 41 | case class ArithmeticPostAggregation(outputName: String, fn: String, fields: Seq[PostAggregationFieldSpec]) extends PostAggregation { 42 | def toJson: JValue = JObject( 43 | "type" -> "arithmetic", 44 | "name" -> outputName, 45 | "fn" -> fn, 46 | "fields" -> fields.map(_.toJson) 47 | ) 48 | 49 | def as(outputName: String) = copy(outputName = outputName) 50 | } -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/DSL.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | object DSL { 4 | import QueryFilter._ 5 | import PostAggregation._ 6 | case class FilterOps(dimension: String) { 7 | def ===(value: String) = where(dimension, value) 8 | def =*=(pattern: String) = regex(dimension, pattern) 9 | } 10 | implicit def string2FilterOps(s: String) : FilterOps = FilterOps(s) 11 | 12 | case class PostAggStringOps(lhs: String) { 13 | def /(rhs: String) = ArithmeticPostAggregation("%s_by_%s".format(lhs, rhs), "/", Seq(FieldAccess(lhs), FieldAccess(rhs))) 14 | def *(rhs: String) = ArithmeticPostAggregation("%s_times_%s".format(lhs, rhs), "*", Seq(FieldAccess(lhs), FieldAccess(rhs))) 15 | def -(rhs: String) = ArithmeticPostAggregation("%s_minus_%s".format(lhs, rhs), "-", Seq(FieldAccess(lhs), FieldAccess(rhs))) 16 | def +(rhs: String) = ArithmeticPostAggregation("%s_plus_%s".format(lhs, rhs), "-", Seq(FieldAccess(lhs), FieldAccess(rhs))) 17 | } 18 | 19 | implicit def string2PostAggOps(s: String) : PostAggStringOps = PostAggStringOps(s) 20 | implicit def string2PostAgg(s: String) : PostAggregationFieldSpec = ArithmeticPostAggregation("no_name", "*", Seq(FieldAccess(s), constant(1))) 21 | implicit def numericToConstant[T](n: T)(implicit num: Numeric[T]) : ConstantPostAggregation = constant(num.toDouble(n)) 22 | 23 | case class OrderByStringOps(col: String) { 24 | def asc = ColumnOrder(col, "ASCENDING") 25 | def desc = ColumnOrder(col, "DESCENDING") 26 | } 27 | implicit def string2OrderByOps(s: String) : OrderByStringOps = OrderByStringOps(s) 28 | 29 | def sum(fieldName: String) = Aggregation("longSum", fieldName, fieldName + "_sum") 30 | def doubleSum(fieldName: String) = Aggregation("doubleSum", fieldName, fieldName + "_sum") 31 | def count = Aggregation("count", "na", "row_count") 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/TopNQuery.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.joda.time.{DateTime, Interval} 4 | import org.json4s.JsonAST._ 5 | import org.json4s.JsonDSL._ 6 | import org.json4s.DefaultFormats._ 7 | import org.joda.time.format.ISODateTimeFormat 8 | 9 | case class TopNQuery(source: String, 10 | interval: Interval, 11 | granularity: Granularity, 12 | dimension: String, 13 | metric: String, 14 | threshold: Int, 15 | aggregate: Seq[Aggregation], 16 | postAggregate: Seq[PostAggregation] = Nil, 17 | filter: QueryFilter = QueryFilter.All) { 18 | def toJson: JValue = { 19 | JObject( 20 | "queryType" -> "topN", 21 | "dataSource" -> source, 22 | "granularity" -> granularity.name, 23 | "dimension" -> dimension, 24 | "threshold" -> threshold, 25 | "metric" -> metric, 26 | "aggregations" -> aggregate.map(_.toJson), 27 | "postAggregations" -> postAggregate.map(_.toJson), 28 | "intervals" -> Time.intervalToString(interval), 29 | "filter" -> filter.toJson 30 | ) 31 | } 32 | } 33 | 34 | case class TopNResponse(data: Seq[(DateTime, Seq[Map[String, Any]])]) 35 | object TopNResponse { 36 | implicit val formats = org.json4s.DefaultFormats 37 | def parse(js: JValue) : TopNResponse = { 38 | js match { 39 | case JArray(results) => 40 | val data = results.map { r => 41 | val time = Time.parse((r \ "timestamp").extract[String]) 42 | val results = (r \ "result").asInstanceOf[JArray] 43 | val valueSeq : Seq[Map[String, Any]] = results.arr.map { v => 44 | v.asInstanceOf[JObject].values 45 | } 46 | time -> valueSeq 47 | } 48 | TopNResponse(data) 49 | case err @ _ => 50 | throw new IllegalArgumentException("Invalid top N response: " + err) 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/QueryFilter.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s.JsonAST.{JNull, JArray, JObject, JValue} 4 | import org.json4s.JsonDSL._ 5 | 6 | sealed trait QueryFilter extends Expression { 7 | def and(other: QueryFilter) : QueryFilter = And(Seq(this, other)) 8 | def or(other: QueryFilter) : QueryFilter = Or(Seq(this, other)) 9 | } 10 | 11 | case class And(filters: Seq[Expression]) extends QueryFilter { 12 | 13 | override def and(other: QueryFilter): QueryFilter = copy(other +: filters) 14 | 15 | def toJson: JValue = JObject( 16 | "type" -> "and", 17 | "fields" -> JArray(filters.toList.map(_.toJson)) 18 | ) 19 | } 20 | case class Or(filters: Seq[Expression]) extends QueryFilter { 21 | 22 | override def or(other: QueryFilter): QueryFilter = copy(other +: filters) 23 | 24 | def toJson: JValue = JObject( 25 | "type" -> "or", 26 | "fields" -> JArray(filters.toList.map(_.toJson)) 27 | ) 28 | } 29 | 30 | case class ExprQueryFilter(typeName: String, dimension: String, value: String) extends QueryFilter { 31 | def toJson: JValue = JObject( 32 | "type" -> typeName, 33 | "dimension" -> dimension, 34 | "value" -> value 35 | ) 36 | } 37 | case class SelectorQueryFilter(dimension: String, value: String) extends QueryFilter { 38 | def toJson: JValue = JObject( 39 | "type" -> "selector", 40 | "dimension" -> dimension, 41 | "value" -> value 42 | ) 43 | } 44 | case class RegexQueryFilter(dimension: String, pattern: String) extends QueryFilter { 45 | def toJson: JValue = JObject( 46 | "type" -> "regex", 47 | "dimension" -> dimension, 48 | "pattern" -> pattern 49 | ) 50 | } 51 | 52 | object QueryFilter { 53 | 54 | def custom(typeName: String, dimension: String, value: String) = ExprQueryFilter(typeName, dimension, value) 55 | def where(dimension: String, value: String) = SelectorQueryFilter(dimension, value) 56 | def regex(dimension: String, pattern: String) = RegexQueryFilter(dimension, pattern) 57 | 58 | val All = new QueryFilter { 59 | def toJson: JValue = JNull 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/DruidClient.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.json4s._ 4 | import org.json4s.jackson._ 5 | import org.json4s.jackson.JsonMethods._ 6 | import com.ning.http.client._ 7 | 8 | import scala.concurrent.{ExecutionContext, Promise, Future} 9 | 10 | case class DruidClient(serverUrl: String)(implicit val executionContext: ExecutionContext) { 11 | private val config = new AsyncHttpClientConfig.Builder() 12 | private val client = new AsyncHttpClient(config.build()) 13 | private val url = s"$serverUrl/druid/v2/?pretty" 14 | 15 | private def JsonPost(body: String) = { 16 | client.preparePost(url) 17 | .setHeader("Content-Type", "application/json") 18 | .setBody(body) 19 | } 20 | 21 | private def parseJson(resp: Response): JValue = { 22 | val body = resp.getResponseBody("UTF-8") 23 | parse(body) 24 | } 25 | 26 | private def execute[R](js: JValue, parser: JValue => R) : Future[R] = { 27 | val p = Promise[Response] 28 | val body = compactJson(js) 29 | JsonPost(body).execute(new AsyncCompletionHandler[Response] { 30 | override def onCompleted(response: Response): Response = { 31 | p.success(response) 32 | response 33 | } 34 | }) 35 | p.future.map(parseJson).map(parser) 36 | } 37 | 38 | def apply(ts: TimeSeriesQuery) : Future[TimeSeriesResponse] = execute(ts.toJson, TimeSeriesResponse.parse) 39 | def apply(ts: GroupByQuery) : Future[GroupByResponse] = execute(ts.toJson, GroupByResponse.parse) 40 | 41 | def queryTimeSeries(query: String) : Future[TimeSeriesResponse] = { 42 | Grammar.parser.parseAll(Grammar.parser.timeSeries, query) match { 43 | case Grammar.parser.Success(ts, _) => execute(ts.toJson, TimeSeriesResponse.parse) 44 | case failure => throw new IllegalArgumentException(failure.toString) 45 | } 46 | } 47 | 48 | def queryGroupBy(query: String) : Future[GroupByResponse] = { 49 | Grammar.parser.parseAll(Grammar.parser.groupByQuery, query) match { 50 | case Grammar.parser.Success(ts, _) => execute(ts.toJson, GroupByResponse.parse) 51 | case failure => throw new IllegalArgumentException(failure.toString) 52 | } 53 | } 54 | 55 | def queryTopN(query: String) : Future[TopNResponse] = { 56 | Grammar.parser.parseAll(Grammar.parser.topNQuery, query) match { 57 | case Grammar.parser.Success(ts, _) => execute(ts.toJson, TopNResponse.parse) 58 | case failure => throw new IllegalArgumentException(failure.toString) 59 | } 60 | } 61 | 62 | } 63 | 64 | -------------------------------------------------------------------------------- /src/test/scala/com/tapad/druid/client/Grammarspec.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import org.scalatest._ 4 | import org.joda.time.format.DateTimeFormat 5 | import org.joda.time.{DateTime, Interval} 6 | 7 | class GrammarSpec extends FlatSpec with Matchers { 8 | 9 | import Grammar._ 10 | def date(s: String) = DateTimeFormat.forPattern("YYYY-MM-dd").parseDateTime(s) 11 | 12 | def expr(s: String): Expression = parser.parseAll(parser.filterExpression, s) match { 13 | case parser.Success(r, _) => r.asInstanceOf[Expression] 14 | case x => fail(x.toString) 15 | } 16 | 17 | "The expression parser" should "parse simple numeric filter expressions" in { 18 | expr("age = 9000") should be(SelectorQueryFilter("age", "9000")) 19 | } 20 | it should "parse simple string double quote filter expressions" in { 21 | expr( """name = "Druid"""") should be(SelectorQueryFilter("name", "Druid")) 22 | } 23 | it should "parse simple string single quote filter expressions" in { 24 | expr( """name = 'Druid'""") should be(SelectorQueryFilter("name", "Druid")) 25 | } 26 | it should "parse escaped string filter expressions" in { 27 | expr( """name = "Druid has a ""name"" " """) should be(SelectorQueryFilter("name", """Druid has a "name" """)) 28 | } 29 | it should "parse nested expressions" in { 30 | expr( """name = "Druid" and age = 9000 or age = 21""") should be( 31 | Or(Seq( 32 | And( 33 | Seq( 34 | SelectorQueryFilter("name", "Druid"), 35 | SelectorQueryFilter("age", "9000") 36 | )), 37 | SelectorQueryFilter("age", "21") 38 | )) 39 | ) 40 | } 41 | it should "parse honor parens in expressions" in { 42 | val res = expr( """(name = "Druid") and (age = 9000 or (age = 21 and level = "VIP"))""") 43 | res should be( 44 | And(Seq( 45 | SelectorQueryFilter("name", "Druid"), 46 | Or( 47 | Seq( 48 | SelectorQueryFilter("age", "9000"), 49 | And(Seq( 50 | SelectorQueryFilter("age", "21"), 51 | SelectorQueryFilter("level", "VIP") 52 | )) 53 | ) 54 | ))) 55 | ) 56 | } 57 | 58 | 59 | "The query parser" should "parse time series expressions" in { 60 | def ts(s: String): TimeSeriesQuery = parser.parseAll(parser.timeSeries, s) match { 61 | case parser.Success(r, _) => r.asInstanceOf[TimeSeriesQuery] 62 | case x => fail(x.toString) 63 | } 64 | ts("hourly between '2013-01-01' and '2013-01-31' select longSum(users), longSum(pageViews) as pages from users where age = 50") should be ( 65 | TimeSeriesQuery( 66 | source = "users", 67 | interval = new Interval(date("2013-01-01"), date("2013-01-31")), 68 | granularity = Granularity.Hour, 69 | aggregate = Seq( 70 | Aggregation("longSum", "users", "users"), 71 | Aggregation("longSum", "pageViews", "pages") 72 | ), 73 | postAggregate = Nil, 74 | filter = QueryFilter.where("age", "50") 75 | ) 76 | ) 77 | ts("daily (between '2013-01-01' and '2013-01-31T16:00:00') select longSum(users), longSum(pageViews) as pages from users where age = 50") should be ( 78 | TimeSeriesQuery( 79 | source = "users", 80 | interval = new Interval(date("2013-01-01"), date("2013-01-31").plusHours(16)), 81 | granularity = Granularity.Day, 82 | aggregate = Seq( 83 | Aggregation("longSum", "users", "users"), 84 | Aggregation("longSum", "pageViews", "pages") 85 | ), 86 | postAggregate = Nil, 87 | filter = QueryFilter.where("age", "50") 88 | ) 89 | ) 90 | } 91 | 92 | "The query parser" should "parse top N expressions" in { 93 | def p(s: String): TopNQuery = parser.parseAll(parser.topNQuery, s) match { 94 | case parser.Success(r, _) => r.asInstanceOf[TopNQuery] 95 | case x => fail(x.toString) 96 | } 97 | p("daily (between '2013-01-01' and '2013-01-31T16:00:00') select top 10 tactic_id, longSum(users), longSum(pageViews) as pages from users where age = 50 order by users") should be ( 98 | TopNQuery( 99 | source = "users", 100 | interval = new Interval(date("2013-01-01"), date("2013-01-31").plusHours(16)), 101 | granularity = Granularity.Day, 102 | threshold = 10, 103 | dimension = "tactic_id", 104 | metric = "users", 105 | aggregate = Seq( 106 | Aggregation("longSum", "users", "users"), 107 | Aggregation("longSum", "pageViews", "pages") 108 | ), 109 | postAggregate = Nil, 110 | filter = QueryFilter.where("age", "50") 111 | ) 112 | ) 113 | } 114 | } -------------------------------------------------------------------------------- /src/main/scala/com/tapad/druid/client/Grammar.scala: -------------------------------------------------------------------------------- 1 | package com.tapad.druid.client 2 | 3 | import scala.util.parsing.combinator.Parsers 4 | import scala.util.parsing.combinator.RegexParsers 5 | import org.joda.time.{DateTime, Interval} 6 | import org.joda.time.format.{ISODateTimeFormat, DateTimeFormatterBuilder, DateTimeFormat} 7 | 8 | object Grammar extends Parsers { 9 | 10 | /** 11 | * Convenience for construction filter expressions without using the full parser. 12 | */ 13 | def parseFilter(filterString: String) : Either[String, QueryFilter] = { 14 | parser.parseAll(parser.filterExpression, filterString) match { 15 | case parser.Success(r, _) => Right(r.asInstanceOf[QueryFilter]) 16 | case failure => Left(failure.toString) 17 | } 18 | } 19 | 20 | val parser = new RegexParsers { 21 | override type Elem = Char 22 | 23 | def keyword = "and|or".r 24 | 25 | // Single and double quoted strings with double escape 26 | def stringLiteral: Parser[String] = 27 | "\"([^\"]|[\"\"]{2})*\"".r ^^ { s => s.substring(1, s.length - 1 ).replaceAll("\"\"", "\"") } | 28 | "\'([^\']|[\'\']{2})*\'".r ^^ { s => s.substring(1, s.length - 1 ).replaceAll("''", "'") } 29 | 30 | 31 | def integer = """(0|[1-9]\d*)""".r 32 | def literal = stringLiteral | integer 33 | def identifier = """[_\p{L}][_\p{L}\p{Nd}]*""".r 34 | 35 | def selectorFilter : Parser[QueryFilter] = identifier~"="~literal ^^ 36 | { case dim~op~value => SelectorQueryFilter(dim, value) } 37 | 38 | def parens = "\\(".r ~> filterExpression <~ "\\)".r 39 | def term = parens | selectorFilter 40 | 41 | def filterExpression : Parser[QueryFilter] = term * ( 42 | "and" ^^^ { (e1: QueryFilter, e2: QueryFilter) => And(Seq(e1, e2))} | 43 | "or" ^^^ { (e1: QueryFilter, e2: QueryFilter) => Or(Seq(e1, e2))} 44 | ) 45 | 46 | def whereClause : Parser[QueryFilter] = "where".r ~ filterExpression ^^ { case _ ~ filter => filter} 47 | 48 | def columnOrder : Parser[ColumnOrder] = identifier ~ (("asc".r | "desc".r)?) ^^ 49 | { case dim~direction => ColumnOrder(dim, direction.getOrElse("desc"))} 50 | 51 | def groupByClause : Parser[Seq[String]] = "group by".r ~ repsep(identifier, ",") ^^ { case _ ~ cols => cols } 52 | 53 | def limit : Parser[Int] = "limit".r ~ integer ^^ { 54 | case _ ~ limit => limit.toInt 55 | } 56 | 57 | def orderByClause : Parser[OrderBy] = "order by".r ~ repsep(columnOrder, ",".r) ~ (limit ?) ^^ { 58 | case _ ~ cols ~ limit => OrderBy(cols, limit) 59 | } 60 | 61 | def aggregationAlias : Parser[String] = "as".r ~ identifier ^^ { case _~alias => alias } 62 | def aggregation : Parser[Aggregation] = identifier ~ ("\\(".r ~> identifier <~ "\\)".r) ~ (aggregationAlias ?) ^^ { 63 | case agg~dimension~alias => 64 | Aggregation(typeName = agg, fieldName = dimension, outputName = alias.getOrElse(dimension)) 65 | } 66 | 67 | 68 | final val dateFormat = ISODateTimeFormat.dateOptionalTimeParser 69 | 70 | def absoluteDateTime : Parser[DateTime] = stringLiteral ^^ { s => dateFormat.parseDateTime(s) } 71 | def now : Parser[DateTime] = "now\\(\\)".r ^^^ { new DateTime() } 72 | def dateTime : Parser[DateTime] = (now | absoluteDateTime).withFailureMessage("Expected date literal or expression") 73 | def between : Parser[Interval] = "between".r ~ dateTime ~ "and".r ~ dateTime ^^ { 74 | case _ ~ from ~ _ ~ to => new Interval(from, to) 75 | } 76 | def lastN : Parser[Interval] = "last".r ~ integer ~ """days|day|hours|hour|minutes|minute""".r ^^ { 77 | case _~amount~unit => 78 | val now = new DateTime() 79 | val n = amount.toInt 80 | unit match { 81 | case "days" | "day" => new Interval(now.minusDays(n), now) 82 | case "hours" | "hour" => new Interval(now.minusHours(n), now) 83 | case "minutes" | "minute" => new Interval(now.minusMinutes(n), now) 84 | } 85 | } 86 | def interval : Parser[Interval] = between | lastN | "\\(".r ~> interval <~ "\\)".r 87 | 88 | def granularity : Parser[Granularity] = """all|1h|hourly|1min|15min|30min|1d|daily""".r ^^ { 89 | case "1h" | "hourly" => Granularity.Hour 90 | case "1min" => Granularity.Minute 91 | case "15min" => Granularity.FifteenMinute 92 | case "30min" => Granularity.ThirtyMinute 93 | case "1d" | "daily" => Granularity.Day 94 | case "all" => Granularity.All 95 | } 96 | 97 | def timeSeries : Parser[TimeSeriesQuery] = 98 | granularity ~ 99 | interval ~ "select".r ~ 100 | repsep(aggregation, ",") ~ 101 | "from" ~ identifier ~(whereClause?) ^^ { 102 | case granularity ~ interval ~ _ ~ aggregates ~ _ ~ source ~ filter => 103 | TimeSeriesQuery( 104 | source = source, 105 | interval = interval, 106 | granularity = granularity, 107 | aggregate = aggregates, 108 | postAggregate = Nil, 109 | filter = filter.getOrElse(QueryFilter.All) 110 | ) 111 | } 112 | 113 | def groupByQuery : Parser[GroupByQuery] = 114 | granularity ~ 115 | interval ~ "select" ~ 116 | repsep(aggregation, ",") ~ 117 | "from" ~ identifier ~(whereClause?) ~ (groupByClause?) ^^ { 118 | case granularity ~ interval ~ _ ~ aggregates ~ _ ~ source ~ filter ~ groupBy => 119 | GroupByQuery( 120 | source = source, 121 | dimensions = groupBy.getOrElse(Nil), 122 | interval = interval, 123 | granularity = granularity, 124 | aggregate = aggregates, 125 | postAggregate = Nil, 126 | filter = filter.getOrElse(QueryFilter.All) 127 | ) 128 | } 129 | 130 | 131 | def topNQuery : Parser[TopNQuery] = 132 | granularity ~ 133 | interval ~ "select" ~ "top" ~ integer ~ identifier ~ "," ~ 134 | repsep(aggregation, ",") ~ 135 | "from" ~ identifier ~(whereClause?) ~ "order by".r ~ identifier ^^ { 136 | case granularity ~ interval ~ _ ~ _ ~ threshold ~ identifier ~ _ ~ aggregates ~ _ ~ source ~ filter ~ _ ~ metric => 137 | TopNQuery( 138 | source = source, 139 | dimension = identifier, 140 | threshold = threshold.toInt, 141 | metric = metric, 142 | interval = interval, 143 | granularity = granularity, 144 | aggregate = aggregates, 145 | postAggregate = Nil, 146 | filter = filter.getOrElse(QueryFilter.All) 147 | ) 148 | } 149 | } 150 | 151 | } 152 | --------------------------------------------------------------------------------