├── .gitignore ├── .travis.yml ├── README.md ├── build.sbt ├── build ├── sbt └── sbt-launch-lib.bash ├── codecov.yml ├── conf ├── log4j.properties └── spear.conf ├── project ├── Dependencies.scala ├── build.properties └── plugins.sbt ├── scalariform.properties ├── scalastyle-config.xml ├── spear-core └── src │ ├── main │ ├── java │ │ └── spear │ │ │ ├── parsers │ │ │ └── annotations │ │ │ │ └── ExtendedSQLSyntax.java │ │ │ └── plans │ │ │ └── logical │ │ │ └── annotations │ │ │ └── Explain.java │ ├── resources │ │ └── spear-reference.conf │ └── scala │ │ └── spear │ │ ├── Catalog.scala │ │ ├── Context.scala │ │ ├── DataFrame.scala │ │ ├── FunctionRegistry.scala │ │ ├── Name.scala │ │ ├── QueryCompiler.scala │ │ ├── Row.scala │ │ ├── RowOrdering.scala │ │ ├── config │ │ ├── Settings.scala │ │ └── package.scala │ │ ├── exceptions │ │ └── exceptions.scala │ │ ├── execution │ │ └── Projection.scala │ │ ├── expressions │ │ ├── Cast.scala │ │ ├── Expression.scala │ │ ├── InternalAlias.scala │ │ ├── Literal.scala │ │ ├── Predicate.scala │ │ ├── SortOrder.scala │ │ ├── aggregates │ │ │ ├── basic.scala │ │ │ ├── interfaces.scala │ │ │ ├── logical.scala │ │ │ └── numeric.scala │ │ ├── arithmetics.scala │ │ ├── comparisons.scala │ │ ├── complexTypes.scala │ │ ├── dsl │ │ │ └── ExpressionDSL.scala │ │ ├── functions │ │ │ └── package.scala │ │ ├── logicalOperators.scala │ │ ├── misc.scala │ │ ├── named.scala │ │ ├── nullExpressions.scala │ │ ├── object.scala │ │ ├── package.scala │ │ ├── patterns.scala │ │ ├── stateful.scala │ │ ├── stringExpressions.scala │ │ ├── typecheck │ │ │ └── TypeConstraint.scala │ │ └── windows │ │ │ ├── WindowFunction.scala │ │ │ └── WindowSpec.scala │ │ ├── package.scala │ │ ├── parsers │ │ ├── DataTypeParser.scala │ │ ├── DirectlyExecutableStatementParser.scala │ │ ├── IdentifierParser.scala │ │ ├── KeywordParser.scala │ │ ├── LoggingParser.scala │ │ ├── ParserImplicits.scala │ │ ├── QueryExpressionParser.scala │ │ ├── SeparatorParser.scala │ │ ├── ValueExpressionParser.scala │ │ └── package.scala │ │ ├── plans │ │ ├── CompiledQuery.scala │ │ ├── QueryPlan.scala │ │ ├── QueryPlanner.scala │ │ ├── logical │ │ │ ├── LogicalPlan.scala │ │ │ ├── Optimizer.scala │ │ │ ├── analysis │ │ │ │ ├── Analyzer.scala │ │ │ │ ├── aggregationAnalysis.scala │ │ │ │ ├── expressionsAnalysis.scala │ │ │ │ ├── postAnalysisCheck.scala │ │ │ │ └── windowAnalysis.scala │ │ │ ├── package.scala │ │ │ └── patterns │ │ │ │ └── package.scala │ │ └── physical │ │ │ └── PhysicalPlan.scala │ │ ├── reflection │ │ └── package.scala │ │ └── types │ │ ├── DataType.scala │ │ ├── complexTypes.scala │ │ └── numericTypes.scala │ └── test │ ├── resources │ ├── log4j.properties │ └── spear-test.conf │ └── scala │ └── spear │ ├── DataFrameSuite.scala │ ├── ExpressionSQLBuilderSuite.scala │ ├── NameSuite.scala │ ├── SQLBuilderTest.scala │ ├── Test.scala │ ├── TestQueryCompiler.scala │ ├── TestUtils.scala │ ├── config │ └── SettingsSuite.scala │ ├── execution │ └── ProjectionSuite.scala │ ├── expressions │ ├── ArithmeticExpressionSuite.scala │ ├── CastSuite.scala │ ├── ComparisonSuite.scala │ ├── LogicalOperatorSuite.scala │ ├── NullExpressionsSuite.scala │ ├── ObjectExpressionsSuite.scala │ ├── StringExpressionsSuite.scala │ ├── typecheck │ │ └── TypeConstraintSuite.scala │ └── windows │ │ └── WindowSpecSuite.scala │ ├── generators │ ├── expressions │ │ └── package.scala │ ├── package.scala │ ├── types │ │ └── package.scala │ └── values │ │ └── package.scala │ ├── parsers │ ├── DataTypeParserSuite.scala │ ├── DirectlyExecutableStatementParserSuite.scala │ ├── IdentifierParserSuite.scala │ └── ValueExpressionParserSuite.scala │ ├── plans │ └── logical │ │ ├── OptimizerSuite.scala │ │ └── analysis │ │ ├── AggregationAnalysisSuite.scala │ │ ├── AnalyzerTest.scala │ │ ├── CTEAnalysisSuite.scala │ │ ├── ExpressionAnalysisSuite.scala │ │ ├── MiscAnalysisSuite.scala │ │ ├── PostAnalysisCheckSuite.scala │ │ ├── WindowAnalysisTest.scala │ │ ├── WindowAnalysisWithGroupBySuite.scala │ │ └── WindowAnalysisWithoutGroupBySuite.scala │ ├── reflection │ └── SchemaReflectionSuite.scala │ ├── trees │ └── TreeNodeSuite.scala │ └── types │ └── DataTypeSuite.scala ├── spear-docs └── src │ └── sphinx │ ├── Makefile │ ├── _static │ └── .git-keep │ ├── conf.py │ ├── index.rst │ └── overview.rst ├── spear-examples └── src │ └── main │ └── scala │ └── BasicExample.scala ├── spear-local └── src │ ├── main │ └── scala │ │ └── spear │ │ └── local │ │ ├── LocalQueryCompiler.scala │ │ └── plans │ │ └── physical │ │ ├── HashAggregate.scala │ │ ├── basicOperators.scala │ │ └── dsl │ │ └── package.scala │ └── test │ ├── resources │ └── log4j.properties │ └── scala │ └── spear │ ├── LocalQueryCompilerSuite.scala │ └── local │ └── plans │ └── physical │ └── LocalPhysicalPlanSuite.scala ├── spear-repl └── src │ └── main │ ├── resources │ ├── log4j.properties │ ├── predef.scala │ └── spear-full-reference.conf │ └── scala │ └── spear │ └── repl │ └── Main.scala ├── spear-trees └── src │ └── main │ └── scala │ └── spear │ └── trees │ ├── Transformer.scala │ └── TreeNode.scala └── spear-utils └── src ├── main └── scala │ └── spear │ └── utils │ ├── Logging.scala │ └── package.scala └── test └── scala └── spear └── LoggingFunSuite.scala /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | build/sbt-launch-*.jar 3 | project/project/ 4 | project/target/ 5 | target/ 6 | *.iml 7 | drafts/ 8 | logs/ 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | sudo: false 3 | cache: 4 | directories: 5 | - $HOME/.ivy2 6 | matrix: 7 | include: 8 | - jdk: openjdk7 9 | scala: 2.11.7 10 | script: 11 | - sbt coverage test coverageReport doc 12 | after_script: 13 | - bash <(curl -s https://codecov.io/bash) 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | [![Build Status][travis-ci-badge]][travis-ci] [![codecov.io][codecov-badge]][codecov] 4 | 5 | ![Codecov.io][codecov-history] 6 | 7 | [travis-ci-badge]: https://travis-ci.org/liancheng/spear.svg?branch=master 8 | [travis-ci]: https://travis-ci.org/liancheng/spear 9 | [codecov-badge]: https://codecov.io/github/liancheng/spear/coverage.svg?branch=master 10 | [codecov]: https://codecov.io/github/liancheng/spear?branch=master 11 | [codecov-history]: https://codecov.io/github/liancheng/spear/branch.svg?branch=master 12 | 13 | This project is a sandbox and playground of mine for experimenting ideas and potential improvements to Spark SQL. It consists of: 14 | 15 | - A parser that parses a small SQL dialect into unresolved logical plans 16 | - A semantic analyzer that resolves unresolved logical plans into resolved ones 17 | - A query optimizer that optimizes resolved query plans into equivalent but more performant ones 18 | - A query planner that turns (optimized) logical plans into executable physical plans 19 | 20 | Currently Spear only works with local Scala collections. 21 | 22 | # Build 23 | 24 | Building Spear is as easy as: 25 | 26 | ``` 27 | $ ./build/sbt package 28 | ``` 29 | 30 | # Run the REPL 31 | 32 | Spear has an Ammonite-based REPL for interactive experiments. To start it: 33 | 34 | ``` 35 | $ ./build/sbt spear-repl/run 36 | ``` 37 | 38 | Let's create a simple DataFrame of numbers: 39 | 40 | ```scala 41 | @ context range 10 show () 42 | ``` 43 | 44 | ``` 45 | ╒══╕ 46 | │id│ 47 | ├──┤ 48 | │ 0│ 49 | │ 1│ 50 | │ 2│ 51 | │ 3│ 52 | │ 4│ 53 | │ 5│ 54 | │ 6│ 55 | │ 7│ 56 | │ 8│ 57 | │ 9│ 58 | ╘══╛ 59 | ``` 60 | 61 | A sample query using the DataFrame API: 62 | 63 | ```scala 64 | @ context. 65 | range(10). 66 | select('id as 'key, (rand(42) * 100) cast IntType as 'value). 67 | where('value % 2 === 0). 68 | orderBy('value.desc). 69 | show() 70 | ``` 71 | 72 | ``` 73 | ╒═══╤═════╕ 74 | │key│value│ 75 | ├───┼─────┤ 76 | │ 5│ 90│ 77 | │ 9│ 78│ 78 | │ 0│ 72│ 79 | │ 1│ 68│ 80 | │ 4│ 66│ 81 | │ 8│ 46│ 82 | │ 6│ 36│ 83 | │ 2│ 30│ 84 | ╘═══╧═════╛ 85 | ``` 86 | 87 | Equivalent sample query using SQL: 88 | 89 | ```scala 90 | @ context range 10 asTable 't // Registers a temporary table first 91 | 92 | @ context.sql( 93 | """SELECT * FROM ( 94 | | SELECT id AS key, CAST(RAND(42) * 100 AS INT) AS value FROM t 95 | |) s 96 | |WHERE value % 2 = 0 97 | |ORDER BY value DESC 98 | |""".stripMargin 99 | ).show() 100 | ``` 101 | 102 | ``` 103 | ╒═══╤═════╕ 104 | │key│value│ 105 | ├───┼─────┤ 106 | │ 5│ 90│ 107 | │ 9│ 78│ 108 | │ 0│ 72│ 109 | │ 1│ 68│ 110 | │ 4│ 66│ 111 | │ 8│ 46│ 112 | │ 6│ 36│ 113 | │ 2│ 30│ 114 | ╘═══╧═════╛ 115 | ``` 116 | 117 | We can also check the query plan using `explain()`: 118 | 119 | ```scala 120 | @ context. 121 | range(10). 122 | select('id as 'key, (rand(42) * 100) cast IntType as 'value). 123 | where('value % 2 === 0). 124 | orderBy('value.desc). 125 | explain(true) 126 | ``` 127 | 128 | ``` 129 | # Logical plan 130 | Sort: order=[$0] ⇒ [?output?] 131 | │ ╰╴$0: `value` DESC NULLS FIRST 132 | ╰╴Filter: condition=$0 ⇒ [?output?] 133 | │ ╰╴$0: ((`value` % 2:INT) = 0:INT) 134 | ╰╴Project: projectList=[$0, $1] ⇒ [?output?] 135 | │ ├╴$0: (`id` AS `key`#11) 136 | │ ╰╴$1: (CAST((RAND(42:INT) * 100:INT) AS INT) AS `value`#12) 137 | ╰╴LocalRelation: data= ⇒ [`id`#10:BIGINT!] 138 | 139 | # Analyzed plan 140 | Sort: order=[$0] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 141 | │ ╰╴$0: `value`#12:INT! DESC NULLS FIRST 142 | ╰╴Filter: condition=$0 ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 143 | │ ╰╴$0: ((`value`#12:INT! % 2:INT) = 0:INT) 144 | ╰╴Project: projectList=[$0, $1] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 145 | │ ├╴$0: (`id`#10:BIGINT! AS `key`#11) 146 | │ ╰╴$1: (CAST((RAND(CAST(42:INT AS BIGINT)) * CAST(100:INT AS DOUBLE)) AS INT) AS `value`#12) 147 | ╰╴LocalRelation: data= ⇒ [`id`#10:BIGINT!] 148 | 149 | # Optimized plan 150 | Sort: order=[$0] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 151 | │ ╰╴$0: `value`#12:INT! DESC NULLS FIRST 152 | ╰╴Filter: condition=$0 ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 153 | │ ╰╴$0: ((`value`#12:INT! % 2:INT) = 0:INT) 154 | ╰╴Project: projectList=[$0, $1] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 155 | │ ├╴$0: (`id`#10:BIGINT! AS `key`#11) 156 | │ ╰╴$1: (CAST((RAND(42:BIGINT) * 100.0:DOUBLE) AS INT) AS `value`#12) 157 | ╰╴LocalRelation: data= ⇒ [`id`#10:BIGINT!] 158 | 159 | # Physical plan 160 | Sort: order=[$0] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 161 | │ ╰╴$0: `value`#12:INT! DESC NULLS FIRST 162 | ╰╴Filter: condition=$0 ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 163 | │ ╰╴$0: ((`value`#12:INT! % 2:INT) = 0:INT) 164 | ╰╴Project: projectList=[$0, $1] ⇒ [`key`#11:BIGINT!, `value`#12:INT!] 165 | │ ├╴$0: (`id`#10:BIGINT! AS `key`#11) 166 | │ ╰╴$1: (CAST((RAND(42:BIGINT) * 100.0:DOUBLE) AS INT) AS `value`#12) 167 | ╰╴LocalRelation: data= ⇒ [`id`#10:BIGINT!] 168 | ``` 169 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | lazy val modules: Seq[ProjectReference] = Seq( 2 | `spear-core`, 3 | `spear-docs`, 4 | `spear-examples`, 5 | `spear-local`, 6 | `spear-repl`, 7 | `spear-trees`, 8 | `spear-utils` 9 | ) 10 | 11 | lazy val spear = { 12 | lazy val repl = taskKey[Unit]("Runs the Spear REPL.") 13 | 14 | Project(id = "spear", base = file(".")) 15 | .aggregate(modules: _*) 16 | // Creates a SBT task alias "repl" that starts the REPL within an SBT session. 17 | .settings(repl := (run in `spear-repl` in Compile toTask "").value) 18 | } 19 | 20 | def spearModule(name: String): Project = 21 | Project(id = name, base = file(name)) 22 | .enablePlugins(commonPlugins: _*) 23 | .settings(commonSettings) 24 | 25 | lazy val `spear-utils` = spearModule("spear-utils") 26 | .settings(libraryDependencies ++= Dependencies.logging) 27 | .settings(libraryDependencies ++= Dependencies.scala) 28 | .settings(libraryDependencies ++= Dependencies.testing) 29 | 30 | lazy val `spear-trees` = spearModule("spear-trees") 31 | .dependsOn(`spear-utils` % "compile->compile;test->test") 32 | 33 | lazy val `spear-core` = spearModule("spear-core") 34 | .dependsOn(`spear-trees` % "compile->compile;test->test") 35 | .settings(libraryDependencies ++= Dependencies.fastparse) 36 | .settings(libraryDependencies ++= Dependencies.typesafeConfig) 37 | 38 | lazy val `spear-local` = spearModule("spear-local") 39 | .dependsOn(`spear-core` % "compile->compile;test->test") 40 | 41 | lazy val `spear-repl` = spearModule("spear-repl") 42 | .dependsOn(`spear-core` % "compile->compile;test->test") 43 | .dependsOn(`spear-local` % "compile->compile;test->test;compile->test") 44 | .enablePlugins(JavaAppPackaging) 45 | .settings(runtimeConfSettings) 46 | .settings(javaPackagingSettings) 47 | .settings(libraryDependencies ++= Dependencies.ammonite) 48 | .settings(libraryDependencies ++= Dependencies.scopt) 49 | 50 | lazy val `spear-examples` = spearModule("spear-examples") 51 | .dependsOn(`spear-core`, `spear-local`) 52 | .enablePlugins(JavaAppPackaging) 53 | .settings(runtimeConfSettings) 54 | .settings(javaPackagingSettings) 55 | 56 | lazy val `spear-docs` = spearModule("spear-docs") 57 | .dependsOn(`spear-core`, `spear-local`) 58 | .enablePlugins(SphinxPlugin) 59 | 60 | lazy val javaPackagingSettings = { 61 | import NativePackagerHelper.directory 62 | 63 | Seq( 64 | // Adds the "conf" directory into the package. 65 | mappings in Universal ++= directory(baseDirectory(_.getParentFile / "conf").value), 66 | // Adds the "conf" directory to runtime classpath (relative to "$app_home/../lib"). 67 | scriptClasspath += "../conf" 68 | ) 69 | } 70 | 71 | lazy val commonPlugins = Seq( 72 | // For Scala code formatting 73 | SbtScalariform, 74 | // For Scala test coverage reporting 75 | ScoverageSbtPlugin 76 | ) 77 | 78 | lazy val commonSettings = { 79 | val buildSettings = Seq( 80 | organization := "spear", 81 | version := "0.1.0-SNAPSHOT", 82 | scalaVersion := Dependencies.Versions.scala, 83 | scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature"), 84 | scalacOptions ++= Seq("-Ywarn-unused-import", "-Xlint"), 85 | javacOptions ++= Seq("-source", "1.7", "-target", "1.7", "-g", "-Xlint:-options") 86 | ) 87 | 88 | val commonTestSettings = Seq( 89 | // Disables parallel test execution to ensure logging order. 90 | parallelExecution in Test := false, 91 | // Does not fork a new JVM process to run the tests. 92 | fork := false, 93 | // Shows duration and full exception stack trace 94 | testOptions in Test += Tests.Argument("-oDF") 95 | ) 96 | 97 | val commonDependencySettings = { 98 | import net.virtualvoid.sbt.graph.Plugin.graphSettings 99 | 100 | graphSettings ++ Seq( 101 | // Avoids copying managed dependencies into `lib_managed` 102 | retrieveManaged := false, 103 | // Enables extra resolvers 104 | resolvers ++= Dependencies.extraResolvers, 105 | // Disables auto conflict resolution 106 | conflictManager := ConflictManager.strict, 107 | // Explicitly overrides all conflicting transitive dependencies 108 | dependencyOverrides ++= Dependencies.overrides 109 | ) 110 | } 111 | 112 | val scalariformPluginSettings = { 113 | import com.typesafe.sbt.SbtScalariform.scalariformSettings 114 | import com.typesafe.sbt.SbtScalariform.ScalariformKeys.preferences 115 | import scalariform.formatter.preferences.PreferencesImporterExporter.loadPreferences 116 | 117 | scalariformSettings ++ Seq( 118 | preferences := loadPreferences("scalariform.properties") 119 | ) 120 | } 121 | 122 | val taskSettings = Seq( 123 | // Runs scalastyle before compilation 124 | compile in Compile := (compile in Compile dependsOn (scalastyle in Compile toTask "")).value, 125 | // Runs scalastyle before running tests 126 | test in Test := (test in Test dependsOn (scalastyle in Test toTask "")).value 127 | ) 128 | 129 | Seq( 130 | buildSettings, 131 | commonTestSettings, 132 | commonDependencySettings, 133 | scalariformPluginSettings, 134 | taskSettings 135 | ).flatten 136 | } 137 | 138 | lazy val runtimeConfSettings = Seq( 139 | unmanagedClasspath in Runtime += baseDirectory { _.getParentFile / "conf" }.value 140 | ) 141 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | comment: 2 | layout: header, changes, diff, sunburst 3 | coverage: 4 | status: 5 | patch: false 6 | -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootCategory = INFO, console 2 | log4j.appender.console = org.apache.log4j.ConsoleAppender 3 | log4j.appender.console.target = System.err 4 | log4j.appender.console.layout = org.apache.log4j.PatternLayout 5 | log4j.appender.console.layout.ConversionPattern = %d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 6 | -------------------------------------------------------------------------------- /conf/spear.conf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liancheng/spear/66216aa44f5c5d42a39addb0a6f3ccff6901fa40/conf/spear.conf -------------------------------------------------------------------------------- /project/Dependencies.scala: -------------------------------------------------------------------------------- 1 | import sbt._ 2 | 3 | object Dependencies { 4 | val extraResolvers = Seq( 5 | Resolver.mavenLocal, 6 | "Twitter Maven" at "http://maven.twttr.com" 7 | ) 8 | 9 | object Versions { 10 | val ammonite = "1.0.0" 11 | val config = "1.2.1" 12 | val fastparse = "0.4.3" 13 | val log4j = "1.2.17" 14 | val mockito = "2.1.0-beta.120" 15 | val scala = "2.11.11" 16 | val scalaCheck = "1.12.5" 17 | val scalaTest = "2.2.5" 18 | val scalaXml = "1.0.4" 19 | val scopt = "3.5.0" 20 | val slf4j = "1.7.13" 21 | 22 | val sourcecode = "0.1.3" 23 | } 24 | 25 | val ammonite = Seq( 26 | "com.lihaoyi" % s"ammonite_${Versions.scala}" % Versions.ammonite 27 | ) 28 | 29 | val fastparse = Seq( 30 | "com.lihaoyi" %% "fastparse" % Versions.fastparse 31 | ) 32 | 33 | val log4j = Seq( 34 | "log4j" % "log4j" % Versions.log4j 35 | ) 36 | 37 | val mockito = Seq( 38 | "org.mockito" % "mockito-core" % Versions.mockito % "test" 39 | ) 40 | 41 | val scala = Seq( 42 | "org.scala-lang" % "scala-library" % Versions.scala, 43 | "org.scala-lang" % "scala-reflect" % Versions.scala, 44 | "org.scala-lang.modules" %% "scala-xml" % Versions.scalaXml 45 | ) 46 | 47 | val scalaCheck = Seq( 48 | "org.scalacheck" %% "scalacheck" % Versions.scalaCheck % "test" 49 | ) 50 | 51 | val scalaTest = Seq( 52 | "org.scalatest" %% "scalatest" % Versions.scalaTest % "test" 53 | ) 54 | 55 | val scopt = Seq( 56 | "com.github.scopt" %% "scopt" % Versions.scopt 57 | ) 58 | 59 | val slf4j = Seq( 60 | "org.slf4j" % "slf4j-api" % Versions.slf4j, 61 | "org.slf4j" % "slf4j-log4j12" % Versions.slf4j, 62 | "org.slf4j" % "jul-to-slf4j" % Versions.slf4j 63 | ) 64 | 65 | val sourcecode = Seq( 66 | "com.lihaoyi" %% "sourcecode" % Versions.sourcecode 67 | ) 68 | 69 | val typesafeConfig = Seq( 70 | "com.typesafe" % "config" % Versions.config 71 | ) 72 | 73 | val testing: Seq[ModuleID] = mockito ++ scalaCheck ++ scalaTest 74 | 75 | val logging: Seq[ModuleID] = log4j ++ slf4j 76 | 77 | val overrides: Set[ModuleID] = (log4j ++ scala ++ slf4j ++ sourcecode).toSet 78 | } 79 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.12 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.typesafe.sbt" % "sbt-native-packager" % "1.1.6") 2 | addSbtPlugin("com.typesafe.sbt" % "sbt-site" % "1.2.0") 3 | addSbtPlugin("net.virtual-void" % "sbt-dependency-graph" % "0.7.5") 4 | addSbtPlugin("org.scalariform" % "sbt-scalariform" % "1.6.0") 5 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.5") 6 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") 7 | addSbtPlugin("org.tpolecat" % "tut-plugin" % "0.5.1") 8 | -------------------------------------------------------------------------------- /scalariform.properties: -------------------------------------------------------------------------------- 1 | alignSingleLineCaseStatements = true 2 | alignSingleLineCaseStatements.maxArrowIndent = 40 3 | compactControlReadability = false 4 | danglingCloseParenthesis = Preserve 5 | doubleIndentClassDeclaration = false 6 | newlineAtEndOfFile = true 7 | preserveSpaceBeforeArguments = true 8 | spacesAroundMultiImports = false 9 | -------------------------------------------------------------------------------- /spear-core/src/main/java/spear/parsers/annotations/ExtendedSQLSyntax.java: -------------------------------------------------------------------------------- 1 | package spear.parsers.annotations; 2 | 3 | import java.lang.annotation.ElementType; 4 | import java.lang.annotation.Target; 5 | 6 | @Target({ 7 | ElementType.FIELD, 8 | ElementType.LOCAL_VARIABLE 9 | }) 10 | public @interface ExtendedSQLSyntax {} 11 | -------------------------------------------------------------------------------- /spear-core/src/main/java/spear/plans/logical/annotations/Explain.java: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.annotations; 2 | 3 | import java.lang.annotation.ElementType; 4 | import java.lang.annotation.Retention; 5 | import java.lang.annotation.RetentionPolicy; 6 | import java.lang.annotation.Target; 7 | 8 | @Retention(RetentionPolicy.RUNTIME) 9 | @Target({ ElementType.PARAMETER }) 10 | public @interface Explain { 11 | boolean hidden() default false; 12 | 13 | boolean nestedTree() default false; 14 | } 15 | 16 | -------------------------------------------------------------------------------- /spear-core/src/main/resources/spear-reference.conf: -------------------------------------------------------------------------------- 1 | spear { 2 | query-compiler = "spear.local.LocalQueryCompiler" 3 | } 4 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/Catalog.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.collection.mutable 4 | 5 | import spear.exceptions.TableNotFoundException 6 | import spear.plans.logical.LogicalPlan 7 | 8 | trait Catalog { 9 | val functionRegistry: FunctionRegistry 10 | 11 | def registerRelation(tableName: Name, analyzedPlan: LogicalPlan): Unit 12 | 13 | def removeRelation(tableName: Name): Unit 14 | 15 | def lookupRelation(tableName: Name): LogicalPlan 16 | } 17 | 18 | class InMemoryCatalog extends Catalog { 19 | override val functionRegistry: FunctionRegistry = new InMemoryFunctionRegistry 20 | 21 | override def registerRelation(tableName: Name, analyzedPlan: LogicalPlan): Unit = 22 | tables(tableName) = analyzedPlan 23 | 24 | override def removeRelation(tableName: Name): Unit = tables -= tableName 25 | 26 | override def lookupRelation(tableName: Name): LogicalPlan = 27 | tables 28 | .get(tableName) 29 | .map { _ subquery tableName } 30 | .getOrElse { throw new TableNotFoundException(tableName) } 31 | 32 | private val tables: mutable.Map[Name, LogicalPlan] = mutable.Map.empty 33 | } 34 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/Context.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.collection.Iterable 4 | import scala.reflect.runtime.universe.WeakTypeTag 5 | 6 | import spear.config.{QueryCompilerClass, Settings} 7 | import spear.expressions.Expression 8 | import spear.plans.logical.{LocalRelation, SingleRowRelation} 9 | import spear.types.{LongType, StructType} 10 | 11 | class Context(val queryCompiler: QueryCompiler) { 12 | def this(settings: Settings) = this( 13 | Class.forName(settings(QueryCompilerClass)).newInstance() match { 14 | case q: QueryCompiler => q 15 | } 16 | ) 17 | 18 | private lazy val values: DataFrame = new DataFrame(SingleRowRelation, this) 19 | 20 | def values(first: Expression, rest: Expression*): DataFrame = values select first +: rest 21 | 22 | def sql(query: String): DataFrame = new DataFrame(queryCompiler parse query, this) 23 | 24 | def table(name: Name): DataFrame = 25 | new DataFrame(queryCompiler.catalog lookupRelation name, this) 26 | 27 | def lift[T <: Product: WeakTypeTag](data: Iterable[T]): DataFrame = 28 | new DataFrame(LocalRelation(data), this) 29 | 30 | def lift[T <: Product: WeakTypeTag](first: T, rest: T*): DataFrame = lift(first +: rest) 31 | 32 | def range(end: Long): DataFrame = range(0, end) 33 | 34 | def range(begin: Long, end: Long): DataFrame = range(begin, end, 1L) 35 | 36 | def range(begin: Long, end: Long, step: Long): DataFrame = { 37 | val rows = begin until end by step map { Row apply _ } 38 | val output = StructType('id -> LongType.!).toAttributes 39 | new DataFrame(LocalRelation(rows, output), this) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/FunctionRegistry.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.collection.mutable 4 | import scala.reflect.ClassTag 5 | import scala.util.{Failure, Success, Try} 6 | 7 | import spear.exceptions.{FunctionInstantiationException, FunctionNotFoundException} 8 | import spear.expressions._ 9 | import spear.expressions.aggregates._ 10 | 11 | case class FunctionInfo( 12 | name: Name, 13 | builder: Seq[Expression] => Expression 14 | ) 15 | 16 | object FunctionInfo { 17 | type FunctionBuilder = Seq[Expression] => Expression 18 | } 19 | 20 | trait FunctionRegistry { 21 | def registerFunction(fn: FunctionInfo): Unit 22 | 23 | def removeFunction(name: Name): Unit 24 | 25 | def lookupFunction(name: Name): FunctionInfo 26 | } 27 | 28 | class InMemoryFunctionRegistry extends FunctionRegistry { 29 | override def registerFunction(fn: FunctionInfo): Unit = map(fn.name) = fn 30 | 31 | override def removeFunction(name: Name): Unit = map remove name 32 | 33 | override def lookupFunction(name: Name): FunctionInfo = 34 | map.getOrElse(name, throw new FunctionNotFoundException(name)) 35 | 36 | private val map: mutable.Map[Name, FunctionInfo] = mutable.Map.empty[Name, FunctionInfo] 37 | 38 | Seq( 39 | function[Coalesce](i"coalesce"), 40 | function[Rand](i"rand"), 41 | function[Length](i"length"), 42 | 43 | function[ArrayAgg](i"array_agg"), 44 | 45 | function[Count](i"count"), 46 | function[First](i"first"), 47 | function[First](i"first_value"), 48 | function[Last](i"last"), 49 | function[Last](i"last_value"), 50 | function[Max](i"max"), 51 | function[Min](i"min"), 52 | function[Average](i"average"), 53 | function[Average](i"avg"), 54 | function[Sum](i"sum"), 55 | function[Product_](i"product"), 56 | function[BoolAnd](i"bool_and"), 57 | function[BoolOr](i"bool_or"), 58 | 59 | function[Concat](i"concat"), 60 | 61 | function[MakeNamedStruct](i"named_struct"), 62 | function[MakeArray](i"array"), 63 | function[MakeMap](i"map") 64 | ) foreach registerFunction 65 | 66 | private def function[T <: Expression: ClassTag](name: Name): FunctionInfo = { 67 | val classTag = implicitly[ClassTag[T]] 68 | 69 | def instantiateWithVararg(args: Seq[Expression]): Any = { 70 | val argClasses = classOf[Seq[Expression]] 71 | val constructor = classTag.runtimeClass.getDeclaredConstructor(argClasses) 72 | constructor.newInstance(args) 73 | } 74 | 75 | def instantiate(args: Seq[Expression]): Any = { 76 | val argClasses = Seq.fill(args.length)(classOf[Expression]) 77 | val constructor = classTag.runtimeClass.getDeclaredConstructor(argClasses: _*) 78 | constructor.newInstance(args: _*) 79 | } 80 | 81 | val builder = (args: Seq[Expression]) => { 82 | Try(instantiateWithVararg(args)) orElse Try(instantiate(args)) match { 83 | case Success(fn: Expression) => fn 84 | case Failure(cause) => throw new FunctionInstantiationException(name, cause) 85 | case _ => throw new FunctionInstantiationException(name) 86 | } 87 | } 88 | 89 | FunctionInfo(name, builder) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/Name.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | class Name(private val impl: Name.CaseSensitivityAware, val namespace: String = "") { 4 | def isCaseSensitive: Boolean = impl.isCaseSensitive 5 | 6 | def casePreserving: String = impl.casePreserving 7 | 8 | def withNamespace(namespace: String): Name = new Name(impl, namespace) 9 | 10 | override def toString: String = s"$impl${if (namespace.isEmpty) "" else s"@$namespace"}" 11 | 12 | override def hashCode(): Int = casePreserving.toUpperCase.hashCode 13 | 14 | override def equals(other: Any): Boolean = other match { 15 | case that: Name if this.isCaseSensitive || that.isCaseSensitive => 16 | this.namespace == that.namespace && this.casePreserving == that.casePreserving 17 | 18 | case that: Name => 19 | this.namespace == that.namespace && this.casePreserving.equalsIgnoreCase(that.casePreserving) 20 | 21 | case _ => 22 | false 23 | } 24 | } 25 | 26 | object Name { 27 | def apply(name: String, isCaseSensitive: Boolean): Name = 28 | if (isCaseSensitive) caseSensitive(name) else caseInsensitive(name) 29 | 30 | def caseSensitive(name: String): Name = new Name(CaseSensitive(name)) 31 | 32 | def caseInsensitive(name: String): Name = new Name(CaseInsensitive(name)) 33 | 34 | private sealed trait CaseSensitivityAware { 35 | def isCaseSensitive: Boolean 36 | 37 | def casePreserving: String 38 | } 39 | 40 | private case class CaseSensitive(casePreserving: String) extends CaseSensitivityAware { 41 | override def isCaseSensitive: Boolean = true 42 | 43 | override def toString: String = quote(casePreserving) 44 | } 45 | 46 | private case class CaseInsensitive(casePreserving: String) extends CaseSensitivityAware { 47 | override def isCaseSensitive: Boolean = false 48 | 49 | override def toString: String = casePreserving 50 | } 51 | 52 | private def quote(name: String): String = "\"" + name.replace("\"", "\"\"") + "\"" 53 | } 54 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/QueryCompiler.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import spear.parsers.DirectlyExecutableStatementParser.directlyExecutableStatement 4 | import spear.plans.CompiledQuery 5 | import spear.plans.logical.{LogicalPlan, Optimizer} 6 | import spear.plans.logical.analysis.Analyzer 7 | import spear.plans.physical.PhysicalPlan 8 | 9 | trait QueryCompiler { 10 | def catalog: Catalog 11 | 12 | /** 13 | * Parses given query string to a [[spear.plans.logical.LogicalPlan logical plan]]. 14 | */ 15 | def parse(query: String): LogicalPlan 16 | 17 | /** 18 | * Analyzes an unresolved [[spear.plans.logical.LogicalPlan logical plan]] and outputs its 19 | * strictly-typed version. 20 | */ 21 | def analyze(plan: LogicalPlan): LogicalPlan 22 | 23 | /** 24 | * Optimizes a resolved [[spear.plans.logical.LogicalPlan logical plan]] into another equivalent 25 | * but more performant version. 26 | */ 27 | def optimize(plan: LogicalPlan): LogicalPlan 28 | 29 | /** 30 | * Plans a [[spear.plans.logical.LogicalPlan logical plan]] into an executable 31 | * [[spear.plans.physical.PhysicalPlan physical plan]]. 32 | */ 33 | def plan(plan: LogicalPlan): PhysicalPlan 34 | 35 | def compile(context: Context, plan: LogicalPlan): CompiledQuery = new CompiledQuery(context, plan) 36 | } 37 | 38 | trait BasicQueryCompiler extends QueryCompiler { 39 | override val catalog: Catalog = new InMemoryCatalog 40 | 41 | override def parse(query: String): LogicalPlan = { 42 | import fastparse.all.{parserApi, End, Start} 43 | 44 | (Start ~ directlyExecutableStatement ~ End parse query).get.value 45 | } 46 | 47 | override def analyze(plan: LogicalPlan): LogicalPlan = analyzer apply plan 48 | 49 | override def optimize(plan: LogicalPlan): LogicalPlan = optimizer apply plan 50 | 51 | private val analyzer = new Analyzer(catalog) 52 | 53 | private val optimizer = new Optimizer 54 | } 55 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/Row.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.collection.mutable.ArrayBuffer 4 | 5 | trait SpecializedGetter { 6 | def getBoolean(ordinal: Int): Boolean 7 | 8 | def getByte(ordinal: Int): Byte 9 | 10 | def getShort(ordinal: Int): Short 11 | 12 | def getInt(ordinal: Int): Int 13 | 14 | def getLong(ordinal: Int): Long 15 | 16 | def getFloat(ordinal: Int): Float 17 | 18 | def getDouble(ordinal: Int): Double 19 | } 20 | 21 | trait SpecializedSetter { 22 | def setBoolean(ordinal: Int, value: Boolean): Unit 23 | 24 | def setByte(ordinal: Int, value: Byte): Unit 25 | 26 | def setShort(ordinal: Int, value: Short): Unit 27 | 28 | def setInt(ordinal: Int, value: Int): Unit 29 | 30 | def setLong(ordinal: Int, value: Long): Unit 31 | 32 | def setFloat(ordinal: Int, value: Float): Unit 33 | 34 | def setDouble(ordinal: Int, value: Double): Unit 35 | } 36 | 37 | trait Row extends Seq[Any] with SpecializedGetter { 38 | override def getBoolean(ordinal: Int): Boolean = apply(ordinal).asInstanceOf[Boolean] 39 | 40 | override def getDouble(ordinal: Int): Double = apply(ordinal).asInstanceOf[Double] 41 | 42 | override def getFloat(ordinal: Int): Float = apply(ordinal).asInstanceOf[Float] 43 | 44 | override def getLong(ordinal: Int): Long = apply(ordinal).asInstanceOf[Long] 45 | 46 | override def getByte(ordinal: Int): Byte = apply(ordinal).asInstanceOf[Byte] 47 | 48 | override def getShort(ordinal: Int): Short = apply(ordinal).asInstanceOf[Short] 49 | 50 | override def getInt(ordinal: Int): Int = apply(ordinal).asInstanceOf[Int] 51 | 52 | def copy(): Row = { 53 | val copy = Array.fill[Any](length)(null) 54 | copyToArray(copy) 55 | Row.fromSeq(copy) 56 | } 57 | } 58 | 59 | object Row { 60 | val empty = new BasicRow(Nil) 61 | 62 | def apply(first: Any, rest: Any*): Row = new BasicRow(first +: rest) 63 | 64 | def fromSeq(values: Seq[Any]): Row = new BasicRow(values) 65 | 66 | def unapplySeq(row: Row): Some[Seq[Any]] = Some(row) 67 | } 68 | 69 | trait MutableRow extends Row with SpecializedSetter { 70 | def update(ordinal: Int, value: Any): Unit 71 | 72 | def setBoolean(ordinal: Int, value: Boolean): Unit = update(ordinal, value) 73 | 74 | def setByte(ordinal: Int, value: Byte): Unit = update(ordinal, value) 75 | 76 | def setShort(ordinal: Int, value: Short): Unit = update(ordinal, value) 77 | 78 | def setInt(ordinal: Int, value: Int): Unit = update(ordinal, value) 79 | 80 | def setLong(ordinal: Int, value: Long): Unit = update(ordinal, value) 81 | 82 | def setFloat(ordinal: Int, value: Float): Unit = update(ordinal, value) 83 | 84 | def setDouble(ordinal: Int, value: Double): Unit = update(ordinal, value) 85 | } 86 | 87 | class BasicRow(values: Seq[Any]) extends Row { 88 | override def length: Int = values.length 89 | 90 | override def apply(ordinal: Int): Any = values.apply(ordinal) 91 | 92 | override def iterator: Iterator[Any] = values.iterator 93 | } 94 | 95 | class BasicMutableRow(values: ArrayBuffer[Any]) extends BasicRow(values) with MutableRow { 96 | def this(size: Int) = this(ArrayBuffer.fill(size)(null: Any)) 97 | 98 | override def update(ordinal: Int, value: Any): Unit = values(ordinal) = value 99 | 100 | override def length: Int = values.length 101 | 102 | override def apply(ordinal: Int): Any = values(ordinal) 103 | 104 | override def iterator: Iterator[Any] = values.iterator 105 | } 106 | 107 | class JoinedRow(private var left: Row, private var right: Row) extends Row { 108 | def this() = this(null, null) 109 | 110 | override def length: Int = left.length + right.length 111 | 112 | override def apply(ordinal: Int): Any = 113 | if (ordinal < left.length) left(ordinal) else right(ordinal - left.length) 114 | 115 | override def iterator: Iterator[Any] = left.iterator ++ right.iterator 116 | 117 | def apply(lhs: Row, rhs: Row): this.type = { 118 | left = lhs 119 | right = rhs 120 | this 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/RowOrdering.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import spear.expressions.{Ascending, Attribute, Descending, SortOrder} 4 | import spear.types.{DataType, OrderedType} 5 | 6 | class NullSafeOrdering(dataType: DataType, nullsLarger: Boolean) extends Ordering[Any] { 7 | private val baseOrdering = OrderedType.orderingOf(dataType) 8 | 9 | override def compare(lhs: Any, rhs: Any): Int = (lhs, rhs) match { 10 | case (null, null) => 0 11 | case (null, _) => if (nullsLarger) 1 else -1 12 | case (_, null) => if (nullsLarger) -1 else 1 13 | case _ => baseOrdering.compare(lhs, rhs) 14 | } 15 | } 16 | 17 | class RowOrdering(boundSortOrders: Seq[SortOrder]) extends Ordering[Row] { 18 | def this(unboundSortOrders: Seq[SortOrder], inputSchema: Seq[Attribute]) = this( 19 | unboundSortOrders 20 | map { _ bindTo inputSchema } 21 | map { case e: SortOrder => e } 22 | ) 23 | 24 | private val nullSafeOrderings = boundSortOrders.map { 25 | case order @ SortOrder(_, Ascending, _) => 26 | new NullSafeOrdering(order.dataType, order.isNullLarger) 27 | 28 | case order @ SortOrder(_, Descending, _) => 29 | new NullSafeOrdering(order.dataType, order.isNullLarger).reverse 30 | } 31 | 32 | def compare(a: Row, b: Row): Int = { 33 | val children = boundSortOrders map { _.child } 34 | (children zip nullSafeOrderings).iterator map { 35 | case (e, ordering) => 36 | ordering.compare(e evaluate a, e evaluate b) 37 | } find { _ != 0 } getOrElse 0 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/config/Settings.scala: -------------------------------------------------------------------------------- 1 | package spear.config 2 | 3 | import java.util.concurrent.TimeUnit 4 | 5 | import scala.concurrent.duration._ 6 | import scala.util.{Success, Try} 7 | import scala.util.control.NonFatal 8 | 9 | import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} 10 | 11 | import spear.config.Settings.Key 12 | import spear.exceptions.SettingsValidationException 13 | 14 | class Settings(val config: Config) { 15 | def apply[T](key: Key[T]): T = (key validator (key get config)).recover { 16 | case NonFatal(cause) => 17 | throw new SettingsValidationException( 18 | s"Configured value of settings key ${key.name} didn't pass validation: ${cause.getMessage}", 19 | cause 20 | ) 21 | }.get 22 | 23 | def withValue(key: String, value: AnyRef): Settings = 24 | Settings(config.withValue(key, ConfigValueFactory.fromAnyRef(value))) 25 | 26 | def withValue[T](key: Key[T], value: T): Settings = 27 | withValue(key.name, value.asInstanceOf[AnyRef]) 28 | } 29 | 30 | object Settings { 31 | case class Key[T](name: String, get: Config => T, validator: T => Try[T] = Success(_: T)) { 32 | def validate(validator: T => Try[T]): Key[T] = copy(validator = validator) 33 | 34 | override def toString: String = name 35 | } 36 | 37 | object Key { 38 | case class KeyBuilder(name: String) { 39 | def boolean: Key[Boolean] = Key[Boolean](name, _ getBoolean name) 40 | def number: Key[Number] = Key[Number](name, _ getNumber name) 41 | def string: Key[String] = Key[String](name, _ getString name) 42 | def int: Key[Int] = Key[Int](name, _ getInt name) 43 | def long: Key[Long] = Key[Long](name, _ getLong name) 44 | def double: Key[Double] = Key[Double](name, _ getDouble name) 45 | def anyRef: Key[AnyRef] = Key[AnyRef](name, _ getAnyRef name) 46 | 47 | private def duration(config: Config, unit: TimeUnit): Long = config getDuration (name, unit) 48 | 49 | def nanos: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.NANOSECONDS).nanos) 50 | def micros: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.MICROSECONDS).micros) 51 | def millis: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.MILLISECONDS).millis) 52 | def seconds: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.SECONDS).seconds) 53 | def minutes: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.MINUTES).minutes) 54 | def hours: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.HOURS).hours) 55 | def days: Key[Duration] = Key[Duration](name, duration(_, TimeUnit.DAYS).days) 56 | } 57 | 58 | def apply(name: String): KeyBuilder = KeyBuilder(name) 59 | } 60 | 61 | val empty: Settings = new Settings(ConfigFactory.empty()) 62 | 63 | def apply(config: Config): Settings = new Settings(config) 64 | 65 | def load(first: String, rest: String*): Settings = Settings( 66 | ConfigFactory 67 | // Environment variables takes highest priority and overrides everything else 68 | .systemEnvironment() 69 | // System properties comes after environment variables 70 | .withFallback(ConfigFactory.systemProperties()) 71 | // Then follows user provided configuration files 72 | .withFallback(first +: rest map ConfigFactory.parseResources reduce { _ withFallback _ }) 73 | // Then follows the reference configuration file 74 | .withFallback(ConfigFactory.parseResources("spear-reference.conf")) 75 | // Configurations of all other components (like Akka) 76 | .withFallback(ConfigFactory.load()) 77 | .resolve() 78 | ) 79 | 80 | def load(): Settings = load("spear.conf") 81 | } 82 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/config/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import spear.config.Settings.Key 4 | 5 | package object config { 6 | val QueryCompilerClass: Key[String] = Key("spear.query-compiler").string 7 | } 8 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/execution/Projection.scala: -------------------------------------------------------------------------------- 1 | package spear.execution 2 | 3 | import spear.{BasicMutableRow, MutableRow, Row} 4 | import spear.expressions.Expression 5 | 6 | trait Projection extends (Row => Row) with (() => Row) 7 | 8 | object Projection { 9 | def apply(expressions: Seq[Expression]): Projection = new Projection { 10 | override def apply(input: Row): Row = Row.fromSeq(expressions map { _ evaluate input }) 11 | 12 | override def apply(): Row = apply(null) 13 | } 14 | } 15 | 16 | trait MutableProjection extends Projection { 17 | def target(mutableRow: MutableRow): this.type 18 | } 19 | 20 | object MutableProjection { 21 | def apply(expressions: Seq[Expression]): MutableProjection = new MutableProjection { 22 | private[this] var mutableRow: MutableRow = new BasicMutableRow(expressions.length) 23 | 24 | override def target(mutableRow: MutableRow): this.type = { 25 | this.mutableRow = mutableRow 26 | this 27 | } 28 | 29 | override def apply(input: Row): Row = { 30 | expressions.map { _ evaluate input }.zipWithIndex foreach { 31 | case (value, ordinal) => mutableRow(ordinal) = value 32 | } 33 | 34 | mutableRow 35 | } 36 | 37 | override def apply(): Row = apply(null) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/InternalAlias.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear._ 4 | import spear.expressions.NamedExpression.newExpressionID 5 | import spear.expressions.aggregates.AggregateFunction 6 | import spear.expressions.windows.WindowFunction 7 | import spear.types._ 8 | 9 | sealed trait InternalAlias extends UnaryExpression with NamedExpression with NonSQLExpression { 10 | def namespace: String 11 | 12 | override def name: Name = i"" withNamespace namespace 13 | 14 | override def debugString: String = 15 | s"(${child.debugString} AS ${name.toString}#${expressionID.id})" 16 | 17 | override def dataType: DataType = child.dataType 18 | 19 | override def isNullable: Boolean = child.isNullable 20 | 21 | override lazy val isFoldable: Boolean = false 22 | 23 | override def attr: AttributeRef = name of dataType nullable isNullable withID expressionID 24 | } 25 | 26 | object InternalAlias { 27 | val GroupingKeyNamespace: String = "G" 28 | 29 | val AggregateFunctionNamespace: String = "A" 30 | 31 | val WindowFunctionNamespace: String = "W" 32 | 33 | val SortOrderNamespace: String = "S" 34 | 35 | def buildRewriter(aliases: Seq[InternalAlias]): Map[Expression, Expression] = 36 | aliases.map { a => a.child -> (a.attr: Expression) }.toMap 37 | 38 | def buildRestorer(aliases: Seq[InternalAlias]): Map[Expression, Expression] = 39 | aliases.map { a => (a.attr: Expression) -> a.child }.toMap 40 | } 41 | 42 | case class GroupingKeyAlias( 43 | child: Expression, override val expressionID: ExpressionID = newExpressionID() 44 | ) extends InternalAlias { 45 | 46 | override val namespace: String = InternalAlias.GroupingKeyNamespace 47 | 48 | override def withID(id: ExpressionID): GroupingKeyAlias = copy(expressionID = id) 49 | } 50 | 51 | case class AggregateFunctionAlias( 52 | child: AggregateFunction, override val expressionID: ExpressionID = newExpressionID() 53 | ) extends InternalAlias { 54 | 55 | override val namespace: String = InternalAlias.AggregateFunctionNamespace 56 | 57 | override def withID(id: ExpressionID): AggregateFunctionAlias = copy(expressionID = id) 58 | } 59 | 60 | case class WindowFunctionAlias( 61 | child: WindowFunction, override val expressionID: ExpressionID = newExpressionID() 62 | ) extends InternalAlias { 63 | 64 | override val namespace: String = InternalAlias.WindowFunctionNamespace 65 | 66 | override def withID(id: ExpressionID): WindowFunctionAlias = copy(expressionID = id) 67 | } 68 | 69 | case class SortOrderAlias( 70 | child: Expression, alias: Name, override val expressionID: ExpressionID = newExpressionID() 71 | ) extends InternalAlias { 72 | 73 | override val namespace: String = InternalAlias.SortOrderNamespace 74 | 75 | override def name: Name = alias withNamespace namespace 76 | 77 | override def withID(id: ExpressionID): SortOrderAlias = copy(expressionID = id) 78 | } 79 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/Literal.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import scala.util.Try 4 | 5 | import spear.Row 6 | import spear.types._ 7 | import spear.utils.quote 8 | 9 | case class Literal(value: Any, override val dataType: DataType) extends LeafExpression { 10 | override def isNullable: Boolean = value == null 11 | 12 | override def evaluate(input: Row): Any = value 13 | 14 | override def debugString: String = value match { 15 | case v: String => s"${quote(v)}:${dataType.sql}" 16 | case v => s"$v:${dataType.sql}" 17 | } 18 | 19 | override def sql: Try[String] = Try((value, dataType) match { 20 | case (v: String, StringType) => quote(v) 21 | case (v: Boolean, BooleanType) => v.toString.toUpperCase 22 | case (v: Byte, ByteType) => s"CAST($v AS ${ByteType.sql})" 23 | case (v: Short, ShortType) => s"CAST($v AS ${ShortType.sql})" 24 | case (v: Long, LongType) => s"CAST($v AS ${LongType.sql})" 25 | case (v: Float, FloatType) => s"CAST($v AS ${FloatType.sql})" 26 | case (v: Double, DoubleType) => s"CAST($v AS ${DoubleType.sql})" 27 | case (v, _) if v == null => "NULL" 28 | case (_, _: ComplexType) => throw new UnsupportedOperationException 29 | case (v, _) => v.toString 30 | }) 31 | } 32 | 33 | object Literal { 34 | val True: Literal = Literal(true) 35 | 36 | val False: Literal = Literal(false) 37 | 38 | def apply(value: Any): Literal = { 39 | value match { 40 | case v: Boolean => Literal(v, BooleanType) 41 | case v: Byte => Literal(v, ByteType) 42 | case v: Short => Literal(v, ShortType) 43 | case v: Int => Literal(v, IntType) 44 | case v: Long => Literal(v, LongType) 45 | case v: Float => Literal(v, FloatType) 46 | case v: Double => Literal(v, DoubleType) 47 | case v: String => Literal(v, StringType) 48 | case null => Literal(null, NullType) 49 | case v => throw new UnsupportedOperationException( 50 | s"Unsupported literal type ${v.getClass} $v" 51 | ) 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/Predicate.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.trees.{FixedPoint, Rule, RuleGroup, Transformer} 4 | 5 | object Predicate { 6 | private[spear] def splitConjunction(predicate: Expression): Seq[Expression] = predicate match { 7 | case left && right => splitConjunction(left) ++ splitConjunction(right) 8 | case _ => predicate :: Nil 9 | } 10 | 11 | private[spear] def toCNF(predicate: Expression): Expression = cnfConverter(predicate) 12 | 13 | private val cnfConverter = new Transformer(RuleGroup(FixedPoint, CNFConversion :: Nil)) 14 | 15 | private object CNFConversion extends Rule[Expression] { 16 | override def transform(tree: Expression): Expression = tree transformDown { 17 | case !(x || y) => !x && !y 18 | case !(x && y) => !x || !y 19 | case (x && y) || z => (x || z) && (y || z) 20 | case x || (y && z) => (x || y) && (x || z) 21 | case x && y if x same y => x 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/SortOrder.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.expressions.typecheck.TypeConstraint 4 | import spear.types.{DataType, OrderedType} 5 | 6 | sealed trait SortDirection 7 | 8 | case object Ascending extends SortDirection { 9 | override def toString: String = "ASC" 10 | } 11 | 12 | case object Descending extends SortDirection { 13 | override def toString: String = "DESC" 14 | } 15 | 16 | case class SortOrder(child: Expression, direction: SortDirection, isNullLarger: Boolean) 17 | extends UnaryExpression with UnevaluableExpression { 18 | 19 | override def dataType: DataType = child.dataType 20 | 21 | override def isNullable: Boolean = child.isNullable 22 | 23 | def nullsFirst: SortOrder = copy(isNullLarger = !isAscending) 24 | 25 | def nullsLast: SortOrder = copy(isNullLarger = isAscending) 26 | 27 | def nullsLarger: SortOrder = copy(isNullLarger = true) 28 | 29 | def isAscending: Boolean = direction == Ascending 30 | 31 | def isNullsFirst: Boolean = isAscending ^ isNullLarger 32 | 33 | override protected lazy val typeConstraint: TypeConstraint = child subtypeOf OrderedType 34 | 35 | override protected def template(childString: String): String = 36 | s"$childString $direction NULLS ${if (isNullsFirst) "FIRST" else "LAST"}" 37 | } 38 | 39 | object SortOrder { 40 | def apply(expression: Expression): SortOrder = expression match { 41 | case e: SortOrder => e 42 | case e => e.asc 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/aggregates/basic.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.aggregates 2 | 3 | import scala.collection.mutable.ArrayBuffer 4 | 5 | import spear.Row 6 | import spear.expressions._ 7 | import spear.expressions.aggregates.FoldLeft.{AccumulateFunction, MergeFunction} 8 | import spear.expressions.functions._ 9 | import spear.expressions.typecheck.TypeConstraint 10 | import spear.types.{ArrayType, BooleanType, DataType, OrderedType} 11 | 12 | case class Count(child: Expression) extends FoldLeft { 13 | override lazy val zeroValue: Expression = 0L 14 | 15 | override lazy val accumulateFunction: AccumulateFunction = if (child.isNullable) { 16 | (count: Expression, input: Expression) => count + If(input.isNull, 0L, 1L) 17 | } else { 18 | (count: Expression, _) => count + 1L 19 | } 20 | 21 | override def mergeFunction: MergeFunction = Plus 22 | 23 | override protected lazy val value: AttributeRef = 'value of dataType.! 24 | } 25 | 26 | case class Max(child: Expression) extends NullableReduceLeft with DuplicateInsensitive { 27 | override val accumulateFunction: AccumulateFunction = Greatest(_, _) 28 | 29 | override protected def typeConstraint: TypeConstraint = children sameSubtypeOf OrderedType 30 | } 31 | 32 | case class Min(child: Expression) extends NullableReduceLeft with DuplicateInsensitive { 33 | override val accumulateFunction: AccumulateFunction = Least(_, _) 34 | 35 | override protected def typeConstraint: TypeConstraint = children sameSubtypeOf OrderedType 36 | } 37 | 38 | abstract class FirstLike(child: Expression, ignoresNull: Expression) 39 | extends AggregateFunction with DuplicateInsensitive { 40 | 41 | override lazy val isPure: Boolean = false 42 | 43 | override def children: Seq[Expression] = Seq(child, ignoresNull) 44 | 45 | override def isNullable: Boolean = child.isNullable 46 | 47 | override protected def typeConstraint: TypeConstraint = child.anyType concat ignoresNull.foldable 48 | 49 | override protected lazy val strictDataType: DataType = child.dataType 50 | 51 | protected lazy val ignoresNullBool: Boolean = ignoresNull.evaluated.asInstanceOf[Boolean] 52 | } 53 | 54 | case class First(child: Expression, ignoresNull: Expression) extends FirstLike(child, ignoresNull) { 55 | def this(child: Expression) = this(child, lit(true)) 56 | 57 | override def nodeName: String = "first_value" 58 | 59 | override lazy val stateAttributes: Seq[Attribute] = Seq(first, valueSet) 60 | 61 | override lazy val initialValues: Seq[Expression] = Seq(Literal(null, child.dataType), false) 62 | 63 | override lazy val accumulateExpressions: Seq[Expression] = 64 | if (child.isNullable && ignoresNullBool) { 65 | Seq( 66 | If(!valueSet, coalesce(child, first), first), 67 | valueSet || child.isNotNull 68 | ) 69 | } else { 70 | Seq( 71 | If(valueSet, first, child), 72 | true 73 | ) 74 | } 75 | 76 | override lazy val mergeExpressions: Seq[Expression] = Seq( 77 | If(valueSet.left, first.left, first.right), 78 | valueSet.left || valueSet.right 79 | ) 80 | 81 | override lazy val resultExpression: Expression = first 82 | 83 | private lazy val first = 'first of dataType nullable isNullable 84 | 85 | private lazy val valueSet = 'valueSet of BooleanType.! 86 | } 87 | 88 | case class Last(child: Expression, ignoresNull: Expression) extends FirstLike(child, ignoresNull) { 89 | def this(child: Expression) = this(child, lit(true)) 90 | 91 | override def nodeName: String = "last_value" 92 | 93 | override lazy val stateAttributes: Seq[Attribute] = Seq(last) 94 | 95 | override lazy val initialValues: Seq[Expression] = Seq(Literal(null, child.dataType)) 96 | 97 | override lazy val accumulateExpressions: Seq[Expression] = Seq( 98 | if (child.isNullable && ignoresNullBool) coalesce(child, last) else child 99 | ) 100 | 101 | override lazy val mergeExpressions: Seq[Expression] = Seq( 102 | coalesce(last.right, last.left) 103 | ) 104 | 105 | override lazy val resultExpression: Expression = last 106 | 107 | private lazy val last = 'last of dataType nullable isNullable 108 | } 109 | 110 | case class ArrayAgg(child: Expression) 111 | extends ImperativeAggregateFunction[ArrayBuffer[Any]] with UnaryExpression { 112 | 113 | override def isNullable: Boolean = false 114 | 115 | override protected lazy val strictDataType: DataType = ArrayType(child.dataType, child.isNullable) 116 | 117 | override def nodeName: String = "array_agg" 118 | 119 | override def initialState: State = ArrayBuffer.empty[Any] 120 | 121 | override def update(state: State, input: Row): State = state += child.evaluate(input) 122 | 123 | override def merge(state: State, inputState: State): State = state ++= inputState 124 | 125 | override def result(state: State): Any = state 126 | } 127 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/aggregates/logical.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.aggregates 2 | 3 | import spear.expressions.{And, Expression, Or} 4 | import spear.expressions.aggregates.FoldLeft.AccumulateFunction 5 | import spear.expressions.typecheck.TypeConstraint 6 | import spear.types.BooleanType 7 | 8 | case class BoolAnd(child: Expression) extends NullableReduceLeft { 9 | override def nodeName: String = "bool_and" 10 | 11 | override val accumulateFunction: AccumulateFunction = And 12 | 13 | override protected lazy val typeConstraint: TypeConstraint = children sameTypeAs BooleanType 14 | } 15 | 16 | case class BoolOr(child: Expression) extends NullableReduceLeft { 17 | override def nodeName: String = "bool_or" 18 | 19 | override val accumulateFunction: AccumulateFunction = Or 20 | 21 | override protected lazy val typeConstraint: TypeConstraint = children sameTypeAs BooleanType 22 | } 23 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/aggregates/numeric.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.aggregates 2 | 3 | import spear.expressions._ 4 | import spear.expressions.aggregates.FoldLeft.AccumulateFunction 5 | import spear.expressions.functions._ 6 | import spear.expressions.typecheck.TypeConstraint 7 | import spear.types.{DataType, DoubleType, NumericType} 8 | 9 | case class Average(child: Expression) extends UnaryExpression with AggregateFunction { 10 | override def nodeName: String = "avg" 11 | 12 | override def dataType: DataType = DoubleType 13 | 14 | override lazy val stateAttributes: Seq[Attribute] = Seq(sum, count) 15 | 16 | override lazy val initialValues: Seq[Expression] = Seq(Literal(null, child.dataType), 0L) 17 | 18 | override lazy val accumulateExpressions: Seq[Expression] = Seq( 19 | coalesce((child cast dataType) + sum, child cast dataType, sum), 20 | if (child.isNullable) If(child.isNull, count, count + 1L) else count + 1L 21 | ) 22 | 23 | override lazy val mergeExpressions: Seq[Expression] = Seq( 24 | sum.left + sum.right, 25 | count.left + count.right 26 | ) 27 | 28 | override lazy val resultExpression: Expression = 29 | If(count === 0L, lit(null), sum / (count cast dataType)) 30 | 31 | override protected lazy val typeConstraint: TypeConstraint = child subtypeOf NumericType 32 | 33 | private lazy val sum = 'sum of dataType nullable child.isNullable 34 | 35 | private lazy val count = 'count.long.! 36 | } 37 | 38 | case class Sum(child: Expression) extends NullableReduceLeft { 39 | override val accumulateFunction: AccumulateFunction = Plus 40 | 41 | override protected lazy val typeConstraint: TypeConstraint = children sameSubtypeOf NumericType 42 | } 43 | 44 | case class Product_(child: Expression) extends NullableReduceLeft { 45 | override def nodeName: String = "product" 46 | 47 | override val accumulateFunction: AccumulateFunction = Multiply 48 | 49 | override protected lazy val typeConstraint: TypeConstraint = children sameSubtypeOf NumericType 50 | } 51 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/comparisons.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.Row 4 | import spear.expressions.typecheck.TypeConstraint 5 | import spear.types.{BooleanType, DataType, OrderedType} 6 | 7 | trait BinaryComparison extends BinaryOperator { 8 | override def dataType: DataType = BooleanType 9 | 10 | protected lazy val ordering: Ordering[Any] = whenStrictlyTyped { 11 | OrderedType.orderingOf(left.dataType) 12 | } 13 | 14 | override protected lazy val typeConstraint: TypeConstraint = children sameSubtypeOf OrderedType 15 | } 16 | 17 | case class Eq(left: Expression, right: Expression) extends BinaryComparison { 18 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = ordering.equiv(lhs, rhs) 19 | 20 | override def operator: String = "=" 21 | } 22 | 23 | case class NotEq(left: Expression, right: Expression) extends BinaryComparison { 24 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = !ordering.equiv(lhs, rhs) 25 | 26 | override def operator: String = "<>" 27 | } 28 | 29 | case class NullSafeEq(left: Expression, right: Expression) extends BinaryComparison { 30 | override def isNullable: Boolean = false 31 | 32 | override def evaluate(input: Row): Any = children map { _ evaluate input } match { 33 | case Seq(null, null) => true 34 | case Seq(null, _) => false 35 | case Seq(_, null) => false 36 | case Seq(lhs, rhs) => ordering.equiv(lhs, rhs) 37 | } 38 | 39 | override def operator: String = "<=>" 40 | } 41 | 42 | case class Gt(left: Expression, right: Expression) extends BinaryComparison { 43 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = ordering.gt(lhs, rhs) 44 | 45 | override def operator: String = ">" 46 | } 47 | 48 | case class Lt(left: Expression, right: Expression) extends BinaryComparison { 49 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = ordering.lt(lhs, rhs) 50 | 51 | override def operator: String = "<" 52 | } 53 | 54 | case class GtEq(left: Expression, right: Expression) extends BinaryComparison { 55 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = ordering.gteq(lhs, rhs) 56 | 57 | override def operator: String = ">=" 58 | } 59 | 60 | case class LtEq(left: Expression, right: Expression) extends BinaryComparison { 61 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = ordering.lteq(lhs, rhs) 62 | 63 | override def operator: String = "<=" 64 | } 65 | 66 | case class In(test: Expression, list: Seq[Expression]) extends Expression { 67 | override def children: Seq[Expression] = test +: list 68 | 69 | override protected lazy val strictDataType: DataType = BooleanType 70 | 71 | override protected lazy val typeConstraint: TypeConstraint = 72 | test.anyType concat (list sameTypeAs test.dataType) 73 | 74 | override def evaluate(input: Row): Any = { 75 | val testValue = test evaluate input 76 | val listValues = list map { _ evaluate input } 77 | listValues contains testValue 78 | } 79 | 80 | override protected def template(childList: Seq[String]): String = { 81 | val Seq(testString, listString @ _*) = childList 82 | s"($testString IN (${listString mkString ", "}))" 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/complexTypes.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.{Name, Row} 4 | import spear.expressions.typecheck.{Foldable, TypeConstraint} 5 | import spear.types._ 6 | 7 | case class MakeNamedStruct(children: Seq[Expression]) extends Expression { 8 | assert(children.length % 2 == 0) 9 | 10 | override def isNullable: Boolean = false 11 | 12 | override def nodeName: String = "named_struct" 13 | 14 | override def evaluate(input: Row): Any = Row.fromSeq(values map { _ evaluate input }) 15 | 16 | override protected def typeConstraint: TypeConstraint = 17 | names sameTypeAs StringType andAlso Foldable concat values.anyType 18 | 19 | override protected lazy val strictDataType: DataType = { 20 | val fields = ( 21 | evaluatedNames, 22 | values map { _.dataType }, 23 | values map { _.isNullable } 24 | ).zipped map { 25 | (name, dataType, nullable) => StructField(Name.caseSensitive(name), dataType, nullable) 26 | } 27 | 28 | StructType(fields) 29 | } 30 | 31 | override protected def template(childList: Seq[String]): String = { 32 | val (nameStrings, valueStrings) = childList splitAt names.length 33 | val argStrings = nameStrings zip valueStrings flatMap { case (name, value) => Seq(name, value) } 34 | argStrings mkString (s"$nodeName(", ", ", ")") 35 | } 36 | 37 | private lazy val (names, values) = children.splitAt(children.length / 2) 38 | 39 | private lazy val evaluatedNames: Seq[String] = names map { _.evaluated } map { 40 | case n: String => n 41 | } 42 | } 43 | 44 | object MakeNamedStruct { 45 | def apply(names: Seq[Expression], values: Seq[Expression]): MakeNamedStruct = 46 | MakeNamedStruct(names ++ values) 47 | } 48 | 49 | case class MakeArray(values: Seq[Expression]) extends Expression { 50 | assert(values.nonEmpty) 51 | 52 | override def isNullable: Boolean = false 53 | 54 | override def children: Seq[Expression] = values 55 | 56 | override def nodeName: String = "array" 57 | 58 | override def evaluate(input: Row): Any = values map { _ evaluate input } 59 | 60 | override protected def typeConstraint: TypeConstraint = values.sameType 61 | 62 | override protected lazy val strictDataType: DataType = 63 | ArrayType(values.head.dataType, values exists (_.isNullable)) 64 | } 65 | 66 | case class MakeMap(children: Seq[Expression]) extends Expression { 67 | assert(children.length % 2 == 0) 68 | 69 | override def nodeName: String = "map" 70 | 71 | override def isNullable: Boolean = false 72 | 73 | override def evaluate(input: Row): Any = 74 | (keys.map { _ evaluate input } zip values.map { _ evaluate input }).toMap 75 | 76 | override protected def typeConstraint: TypeConstraint = keys.sameType concat values.sameType 77 | 78 | override protected lazy val strictDataType: DataType = { 79 | val valueNullable = values exists { _.isNullable } 80 | MapType(keys.head.dataType, values.head.dataType, valueNullable) 81 | } 82 | 83 | private lazy val (keys, values) = children.splitAt(children.length / 2) 84 | } 85 | 86 | object MakeMap { 87 | def apply(keys: Seq[Expression], values: Seq[Expression]): MakeMap = MakeMap(keys ++ values) 88 | } 89 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/dsl/ExpressionDSL.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.dsl 2 | 3 | import scala.language.implicitConversions 4 | 5 | import spear.Name 6 | import spear.expressions._ 7 | import spear.expressions.NamedExpression.newExpressionID 8 | import spear.expressions.windows.{Window, WindowFunction, WindowSpec} 9 | import spear.types.DataType 10 | 11 | trait ExpressionDSL { 12 | val self: Expression 13 | 14 | def +(that: Expression): Plus = Plus(self, that) 15 | 16 | def -(that: Expression): Minus = Minus(self, that) 17 | 18 | def *(that: Expression): Multiply = Multiply(self, that) 19 | 20 | def /(that: Expression): Divide = Divide(self, that) 21 | 22 | def %(that: Expression): Remainder = Remainder(self, that) 23 | 24 | def ^(that: Expression): Power = Power(self, that) 25 | 26 | def unary_- : Negate = Negate(self) 27 | 28 | def >(that: Expression): Gt = Gt(self, that) 29 | 30 | def <(that: Expression): Lt = Lt(self, that) 31 | 32 | def >=(that: Expression): GtEq = GtEq(self, that) 33 | 34 | def <=(that: Expression): LtEq = LtEq(self, that) 35 | 36 | def ===(that: Expression): Eq = Eq(self, that) 37 | 38 | def =/=(that: Expression): NotEq = NotEq(self, that) 39 | 40 | def <=>(that: Expression): NullSafeEq = NullSafeEq(self, that) 41 | 42 | def &&(that: Expression): And = And(self, that) 43 | 44 | def ||(that: Expression): Or = Or(self, that) 45 | 46 | def unary_! : Not = Not(self) 47 | 48 | def isNaN: IsNaN = IsNaN(self) 49 | 50 | def as(alias: Name): Alias = Alias(self, alias, newExpressionID()) 51 | 52 | def cast(dataType: DataType): Cast = Cast(self, dataType) 53 | 54 | def isNull: IsNull = IsNull(self) 55 | 56 | def isNotNull: IsNotNull = IsNotNull(self) 57 | 58 | def asc: SortOrder = SortOrder(self, Ascending, isNullLarger = true) 59 | 60 | def desc: SortOrder = SortOrder(self, Descending, isNullLarger = true) 61 | 62 | def in(list: Seq[Expression]): In = In(self, list) 63 | 64 | def in(first: Expression, rest: Expression*): In = In(self, first +: rest) 65 | 66 | def rlike(pattern: Expression): RLike = RLike(self, pattern) 67 | 68 | def over(window: WindowSpec): WindowFunction = WindowFunction(self, window) 69 | 70 | def over(windowName: Name): WindowFunction = WindowFunction(self, Window(windowName)) 71 | 72 | def over(): WindowFunction = WindowFunction(self, Window.Default) 73 | } 74 | 75 | trait LowPriorityImplicits { 76 | implicit def `Boolean->ExpressionDSL`(value: Boolean): ExpressionDSL = new ExpressionDSL { 77 | override val self: Expression = `Boolean->Literal`(value) 78 | } 79 | 80 | implicit def `Byte->ExpressionDSL`(value: Byte): ExpressionDSL = new ExpressionDSL { 81 | override val self: Expression = `Byte->Literal`(value) 82 | } 83 | 84 | implicit def `Short->ExpressionDSL`(value: Short): ExpressionDSL = new ExpressionDSL { 85 | override val self: Expression = `Short->Literal`(value) 86 | } 87 | 88 | implicit def `Int->ExpressionDSL`(value: Int): ExpressionDSL = new ExpressionDSL { 89 | override val self: Expression = `Int->Literal`(value) 90 | } 91 | 92 | implicit def `Long->ExpressionDSL`(value: Long): ExpressionDSL = new ExpressionDSL { 93 | override val self: Expression = `Long->Literal`(value) 94 | } 95 | 96 | implicit def `Float->ExpressionDSL`(value: Float): ExpressionDSL = new ExpressionDSL { 97 | override val self: Expression = `Float->Literal`(value) 98 | } 99 | 100 | implicit def `Double->ExpressionDSL`(value: Double): ExpressionDSL = new ExpressionDSL { 101 | override val self: Expression = `Double->Literal`(value) 102 | } 103 | 104 | implicit def `String->ExpressionDSL`(value: String): ExpressionDSL = new ExpressionDSL { 105 | override val self: Expression = `String->Literal`(value) 106 | } 107 | 108 | implicit def `Symbol->ExpressionDSL`(name: Symbol): ExpressionDSL = new ExpressionDSL { 109 | override val self: Expression = `Symbol->UnresolvedAttribute`(name) 110 | } 111 | 112 | implicit def `Name->ExpressionDSL`(name: Name): ExpressionDSL = new ExpressionDSL { 113 | override val self: Expression = `Name->UnresolvedAttribute`(name) 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/functions/package.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.expressions.aggregates._ 4 | 5 | package object functions { 6 | def lit(value: Any): Literal = Literal(value) 7 | 8 | def not(predicate: Expression): Not = Not(predicate) 9 | 10 | def when(condition: Expression, consequence: Expression): CaseWhen = 11 | CaseWhen(condition :: Nil, consequence :: Nil, None) 12 | 13 | def coalesce(first: Expression, second: Expression, rest: Expression*): Coalesce = 14 | Coalesce(Seq(first, second) ++ rest) 15 | 16 | def rand(seed: Int): Rand = Rand(seed) 17 | 18 | // ------------------- 19 | // Aggregate functions 20 | // ------------------- 21 | 22 | def count(expression: Expression): Count = Count(expression) 23 | 24 | def count(): Count = Count(*) 25 | 26 | def first(expression: Expression, ignoresNull: Boolean = true): First = 27 | First(expression, ignoresNull) 28 | 29 | def last(expression: Expression, ignoresNull: Boolean = true): Last = 30 | Last(expression, ignoresNull) 31 | 32 | def average(expression: Expression): Average = Average(expression) 33 | 34 | def avg(expression: Expression): Average = average(expression) 35 | 36 | def sum(expression: Expression): Sum = Sum(expression) 37 | 38 | def product(expression: Expression): Product_ = Product_(expression) 39 | 40 | def max(expression: Expression): Max = Max(expression) 41 | 42 | def min(expression: Expression): Min = Min(expression) 43 | 44 | def bool_and(expression: Expression): BoolAnd = BoolAnd(expression) 45 | 46 | def bool_or(expression: Expression): BoolOr = BoolOr(expression) 47 | 48 | def array_agg(expression: Expression): ArrayAgg = ArrayAgg(expression) 49 | 50 | def distinct(agg: AggregateFunction): DistinctAggregateFunction = DistinctAggregateFunction(agg) 51 | 52 | // ---------------- 53 | // String functions 54 | // ---------------- 55 | 56 | def concat(expressions: Seq[Expression]): Concat = Concat(expressions) 57 | 58 | def concat(first: Expression, rest: Expression*): Concat = Concat(first +: rest) 59 | 60 | def rlike(string: Expression, pattern: Expression): RLike = RLike(string, pattern) 61 | 62 | def rlike(string: Expression, pattern: String): RLike = RLike(string, pattern) 63 | 64 | // ------------------------- 65 | // Complex type constructors 66 | // ------------------------- 67 | 68 | def named_struct( 69 | first: (Expression, Expression), rest: (Expression, Expression)* 70 | ): MakeNamedStruct = { 71 | val (names, values) = (first +: rest).unzip 72 | MakeNamedStruct(names, values) 73 | } 74 | 75 | def struct(first: Expression, rest: Expression*): MakeNamedStruct = struct(first +: rest) 76 | 77 | def struct(args: Seq[Expression]): MakeNamedStruct = { 78 | val fieldNames = args.indices map { "c" + _ } map lit 79 | MakeNamedStruct(fieldNames, args) 80 | } 81 | 82 | def array(first: Expression, rest: Expression*): MakeArray = MakeArray(first +: rest) 83 | 84 | def map(keyValues: Expression*): MakeMap = { 85 | require(keyValues.length % 2 == 0) 86 | val (keys, values) = keyValues.sliding(2, 2).map { 87 | case Seq(key, value) => key -> value 88 | }.toSeq.unzip 89 | MakeMap(keys, values) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/logicalOperators.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import java.lang.{Boolean => JBoolean} 4 | 5 | import spear.Row 6 | import spear.expressions.typecheck.TypeConstraint 7 | import spear.types.{BooleanType, DataType} 8 | 9 | trait BinaryLogicalPredicate extends BinaryOperator { 10 | override lazy val dataType: DataType = BooleanType 11 | 12 | override protected lazy val typeConstraint: TypeConstraint = children sameTypeAs BooleanType 13 | } 14 | 15 | case class And(left: Expression, right: Expression) extends BinaryLogicalPredicate { 16 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = { 17 | lhs.asInstanceOf[Boolean] && rhs.asInstanceOf[Boolean] 18 | } 19 | 20 | override def operator: String = "AND" 21 | } 22 | 23 | case class Or(left: Expression, right: Expression) extends BinaryLogicalPredicate { 24 | override def nullSafeEvaluate(lhs: Any, rhs: Any): Any = 25 | lhs.asInstanceOf[Boolean] || rhs.asInstanceOf[Boolean] 26 | 27 | override def operator: String = "OR" 28 | } 29 | 30 | case class Not(child: Expression) extends UnaryOperator { 31 | override lazy val dataType: DataType = BooleanType 32 | 33 | override protected lazy val typeConstraint: TypeConstraint = children sameTypeAs BooleanType 34 | 35 | override def nullSafeEvaluate(value: Any): Any = !value.asInstanceOf[Boolean] 36 | 37 | override def operator: String = "NOT" 38 | 39 | override protected def template(childString: String): String = s"($operator $childString)" 40 | } 41 | 42 | case class If(test: Expression, yes: Expression, no: Expression) extends Expression { 43 | override protected lazy val strictDataType: DataType = yes.dataType 44 | 45 | override def children: Seq[Expression] = Seq(test, yes, no) 46 | 47 | override protected lazy val typeConstraint: TypeConstraint = 48 | test sameTypeAs BooleanType concat Seq(yes, no).sameType 49 | 50 | override def evaluate(input: Row): Any = test.evaluate(input) match { 51 | case null => null 52 | case true => yes evaluate input 53 | case false => no evaluate input 54 | } 55 | } 56 | 57 | case class CaseWhen( 58 | conditions: Seq[Expression], 59 | consequences: Seq[Expression], 60 | alternative: Option[Expression] 61 | ) extends Expression { 62 | 63 | require(conditions.nonEmpty && conditions.length == consequences.length) 64 | 65 | override def nodeName: String = "CASE" 66 | 67 | override def children: Seq[Expression] = conditions ++ consequences ++ alternative 68 | 69 | override def evaluate(input: Row): Any = { 70 | val branches = conditions.iterator map { _ evaluate input } zip consequences.iterator 71 | def alternativeValue = alternative map { _ evaluate input } 72 | 73 | val hitBranchValue = branches collectFirst { 74 | case (JBoolean.TRUE, consequence) => consequence evaluate input 75 | } 76 | 77 | (hitBranchValue orElse alternativeValue).orNull 78 | } 79 | 80 | def when(condition: Expression, consequence: Expression): CaseWhen = copy( 81 | conditions = conditions :+ condition, 82 | consequences = consequences :+ consequence 83 | ) 84 | 85 | def otherwise(expression: Expression): CaseWhen = copy(alternative = Some(expression)) 86 | 87 | override protected lazy val typeConstraint: TypeConstraint = 88 | conditions sameTypeAs BooleanType concat (consequences ++ alternative).sameType 89 | 90 | override protected lazy val strictDataType: DataType = consequences.head.dataType 91 | 92 | override protected def template(childList: Seq[String]): String = { 93 | val (tests, rest) = childList splitAt conditions.length 94 | val (values, alternativeString) = rest splitAt conditions.length 95 | val elsePart = alternativeString.headOption map { " ELSE " + _ } getOrElse "" 96 | val cases = (tests, values).zipped map { "WHEN " + _ + " THEN " + _ } mkString " " 97 | s"CASE $cases$elsePart END" 98 | } 99 | } 100 | 101 | object CaseKeyWhen { 102 | def apply( 103 | key: Expression, 104 | candidates: Seq[Expression], 105 | consequences: Seq[Expression], 106 | alternative: Option[Expression] 107 | ): CaseWhen = CaseWhen(candidates map { key === _ }, consequences, alternative) 108 | } 109 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/misc.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.Row 4 | import spear.expressions.typecheck.TypeConstraint 5 | import spear.types._ 6 | 7 | case class Length(child: Expression) extends UnaryExpression { 8 | override protected def typeConstraint: TypeConstraint = 9 | child oneOf (StringType, ArrayType, MapType) 10 | 11 | override def dataType: DataType = IntType 12 | 13 | override def evaluate(input: Row): Any = child evaluate input match { 14 | case v: String => v.length 15 | case v: Array[_] => v.length 16 | case v: TraversableOnce[_] => v.size 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/nullExpressions.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.Row 4 | import spear.expressions.typecheck.TypeConstraint 5 | import spear.types.{BooleanType, DataType} 6 | 7 | case class Coalesce(children: Seq[Expression]) extends Expression { 8 | override protected lazy val strictDataType: DataType = children.head.dataType 9 | 10 | override protected lazy val typeConstraint: TypeConstraint = children.sameType 11 | 12 | override def evaluate(input: Row): Any = 13 | children.iterator.map { _ evaluate input }.find { _ != null }.orNull 14 | } 15 | 16 | object Coalesce { 17 | def apply(first: Expression, second: Expression, rest: Expression*): Coalesce = 18 | Coalesce(Seq(first, second) ++ rest) 19 | } 20 | 21 | case class IsNull(child: Expression) extends UnaryExpression { 22 | override def evaluate(input: Row): Any = (child evaluate input) == null 23 | 24 | override lazy val dataType: DataType = BooleanType 25 | 26 | override protected def template(childString: String): String = s"($childString IS NULL)" 27 | } 28 | 29 | case class IsNotNull(child: Expression) extends UnaryExpression { 30 | override def evaluate(input: Row): Any = (child evaluate input) != null 31 | 32 | override lazy val dataType: DataType = BooleanType 33 | 34 | override protected def template(childString: String): String = s"($childString IS NOT NULL)" 35 | } 36 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/object.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.Row 4 | import spear.expressions.typecheck.TypeConstraint 5 | import spear.types.{DataType, ObjectType} 6 | 7 | case class Invoke( 8 | target: Expression, 9 | methodName: String, 10 | override val dataType: DataType, 11 | args: Seq[Expression] 12 | ) extends Expression { 13 | 14 | override def children: Seq[Expression] = target +: args 15 | 16 | override def evaluate(input: Row): Any = { 17 | val evaluatedArgs = args map { _ evaluate input } map { _.asInstanceOf[AnyRef] } 18 | method.invoke(target evaluate input, evaluatedArgs: _*) 19 | } 20 | 21 | override protected def typeConstraint: TypeConstraint = 22 | target subtypeOf ObjectType concat args.anyType 23 | 24 | private lazy val targetClass: Class[_] = target.dataType match { 25 | case t: ObjectType => Class forName t.className 26 | } 27 | 28 | private lazy val method = targetClass.getDeclaredMethods.find { _.getName == methodName }.get 29 | } 30 | 31 | case class StaticInvoke( 32 | className: String, 33 | methodName: String, 34 | override val dataType: DataType, 35 | args: Seq[Expression] 36 | ) extends Expression { 37 | 38 | override def children: Seq[Expression] = args 39 | 40 | override def evaluate(input: Row): Any = { 41 | val evaluatedArgs = args map { _ evaluate input } map { _.asInstanceOf[AnyRef] } 42 | method.invoke(null, evaluatedArgs: _*) 43 | } 44 | 45 | private lazy val targetClass: Class[_] = Class.forName(className) 46 | 47 | private lazy val method = targetClass.getDeclaredMethods.find(_.getName == methodName).get 48 | } 49 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.language.implicitConversions 4 | 5 | import spear.expressions.dsl.ExpressionDSL 6 | import spear.expressions.typecheck._ 7 | import spear.parsers._ 8 | import spear.types._ 9 | 10 | package object expressions extends expressions.dsl.LowPriorityImplicits { 11 | val * : Star = Star(None) 12 | 13 | implicit def `Expression->ExpressionDSL`(expression: Expression): ExpressionDSL = 14 | new ExpressionDSL { 15 | override val self: Expression = expression 16 | } 17 | 18 | implicit def `Boolean->Literal`(value: Boolean): Literal = Literal(value, BooleanType) 19 | 20 | implicit def `Byte->Literal`(value: Byte): Literal = Literal(value, ByteType) 21 | 22 | implicit def `Short->Literal`(value: Short): Literal = Literal(value, ShortType) 23 | 24 | implicit def `Int->Literal`(value: Int): Literal = Literal(value, IntType) 25 | 26 | implicit def `Long->Literal`(value: Long): Literal = Literal(value, LongType) 27 | 28 | implicit def `Float->Literal`(value: Float): Literal = Literal(value, FloatType) 29 | 30 | implicit def `Double->Literal`(value: Double): Literal = Literal(value, DoubleType) 31 | 32 | implicit def `String->Literal`(value: String): Literal = Literal(value, StringType) 33 | 34 | implicit class OfDataType(value: Any) { 35 | def of(dataType: DataType): Literal = Literal(value, dataType) 36 | } 37 | 38 | implicit def `Symbol->UnresolvedAttribute`(name: Symbol): UnresolvedAttribute = 39 | UnresolvedAttribute(name) 40 | 41 | implicit def `Name->UnresolvedAttribute`(name: Name): UnresolvedAttribute = 42 | UnresolvedAttribute(name) 43 | 44 | implicit class ParsedUnresolvedAttribute(sc: StringContext) { 45 | import ColumnReferenceParser._ 46 | import QuerySpecificationParser._ 47 | import fastparse.all._ 48 | 49 | private val parser: P[NamedExpression] = ( 50 | "*" ~ PassWith(*) 51 | | qualifiedAsterisk 52 | | columnReference 53 | ) 54 | 55 | def $(args: Any*): NamedExpression = (parser parse sc.s(args: _*)).get.value 56 | } 57 | 58 | private[spear] implicit class NamedExpressionSet[E <: NamedExpression](set: Set[E]) { 59 | require(set forall { _.isResolved }) 60 | 61 | def intersectByID(other: Set[E]): Set[E] = { 62 | require(other forall { _.isResolved }) 63 | val otherIDs = other map { _.expressionID } 64 | set filter { e => otherIDs contains e.expressionID } 65 | } 66 | 67 | def subsetOfByID(other: Set[E]): Boolean = { 68 | require(other forall { _.isResolved }) 69 | val otherIDs = other map { _.expressionID } 70 | set forall { e => otherIDs contains e.expressionID } 71 | } 72 | } 73 | 74 | def function(name: Name, args: Expression*): UnresolvedFunction = 75 | UnresolvedFunction(name, args, isDistinct = false) 76 | 77 | def distinctFunction(name: Name, args: Expression*): UnresolvedFunction = 78 | UnresolvedFunction(name, args, isDistinct = true) 79 | 80 | implicit class UnresolvedFunctionDSL(name: Symbol) { 81 | def apply(args: Expression*): UnresolvedFunction = function(name, args: _*) 82 | } 83 | 84 | implicit class TypeConstraintDSL(input: Seq[Expression]) { 85 | def anyType: TypeConstraint = StrictlyTyped(input) 86 | 87 | def sameTypeAs(dataType: DataType): TypeConstraint = SameTypeAs(input, dataType) 88 | 89 | def sameSubtypeOf(supertype: AbstractDataType): TypeConstraint = SameSubtypeOf(input, supertype) 90 | 91 | def sameType: TypeConstraint = SameType(input) 92 | 93 | def foldable: TypeConstraint = Foldable(input) 94 | 95 | def castTo(dataType: DataType): TypeConstraint = CastTo(input, dataType) 96 | } 97 | 98 | implicit class SingleExpressionTypeConstraintDSL(input: Expression) { 99 | def anyType: TypeConstraint = StrictlyTyped(Seq(input)) 100 | 101 | def sameTypeAs(dataType: DataType): TypeConstraint = Seq(input) sameTypeAs dataType 102 | 103 | def subtypeOf(supertype: AbstractDataType): TypeConstraint = Seq(input) sameSubtypeOf supertype 104 | 105 | def oneOf(first: AbstractDataType, rest: AbstractDataType*): TypeConstraint = 106 | Seq(input) sameSubtypeOf OneOf(first +: rest) 107 | 108 | def foldable: TypeConstraint = Foldable(Seq(input)) 109 | 110 | def castTo(dataType: DataType): TypeConstraint = CastTo(Seq(input), dataType) 111 | } 112 | 113 | implicit class Invoker(expression: Expression) { 114 | def invoke(methodName: String, returnType: DataType): InvokeBuilder = 115 | new InvokeBuilder(methodName, returnType) 116 | 117 | class InvokeBuilder(methodName: String, returnType: DataType) { 118 | def withArgs(args: Expression*): Invoke = Invoke(expression, methodName, returnType, args) 119 | 120 | def noArgs: Invoke = Invoke(expression, methodName, returnType, Nil) 121 | } 122 | } 123 | 124 | implicit class StaticInvoker(targetClass: Class[_]) { 125 | def invoke(methodName: String, returnType: DataType): StaticInvokeBuilder = 126 | new StaticInvokeBuilder(methodName, returnType) 127 | 128 | class StaticInvokeBuilder(methodName: String, returnType: DataType) { 129 | def withArgs(args: Expression*): StaticInvoke = 130 | StaticInvoke(targetClass.getName, methodName, returnType, args) 131 | 132 | def noArgs: StaticInvoke = 133 | StaticInvoke(targetClass.getName, methodName, returnType, Nil) 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/patterns.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | private[spear] trait BinaryOperatorPattern[T <: BinaryExpression] { 4 | def unapply(op: T): Option[(Expression, Expression)] = Some((op.left, op.right)) 5 | } 6 | 7 | private[spear] object ! { 8 | def unapply(not: Not): Option[Expression] = Some(not.child) 9 | } 10 | 11 | private[spear] object || extends BinaryOperatorPattern[Or] 12 | 13 | private[spear] object && extends BinaryOperatorPattern[And] 14 | 15 | private[spear] object === extends BinaryOperatorPattern[Eq] 16 | 17 | private[spear] object =/= extends BinaryOperatorPattern[NotEq] 18 | 19 | private[spear] object > extends BinaryOperatorPattern[Gt] 20 | 21 | private[spear] object < extends BinaryOperatorPattern[Lt] 22 | 23 | private[spear] object >= extends BinaryOperatorPattern[GtEq] 24 | 25 | private[spear] object <= extends BinaryOperatorPattern[LtEq] 26 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/stateful.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import scala.util.Random 4 | 5 | import spear.Row 6 | import spear.expressions.typecheck.{Foldable, TypeConstraint} 7 | import spear.types.{DataType, DoubleType, LongType} 8 | 9 | case class Rand(seed: Expression) extends UnaryExpression with StatefulExpression[Random] { 10 | override def dataType: DataType = DoubleType 11 | 12 | override def child: Expression = seed 13 | 14 | override protected def typeConstraint: TypeConstraint = 15 | seed sameTypeAs LongType andAlso Foldable 16 | 17 | private lazy val evaluatedSeed: Long = seed.evaluated.asInstanceOf[Long] 18 | 19 | override protected lazy val initialState: Random = new Random(evaluatedSeed) 20 | 21 | override protected def statefulEvaluate(state: Random, input: Row): (Any, Random) = 22 | (state.nextDouble(), state) 23 | 24 | override protected def template(childString: String): String = s"RAND($childString)" 25 | } 26 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/stringExpressions.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import java.util.regex.Pattern 4 | 5 | import scala.util.Try 6 | 7 | import spear.Row 8 | import spear.expressions.typecheck.{Foldable, TypeConstraint} 9 | import spear.types.{BooleanType, DataType, StringType} 10 | import spear.utils.trySequence 11 | 12 | case class Concat(children: Seq[Expression]) extends Expression { 13 | override def dataType: DataType = StringType 14 | 15 | override protected def typeConstraint: TypeConstraint = children sameTypeAs StringType 16 | 17 | override def evaluate(input: Row): Any = 18 | children.map { _ evaluate input }.map { _.asInstanceOf[String] }.filter { _ != null }.mkString 19 | 20 | override def sql: Try[String] = 21 | trySequence(children map { _.sql }) map { _ mkString ("(", " || ", ")") } 22 | } 23 | 24 | case class RLike(left: Expression, right: Expression) extends BinaryOperator { 25 | override def operator: String = "RLIKE" 26 | 27 | override def dataType: DataType = BooleanType 28 | 29 | override protected def typeConstraint: TypeConstraint = 30 | left sameTypeAs StringType concat (right sameTypeAs StringType andAlso Foldable) 31 | 32 | private lazy val compiledPattern = 33 | Pattern.compile(right.evaluated match { case pattern: String => pattern }) 34 | 35 | override def evaluate(input: Row): Any = 36 | compiledPattern.matcher(left.evaluate(input).asInstanceOf[String]).matches() 37 | } 38 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/expressions/windows/WindowFunction.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.windows 2 | 3 | import spear.expressions.{BinaryExpression, Expression} 4 | import spear.types.DataType 5 | 6 | case class WindowFunction(function: Expression, window: WindowSpec) extends BinaryExpression { 7 | override def dataType: DataType = function.dataType 8 | 9 | override def isNullable: Boolean = function.isNullable 10 | 11 | override lazy val isFoldable: Boolean = function.isFoldable 12 | 13 | override def left: Expression = function 14 | 15 | override def right: Expression = window 16 | 17 | override protected def template(childList: Seq[String]): String = childList mkString " OVER " 18 | } 19 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/package.scala: -------------------------------------------------------------------------------- 1 | import scala.language.implicitConversions 2 | 3 | import spear.Name.{caseInsensitive, caseSensitive} 4 | 5 | package object spear { 6 | implicit def `String->CaseSensitiveName`(string: String): Name = caseSensitive(string) 7 | 8 | implicit def `Symbol->CaseInsensitiveName`(symbol: Symbol): Name = caseInsensitive(symbol.name) 9 | 10 | implicit class NameHelper(sc: StringContext) { 11 | def i(args: Any*): Name = caseInsensitive(sc.s(args: _*)) 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/DataTypeParser.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.all.{LiteralStr, P, PassWith} 4 | 5 | import spear.parsers.annotations.ExtendedSQLSyntax 6 | import spear.types._ 7 | 8 | object DataTypeParser extends LoggingParser { 9 | import KeywordParser._ 10 | import NameParser._ 11 | import WhitespaceApi.parserApi 12 | 13 | @ExtendedSQLSyntax 14 | private val extendedNumericType: P[DataType] = TINYINT ~ PassWith(ByteType) opaque "extended-type" 15 | 16 | private val exactNumericType: P[DataType] = ( 17 | BIGINT ~ PassWith(LongType) 18 | | INT ~ PassWith(IntType) 19 | | INTEGER ~ PassWith(IntType) 20 | | SMALLINT ~ PassWith(ShortType) 21 | | extendedNumericType 22 | opaque "exact-numeric-type" 23 | ) 24 | 25 | private val approximateNumericType: P[DataType] = ( 26 | FLOAT ~ PassWith(FloatType) 27 | | REAL ~ PassWith(DoubleType) 28 | | DOUBLE ~ PassWith(DoubleType) 29 | opaque "approximate-numeric-type" 30 | ) 31 | 32 | private val numericType: P[DataType] = 33 | exactNumericType | approximateNumericType opaque "numeric-type" 34 | 35 | private val booleanType: P[DataType] = BOOLEAN ~ PassWith(BooleanType) opaque "boolean-type" 36 | 37 | private val datetimeType: P[DataType] = DATE ~ PassWith(DateType) opaque "datetime-type" 38 | 39 | @ExtendedSQLSyntax 40 | private val stringType: P[DataType] = STRING ~ PassWith(StringType) opaque "string-type" 41 | 42 | private val characterStringType: P[DataType] = stringType opaque "character-string-type" 43 | 44 | private val predefinedType: P[DataType] = ( 45 | numericType 46 | | booleanType 47 | | datetimeType 48 | | characterStringType 49 | opaque "predefined-type" 50 | ) 51 | 52 | private val fieldDefinition: P[StructField] = ( 53 | fieldName ~ P(dataType) 54 | map { case (name, fieldType) => StructField(name, fieldType.?) } 55 | opaque "field-definition" 56 | ) 57 | 58 | private val rowTypeBody: P[StructType] = ( 59 | "(" ~ fieldDefinition.rep(min = 1, sep = ",") ~ ")" 60 | map StructType.apply 61 | opaque "row-type-body" 62 | ) 63 | 64 | private val rowType: P[StructType] = ROW ~ rowTypeBody opaque "row-type" 65 | 66 | @ExtendedSQLSyntax 67 | private val structField: P[StructField] = ( 68 | fieldName ~ ":" ~ P(dataType) 69 | map { case (name, fieldType) => StructField(name, fieldType, isNullable = true) } 70 | opaque "struct-field" 71 | ) 72 | 73 | @ExtendedSQLSyntax 74 | private val structType: P[StructType] = ( 75 | STRUCT ~ "<" ~ structField.rep(sep = ",") ~ ">" 76 | map StructType.apply 77 | opaque "struct-type" 78 | ) 79 | 80 | @ExtendedSQLSyntax 81 | private val arrayType: P[ArrayType] = ( 82 | ARRAY ~ "<" ~ P(dataType) ~ ">" 83 | map { ArrayType(_, isElementNullable = true) } 84 | opaque "array-type" 85 | ) 86 | 87 | @ExtendedSQLSyntax 88 | private val mapType: P[MapType] = ( 89 | MAP ~ "<" ~ P(dataType) ~ "," ~ P(dataType) ~ ">" 90 | map { case (keyType, valueType) => MapType(keyType, valueType, isValueNullable = true) } 91 | opaque "map-type" 92 | ) 93 | 94 | @ExtendedSQLSyntax 95 | private val extendedNestedType: P[DataType] = 96 | structType | arrayType | mapType opaque "extended-nested-type" 97 | 98 | lazy val dataType: P[DataType] = predefinedType | rowType | extendedNestedType opaque "data-type" 99 | } 100 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/DirectlyExecutableStatementParser.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.all.P 4 | 5 | import spear.plans.logical.LogicalPlan 6 | 7 | // SQL06 section 22.1 8 | object DirectlyExecutableStatementParser extends LoggingParser { 9 | import OrderByClauseParser._ 10 | import QueryExpressionParser._ 11 | import WhitespaceApi.parserApi 12 | 13 | private val cursorSpecification: P[LogicalPlan] = queryExpression ~ orderByClause.? map { 14 | case (plan, orderBy) => orderBy.orIdentity apply plan 15 | } opaque "cursor-specification" 16 | 17 | private val directSelectStatementMultipleRows: P[LogicalPlan] = 18 | cursorSpecification opaque "direct-select-statement-multiple-rows" 19 | 20 | private val directSQLDataStatement: P[LogicalPlan] = 21 | directSelectStatementMultipleRows opaque "direct-sql-data-statement" 22 | 23 | val directlyExecutableStatement: P[LogicalPlan] = 24 | directSQLDataStatement opaque "directly-executable-statement" 25 | } 26 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/KeywordParser.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import scala.collection.mutable 4 | 5 | import fastparse.all._ 6 | 7 | // SQL06 section 5.2 8 | object KeywordParser { 9 | private val reservedWords: mutable.ArrayBuffer[P0] = mutable.ArrayBuffer.empty[P0] 10 | 11 | private def mkKeyword(literal: String): P0 = { 12 | val parser = P(IgnoreCase(literal)) opaque literal.toUpperCase 13 | reservedWords += parser 14 | parser 15 | } 16 | 17 | val ALL: P0 = mkKeyword("ALL") 18 | val AND: P0 = mkKeyword("AND") 19 | val ARRAY: P0 = mkKeyword("ARRAY") 20 | val AS: P0 = mkKeyword("AS") 21 | val ASC: P0 = mkKeyword("ASC") 22 | val BETWEEN: P0 = mkKeyword("BETWEEN") 23 | val BIGINT: P0 = mkKeyword("BIGINT") 24 | val BINARY: P0 = mkKeyword("BINARY") 25 | val BLOB: P0 = mkKeyword("BLOB") 26 | val BOOLEAN: P0 = mkKeyword("BOOLEAN") 27 | val BY: P0 = mkKeyword("BY") 28 | val CASE: P0 = mkKeyword("CASE") 29 | val CAST: P0 = mkKeyword("CAST") 30 | val COALESCE: P0 = mkKeyword("COALESCE") 31 | val CROSS: P0 = mkKeyword("CROSS") 32 | val CURRENT: P0 = mkKeyword("CURRENT") 33 | val DATE: P0 = mkKeyword("DATE") 34 | val DAY: P0 = mkKeyword("DAY") 35 | val DEC: P0 = mkKeyword("DEC") 36 | val DECIMAL: P0 = mkKeyword("DECIMAL") 37 | val DESC: P0 = mkKeyword("DESC") 38 | val DISTINCT: P0 = mkKeyword("DISTINCT") 39 | val DOUBLE: P0 = mkKeyword("DOUBLE") 40 | val ELSE: P0 = mkKeyword("ELSE") 41 | val END: P0 = mkKeyword("END") 42 | val EXCEPT: P0 = mkKeyword("EXCEPT") 43 | val FALSE: P0 = mkKeyword("FALSE") 44 | val FIRST: P0 = mkKeyword("FIRST") 45 | val FLOAT: P0 = mkKeyword("FLOAT") 46 | val FOLLOWING: P0 = mkKeyword("FOLLOWING") 47 | val FROM: P0 = mkKeyword("FROM") 48 | val FULL: P0 = mkKeyword("FULL") 49 | val GROUP: P0 = mkKeyword("GROUP") 50 | val HAVING: P0 = mkKeyword("HAVING") 51 | val HOUR: P0 = mkKeyword("HOUR") 52 | val IF: P0 = mkKeyword("IF") 53 | val INNER: P0 = mkKeyword("INNER") 54 | val INT: P0 = mkKeyword("INT") 55 | val INTEGER: P0 = mkKeyword("INTEGER") 56 | val INTERSECT: P0 = mkKeyword("INTERSECT") 57 | val INTERVAL: P0 = mkKeyword("INTERVAL") 58 | val IS: P0 = mkKeyword("IS") 59 | val JOIN: P0 = mkKeyword("JOIN") 60 | val LARGE: P0 = mkKeyword("LARGE") 61 | val LAST: P0 = mkKeyword("LAST") 62 | val LEFT: P0 = mkKeyword("LEFT") 63 | val LIMIT: P0 = mkKeyword("LIMIT") 64 | val MAP: P0 = mkKeyword("MAP") 65 | val MINUTE: P0 = mkKeyword("MINUTE") 66 | val MODULE: P0 = mkKeyword("MODULE") 67 | val MONTH: P0 = mkKeyword("MONTH") 68 | val NATURAL: P0 = mkKeyword("NATURAL") 69 | val NOT: P0 = mkKeyword("NOT") 70 | val NULL: P0 = mkKeyword("NULL") 71 | val NULLIF: P0 = mkKeyword("NULLIF") 72 | val NULLS: P0 = mkKeyword("NULLS") 73 | val NUMERIC: P0 = mkKeyword("NUMERIC") 74 | val OBJECT: P0 = mkKeyword("OBJECT") 75 | val ON: P0 = mkKeyword("ON") 76 | val ORDER: P0 = mkKeyword("ORDER") 77 | val OR: P0 = !ORDER ~ mkKeyword("OR") 78 | val OUTER: P0 = mkKeyword("OUTER") 79 | val OVER: P0 = mkKeyword("OVER") 80 | val PARTITION: P0 = mkKeyword("PARTITION") 81 | val PRECEDING: P0 = mkKeyword("PRECEDING") 82 | val PRECISION: P0 = mkKeyword("PRECISION") 83 | val RANGE: P0 = mkKeyword("RANGE") 84 | val REAL: P0 = mkKeyword("REAL") 85 | val RIGHT: P0 = mkKeyword("RIGHT") 86 | val ROW: P0 = mkKeyword("ROW") 87 | val ROWS: P0 = mkKeyword("ROWS") 88 | val SECOND: P0 = mkKeyword("SECOND") 89 | val SELECT: P0 = mkKeyword("SELECT") 90 | val SMALLINT: P0 = mkKeyword("SMALLINT") 91 | val STRING: P0 = mkKeyword("STRING") 92 | val STRUCT: P0 = mkKeyword("STRUCT") 93 | val THEN: P0 = mkKeyword("THEN") 94 | val TIME: P0 = mkKeyword("TIME") 95 | val TIMESTAMP: P0 = mkKeyword("TIMESTAMP") 96 | val TINYINT: P0 = mkKeyword("TINYINT") 97 | val TO: P0 = mkKeyword("TO") 98 | val TRUE: P0 = mkKeyword("TRUE") 99 | val UESCAPE: P0 = mkKeyword("UESCAPE") 100 | val UNBOUNDED: P0 = mkKeyword("UNBOUNDED") 101 | val UNION: P0 = mkKeyword("UNION") 102 | val UNKNOWN: P0 = mkKeyword("UNKNOWN") 103 | val WHEN: P0 = mkKeyword("WHEN") 104 | val WHERE: P0 = mkKeyword("WHERE") 105 | val WINDOW: P0 = mkKeyword("WINDOW") 106 | val WITH: P0 = mkKeyword("WITH") 107 | val WITHOUT: P0 = mkKeyword("WITHOUT") 108 | val YEAR: P0 = mkKeyword("YEAR") 109 | val ZONE: P0 = mkKeyword("ZONE") 110 | 111 | val keyword: P0 = reservedWords reduce { _ | _ } opaque "keyword" 112 | } 113 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/LoggingParser.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.core.Logger 4 | 5 | import spear.utils.Logging 6 | 7 | trait LoggingParser extends Logging { 8 | protected implicit val parsingLogger = Logger(logTrace(_)) 9 | } 10 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/ParserImplicits.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | trait ParserImplicits[+T] { 4 | import fastparse.all.{P, Parser} 5 | 6 | val self: Parser[T] 7 | 8 | def fold[U >: T](sep: => P[(U, U) => U], min: Int = 0): P[U] = { 9 | import WhitespaceApi.parserApi 10 | 11 | self ~ (sep ~ self rep (min = min)) map { 12 | case (head, tail) => 13 | tail.foldLeft(head: U) { 14 | case (lhs, (op, rhs)) => 15 | op(lhs, rhs) 16 | } 17 | } 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/SeparatorParser.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.all.{parserApi, AnyChar, CharPred, LiteralStr, P0} 4 | 5 | // SQL06 section 5.2 6 | object SeparatorParser { 7 | private val EoF = '\u001a' 8 | 9 | val whitespace: P0 = CharPred { ch => ch <= ' ' && ch != EoF } opaque "whitespace" 10 | 11 | private val comment: P0 = ( 12 | "--" ~ (!"\n" ~ AnyChar).rep 13 | | "/*" ~ (!"*" ~ AnyChar).rep ~ "*/" 14 | opaque "comment" 15 | ) 16 | 17 | val separator: P0 = comment | whitespace opaque "separator" 18 | } 19 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/parsers/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.language.implicitConversions 4 | 5 | import fastparse.WhitespaceApi.Wrapper 6 | 7 | // SQL06 section 5.2 8 | package object parsers { 9 | import fastparse.all._ 10 | 11 | val WhitespaceApi: Wrapper = Wrapper(SeparatorParser.separator.rep) 12 | 13 | implicit def `Parser->ParserImplicits`[T](parser: Parser[T]): ParserImplicits[T] = 14 | new ParserImplicits[T] { override val self: Parser[T] = parser } 15 | 16 | implicit def `String->ParserImplicits`(literal: String): ParserImplicits[Unit] = 17 | new ParserImplicits[Unit] { override val self: Parser[Unit] = P(literal) } 18 | 19 | implicit class OrIdentity[A](maybeTransform: Option[A => A]) { 20 | def orIdentity: A => A = maybeTransform getOrElse identity[A] _ 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/CompiledQuery.scala: -------------------------------------------------------------------------------- 1 | package spear.plans 2 | 3 | import spear.Context 4 | import spear.plans.logical.LogicalPlan 5 | import spear.plans.physical.PhysicalPlan 6 | 7 | class CompiledQuery(val context: Context, val logicalPlan: LogicalPlan) { 8 | lazy val analyzedPlan: LogicalPlan = context.queryCompiler analyze logicalPlan 9 | 10 | lazy val optimizedPlan: LogicalPlan = context.queryCompiler optimize analyzedPlan 11 | 12 | lazy val physicalPlan: PhysicalPlan = context.queryCompiler plan optimizedPlan 13 | } 14 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/QueryPlanner.scala: -------------------------------------------------------------------------------- 1 | package spear.plans 2 | 3 | import spear.trees.TreeNode 4 | import spear.utils.Logging 5 | 6 | trait QueryPlanner[Logical <: QueryPlan[Logical], Physical <: TreeNode[Physical]] extends Logging { 7 | trait Strategy extends (Logical => Seq[Physical]) 8 | 9 | def strategies: Seq[Strategy] 10 | 11 | def planLater(logicalPlan: Logical): Physical = plan(logicalPlan) 12 | 13 | private def plan(logicalPlan: Logical): Physical = { 14 | val physicalPlans = for { 15 | strategy <- strategies 16 | physicalPlan <- strategy(logicalPlan) 17 | } yield physicalPlan 18 | 19 | assert( 20 | physicalPlans.nonEmpty, 21 | s"""Don't know how to compile the following logical query plan fragment: 22 | | 23 | |${logicalPlan.prettyTree} 24 | |""".stripMargin 25 | ) 26 | 27 | physicalPlans.head 28 | } 29 | 30 | def apply(logicalPlan: Logical): Physical = { 31 | logDebug( 32 | s"""Planning logical query plan: 33 | | 34 | |${logicalPlan.prettyTree} 35 | |""".stripMargin 36 | ) 37 | 38 | val physicalPlan = plan(logicalPlan) 39 | 40 | logDebug( 41 | s"""Compiled logical query plan 42 | | 43 | |${logicalPlan.prettyTree} 44 | | 45 | |to physical query plan 46 | | 47 | |${physicalPlan.prettyTree} 48 | |""".stripMargin 49 | ) 50 | 51 | physicalPlan 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/logical/analysis/expressionsAnalysis.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import scala.util.control.NonFatal 4 | 5 | import spear._ 6 | import spear.exceptions.{AnalysisException, ResolutionFailureException} 7 | import spear.expressions._ 8 | import spear.expressions.NamedExpression.AnonymousColumnName 9 | import spear.expressions.aggregates.{AggregateFunction, Count, DistinctAggregateFunction} 10 | import spear.plans.logical._ 11 | import spear.plans.logical.patterns.{Resolved, Unresolved} 12 | 13 | /** 14 | * This rule expands "`*`" appearing in `SELECT`. 15 | */ 16 | class ExpandStars(val catalog: Catalog) extends AnalysisRule { 17 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformUp { 18 | case Unresolved(Resolved(child) Project projectList) => 19 | child select (projectList flatMap { 20 | case Star(qualifier) => expand(qualifier, child.output) 21 | case e => Seq(e) 22 | }) 23 | } 24 | 25 | private def expand(maybeQualifier: Option[Name], input: Seq[Attribute]): Seq[Attribute] = 26 | (maybeQualifier fold input) { qualifier => 27 | input collect { 28 | case ref: AttributeRef if ref.qualifier contains qualifier => ref 29 | } 30 | } 31 | } 32 | 33 | /** 34 | * This rule tries to resolve [[spear.expressions.UnresolvedAttribute UnresolvedAttribute]]s in 35 | * an logical plan operator using output [[spear.expressions.Attribute Attribute]]s of its 36 | * children. 37 | * 38 | * @throws spear.exceptions.ResolutionFailureException If no candidate or multiple ambiguous 39 | * candidate input attributes can be found. 40 | */ 41 | class ResolveReferences(val catalog: Catalog) extends AnalysisRule { 42 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformUp { 43 | case Unresolved(plan) if plan.isDeduplicated => 44 | val input = plan.children flatMap { _.output } 45 | plan transformExpressionsDown { 46 | case a: UnresolvedAttribute => 47 | try a tryResolveUsing input catch { 48 | case NonFatal(cause) => 49 | throw new ResolutionFailureException( 50 | s"""Failed to resolve attribute ${a.sqlLike} in logical plan: 51 | | 52 | |${plan.prettyTree} 53 | |""".stripMargin, 54 | cause 55 | ) 56 | } 57 | } 58 | } 59 | } 60 | 61 | /** 62 | * This rule converts [[spear.expressions.UnresolvedAlias unresolved aliases]] into proper 63 | * [[spear.expressions.Alias aliases]]. 64 | */ 65 | class ResolveAliases(val catalog: Catalog) extends AnalysisRule { 66 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformDown { 67 | case node => 68 | node transformExpressionsDown { 69 | case UnresolvedAlias(Resolved(child: Expression)) => 70 | child as Name.caseSensitive(child.sql getOrElse AnonymousColumnName) 71 | } 72 | } 73 | } 74 | 75 | /** 76 | * This rule resolves [[spear.expressions.UnresolvedFunction unresolved functions]] by looking 77 | * up function names from the [[Catalog]]. 78 | */ 79 | class ResolveFunctions(val catalog: Catalog) extends AnalysisRule { 80 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformDown { 81 | case node => 82 | node transformExpressionsDown { 83 | case UnresolvedFunction(name, Seq(_: Star), false) if name == i"count" => 84 | Count(1) 85 | 86 | case Count(_: Star) => 87 | Count(1) 88 | 89 | case UnresolvedFunction(_, Seq(_: Star), true) => 90 | throw new AnalysisException("DISTINCT cannot be used together with star") 91 | 92 | case UnresolvedFunction(_, Seq(_: Star), _) => 93 | throw new AnalysisException("Only function \"count\" may have star as argument") 94 | 95 | case UnresolvedFunction(name, args, isDistinct) if args forall { _.isResolved } => 96 | val fnInfo = catalog.functionRegistry lookupFunction name 97 | fnInfo.builder apply args match { 98 | case f: AggregateFunction if isDistinct => 99 | DistinctAggregateFunction(f) 100 | 101 | case _ if isDistinct => 102 | throw new AnalysisException(s"Cannot apply DISTINCT to non-aggregate function $name") 103 | 104 | case f => 105 | f 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/logical/analysis/postAnalysisCheck.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import spear._ 4 | import spear.exceptions.{AnalysisException, ResolutionFailureException} 5 | import spear.expressions._ 6 | import spear.expressions.aggregates.DistinctAggregateFunction 7 | import spear.plans.logical._ 8 | import spear.plans.logical.patterns.Unresolved 9 | import spear.utils._ 10 | 11 | class RejectUnresolvedExpressions(val catalog: Catalog) extends AnalysisRule { 12 | override def transform(tree: LogicalPlan): LogicalPlan = tree.transformExpressionsDown { 13 | // Tries to collect a "minimum" unresolved expression. 14 | case Unresolved(e) if e.children forall { _.isResolved } => 15 | throw new ResolutionFailureException( 16 | s"""Failed to resolve expression ${e.sqlLike} in the analyzed logical plan: 17 | | 18 | |${tree.prettyTree} 19 | |""".stripMargin 20 | ) 21 | } 22 | } 23 | 24 | class RejectUnresolvedPlans(val catalog: Catalog) extends AnalysisRule { 25 | override def transform(tree: LogicalPlan): LogicalPlan = tree.transformDown { 26 | // Tries to collect a "minimum" unresolved logical plan node. 27 | case Unresolved(plan) if plan.children forall { _.isResolved } => 28 | throw new ResolutionFailureException( 29 | s"""Failed to resolve the following logical plan operator 30 | | 31 | |${plan.prettyTree} 32 | | 33 | |in the analyzed plan: 34 | | 35 | |${tree.prettyTree} 36 | |""".stripMargin 37 | ) 38 | } 39 | } 40 | 41 | class RejectTopLevelInternalAttributes(val catalog: Catalog) extends AnalysisRule { 42 | override def transform(tree: LogicalPlan): LogicalPlan = { 43 | val internal = tree.output.collect { case e: NamedExpression if e.namespace.nonEmpty => e } 44 | 45 | if (internal.nonEmpty) { 46 | val internalList = internal map { _.sqlLike } mkString ("[", ", ", "]") 47 | val suggestion = 48 | """You probably hit an internal bug since internal attributes are only used internally by 49 | |the analyzer and should never appear in a fully analyzed logical plan. 50 | |""".oneLine 51 | 52 | throw new ResolutionFailureException( 53 | s"""Internal attributes $internalList found in the analyzed logical plan: 54 | | 55 | |${tree.prettyTree} 56 | | 57 | |$suggestion 58 | |""".stripMargin 59 | ) 60 | } 61 | 62 | tree 63 | } 64 | } 65 | 66 | class RejectDistinctAggregateFunctions(val catalog: Catalog) extends AnalysisRule { 67 | override def transform(tree: LogicalPlan): LogicalPlan = { 68 | val distinctAggs = tree.collectDown { 69 | case node => 70 | node.expressions flatMap { 71 | _ collectDown { 72 | case agg: DistinctAggregateFunction => agg 73 | } 74 | } 75 | }.flatten 76 | 77 | if (distinctAggs.nonEmpty) { 78 | val distinctAggList = distinctAggs map { _.sqlLike } mkString ("[", ", ", "]") 79 | val suggestion = 80 | """You probably hit an internal bug since all distinct aggregate functions should have 81 | |been resolved into normal aggregate functions by the analyzer. 82 | |""".oneLine 83 | 84 | throw new ResolutionFailureException( 85 | s"""Distinct aggregate functions $distinctAggList found in the analyzed logical plan: 86 | | 87 | |${tree.prettyTree} 88 | | 89 | |$suggestion 90 | |""".stripMargin 91 | ) 92 | } 93 | 94 | tree 95 | } 96 | } 97 | 98 | class RejectOrphanAttributeRefs(val catalog: Catalog) extends AnalysisRule { 99 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformUp { 100 | case plan: LeafLogicalPlan => 101 | plan 102 | 103 | case plan => 104 | val inputSet = plan.children.flatMap { _.outputSet } ++ plan.derivedInput 105 | val orphans = plan.references filterNot inputSet.contains 106 | 107 | if (orphans.nonEmpty) { 108 | val message = 109 | s"""Orphan attribute references ${orphans map { _.sqlLike } mkString ("[", ", ", "]")} 110 | |found in the following logical plan operator. They are neither output of child 111 | |operators nor derived by the problematic operator itself. 112 | |""".oneLine 113 | 114 | throw new AnalysisException( 115 | s"""$message 116 | | 117 | |${plan.prettyTree} 118 | |""".stripMargin 119 | ) 120 | } 121 | 122 | plan 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/logical/analysis/windowAnalysis.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import spear._ 4 | import spear.exceptions.WindowAnalysisException 5 | import spear.expressions._ 6 | import spear.expressions.InternalAlias.buildRewriter 7 | import spear.expressions.windows.{WindowFunction, WindowSpecRef} 8 | import spear.plans.logical._ 9 | import spear.plans.logical.analysis.AggregationAnalysis.hasAggregateFunction 10 | import spear.plans.logical.analysis.WindowAnalysis._ 11 | import spear.plans.logical.patterns.Resolved 12 | import spear.utils._ 13 | 14 | /** 15 | * This rule extracts window functions from `ORDER BY` clauses and moves them into separate `Window` 16 | * operators. 17 | */ 18 | class ExtractWindowFunctionsFromSorts(val catalog: Catalog) extends AnalysisRule { 19 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformDown { 20 | // Waits until all aggregations are resolved. 21 | case plan: UnresolvedAggregate => 22 | plan 23 | 24 | // Waits until all global aggregations are resolved. 25 | case plan @ Resolved(_ Project projectList) if hasAggregateFunction(projectList) => 26 | plan 27 | 28 | case Resolved(child Sort order) if hasWindowFunction(order) => 29 | val winAliases = collectWindowFunctions(order) map { WindowFunctionAlias(_) } 30 | val rewrittenOrder = order map { _ transformDown buildRewriter(winAliases) } 31 | child windows winAliases orderBy rewrittenOrder select child.output 32 | } 33 | } 34 | 35 | /** 36 | * This rule extracts window functions from `SELECT` clauses and moves them into separate `Window` 37 | * operators. 38 | */ 39 | class ExtractWindowFunctionsFromProjections(val catalog: Catalog) extends AnalysisRule { 40 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformDown { 41 | // Waits until all aggregations are resolved. 42 | case plan: UnresolvedAggregate => 43 | plan 44 | 45 | // Waits until all global aggregations are resolved. 46 | case plan @ Resolved(_ Project projectList) if hasAggregateFunction(projectList) => 47 | plan 48 | 49 | case Resolved(child Project projectList) if hasWindowFunction(projectList) => 50 | val winAliases = collectWindowFunctions(projectList) map { WindowFunctionAlias(_) } 51 | val rewrittenProjectList = projectList map { _ transformDown buildRewriter(winAliases) } 52 | child windows winAliases select rewrittenProjectList 53 | } 54 | } 55 | 56 | class InlineWindowDefinitions(val catalog: Catalog) extends AnalysisRule { 57 | override def transform(tree: LogicalPlan): LogicalPlan = tree transformUp { 58 | case WindowDef(child, name, windowSpec) => 59 | child transformDown { 60 | case node => 61 | node transformExpressionsDown { 62 | case WindowSpecRef(`name`, maybeFrame) => 63 | for { 64 | existingFrame <- windowSpec.windowFrame 65 | newFrame <- maybeFrame 66 | } throw new WindowAnalysisException( 67 | s"""Cannot decorate window $name with frame $newFrame 68 | |because it already has a frame $existingFrame 69 | |""".oneLine 70 | ) 71 | 72 | windowSpec between (windowSpec.windowFrame orElse maybeFrame) 73 | } 74 | } 75 | } 76 | } 77 | 78 | class RejectUndefinedWindowSpecRefs(val catalog: Catalog) extends AnalysisRule { 79 | override def transform(tree: LogicalPlan): LogicalPlan = { 80 | tree collectDown { 81 | case node => 82 | node.expressions flatMap { 83 | _ collectDown { 84 | case WindowSpecRef(name, _) => 85 | throw new WindowAnalysisException( 86 | s"Window specification references $name is undefined" 87 | ) 88 | } 89 | } 90 | } 91 | 92 | tree 93 | } 94 | } 95 | 96 | object WindowAnalysis { 97 | def hasWindowFunction(expressions: Seq[Expression]): Boolean = 98 | expressions exists hasWindowFunction 99 | 100 | /** 101 | * Collects all distinct window functions from `expressions`. 102 | */ 103 | def collectWindowFunctions(expressions: Seq[Expression]): Seq[WindowFunction] = 104 | (expressions flatMap collectWindowFunctions).distinct 105 | 106 | /** 107 | * Collects all distinct window functions from `expression`. 108 | */ 109 | def collectWindowFunctions(expression: Expression): Seq[WindowFunction] = 110 | expression.collectDown { case e: WindowFunction => e }.distinct 111 | 112 | private def hasWindowFunction(expression: Expression): Boolean = 113 | expression.collectFirstDown { case _: WindowFunction => }.nonEmpty 114 | } 115 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/logical/package.scala: -------------------------------------------------------------------------------- 1 | package spear.plans 2 | 3 | import spear.Name 4 | import spear.expressions.Expression 5 | import spear.expressions.windows.{WindowSpec, WindowSpecRef} 6 | 7 | package object logical { 8 | def table(name: Name): UnresolvedRelation = UnresolvedRelation(name) 9 | 10 | def values(expressions: Seq[Expression]): Project = SingleRowRelation select expressions 11 | 12 | def values(first: Expression, rest: Expression*): Project = values(first +: rest) 13 | 14 | def let(name: Name, plan: LogicalPlan)(body: LogicalPlan): With = { 15 | With(body, name, plan, None) 16 | } 17 | 18 | def let(name: Name, windowSpec: WindowSpec)(body: LogicalPlan): WindowDef = 19 | WindowDef(body, name, windowSpec) 20 | 21 | def let(name: Name, windowName: Name)(body: LogicalPlan): WindowDef = 22 | WindowDef(body, name, WindowSpecRef(windowName)) 23 | } 24 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/logical/patterns/package.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical 2 | 3 | import spear.expressions.{AggregateFunctionAlias, Expression, GroupingKeyAlias, WindowFunctionAlias} 4 | import spear.expressions.InternalAlias.buildRestorer 5 | 6 | package object patterns { 7 | /** A simple pattern that matches unresolved logical plans and expressions */ 8 | object Unresolved { 9 | def unapply(plan: LogicalPlan): Option[LogicalPlan] = 10 | Some(plan) filter { !_.isResolved } 11 | 12 | def unapply(expression: Expression): Option[Expression] = 13 | Some(expression) filter { !_.isResolved } 14 | } 15 | 16 | /** A simple pattern that matches resolved logical plans and expressions */ 17 | object Resolved { 18 | def unapply(plan: LogicalPlan): Option[LogicalPlan] = 19 | Some(plan) filter { _.isResolved } 20 | 21 | def unapply(expression: Expression): Option[Expression] = 22 | Some(expression) filter { _.isResolved } 23 | } 24 | 25 | object ResolvedAggregate { 26 | // format: OFF 27 | type ResultType = ( 28 | Seq[Expression], // Project list 29 | Seq[Expression], // Sort order 30 | Option[Expression], // HAVING condition 31 | Seq[Expression], // Window functions 32 | LogicalPlan // Child plan 33 | ) 34 | // format: ON 35 | 36 | def unapply(tree: LogicalPlan): Option[ResultType] = tree match { 37 | case (a: Aggregate) WindowSeq windowFunctions Project output => 38 | val restore = restoreInternalAttributes(a.keys, a.functions, windowFunctions) 39 | Some((output map restore, Nil, None, a.keys map { _.child }, a.child)) 40 | 41 | case (a: Aggregate) Filter condition WindowSeq windowFunctions Project output => 42 | val restore = restoreInternalAttributes(a.keys, a.functions, windowFunctions) 43 | Some(( 44 | output map restore, 45 | Nil, 46 | Some(restore(condition)), 47 | a.keys map { _.child }, 48 | a.child 49 | )) 50 | 51 | case (a: Aggregate) WindowSeq windowFunctions Sort order Project output => 52 | val restore = restoreInternalAttributes(a.keys, a.functions, windowFunctions) 53 | Some(( 54 | output map restore, 55 | order map restore, 56 | None, 57 | a.keys map { _.child }, 58 | a.child 59 | )) 60 | 61 | case (a: Aggregate) Filter condition WindowSeq windowFunctions Sort order Project output => 62 | val restore = restoreInternalAttributes(a.keys, a.functions, windowFunctions) 63 | Some(( 64 | output map restore, 65 | order map restore, 66 | Some(restore(condition)), 67 | a.keys map { _.child }, 68 | a.child 69 | )) 70 | } 71 | 72 | def restoreInternalAttributes( 73 | keys: Seq[GroupingKeyAlias], 74 | aggs: Seq[AggregateFunctionAlias], 75 | wins: Seq[WindowFunctionAlias] 76 | ): Expression => Expression = { 77 | val restoreKeys = (_: Expression) transformUp buildRestorer(keys) 78 | val restoreAggs = (_: Expression) transformUp buildRestorer(aggs) 79 | val restoreWins = (_: Expression) transformUp buildRestorer(wins) 80 | restoreWins andThen restoreAggs andThen restoreKeys 81 | } 82 | } 83 | 84 | object WindowSeq { 85 | def unapply(tree: LogicalPlan): Option[(LogicalPlan, Seq[WindowFunctionAlias])] = tree match { 86 | case plan @ Window(_, _, _, WindowSeq(child, functions)) => 87 | Some((child, plan.functions ++ functions)) 88 | 89 | case plan => 90 | Some((plan, Nil)) 91 | } 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/plans/physical/PhysicalPlan.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.physical 2 | 3 | import spear._ 4 | import spear.expressions._ 5 | import spear.plans.QueryPlan 6 | import spear.plans.logical.annotations.Explain 7 | 8 | trait PhysicalPlan extends QueryPlan[PhysicalPlan] { 9 | def iterator: Iterator[Row] 10 | 11 | def requireMaterialization: Boolean = children exists { _.requireMaterialization } 12 | } 13 | 14 | trait LeafPhysicalPlan extends PhysicalPlan { 15 | override def children: Seq[PhysicalPlan] = Nil 16 | } 17 | 18 | trait UnaryPhysicalPlan extends PhysicalPlan { 19 | def child: PhysicalPlan 20 | 21 | override def children: Seq[PhysicalPlan] = Seq(child) 22 | } 23 | 24 | trait BinaryPhysicalPlan extends PhysicalPlan { 25 | def left: PhysicalPlan 26 | 27 | def right: PhysicalPlan 28 | 29 | override def children: Seq[PhysicalPlan] = Seq(left, right) 30 | } 31 | 32 | case object SingleRowRelation extends LeafPhysicalPlan { 33 | override def iterator: Iterator[Row] = Iterator single Row.empty 34 | 35 | override val output: Seq[Attribute] = Nil 36 | } 37 | 38 | case class NotImplemented( 39 | logicalPlanName: String, 40 | @Explain(hidden = true) input: Seq[PhysicalPlan], 41 | output: Seq[Attribute] 42 | ) extends PhysicalPlan { 43 | override def children: Seq[PhysicalPlan] = input 44 | 45 | override def iterator: Iterator[Row] = throw new UnsupportedOperationException( 46 | s"$logicalPlanName is not implemented yet" 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/reflection/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.reflect.runtime.universe._ 4 | import scala.reflect.runtime.universe.definitions._ 5 | 6 | import spear.types._ 7 | 8 | package object reflection { 9 | def fieldSpecFor[T: WeakTypeTag]: FieldSpec = fieldSpecFor(weakTypeOf[T]) 10 | 11 | private val fieldSpecFor: PartialFunction[Type, FieldSpec] = ( 12 | fieldSpecForUnboxedPrimitiveType 13 | orElse fieldSpecForBoxedPrimitiveType 14 | orElse fieldSpecForCompoundType 15 | ) 16 | 17 | private def fieldSpecForCompoundType: PartialFunction[Type, FieldSpec] = { 18 | case t if t <:< weakTypeOf[Option[_]] => 19 | val Seq(elementType) = t.typeArgs 20 | fieldSpecFor(elementType).dataType.? 21 | 22 | case t if t <:< weakTypeOf[Seq[_]] || t <:< weakTypeOf[Array[_]] => 23 | val Seq(elementType) = t.typeArgs 24 | val FieldSpec(elementDataType, elementOptional) = fieldSpecFor(elementType) 25 | ArrayType(elementDataType, elementOptional).? 26 | 27 | case t if t <:< weakTypeOf[Map[_, _]] => 28 | val Seq(keyType, valueType) = t.typeArgs 29 | val FieldSpec(keyDataType, _) = fieldSpecFor(keyType) 30 | val FieldSpec(valueDataType, valueOptional) = fieldSpecFor(valueType) 31 | MapType(keyDataType, valueDataType, valueOptional).? 32 | 33 | case t if t <:< weakTypeOf[Product] => 34 | val formalTypeArgs = t.typeSymbol.asClass.typeParams 35 | val TypeRef(_, _, actualTypeArgs) = t 36 | val params = constructorParams(t) 37 | StructType(params.map { param => 38 | val paramType = param.typeSignature.substituteTypes(formalTypeArgs, actualTypeArgs) 39 | val FieldSpec(dataType, nullable) = fieldSpecFor(paramType) 40 | StructField(Name.caseSensitive(param.name.toString), dataType, nullable) 41 | }).? 42 | } 43 | 44 | private def fieldSpecForBoxedPrimitiveType: PartialFunction[Type, FieldSpec] = { 45 | case t if t <:< weakTypeOf[String] => StringType.? 46 | case t if t <:< weakTypeOf[Integer] => IntType.? 47 | case t if t <:< weakTypeOf[java.lang.Boolean] => BooleanType.? 48 | case t if t <:< weakTypeOf[java.lang.Byte] => ByteType.? 49 | case t if t <:< weakTypeOf[java.lang.Short] => ShortType.? 50 | case t if t <:< weakTypeOf[java.lang.Long] => LongType.? 51 | case t if t <:< weakTypeOf[java.lang.Float] => FloatType.? 52 | case t if t <:< weakTypeOf[java.lang.Double] => DoubleType.? 53 | } 54 | 55 | private def fieldSpecForUnboxedPrimitiveType: PartialFunction[Type, FieldSpec] = { 56 | case t if t <:< IntTpe => IntType.! 57 | case t if t <:< BooleanTpe => BooleanType.! 58 | case t if t <:< ByteTpe => ByteType.! 59 | case t if t <:< ShortTpe => ShortType.! 60 | case t if t <:< LongTpe => LongType.! 61 | case t if t <:< FloatTpe => FloatType.! 62 | case t if t <:< DoubleTpe => DoubleType.! 63 | case t if t <:< NullTpe => NullType.? 64 | } 65 | 66 | def constructorParams(clazz: Class[_]): List[Symbol] = { 67 | val selfType = runtimeMirror(clazz.getClassLoader).staticClass(clazz.getName).selfType 68 | constructorParams(selfType) 69 | } 70 | 71 | def constructorParams(tpe: Type): List[Symbol] = { 72 | val constructorSymbol = tpe.member(termNames.CONSTRUCTOR) 73 | 74 | val constructor = if (constructorSymbol.isMethod && constructorSymbol.asMethod.isPublic) { 75 | // The type has only one public constructor 76 | constructorSymbol.asMethod 77 | } else { 78 | // The type has multiple constructors. Let's pick the primary one. 79 | constructorSymbol.asTerm.alternatives.find { symbol => 80 | symbol.isMethod && symbol.asMethod.isPrimaryConstructor && symbol.asMethod.isPublic 81 | }.map { _.asMethod } getOrElse { 82 | throw ScalaReflectionException(s"Type $tpe doesn't have a public primary constructor") 83 | } 84 | } 85 | 86 | constructor.paramLists.flatten 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /spear-core/src/main/scala/spear/types/numericTypes.scala: -------------------------------------------------------------------------------- 1 | package spear.types 2 | 3 | trait NumericType extends PrimitiveType { 4 | val numeric: Numeric[InternalType] 5 | 6 | def genericNumeric: Numeric[Any] = numeric.asInstanceOf[Numeric[Any]] 7 | } 8 | 9 | object NumericType extends AbstractDataType { 10 | override def isSupertypeOf(dataType: DataType): Boolean = dataType match { 11 | case _: NumericType => true 12 | case _ => false 13 | } 14 | 15 | override def toString: String = "numeric type" 16 | } 17 | 18 | trait IntegralType extends NumericType { 19 | val integral: Integral[InternalType] 20 | 21 | def genericIntegral: Integral[Any] = integral.asInstanceOf[Integral[Any]] 22 | } 23 | 24 | object IntegralType extends AbstractDataType { 25 | override def isSupertypeOf(dataType: DataType): Boolean = dataType match { 26 | case _: IntegralType => true 27 | case _ => false 28 | } 29 | 30 | override def toString: String = "integral type" 31 | } 32 | 33 | case object ByteType extends IntegralType { 34 | override type InternalType = Byte 35 | 36 | override val integral: Integral[Byte] = implicitly[Integral[Byte]] 37 | 38 | override val numeric: Numeric[Byte] = implicitly[Numeric[Byte]] 39 | 40 | override val ordering: Option[Ordering[Byte]] = Some(Ordering.Byte) 41 | 42 | override def sql: String = "TINYINT" 43 | } 44 | 45 | case object ShortType extends IntegralType { 46 | override type InternalType = Short 47 | 48 | override val integral: Integral[Short] = implicitly[Integral[Short]] 49 | 50 | override val numeric: Numeric[Short] = implicitly[Numeric[Short]] 51 | 52 | override val ordering: Option[Ordering[Short]] = Some(Ordering.Short) 53 | 54 | override def sql: String = "SMALLINT" 55 | } 56 | 57 | case object IntType extends IntegralType { 58 | override type InternalType = Int 59 | 60 | override val integral: Integral[Int] = implicitly[Integral[Int]] 61 | 62 | override val numeric: Numeric[Int] = implicitly[Numeric[Int]] 63 | 64 | override val ordering: Option[Ordering[Int]] = Some(Ordering.Int) 65 | 66 | override def sql: String = "INT" 67 | } 68 | 69 | case object LongType extends IntegralType { 70 | override type InternalType = Long 71 | 72 | override val integral: Integral[Long] = implicitly[Integral[Long]] 73 | 74 | override val numeric: Numeric[Long] = implicitly[Numeric[Long]] 75 | 76 | override val ordering: Option[Ordering[Long]] = Some(Ordering.Long) 77 | 78 | override def sql: String = "BIGINT" 79 | } 80 | 81 | trait FractionalType extends NumericType { 82 | val fractional: Fractional[InternalType] 83 | 84 | def genericFractional: Fractional[Any] = fractional.asInstanceOf[Fractional[Any]] 85 | } 86 | 87 | object FractionalType extends AbstractDataType { 88 | override def isSupertypeOf(dataType: DataType): Boolean = dataType match { 89 | case _: FractionalType => true 90 | case _ => false 91 | } 92 | 93 | override def toString: String = "fractional type" 94 | } 95 | 96 | case object FloatType extends FractionalType { 97 | override type InternalType = Float 98 | 99 | override val fractional: Fractional[Float] = implicitly[Fractional[Float]] 100 | 101 | override val numeric: Numeric[Float] = implicitly[Numeric[Float]] 102 | 103 | override val ordering: Option[Ordering[Float]] = Some(Ordering.Float) 104 | 105 | override def sql: String = "FLOAT" 106 | } 107 | 108 | case object DoubleType extends FractionalType { 109 | override type InternalType = Double 110 | 111 | override val fractional: Fractional[Double] = implicitly[Fractional[Double]] 112 | 113 | override val numeric: Numeric[Double] = implicitly[Numeric[Double]] 114 | 115 | override val ordering: Option[Ordering[Double]] = Some(Ordering.Double) 116 | 117 | override def sql: String = "DOUBLE" 118 | } 119 | -------------------------------------------------------------------------------- /spear-core/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger = TRACE, file 2 | 3 | # File appender 4 | log4j.appender.file=org.apache.log4j.FileAppender 5 | log4j.appender.file.threshold=TRACE 6 | log4j.appender.file.append=false 7 | log4j.appender.file.file=spear-core/target/unit-tests.log 8 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 10 | -------------------------------------------------------------------------------- /spear-core/src/test/resources/spear-test.conf: -------------------------------------------------------------------------------- 1 | spear { 2 | test { 3 | types { 4 | allow-null-type = true 5 | 6 | allow-empty-struct-type = true 7 | 8 | allow-nullable-complex-type = true 9 | 10 | allow-nullable-array-type = ${spear.test.types.allow-nullable-complex-type} 11 | 12 | allow-nullable-map-type = ${spear.test.types.allow-nullable-complex-type} 13 | 14 | allow-nullable-struct-field = ${spear.test.types.allow-nullable-complex-type} 15 | 16 | allow-nested-struct-type = true 17 | 18 | max-struct-type-width = 5 19 | } 20 | 21 | expressions { 22 | max-repetition = 8 23 | 24 | chances { 25 | null = 0 26 | } 27 | 28 | only-logical-operators-in-predicate = false 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/DataFrameSuite.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import java.io.{ByteArrayOutputStream, PrintStream} 4 | 5 | import org.scalatest.BeforeAndAfterAll 6 | 7 | import spear.expressions._ 8 | import spear.expressions.functions._ 9 | import spear.plans.logical.{LocalRelation, LogicalPlan} 10 | import spear.types.{IntType, StringType, StructType} 11 | 12 | class DataFrameSuite extends LoggingFunSuite with TestUtils with BeforeAndAfterAll { 13 | private implicit val context = new Context(new TestQueryCompiler) 14 | 15 | import context._ 16 | 17 | private val r1 = LocalRelation.empty('a.int.!, 'b.string.?) 18 | 19 | private val r2 = LocalRelation.empty('a.int.!, 'b.string.?) 20 | 21 | override protected def beforeAll(): Unit = { 22 | context.queryCompiler.catalog.registerRelation('t, r1) 23 | context.queryCompiler.catalog.registerRelation('s, r2) 24 | } 25 | 26 | test("context") { 27 | assert(range(10).context eq context) 28 | } 29 | 30 | test("inferred local data") { 31 | checkTree( 32 | (lift(1 -> "a", 2 -> "b") rename ("x", "y")).schema, 33 | StructType( 34 | 'x -> IntType.!, 35 | 'y -> StringType.? 36 | ) 37 | ) 38 | } 39 | 40 | test("select named expressions") { 41 | checkLogicalPlan( 42 | table('t) select ('a, 'b as 'c), 43 | r1 subquery 't select ('a, 'b as 'c) 44 | ) 45 | } 46 | 47 | test("select arbitrary expression") { 48 | checkLogicalPlan( 49 | table('t) select ('a + 1, 'b cast IntType), 50 | r1 subquery 't select ('a + 1, 'b cast IntType) 51 | ) 52 | } 53 | 54 | test("global aggregation using select") { 55 | checkLogicalPlan( 56 | table('t) select count('a), 57 | r1 subquery 't select count('a) 58 | ) 59 | } 60 | 61 | test("filter") { 62 | val Seq(a: AttributeRef, _) = r1.output 63 | 64 | checkLogicalPlan( 65 | table('t) filter 'a > 3 filter 'b.isNotNull, 66 | r1 subquery 't filter (a of 't) > 3 filter 'b.isNotNull 67 | ) 68 | } 69 | 70 | test("limit") { 71 | checkLogicalPlan( 72 | table('t) limit 3, 73 | r1 subquery 't limit 3 74 | ) 75 | } 76 | 77 | test("distinct") { 78 | checkLogicalPlan( 79 | table('t).distinct, 80 | (r1 subquery 't).distinct 81 | ) 82 | } 83 | 84 | test("order by sort order") { 85 | checkLogicalPlan( 86 | table('t) orderBy 'a.asc.nullsFirst, 87 | r1 subquery 't orderBy 'a.asc.nullsFirst 88 | ) 89 | } 90 | 91 | test("order by arbitrary expression") { 92 | checkLogicalPlan( 93 | table('t) orderBy 'a + 1, 94 | r1 subquery 't orderBy ('a + 1).asc 95 | ) 96 | } 97 | 98 | test("subquery") { 99 | checkLogicalPlan( 100 | table('t) subquery 'x subquery 'y, 101 | r1 subquery 't subquery 'x subquery 'y 102 | ) 103 | } 104 | 105 | test("join") { 106 | checkLogicalPlan( 107 | table('t) subquery 'x join (table('t) subquery 'y) on $"x.a" === $"y.a", 108 | r1 subquery 't subquery 'x join (r1 subquery 't subquery 'y) on ('a of 'x) === ('a of 'y) 109 | ) 110 | } 111 | 112 | test("union") { 113 | checkLogicalPlan( 114 | table('t) union table('t), 115 | r1 subquery 't union (r1 subquery 't) 116 | ) 117 | } 118 | 119 | test("intersect") { 120 | checkLogicalPlan( 121 | table('t) intersect table('s), 122 | r1 subquery 't intersect (r2 subquery 's) 123 | ) 124 | } 125 | 126 | test("except") { 127 | checkLogicalPlan( 128 | table('t) except table('s), 129 | r1 subquery 't except (r2 subquery 's) 130 | ) 131 | } 132 | 133 | test("global aggregation") { 134 | checkLogicalPlan( 135 | table('t) agg count('a), 136 | r1 subquery 't groupBy Nil agg count('a) 137 | ) 138 | } 139 | 140 | test("group by") { 141 | checkLogicalPlan( 142 | table('t) groupBy 'a agg count('b), 143 | r1 subquery 't groupBy 'a agg count('b) 144 | ) 145 | } 146 | 147 | test("asTable") { 148 | withTable('reverse) { 149 | val df = table('t) orderBy 'a.desc 150 | df asTable 'reverse 151 | 152 | checkLogicalPlan( 153 | table('reverse), 154 | df.query.analyzedPlan subquery 'reverse 155 | ) 156 | } 157 | } 158 | 159 | test("explain") { 160 | val out = new PrintStream(new ByteArrayOutputStream()) 161 | // These only check that no exceptions are thrown during analysis, optimization, and planning. 162 | table('t) explain (extended = false, out = out) 163 | table('t) explain (extended = true, out = out) 164 | } 165 | 166 | test("show") { 167 | val out = new PrintStream(new ByteArrayOutputStream()) 168 | // These only check that no exceptions are thrown during analysis, optimization, planning and 169 | // query execution. 170 | table('t) show (out = out) 171 | } 172 | 173 | test("showSchema") { 174 | val out = new ByteArrayOutputStream() 175 | val printStream = new PrintStream(out) 176 | val df = table('t) 177 | 178 | df.showSchema(out = printStream) 179 | assertSideBySide( 180 | out.toString, 181 | table('t).schema.prettyTree + "\n" 182 | ) 183 | } 184 | 185 | private def checkLogicalPlan(df: DataFrame, expected: LogicalPlan): Unit = { 186 | checkPlan(df.query.logicalPlan, expected) 187 | // Triggers analysis phase 188 | df.query.analyzedPlan 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/ExpressionSQLBuilderSuite.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import spear.expressions._ 4 | import spear.expressions.Literal.{False, True} 5 | import spear.types._ 6 | 7 | class ExpressionSQLBuilderSuite extends SQLBuilderTest { 8 | test("literals") { 9 | checkSQL(Literal(null), "NULL") 10 | checkSQL(True, "TRUE") 11 | checkSQL(False, "FALSE") 12 | checkSQL(Literal("foo"), "'foo'") 13 | checkSQL(Literal("'foo"), "'\\'foo'") 14 | checkSQL(Literal(0), "0") 15 | checkSQL(Literal(1: Byte), "CAST(1 AS TINYINT)") 16 | checkSQL(Literal(2: Short), "CAST(2 AS SMALLINT)") 17 | checkSQL(Literal(4L), "CAST(4 AS BIGINT)") 18 | checkSQL(Literal(1.1F), "CAST(1.1 AS FLOAT)") 19 | checkSQL(Literal(1.2D), "CAST(1.2 AS DOUBLE)") 20 | } 21 | 22 | test("arithmetic expressions") { 23 | val a = 'a.int 24 | val b = 'b.int 25 | 26 | checkSQL(a + b, "(a + b)") 27 | checkSQL(a - b, "(a - b)") 28 | checkSQL(a * b, "(a * b)") 29 | checkSQL(a / b, "(a / b)") 30 | checkSQL(-a, "(-a)") 31 | } 32 | 33 | test("logical operators") { 34 | val a = 'a.boolean 35 | val b = 'b.boolean 36 | val c = 'c.boolean 37 | 38 | checkSQL(a && b, "(a AND b)") 39 | checkSQL(a || b, "(a OR b)") 40 | checkSQL(!a, "(NOT a)") 41 | checkSQL(If(a, b, c), "if(a, b, c)") 42 | } 43 | 44 | test("non-SQL expressions") { 45 | intercept[UnsupportedOperationException] { 46 | ('_.int.! at 0).sql.get 47 | } 48 | } 49 | 50 | test("casting") { 51 | val a = 'a.int 52 | 53 | checkSQL(a cast BooleanType, "CAST(a AS BOOLEAN)") 54 | checkSQL(a cast ByteType, "CAST(a AS TINYINT)") 55 | checkSQL(a cast ShortType, "CAST(a AS SMALLINT)") 56 | checkSQL(a cast IntType, "CAST(a AS INT)") 57 | checkSQL(a cast LongType, "CAST(a AS BIGINT)") 58 | checkSQL(a cast FloatType, "CAST(a AS FLOAT)") 59 | checkSQL(a cast DoubleType, "CAST(a AS DOUBLE)") 60 | checkSQL(a cast ArrayType(IntType.?), "CAST(a AS ARRAY)") 61 | checkSQL(a cast MapType(IntType, StringType.?), "CAST(a AS MAP)") 62 | checkSQL( 63 | a cast StructType('name -> StringType.?, 'age -> IntType.?), 64 | "CAST(a AS STRUCT)" 65 | ) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/NameSuite.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import org.scalacheck.Arbitrary.arbitrary 4 | import org.scalacheck.Gen 5 | import org.scalacheck.Gen.alphaStr 6 | import org.scalacheck.Prop.{all, forAll, BooleanOperators} 7 | import org.scalatest.prop.Checkers 8 | 9 | import spear.Name.{caseInsensitive, caseSensitive} 10 | 11 | class NameSuite extends LoggingFunSuite with Checkers { 12 | test("basic properties") { 13 | check { 14 | forAll(alphaStr) { name => 15 | all( 16 | caseSensitive(name).isCaseSensitive, 17 | 18 | !caseInsensitive(name).isCaseSensitive, 19 | 20 | caseSensitive(name).casePreserving == name, 21 | 22 | caseInsensitive(name).casePreserving == name 23 | ) 24 | } 25 | } 26 | } 27 | 28 | test("equality") { 29 | check { 30 | forAll(alphaStr) { name => 31 | all( 32 | (name != name.toLowerCase) ==> 33 | all( 34 | "inequality: case-insensitive v.s. case-sensitive" |: 35 | caseInsensitive(name.toLowerCase) != caseSensitive(name), 36 | 37 | "inequality: case-sensitive v.s. case-insensitive" |: 38 | caseSensitive(name.toLowerCase) != caseInsensitive(name), 39 | 40 | "inequality: case-sensitive v.s. case-sensitive" |: 41 | caseSensitive(name.toLowerCase) != caseSensitive(name), 42 | 43 | "inequality: case-sensitive v.s. other classes" |: 44 | caseSensitive(name) != (name: Any), 45 | 46 | "inequality: case-insensitive v.s. other classes" |: 47 | caseSensitive(name) != (name: Any) 48 | ), 49 | 50 | "equality: case-sensitive v.s. case-sensitive" |: 51 | caseSensitive(name) == caseSensitive(name), 52 | 53 | "equality: case-sensitive v.s. case-insensitive" |: 54 | caseSensitive(name) == caseInsensitive(name), 55 | 56 | "equality: case-insensitive v.s. case-sensitive" |: 57 | caseInsensitive(name) == caseSensitive(name), 58 | 59 | "equality: case-insensitive v.s. case-insensitive (in the same case)" |: 60 | caseInsensitive(name) == caseInsensitive(name), 61 | 62 | "equality: case-insensitive v.s. case-insensitive (in different cases)" |: 63 | caseInsensitive(name) == caseInsensitive(name.toLowerCase) 64 | ) 65 | } 66 | } 67 | } 68 | 69 | test("hashCode and equals contract") { 70 | val genName: Gen[Name] = for { 71 | name <- alphaStr 72 | isCaseSensitive <- arbitrary[Boolean] 73 | } yield Name(name, isCaseSensitive) 74 | 75 | check { 76 | forAll(genName, genName) { 77 | case (lhs: Name, rhs: Name) if lhs.## != rhs.## => lhs != rhs 78 | // TODO Probability of reaching this branch is too small, can't be well tested 79 | case (lhs: Name, rhs: Name) if lhs == rhs => lhs.## == rhs.## 80 | case _ => true 81 | } 82 | } 83 | } 84 | 85 | test("interpolation") { 86 | val caseInsensitive = i"Hello" 87 | assert(!caseInsensitive.isCaseSensitive) 88 | assert(caseInsensitive.casePreserving == "Hello") 89 | } 90 | 91 | test("implicit conversion") { 92 | val caseSensitive: Name = "Hello" 93 | val caseInsensitive: Name = 'Hello 94 | 95 | assert(caseSensitive.isCaseSensitive) 96 | assert(caseSensitive.casePreserving == "Hello") 97 | 98 | assert(!caseInsensitive.isCaseSensitive) 99 | assert(caseInsensitive.casePreserving == "Hello") 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/SQLBuilderTest.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.util.control.NonFatal 4 | 5 | import org.scalatest.BeforeAndAfterAll 6 | 7 | import spear.expressions.Expression 8 | 9 | abstract class SQLBuilderTest 10 | extends LoggingFunSuite with TestUtils with BeforeAndAfterAll { 11 | 12 | protected def checkSQL(e: Expression, expectedSQL: String): Unit = { 13 | try { 14 | assert(e.sql.get == expectedSQL) 15 | } catch { 16 | case NonFatal(cause) => 17 | fail( 18 | s"""Wrong SQL generated for the following expression: 19 | |${e.prettyTree} 20 | | 21 | |Expected: $expectedSQL 22 | |Actual: ${e.sql.get} 23 | |""".stripMargin, 24 | cause 25 | ) 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/Test.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import spear.config.Settings 4 | 5 | object Test { 6 | implicit val defaultSettings: Settings = Settings.load("spear-test.conf") 7 | } 8 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/TestQueryCompiler.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import org.mockito.Mockito._ 4 | 5 | import spear.plans.logical.LogicalPlan 6 | import spear.plans.physical.PhysicalPlan 7 | 8 | class TestQueryCompiler extends BasicQueryCompiler { 9 | override def plan(plan: LogicalPlan): PhysicalPlan = when { 10 | mock(classOf[PhysicalPlan]).iterator 11 | }.thenReturn { 12 | Iterator.empty 13 | }.getMock[PhysicalPlan] 14 | } 15 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/TestUtils.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.util.control.NonFatal 4 | 5 | import org.scalatest.FunSuite 6 | 7 | import spear.expressions._ 8 | import spear.plans.QueryPlan 9 | import spear.plans.logical.LogicalPlan 10 | import spear.trees.TreeNode 11 | import spear.types.DataType 12 | import spear.utils._ 13 | 14 | trait TestUtils { this: FunSuite => 15 | def assertSideBySide(actual: String, expected: String): Unit = { 16 | if (expected != actual) { 17 | fail(sideBySide( 18 | s"""Actual 19 | |$expected 20 | |""".stripMargin, 21 | 22 | s"""Expected 23 | |$actual 24 | |""".stripMargin, 25 | 26 | withHeader = true 27 | )) 28 | } 29 | } 30 | 31 | def assertSideBySide[T <: TreeNode[T]](actual: TreeNode[T], expected: TreeNode[T]): Unit = 32 | if (actual != expected) { 33 | fail(sideBySide( 34 | s"""Actual 35 | |${actual.prettyTree} 36 | |""".stripMargin, 37 | 38 | s"""Expected 39 | |${expected.prettyTree} 40 | |""".stripMargin, 41 | 42 | withHeader = true 43 | )) 44 | } 45 | 46 | def checkTree[T <: TreeNode[T]](actual: TreeNode[T], expected: TreeNode[T]): Unit = 47 | assertSideBySide(actual, expected) 48 | 49 | def checkPlan[Plan <: QueryPlan[Plan]](actual: Plan, expected: Plan): Unit = 50 | checkTree(QueryPlan.normalizeExpressionIDs(actual), QueryPlan.normalizeExpressionIDs(expected)) 51 | 52 | def checkDataFrame(actual: DataFrame, expected: DataFrame): Unit = 53 | checkDataFrame(actual, expected.toSeq) 54 | 55 | def checkDataFrame(df: DataFrame, first: Row, rest: Row*): Unit = 56 | checkDataFrame(df, first +: rest) 57 | 58 | def checkDataFrame(df: DataFrame, expected: Seq[Row]): Unit = { 59 | val actual = try { 60 | df.query.physicalPlan.iterator.toSeq 61 | } catch { 62 | case NonFatal(cause) => 63 | fail( 64 | s"""Query execution failed: 65 | | 66 | |${df.explanation(extended = true)} 67 | |""".stripMargin, 68 | cause 69 | ) 70 | } 71 | 72 | if (actual != expected) { 73 | val answerDiff = sideBySide( 74 | s"""Actual 75 | |${actual mkString "\n"} 76 | |""".stripMargin, 77 | 78 | s"""Expected 79 | |${expected mkString "\n"} 80 | |""".stripMargin, 81 | 82 | withHeader = true 83 | ) 84 | 85 | fail( 86 | s"""Unexpected row(s) detected: 87 | |$answerDiff 88 | | 89 | |Query plan details: 90 | | 91 | |${df explanation (extended = true)} 92 | |""".stripMargin 93 | ) 94 | } 95 | } 96 | 97 | def checkWellTyped(e: Expression, dataType: DataType): Unit = { 98 | if (!e.isWellTyped) { 99 | fail( 100 | s"""Expression ${e.debugString} is not well-typed: 101 | |${e.prettyTree} 102 | |""".stripMargin 103 | ) 104 | } 105 | 106 | val actualType = e.strictlyTyped.dataType 107 | if (actualType != dataType) { 108 | fail( 109 | s"""Strictly typed form of ${e.debugString} has wrong data type $actualType: 110 | |${e.strictlyTyped.prettyTree} 111 | |""".stripMargin 112 | ) 113 | } 114 | } 115 | 116 | def checkStrictlyTyped(e: Expression, dataType: DataType): Unit = { 117 | if (!e.isStrictlyTyped) { 118 | fail( 119 | s"""Expression ${e.debugString} is not strictly-typed: 120 | |${e.prettyTree} 121 | |""".stripMargin 122 | ) 123 | } 124 | 125 | val actualType = e.strictlyTyped.dataType 126 | if (actualType != dataType) { 127 | fail( 128 | s"""Strictly typed form of ${e.debugString} has wrong data type $actualType: 129 | |${e.prettyTree} 130 | |""".stripMargin 131 | ) 132 | } 133 | } 134 | 135 | def checkWellTyped(plan: LogicalPlan): Unit = 136 | if (!plan.isWellTyped) { 137 | fail( 138 | s"""Logical plan not well-typed: 139 | |${plan.prettyTree} 140 | |""".stripMargin 141 | ) 142 | } 143 | 144 | def checkStrictlyTyped(plan: LogicalPlan): Unit = { 145 | checkWellTyped(plan) 146 | 147 | if (!plan.isStrictlyTyped) { 148 | fail( 149 | s"""Logical plan is well typed but not strictly-typed: 150 | | 151 | |# Original logical plan: 152 | |${plan.prettyTree} 153 | | 154 | |# Well-typed logical plan: 155 | |${plan.strictlyTyped.prettyTree} 156 | |""".stripMargin 157 | ) 158 | } 159 | } 160 | 161 | def withTable(context: Context, name: Name)(f: => Unit): Unit = try f finally { 162 | context.queryCompiler.catalog.removeRelation(name) 163 | } 164 | 165 | def withTable(name: Name)(f: => Unit)(implicit context: Context): Unit = 166 | withTable(context, name)(f) 167 | } 168 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/config/SettingsSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.config 2 | 3 | import scala.collection.JavaConverters._ 4 | import scala.concurrent.duration._ 5 | import scala.util.{Failure, Try} 6 | 7 | import com.typesafe.config.{ConfigException, ConfigFactory} 8 | 9 | import spear.LoggingFunSuite 10 | import spear.config.Settings.Key 11 | import spear.config.SettingsSuite._ 12 | import spear.exceptions.SettingsValidationException 13 | 14 | class SettingsSuite extends LoggingFunSuite { 15 | private val config = ConfigFactory.parseString( 16 | s"""$booleanKey = true 17 | |$numberKey = 2.5 18 | |$stringKey = "foo" 19 | |$intKey = 1 20 | |$longKey = 2 21 | |$doubleKey = 3.5 22 | |$anyRefKey = { some-key = "some-value" } 23 | |$nanosKey = 1 ns 24 | |$microsKey = 1 us 25 | |$millisKey = 1 ms 26 | |$secondsKey = 1 s 27 | |$minutesKey = 1 m 28 | |$hoursKey = 1 h 29 | |$daysKey = 1 d 30 | |""".stripMargin 31 | ) 32 | 33 | private val settings = Settings(config) 34 | 35 | test("key accessors") { 36 | assert(settings(booleanKey)) 37 | assert(settings(numberKey) == (2.5F: Number)) 38 | assert(settings(stringKey) == "foo") 39 | assert(settings(intKey) == 1) 40 | assert(settings(longKey) == 2L) 41 | assert(settings(doubleKey) == 3.5D) 42 | assert(settings(anyRefKey) == Map("some-key" -> "some-value").asJava) 43 | assert(settings(nanosKey) == 1.nano) 44 | assert(settings(microsKey) == 1.micro) 45 | assert(settings(millisKey) == 1.milli) 46 | assert(settings(secondsKey) == 1.second) 47 | assert(settings(hoursKey) == 1.hour) 48 | assert(settings(daysKey) == 1.day) 49 | 50 | intercept[SettingsValidationException](settings(validatedKey)) 51 | intercept[ConfigException.Missing](settings(invalidKey)) 52 | } 53 | } 54 | 55 | object SettingsSuite { 56 | val booleanKey: Key[Boolean] = Key("boolean-value").boolean 57 | 58 | val numberKey: Key[Number] = Key("number-value").number 59 | 60 | val stringKey: Key[String] = Key("string-value").string 61 | 62 | val intKey: Key[Int] = Key("int-value").int 63 | 64 | val longKey: Key[Long] = Key("long-value").long 65 | 66 | val doubleKey: Key[Double] = Key("double-value").double 67 | 68 | val anyRefKey: Key[AnyRef] = Key("anyref-value").anyRef 69 | 70 | val nanosKey: Key[Duration] = Key("nanos-value").nanos 71 | 72 | val microsKey: Key[Duration] = Key("micros-value").micros 73 | 74 | val millisKey: Key[Duration] = Key("millis-value").millis 75 | 76 | val secondsKey: Key[Duration] = Key("seconds-value").seconds 77 | 78 | val minutesKey: Key[Duration] = Key("minutes-value").minutes 79 | 80 | val hoursKey: Key[Duration] = Key("hours-value").hours 81 | 82 | val daysKey: Key[Duration] = Key("days-value").days 83 | 84 | val validatedKey: Key[Int] = Key("int-value").int.validate { 85 | Try(_) filter { _ % 2 == 0 } orElse Failure(new RuntimeException("Expecting an even number.")) 86 | } 87 | 88 | val invalidKey: Key[Int] = Key("invalid").int 89 | } 90 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/execution/ProjectionSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.execution 2 | 3 | import scala.collection.mutable.ArrayBuffer 4 | 5 | import spear.{BasicMutableRow, LoggingFunSuite, Row} 6 | import spear.expressions._ 7 | 8 | class ProjectionSuite extends LoggingFunSuite { 9 | test("projection") { 10 | val projection = Projection(Seq(1, "foo")) 11 | assert(projection() == Row(1, "foo")) 12 | } 13 | 14 | test("mutable projection") { 15 | val projection = MutableProjection(Seq('_.int at 0, '_.string at 1)) 16 | assert(projection(Row(1, "foo")) == Row(1, "foo")) 17 | assert(projection(Row(2, "bar")) == Row(2, "bar")) 18 | } 19 | 20 | test("mutable projection - target change") { 21 | val row = new BasicMutableRow(ArrayBuffer[Any](1, "foo")) 22 | val projection = MutableProjection(Seq('_.int at 0, '_.string at 1)) target row 23 | assert(projection(Row(2, "bar")) == Row(2, "bar")) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/ArithmeticExpressionSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.Prop.forAll 5 | import org.scalatest.prop.Checkers 6 | 7 | import spear.{LoggingFunSuite, TestUtils} 8 | import spear.expressions.functions._ 9 | import spear.generators.types._ 10 | import spear.generators.values._ 11 | import spear.types._ 12 | 13 | class ArithmeticExpressionSuite extends LoggingFunSuite with TestUtils with Checkers { 14 | private val genNumericLiteral: Gen[Literal] = for { 15 | t <- genNumericType 16 | v <- genValueForNumericType(t) 17 | } yield Literal(v, t) 18 | 19 | private val genNumericLiteralPair = for { 20 | t <- genNumericType 21 | a <- genValueForNumericType(t) 22 | b <- genValueForNumericType(t) 23 | } yield (Literal(a, t), Literal(b, t)) 24 | 25 | test("plus") { 26 | check(forAll(genNumericLiteralPair) { 27 | case (a @ Literal(_, t: NumericType), b) => 28 | Plus(a, b).evaluated == t.genericNumeric.plus(a.value, b.value) 29 | }) 30 | } 31 | 32 | test("minus") { 33 | check(forAll(genNumericLiteralPair) { 34 | case (a @ Literal(_, t: NumericType), b) => 35 | (a - b).evaluated == t.genericNumeric.minus(a.value, b.value) 36 | }) 37 | } 38 | 39 | test("multiply") { 40 | check(forAll(genNumericLiteralPair) { 41 | case (a @ Literal(_, t: NumericType), b) => 42 | (a * b).evaluated == t.genericNumeric.times(a.value, b.value) 43 | }) 44 | } 45 | 46 | test("divide") { 47 | check(forAll(genNumericLiteralPair) { 48 | case (a @ Literal(_, t: IntegralType), b) => 49 | if (b.value == 0) { 50 | intercept[ArithmeticException](Divide(a, b).evaluated) 51 | true 52 | } else { 53 | Divide(a, b).evaluated == t.genericIntegral.quot(a.value, b.value) 54 | } 55 | 56 | case (a @ Literal(_, t: FractionalType), b) => 57 | if (b.value == 0D) { 58 | intercept[ArithmeticException]((a / b).evaluated) 59 | true 60 | } else { 61 | (a / b).evaluated == t.genericFractional.div(a.value, b.value) 62 | } 63 | }) 64 | } 65 | 66 | test("negate") { 67 | check(forAll(genNumericLiteral) { 68 | case lit @ Literal(v, t: NumericType) => 69 | val numeric = t.numeric.asInstanceOf[Numeric[Any]] 70 | Negate(lit).evaluated == numeric.negate(v) 71 | }) 72 | } 73 | 74 | test("binary arithmetic expression type check") { 75 | checkStrictlyTyped(lit(1) + 1, IntType) 76 | checkStrictlyTyped(lit(1L) + 1L, LongType) 77 | 78 | checkWellTyped(lit(1) + "1", IntType) 79 | checkWellTyped(lit("1") + 1, IntType) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/ComparisonSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import org.scalatest.prop.Checkers 4 | 5 | import spear.{LoggingFunSuite, TestUtils} 6 | import spear.types.{BooleanType, IntType} 7 | 8 | class ComparisonSuite extends LoggingFunSuite with TestUtils with Checkers { 9 | test("a = b") { 10 | check { (a: Int, b: Int) => 11 | Eq(Literal(a), Literal(b)).evaluated == (a == b) 12 | } 13 | } 14 | 15 | test("a > b") { 16 | check { (a: Int, b: Int) => 17 | Gt(Literal(a), Literal(b)).evaluated == (a > b) 18 | } 19 | } 20 | 21 | test("a < b") { 22 | check { (a: Int, b: Int) => 23 | Lt(Literal(a), Literal(b)).evaluated == (a < b) 24 | } 25 | } 26 | 27 | test("a >= b") { 28 | check { (a: Int, b: Int) => 29 | GtEq(Literal(a), Literal(b)).evaluated == (a >= b) 30 | } 31 | } 32 | 33 | test("a <= b") { 34 | check { (a: Int, b: Int) => 35 | LtEq(Literal(a), Literal(b)).evaluated == (a <= b) 36 | } 37 | } 38 | 39 | test("binary comparison type check") { 40 | checkStrictlyTyped('a.int === 'b.int, BooleanType) 41 | checkStrictlyTyped(('a.long cast IntType) === 'b.int, BooleanType) 42 | 43 | checkWellTyped('a.int === 'b.long, BooleanType) 44 | checkWellTyped('a.string === 'b.long, BooleanType) 45 | } 46 | 47 | test("IN") { 48 | checkWellTyped("1" in (1, 2, 3), BooleanType) 49 | checkWellTyped("1.0" in (1.0, 2, 3), BooleanType) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/LogicalOperatorSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import org.scalatest.prop.Checkers 4 | 5 | import spear.{LoggingFunSuite, TestUtils} 6 | 7 | class LogicalOperatorSuite extends LoggingFunSuite with TestUtils with Checkers { 8 | test("a AND b") { 9 | assert(!And(Literal(true), Literal(0L)).isWellTyped) 10 | assert(And(Literal(true), Literal(0)).isWellTyped) 11 | assert(And(Literal(true), Literal(false)).isStrictlyTyped) 12 | 13 | check { (a: Boolean, b: Boolean) => 14 | And(Literal(a), Literal(b)).evaluated == (a && b) 15 | } 16 | } 17 | 18 | test("a OR b") { 19 | assert(!Or(Literal(true), Literal(0L)).isWellTyped) 20 | assert(Or(Literal(true), Literal(0)).isWellTyped) 21 | assert(Or(Literal(true), Literal(false)).isStrictlyTyped) 22 | 23 | check { (a: Boolean, b: Boolean) => 24 | Or(Literal(a), Literal(b)).evaluated == (a || b) 25 | } 26 | } 27 | 28 | test("NOT a") { 29 | assert(!Not(Literal(0L)).isWellTyped) 30 | assert(Not(Literal(0)).isWellTyped) 31 | assert(Not(Literal(true)).isStrictlyTyped) 32 | 33 | check { a: Boolean => 34 | Not(Literal(a)).evaluated == !a 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/NullExpressionsSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.{LoggingFunSuite, TestUtils} 4 | import spear.types.{IntType, LongType} 5 | 6 | class NullExpressionsSuite extends LoggingFunSuite with TestUtils { 7 | test("if - type check") { 8 | checkStrictlyTyped(If(true, 'a.int, 'b.int), IntType) 9 | checkStrictlyTyped(If('a.int === 1, 'a.int, 'b.int), IntType) 10 | 11 | checkWellTyped(If(true, 'a.long, 'b.int), LongType) 12 | checkWellTyped(If('a.int === 1L, 'a.int, 'b.int), IntType) 13 | checkWellTyped(If('a.int === 1L, 'a.long, 'b.int), LongType) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/ObjectExpressionsSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import spear.LoggingFunSuite 4 | import spear.types.{ByteType, IntType, ObjectType} 5 | 6 | class ObjectExpressionsSuite extends LoggingFunSuite { 7 | test("static invoke") { 8 | assertResult(1: Integer) { 9 | classOf[Integer] 10 | .invoke("decode", IntType) 11 | .withArgs("1") 12 | .evaluated 13 | } 14 | } 15 | 16 | test("invoke") { 17 | assertResult(1: Byte) { 18 | Literal(1: Integer, ObjectType("java.lang.Integer")) 19 | .invoke("byteValue", ByteType) 20 | .noArgs 21 | .evaluated 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/StringExpressionsSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions 2 | 3 | import org.scalacheck.Gen 4 | import org.scalacheck.Prop.forAll 5 | import org.scalatest.prop.Checkers 6 | 7 | import spear.expressions.functions._ 8 | import spear.LoggingFunSuite 9 | 10 | class StringExpressionsSuite extends LoggingFunSuite with Checkers { 11 | test("concat") { 12 | check { 13 | forAll { (a: String, b: String) => 14 | concat(a, b).strictlyTyped.evaluated == a + b 15 | } 16 | } 17 | } 18 | 19 | test("rlike") { 20 | check { 21 | forAll(Gen.alphaStr) { a: String => 22 | rlike(a, "a.*").strictlyTyped.evaluated == a.startsWith("a") 23 | } 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/typecheck/TypeConstraintSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.typecheck 2 | 3 | import spear.LoggingFunSuite 4 | import spear.exceptions.{ImplicitCastException, TypeMismatchException} 5 | import spear.expressions._ 6 | import spear.expressions.functions._ 7 | import spear.types._ 8 | 9 | class TypeConstraintSuite extends LoggingFunSuite { 10 | testTypeConstraint(classOf[StrictlyTyped]) { 11 | expectExpressions(1) { 12 | lit(1).anyType 13 | } 14 | 15 | expectExpressions((1 cast LongType) + 1L) { 16 | (lit(1) + 1L).anyType 17 | } 18 | } 19 | 20 | testTypeConstraint(classOf[SameTypeAs]) { 21 | expectExpressions(true, 1 cast BooleanType) { 22 | Seq[Expression](true, 1) sameTypeAs BooleanType 23 | } 24 | 25 | expectException[ImplicitCastException] { 26 | Seq[Expression](true, false) sameTypeAs LongType 27 | } 28 | } 29 | 30 | testTypeConstraint(classOf[SameSubtypeOf]) { 31 | expectExpressions((1: Byte) cast IntType, (1: Short) cast IntType, lit(1)) { 32 | Seq[Expression](1: Byte, 1: Short, 1) sameSubtypeOf IntegralType 33 | } 34 | 35 | expectException[TypeMismatchException] { 36 | Seq[Expression](1F, 1L) sameSubtypeOf IntegralType 37 | } 38 | } 39 | 40 | testTypeConstraint(classOf[SameType]) { 41 | expectExpressions(1 cast LongType, 1L) { 42 | SameType(Seq(1, 1L)) 43 | } 44 | } 45 | 46 | testTypeConstraint(classOf[Foldable]) { 47 | expectExpressions(1, "foo") { 48 | Foldable(Seq(lit(1), "foo")) 49 | } 50 | 51 | expectException[TypeMismatchException] { 52 | ('a of IntType.!).foldable 53 | } 54 | } 55 | 56 | test("andAlso") { 57 | expectExpressions(1 cast LongType, 1L) { 58 | Seq[Expression](1, 1L) sameSubtypeOf OrderedType andAlso { _.sameType } 59 | } 60 | } 61 | 62 | test("orElse") { 63 | expectExpressions(1 cast StringType) { 64 | lit(1) sameTypeAs StringType orElse (lit(1) subtypeOf ArrayType) 65 | } 66 | 67 | expectExpressions(1 cast StringType) { 68 | lit(1) subtypeOf ArrayType orElse (lit(1) sameTypeAs StringType) 69 | } 70 | } 71 | 72 | private def testTypeConstraint(constraintsClass: Class[_ <: TypeConstraint])(f: => Unit): Unit = { 73 | test(constraintsClass.getSimpleName)(f) 74 | } 75 | 76 | private def check(expected: Seq[Expression])(constraint: TypeConstraint): Unit = { 77 | assertResult(expected)(constraint.enforced) 78 | } 79 | 80 | private def expectExpressions( 81 | first: Expression, rest: Expression* 82 | )(constraint: TypeConstraint): Unit = { 83 | check(first +: rest)(constraint) 84 | } 85 | 86 | private def expectException[T <: Throwable: Manifest](constraint: TypeConstraint): Unit = { 87 | intercept[T](constraint.enforced) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/expressions/windows/WindowSpecSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.expressions.windows 2 | 3 | import scala.util.Try 4 | 5 | import spear.LoggingFunSuite 6 | import spear.exceptions.TypeMismatchException 7 | import spear.expressions._ 8 | import spear.types.{DoubleType, IntType, StringType} 9 | 10 | class WindowSpecSuite extends LoggingFunSuite { 11 | test("rows window frame") { 12 | assertResult("ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING") { 13 | WindowFrame(RowsFrame, UnboundedPreceding, UnboundedFollowing).sql.get 14 | } 15 | 16 | assertResult("ROWS BETWEEN 10 PRECEDING AND CURRENT ROW") { 17 | WindowFrame(RowsFrame, Preceding(10), CurrentRow).sql.get 18 | } 19 | 20 | assertResult("ROWS BETWEEN CURRENT ROW AND 10 FOLLOWING") { 21 | WindowFrame(RowsFrame, CurrentRow, Following(10)).sql.get 22 | } 23 | 24 | assertResult("ROWS BETWEEN 2 PRECEDING AND 1 PRECEDING") { 25 | WindowFrame(RowsFrame, Preceding(2), Preceding(1)).sql.get 26 | } 27 | 28 | assertResult("ROWS BETWEEN 1 FOLLOWING AND 2 FOLLOWING") { 29 | WindowFrame(RowsFrame, Following(1), Following(2)).sql.get 30 | } 31 | } 32 | 33 | test("range window frame") { 34 | assertResult("RANGE BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING") { 35 | WindowFrame(RangeFrame, UnboundedPreceding, UnboundedFollowing).sql.get 36 | } 37 | 38 | assertResult("RANGE BETWEEN 10 PRECEDING AND CURRENT ROW") { 39 | WindowFrame(RangeFrame, Preceding(10), CurrentRow).sql.get 40 | } 41 | 42 | assertResult("RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING") { 43 | WindowFrame(RangeFrame, CurrentRow, Following(10)).sql.get 44 | } 45 | 46 | assertResult("RANGE BETWEEN 2 PRECEDING AND 1 PRECEDING") { 47 | WindowFrame(RangeFrame, Preceding(2), Preceding(1)).sql.get 48 | } 49 | 50 | assertResult("RANGE BETWEEN 1 FOLLOWING AND 2 FOLLOWING") { 51 | WindowFrame(RangeFrame, Following(1), Following(2)).sql.get 52 | } 53 | } 54 | 55 | test("arbitrary expressions in window frame boundary") { 56 | Following('id.long + 1).strictlyTyped 57 | Preceding('id.long + "1").strictlyTyped 58 | } 59 | 60 | test("illegal window frame boundary") { 61 | intercept[TypeMismatchException](Preceding(true).strictlyTyped) 62 | intercept[TypeMismatchException](Following(true).strictlyTyped) 63 | } 64 | 65 | test("window spec") { 66 | def checkWindowSpec(sql: String)(spec: => WindowSpec): Unit = assertResult(Try(sql))(spec.sql) 67 | 68 | val rowsFrame = WindowFrame.Default.copy(frameType = RowsFrame) 69 | val rangeFrame = WindowFrame.Default.copy(frameType = RangeFrame) 70 | 71 | checkWindowSpec(s"(PARTITION BY a, b ORDER BY c ASC NULLS FIRST)") { 72 | Window partitionBy (a, b) orderBy c.asc.nullsFirst 73 | } 74 | 75 | checkWindowSpec(s"(PARTITION BY a, b ORDER BY c ASC NULLS FIRST)") { 76 | Window orderBy c.asc.nullsFirst partitionBy (a, b) 77 | } 78 | 79 | checkWindowSpec(s"(ORDER BY c ASC NULLS FIRST)") { 80 | Window orderBy c.asc.nullsFirst 81 | } 82 | 83 | checkWindowSpec(s"(PARTITION BY a, b)") { 84 | Window partitionBy (a, b) 85 | } 86 | 87 | checkWindowSpec("()") { 88 | Window.Default 89 | } 90 | 91 | Seq(rowsFrame, rangeFrame) foreach { frame => 92 | checkWindowSpec(s"(PARTITION BY a, b ORDER BY c ASC NULLS FIRST $frame)") { 93 | Window partitionBy (a, b) orderBy c.asc.nullsFirst between frame 94 | } 95 | 96 | checkWindowSpec(s"(PARTITION BY a, b ORDER BY c ASC NULLS FIRST $frame)") { 97 | Window orderBy c.asc.nullsFirst partitionBy (a, b) between frame 98 | } 99 | 100 | checkWindowSpec(s"(ORDER BY c ASC NULLS FIRST $frame)") { 101 | Window orderBy c.asc.nullsFirst between frame 102 | } 103 | 104 | checkWindowSpec(s"(PARTITION BY a, b $frame)") { 105 | Window partitionBy (a, b) between frame 106 | } 107 | 108 | checkWindowSpec(s"($frame)") { 109 | Window between frame 110 | } 111 | } 112 | 113 | checkWindowSpec(s"($rowsFrame)") { 114 | Window rowsBetween (UnboundedPreceding, CurrentRow) 115 | } 116 | 117 | checkWindowSpec(s"($rangeFrame)") { 118 | Window rangeBetween (UnboundedPreceding, CurrentRow) 119 | } 120 | } 121 | 122 | private val (a, b, c) = ('a of IntType, 'b of StringType, 'c of DoubleType) 123 | } 124 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/generators/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.util.{Failure, Success} 4 | 5 | import org.scalacheck.Gen 6 | 7 | import spear.config.Settings.Key 8 | import spear.exceptions.SettingsValidationException 9 | 10 | package object generators { 11 | def genRandomPartitions(sum: Int, partitionNum: Int): Gen[Seq[Int]] = for { 12 | ns <- Gen pick (partitionNum - 1, 1 until sum) 13 | sorted = ns.sorted 14 | } yield (0 +: sorted, sorted :+ sum).zipped.map(_ - _) 15 | 16 | def chance[T](gs: (Double, Gen[T])*): Gen[T] = { 17 | val (chances, gens) = gs.unzip 18 | val frequencies = chances map { _ * 100 } map { _.toInt } 19 | Gen.frequency(frequencies zip gens: _*) 20 | } 21 | 22 | def chanceOption[T](chance: Double, gen: Gen[T]): Gen[Option[T]] = Gen.sized { 23 | case 0 => Gen const None 24 | case _ => generators.chance(chance -> gen.map(Some.apply), (1 - chance) -> Gen.const(None)) 25 | } 26 | 27 | object Keys { 28 | // --------------------------------- 29 | // Settings keys for type generators 30 | // --------------------------------- 31 | 32 | val AllowNullType: Key[Boolean] = 33 | Key("spear.test.types.allow-null-type").boolean 34 | 35 | val AllowEmptyStructType: Key[Boolean] = 36 | Key("spear.test.types.allow-empty-struct-type").boolean 37 | 38 | val AllowNullableComplexType: Key[Boolean] = 39 | Key("spear.test.types.allow-nullable-complex-type").boolean 40 | 41 | val AllowNullableArrayType: Key[Boolean] = 42 | Key("spear.test.types.allow-nullable-array-type").boolean 43 | 44 | val AllowNullableMapType: Key[Boolean] = 45 | Key("spear.test.types.allow-nullable-map-type").boolean 46 | 47 | val AllowNullableStructField: Key[Boolean] = 48 | Key("spear.test.types.allow-nullable-struct-field").boolean 49 | 50 | val AllowNestedStructType: Key[Boolean] = 51 | Key("spear.test.types.allow-nested-struct-type").boolean 52 | 53 | val MaxStructTypeWidth: Key[Int] = 54 | Key("spear.test.types.max-struct-type-width").int 55 | 56 | // ---------------------------------- 57 | // Settings keys for value generators 58 | // ---------------------------------- 59 | 60 | val MaxRepetition: Key[Int] = 61 | Key("spear.test.expressions.max-repetition").int 62 | 63 | val NullChances: Key[Double] = 64 | Key("spear.test.expressions.chances.null").double.validate { 65 | case v if v >= 0D && v <= 1.0D => Success(v) 66 | case v => Failure(new SettingsValidationException( 67 | s"Illegal null chance $v, value must be within range [0.0, 1.0]." 68 | )) 69 | } 70 | 71 | val OnlyLogicalOperatorsInPredicate: Key[Boolean] = 72 | Key("spear.test.expressions.only-logical-operators-in-predicate").boolean 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/generators/values/package.scala: -------------------------------------------------------------------------------- 1 | package spear.generators 2 | 3 | import scala.collection.JavaConverters._ 4 | 5 | import org.scalacheck.Arbitrary.arbitrary 6 | import org.scalacheck.Gen 7 | 8 | import spear.Row 9 | import spear.config.Settings 10 | import spear.generators.Keys.MaxRepetition 11 | import spear.types._ 12 | 13 | package object values { 14 | def genValueForDataType(dataType: DataType)(implicit settings: Settings): Gen[Any] = 15 | dataType match { 16 | case t: PrimitiveType => genValueForPrimitiveType(t) 17 | case t: ComplexType => genValueForComplexType(t)(settings) 18 | } 19 | 20 | def genValueForPrimitiveType(dataType: PrimitiveType): Gen[Any] = dataType match { 21 | case NullType => Gen.const(null) 22 | case BooleanType => arbitrary[Boolean] 23 | case StringType => Gen.alphaStr 24 | case t: NumericType => genValueForNumericType(t) 25 | } 26 | 27 | def genValueForNumericType(dataType: NumericType): Gen[Any] = dataType match { 28 | case t: IntegralType => genValueForIntegralType(t) 29 | case t: FractionalType => genValueForFractionalType(t) 30 | } 31 | 32 | def genValueForIntegralType(dataType: IntegralType): Gen[Any] = dataType match { 33 | case ByteType => arbitrary[Byte] 34 | case ShortType => arbitrary[Short] 35 | case IntType => arbitrary[Int] 36 | case LongType => arbitrary[Long] 37 | } 38 | 39 | def genValueForFractionalType(dataType: FractionalType): Gen[Any] = dataType match { 40 | case FloatType => arbitrary[Float] 41 | case DoubleType => arbitrary[Double] 42 | } 43 | 44 | def genValueForComplexType(dataType: ComplexType)(implicit settings: Settings): Gen[Any] = 45 | dataType match { 46 | case t: ArrayType => genValueForArrayType(t)(settings) 47 | case t: MapType => genValueForMapType(t)(settings) 48 | case t: StructType => genValueForStructType(t)(settings) 49 | } 50 | 51 | def genValueForArrayType(dataType: ArrayType)(implicit settings: Settings): Gen[Seq[_]] = { 52 | val genElement = if (dataType.isElementNullable) { 53 | Gen.option(genValueForDataType(dataType)(settings)) map { _.orNull } 54 | } else { 55 | genValueForDataType(dataType)(settings) 56 | } 57 | 58 | for { 59 | repetition <- Gen.choose(0, settings(MaxRepetition)) 60 | elements <- Gen.listOfN(repetition, genElement) 61 | } yield elements 62 | } 63 | 64 | def genValueForMapType(dataType: MapType)(implicit settings: Settings): Gen[Map[_, _]] = { 65 | val genKey = genValueForDataType(dataType.keyType)(settings) 66 | val genValue = { 67 | val genNonNullValue = genValueForDataType(dataType)(settings) 68 | if (dataType.isValueNullable) { 69 | Gen.option(genNonNullValue) map { _.orNull } 70 | } else { 71 | genNonNullValue 72 | } 73 | } 74 | 75 | for { 76 | repetition <- Gen.choose(0, settings(MaxRepetition)) 77 | keys <- Gen.listOfN(repetition, genKey) 78 | values <- Gen.listOfN(repetition, genValue) 79 | } yield (keys zip values).toMap 80 | } 81 | 82 | def genValueForStructType(dataType: StructType)(implicit settings: Settings): Gen[Row] = { 83 | val genFields = Gen.sequence(dataType.fields map { 84 | case StructField(_, fieldType, nullable) => 85 | val genNonNullField = genValueForDataType(fieldType)(settings) 86 | if (nullable) Gen.option(genNonNullField) map { _.orNull } else genNonNullField 87 | }) 88 | 89 | for (fields <- genFields) yield Row.fromSeq(fields.asScala) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/parsers/DataTypeParserSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.core.Logger 4 | 5 | import spear.{LoggingFunSuite, TestUtils} 6 | import spear.types._ 7 | 8 | class DataTypeParserSuite extends LoggingFunSuite with TestUtils { 9 | import fastparse.all._ 10 | 11 | testDataTypeParsing("BOOLEAN", BooleanType) 12 | 13 | testDataTypeParsing("TINYINT", ByteType) 14 | 15 | testDataTypeParsing("SMALLINT", ShortType) 16 | 17 | testDataTypeParsing("INT", IntType) 18 | 19 | testDataTypeParsing("BIGINT", LongType) 20 | 21 | testDataTypeParsing("FLOAT", FloatType) 22 | 23 | testDataTypeParsing("DOUBLE", DoubleType) 24 | 25 | testDataTypeParsing("ARRAY", ArrayType(IntType.?)) 26 | 27 | testDataTypeParsing("MAP", MapType(IntType, StringType.?)) 28 | 29 | testDataTypeParsing( 30 | "STRUCT", 31 | StructType( 32 | 'name -> StringType.?, 33 | 'age -> IntType.? 34 | ) 35 | ) 36 | 37 | testDataTypeParsing( 38 | "STRUCT<\"name\": STRING, \"age\": INT>", 39 | StructType( 40 | 'name -> StringType.?, 41 | 'age -> IntType.? 42 | ) 43 | ) 44 | 45 | testDataTypeParsing( 46 | "ROW(name STRING, age INT)", 47 | StructType( 48 | 'NAME -> StringType.?, 49 | 'AGE -> IntType.? 50 | ) 51 | ) 52 | 53 | testDataTypeParsing( 54 | "ROW(\"name\" STRING, \"age\" INT)", 55 | StructType( 56 | "name" -> StringType.?, 57 | "age" -> IntType.? 58 | ) 59 | ) 60 | 61 | private def testDataTypeParsing(input: String, expected: DataType): Unit = { 62 | test(s"parsing expression: $input") { 63 | checkTree(parse(input), expected) 64 | } 65 | } 66 | 67 | private def parse(input: String): DataType = { 68 | implicit val parserLogger = Logger(logInfo(_)) 69 | (Start ~ DataTypeParser.dataType.log() ~ End parse input.trim).get.value 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/parsers/IdentifierParserSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.all._ 4 | import fastparse.core.Logger 5 | 6 | import spear._ 7 | 8 | class IdentifierParserSuite extends LoggingFunSuite { 9 | import WhitespaceApi._ 10 | 11 | private implicit val parsingLogger = Logger(logInfo(_)) 12 | 13 | private def fullyParse[T](parser: P[T], input: String) = { 14 | (Start ~~ parser ~~ End parse input).get.value 15 | } 16 | 17 | private def testParsingLegalIdentifier[T](input: String, expected: T): Unit = { 18 | test(s"parsing legal identifier: $input") { 19 | assertResult(expected) { 20 | fullyParse(IdentifierParser.identifier, input) 21 | } 22 | } 23 | } 24 | 25 | private def testParsingIllegalIdentifier[T](input: String): Unit = { 26 | test(s"parsing illegal identifier: $input") { 27 | val cause = intercept[ParseError] { 28 | fullyParse(IdentifierParser.identifier, input) 29 | } 30 | 31 | logInfo(s"Expected parsing failure: ${cause.getMessage}") 32 | } 33 | } 34 | 35 | val successfulCases: Seq[(String, Name)] = Seq( 36 | // Regular identifiers 37 | "data" -> i"data", 38 | "数据" -> i"数据", 39 | 40 | // Delimited identifiers 41 | "\"data\"" -> "data", 42 | "\"数据\"" -> "数据", 43 | "\"double\"\"quote\"" -> "double\"quote", 44 | 45 | // Unicode delimited identifiers 46 | "U&\"data\"" -> "data", 47 | "U&\"\\6570\\636e\"" -> "数据", 48 | "U&\"\\0064\\0061\\0074\\0061\"" -> "data", 49 | "U&\"!!\"" -> "!!", 50 | "U&\"\\\\\"" -> "\\", 51 | 52 | // Unicode delimited identifiers with unicode escape specifier 53 | "U&\"d!0061t!+000061\" UESCAPE '!'" -> "data", 54 | "U&\"!!\" UESCAPE '!'" -> "!", 55 | "U&\"\\\\\" UESCAPE '!'" -> "\\\\" 56 | ) 57 | 58 | successfulCases foreach (testParsingLegalIdentifier _).tupled 59 | 60 | val failedCases = Seq( 61 | "_data", 62 | "\"double\\\"quote\"", 63 | "U&\"!\" UESCAPE '!'", 64 | "U&\"\\\"" 65 | ) 66 | 67 | failedCases foreach testParsingIllegalIdentifier 68 | } 69 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/parsers/ValueExpressionParserSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.parsers 2 | 3 | import fastparse.core.Logger 4 | 5 | import spear.{LoggingFunSuite, TestUtils} 6 | import spear.expressions._ 7 | import spear.expressions.functions._ 8 | import spear.types.IntType 9 | 10 | class ValueExpressionParserSuite extends LoggingFunSuite with TestUtils { 11 | import fastparse.all._ 12 | 13 | testExpressionParsing("1", 1) 14 | 15 | testExpressionParsing(Int.MaxValue.toString, Int.MaxValue) 16 | 17 | testExpressionParsing(Int.MinValue.toString, Int.MinValue) 18 | 19 | testExpressionParsing((Int.MaxValue.toLong + 1).toString, Int.MaxValue.toLong + 1) 20 | 21 | testExpressionParsing((Int.MinValue.toLong - 1).toString, Int.MinValue.toLong - 1) 22 | 23 | testExpressionParsing("'1'", "1") 24 | 25 | testExpressionParsing("'a' || 'b' || 'c'", concat(concat("a", "b"), "c")) 26 | 27 | testExpressionParsing("('a' || 'b') || 'c'", concat(concat("a", "b"), "c")) 28 | 29 | testExpressionParsing("'a' || ('b' || 'c')", concat("a", concat("b", "c"))) 30 | 31 | testExpressionParsing("'a' 'b'", "ab") 32 | 33 | testExpressionParsing("true", true) 34 | 35 | testExpressionParsing("false", false) 36 | 37 | testExpressionParsing("\"a\"", 'a) 38 | 39 | testExpressionParsing("\"a\"\"b\"", Symbol("a\"b")) 40 | 41 | testExpressionParsing("(a = 1)", 'a === 1) 42 | 43 | testExpressionParsing("a AND b", 'a && 'b) 44 | 45 | testExpressionParsing("a OR b", 'a || 'b) 46 | 47 | testExpressionParsing("NOT a", !'a) 48 | 49 | testExpressionParsing("(a AND a) AND a", ('a && 'a) && 'a) 50 | 51 | testExpressionParsing("a = b", 'a === 'b) 52 | 53 | testExpressionParsing("a <> b", 'a =/= 'b) 54 | 55 | testExpressionParsing("a > b", 'a > 'b) 56 | 57 | testExpressionParsing("a >= b", 'a >= 'b) 58 | 59 | testExpressionParsing("a < b", 'a < 'b) 60 | 61 | testExpressionParsing("a <= b", 'a <= 'b) 62 | 63 | testExpressionParsing("a IS NULL", 'a.isNull) 64 | 65 | testExpressionParsing("a IS NOT NULL", 'a.isNotNull) 66 | 67 | testExpressionParsing("-a", -'a) 68 | 69 | testExpressionParsing("a + b", 'a + 'b) 70 | 71 | testExpressionParsing("a - b", 'a - 'b) 72 | 73 | testExpressionParsing("a * b", 'a * 'b) 74 | 75 | testExpressionParsing("a / b", 'a / 'b) 76 | 77 | testExpressionParsing("a % b", 'a % 'b) 78 | 79 | testExpressionParsing("a ^ b", 'a ^ 'b) 80 | 81 | testExpressionParsing("a + b * c - d / e", 'a + ('b * 'c) - ('d / 'e)) 82 | 83 | testExpressionParsing("a + b * (c - d) / e", 'a + 'b * ('c - 'd) / 'e) 84 | 85 | testExpressionParsing("a + b * c ^ d", 'a + ('b * ('c ^ 'd))) 86 | 87 | testExpressionParsing("(a + b) + c", ('a + 'b) + 'c) 88 | 89 | testExpressionParsing("a + (b + c)", 'a + ('b + 'c)) 90 | 91 | testExpressionParsing("CAST(RAND(42) * 100 AS INT)", ('rand(42) * 100) cast IntType) 92 | 93 | testExpressionParsing( 94 | "CASE WHEN 1 THEN 'x' WHEN 2 THEN 'y' END", 95 | when(1, "x") when (2, "y") 96 | ) 97 | 98 | testExpressionParsing( 99 | "CASE WHEN 1 THEN 'x' WHEN 2 THEN 'y' ELSE 'z' END", 100 | when(1, "x") when (2, "y") otherwise "z" 101 | ) 102 | 103 | testExpressionParsing( 104 | "CASE a WHEN 1 THEN 'x' WHEN 2 THEN 'y' END", 105 | when('a === 1, "x") when ('a === 2, "y") 106 | ) 107 | 108 | testExpressionParsing( 109 | "CASE a WHEN 1 THEN 'x' WHEN 2 THEN 'y' ELSE 'z' END", 110 | when('a === 1, "x") when ('a === 2, "y") otherwise "z" 111 | ) 112 | 113 | testExpressionParsing( 114 | "CASE a WHEN 1, 2 THEN 'x' WHEN 3, 4 THEN 'y' ELSE 'z' END", 115 | when('a === 1 || 'a === 2, "x") when ('a === 3 || 'a === 4, "y") otherwise "z" 116 | ) 117 | 118 | testExpressionParsing( 119 | "IF(a > 0, 1, 2)", 120 | If('a > 0, 1, 2) 121 | ) 122 | 123 | private def testExpressionParsing(input: String, expected: Expression): Unit = { 124 | test(s"parsing expression: $input") { 125 | checkTree(parse(input), expected) 126 | } 127 | } 128 | 129 | private def parse(input: String): Expression = { 130 | implicit val parserLogger = Logger(logInfo(_)) 131 | (Start ~ ValueExpressionParser.valueExpression.log() ~ End parse input.trim).get.value 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/plans/logical/analysis/AnalyzerTest.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import org.scalatest.BeforeAndAfterAll 4 | 5 | import spear.{InMemoryCatalog, LoggingFunSuite, TestUtils} 6 | import spear.parsers.DirectlyExecutableStatementParser.directlyExecutableStatement 7 | import spear.plans.logical.LogicalPlan 8 | 9 | abstract class AnalyzerTest extends LoggingFunSuite with TestUtils with BeforeAndAfterAll { 10 | protected val catalog = new InMemoryCatalog 11 | 12 | protected val analyze = new Analyzer(catalog) 13 | 14 | protected def checkAnalyzedPlan(sql: String, expected: LogicalPlan): Unit = 15 | checkAnalyzedPlan(parse(sql), expected) 16 | 17 | protected def checkAnalyzedPlan(unresolved: LogicalPlan, expected: LogicalPlan): Unit = 18 | checkPlan(analyze(unresolved), expected) 19 | 20 | protected def checkSQLAnalysis( 21 | sql: String, expectedParsedPlan: LogicalPlan, expectedAnalyzedPlan: LogicalPlan 22 | ): Unit = { 23 | val actualParsed = parse(sql) 24 | 25 | withClue("Checking SQL parsing result") { 26 | checkPlan(actualParsed, expectedParsedPlan) 27 | } 28 | 29 | withClue("Checking analysis result") { 30 | checkPlan(analyze(actualParsed), expectedAnalyzedPlan) 31 | } 32 | } 33 | 34 | protected def checkMessage[T <: Throwable: Manifest](patterns: String*)(f: => Any): Unit = { 35 | val cause = intercept[T](f) 36 | 37 | (patterns foldLeft cause.getMessage) { (text, pattern) => 38 | val index = text.indexOf(pattern) 39 | 40 | if (index == -1) { 41 | fail( 42 | s"""Failed to find the following pattern in the error message: 43 | | 44 | | $pattern 45 | | 46 | |Full error message: 47 | | 48 | | ${cause.getMessage} 49 | |""".stripMargin 50 | ) 51 | } 52 | 53 | text.drop(index + pattern.length) 54 | } 55 | } 56 | 57 | protected def checkMessageRegex[T <: Throwable: Manifest](patterns: String*)(f: => Any): Unit = { 58 | val regex = (patterns mkString ".*").r 59 | val cause = intercept[T](f) 60 | 61 | if (regex.findFirstIn(cause.getMessage).isEmpty) { 62 | fail( 63 | s"""Failed to find all of the following regex patterns in the error message: 64 | | 65 | |${patterns map { " - " + _ } mkString "\n"} 66 | | 67 | |Full error message: 68 | | 69 | | ${cause.getMessage} 70 | |""".stripMargin 71 | ) 72 | } 73 | } 74 | 75 | private def parse(sql: String): LogicalPlan = { 76 | import fastparse.all._ 77 | (Start ~ directlyExecutableStatement ~ End).parse(sql).get.value 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/plans/logical/analysis/MiscAnalysisSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import spear._ 4 | import spear.expressions._ 5 | import spear.expressions.NamedExpression.newExpressionID 6 | import spear.plans.logical._ 7 | 8 | class MiscAnalysisSuite extends AnalyzerTest { 9 | test("self-join") { 10 | checkAnalyzedPlan( 11 | relation join relation, 12 | relation join relation.newInstance() 13 | ) 14 | } 15 | 16 | test("self-join in SQL") { 17 | val relation0 = relation.newInstance() 18 | val Seq(a0: AttributeRef, b0: AttributeRef) = relation0.output 19 | 20 | checkAnalyzedPlan( 21 | "SELECT * FROM t JOIN t", 22 | relation 23 | subquery 't 24 | join (relation0 subquery 't) 25 | select (a of 't, b of 't, a0 of 't, b0 of 't) 26 | ) 27 | } 28 | 29 | test("duplicated aliases") { 30 | val alias = 1 as 'a 31 | val newAlias = alias withID newExpressionID() 32 | 33 | checkAnalyzedPlan( 34 | SingleRowRelation select alias union (SingleRowRelation select alias), 35 | SingleRowRelation select alias union (SingleRowRelation select newAlias) 36 | ) 37 | 38 | checkAnalyzedPlan( 39 | SingleRowRelation select alias intersect (SingleRowRelation select alias), 40 | SingleRowRelation select alias intersect (SingleRowRelation select newAlias) 41 | ) 42 | 43 | checkAnalyzedPlan( 44 | SingleRowRelation select alias except (SingleRowRelation select alias), 45 | SingleRowRelation select alias except (SingleRowRelation select newAlias) 46 | ) 47 | } 48 | 49 | test("order by columns not appearing in project list") { 50 | val `@S: a + 1` = SortOrderAlias(a + 1, "order0") 51 | val `@S: a * 2` = SortOrderAlias(a * 2, "order1") 52 | 53 | checkAnalyzedPlan( 54 | relation 55 | select 'b 56 | orderBy (('a + 1).asc, ('a * 2).desc, 'b.desc), 57 | 58 | relation 59 | select (b, `@S: a + 1`, `@S: a * 2`) 60 | orderBy (`@S: a + 1`.attr.asc, `@S: a * 2`.attr.desc, b.desc) 61 | select b 62 | ) 63 | } 64 | 65 | test("order by columns not appearing in project list in SQL") { 66 | val `@S: a + 1` = SortOrderAlias((a of 't) + 1, "order0") 67 | val `@S: a * 2` = SortOrderAlias((a of 't) * 2, "order1") 68 | 69 | checkAnalyzedPlan( 70 | "SELECT b FROM t ORDER BY a + 1 ASC, a * 2 DESC, b DESC", 71 | 72 | relation 73 | subquery 't 74 | select (b of 't, `@S: a + 1`, `@S: a * 2`) 75 | orderBy (`@S: a + 1`.attr.asc, `@S: a * 2`.attr.desc, (b of 't).desc) 76 | select (b of 't) 77 | ) 78 | } 79 | 80 | override protected def beforeAll(): Unit = catalog.registerRelation('t, relation) 81 | 82 | private val (a, b) = ('a.int.!, 'b.string.?) 83 | 84 | private val relation = LocalRelation.empty(a, b) 85 | } 86 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/plans/logical/analysis/PostAnalysisCheckSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import spear._ 4 | import spear.exceptions.{AnalysisException, ResolutionFailureException} 5 | import spear.expressions._ 6 | import spear.expressions.functions._ 7 | import spear.plans.logical.LocalRelation 8 | 9 | class PostAnalysisCheckSuite extends AnalyzerTest { 10 | test("post-analysis check - reject unresolved expressions") { 11 | val rule = new RejectUnresolvedExpressions(catalog) 12 | 13 | intercept[ResolutionFailureException] { 14 | rule(relation select 'a) 15 | } 16 | } 17 | 18 | test("post-analysis check - reject unresolved plans") { 19 | val rule = new RejectUnresolvedPlans(catalog) 20 | 21 | intercept[ResolutionFailureException] { 22 | rule(relation groupBy Nil agg (1 as 'a)) 23 | } 24 | } 25 | 26 | test("post-analysis check - reject top-level `InternalAttribute`s") { 27 | val rule = new RejectTopLevelInternalAttributes(catalog) 28 | 29 | intercept[ResolutionFailureException] { 30 | rule(LocalRelation.empty(GroupingKeyAlias('a.int.!).attr)) 31 | } 32 | } 33 | 34 | test("post-analysis check - reject distinct aggregate functions") { 35 | val rule = new RejectDistinctAggregateFunctions(catalog) 36 | 37 | intercept[ResolutionFailureException] { 38 | rule(relation select distinct(count(a))) 39 | } 40 | } 41 | 42 | test("post-analysis check - reject orphan attribute references") { 43 | val rule = new RejectOrphanAttributeRefs(catalog) 44 | 45 | intercept[AnalysisException] { 46 | rule(relation select 'c.int.!) 47 | } 48 | } 49 | 50 | private val (a, b) = ('a.int.!, 'b.string.?) 51 | 52 | private val relation = LocalRelation.empty(a, b) 53 | } 54 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/plans/logical/analysis/WindowAnalysisTest.scala: -------------------------------------------------------------------------------- 1 | package spear.plans.logical.analysis 2 | 3 | import spear.expressions._ 4 | import spear.plans.logical.{LocalRelation, LogicalPlan} 5 | 6 | abstract class WindowAnalysisTest extends AnalyzerTest { self => 7 | override protected def afterAll(): Unit = catalog.removeRelation('t) 8 | 9 | protected val relation: LogicalPlan = { 10 | catalog.registerRelation('t, LocalRelation.empty('a.int.!, 'b.string.?)) 11 | catalog.lookupRelation('t) 12 | } 13 | 14 | protected val Seq(a: AttributeRef, b: AttributeRef) = relation.output 15 | } 16 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/trees/TreeNodeSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.trees 2 | 3 | import scala.collection.Iterator.iterate 4 | import scala.collection.JavaConverters._ 5 | import scala.collection.immutable.Stream.Empty 6 | import scala.language.implicitConversions 7 | 8 | import org.scalacheck._ 9 | import org.scalacheck.Prop.{all, BooleanOperators} 10 | import org.scalacheck.util.Pretty 11 | import org.scalatest.prop.Checkers 12 | 13 | import spear.{LoggingFunSuite, TestUtils} 14 | import spear.generators.genRandomPartitions 15 | import spear.trees.TreeNodeSuite.Node 16 | 17 | class TreeNodeSuite extends LoggingFunSuite with TestUtils with Checkers { 18 | def genNode: Gen[Node] = Gen.parameterized { param => 19 | param.size match { 20 | case size if size < 2 => Node(1, Nil) 21 | case size => 22 | for { 23 | width <- Gen choose (1, size - 1) 24 | childrenSizes <- genRandomPartitions(size - 1, width) 25 | children <- Gen sequence (childrenSizes map { Gen.resize(_, genNode) }) 26 | } yield Node(1, children.asScala) 27 | } 28 | } 29 | 30 | implicit val arbNode = Arbitrary(genNode) 31 | 32 | implicit val shrinkNode: Shrink[Node] = Shrink { 33 | case node if node.isLeaf => Empty 34 | case node => node.children.toStream :+ node.stripLeaves 35 | } 36 | 37 | implicit def prettyNode(tree: Node): Pretty = Pretty { 38 | _ => "\n" + tree.prettyTree 39 | } 40 | 41 | test("transformDown") { 42 | check { 43 | (_: Node) transformDown { 44 | case node @ Node(_, children) => 45 | node.copy(value = children.map(_.value).sum) 46 | } forall { 47 | case Node(value, Nil) => value == 0 48 | case Node(value, children) => value == children.size 49 | } 50 | } 51 | } 52 | 53 | test("transformUp") { 54 | check { 55 | (_: Node) transformUp { 56 | case n @ Node(_, children) => 57 | n.copy(value = children.map(_.value).sum) 58 | } forall { 59 | case Node(value, _) => value == 0 60 | } 61 | } 62 | } 63 | 64 | test("collect") { 65 | check { tree: Node => 66 | val even = tree collectDown { case n if n.children.size % 2 == 0 => n } 67 | val odd = tree collectDown { case n if n.children.size % 2 == 1 => n } 68 | val nodes = tree collectDown { case n => n } 69 | 70 | all( 71 | "all nodes should be collected" |: 72 | (nodes.size == tree.size), 73 | 74 | "a node can't be both even and odd" |: 75 | (even intersect odd).isEmpty, 76 | 77 | "a node must be either even or odd" |: 78 | (even.size + odd.size == nodes.size), 79 | 80 | "even nodes should be even" |: 81 | (even forall (_.children.size % 2 == 0)), 82 | 83 | "odd nodes should be odd" |: 84 | (odd forall (_.children.size % 2 == 1)) 85 | ) 86 | } 87 | } 88 | 89 | test("forall") { 90 | check { tree: Node => 91 | all( 92 | "the generator we are using only generates nodes with value 1" |: 93 | tree.forall(_.value == 1), 94 | 95 | "for trees that has more than 1 node, there must be non-leaf nodes" |: 96 | (tree.size > 1) ==> !tree.forall(_.isLeaf) 97 | ) 98 | } 99 | } 100 | 101 | test("exists") { 102 | check { tree: Node => 103 | all( 104 | "a tree must have leaf node(s)" |: 105 | tree.exists(_.isLeaf), 106 | 107 | "the generator we are using only generates nodes with value 1" |: 108 | !tree.exists(_.value == 2) 109 | ) 110 | } 111 | } 112 | 113 | test("size") { 114 | check { tree: Node => 115 | tree.size == (tree collectDown { case n => n.value }).sum 116 | } 117 | } 118 | 119 | test("depth") { 120 | check { tree: Node => 121 | // Computes the depth by iterating over the tree and removing all the leaf nodes during each 122 | // iteration until only the root node is left. 123 | val iterations = iterate(tree) { 124 | _ transformDown { case n => n.stripLeaves } 125 | } takeWhile (_.size > 1) 126 | 127 | tree.depth == iterations.size + 1 128 | } 129 | } 130 | 131 | test("withChildren") { 132 | val children = (0 until 3) map { Node(_, Nil) } 133 | val node = Node(3, children) 134 | assert(node.withChildren(children) == node) 135 | } 136 | } 137 | 138 | object TreeNodeSuite { 139 | case class Node(value: Int, children: Seq[Node]) extends TreeNode[Node] { 140 | override def caption: String = s"Node($value)" 141 | 142 | def stripLeaves: Node = 143 | if (children forall (!_.isLeaf)) this else copy(children = children filterNot (_.isLeaf)) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /spear-core/src/test/scala/spear/types/DataTypeSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.types 2 | 3 | import scala.language.implicitConversions 4 | 5 | import org.scalacheck.util.Pretty 6 | import org.scalatest.prop.Checkers 7 | 8 | import spear._ 9 | import spear.exceptions.TypeMismatchException 10 | import spear.expressions._ 11 | import spear.generators.types._ 12 | 13 | class DataTypeSuite extends LoggingFunSuite with TestUtils with Checkers { 14 | // ScalaCheck pretty printing support for `DataType` 15 | private implicit def prettyDataType(dataType: DataType): Pretty = Pretty { 16 | _ => "\n" + dataType.prettyTree 17 | } 18 | 19 | test("size of generated DataType") { 20 | check { t: PrimitiveType => 21 | t.size == 1 22 | } 23 | 24 | check { t: ArrayType => 25 | t.size == 1 + t.elementType.size 26 | } 27 | 28 | check { t: MapType => 29 | t.size == 1 + t.keyType.size + t.valueType.size 30 | } 31 | } 32 | 33 | test("depth of generated DataType") { 34 | check { t: PrimitiveType => 35 | t.depth == 1 36 | } 37 | 38 | check { t: ArrayType => 39 | t.depth == 1 + t.elementType.depth 40 | } 41 | 42 | check { t: MapType => 43 | t.depth == 1 + (t.keyType.depth max t.valueType.depth) 44 | } 45 | } 46 | 47 | test("ArrayType instantiation") { 48 | checkTree( 49 | ArrayType(IntType, isElementNullable = true), 50 | ArrayType(IntType) 51 | ) 52 | 53 | checkTree( 54 | ArrayType(IntType, isElementNullable = false), 55 | ArrayType(IntType.!) 56 | ) 57 | } 58 | 59 | test("MapType instantiation") { 60 | checkTree( 61 | MapType(IntType, StringType, isValueNullable = true), 62 | MapType(IntType, StringType) 63 | ) 64 | 65 | checkTree( 66 | MapType(IntType, StringType, isValueNullable = false), 67 | MapType(IntType, StringType.!) 68 | ) 69 | } 70 | 71 | test("StructType instantiation") { 72 | checkTree( 73 | StructType(StructField('f1, IntType, isNullable = false) :: Nil), 74 | StructType('f1 -> IntType.!) 75 | ) 76 | 77 | checkTree( 78 | StructType(Seq( 79 | StructField('f1, IntType, isNullable = true), 80 | StructField('f2, DoubleType, isNullable = false) 81 | )), 82 | StructType( 83 | 'f1 -> IntType, 84 | 'f2 -> DoubleType.! 85 | ) 86 | ) 87 | } 88 | 89 | test("StructType.rename") { 90 | checkTree( 91 | StructType( 92 | 'f1 -> IntType.!, 93 | 'f2 -> StringType.? 94 | ) rename ("c1", "c2"), 95 | StructType( 96 | 'c1 -> IntType.!, 97 | 'c2 -> StringType.? 98 | ) 99 | ) 100 | } 101 | 102 | test("StructType.toAttribute") { 103 | def resetID(attribute: Attribute): Attribute = attribute withID ExpressionID(0L) 104 | 105 | assertResult(Seq('f1.int.!, 'f2.string.?) map resetID) { 106 | StructType( 107 | "f1" -> IntType.!, 108 | "f2" -> StringType.? 109 | ).toAttributes map resetID 110 | } 111 | } 112 | 113 | private val testSchema = 114 | StructType( 115 | 'name -> StringType.!, 116 | 'age -> IntType, 117 | 'gender -> StringType, 118 | 'location -> StructType( 119 | 'latitude -> DoubleType.!, 120 | 'longitude -> DoubleType.! 121 | ), 122 | i"phone-numbers" -> ArrayType(StringType.!), 123 | 'addresses -> MapType(StringType, StringType.!) 124 | ) 125 | 126 | test("StructType field types") { 127 | assertResult(IntType :: DoubleType :: Nil) { 128 | StructType( 129 | 'f0 -> IntType.!, 130 | 'f1 -> DoubleType 131 | ).fieldTypes 132 | } 133 | } 134 | 135 | test("pretty tree string of a StructType") { 136 | assertSideBySide( 137 | """struct 138 | |├╴name: string 139 | |├╴age: int? 140 | |├╴gender: string? 141 | |├╴location: struct? 142 | |│ ├╴latitude: double 143 | |│ └╴longitude: double 144 | |├╴phone-numbers: array? 145 | |│ └╴element: string 146 | |└╴addresses: map? 147 | | ├╴key: string 148 | | └╴value: string 149 | |""".stripMargin.trim, 150 | testSchema.prettyTree 151 | ) 152 | } 153 | 154 | test("orderable ArrayType") { 155 | val arrayType = ArrayType(IntType) 156 | assert(arrayType isSubtypeOf OrderedType) 157 | } 158 | 159 | test("non-orderable ArrayType") { 160 | val arrayType = ArrayType(MapType(IntType, StringType)) 161 | assert(!(arrayType isSubtypeOf OrderedType)) 162 | } 163 | 164 | test("orderable StructType") { 165 | val schema = StructType( 166 | 'f0 -> IntType.!, 167 | 'f1 -> DoubleType.! 168 | ) 169 | 170 | assert(schema isSubtypeOf OrderedType) 171 | } 172 | 173 | test("non-orderable StructType") { 174 | val schema = StructType( 175 | 'f0 -> IntType.!, 176 | 'f1 -> MapType(IntType, StringType) 177 | ) 178 | 179 | assert(!(schema isSubtypeOf OrderedType)) 180 | } 181 | 182 | test("ordering") { 183 | OrderedType.orderingOf(IntType) 184 | 185 | intercept[TypeMismatchException] { 186 | OrderedType.orderingOf(MapType(IntType, StringType)) 187 | } 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /spear-docs/src/sphinx/_static/.git-keep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liancheng/spear/66216aa44f5c5d42a39addb0a6f3ccff6901fa40/spear-docs/src/sphinx/_static/.git-keep -------------------------------------------------------------------------------- /spear-docs/src/sphinx/index.rst: -------------------------------------------------------------------------------- 1 | Spear documentation 2 | =================== 3 | 4 | Table of Contents: 5 | 6 | .. toctree:: 7 | :numbered: 8 | :maxdepth: 2 9 | 10 | overview 11 | -------------------------------------------------------------------------------- /spear-docs/src/sphinx/overview.rst: -------------------------------------------------------------------------------- 1 | Overview 2 | ======== 3 | 4 | Spear is a SQL query engine that roots from Spark SQL. It's mostly a playground for the author to experiment and verify various ideas that may or may not be applicable to Spark SQL. 5 | 6 | Spear implements both a subset of ANSI SQL 2006 and a Scala DSL (similar to the untyped DataFrame API provided by Spark SQL). Currently, Spear mostly focuses on the frontend and only provides a fairly trivial physical execution engine that only handles local Scala collections, mostly a PoC for validating the whole pipeline. 7 | 8 | Quick start 9 | ----------- 10 | 11 | Building Spear is as easy as:: 12 | 13 | $ ./build/sbt package 14 | 15 | Run the REPL:: 16 | 17 | $ ./build/sbt repl 18 | -------------------------------------------------------------------------------- /spear-examples/src/main/scala/BasicExample.scala: -------------------------------------------------------------------------------- 1 | import spear.Context 2 | import spear.config.Settings 3 | import spear.expressions._ 4 | import spear.expressions.functions._ 5 | 6 | object BasicExample { 7 | case class Person(name: String, gender: String, age: Int) 8 | 9 | def main(args: Array[String]) { 10 | val context = new Context(Settings.load()) 11 | 12 | import context._ 13 | 14 | val people = lift( 15 | Person("Alice", "F", 9), 16 | Person("Bob", "M", 15), 17 | Person("Charlie", "M", 18), 18 | Person("David", "M", 13), 19 | Person("Eve", "F", 20), 20 | Person("Frank", "M", 19) 21 | ) 22 | 23 | val adults = people filter 'age >= 18 select ('name, 'gender) 24 | adults.explain() 25 | adults.show() 26 | 27 | val countGender = people groupBy 'gender agg ('gender, count() as 'count) 28 | countGender.explain() 29 | countGender.show() 30 | 31 | people.asTable('people) 32 | 33 | val adultsSQL = sql( 34 | """SELECT name, gender 35 | |FROM people 36 | |WHERE age >= 18 37 | |""".stripMargin 38 | ) 39 | 40 | adultsSQL.explain() 41 | adultsSQL.show() 42 | 43 | val countGenderSQL = sql( 44 | """SELECT gender, max(age), count(*) 45 | |FROM people 46 | |GROUP BY gender 47 | |HAVING gender = 'M' 48 | |""".stripMargin 49 | ) 50 | 51 | countGenderSQL.explain() 52 | countGenderSQL.show() 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /spear-local/src/main/scala/spear/local/LocalQueryCompiler.scala: -------------------------------------------------------------------------------- 1 | package spear.local 2 | 3 | import spear._ 4 | import spear.local.plans.physical 5 | import spear.local.plans.physical.HashAggregate 6 | import spear.local.plans.physical.dsl._ 7 | import spear.plans.QueryPlanner 8 | import spear.plans.logical._ 9 | import spear.plans.physical.{NotImplemented, PhysicalPlan} 10 | 11 | class LocalQueryCompiler extends BasicQueryCompiler { 12 | override def plan(plan: LogicalPlan): PhysicalPlan = planner apply plan 13 | 14 | private val planner = new LocalQueryPlanner 15 | } 16 | 17 | class LocalQueryPlanner extends QueryPlanner[LogicalPlan, PhysicalPlan] { 18 | override def strategies: Seq[Strategy] = Seq( 19 | BasicOperators 20 | ) 21 | 22 | object BasicOperators extends Strategy { 23 | override def apply(logicalPlan: LogicalPlan): Seq[PhysicalPlan] = logicalPlan match { 24 | case relation @ LocalRelation(data, _) => 25 | physical.LocalRelation(data, relation.output) :: Nil 26 | 27 | case child Project projectList => 28 | (planLater(child) select projectList) :: Nil 29 | 30 | case Aggregate(child, keys, functions) => 31 | HashAggregate(planLater(child), keys, functions) :: Nil 32 | 33 | case child Filter condition => 34 | (planLater(child) filter condition) :: Nil 35 | 36 | case child Limit n => 37 | (planLater(child) limit n) :: Nil 38 | 39 | case Join(left, right, Inner, Some(condition)) => 40 | (planLater(left) cartesianJoin planLater(right) on condition) :: Nil 41 | 42 | case Join(left, right, Inner, _) => 43 | (planLater(left) cartesianJoin planLater(right)) :: Nil 44 | 45 | case child Sort order => 46 | (planLater(child) orderBy order) :: Nil 47 | 48 | case child Subquery _ => 49 | planLater(child) :: Nil 50 | 51 | case SingleRowRelation => 52 | spear.plans.physical.SingleRowRelation :: Nil 53 | 54 | case left Union right => 55 | (planLater(left) union planLater(right)) :: Nil 56 | 57 | case left Intersect right => 58 | (planLater(left) intersect planLater(right)) :: Nil 59 | 60 | case left Except right => 61 | (planLater(left) except planLater(right)) :: Nil 62 | 63 | case plan => 64 | NotImplemented(plan.nodeName.toString, plan.children map planLater, plan.output) :: Nil 65 | } 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /spear-local/src/main/scala/spear/local/plans/physical/basicOperators.scala: -------------------------------------------------------------------------------- 1 | package spear.local.plans.physical 2 | 3 | import scala.collection.mutable.ArrayBuffer 4 | 5 | import spear._ 6 | import spear.execution.MutableProjection 7 | import spear.expressions._ 8 | import spear.expressions.Literal.True 9 | import spear.plans.logical.annotations.Explain 10 | import spear.plans.physical.{BinaryPhysicalPlan, LeafPhysicalPlan, PhysicalPlan, UnaryPhysicalPlan} 11 | import spear.trees.TreeNode 12 | 13 | case class LocalRelation( 14 | @Explain(hidden = true) data: Iterable[Row], 15 | @Explain(hidden = true) override val output: Seq[Attribute] 16 | ) extends LeafPhysicalPlan { 17 | 18 | override def iterator: Iterator[Row] = data.iterator 19 | 20 | override protected def nestedTrees: Seq[TreeNode[_]] = Nil 21 | } 22 | 23 | case class Project(child: PhysicalPlan, projectList: Seq[NamedExpression]) 24 | extends UnaryPhysicalPlan { 25 | 26 | override lazy val output: Seq[Attribute] = projectList map { _.attr } 27 | 28 | override def requireMaterialization: Boolean = true 29 | 30 | private lazy val projection = MutableProjection(projectList map { _ bindTo child.output }) 31 | 32 | override def iterator: Iterator[Row] = child.iterator map projection 33 | } 34 | 35 | case class Filter(child: PhysicalPlan, condition: Expression) extends UnaryPhysicalPlan { 36 | override lazy val output: Seq[Attribute] = child.output 37 | 38 | private lazy val boundCondition = condition bindTo child.output 39 | 40 | override def iterator: Iterator[Row] = child.iterator filter { 41 | boundCondition.evaluate(_).asInstanceOf[Boolean] 42 | } 43 | } 44 | 45 | case class Limit(child: PhysicalPlan, limit: Expression) extends UnaryPhysicalPlan { 46 | override lazy val output: Seq[Attribute] = child.output 47 | 48 | override def iterator: Iterator[Row] = child.iterator take limit.evaluated.asInstanceOf[Int] 49 | } 50 | 51 | case class Union(left: PhysicalPlan, right: PhysicalPlan) extends BinaryPhysicalPlan { 52 | override lazy val output: Seq[Attribute] = 53 | left.output zip right.output map { 54 | case (a1, a2) => 55 | a1.nullable(a1.isNullable || a2.isNullable) 56 | } 57 | 58 | override def iterator: Iterator[Row] = left.iterator ++ right.iterator 59 | } 60 | 61 | case class Intersect(left: PhysicalPlan, right: PhysicalPlan) extends BinaryPhysicalPlan { 62 | override lazy val output: Seq[Attribute] = 63 | left.output zip right.output map { 64 | case (a1, a2) => 65 | a1.nullable(a1.isNullable && a2.isNullable) 66 | } 67 | 68 | override def iterator: Iterator[Row] = 69 | (left.iterator.toSeq intersect right.iterator.toSeq).iterator 70 | } 71 | 72 | case class Except(left: PhysicalPlan, right: PhysicalPlan) extends BinaryPhysicalPlan { 73 | override lazy val output: Seq[Attribute] = left.output 74 | 75 | override def iterator: Iterator[Row] = (left.iterator.toSeq diff right.iterator.toSeq).iterator 76 | } 77 | 78 | case class CartesianJoin( 79 | left: PhysicalPlan, 80 | right: PhysicalPlan, 81 | condition: Option[Expression] 82 | ) extends BinaryPhysicalPlan { 83 | def evaluateBoundCondition(input: Row): Boolean = 84 | boundCondition evaluate input match { case result: Boolean => result } 85 | 86 | override def output: Seq[Attribute] = left.output ++ right.output 87 | 88 | override def iterator: Iterator[Row] = for { 89 | leftRow <- left.iterator 90 | rightRow <- right.iterator if evaluateBoundCondition(join(leftRow, rightRow)) 91 | } yield join 92 | 93 | override def requireMaterialization: Boolean = true 94 | 95 | def on(condition: Expression): CartesianJoin = copy(condition = Some(condition)) 96 | 97 | private lazy val boundCondition = condition map { _ bindTo output } getOrElse True 98 | 99 | private val join = new JoinedRow() 100 | } 101 | 102 | case class Sort(child: PhysicalPlan, order: Seq[SortOrder]) extends UnaryPhysicalPlan { 103 | override lazy val output: Seq[Attribute] = child.output 104 | 105 | private lazy val rowOrdering = new RowOrdering(order, child.output) 106 | 107 | override def iterator: Iterator[Row] = { 108 | val buffer = ArrayBuffer.empty[Row] 109 | child.iterator foreach { buffer += _.copy() } 110 | buffer.sorted(rowOrdering).iterator 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /spear-local/src/main/scala/spear/local/plans/physical/dsl/package.scala: -------------------------------------------------------------------------------- 1 | package spear.local.plans.physical 2 | 3 | import spear.expressions._ 4 | import spear.expressions.functions._ 5 | import spear.plans.physical.PhysicalPlan 6 | 7 | package object dsl { 8 | implicit class PhysicalPlanDSL(plan: PhysicalPlan) { 9 | def select(projectList: Seq[NamedExpression]): Project = Project(plan, projectList) 10 | 11 | def select(first: NamedExpression, rest: NamedExpression*): Project = select(first +: rest) 12 | 13 | def filter(condition: Expression): Filter = Filter(plan, condition) 14 | 15 | def limit(n: Expression): Limit = Limit(plan, n) 16 | 17 | def limit(n: Int): Limit = limit(lit(n)) 18 | 19 | def orderBy(order: Seq[SortOrder]): Sort = Sort(plan, order) 20 | 21 | def orderBy(first: SortOrder, rest: SortOrder*): Sort = plan orderBy (first +: rest) 22 | 23 | def cartesianJoin(that: PhysicalPlan): CartesianJoin = CartesianJoin(plan, that, None) 24 | 25 | def union(that: PhysicalPlan): Union = Union(plan, that) 26 | 27 | def intersect(that: PhysicalPlan): Intersect = Intersect(plan, that) 28 | 29 | def except(that: PhysicalPlan): Except = Except(plan, that) 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /spear-local/src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger = TRACE, file 2 | 3 | # File appender 4 | log4j.appender.file=org.apache.log4j.FileAppender 5 | log4j.appender.file.threshold=TRACE 6 | log4j.appender.file.append=false 7 | log4j.appender.file.file=spear-local/target/unit-tests.log 8 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 10 | -------------------------------------------------------------------------------- /spear-local/src/test/scala/spear/local/plans/physical/LocalPhysicalPlanSuite.scala: -------------------------------------------------------------------------------- 1 | package spear.local.plans.physical 2 | 3 | import spear.{LoggingFunSuite, Row, TestUtils} 4 | import spear.expressions._ 5 | import spear.local.plans.physical.dsl._ 6 | import spear.plans.physical.{PhysicalPlan, SingleRowRelation} 7 | 8 | class LocalPhysicalPlanSuite extends LoggingFunSuite with TestUtils { 9 | private val Seq(a1, b1) = Seq('a.int.!, 'b.string.?) 10 | 11 | private val r1 = LocalRelation( 12 | Seq(Row(1, "a"), Row(2, "b")), 13 | Seq(a1, b1) 14 | ) 15 | 16 | private val Seq(a2, b2) = Seq('a.int.!, 'b.string.?) 17 | 18 | private val r2 = LocalRelation( 19 | Seq(Row(1, "a"), Row(3, "c")), 20 | Seq(a2, b2) 21 | ) 22 | 23 | private val Seq(a3, b3) = Seq('a.int.!, 'b.string.?) 24 | 25 | private val r3 = LocalRelation( 26 | Seq( 27 | Row(1: Integer, "a"), 28 | Row(3: Integer, "c"), 29 | Row(null: Integer, "b"), 30 | Row(4: Integer, null) 31 | ), 32 | Seq(a3, b3) 33 | ) 34 | 35 | def checkPhysicalPlan(plan: PhysicalPlan, expectedRows: Traversable[Row]): Unit = { 36 | val planOrdered = plan.collectFirstDown { case _: Sort => }.nonEmpty 37 | 38 | if (planOrdered) { 39 | assert(plan.iterator.toSeq == expectedRows.toSeq) 40 | } else { 41 | val sort = Sort(_: PhysicalPlan, plan.output map { _.asc }) 42 | val actual = sort(plan).iterator.toSeq 43 | val expected = sort(LocalRelation(expectedRows.toIterable, plan.output)).iterator.toSeq 44 | assert(actual == expected) 45 | } 46 | } 47 | 48 | def checkPhysicalPlan(plan: PhysicalPlan, first: Row, rest: Row*): Unit = { 49 | checkPhysicalPlan(plan, first +: rest) 50 | } 51 | 52 | test("project") { 53 | checkPhysicalPlan( 54 | SingleRowRelation select (1 as 'a), 55 | Row(1) 56 | ) 57 | 58 | checkPhysicalPlan( 59 | r1 select (a1 + 1 as 'f), 60 | Row(2), Row(3) 61 | ) 62 | } 63 | 64 | test("filter") { 65 | checkPhysicalPlan( 66 | r1 filter a1 > 1, 67 | Row(2, "b") 68 | ) 69 | } 70 | 71 | test("limit") { 72 | checkPhysicalPlan( 73 | r1 limit 0, 74 | Nil 75 | ) 76 | 77 | checkPhysicalPlan( 78 | r1 limit 1, 79 | Row(1, "a") 80 | ) 81 | 82 | checkPhysicalPlan( 83 | r1 limit 3, 84 | Row(1, "a"), 85 | Row(2, "b") 86 | ) 87 | } 88 | 89 | test("sort") { 90 | checkPhysicalPlan( 91 | r1 orderBy a1.desc, 92 | Row(2, "b"), Row(1, "a") 93 | ) 94 | 95 | checkPhysicalPlan( 96 | r1 orderBy a1.asc, 97 | Row(1, "a"), Row(2, "b") 98 | ) 99 | 100 | checkPhysicalPlan( 101 | r3 orderBy a3.asc.nullsFirst, 102 | Row(null: Integer, "b"), 103 | Row(1: Integer, "a"), 104 | Row(3: Integer, "c"), 105 | Row(4: Integer, null) 106 | ) 107 | 108 | checkPhysicalPlan( 109 | r3 orderBy a3.asc.nullsLast, 110 | Row(1: Integer, "a"), 111 | Row(3: Integer, "c"), 112 | Row(4: Integer, null), 113 | Row(null: Integer, "b") 114 | ) 115 | 116 | checkPhysicalPlan( 117 | r3 orderBy b3.desc.nullsFirst, 118 | Row(4: Integer, null), 119 | Row(3: Integer, "c"), 120 | Row(null: Integer, "b"), 121 | Row(1: Integer, "a") 122 | ) 123 | 124 | checkPhysicalPlan( 125 | r3 orderBy b3.desc.nullsLast, 126 | Row(3: Integer, "c"), 127 | Row(null: Integer, "b"), 128 | Row(1: Integer, "a"), 129 | Row(4: Integer, null) 130 | ) 131 | } 132 | 133 | test("union") { 134 | checkPhysicalPlan( 135 | r1 union r2, 136 | Row(1, "a"), 137 | Row(2, "b"), 138 | Row(1, "a"), 139 | Row(3, "c") 140 | ) 141 | } 142 | 143 | test("intersect") { 144 | checkPhysicalPlan( 145 | r1 intersect r2, 146 | Row(1, "a") 147 | ) 148 | } 149 | 150 | test("except") { 151 | checkPhysicalPlan( 152 | r1 except r2, 153 | Row(2, "b") 154 | ) 155 | } 156 | 157 | test("cartesian join") { 158 | checkPhysicalPlan( 159 | r1 cartesianJoin r2, 160 | Row(1, "a", 1, "a"), 161 | Row(1, "a", 3, "c"), 162 | Row(2, "b", 1, "a"), 163 | Row(2, "b", 3, "c") 164 | ) 165 | 166 | checkPhysicalPlan( 167 | r1 cartesianJoin r2 on a1 === a2, 168 | Row(1, "a", 1, "a") 169 | ) 170 | 171 | checkPhysicalPlan( 172 | r1 cartesianJoin r2 on a1 > a2, 173 | Row(2, "b", 1, "a") 174 | ) 175 | } 176 | } 177 | -------------------------------------------------------------------------------- /spear-repl/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger = ERROR, console, file 2 | 3 | # Console appender 4 | log4j.appender.console=org.apache.log4j.ConsoleAppender 5 | log4j.appender.console.threshold=ALL 6 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 7 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 8 | 9 | # File appender 10 | log4j.appender.file=org.apache.log4j.FileAppender 11 | log4j.appender.file.threshold=TRACE 12 | log4j.appender.file.append=false 13 | log4j.appender.file.file=spear-repl/logs/spear-repl.log 14 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 15 | log4j.appender.file.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 16 | -------------------------------------------------------------------------------- /spear-repl/src/main/resources/predef.scala: -------------------------------------------------------------------------------- 1 | import org.apache.log4j.{Level, Logger} 2 | 3 | import spear.Context 4 | import spear.config.Settings 5 | import spear.expressions._ 6 | import spear.expressions.dsl._ 7 | import spear.expressions.functions._ 8 | import spear.expressions.windows._ 9 | import spear.Row 10 | import spear.repl.Main.% 11 | import spear.types._ 12 | 13 | implicit val context = new Context(Settings.load("spear.conf", "spear-reference.conf")) 14 | 15 | import context._ 16 | 17 | def setLogLevel(level: String) { 18 | Logger.getRootLogger.setLevel(Level.toLevel(level)) 19 | } 20 | -------------------------------------------------------------------------------- /spear-repl/src/main/resources/spear-full-reference.conf: -------------------------------------------------------------------------------- 1 | spear { 2 | language { 3 | case-sensitive = true 4 | 5 | # Whether nulls appear before or after non-null values in sort ordering. By default, null value 6 | # sort as if larger than any non-null value. 7 | nulls-larger = true 8 | } 9 | 10 | test { 11 | types { 12 | allow-null-type = true 13 | 14 | allow-empty-struct-type = true 15 | 16 | allow-nullable-complex-type = true 17 | 18 | allow-nullable-array-type = ${spear.test.types.allow-nullable-complex-type} 19 | 20 | allow-nullable-map-type = ${spear.test.types.allow-nullable-complex-type} 21 | 22 | allow-nullable-struct-field = ${spear.test.types.allow-nullable-complex-type} 23 | 24 | allow-nested-struct-type = false 25 | 26 | max-struct-type-width = 4 27 | } 28 | 29 | expressions { 30 | max-repetition = 8 31 | 32 | chances { 33 | # Chance of null values 34 | null = 0 35 | } 36 | 37 | only-logical-operators-in-predicate = false 38 | } 39 | 40 | plans { 41 | max-join-num = 1 42 | 43 | max-project-width = 4 44 | 45 | max-limit = 10 46 | 47 | max-expression-size = 8 48 | 49 | max-where-predicate-size = ${spear.test.plans.max-expression-size} 50 | 51 | max-select-expression-size = ${spear.test.plans.max-expression-size} 52 | 53 | chances { 54 | select-clause = 0.8 55 | 56 | from-clause = 0.5 57 | 58 | where-clause = 0.5 59 | 60 | limit-clause = 0.5 61 | 62 | subquery = 0.2 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /spear-repl/src/main/scala/spear/repl/Main.scala: -------------------------------------------------------------------------------- 1 | package spear.repl 2 | 3 | import ammonite.ops.{read, Path, ResourcePath} 4 | import ammonite.runtime.Storage 5 | import scopt._ 6 | 7 | import spear.Context 8 | import spear.utils._ 9 | 10 | object Main { 11 | object % { 12 | def sql(query: String)(implicit context: Context): Unit = 13 | context.sql(query).show(rowCount = None, truncate = false, out = System.out) 14 | } 15 | 16 | case class Config(remoteLogging: Boolean = true) 17 | 18 | def main(args: Array[String]) { 19 | val optionParser = new OptionParser[Config]("spear-shell") { 20 | opt[Unit]("no-remote-logging") 21 | .text("Disable remote logging of the number of times a REPL starts and runs command") 22 | .action { (_, config) => 23 | config.copy(remoteLogging = false) 24 | } 25 | } 26 | 27 | for (Config(remoteLoggingEnabled) <- optionParser.parse(args, Config())) { 28 | ammonite.Main( 29 | predefCode = read(ResourcePath.resource(this.getClass.getClassLoader) / "predef.scala"), 30 | storageBackend = new Storage.Folder(Path.home / ".spear"), 31 | welcomeBanner = Some(banner(remoteLoggingEnabled)), 32 | remoteLogging = remoteLoggingEnabled 33 | ).run() 34 | } 35 | } 36 | 37 | private val scalaVersion = scala.util.Properties.versionNumberString 38 | 39 | private val javaVersion = System.getProperty("java.version") 40 | 41 | private def banner(remoteLoggingEnabled: Boolean): String = { 42 | val banner = 43 | raw"""Welcome to 44 | | ____ 45 | | / __/__ ___ ___ _____ 46 | | _\ \/ _ \/ -_) _ `/ __/ 47 | |/___/ .__/\__/\_,_/_/ 48 | | /_/ 49 | | 50 | |Scala version: $scalaVersion 51 | |Java version: $javaVersion 52 | | 53 | |The default context object is available as `context'. 54 | |""".stripMargin 55 | 56 | val remoteLoggingNote = 57 | s"""NOTE: This project uses the Ammonite REPL. Ammonite anonymously logs the times you start a 58 | |REPL session and the count of how many commands get run. This is to allow the author of 59 | |Ammonite to understand usage patterns and prioritize improvements. If you wish to disable 60 | |it, pass in the "--no-remote-logging" command-line flag. 61 | |""".oneLine 62 | 63 | Seq(banner) ++ Option(remoteLoggingNote).filter { _ => remoteLoggingEnabled } mkString "\n" 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /spear-trees/src/main/scala/spear/trees/Transformer.scala: -------------------------------------------------------------------------------- 1 | package spear.trees 2 | 3 | import scala.annotation.tailrec 4 | 5 | import spear.utils.{sideBySide, Logging} 6 | 7 | trait RuleLike[Base <: TreeNode[Base]] extends (Base => Base) with Logging { 8 | protected def logTransformation(action: String, before: Base)(tree: => Base): Base = { 9 | val after = tree 10 | 11 | if (!before.same(after)) { 12 | logTrace { 13 | val diff = sideBySide( 14 | s"""Before $action 15 | |${before.prettyTree} 16 | |""".stripMargin, 17 | 18 | s"""After $action 19 | |${after.prettyTree} 20 | |""".stripMargin, 21 | 22 | withHeader = true 23 | ) 24 | 25 | s"""Applied $action 26 | |$diff 27 | |""".stripMargin 28 | } 29 | } 30 | 31 | after 32 | } 33 | } 34 | 35 | trait Rule[Base <: TreeNode[Base]] extends RuleLike[Base] { 36 | private val name = getClass.getSimpleName stripSuffix "$" 37 | 38 | override final def apply(tree: Base): Base = logTransformation(s"rule '$name'", tree) { 39 | transform(tree) 40 | } 41 | 42 | def transform(tree: Base): Base 43 | } 44 | 45 | case class RuleGroup[Base <: TreeNode[Base]]( 46 | convergenceTest: ConvergenceTest, 47 | rules: Seq[Base => Base] 48 | ) extends RuleLike[Base] { 49 | 50 | require(rules.nonEmpty) 51 | 52 | override def apply(tree: Base): Base = converge(tree) 53 | 54 | @tailrec private def converge(before: Base): Base = { 55 | val after = composedRules(before) 56 | if (convergenceTest.test(before, after)) after else converge(after) 57 | } 58 | 59 | private val composedRules = rules reduce { _ andThen _ } 60 | } 61 | 62 | class Transformer[Base <: TreeNode[Base]](rules: Seq[Base => Base]) extends RuleLike[Base] { 63 | require(rules.nonEmpty) 64 | 65 | def this(first: RuleGroup[Base], rest: RuleGroup[Base]*) = this(first +: rest) 66 | 67 | def apply(tree: Base): Base = composedRules(tree) 68 | 69 | private val composedRules = rules reduce { _ andThen _ } 70 | } 71 | 72 | sealed trait ConvergenceTest { 73 | def test[Base <: TreeNode[Base]](before: Base, after: Base): Boolean 74 | } 75 | 76 | case object Once extends ConvergenceTest { 77 | def test[Base <: TreeNode[Base]](before: Base, after: Base): Boolean = true 78 | } 79 | 80 | case object FixedPoint extends ConvergenceTest { 81 | def test[Base <: TreeNode[Base]](before: Base, after: Base): Boolean = before same after 82 | } 83 | -------------------------------------------------------------------------------- /spear-utils/src/main/scala/spear/utils/Logging.scala: -------------------------------------------------------------------------------- 1 | package spear.utils 2 | 3 | import org.slf4j.{Logger, LoggerFactory} 4 | 5 | trait Logging { 6 | protected val logger: Logger = LoggerFactory.getLogger(getClass) 7 | 8 | protected def logTrace(message: => String): Unit = { 9 | if (logger.isTraceEnabled) { 10 | logger.trace(message) 11 | } 12 | } 13 | 14 | protected def logDebug(message: => String): Unit = { 15 | if (logger.isDebugEnabled) { 16 | logger.debug(message) 17 | } 18 | } 19 | 20 | protected def logInfo(message: => String): Unit = { 21 | if (logger.isInfoEnabled) { 22 | logger.info(message) 23 | } 24 | } 25 | 26 | protected def logWarn(message: => String): Unit = { 27 | if (logger.isWarnEnabled) { 28 | logger.warn(message) 29 | } 30 | } 31 | 32 | protected def logError(message: => String): Unit = { 33 | if (logger.isErrorEnabled) { 34 | logger.error(message) 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /spear-utils/src/main/scala/spear/utils/package.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import scala.util.Try 4 | 5 | package object utils { 6 | def sideBySide(lhs: String, rhs: String, withHeader: Boolean): String = { 7 | val lhsLines = lhs split "\n" 8 | val rhsLines = rhs split "\n" 9 | 10 | val lhsWidth = lhsLines.map { _.length }.max 11 | val rhsWidth = rhsLines.map { _.length }.max 12 | 13 | val height = lhsLines.length max rhsLines.length 14 | val paddedLhs = lhsLines map { _ padTo (lhsWidth, ' ') } padTo (height, " " * lhsWidth) 15 | val paddedRhs = rhsLines map { _ padTo (rhsWidth, ' ') } padTo (height, " " * rhsWidth) 16 | 17 | val zipped = paddedLhs zip paddedRhs 18 | val header = if (withHeader) zipped.headOption else None 19 | val contents = if (withHeader) zipped.tail else zipped 20 | 21 | def trimRight(str: String): String = str.replaceAll("\\s+$", "") 22 | 23 | val pipe = "\u2502" 24 | 25 | header.map { 26 | case (lhsLine, rhsLine) => 27 | trimRight(s"# $lhsLine # $rhsLine") 28 | } ++ contents.map { 29 | case (lhsLine, rhsLine) => 30 | val diffIndicator = if (trimRight(lhsLine) != trimRight(rhsLine)) "!" else " " 31 | trimRight(s"$diffIndicator $lhsLine $pipe $rhsLine") 32 | } 33 | } mkString ("\n", "\n", "") 34 | 35 | implicit class OneLineString(string: String) { 36 | def oneLine: String = oneLine('|', " ") 37 | 38 | def oneLine(joiner: String): String = oneLine('|', joiner) 39 | 40 | def oneLine(marginChar: Char, joiner: String): String = 41 | ((string stripMargin marginChar).lines mkString joiner).trim 42 | } 43 | 44 | def trySequence[T](seq: Seq[Try[T]]): Try[Seq[T]] = seq match { 45 | case xs if xs.isEmpty => Try(Nil) 46 | case Seq(x, xs @ _*) => for (head <- x; tail <- trySequence(xs)) yield head +: tail 47 | } 48 | 49 | def quote(string: String): String = "'" + string.replace("\\", "\\\\").replace("'", "\\'") + "'" 50 | } 51 | -------------------------------------------------------------------------------- /spear-utils/src/test/scala/spear/LoggingFunSuite.scala: -------------------------------------------------------------------------------- 1 | package spear 2 | 3 | import org.scalatest.{FunSuite, Outcome} 4 | 5 | import spear.utils.Logging 6 | 7 | class LoggingFunSuite extends FunSuite with Logging { 8 | override protected def withFixture(test: NoArgTest): Outcome = { 9 | try { 10 | logInfo(s"Log output for test '${test.text}' {{{") 11 | test() 12 | } finally { 13 | logInfo(s"}}}\n") 14 | } 15 | } 16 | } 17 | --------------------------------------------------------------------------------