├── .github ├── release-drafter.yml └── workflows │ ├── ci.yml │ └── release-drafter.yml ├── .gitignore ├── .scalafmt.conf ├── LICENSE ├── README.md ├── build.sbt ├── core ├── sql │ └── createUDFs.sql └── src │ ├── main │ └── scala │ │ └── com │ │ └── carto │ │ └── analyticstoolbox │ │ ├── core │ │ ├── ST_AntimeridianSafeGeom.scala │ │ ├── ST_Area.scala │ │ ├── ST_AsBinary.scala │ │ ├── ST_AsGeoHash.scala │ │ ├── ST_AsGeoJson.scala │ │ ├── ST_AsLatLonText.scala │ │ ├── ST_AsTWKB.scala │ │ ├── ST_AsText.scala │ │ ├── ST_Boundary.scala │ │ ├── ST_BufferPoint.scala │ │ ├── ST_ByteArray.scala │ │ ├── ST_CastToGeometry.scala │ │ ├── ST_CastToLineString.scala │ │ ├── ST_CastToPoint.scala │ │ ├── ST_CastToPolygon.scala │ │ ├── ST_Centroid.scala │ │ ├── ST_ClosestPoint.scala │ │ ├── ST_Contains.scala │ │ ├── ST_ConvexHull.scala │ │ ├── ST_CoordDim.scala │ │ ├── ST_Covers.scala │ │ ├── ST_Crosses.scala │ │ ├── ST_Difference.scala │ │ ├── ST_Dimension.scala │ │ ├── ST_Disjoint.scala │ │ ├── ST_Distance.scala │ │ ├── ST_DistanceSphere.scala │ │ ├── ST_Envelope.scala │ │ ├── ST_Equals.scala │ │ ├── ST_ExteriorRing.scala │ │ ├── ST_GeoHash.scala │ │ ├── ST_GeomFromGeoHash.scala │ │ ├── ST_GeomFromGeoJson.scala │ │ ├── ST_GeomFromTWKB.scala │ │ ├── ST_GeomFromWKB.scala │ │ ├── ST_GeomFromWKT.scala │ │ ├── ST_GeometryN.scala │ │ ├── ST_InteriorRingN.scala │ │ ├── ST_Intersection.scala │ │ ├── ST_Intersects.scala │ │ ├── ST_IsClosed.scala │ │ ├── ST_IsCollection.scala │ │ ├── ST_IsEmpty.scala │ │ ├── ST_IsGeomField.scala │ │ ├── ST_IsRing.scala │ │ ├── ST_IsSimple.scala │ │ ├── ST_IsValid.scala │ │ ├── ST_Length.scala │ │ ├── ST_LengthSphere.scala │ │ ├── ST_MLineFromText.scala │ │ ├── ST_MPointFromText.scala │ │ ├── ST_MPolyFromText.scala │ │ ├── ST_MakeBBOX.scala │ │ ├── ST_MakeBox2D.scala │ │ ├── ST_MakeLine.scala │ │ ├── ST_MakePoint.scala │ │ ├── ST_MakePointM.scala │ │ ├── ST_MakePolygon.scala │ │ ├── ST_NumGeometries.scala │ │ ├── ST_NumPoints.scala │ │ ├── ST_Overlaps.scala │ │ ├── ST_PointFromGeoHash.scala │ │ ├── ST_PointFromText.scala │ │ ├── ST_PointFromWKB.scala │ │ ├── ST_PointN.scala │ │ ├── ST_PolygonFromText.scala │ │ ├── ST_Relate.scala │ │ ├── ST_RelateBool.scala │ │ ├── ST_Simplify.scala │ │ ├── ST_SimplifyPreserveTopology.scala │ │ ├── ST_Touches.scala │ │ ├── ST_Translate.scala │ │ ├── ST_Within.scala │ │ ├── ST_X.scala │ │ ├── ST_Y.scala │ │ ├── ST_lineFromText.scala │ │ └── package.scala │ │ ├── index │ │ ├── H3_ToParent.scala │ │ ├── ST_CrsFromText.scala │ │ ├── ST_ExtentFromGeom.scala │ │ ├── ST_ExtentToGeom.scala │ │ ├── ST_GeomReproject.scala │ │ ├── ST_MakeExtent.scala │ │ ├── ST_PartitionCentroid.scala │ │ ├── ST_Z2LatLon.scala │ │ ├── h3 │ │ │ └── H3CoreV3Producer.scala │ │ └── package.scala │ │ └── spark │ │ ├── geotrellis │ │ ├── Z2Index.scala │ │ └── encoders │ │ │ └── StandardEncoders.scala │ │ ├── spatial │ │ ├── OptimizeSpatial.scala │ │ └── package.scala │ │ └── sql │ │ ├── SpatialFilterPushdownOptimizations.scala │ │ └── rules │ │ ├── STContainsRule.scala │ │ ├── STIntersectsRule.scala │ │ └── SpatialFilterPushdownRules.scala │ └── test │ ├── resources │ ├── polygons.csv │ └── polygons.snappy.parquet │ └── scala │ └── com │ └── carto │ └── analyticstoolbox │ ├── HiveTestEnvironment.scala │ ├── InjectOptimizerTestEnvironment.scala │ ├── TestTables.scala │ ├── core │ └── STCoreSpec.scala │ ├── index │ └── STIndexSpec.scala │ └── spark │ └── sql │ └── STIndexInjectorSpec.scala └── project ├── build.properties └── plugins.sbt /.github/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name-template: '$NEXT_MINOR_VERSION' 2 | tag-template: 'v$NEXT_MINOR_VERSION' 3 | categories: 4 | - title: 'Added' 5 | labels: 6 | - 'feature' 7 | - title: 'Changed' 8 | labels: 9 | - 'enhancement' 10 | - 'dependency-update' 11 | - 'dependencies' 12 | - title: 'Fixed' 13 | labels: 14 | - 'fix' 15 | - 'bugfix' 16 | - 'bug' 17 | include-labels: 18 | - 'feature' 19 | - 'enhancement' 20 | - 'dependency-update' 21 | - 'dependencies' 22 | - 'fix' 23 | - 'bugfix' 24 | - 'bug' 25 | exclude-labels: 26 | - 'skip-changelog' 27 | - 'docs' 28 | - 'build' 29 | change-template: '- $TITLE [#$NUMBER](https://github.com/CartoDB/analytics-toolbox-databricks/pull/$NUMBER) (@$AUTHOR)' 30 | template: | 31 | $CHANGES 32 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: 3 | pull_request: 4 | branches: ['**'] 5 | push: 6 | branches: ['**'] 7 | tags: [v*] 8 | jobs: 9 | build: 10 | name: Build and Test 11 | if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != 'CartoDB/analytics-toolbox-databricks' 12 | strategy: 13 | matrix: 14 | os: [ubuntu-latest] 15 | java: [8, 11] 16 | distribution: [temurin] 17 | runs-on: ${{ matrix.os }} 18 | steps: 19 | - uses: actions/checkout@v2 20 | with: 21 | fetch-depth: 0 22 | - uses: coursier/cache-action@v6 23 | - uses: actions/setup-java@v2 24 | with: 25 | distribution: ${{ matrix.distribution }} 26 | java-version: ${{ matrix.java }} 27 | 28 | - name: Check formatting 29 | run: sbt scalafmtCheckAll 30 | 31 | - name: Build project 32 | run: sbt test 33 | 34 | publish: 35 | name: Publish Artifacts 36 | needs: [build] 37 | if: github.event_name != 'pull_request' && (github.ref == 'refs/heads/main' || startsWith(github.ref, 'refs/tags/v')) 38 | strategy: 39 | matrix: 40 | os: [ubuntu-latest] 41 | java: [8] 42 | distribution: [temurin] 43 | runs-on: ${{ matrix.os }} 44 | steps: 45 | - uses: actions/checkout@v2 46 | with: 47 | fetch-depth: 0 48 | - uses: coursier/cache-action@v6 49 | - uses: actions/setup-java@v2 50 | with: 51 | distribution: ${{ matrix.distribution }} 52 | java-version: ${{ matrix.java }} 53 | 54 | - name: Release 55 | run: sbt ci-release 56 | env: 57 | PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }} 58 | PGP_SECRET: ${{ secrets.PGP_SECRET }} 59 | SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }} 60 | SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }} 61 | if: ${{ env.SONATYPE_PASSWORD != '' && env.SONATYPE_USERNAME != '' }} 62 | -------------------------------------------------------------------------------- /.github/workflows/release-drafter.yml: -------------------------------------------------------------------------------- 1 | name: Release Drafter 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | types: [opened, reopened, synchronize] 9 | 10 | jobs: 11 | update_release_draft: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: release-drafter/release-drafter@v5.15.0 15 | env: 16 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Operating System Files 2 | 3 | *.DS_Store 4 | Thumbs.db 5 | 6 | # Build Files 7 | 8 | bin 9 | target 10 | build/ 11 | .gradle 12 | cmake-build-debug 13 | 14 | # Eclipse Project Files 15 | 16 | .classpath 17 | .project 18 | .settings 19 | 20 | # IntelliJ IDEA Files 21 | 22 | *.iml 23 | *.ipr 24 | *.iws 25 | *.idea 26 | 27 | # Spring Bootstrap artifacts 28 | 29 | dependency-reduced-pom.xml 30 | README.html 31 | 32 | # Sublime files 33 | 34 | *.sublime-workspace 35 | 36 | # VSCode files 37 | 38 | .vscode 39 | .history 40 | 41 | # Metals 42 | 43 | .metals 44 | .bloop 45 | metals.sbt 46 | 47 | # SBT 48 | 49 | .bsp 50 | 51 | # Test data files 52 | 53 | java/data 54 | 55 | # Compiled libs 56 | 57 | java/*.dylib 58 | java/*.so 59 | java/*dll 60 | 61 | # Spark files 62 | 63 | metastore_db 64 | derby.log 65 | -------------------------------------------------------------------------------- /.scalafmt.conf: -------------------------------------------------------------------------------- 1 | version = 3.4.3 2 | runner.dialect = scala212 3 | indent.main = 2 4 | indent.significant = 2 5 | maxColumn = 150 6 | indent.callSite = 2 7 | indent.defnSite = 2 8 | continuationIndent.callSite = 2 9 | continuationIndent.defnSite = 2 10 | danglingParentheses.preset = true 11 | assumeStandardLibraryStripMargin = true 12 | danglingParentheses.preset = true 13 | rewrite.rules = [SortImports, RedundantBraces, RedundantParens, SortModifiers] 14 | docstrings.style = Asterisk 15 | align.preset = more 16 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | SPDX short identifier: BSD-3-Clause 2 | 3 | Copyright (c) 2021, CARTO 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 9 | 10 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 11 | 12 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CARTO Analytics Toolbox 2 | 3 | [![CI](https://github.com/cartodb/analytics-toolbox-databricks/actions/workflows/ci.yml/badge.svg)](https://github.com/cartodb/analytics-toolbox-databricks/actions/workflows/ci.yml) 4 | [![Maven Badge](https://img.shields.io/maven-central/v/com.carto.analyticstoolbox/core_2.12?color=blue)](https://search.maven.org/search?q=g:com.carto.analyticstoolbox%20and%20core) 5 | [![Snapshots Badge](https://img.shields.io/nexus/s/https/s01.oss.sonatype.org/com.carto.analyticstoolbox/core_2.12)](https://s01.oss.sonatype.org/content/repositories/snapshots/com/carto/analyticstoolbox/core_2.12/) 6 | 7 | CARTO Analytics Toolbox for Databricks provides geospatial functionality leveraging the GeoMesa SparkSQL capabilities. It implements Spatial Hive UDFs and consists of the following modules: 8 | 9 | * `core` with Hive GIS UDFs (depends on [GeoMesa](https://github.com/locationtech/geomesa), [GeoTrellis](https://github.com/locationtech/geotrellis), and [Hiveless](https://github.com/azavea/hiveless)) 10 | 11 | ## Quick Start 12 | 13 | ```scala 14 | resolvers ++= Seq( 15 | // for snapshot artifacts only 16 | "s01-oss-sonatype" at "https://s01.oss.sonatype.org/content/repositories/snapshots" 17 | ) 18 | 19 | libraryDependencies ++= Seq( 20 | "com.carto.analyticstoolbox" %% "core" % "" 21 | ) 22 | ``` 23 | 24 | ## Supported GIS functions (core) 25 | 26 | ```sql 27 | CREATE OR REPLACE FUNCTION st_geometryFromText as 'com.carto.analyticstoolbox.core.ST_GeomFromWKT'; 28 | CREATE OR REPLACE FUNCTION st_intersects as 'com.carto.analyticstoolbox.core.ST_Intersects'; 29 | CREATE OR REPLACE FUNCTION st_simplify as 'com.carto.analyticstoolbox.core.ST_Simplify'; 30 | -- ...and more 31 | ``` 32 | 33 | The full list of supported functions can be found [here](./core/sql/createUDFs.sql). 34 | 35 | ## Spatial Query Optimizations 36 | 37 | There are two types of supported optimizations: `ST_Intersects` and `ST_Contains`, which allow Spark to push down predicates when possible. 38 | 39 | To enable optimizations: 40 | 41 | ```scala 42 | com.carto.analyticstoolbox.spark.sql.rules.SpatialFilterPushdownRules 43 | 44 | val spark: SparkSession = ??? 45 | SpatialFilterPushdownRules.registerOptimizations(sparkContext.sqlContext) 46 | ``` 47 | 48 | It is also possible to set it through the Spark configuration via the optimizations injector: 49 | 50 | ```scala 51 | import com.carto.analyticstoolbox.spark.sql.SpatialFilterPushdownOptimizations 52 | 53 | val conf: SparkConfig = ??? 54 | config.set("spark.sql.extensions", classOf[SpatialFilterPushdownOptimizations].getName) 55 | ``` 56 | 57 | In case there is a need to have optimizations enabled on a DataBricks cluster by default, follow [Enabling CARTO Query Optimizations on Databricks](#enabling-carto-query-optimizations-on-databricks) section. 58 | 59 | ## Table Optimization 60 | 61 | There are two functions defined to help with the raw table preparations. Both transform the input table 62 | into a shape optimized for intersection queries; for more details see [OptimizeSpatial.scala](./core/src/main/scala/com/carto/analyticstoolbox/spark/spatial/OptimizeSpatial.scala): 63 | 64 | 1. **optimizeSpatial** 65 | * Uses heuristics to compute the optimal parquet block size 66 | 2. **optimizeSpatialManual** 67 | * Uses the user input to set the output parquet file block size 68 | 69 | ```scala 70 | import com.carto.analyticstoolbox.spark.spatial._ 71 | 72 | val sourceTable: String = ??? 73 | val outputTable: String = ??? 74 | val outputLocation: String = ??? 75 | 76 | // optimize with the block size computation 77 | spark.optimizeSpatial(sourceTable, outputTable, outputLocation, geomColumn = "bbox") 78 | // optimize with the user defined block size 79 | spark.optimizeSpatialManual(sourceTable, outputTable, outputLocation, geomColumn = "bbox", blockSize = 20097000) 80 | ``` 81 | 82 | ### Enabling CARTO Query Optimizations on Databricks 83 | 84 | > This section and approach are based on [docs](https://sedona.apache.org/setup/databricks/) 85 | > from Apache Sedona. 86 | 87 | #### Create Initialization Script 88 | 89 | First, write a script to DBFS which can be used to copy jars from 90 | [DBFS](https://docs.databricks.com/data/databricks-file-system.html) to 91 | [the default class path](https://kb.databricks.com/libraries/replace-default-jar-new-jar.html) 92 | cluster directory on master. 93 | 94 | These scripts can be written using notebook cells: 95 | 96 | ```bash 97 | %sh 98 | rm -rf /dbfs/FileStore/jars-carto 99 | 100 | # Create JAR directory for CARTO Analytics Toolbox 101 | mkdir -p /dbfs/FileStore/jars-carto/ 102 | 103 | # Download assembly har from GitHub into DBFS 104 | curl -L -o /dbfs/FileStore/jars-carto/core-assembly-{version}.jar "https://github.com/CartoDB/analytics-toolbox-databricks/releases/download/v{version}/core-assembly-{version}.jar" 105 | ``` 106 | 107 | ```bash 108 | %sh 109 | rm -rf /dbfs/FileStore/carto/ 110 | 111 | # Create JAR directory for CARTO 112 | mkdir -p /dbfs/FileStore/carto/ 113 | 114 | # Create init script 115 | cat > /dbfs/FileStore/carto/carto-init.sh <<'EOF' 116 | #!/bin/bash 117 | # 118 | # 119 | # On cluster startup, this script will copy the CARTO jars to the cluster's default jar directory. 120 | # In order to activate CARTO Spatial optimizations: "com.carto.analyticstoolbox.spark.sql.rules.SpatialFilterPushdownRules" 121 | 122 | cp /dbfs/FileStore/jars-carto/core-assembly-{version}.jar /databricks/jars 123 | 124 | EOF 125 | ``` 126 | 127 | Where `/dbfs/FileStore/jars-carto/core-assembly-{version}.jar` is a path to the installed CARTO Analytics Toolbox 128 | package of a certain version. 129 | 130 | #### Update Cluster Configuration 131 | 132 | Next, we need to update the spark config and inform the cluster of its new initialization script. 133 | Navigate to cluster settings and find the 'Advanced options'. From your cluster configuration 134 | activate the CARTO Spatial optimizations by adding to the Spark Config 135 | (`Cluster` -> `Edit` -> `Configuration` -> `Advanced options` -> `Spark`). 136 | 137 | To the spark config, add 138 | ```cfg 139 | spark.sql.extensions com.carto.analyticstoolbox.spark.sql.SpatialFilterPushdownOptimizations 140 | ``` 141 | 142 | This will inform spark of the class which will register sql extensions. Move from the `Spark` tab 143 | of advanced options to the `Init scripts` tab and add an entry for the initialization script 144 | written above (`dbfs:/FileStore/carto/carto-init.sh`). 145 | 146 | Restart the cluster and predicate pushdown for spatial intersection is enabled, 147 | allowing certain workflows to run far more efficiently. 148 | 149 | #### Why is this necessary? 150 | 151 | The startup of a databricks cluster looks something like: 152 | 153 | 1. The JVM process starts with the cluster default classpath 154 | 2. The Spark config is initialized (Here's where we want to enable optimizations) 155 | 3. [VFS](https://commons.apache.org/proper/commons-vfs/) / [DBFS](https://docs.databricks.com/data/databricks-file-system.html) user class paths are mounted 156 | 157 | The jar which contains classes that are referenced in step 2 isn't available prior to step 3! 158 | Fortunately, it is possible to set up Databricks 159 | [initialization scripts](https://docs.databricks.com/clusters/init-scripts.html) 160 | which run prior to step 1 and which we can use on databricks to ensure our classes are 161 | available by the time a cluster loads its spark config. 162 | -------------------------------------------------------------------------------- /build.sbt: -------------------------------------------------------------------------------- 1 | import de.heikoseeberger.sbtheader._ 2 | import java.time.Year 3 | 4 | val scalaVersions = Seq("2.12.15") 5 | 6 | val catsVersion = "2.7.0" 7 | val shapelessVersion = "2.3.3" // to be compatible with Spark 3.1.x 8 | val scalaTestVersion = "3.2.11" 9 | val jtsVersion = "1.18.1" 10 | val geomesaVersion = "3.3.0" 11 | val hivelessVersion = "0.0.12" 12 | val geotrellisVersion = "3.6.2" 13 | val h3Version = "4.0.0-rc1" 14 | 15 | // GeoTrellis depends on Shapeless 2.3.7 16 | // To maintain better compat with Spark 3.1.x and DataBricks 9.1 we need to depend on Shapeless 2.3.3 17 | val excludedDependencies = List( 18 | ExclusionRule("com.chuusai", "shapeless_2.12"), 19 | ExclusionRule("com.chuusai", "shapeless_2.13") 20 | ) 21 | 22 | def ver(for212: String, for213: String) = Def.setting { 23 | CrossVersion.partialVersion(scalaVersion.value) match { 24 | case Some((2, 12)) => for212 25 | case Some((2, 13)) => for213 26 | case _ => sys.error("not good") 27 | } 28 | } 29 | 30 | def spark(module: String) = Def.setting { 31 | "org.apache.spark" %% s"spark-$module" % ver("3.1.3", "3.2.1").value 32 | } 33 | 34 | // https://github.com/xerial/sbt-sonatype/issues/276 35 | ThisBuild / sonatypeCredentialHost := "s01.oss.sonatype.org" 36 | 37 | lazy val commonSettings = Seq( 38 | scalaVersion := scalaVersions.head, 39 | crossScalaVersions := scalaVersions, 40 | organization := "com.carto.analyticstoolbox", 41 | scalacOptions ++= Seq( 42 | "-deprecation", 43 | "-unchecked", 44 | "-language:implicitConversions", 45 | "-language:reflectiveCalls", 46 | "-language:higherKinds", 47 | "-language:postfixOps", 48 | "-language:existentials", 49 | "-feature", 50 | "-target:jvm-1.8" // , 51 | // "-Xsource:3" 52 | ), 53 | licenses := Seq("BSD-3-Clause" -> url("https://github.com/CartoDB/analytics-toolbox-databricks/blob/master/LICENSE")), 54 | homepage := Some(url("https://github.com/CartoDB/analytics-toolbox-databricks")), 55 | versionScheme := Some("semver-spec"), 56 | Test / publishArtifact := false, 57 | Test / fork := true, 58 | developers := List( 59 | Developer( 60 | "pomadchin", 61 | "Grigory Pomadchin", 62 | "@pomadchin", 63 | url("https://github.com/pomadchin") 64 | ) 65 | ), 66 | headerLicense := Some(HeaderLicense.ALv2(Year.now.getValue.toString, "Azavea")), 67 | headerMappings := Map( 68 | FileType.scala -> CommentStyle.cStyleBlockComment.copy( 69 | commentCreator = { (text, existingText) => 70 | // preserve year of old headers 71 | val newText = CommentStyle.cStyleBlockComment.commentCreator.apply(text, existingText) 72 | existingText.flatMap(_ => existingText.map(_.trim)).getOrElse(newText) 73 | } 74 | ) 75 | ), 76 | // resolver for hiveless SNAPSHOT dependencies 77 | resolvers += "oss-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots", 78 | addCompilerPlugin("org.typelevel" % "kind-projector" % "0.13.2" cross CrossVersion.full), 79 | libraryDependencies += "org.scalatest" %% "scalatest" % scalaTestVersion % Test, 80 | // sonatype settings 81 | sonatypeProfileName := "com.carto", 82 | sonatypeCredentialHost := "s01.oss.sonatype.org", 83 | sonatypeRepository := "https://s01.oss.sonatype.org/service/local" 84 | ) 85 | 86 | lazy val root = (project in file(".")) 87 | .settings(commonSettings) 88 | .settings(name := "analyticstoolbox") 89 | .settings( 90 | scalaVersion := scalaVersions.head, 91 | crossScalaVersions := Nil, 92 | publish := {}, 93 | publishLocal := {} 94 | ) 95 | .aggregate(core) 96 | 97 | lazy val core = project 98 | .settings(commonSettings) 99 | .settings(name := "core") 100 | .settings( 101 | libraryDependencies ++= Seq( 102 | "com.azavea" %% "hiveless-core" % hivelessVersion, 103 | "com.azavea" %% "hiveless-jts" % hivelessVersion, 104 | "org.locationtech.geomesa" %% "geomesa-spark-jts" % geomesaVersion, 105 | "com.uber" % "h3" % h3Version, 106 | spark("hive").value % Provided 107 | ) ++ Seq( 108 | "org.locationtech.geotrellis" %% "geotrellis-store" % geotrellisVersion, 109 | "org.locationtech.geotrellis" %% "geotrellis-spark-testkit" % geotrellisVersion % Test 110 | ).map(_ excludeAll (excludedDependencies: _*)), 111 | assembly / test := {}, 112 | assembly / assemblyShadeRules := { 113 | val shadePackage = "com.carto.analytics" 114 | Seq( 115 | ShadeRule.rename("shapeless.**" -> s"$shadePackage.shapeless.@1").inAll, 116 | ShadeRule.rename("cats.kernel.**" -> s"$shadePackage.cats.kernel.@1").inAll 117 | ) 118 | }, 119 | assembly / assemblyMergeStrategy := { 120 | case s if s.startsWith("META-INF/services") => MergeStrategy.concat 121 | case "reference.conf" | "application.conf" => MergeStrategy.concat 122 | case "META-INF/MANIFEST.MF" | "META-INF\\MANIFEST.MF" => MergeStrategy.discard 123 | case "META-INF/ECLIPSEF.RSA" | "META-INF/ECLIPSEF.SF" => MergeStrategy.discard 124 | case _ => MergeStrategy.first 125 | } 126 | ) 127 | -------------------------------------------------------------------------------- /core/sql/createUDFs.sql: -------------------------------------------------------------------------------- 1 | CREATE OR REPLACE FUNCTION st_antimeridianSafeGeom as 'com.carto.analyticstoolbox.core.ST_AntimeridianSafeGeom'; 2 | CREATE OR REPLACE FUNCTION st_area as 'com.carto.analyticstoolbox.core.ST_Area'; 3 | CREATE OR REPLACE FUNCTION st_asBinary as 'com.carto.analyticstoolbox.core.ST_AsBinary'; 4 | CREATE OR REPLACE FUNCTION st_asGeoHash as 'com.carto.analyticstoolbox.core.ST_AsGeoHash'; 5 | CREATE OR REPLACE FUNCTION st_asGeoJson as 'com.carto.analyticstoolbox.core.ST_AsGeoJson'; 6 | CREATE OR REPLACE FUNCTION st_asLatLonText as 'com.carto.analyticstoolbox.core.ST_AsLatLonText'; 7 | CREATE OR REPLACE FUNCTION st_asText as 'com.carto.analyticstoolbox.core.ST_AsText'; 8 | CREATE OR REPLACE FUNCTION st_asTWKB as 'com.carto.analyticstoolbox.core.ST_AsTWKB'; 9 | CREATE OR REPLACE FUNCTION st_boundary as 'com.carto.analyticstoolbox.core.ST_Boundary'; 10 | CREATE OR REPLACE FUNCTION st_box2DFromGeoHash as 'com.carto.analyticstoolbox.core.ST_GeomFromGeoHash'; 11 | CREATE OR REPLACE FUNCTION st_bufferPoint as 'com.carto.analyticstoolbox.core.ST_BufferPoint'; 12 | CREATE OR REPLACE FUNCTION st_byteArray as 'com.carto.analyticstoolbox.core.ST_ByteArray'; 13 | CREATE OR REPLACE FUNCTION st_castToGeometry as 'com.carto.analyticstoolbox.core.ST_CastToGeometry'; 14 | CREATE OR REPLACE FUNCTION st_castToLineString as 'com.carto.analyticstoolbox.core.ST_CastToLineString'; 15 | CREATE OR REPLACE FUNCTION st_castToPoint as 'com.carto.analyticstoolbox.core.ST_CastToPoint'; 16 | CREATE OR REPLACE FUNCTION st_castToPolygon as 'com.carto.analyticstoolbox.core.ST_CastToPolygon'; 17 | CREATE OR REPLACE FUNCTION st_centroid as 'com.carto.analyticstoolbox.core.ST_Centroid'; 18 | CREATE OR REPLACE FUNCTION st_closestPoint as 'com.carto.analyticstoolbox.core.ST_ClosestPoint'; 19 | CREATE OR REPLACE FUNCTION st_contains as 'com.carto.analyticstoolbox.core.ST_Contains'; 20 | CREATE OR REPLACE FUNCTION st_convexHull as 'com.carto.analyticstoolbox.core.ST_ConvexHull'; 21 | CREATE OR REPLACE FUNCTION st_coordDim as 'com.carto.analyticstoolbox.core.ST_CoordDim'; 22 | CREATE OR REPLACE FUNCTION st_covers as 'com.carto.analyticstoolbox.core.ST_Covers'; 23 | CREATE OR REPLACE FUNCTION st_crosses as 'com.carto.analyticstoolbox.core.ST_Crosses'; 24 | CREATE OR REPLACE FUNCTION st_difference as 'com.carto.analyticstoolbox.core.ST_Difference'; 25 | CREATE OR REPLACE FUNCTION st_dimension as 'com.carto.analyticstoolbox.core.ST_Dimension'; 26 | CREATE OR REPLACE FUNCTION st_disjoint as 'com.carto.analyticstoolbox.core.ST_Disjoint'; 27 | CREATE OR REPLACE FUNCTION st_distance as 'com.carto.analyticstoolbox.core.ST_Distance'; 28 | CREATE OR REPLACE FUNCTION st_distanceSphere as 'com.carto.analyticstoolbox.core.ST_DistanceSphere'; 29 | CREATE OR REPLACE FUNCTION st_envelope as 'com.carto.analyticstoolbox.core.ST_Envelope'; 30 | CREATE OR REPLACE FUNCTION st_equals as 'com.carto.analyticstoolbox.core.ST_Equals'; 31 | CREATE OR REPLACE FUNCTION st_exteriorRing as 'com.carto.analyticstoolbox.core.ST_ExteriorRing'; 32 | CREATE OR REPLACE FUNCTION st_geoHash as 'com.carto.analyticstoolbox.core.ST_GeoHash'; 33 | CREATE OR REPLACE FUNCTION st_geomFromGeoHash as 'com.carto.analyticstoolbox.core.ST_GeomFromGeoHash'; 34 | CREATE OR REPLACE FUNCTION st_geomFromGeoJson as 'com.carto.analyticstoolbox.core.ST_GeomFromGeoJson'; 35 | CREATE OR REPLACE FUNCTION st_geometryFromText as 'com.carto.analyticstoolbox.core.ST_GeomFromWKT'; 36 | CREATE OR REPLACE FUNCTION st_geomFromTWKB as 'com.carto.analyticstoolbox.core.ST_GeomFromTWKB'; 37 | CREATE OR REPLACE FUNCTION st_geomFromWKB as 'com.carto.analyticstoolbox.core.ST_GeomFromWKB'; 38 | CREATE OR REPLACE FUNCTION st_geomFromWKT as 'com.carto.analyticstoolbox.core.ST_GeomFromWKT'; 39 | CREATE OR REPLACE FUNCTION st_geometryN as 'com.carto.analyticstoolbox.core.ST_GeometryN'; 40 | CREATE OR REPLACE FUNCTION st_idlSafeGeom as 'com.carto.analyticstoolbox.core.ST_AntimeridianSafeGeom'; 41 | CREATE OR REPLACE FUNCTION st_interiorRingN as 'com.carto.analyticstoolbox.core.ST_InteriorRingN'; 42 | CREATE OR REPLACE FUNCTION st_intersection as 'com.carto.analyticstoolbox.core.ST_Intersection'; 43 | CREATE OR REPLACE FUNCTION st_intersects as 'com.carto.analyticstoolbox.core.ST_Intersects'; 44 | CREATE OR REPLACE FUNCTION st_icClosed as 'com.carto.analyticstoolbox.core.ST_IsClosed'; 45 | CREATE OR REPLACE FUNCTION st_isCollection as 'com.carto.analyticstoolbox.core.ST_IsCollection'; 46 | CREATE OR REPLACE FUNCTION st_isEmpty as 'com.carto.analyticstoolbox.core.ST_IsEmpty'; 47 | CREATE OR REPLACE FUNCTION st_isGeomField as 'com.carto.analyticstoolbox.core.ST_IsGeomField'; 48 | CREATE OR REPLACE FUNCTION st_isRing as 'com.carto.analyticstoolbox.core.ST_IsRing'; 49 | CREATE OR REPLACE FUNCTION st_isSimple as 'com.carto.analyticstoolbox.core.ST_IsSimple'; 50 | CREATE OR REPLACE FUNCTION st_isValid as 'com.carto.analyticstoolbox.core.ST_IsValid'; 51 | CREATE OR REPLACE FUNCTION st_length as 'com.carto.analyticstoolbox.core.ST_Length'; 52 | CREATE OR REPLACE FUNCTION st_lengthSphere as 'com.carto.analyticstoolbox.core.ST_LengthSphere'; 53 | CREATE OR REPLACE FUNCTION st_lineFromText as 'com.carto.analyticstoolbox.core.ST_LineFromText'; 54 | CREATE OR REPLACE FUNCTION st_makeBBOX as 'com.carto.analyticstoolbox.core.ST_MakeBBOX'; 55 | CREATE OR REPLACE FUNCTION st_makeBox2D as 'com.carto.analyticstoolbox.core.ST_MakeBox2D'; 56 | CREATE OR REPLACE FUNCTION st_makeLine as 'com.carto.analyticstoolbox.core.ST_MakeLine'; 57 | CREATE OR REPLACE FUNCTION st_makePoint as 'com.carto.analyticstoolbox.core.ST_MakePoint'; 58 | CREATE OR REPLACE FUNCTION st_makePointM as 'com.carto.analyticstoolbox.core.ST_MakePointM'; 59 | CREATE OR REPLACE FUNCTION st_makePolygon as 'com.carto.analyticstoolbox.core.ST_MakePolygon'; 60 | CREATE OR REPLACE FUNCTION st_mLineFromText as 'com.carto.analyticstoolbox.core.ST_MLineFromText'; 61 | CREATE OR REPLACE FUNCTION st_mPointFromText as 'com.carto.analyticstoolbox.core.ST_MPointFromText'; 62 | CREATE OR REPLACE FUNCTION st_mPolyFromText as 'com.carto.analyticstoolbox.core.ST_MPolyFromText'; 63 | CREATE OR REPLACE FUNCTION st_numGeometries as 'com.carto.analyticstoolbox.core.ST_NumGeometries'; 64 | CREATE OR REPLACE FUNCTION st_numPoints as 'com.carto.analyticstoolbox.core.ST_NumPoints'; 65 | CREATE OR REPLACE FUNCTION st_overlaps as 'com.carto.analyticstoolbox.core.ST_Overlaps'; 66 | CREATE OR REPLACE FUNCTION st_pointFromGeoHash as 'com.carto.analyticstoolbox.core.ST_PointFromGeoHash'; 67 | CREATE OR REPLACE FUNCTION st_pointFromText as 'com.carto.analyticstoolbox.core.ST_PointFromText'; 68 | CREATE OR REPLACE FUNCTION st_pointFromWKB as 'com.carto.analyticstoolbox.core.ST_PointFromWKB'; 69 | CREATE OR REPLACE FUNCTION st_pointN as 'com.carto.analyticstoolbox.core.ST_PointN'; 70 | CREATE OR REPLACE FUNCTION st_polygonFromText as 'com.carto.analyticstoolbox.core.ST_PolygonFromText'; 71 | CREATE OR REPLACE FUNCTION st_relate as 'com.carto.analyticstoolbox.core.ST_Relate'; 72 | CREATE OR REPLACE FUNCTION st_relateBool as 'com.carto.analyticstoolbox.core.ST_RelateBool'; 73 | CREATE OR REPLACE FUNCTION st_simplify as 'com.carto.analyticstoolbox.core.ST_Simplify'; 74 | CREATE OR REPLACE FUNCTION st_simplifyPreserveTopology as 'com.carto.analyticstoolbox.core.ST_SimplifyPreserveTopology'; 75 | CREATE OR REPLACE FUNCTION st_touches as 'com.carto.analyticstoolbox.core.ST_Touches'; 76 | CREATE OR REPLACE FUNCTION st_translate as 'com.carto.analyticstoolbox.core.ST_Translate'; 77 | CREATE OR REPLACE FUNCTION st_within as 'com.carto.analyticstoolbox.core.ST_Within'; 78 | CREATE OR REPLACE FUNCTION st_x as 'com.carto.analyticstoolbox.core.ST_X'; 79 | CREATE OR REPLACE FUNCTION st_y as 'com.carto.analyticstoolbox.core.ST_Y'; 80 | -- indexing functions 81 | CREATE OR REPLACE FUNCTION H3_ToParent as 'com.carto.analyticstoolbox.index.H3_ToParent'; 82 | CREATE OR REPLACE FUNCTION st_crsFromText as 'com.carto.analyticstoolbox.index.ST_CrsFromText'; 83 | CREATE OR REPLACE FUNCTION st_extentFromGeom as 'com.carto.analyticstoolbox.index.ST_ExtentFromGeom'; 84 | CREATE OR REPLACE FUNCTION st_extentToGeom as 'com.carto.analyticstoolbox.index.ST_ExtentToGeom'; 85 | CREATE OR REPLACE FUNCTION st_geomReproject as 'com.carto.analyticstoolbox.index.ST_GeomReproject'; 86 | CREATE OR REPLACE FUNCTION st_makeExtent as 'com.carto.analyticstoolbox.index.ST_MakeExtent'; 87 | CREATE OR REPLACE FUNCTION st_partitionCentroid as 'com.carto.analyticstoolbox.index.ST_PartitionCentroid'; 88 | CREATE OR REPLACE FUNCTION st_z2LatLon as 'com.carto.analyticstoolbox.index.ST_Z2LatLon'; 89 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AntimeridianSafeGeom.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricProcessingFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_AntimeridianSafeGeom extends HUDF[Geometry, Geometry] { 24 | def function = GeometricProcessingFunctions.ST_antimeridianSafeGeom 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Area.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_Area extends HUDF[Geometry, jl.Double] { 26 | def function = SpatialRelationFunctions.ST_Area 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsBinary.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_AsBinary extends HUDF[Geometry, Array[Byte]] { 24 | def function = GeometricOutputFunctions.ST_AsBinary 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsGeoHash.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_AsGeoHash extends HUDF[(Geometry, Int), String] { 25 | def function = GeometricOutputFunctions.ST_GeoHash 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsGeoJson.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_AsGeoJson extends HUDF[Geometry, String] { 24 | def function = GeometricOutputFunctions.ST_AsGeoJSON 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsLatLonText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 21 | import org.locationtech.jts.geom.Point 22 | 23 | class ST_AsLatLonText extends HUDF[Point, String] { 24 | def function = GeometricOutputFunctions.ST_AsLatLonText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsTWKB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.spatial.util.TWKBUtils 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_AsTWKB extends HUDF[Geometry, Array[Byte]] { 24 | def function = TWKBUtils.write 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_AsText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_AsText extends HUDF[Geometry, String] { 24 | def function = GeometricOutputFunctions.ST_AsText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Boundary.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_Boundary extends HUDF[Geometry, Geometry] { 24 | def function = GeometricAccessorFunctions.ST_Boundary 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_BufferPoint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricProcessingFunctions 22 | import org.locationtech.jts.geom.{Geometry, Point} 23 | 24 | class ST_BufferPoint extends HUDF[(Point, Double), Geometry] { 25 | def function = GeometricProcessingFunctions.ST_BufferPoint 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_ByteArray.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricCastFunctions 21 | 22 | class ST_ByteArray extends HUDF[String, Array[Byte]] { 23 | def function = GeometricCastFunctions.ST_ByteArray 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_CastToGeometry.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricCastFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_CastToGeometry extends HUDF[Geometry, Geometry] { 24 | def function = GeometricCastFunctions.ST_CastToGeometry 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_CastToLineString.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricCastFunctions 21 | import org.locationtech.jts.geom.{Geometry, LineString} 22 | 23 | class ST_CastToLineString extends HUDF[Geometry, LineString] { 24 | def function = GeometricCastFunctions.ST_CastToLineString 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_CastToPoint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricCastFunctions 21 | import org.locationtech.jts.geom.{Geometry, Point} 22 | 23 | class ST_CastToPoint extends HUDF[Geometry, Point] { 24 | def function = GeometricCastFunctions.ST_CastToPoint 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_CastToPolygon.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricCastFunctions 21 | import org.locationtech.jts.geom.{Geometry, Polygon} 22 | 23 | class ST_CastToPolygon extends HUDF[Geometry, Polygon] { 24 | def function = GeometricCastFunctions.ST_CastToPolygon 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Centroid.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 21 | import org.locationtech.jts.geom.{Geometry, Point} 22 | 23 | class ST_Centroid extends HUDF[Geometry, Point] { 24 | def function = SpatialRelationFunctions.ST_Centroid 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_ClosestPoint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.{Geometry, Point} 23 | 24 | class ST_ClosestPoint extends HUDF[(Geometry, Geometry), Point] { 25 | def function = SpatialRelationFunctions.ST_ClosestPoint 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Contains.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.carto.analyticstoolbox.index._ 20 | 21 | import com.azavea.hiveless.HUDF 22 | import com.azavea.hiveless.implicits.tupler._ 23 | 24 | class ST_Contains extends HUDF[(ST_Contains.Arg, ST_Contains.Arg), Boolean] { 25 | def function = ST_Contains.function 26 | } 27 | 28 | object ST_Contains { 29 | type Arg = ST_Intersects.Arg 30 | 31 | def function(left: Arg, right: Arg): Boolean = { 32 | val (l, r) = (ST_Intersects.parseGeometryUnsafe(left, "first"), ST_Intersects.parseGeometryUnsafe(right, "second")) 33 | 34 | l.contains(r) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_ConvexHull.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.{HAggregationBuffer, HGenericUDAFEvaluator} 20 | import org.apache.hadoop.hive.ql.udf.generic.{AbstractGenericUDAFResolver, GenericUDAFEvaluator} 21 | import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_ConvexHull extends AbstractGenericUDAFResolver { 25 | implicit def geometryConvexHullBuffer: HAggregationBuffer[Geometry] = new HAggregationBuffer[Geometry] { 26 | protected var accumulator: Geometry = _ 27 | 28 | def add(argument: Geometry): Unit = 29 | if (argument != null) 30 | if (accumulator == null) accumulator = argument.convexHull() 31 | // convexHull of the argument to avoid collecting union into the Geometry Collection 32 | else accumulator = accumulator.union(argument.convexHull()).convexHull() 33 | 34 | def reset: Unit = accumulator = null 35 | } 36 | 37 | override def getEvaluator(info: Array[TypeInfo]): GenericUDAFEvaluator = HGenericUDAFEvaluator[Geometry] 38 | } 39 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_CoordDim.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_CoordDim extends HUDF[Geometry, jl.Integer] { 26 | def function = GeometricAccessorFunctions.ST_CoordDim 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Covers.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Covers extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Covers 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Crosses.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Crosses extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Crosses 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Difference.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_Difference extends HUDF[(Geometry, Geometry), Geometry] { 25 | def function = SpatialRelationFunctions.ST_Difference 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Dimension.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_Dimension extends HUDF[Geometry, jl.Integer] { 26 | def function = GeometricAccessorFunctions.ST_Dimension 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Disjoint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Disjoint extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Disjoint 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Distance.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Distance extends HUDF[(Geometry, Geometry), jl.Double] { 27 | def function = SpatialRelationFunctions.ST_Distance 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_DistanceSphere.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_DistanceSphere extends HUDF[(Geometry, Geometry), jl.Double] { 27 | def function = SpatialRelationFunctions.ST_DistanceSphere 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Envelope.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_Envelope extends HUDF[Geometry, Geometry] { 24 | def function = GeometricAccessorFunctions.ST_Envelope 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Equals.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Equals extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Equals 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_ExteriorRing.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_ExteriorRing extends HUDF[Geometry, Geometry] { 24 | def function = GeometricAccessorFunctions.ST_ExteriorRing 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeoHash.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricOutputFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_GeoHash extends HUDF[(Geometry, Int), String] { 25 | def function = GeometricOutputFunctions.ST_GeoHash 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeomFromGeoHash.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_GeomFromGeoHash extends HUDF[(String, Int), Geometry] { 25 | def function = GeometricConstructorFunctions.ST_GeomFromGeoHash 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeomFromGeoJson.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_GeomFromGeoJson extends HUDF[String, Geometry] { 24 | def function = GeometricConstructorFunctions.ST_GeomFromGeoJSON 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeomFromTWKB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.spatial.util.TWKBUtils 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_GeomFromTWKB extends HUDF[Array[Byte], Geometry] { 24 | def function = TWKBUtils.read 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeomFromWKB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_GeomFromWKB extends HUDF[Array[Byte], Geometry] { 24 | def function = GeometricConstructorFunctions.ST_GeomFromWKB 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeomFromWKT.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_GeomFromWKT extends HUDF[String, Geometry] { 24 | def function = GeometricConstructorFunctions.ST_GeomFromWKT 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_GeometryN.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_GeometryN extends HUDF[(Geometry, Int), Geometry] { 25 | def function = GeometricAccessorFunctions.ST_GeometryN 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_InteriorRingN.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_InteriorRingN extends HUDF[(Geometry, Int), Geometry] { 25 | def function = GeometricAccessorFunctions.ST_InteriorRingN 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Intersection.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_Intersection extends HUDF[(Geometry, Geometry), Geometry] { 25 | def function = SpatialRelationFunctions.ST_Intersection 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Intersects.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.carto.analyticstoolbox.index._ 20 | 21 | import com.azavea.hiveless.HUDF 22 | import com.azavea.hiveless.implicits.tupler._ 23 | import com.azavea.hiveless.serializers.HDeserializer.Errors.ProductDeserializationError 24 | import geotrellis.vector._ 25 | import shapeless._ 26 | 27 | class ST_Intersects extends HUDF[(ST_Intersects.Arg, ST_Intersects.Arg), Boolean] { 28 | def function = ST_Intersects.function 29 | } 30 | 31 | object ST_Intersects { 32 | // We could use Either[Extent, Geometry], but Either has no safe fall back CNil 33 | // which may lead to derivation error messages rather than parsing 34 | type Arg = Extent :+: Geometry :+: CNil 35 | 36 | def parseGeometry(a: Arg): Option[Geometry] = a.select[Geometry].orElse(a.select[Extent].map(_.toPolygon())) 37 | 38 | def parseGeometryUnsafe(a: Arg, aname: String): Geometry = 39 | parseGeometry(a).getOrElse(throw ProductDeserializationError[ST_Intersects, Arg](aname)) 40 | 41 | def function(left: Arg, right: Arg): Boolean = { 42 | val (l, r) = (parseGeometryUnsafe(left, "first"), parseGeometryUnsafe(right, "second")) 43 | 44 | l.intersects(r) 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsClosed.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsClosed extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsClosed 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsCollection.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsCollection extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsCollection 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsEmpty.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsEmpty extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsEmpty 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsGeomField.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDTF 20 | import org.apache.spark.unsafe.types.UTF8String 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import scala.util.{Success, Try} 25 | import java.{lang => jl} 26 | 27 | class ST_IsGeomField extends HUDTF[Boolean] { 28 | val circularInspectors: Boolean = true 29 | 30 | def eval(arguments: Array[AnyRef]): Array[AnyRef] = arguments.map { 31 | case _: Geometry => jl.Boolean.TRUE 32 | case s: UTF8String => 33 | Try(GeometricConstructorFunctions.ST_GeomFromWKT(s.toString)) match { 34 | case Success(_) => jl.Boolean.TRUE 35 | case _ => jl.Boolean.FALSE 36 | } 37 | case _ => jl.Boolean.FALSE 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsRing.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsRing extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsRing 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsSimple.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsSimple extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsSimple 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_IsValid.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_IsValid extends HUDF[Geometry, jl.Boolean] { 26 | def function = GeometricAccessorFunctions.ST_IsValid 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Length.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_Length extends HUDF[Geometry, jl.Double] { 26 | def function = SpatialRelationFunctions.ST_Length 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_LengthSphere.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 21 | import org.locationtech.jts.geom.LineString 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_LengthSphere extends HUDF[LineString, jl.Double] { 26 | def function = SpatialRelationFunctions.ST_LengthSphere 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MLineFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.MultiLineString 22 | 23 | class ST_MLineFromText extends HUDF[String, MultiLineString] { 24 | def function = GeometricConstructorFunctions.ST_MLineFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MPointFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.MultiPoint 22 | 23 | class ST_MPointFromText extends HUDF[String, MultiPoint] { 24 | def function = GeometricConstructorFunctions.ST_MPointFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MPolyFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.MultiPolygon 22 | 23 | class ST_MPolyFromText extends HUDF[String, MultiPolygon] { 24 | def function = GeometricConstructorFunctions.ST_MPolyFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakeBBOX.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_MakeBBOX extends HUDF[(Double, Double, Double, Double), Geometry] { 25 | def function = GeometricConstructorFunctions.ST_MakeBBOX 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakeBox2D.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.{Geometry, Point} 23 | 24 | class ST_MakeBox2D extends HUDF[(Point, Point), Geometry] { 25 | def function = GeometricConstructorFunctions.ST_MakeBox2D 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakeLine.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.{LineString, Point} 22 | 23 | class ST_MakeLine extends HUDF[Array[Point], LineString] { 24 | def function = array => GeometricConstructorFunctions.ST_MakeLine(array) 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakePoint.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Point 23 | 24 | class ST_MakePoint extends HUDF[(Double, Double), Point] { 25 | def function = GeometricConstructorFunctions.ST_MakePoint 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakePointM.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Point 23 | 24 | class ST_MakePointM extends HUDF[(Double, Double, Double), Point] { 25 | def function = GeometricConstructorFunctions.ST_MakePointM 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_MakePolygon.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.{LineString, Polygon} 22 | 23 | class ST_MakePolygon extends HUDF[LineString, Polygon] { 24 | def function = GeometricConstructorFunctions.ST_MakePolygon 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_NumGeometries.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_NumGeometries extends HUDF[Geometry, jl.Integer] { 26 | def function = GeometricAccessorFunctions.ST_NumGeometries 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_NumPoints.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_NumPoints extends HUDF[Geometry, jl.Integer] { 26 | def function = GeometricAccessorFunctions.ST_NumPoints 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Overlaps.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Overlaps extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Overlaps 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_PointFromGeoHash.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 22 | import org.locationtech.jts.geom.Point 23 | 24 | class ST_PointFromGeoHash extends HUDF[(String, Int), Point] { 25 | def function = GeometricConstructorFunctions.ST_PointFromGeoHash 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_PointFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Point 22 | 23 | class ST_PointFromText extends HUDF[String, Point] { 24 | def function = GeometricConstructorFunctions.ST_PointFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_PointFromWKB.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Point 22 | 23 | class ST_PointFromWKB extends HUDF[Array[Byte], Point] { 24 | def function = GeometricConstructorFunctions.ST_PointFromWKB 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_PointN.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 22 | import org.locationtech.jts.geom.{Geometry, Point} 23 | 24 | class ST_PointN extends HUDF[(Geometry, Int), Point] { 25 | def function = GeometricAccessorFunctions.ST_PointN 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_PolygonFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | class ST_PolygonFromText extends HUDF[String, Geometry] { 24 | def function = GeometricConstructorFunctions.ST_PolygonFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Relate.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_Relate extends HUDF[(Geometry, Geometry), String] { 25 | def function = SpatialRelationFunctions.ST_Relate 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_RelateBool.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_RelateBool extends HUDF[(Geometry, Geometry, String), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_RelateBool 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Simplify.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.jts.geom.Geometry 22 | import org.locationtech.jts.simplify.DouglasPeuckerSimplifier 23 | 24 | class ST_Simplify extends HUDF[(Geometry, Double), Geometry] { 25 | def function = { (g: Geometry, p: Double) => DouglasPeuckerSimplifier.simplify(g, p) } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_SimplifyPreserveTopology.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.jts.geom.Geometry 22 | import org.locationtech.jts.simplify.TopologyPreservingSimplifier 23 | 24 | class ST_SimplifyPreserveTopology extends HUDF[(Geometry, Double), Geometry] { 25 | def function = { (g: Geometry, p: Double) => TopologyPreservingSimplifier.simplify(g, p) } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Touches.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Touches extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Touches 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Translate.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | class ST_Translate extends HUDF[(Geometry, Double, Double), Geometry] { 25 | def function = SpatialRelationFunctions.ST_Translate 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Within.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.azavea.hiveless.implicits.tupler._ 21 | import org.locationtech.geomesa.spark.jts.udf.SpatialRelationFunctions 22 | import org.locationtech.jts.geom.Geometry 23 | 24 | import java.{lang => jl} 25 | 26 | class ST_Within extends HUDF[(Geometry, Geometry), jl.Boolean] { 27 | def function = SpatialRelationFunctions.ST_Within 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_X.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_X extends HUDF[Geometry, jl.Float] { 26 | def function = GeometricAccessorFunctions.ST_X 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_Y.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricAccessorFunctions 21 | import org.locationtech.jts.geom.Geometry 22 | 23 | import java.{lang => jl} 24 | 25 | class ST_Y extends HUDF[Geometry, jl.Float] { 26 | def function = GeometricAccessorFunctions.ST_Y 27 | } 28 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/ST_lineFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.azavea.hiveless.HUDF 20 | import org.locationtech.geomesa.spark.jts.udf.GeometricConstructorFunctions 21 | import org.locationtech.jts.geom.LineString 22 | 23 | class ST_lineFromText extends HUDF[String, LineString] { 24 | def function = GeometricConstructorFunctions.ST_LineFromText 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/core/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2021 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox 18 | 19 | import com.azavea.hiveless.serializers.{HConverter, HDeserializer, HSerializer} 20 | import com.azavea.hiveless.serializers.syntax._ 21 | import com.azavea.hiveless.spatial.util.TWKBUtils 22 | import cats.Id 23 | import org.locationtech.jts.geom.Geometry 24 | import org.apache.spark.sql.types.{BinaryType, DataType} 25 | // import org.locationtech.geomesa.spark.jts.util.WKBUtils 26 | 27 | package object core extends Serializable { 28 | implicit def geometryConverter[T <: Geometry]: HConverter[T] = new HConverter[T] { 29 | def convert(argument: Any): T = TWKBUtils.read(argument.asInstanceOf[Array[Byte]]).asInstanceOf[T] 30 | } 31 | 32 | implicit def geometryUnaryDeserializer[T <: Geometry: HConverter]: HDeserializer[Id, T] = 33 | (arguments, inspectors) => arguments.deserialize[Array[Byte]](inspectors).convert[T] 34 | 35 | implicit def geometrySerializer[T <: Geometry]: HSerializer[T] = new HSerializer[T] { 36 | def dataType: DataType = BinaryType 37 | def serialize: Geometry => Array[Byte] = TWKBUtils.write 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/H3_ToParent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.carto.analyticstoolbox.index.h3.H3CoreV3Producer 20 | 21 | import com.azavea.hiveless.HUDF 22 | import com.azavea.hiveless.implicits.tupler._ 23 | 24 | class H3_ToParent extends HUDF[(Long, Int), Long] { 25 | def function = { (h3: Long, parentRes: Int) => H3CoreV3Producer.get.h3ToParent(h3, parentRes) } 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_CrsFromText.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.azavea.hiveless.HUDF 20 | import geotrellis.proj4.CRS 21 | 22 | class ST_CrsFromText extends HUDF[String, CRS] { 23 | def function = CRS.fromString 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_ExtentFromGeom.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | 21 | import com.azavea.hiveless.HUDF 22 | import geotrellis.vector._ 23 | 24 | class ST_ExtentFromGeom extends HUDF[Geometry, Extent] { 25 | def function = _.extent 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_ExtentToGeom.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | 21 | import com.azavea.hiveless.HUDF 22 | import geotrellis.vector._ 23 | 24 | class ST_ExtentToGeom extends HUDF[Extent, Geometry] { 25 | def function = _.toPolygon 26 | } 27 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_GeomReproject.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | 21 | import com.azavea.hiveless.HUDF 22 | import com.azavea.hiveless.implicits.tupler._ 23 | import geotrellis.proj4.CRS 24 | import geotrellis.vector._ 25 | 26 | class ST_GeomReproject extends HUDF[(Geometry, CRS, CRS), Geometry] { 27 | def function = { (geom: Geometry, from: CRS, to: CRS) => geom.reproject(from, to) } 28 | } 29 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_MakeExtent.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.azavea.hiveless.HUDF 20 | import geotrellis.vector.Extent 21 | import com.azavea.hiveless.implicits.tupler._ 22 | 23 | class ST_MakeExtent extends HUDF[(Double, Double, Double, Double), Extent] { 24 | def function = Extent.apply 25 | } 26 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_PartitionCentroid.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.carto.analyticstoolbox.core._ 21 | import com.azavea.hiveless.implicits.tupler._ 22 | import geotrellis.layer.{SpatialKey, ZoomedLayoutScheme} 23 | import geotrellis.vector._ 24 | import geotrellis.proj4.{CRS, LatLng} 25 | import geotrellis.store.index.zcurve.Z2 26 | 27 | class ST_PartitionCentroid extends HUDF[(Geometry, Int, Option[Int], Option[Int], Option[CRS], Option[Double]), Long] { 28 | def function = ST_PartitionCentroid.function 29 | } 30 | 31 | object ST_PartitionCentroid { 32 | def function( 33 | geom: Geometry, 34 | zoom: Int, 35 | tileSizeOpt: Option[Int], 36 | bitsOpt: Option[Int], 37 | crsOpt: Option[CRS], 38 | resolutionThresholdOpt: Option[Double] 39 | ): Long = { 40 | val crs = crsOpt.getOrElse(LatLng) 41 | val tileSize = tileSizeOpt.getOrElse(ZoomedLayoutScheme.DEFAULT_TILE_SIZE) 42 | val resolutionThreshold = resolutionThresholdOpt.getOrElse(ZoomedLayoutScheme.DEFAULT_RESOLUTION_THRESHOLD) 43 | val bits = bitsOpt.getOrElse(8) 44 | 45 | val SpatialKey(col, row) = new ZoomedLayoutScheme(crs, tileSize, resolutionThreshold) 46 | .levelForZoom(zoom) 47 | .layout 48 | .mapTransform(geom.extent.center) 49 | 50 | Z2(col, row).z >> bits 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/ST_Z2LatLon.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.azavea.hiveless.HUDF 20 | import com.carto.analyticstoolbox.core._ 21 | import com.carto.analyticstoolbox.spark.geotrellis.Z2Index 22 | import geotrellis.store.index.zcurve.Z2 23 | import geotrellis.vector.Geometry 24 | 25 | class ST_Z2LatLon extends HUDF[Geometry, Z2Index] { 26 | def function = ST_Z2LatLon.function 27 | } 28 | 29 | object ST_Z2LatLon { 30 | def function(geom: Geometry): Z2Index = { 31 | val env = geom.getEnvelopeInternal 32 | Z2Index(z2index(env.getMinX, env.getMinY), z2index(env.getMaxX, env.getMaxY)) 33 | } 34 | def scaleLat(lat: Double): Int = ((lat + 90) / 180 * (1 << 30)).toInt 35 | def scaleLong(lng: Double): Int = ((lng + 180) / 360 * (1 << 30)).toInt 36 | def z2index(x: Double, y: Double): Long = Z2(scaleLong(x), scaleLat(y)).z 37 | } 38 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/h3/H3CoreV3Producer.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index.h3 18 | 19 | import com.uber.h3core.H3CoreV3 20 | 21 | object H3CoreV3Producer extends Serializable { 22 | @transient lazy val get: H3CoreV3 = H3CoreV3.newInstance() 23 | } 24 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/index/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox 18 | 19 | import com.carto.analyticstoolbox.spark.geotrellis.encoders.StandardEncoders 20 | import com.carto.analyticstoolbox.spark.geotrellis.Z2Index 21 | 22 | import com.azavea.hiveless.serializers.{HConverter, HDeserializer, HSerializer} 23 | import com.azavea.hiveless.serializers.syntax._ 24 | import com.azavea.hiveless.spark.encoders.syntax._ 25 | import cats.Id 26 | import geotrellis.proj4.CRS 27 | import geotrellis.vector.Extent 28 | import org.apache.spark.sql.catalyst.InternalRow 29 | import org.apache.spark.sql.types.{DataType, StringType} 30 | 31 | package object index extends StandardEncoders { 32 | implicit def crsConverter: HConverter[CRS] = new HConverter[CRS] { 33 | def convert(argument: Any): CRS = CRS.fromString(argument.convert[String]) 34 | } 35 | 36 | implicit def extentConverter: HConverter[Extent] = new HConverter[Extent] { 37 | def convert(argument: Any): Extent = argument.convert[InternalRow].as[Extent] 38 | } 39 | 40 | implicit def crsUnaryDeserializer: HDeserializer[Id, CRS] = 41 | (arguments, inspectors) => arguments.deserialize[String](inspectors).convert[CRS] 42 | 43 | implicit def crsSerializer: HSerializer[CRS] = new HSerializer[CRS] { 44 | def dataType: DataType = StringType 45 | def serialize: CRS => Any = crs => crs.toProj4String.serialize 46 | } 47 | 48 | /** HSerializer.expressionEncoderSerializer is not used since TypeTags are not Kryo serializable by default. */ 49 | implicit def extentSerializer: HSerializer[Extent] = new HSerializer[Extent] { 50 | def dataType: DataType = extentEncoder.schema 51 | def serialize: Extent => InternalRow = _.toInternalRow 52 | } 53 | 54 | implicit def z2IndexSerializer: HSerializer[Z2Index] = new HSerializer[Z2Index] { 55 | def dataType: DataType = z2IndexEncoder.schema 56 | def serialize: Z2Index => InternalRow = _.toInternalRow 57 | } 58 | 59 | /** UnaryDeserializer.expressionEncoderUnaryDeserializer since TypeTags are not Kryo serializable by default. */ 60 | implicit def extentUnaryDeserializer: HDeserializer[Id, Extent] = 61 | (arguments, inspectors) => arguments.deserialize[InternalRow](inspectors).as[Extent] 62 | } 63 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/geotrellis/Z2Index.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.geotrellis 18 | 19 | case class Z2Index(min: Long, max: Long) 20 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/geotrellis/encoders/StandardEncoders.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.geotrellis.encoders 18 | 19 | import com.carto.analyticstoolbox.spark.geotrellis.Z2Index 20 | import geotrellis.vector.Extent 21 | import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder 22 | import org.apache.spark.sql.types.{DataType, StructType} 23 | 24 | import scala.reflect.runtime.universe.TypeTag 25 | 26 | trait StandardEncoders extends Serializable { 27 | def expressionEncoder[T: TypeTag]: ExpressionEncoder[T] = ExpressionEncoder() 28 | 29 | implicit val extentEncoder: ExpressionEncoder[Extent] = expressionEncoder 30 | implicit val z2IndexEncoder: ExpressionEncoder[Z2Index] = expressionEncoder 31 | } 32 | 33 | object StandardEncoders extends StandardEncoders 34 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/spatial/OptimizeSpatial.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.spatial 18 | 19 | import org.apache.spark.sql.SparkSession 20 | import org.apache.spark.sql.catalyst.TableIdentifier 21 | import org.apache.spark.sql.types.BinaryType 22 | 23 | object OptimizeSpatial extends Serializable { 24 | def apply( 25 | sourceTable: String, 26 | outputTable: String, 27 | outputLocation: String, 28 | geomColumn: String, 29 | zoom: Int, 30 | computeBlockSize: String => Long, 31 | compression: String, 32 | maxRecordsPerFile: Int 33 | )(implicit ssc: SparkSession): Unit = { 34 | // drop tmp views, IF NOT EXISTS is not supported by Spark SQL, that's a DataBricks feature 35 | // using try catch to capture 36 | val catalog = ssc.sessionState.catalog 37 | 38 | catalog.dropTable(TableIdentifier(s"${sourceTable}_idx_view"), ignoreIfNotExists = true, purge = false) 39 | catalog.dropTable(TableIdentifier(outputTable), ignoreIfNotExists = true, purge = false) 40 | 41 | // via SQL 42 | /*try ssc.sql(s"DROP TABLE ${sourceTable}_idx_view;") catch { case _: AnalysisException => // e.printStackTrace() } 43 | // overwrite the output table 44 | try ssc.sql(s"DROP TABLE $outputTable;") catch { case _: AnalysisException => // e.printStackTrace() }*/ 45 | 46 | // Decide, based on column type of geometry, which parsing function to use 47 | val df = ssc.table(sourceTable) 48 | val parseGeom = 49 | if (df.schema(geomColumn).dataType == BinaryType) "" 50 | else "ST_geomFromWKT" 51 | 52 | // view creation 53 | // SQL definition is easier and more readable 54 | ssc.sql( 55 | s""" 56 | |CREATE TEMPORARY VIEW ${sourceTable}_idx_view AS( 57 | | WITH orig_q AS ( 58 | | SELECT 59 | | * EXCEPT($geomColumn), 60 | | $parseGeom($geomColumn) AS geom 61 | | FROM $sourceTable 62 | | ) 63 | | SELECT 64 | | *, 65 | | st_z2LatLon(geom) AS __carto_z2, 66 | | st_extentFromGeom(geom) AS __carto_index, 67 | | st_partitionCentroid(geom, $zoom) AS __carto_partitioning 68 | | FROM orig_q 69 | | DISTRIBUTE BY __carto_partitioning SORT BY __carto_z2.min, __carto_z2.max 70 | | ); 71 | |""".stripMargin 72 | ) 73 | 74 | // configure the output 75 | val blockSize = computeBlockSize(s"${sourceTable}_idx_view") 76 | val conf = ssc.conf 77 | conf.set("parquet.block.size", blockSize) 78 | conf.set("spark.sql.parquet.compression.codec", compression) 79 | conf.set("spark.sql.files.maxRecordsPerFile", maxRecordsPerFile) 80 | 81 | // via SQL 82 | /*ssc.sql(s"SET parquet.block.size = $blockSize;") 83 | ssc.sql(s"SET spark.sql.parquet.compression.codec=$compression;") 84 | ssc.sql(s"SET spark.sql.files.maxRecordsPerFile=$maxRecordsPerFile;")*/ 85 | 86 | ssc.sql( 87 | s""" 88 | |CREATE TABLE $outputTable 89 | |USING PARQUET LOCATION '$outputLocation/$outputTable' 90 | |AS (SELECT * EXCEPT (__carto_partitioning, __carto_z2) FROM ${sourceTable}_idx_view); 91 | |""".stripMargin 92 | ) 93 | } 94 | 95 | /** automatically computes the block size */ 96 | def apply( 97 | sourceTable: String, 98 | outputTable: String, 99 | outputLocation: String, 100 | geomColumn: String, 101 | zoom: Int, 102 | blockSizeDefault: Long, 103 | compression: String, 104 | maxRecordsPerFile: Int 105 | )(implicit ssc: SparkSession): Unit = 106 | apply( 107 | sourceTable, 108 | outputTable, 109 | outputLocation, 110 | geomColumn, 111 | zoom, 112 | computeBlockSize = t => blockSizeCompute(t, blockSizeDefault), 113 | compression, 114 | maxRecordsPerFile 115 | ) 116 | 117 | /** TODO: improve heuristic */ 118 | def blockSizeCompute(table: String, blockSizeDefault: Long)(implicit ssc: SparkSession): Long = { 119 | val df = ssc.sql(s"SELECT __carto_partitioning FROM $table LIMIT 10;") 120 | val p = df.take(1).map(_.getLong(0)).headOption.getOrElse(0) 121 | val dfc = ssc.sql(s"SELECT COUNT(*) FROM $table WHERE __carto_partitioning = $p;") 122 | 123 | math.max(dfc.head.getLong(0) * 10 / 2, blockSizeDefault) 124 | } 125 | 126 | /** Optimization function defaults. */ 127 | val DEFAULT_OUTPUT_LOCATION: String = "/FileStore/tables/carto_default/" 128 | val DEFAULT_GEOM_COLUMN: String = "geom" 129 | val DEFAULT_ZOOM: Int = 8 130 | val DEFAULT_BLOCK_SIZE: Long = 2097000 131 | val DEFAULT_COMPRESSION: String = "lz4" 132 | val DEFAULT_MAX_RECORDS_PER_FILE: Int = 0 133 | } 134 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/spatial/package.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark 18 | 19 | import org.apache.spark.sql.SparkSession 20 | 21 | package object spatial extends Serializable { 22 | implicit class SparkSessionOps(val ssc: SparkSession) extends AnyVal { 23 | import OptimizeSpatial._ 24 | 25 | def optimizeSpatial( 26 | sourceTable: String, 27 | outputTable: String, 28 | outputLocation: String = DEFAULT_OUTPUT_LOCATION, 29 | geomColumn: String = DEFAULT_GEOM_COLUMN, 30 | zoom: Int = DEFAULT_ZOOM, 31 | blockSizeDefault: Long = DEFAULT_BLOCK_SIZE, 32 | compression: String = DEFAULT_COMPRESSION, 33 | maxRecordsPerFile: Int = DEFAULT_MAX_RECORDS_PER_FILE 34 | ): Unit = OptimizeSpatial(sourceTable, outputTable, outputLocation, geomColumn, zoom, blockSizeDefault, compression, maxRecordsPerFile)(ssc) 35 | 36 | def optimizeSpatialManual( 37 | sourceTable: String, 38 | outputTable: String, 39 | outputLocation: String = DEFAULT_OUTPUT_LOCATION, 40 | geomColumn: String = DEFAULT_GEOM_COLUMN, 41 | zoom: Int = DEFAULT_ZOOM, 42 | blockSize: Long = DEFAULT_BLOCK_SIZE, 43 | compression: String = DEFAULT_COMPRESSION, 44 | maxRecordsPerFile: Int = DEFAULT_MAX_RECORDS_PER_FILE 45 | ): Unit = OptimizeSpatial(sourceTable, outputTable, outputLocation, geomColumn, zoom, _ => blockSize, compression, maxRecordsPerFile)(ssc) 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/sql/SpatialFilterPushdownOptimizations.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.sql 18 | 19 | import com.carto.analyticstoolbox.spark.sql.rules.SpatialFilterPushdownRules 20 | import org.apache.spark.sql.SparkSessionExtensions 21 | 22 | class SpatialFilterPushdownOptimizations extends (SparkSessionExtensions => Unit) { 23 | def apply(e: SparkSessionExtensions): Unit = e.injectOptimizerRule(_ => SpatialFilterPushdownRules) 24 | } 25 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/sql/rules/STContainsRule.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.sql.rules 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | import com.carto.analyticstoolbox.index._ 21 | import com.azavea.hiveless.spark.rules.syntax._ 22 | import com.azavea.hiveless.serializers.HDeserializer.Errors.ProductDeserializationError 23 | import com.azavea.hiveless.serializers.syntax._ 24 | import geotrellis.vector._ 25 | import cats.syntax.option._ 26 | import org.apache.commons.lang3.exception.ExceptionUtils 27 | import org.apache.spark.sql.hive.HivelessInternals.HiveGenericUDF 28 | import org.apache.spark.sql.catalyst.expressions._ 29 | import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} 30 | import org.log4s.getLogger 31 | 32 | import scala.util.{Failure, Success, Try} 33 | 34 | object STContainsRule extends Serializable { 35 | @transient private[this] lazy val logger = getLogger 36 | 37 | def apply(f: Filter, condition: HiveGenericUDF, plan: LogicalPlan): Filter = 38 | try { 39 | val Seq(extentExpr, geometryExpr) = condition.children 40 | 41 | // ST_Contains is polymorphic by the first argument 42 | // Optimization is done only when the first argument is Extent 43 | if (!extentExpr.dataType.conformsToSchema(extentEncoder.schema)) 44 | throw new UnsupportedOperationException( 45 | s"${classOf[ST_Contains]} push-down optimization works on the ${classOf[Extent]} column data type only." 46 | ) 47 | 48 | // transform expression 49 | val expr = Try(geometryExpr.eval(null)) match { 50 | // Literals push-down support only 51 | case Success(g) => 52 | // ST_Contains is polymorphic by the second argument 53 | // Extract Extent literal from the right 54 | // The second argument can be Geometry or Extent 55 | val (extent, isGeometry) = Try(g.convert[Geometry].extent -> true) 56 | .orElse(Try(g.convert[Extent] -> false)) 57 | .getOrElse(throw ProductDeserializationError[ST_Contains, ST_Contains.Arg]("second")) 58 | 59 | // transform expression 60 | AndList( 61 | List( 62 | IsNotNull(extentExpr), 63 | GreaterThanOrEqual(GetStructField(extentExpr, 0, "xmin".some), Literal(extent.xmin)), 64 | GreaterThanOrEqual(GetStructField(extentExpr, 1, "ymin".some), Literal(extent.ymin)), 65 | LessThanOrEqual(GetStructField(extentExpr, 2, "xmax".some), Literal(extent.xmax)), 66 | LessThanOrEqual(GetStructField(extentExpr, 3, "ymax".some), Literal(extent.ymax)) 67 | // the old condition node is a secondary filter which is not pushed down 68 | // it is needed in case it is a Geometry intersection 69 | ) ::: (if (isGeometry) List(condition) else Nil) 70 | ) 71 | // Expression 72 | case Failure(_) => 73 | // In case on the right we have an Expression, no further optimizations needed and 74 | // such predicates won't be pushed down. 75 | // 76 | // In case Geometry is on the right, we can't extract Envelope coordinates, to perform it we need to define 77 | // User Defined Expression and that won't be pushed down. 78 | // 79 | // However, it is possible to extract coordinates out of Extent. 80 | // In this case the GetStructField can be used to extract values and transform the request, 81 | // though such predicates are not pushed down as well. 82 | // 83 | // The rough implementation of the idea above (The transformed plan for Extent, which is not pushed down): 84 | /*if (geometryExpr.dataType.conformsToSchema(extentEncoder.schema)) { 85 | AndList( 86 | List( 87 | IsNotNull(extentExpr), 88 | GreaterThanOrEqual(GetStructField(extentExpr, 0, "xmin".some), GetStructField(geometryExpr, 0, "xmin".some)), 89 | GreaterThanOrEqual(GetStructField(extentExpr, 1, "ymin".some), GetStructField(geometryExpr, 1, "ymin".some)), 90 | LessThanOrEqual(GetStructField(extentExpr, 2, "xmax".some), GetStructField(geometryExpr, 2, "xmax".some)), 91 | LessThanOrEqual(GetStructField(extentExpr, 3, "ymax".some), GetStructField(geometryExpr, 3, "ymax".some)) 92 | ) 93 | ) 94 | } else { 95 | throw new UnsupportedOperationException( 96 | s"${classOf[Geometry]} Envelope values extraction is not supported by the internal ${classOf[Geometry]} representation.".stripMargin 97 | ) 98 | }*/ 99 | 100 | throw new UnsupportedOperationException( 101 | s"${classOf[ST_Contains]} push-down optimization works with ${classOf[Geometry]} and ${classOf[Extent]} Literals only." 102 | ) 103 | } 104 | 105 | Filter(expr, plan) 106 | } catch { 107 | // fallback to the unoptimized node if optimization failed 108 | case e: Throwable => 109 | logger.warn( 110 | s""" 111 | |${this.getClass.getName} ${classOf[ST_Contains]} optimization failed, using the original plan. 112 | |StackTrace: ${ExceptionUtils.getStackTrace(e)} 113 | |""".stripMargin 114 | ) 115 | f 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/sql/rules/STIntersectsRule.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.sql.rules 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | import com.carto.analyticstoolbox.index._ 21 | import com.azavea.hiveless.spark.rules.syntax._ 22 | import com.azavea.hiveless.serializers.HDeserializer.Errors.ProductDeserializationError 23 | import com.azavea.hiveless.serializers.syntax._ 24 | import geotrellis.vector._ 25 | import cats.syntax.option._ 26 | import org.apache.commons.lang3.exception.ExceptionUtils 27 | import org.apache.spark.sql.hive.HivelessInternals.HiveGenericUDF 28 | import org.apache.spark.sql.catalyst.expressions._ 29 | import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} 30 | import org.log4s.getLogger 31 | 32 | import scala.util.{Failure, Success, Try} 33 | 34 | object STIntersectsRule extends Serializable { 35 | @transient private[this] lazy val logger = getLogger 36 | 37 | def apply(f: Filter, condition: HiveGenericUDF, plan: LogicalPlan): Filter = 38 | try { 39 | val Seq(extentExpr, geometryExpr) = condition.children 40 | 41 | // ST_Intersects is polymorphic by the first argument 42 | // Optimization is done only when the first argument is Extent 43 | if (!extentExpr.dataType.conformsToSchema(extentEncoder.schema)) 44 | throw new UnsupportedOperationException( 45 | s"${classOf[ST_Intersects]} push-down optimization works on the ${classOf[Extent]} column data type only." 46 | ) 47 | 48 | // transform expression 49 | val expr = Try(geometryExpr.eval(null)) match { 50 | // Literals push-down support only 51 | case Success(g) => 52 | // ST_Intersects is polymorphic by the second argument 53 | // Extract Extent literal from the right 54 | // The second argument can be Geometry or Extent 55 | val (extent, isGeometry) = Try(g.convert[Geometry].extent -> true) 56 | .orElse(Try(g.convert[Extent] -> false)) 57 | .getOrElse(throw ProductDeserializationError[ST_Intersects, ST_Intersects.Arg]("second")) 58 | 59 | // transform expression 60 | val expanded = 61 | And( 62 | IsNotNull(extentExpr), 63 | OrList( 64 | List( 65 | GreaterThanOrEqual(GetStructField(extentExpr, 0, "xmin".some), Literal(extent.xmin)), 66 | GreaterThanOrEqual(GetStructField(extentExpr, 1, "ymin".some), Literal(extent.ymin)), 67 | LessThanOrEqual(GetStructField(extentExpr, 2, "xmax".some), Literal(extent.xmax)), 68 | LessThanOrEqual(GetStructField(extentExpr, 3, "ymax".some), Literal(extent.ymax)) 69 | // the old condition node is a secondary filter which is not pushed down 70 | // it is needed in case it is a Geometry intersection 71 | ) 72 | ) 73 | ) 74 | 75 | if (isGeometry) And(expanded, condition) else expanded 76 | // Expression 77 | case Failure(_) => 78 | // In case on the right we have an Expression, no further optimizations needed and 79 | // such predicates won't be pushed down. 80 | // 81 | // In case Geometry is on the right, we can't extract Envelope coordinates, to perform it we need to define 82 | // User Defined Expression and that won't be pushed down. 83 | // 84 | // However, it is possible to extract coordinates out of Extent. 85 | // In this case the GetStructField can be used to extract values and transform the request, 86 | // though such predicates are not pushed down as well. 87 | // 88 | // The rough implementation of the idea above (The transformed plan for Extent, which is not pushed down): 89 | /*if (geometryExpr.dataType.conformsToSchema(extentEncoder.schema)) { 90 | And( 91 | IsNotNull(extentExpr), 92 | OrList( 93 | List( 94 | GreaterThanOrEqual(GetStructField(extentExpr, 0, "xmin".some), GetStructField(geometryExpr, 0, "xmin".some)), 95 | GreaterThanOrEqual(GetStructField(extentExpr, 1, "ymin".some), GetStructField(geometryExpr, 1, "ymin".some)), 96 | LessThanOrEqual(GetStructField(extentExpr, 2, "xmax".some), GetStructField(geometryExpr, 2, "xmax".some)), 97 | LessThanOrEqual(GetStructField(extentExpr, 3, "ymax".some), GetStructField(geometryExpr, 3, "ymax".some)) 98 | // the old condition node is a secondary filter which is not pushed down 99 | // it is needed in case it is a Geometry intersection 100 | ) 101 | ) 102 | ) 103 | } else { 104 | throw new UnsupportedOperationException( 105 | s"${classOf[Geometry]} Envelope values extraction is not supported by the internal ${classOf[Geometry]} representation.".stripMargin 106 | ) 107 | }*/ 108 | 109 | throw new UnsupportedOperationException( 110 | s"${classOf[ST_Intersects]} push-down optimization works with ${classOf[Geometry]} and ${classOf[Extent]} Literals only." 111 | ) 112 | } 113 | 114 | Filter(expr, plan) 115 | } catch { 116 | // fallback to the unoptimized node if optimization failed 117 | case e: Throwable => 118 | logger.warn( 119 | s""" 120 | |${this.getClass.getName} ${classOf[ST_Intersects]} optimization failed, using the original plan. 121 | |StackTrace: ${ExceptionUtils.getStackTrace(e)} 122 | |""".stripMargin 123 | ) 124 | f 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /core/src/main/scala/com/carto/analyticstoolbox/spark/sql/rules/SpatialFilterPushdownRules.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.sql.rules 18 | 19 | import com.carto.analyticstoolbox.core._ 20 | import com.azavea.hiveless.spark.rules.syntax._ 21 | import org.apache.spark.sql.hive.HivelessInternals.HiveGenericUDF 22 | import org.apache.spark.sql.SQLContext 23 | import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan} 24 | import org.apache.spark.sql.catalyst.rules.Rule 25 | 26 | object SpatialFilterPushdownRules extends Rule[LogicalPlan] { 27 | def apply(plan: LogicalPlan): LogicalPlan = 28 | // format: off 29 | /** 30 | * transform is an alias to transformDown 31 | * The transformDown usage causes the following error on DataBricks 9.1: 32 | * java.lang.NoClassDefFoundError: LogicalPlan.transformDown(Lscala/PartialFunction;)Lorg/apache/spark/sql/catalyst/plans/logical/LogicalPlan; 33 | */ 34 | // format: on 35 | plan.transform { 36 | case f @ Filter(condition: HiveGenericUDF, plan) if condition.of[ST_Intersects] => STIntersectsRule(f, condition, plan) 37 | case f @ Filter(condition: HiveGenericUDF, plan) if condition.of[ST_Contains] => STContainsRule(f, condition, plan) 38 | } 39 | 40 | def registerOptimizations(sqlContext: SQLContext): Unit = 41 | Seq(SpatialFilterPushdownRules).foreach { r => 42 | if (!sqlContext.experimental.extraOptimizations.contains(r)) 43 | sqlContext.experimental.extraOptimizations ++= Seq(r) 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /core/src/test/resources/polygons.csv: -------------------------------------------------------------------------------- 1 | address,bbl,wkt 2 | "",4050660250,"POLYGON((-73.83616 40.75531, -73.83622 40.75555, -73.83723 40.75507, -73.83775 40.75486, -73.83796 40.75479, -73.83833 40.75464, -73.8388 40.75445, -73.83932 40.75426, -73.83976 40.75412, -73.83984 40.7541, -73.83985 40.75405, -73.83985 40.75403, -73.83986 40.75402, -73.83987 40.75401, -73.83988 40.75401, -73.83989 40.754, -73.83993 40.75398, -73.83998 40.75397, -73.84004 40.75395, -73.8401 40.75393, -73.84014 40.75391, -73.84011 40.75385, -73.83984 40.75395, -73.84003 40.75355, -73.83999 40.75356, -73.83931 40.75407, -73.83914 40.75411, -73.83909 40.75405, -73.83884 40.75407, -73.83891 40.75412, -73.83776 40.75452, -73.83658 40.7551, -73.83623 40.75528, -73.83616 40.75531))" 3 | "",1002710036,"POLYGON((-73.98955 40.71278, -73.98958 40.71299, -73.98962 40.71299, -73.98961 40.71292, -73.98959 40.71278, -73.98955 40.71278))" 4 | "",4004900102,"POLYGON((-73.93651 40.77378, -73.93679 40.77387, -73.93681 40.77388, -73.93682 40.77389, -73.93685 40.7739, -73.93687 40.77392, -73.93689 40.77393, -73.93691 40.77395, -73.9371 40.77394, -73.93712 40.77385, -73.93714 40.77377, -73.93714 40.77368, -73.93713 40.7736, -73.93712 40.77351, -73.93709 40.77343, -73.93705 40.77335, -73.93701 40.77327, -73.93696 40.77319, -73.9369 40.77312, -73.93683 40.77305, -73.93675 40.77299, -73.93667 40.77293, -73.93658 40.77288, -73.93649 40.77283, -73.93639 40.77279, -73.93629 40.77275, -73.93603 40.77326, -73.93651 40.77341, -73.93659 40.77363, -73.93651 40.77378))" 5 | "",4024820051,"POLYGON((-73.89244 40.73341, -73.8923 40.73359, -73.8924 40.73359, -73.89262 40.73332, -73.89262 40.73331, -73.8927 40.73322, -73.89279 40.73309, -73.893 40.73281, -73.89299 40.7327, -73.89298 40.73271, -73.8929 40.73282, -73.89283 40.73292, -73.89267 40.73312, -73.89263 40.73318, -73.89257 40.73325, -73.89252 40.73331, -73.89244 40.73341))" 6 | "",4092410183,"POLYGON((-73.82746 40.70538, -73.82755 40.70541, -73.82771 40.70509, -73.82762 40.70506, -73.82749 40.70532, -73.82746 40.70538))" 7 | -------------------------------------------------------------------------------- /core/src/test/resources/polygons.snappy.parquet: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/CartoDB/poc-databricks/551d74d2a1d9ad276694ea3e924a2c5913cb5b21/core/src/test/resources/polygons.snappy.parquet -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/HiveTestEnvironment.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox 18 | 19 | import com.carto.analyticstoolbox.spark.sql.rules.SpatialFilterPushdownRules 20 | import geotrellis.spark.testkit.TestEnvironment 21 | import org.apache.spark.SparkConf 22 | import org.apache.spark.serializer.KryoSerializer 23 | import org.apache.spark.sql.{SQLContext, SparkSession} 24 | import org.scalatest.{BeforeAndAfterAll, Suite} 25 | 26 | import java.io.File 27 | import scala.io.Source 28 | import scala.util.Properties 29 | 30 | trait HiveTestEnvironment extends TestEnvironment { self: Suite with BeforeAndAfterAll => 31 | import HiveTestEnvironment._ 32 | 33 | def loadSQL(path: String): List[String] = 34 | Source 35 | .fromFile(new File(path).toURI) 36 | .using(_.mkString.split(";").toList.map(_.trim).filter(_.nonEmpty)) 37 | 38 | def spatialFunctions: List[String] = loadSQL("../core/sql/createUDFs.sql") 39 | 40 | // function to override Hive SQL functions registration 41 | def registerHiveUDFs(ssc: SparkSession): Unit = 42 | spatialFunctions.foreach(ssc.sql) 43 | 44 | // function to override optimizations 45 | def registerOptimizations(sqlContext: SQLContext): Unit = 46 | SpatialFilterPushdownRules.registerOptimizations(sqlContext) 47 | 48 | def addSparkConfigProperties(config: SparkConf): Unit = {} 49 | 50 | // returns (warehouseDir, derbyConnectionURL) 51 | def warehouseLocation: (String, String) = { 52 | val tmpDir = System.getProperty("java.io.tmpdir") 53 | // a separate warehouse for each spec, JDK 8 is unhappy with the old directory being populated 54 | val wdir = s"${tmpDir}/cartoanalyticstoolbox-warehouse/${self.getClass.getName}" 55 | val ddir = s"${tmpDir}/cartoanalyticstoolbox-db/${self.getClass.getName}" 56 | val connectionURL = s"jdbc:derby:;databaseName=${ddir};create=true" 57 | (wdir, connectionURL) 58 | } 59 | 60 | lazy val sparkConfig: SparkConf = { 61 | val (warehouseDir, derbyConnectionURL) = warehouseLocation 62 | val conf = new SparkConf() 63 | conf 64 | .setMaster(sparkMaster) 65 | .setAppName("Test Hive Context") 66 | .set("spark.default.parallelism", "4") 67 | // Since Spark 3.2.0 this flag is set to true by default 68 | // We need it to be set to false, since it is required by the HBase TableInputFormat 69 | .set("spark.hadoopRDD.ignoreEmptySplits", "false") 70 | .set("spark.sql.warehouse.dir", warehouseDir) 71 | .set("javax.jdo.option.ConnectionURL", derbyConnectionURL) 72 | 73 | // Shortcut out of using Kryo serialization if we want to test against 74 | // java serialization. 75 | if (Properties.envOrNone("GEOTRELLIS_USE_JAVA_SER").isEmpty) { 76 | conf 77 | .set("spark.serializer", classOf[KryoSerializer].getName) 78 | .set("spark.kryoserializer.buffer.max", "500m") 79 | .set("spark.kryo.registrationRequired", "false") 80 | setKryoRegistrator(conf) 81 | } 82 | 83 | addSparkConfigProperties(conf) 84 | conf 85 | } 86 | 87 | // override the SparkSession construction to enable Hive support 88 | override lazy val _ssc: SparkSession = { 89 | System.setProperty("spark.driver.port", "0") 90 | System.setProperty("spark.hostPort", "0") 91 | System.setProperty("spark.ui.enabled", "false") 92 | 93 | val sparkContext = 94 | SparkSession 95 | .builder() 96 | .config(sparkConfig) 97 | .enableHiveSupport() 98 | .getOrCreate() 99 | 100 | System.clearProperty("spark.driver.port") 101 | System.clearProperty("spark.hostPort") 102 | System.clearProperty("spark.ui.enabled") 103 | 104 | registerOptimizations(sparkContext.sqlContext) 105 | registerHiveUDFs(sparkContext) 106 | 107 | sparkContext 108 | } 109 | } 110 | 111 | object HiveTestEnvironment { 112 | implicit class AutoCloseableOps[A <: AutoCloseable](val resource: A) extends AnyVal { 113 | def using[B](f: A => B): B = try f(resource) 114 | finally resource.close() 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/InjectOptimizerTestEnvironment.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox 18 | 19 | import com.carto.analyticstoolbox.spark.sql.SpatialFilterPushdownOptimizations 20 | import org.apache.spark.SparkConf 21 | import org.apache.spark.sql.SQLContext 22 | import org.scalatest.{BeforeAndAfterAll, Suite} 23 | 24 | trait InjectOptimizerTestEnvironment extends HiveTestEnvironment { self: Suite with BeforeAndAfterAll => 25 | 26 | // disable manual optimizations registration 27 | override def registerOptimizations(sqlContext: SQLContext): Unit = {} 28 | 29 | // enable plan optimizations by using the plan injector 30 | override def addSparkConfigProperties(config: SparkConf): Unit = 31 | config.set("spark.sql.extensions", classOf[SpatialFilterPushdownOptimizations].getName) 32 | } 33 | -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/TestTables.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox 18 | 19 | import java.io.File 20 | 21 | trait TestTables { self: HiveTestEnvironment => 22 | // core/src/test/resources/polygons.csv when JVM is not forked 23 | def uriCSV: String = new File("../core/src/test/resources/polygons.csv").toURI.toString 24 | def uriParquet: String = new File("../core/src/test/resources/polygons.snappy.parquet").toURI.toString 25 | 26 | // create tmp views 27 | ssc.read 28 | .option("delimiter", ",") 29 | .option("header", "true") 30 | .csv(uriCSV) 31 | .createOrReplaceTempView("polygons_csv") 32 | 33 | ssc.read 34 | .parquet(uriParquet) 35 | .createOrReplaceTempView("polygons_parquet") 36 | 37 | // create view with a parsed geometry and bbox columns 38 | def createViews(): Unit = 39 | ssc.sql( 40 | """ 41 | |CREATE TEMPORARY VIEW polygons_csv_view AS ( 42 | | SELECT *, ST_GeomFromWKT(wkt) AS geom, ST_ExtentFromGeom(ST_GeomFromWKT(wkt)) as bbox FROM polygons_csv 43 | |); 44 | |""".stripMargin 45 | ) 46 | 47 | createViews() 48 | 49 | // Parquet generation 50 | def createParquet(drop: Boolean = false): Unit = { 51 | if (drop) ssc.sql("DROP TABLE polygons_parquet;") 52 | ssc.sql( 53 | """ 54 | |CREATE TABLE polygons_parquet 55 | |USING PARQUET LOCATION '/tmp/polygons_parquet' 56 | |AS (SELECT * FROM polygons_csv_view); 57 | |""".stripMargin 58 | ) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/core/STCoreSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.core 18 | 19 | import com.carto.analyticstoolbox.{HiveTestEnvironment, TestTables} 20 | import org.scalatest.funspec.AnyFunSpec 21 | 22 | class STCoreSpec extends AnyFunSpec with HiveTestEnvironment with TestTables { 23 | describe("ST Core functions spec") { 24 | it("ST_Intersects should filter a CSV view") { 25 | val df = ssc.sql( 26 | """ 27 | |SELECT * FROM polygons_csv_view WHERE ST_Intersects(geom, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 28 | |""".stripMargin 29 | ) 30 | 31 | df.count() shouldBe 5 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/index/STIndexSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.index 18 | 19 | import com.carto.analyticstoolbox.{HiveTestEnvironment, TestTables} 20 | import com.azavea.hiveless.spark.encoders.syntax._ 21 | import geotrellis.vector.Extent 22 | import org.apache.spark.sql.catalyst.plans.logical.Filter 23 | import org.scalatest.funspec.AnyFunSpec 24 | 25 | class STIndexSpec extends AnyFunSpec with HiveTestEnvironment with TestTables { 26 | 27 | describe("ST Index functions spec") { 28 | it("ST_ExtentFromGeom") { 29 | val df = ssc.sql( 30 | """ 31 | |SELECT ST_ExtentFromGeom(ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 32 | |""".stripMargin 33 | ) 34 | 35 | df.head().getStruct(0).as[Extent] shouldBe Extent(-75.5859375, 40.3251777, -72.4101562, 43.1971673) 36 | } 37 | it("ST_Intersects should filter a CSV file") { 38 | val df = ssc.sql( 39 | """ 40 | |SELECT * FROM polygons_csv_view WHERE ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 41 | |""".stripMargin 42 | ) 43 | 44 | df.count() shouldBe 5 45 | } 46 | 47 | it("ST_Intersects should filter a Parquet file") { 48 | val df = ssc.sql( 49 | """ 50 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 51 | |""".stripMargin 52 | ) 53 | 54 | df.count() shouldBe 5 55 | } 56 | 57 | it("ST_Intersects plan should be optimized") { 58 | val df = ssc.sql( 59 | """ 60 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 61 | |""".stripMargin 62 | ) 63 | 64 | val dfe = ssc.sql( 65 | """ 66 | |SELECT * FROM polygons_parquet 67 | |WHERE isNotNull(bbox) 68 | |AND (((bbox.xmin >= -75.5859375 69 | |OR bbox.ymin >= 40.3251777) 70 | |OR bbox.xmax <= -72.4101562) 71 | |OR bbox.ymax <= 43.1971673) 72 | |AND ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 73 | |""".stripMargin 74 | ) 75 | 76 | df.count() shouldBe dfe.count() 77 | 78 | // compare optimized plans filters 79 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 80 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 81 | 82 | dfc shouldBe dfec 83 | } 84 | 85 | it("ST_Intersects by Extent plan should be optimized") { 86 | val df = ssc.sql( 87 | """ 88 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, ST_MakeExtent(-75.5859375, 40.3251777, -72.4101562, 43.1971673)) 89 | |""".stripMargin 90 | ) 91 | 92 | val dfe = ssc.sql( 93 | """ 94 | |SELECT * FROM polygons_parquet 95 | |WHERE isNotNull(bbox) 96 | |AND ((((bbox.xmin >= -75.5859375) 97 | |OR bbox.ymin >= 40.3251777) 98 | |OR bbox.xmax <= -72.4101562) 99 | |OR bbox.ymax <= 43.1971673) 100 | |""".stripMargin 101 | ) 102 | 103 | df.count() shouldBe dfe.count() 104 | 105 | // compare optimized plans filters 106 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 107 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 108 | 109 | dfc shouldBe dfec 110 | } 111 | 112 | it("ST_Intersects optimization failure (Extent, Extent)") { 113 | val df = ssc.sql( 114 | """ 115 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, bbox) 116 | |""".stripMargin 117 | ) 118 | 119 | val dfe = ssc.sql( 120 | """ 121 | |SELECT * FROM polygons_parquet 122 | |WHERE isNotNull(bbox) 123 | |AND ((((bbox.xmin >= -75.5859375) 124 | |OR bbox.ymin >= 40.3251777) 125 | |OR bbox.xmax <= -72.4101562) 126 | |OR bbox.ymax <= 43.1971673) 127 | |""".stripMargin 128 | ) 129 | 130 | df.count() shouldBe dfe.count() 131 | 132 | // compare optimized plans filters 133 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 134 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 135 | 136 | dfc shouldNot be(dfec) 137 | } 138 | 139 | it("ST_Intersects optimization failure (Extent, Geometry)") { 140 | val df = ssc.sql( 141 | """ 142 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, geom) 143 | |""".stripMargin 144 | ) 145 | 146 | val dfe = ssc.sql( 147 | """ 148 | |SELECT * FROM polygons_parquet 149 | |WHERE isNotNull(bbox) 150 | |AND ((((bbox.xmin >= -75.5859375) 151 | |OR bbox.ymin >= 40.3251777) 152 | |OR bbox.xmax <= -72.4101562) 153 | |OR bbox.ymax <= 43.1971673) 154 | |""".stripMargin 155 | ) 156 | 157 | df.count() shouldBe dfe.count() 158 | 159 | // compare optimized plans filters 160 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 161 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 162 | 163 | dfc shouldNot be(dfec) 164 | } 165 | 166 | it("ST_Intersects optimization failure (Geometry, Geometry)") { 167 | val df = ssc.sql( 168 | """ 169 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(geom, geom) 170 | |""".stripMargin 171 | ) 172 | 173 | val dfe = ssc.sql( 174 | """ 175 | |SELECT * FROM polygons_parquet 176 | |WHERE isNotNull(bbox) 177 | |AND ((((bbox.xmin >= -75.5859375) 178 | |OR bbox.ymin >= 40.3251777) 179 | |OR bbox.xmax <= -72.4101562) 180 | |OR bbox.ymax <= 43.1971673) 181 | |""".stripMargin 182 | ) 183 | 184 | df.count() shouldBe dfe.count() 185 | 186 | // compare optimized plans filters 187 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 188 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 189 | 190 | dfc shouldNot be(dfec) 191 | } 192 | 193 | it("ST_Intersects optimization failure (Geometry, Extent)") { 194 | val df = ssc.sql( 195 | """ 196 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(geom, bbox) 197 | |""".stripMargin 198 | ) 199 | 200 | val dfe = ssc.sql( 201 | """ 202 | |SELECT * FROM polygons_parquet 203 | |WHERE isNotNull(bbox) 204 | |AND ((((bbox.xmin >= -75.5859375) 205 | |OR bbox.ymin >= 40.3251777) 206 | |OR bbox.xmax <= -72.4101562) 207 | |OR bbox.ymax <= 43.1971673) 208 | |""".stripMargin 209 | ) 210 | 211 | df.count() shouldBe dfe.count() 212 | 213 | // compare optimized plans filters 214 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 215 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 216 | 217 | dfc shouldNot be(dfec) 218 | } 219 | } 220 | 221 | it("ST_Contains plan should be optimized") { 222 | val df = ssc.sql( 223 | """ 224 | |SELECT * FROM polygons_parquet WHERE ST_Contains(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 225 | |""".stripMargin 226 | ) 227 | 228 | val dfe = ssc.sql( 229 | """ 230 | |SELECT * FROM polygons_parquet 231 | |WHERE bbox.xmin >= -75.5859375 232 | |AND bbox.ymin >= 40.3251777 233 | |AND bbox.xmax <= -72.4101562 234 | |AND bbox.ymax <= 43.1971673 235 | |AND ST_Contains(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 236 | |""".stripMargin 237 | ) 238 | 239 | df.count() shouldBe dfe.count() 240 | 241 | // compare optimized plans filters 242 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 243 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 244 | 245 | dfc shouldBe dfec 246 | } 247 | 248 | it("H3_ToParent") { 249 | val df = ssc.sql(s"SELECT H3_ToParent(617700169958293503, 7);") 250 | 251 | // 0x872830828ffffffL shouldBe h3.h3ToParent(0x8928308280fffffL, 7) 252 | // https://github.com/uber/h3-java/blob/f628b6e0fcbd330122ee80ef062b0d59b2450c00/src/test/java/com/uber/h3core/v3/TestHierarchy.java#L36 253 | df.collect().head.getLong(0) shouldBe 0x872830828ffffffL 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /core/src/test/scala/com/carto/analyticstoolbox/spark/sql/STIndexInjectorSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2022 Azavea 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.carto.analyticstoolbox.spark.sql 18 | 19 | import com.carto.analyticstoolbox.{InjectOptimizerTestEnvironment, TestTables} 20 | import org.apache.spark.sql.catalyst.plans.logical.Filter 21 | import org.scalatest.funspec.AnyFunSpec 22 | 23 | /** Different from STIndexSpec only by using a different approach to enable optimizations. */ 24 | class STIndexInjectorSpec extends AnyFunSpec with InjectOptimizerTestEnvironment with TestTables { 25 | 26 | describe("ST Index injector spec") { 27 | it("ST_Intersects plan should be optimized") { 28 | val df = ssc.sql( 29 | """ 30 | |SELECT * FROM polygons_parquet WHERE ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 31 | |""".stripMargin 32 | ) 33 | 34 | val dfe = ssc.sql( 35 | """ 36 | |SELECT * FROM polygons_parquet 37 | |WHERE isNotNull(bbox) 38 | |AND (((bbox.xmin >= -75.5859375 39 | |OR bbox.ymin >= 40.3251777) 40 | |OR bbox.xmax <= -72.4101562) 41 | |OR bbox.ymax <= 43.1971673) 42 | |AND ST_Intersects(bbox, ST_GeomFromGeoJSON('{"type":"Polygon","coordinates":[[[-75.5859375,40.32517767999294],[-75.5859375,43.197167282501276],[-72.41015625,43.197167282501276],[-72.41015625,40.32517767999294],[-75.5859375,40.32517767999294]]]}')) 43 | |""".stripMargin 44 | ) 45 | 46 | df.count() shouldBe dfe.count() 47 | 48 | // compare optimized plans filters 49 | val dfc = df.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 50 | val dfec = dfe.queryExecution.optimizedPlan.collect { case Filter(condition, _) => condition } 51 | 52 | dfc shouldBe dfec 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=1.6.2 2 | 3 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("com.timushev.sbt" % "sbt-updates" % "0.6.1") 2 | addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.6.5") 3 | addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3") 4 | addSbtPlugin("com.github.sbt" % "sbt-ci-release" % "1.5.10") 5 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.2.0") 6 | --------------------------------------------------------------------------------