├── .gitignore ├── LICENSE ├── README.md ├── README_zh.md ├── examples ├── README.md ├── preview-example.yml ├── preview-style.json ├── preview.html └── preview.png ├── pom.xml ├── scripts └── SimpleWebServer.sh └── src ├── main └── scala │ └── org │ ├── apache │ └── spark │ │ └── sql │ │ ├── GeometricUtils.scala │ │ └── SQLGeometricExtensions.scala │ └── ieee │ └── codemeow │ └── geometric │ ├── CRSUtils.scala │ ├── Feature.scala │ ├── GeometricUtils.scala │ ├── MvtBuilder.scala │ └── spark │ ├── Configurations.scala │ ├── VectorTileTask.scala │ └── data │ ├── AbstractDataProvider.scala │ └── SQLDataProvider.scala └── test └── scala └── samples ├── junit.scala ├── scalatest.scala └── specs.scala /.gitignore: -------------------------------------------------------------------------------- 1 | *.iml 2 | dependency-reduced-pom.xml 3 | .idea/ 4 | target/ 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Alexander.Zhou 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Vector Tile Spark Process 2 | 3 | The ***Vector Tile Spark Process*** allows developers and data scientists clip geographic data into Hadoop SequeueFiles on Spark platform. 4 | 5 | The effect comes from [Preview Example](https://github.com/codemeow5/Vector-Tile-Spark-Process/tree/master/examples). 6 | ![Preview](examples/preview.png) 7 | 8 | ## Features 9 | 10 | - Parallel processing based on Apache Spark 11 | - Adapting various data sources 12 | - Output standard [Mapbox Vector Tiles](https://github.com/mapbox/vector-tile-spec/tree/master/2.1 "Vector Tile Specification") format 13 | - A configuration file similar to [TileStache.Goodies.VecTiles.Provider](https://github.com/TileStache/TileStache/blob/master/TileStache/Goodies/VecTiles/server.py) 14 | 15 | ## Dependencies 16 | 17 | - [GeoTools](http://www.geotools.org/ "GeoTools") - an open source Java library that provides tools for geospatial data 18 | - [mapbox-vector-tile-java](https://github.com/wdtinc/mapbox-vector-tile-java "mapbox-vector-tile-java") - Java Mapbox Vector Tile Library for Encoding/Decoding 19 | 20 | ## Requirements 21 | 22 | - Hadoop 2.7 and later 23 | - Spark 2.1.1 and above 24 | - Protocol Buffers 3.0.0-beta-2 25 | 26 | ## Getting Started 27 | 28 | ### Build 29 | 30 | $ mvn clean && mvn package 31 | 32 | ### Run 33 | 34 | $SPARK_HOME/bin/spark-submit --class org.ieee.codemeow.geometric.spark.VectorTileTask --master yarn --deploy-mode cluster --jars /path/to/postgresql-42.0.0.jar --driver-class-path /path/to/postgresql-42.0.0.jar /path/to/vectortile-spark-process-1.0-SNAPSHOT.jar hdfs:///path/to/vectortile-spark-process.yml 35 | 36 | ## Configuration File 37 | 38 | --- 39 | # vectortile-spark-process.yml 40 | 41 | appName: "Vector Tile Process" 42 | sequenceFileDir: "hdfs:///path/to" 43 | layers: 44 | - layerName: "layerName" 45 | minZoom: "0" 46 | maxZoom: "22" 47 | dataProvider: "org.ieee.codemeow.geometric.spark.data.SQLDataProvider" 48 | kwargs: 49 | url: "jdbc:postgresql://hostname/dbname" 50 | dbtables: 51 | planet_osm_line: "public.planet_osm_line" 52 | planet_osm_point: "public.planet_osm_point" 53 | planet_osm_polygon: "public.planet_osm_polygon" 54 | planet_osm_roads: "public.planet_osm_roads" 55 | user: "postgres" 56 | password: "postgres" 57 | zooms: 58 | 0: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 59 | 1: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 60 | ... 61 | 22: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 62 | 63 | ## Resources 64 | 65 | - http://tilestache.org/ 66 | - https://github.com/locationtech/geomesa 67 | - http://www.geomesa.org/documentation/current/user/spark/sparksql_functions.html 68 | - https://github.com/mapbox/awesome-vector-tiles 69 | - https://github.com/DataSystemsLab/GeoSpark 70 | - https://github.com/Esri/spatial-framework-for-hadoop 71 | - https://github.com/gbif/maps 72 | - https://github.com/mapbox/mercantile 73 | - https://github.com/modestmaps/modestmaps-processing 74 | - http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/ 75 | 76 | ## Issues 77 | 78 | Find a bug or want to request a new feature? Please let us know by submitting an issue. 79 | 80 | ## Tips 81 | 82 | 1. Upgrade protobuf package version on your Spark cluster 83 | 84 | >cp protobuf-java-3.0.0-beta-2.jar $SPARK_HOME/jars 85 | 86 | 2. Use SparkSQL in the ***zooms*** section of the configuration file 87 | -------------------------------------------------------------------------------- /README_zh.md: -------------------------------------------------------------------------------- 1 | # Vector Tile Spark Process 2 | 3 | ***Vector Tile Spark Process*** 允许开发者和数据研究员使用Spark平台将地理空间数据切分到Hadoop SequeueFiles文件中. 4 | 5 | 效果来自于我们用于[预览的例子](https://github.com/codemeow5/Vector-Tile-Spark-Process/tree/master/examples).   6 | ![Preview](examples/preview.png) 7 | 8 | ## 特性 9 | 10 | - 基于Apache Spark进行并行处理 11 | - 适配多种数据源 12 | - 输出标准 [Mapbox Vector Tiles](https://github.com/mapbox/vector-tile-spec/tree/master/2.1 "Vector Tile Specification") 格式 13 | - 类似于 [TileStache.Goodies.VecTiles.Provider](https://github.com/TileStache/TileStache/blob/master/TileStache/Goodies/VecTiles/server.py)的配置文件 14 | 15 | ## 依赖项 16 | 17 | - [GeoTools](http://www.geotools.org/ "GeoTools") - 提供空间数据工具的开源Java库 18 | - [mapbox-vector-tile-java](https://github.com/wdtinc/mapbox-vector-tile-java "mapbox-vector-tile-java") - Map Vector Tile 编解码Java库 19 | 20 | ## 要求 21 | 22 | - Hadoop 2.7 及以上 23 | - Spark 2.1.1 及以上 24 | - Protocol Buffers 3.0.0-beta-2 25 | 26 | ## 开始工作 27 | 28 | ### 构建 29 | 30 | $ mvn clean && mvn package 31 | 32 | ### 运行 33 | 34 | $SPARK_HOME/bin/spark-submit --class org.ieee.codemeow.geometric.spark.VectorTileTask --master yarn --deploy-mode cluster --jars /path/to/postgresql-42.0.0.jar --driver-class-path /path/to/postgresql-42.0.0.jar /path/to/vectortile-spark-process-1.0-SNAPSHOT.jar hdfs:///path/to/vectortile-spark-process.yml 35 | 36 | ## 配置文件 37 | 38 | --- 39 | # vectortile-spark-process.yml 40 | 41 | appName: "Vector Tile Process" 42 | sequenceFileDir: "hdfs:///path/to" 43 | layers: 44 | - layerName: "layerName" 45 | minZoom: "0" 46 | maxZoom: "22" 47 | dataProvider: "org.ieee.codemeow.geometric.spark.data.SQLDataProvider" 48 | kwargs: 49 | url: "jdbc:postgresql://hostname/dbname" 50 | dbtables: 51 | planet_osm_line: "public.planet_osm_line" 52 | planet_osm_point: "public.planet_osm_point" 53 | planet_osm_polygon: "public.planet_osm_polygon" 54 | planet_osm_roads: "public.planet_osm_roads" 55 | user: "postgres" 56 | password: "postgres" 57 | zooms: 58 | 0: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 59 | 1: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 60 | ... 61 | 22: "SELECT osm_id AS __id__, ST_GeomFromWKB(way) AS __geometry__ FROM ..." 62 | 63 | ## 资源 64 | 65 | - http://tilestache.org/ 66 | - https://github.com/locationtech/geomesa 67 | - http://www.geomesa.org/documentation/current/user/spark/sparksql_functions.html 68 | - https://github.com/mapbox/awesome-vector-tiles 69 | - https://github.com/DataSystemsLab/GeoSpark 70 | - https://github.com/Esri/spatial-framework-for-hadoop 71 | - https://github.com/gbif/maps 72 | - https://github.com/mapbox/mercantile 73 | - https://github.com/modestmaps/modestmaps-processing 74 | - http://www.maptiler.org/google-maps-coordinates-tile-bounds-projection/ 75 | 76 | ## 问题 77 | 78 | 如果你找到一些问题或者希望我们提供一些新特性，请通过提交Issue的方式通知我们. 79 | 80 | ## 贴士 81 | 82 | 1. 升级你的Spark集群的protobuf版本 83 | 84 | >cp protobuf-java-3.0.0-beta-2.jar $SPARK_HOME/jars 85 | 86 | 2. 在配置文件的***zooms***节，请使用SparkSQL 87 | -------------------------------------------------------------------------------- /examples/README.md: -------------------------------------------------------------------------------- 1 | # Preview Example 2 | 3 | Processing Beijing OpenStreetMap data and Preview it 4 | 5 | ![Preview](preview.png) 6 | 7 | ## Download raw data and import to PostGIS database 8 | 9 | 1. Download [beijing_china.osm.pbf](https://s3.amazonaws.com/metro-extracts.mapzen.com/beijing_china.osm.pbf) from [Mapzen](https://mapzen.com/data/metro-extracts/metro/beijing_china/) 10 | 2. Install Osm2pgsql to your enviroment ([http://wiki.openstreetmap.org/wiki/Osm2pgsql#Installation](http://wiki.openstreetmap.org/wiki/Osm2pgsql#Installation)) 11 | 3. Import data into PostgreSQL database 12 | 13 | $ osm2pgsql -c -d beijing\_china\_osm -U postgres -W -H [PGHOST] beijing_china.osm.pbf 14 | 15 | 4. Add column ***wkb_way*** 16 | 17 | ALTER TABLE public.planet_osm_line ADD wkb_way bytea; 18 | UPDATE public.planet_osm_line SET wkb_way = ST_AsBinary(way); 19 | 20 | ## Update your configuration file 21 | 22 | --- 23 | # preview-example.yml 24 | 25 | appName: "Preview Example Task" 26 | sequenceFileDir: "hdfs:///preview_data" 27 | layers: 28 | - layerName: "lines" 29 | minZoom: "3" 30 | maxZoom: "18" 31 | dataProvider: "org.ieee.codemeow.geometric.spark.data.SQLDataProvider" 32 | kwargs: 33 | url: "jdbc:postgresql://hostname/beijing_china_osm" 34 | dbtables: 35 | planet_osm_line: "public.planet_osm_line" 36 | user: "postgres" 37 | password: "postgres" 38 | zooms: 39 | 3: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 40 | 4: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 41 | 5: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 42 | 6: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 43 | 7: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 44 | 8: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 45 | 9: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 46 | 10: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 47 | 11: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 48 | 12: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 49 | 13: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 50 | 14: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 51 | 15: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 52 | 16: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 53 | 17: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 54 | 18: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 55 | 56 | ## Run Spark Job 57 | 58 | $SPARK_HOME/bin/spark-submit --class org.ieee.codemeow.geometric.spark.VectorTileTask --master yarn --deploy-mode cluster --jars /path/to/postgresql-42.0.0.jar --driver-class-path /path/to/postgresql-42.0.0.jar /path/to/vectortile-spark-process-1.0-SNAPSHOT.jar hdfs:///path/to/preview-example.yml 59 | 60 | ## Run Simple Web Server 61 | 62 | #### Install SBT scripts runtime ([http://www.scala-sbt.org/0.13/docs/Scripts.html](http://www.scala-sbt.org/0.13/docs/Scripts.html)) 63 | 64 | // Installing conscript 65 | export CONSCRIPT_HOME="$HOME/.conscript" 66 | export PATH=$CONSCRIPT_HOME/bin:$PATH 67 | wget https://raw.githubusercontent.com/foundweekends/conscript/master/setup.sh -O - | sh 68 | 69 | //Install command screpl and scalas 70 | cs sbt/sbt --branch 0.13 71 | 72 | #### Start server 73 | 74 | ./scripts/SimpleWebServer.sh hdfs://hostname:port/preview_data 75 | 76 | ## Open preview.html 77 | 78 | 1. Modify server address on ***preview-style.json*** 79 | 2. Copy ***preview.html*** and ***preview-style.json*** to web root directory 80 | 3. Visit ***http://127.0.0.1/preivew.html*** 81 | 82 | 83 | -------------------------------------------------------------------------------- /examples/preview-example.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # preview-example.yml 3 | 4 | appName: "Preview Example Task" 5 | sequenceFileDir: "hdfs:///preview_data" 6 | layers: 7 | - layerName: "lines" 8 | minZoom: "3" 9 | maxZoom: "18" 10 | dataProvider: "org.ieee.codemeow.geometric.spark.data.SQLDataProvider" 11 | kwargs: 12 | url: "jdbc:postgresql://hostname/beijing_china_osm" 13 | dbtables: 14 | planet_osm_line: "public.planet_osm_line" 15 | user: "postgres" 16 | password: "postgres" 17 | zooms: 18 | 3: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 19 | 4: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 20 | 5: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 21 | 6: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 22 | 7: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 23 | 8: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 24 | 9: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 25 | 10: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 26 | 11: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 27 | 12: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 28 | 13: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 29 | 14: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 30 | 15: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 31 | 16: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 32 | 17: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 33 | 18: "SELECT osm_id AS __id__, ST_GeomFromWKB(wkb_way) AS __geometry__ FROM planet_osm_line" 34 | -------------------------------------------------------------------------------- /examples/preview-style.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": 8, 3 | "name": "Preview Style For Vector Tile Spark Process", 4 | "sources": { 5 | "simple_web_server": { 6 | "type": "vector", 7 | "tileSize": 512, 8 | "tiles": [ 9 | "http://127.0.0.1:9000/lines/{z}/{x}/{y}.pbf" 10 | ], 11 | "minzoom": 3, 12 | "maxzoom": 18 13 | } 14 | }, 15 | "layers": [ 16 | { 17 | "id": "lines", 18 | "type": "line", 19 | "source": "simple_web_server", 20 | "source-layer": "lines", 21 | "paint": { 22 | "line-color": "#333333" 23 | } 24 | } 25 | ] 26 | } -------------------------------------------------------------------------------- /examples/preview.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Preview - Vector Tile Spark Process 6 | 7 | 8 | 9 | 10 |
11 | 28 | 29 | -------------------------------------------------------------------------------- /examples/preview.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bluenoah1991/Vector-Tile-Spark-Process/2b75f0947b45c991a83c3ec72311fccc0aa7a548/examples/preview.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | org.ieee.codemeow 4 | vectortile-spark-process 5 | 1.0-SNAPSHOT 6 | vectortile-spark-process 7 | 8 | 2017 9 | 10 | 11 | MIT License 12 | http://www.opensource.org/licenses/mit-license.php 13 | repo 14 | 15 | 16 | 17 | 1.7 18 | 1.7 19 | UTF-8 20 | 2.10 21 | 2.11.8 22 | 2.1.0 23 | 2.4.4 24 | 25 | 26 | 27 | 28 | osgeo 29 | Open Source Geospatial Foundation Repository 30 | http://download.osgeo.org/webdav/geotools/ 31 | 32 | 33 | 34 | 35 | 36 | org.scala-lang 37 | scala-library 38 | ${scala.version} 39 | 40 | 41 | org.apache.spark 42 | spark-core_${scala.tools.version} 43 | ${spark.version} 44 | provided 45 | 46 | 47 | org.apache.spark 48 | spark-sql_${scala.tools.version} 49 | ${spark.version} 50 | provided 51 | 52 | 53 | org.geotools 54 | gt-api 55 | 17.0 56 | 57 | 58 | com.wdtinc 59 | mapbox-vector-tile 60 | 1.1.1 61 | 62 | 63 | com.fasterxml.jackson.dataformat 64 | jackson-dataformat-yaml 65 | 66 | ${jackson.spark.version} 67 | 68 | 69 | com.fasterxml.jackson.core 70 | jackson-annotations 71 | 72 | ${jackson.spark.version} 73 | 74 | 75 | com.fasterxml.jackson.core 76 | jackson-core 77 | 78 | ${jackson.spark.version} 79 | 80 | 81 | 82 | 83 | junit 84 | junit 85 | 4.11 86 | test 87 | 88 | 89 | org.specs2 90 | specs2_${scala.tools.version} 91 | 1.13 92 | test 93 | 94 | 95 | org.scalatest 96 | scalatest_${scala.tools.version} 97 | 2.0.M6-SNAP8 98 | test 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | net.alchim31.maven 111 | scala-maven-plugin 112 | 3.2.0 113 | 114 | 115 | 116 | add-source 117 | compile 118 | 119 | 120 | 121 | 122 | -dependencyfile 123 | ${project.build.directory}/.scala_dependencies 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | org.apache.maven.plugins 132 | maven-surefire-plugin 133 | 2.18.1 134 | 135 | false 136 | true 137 | 138 | false 139 | 140 | **/*Test.* 141 | **/*Suite.* 142 | 143 | 144 | 145 | 146 | 147 | org.apache.maven.plugins 148 | maven-shade-plugin 149 | 2.3 150 | 151 | 152 | package 153 | 154 | shade 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | -------------------------------------------------------------------------------- /scripts/SimpleWebServer.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env scalas 2 | 3 | /*** 4 | scalaVersion := "2.11.8" 5 | 6 | libraryDependencies ++= Seq( 7 | "com.tumblr" %% "colossus" % "0.8.3", 8 | "org.apache.hadoop" % "hadoop-common" % "2.2.0", 9 | "org.apache.hadoop" % "hadoop-hdfs" % "2.2.0" 10 | ) 11 | */ 12 | 13 | import java.net.URI 14 | 15 | import colossus._ 16 | import core._ 17 | import service._ 18 | import protocols.http._ 19 | import HttpMethod._ 20 | import akka.actor.ActorSystem 21 | import colossus.protocols.http.UrlParsing._ 22 | import org.apache.hadoop.conf.Configuration 23 | import org.apache.hadoop.fs.{FileSystem, Path} 24 | import org.apache.hadoop.io._ 25 | import org.apache.hadoop.util.ReflectionUtils 26 | 27 | 28 | // load all tiles into memory (so slow) 29 | val cache = scala.collection.mutable.Map[String, Array[Byte]]() 30 | 31 | val config = new Configuration 32 | config.set("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); 33 | 34 | // Ref http://alvinalexander.com/scala/scala-shell-script-command-line-arguments-args 35 | val uri = URI.create(args(0)) 36 | val fs = FileSystem.get(uri, config) 37 | val statusCollection = fs.listStatus(new Path(uri.getPath)) 38 | 39 | 40 | println("Starting load all SequenceFile...") 41 | 42 | statusCollection.filter(status => { 43 | // filter _SUCCESS 44 | status.getPath.getName != "_SUCCESS" 45 | }).foreach(status => { 46 | val path = status.getPath 47 | println(s"load ${path}") 48 | 49 | val reader = new SequenceFile.Reader(config, SequenceFile.Reader.file(path)) 50 | val key = ReflectionUtils.newInstance(reader.getKeyClass, config).asInstanceOf[Text] 51 | val value = ReflectionUtils.newInstance(reader.getValueClass, config).asInstanceOf[BytesWritable] 52 | 53 | while(reader.next(key, value)){ 54 | val code = key.toString() 55 | // Ref http://stackoverflow.com/questions/23211493/how-to-extract-data-from-hadoop-sequence-file 56 | val bytes = value.copyBytes() 57 | cache(code) = bytes 58 | } 59 | }) 60 | 61 | println("Finished loading.") 62 | 63 | // Take samples 64 | 65 | val samples = cache.take(5) ++ cache.takeRight(5) 66 | 67 | println("Take samples of Tiles") 68 | samples.zipWithIndex.foreach(t => { 69 | val tile = t._1 70 | val index = t._2 71 | val (layerName, (row, column, zoom)) = decodeTile(tile._1) 72 | println(s"${index}: Layer is ${layerName} and Row is ${row} and Column is ${column} and Zoom is ${zoom}") 73 | }) 74 | 75 | def encodeTile(layerName: String, tile: (Long, Long, Long)): String ={ 76 | val (row, column, zoom) = tile 77 | val code = (row % 0x1000000L) << 40 | (column % 0x1000000L) << 16 | (zoom % 0x10000L) 78 | s"${layerName}:${code}" 79 | } 80 | 81 | def decodeTile(key: String): (String, (Long, Long, Long)) ={ 82 | val x = key.split(":").toSeq 83 | val layerName = x(0) 84 | val code = x(1).toLong 85 | val row = code >> 40 86 | val column = (code >> 16) % 0x1000000L 87 | val zoom = code % 0x10000L 88 | (layerName, (row, column, zoom)) 89 | } 90 | 91 | class ArrayByteEncoder extends HttpBodyEncoder[Array[Byte]]{ 92 | def encode(data : Array[Byte]) : HttpBody ={ 93 | 94 | val contentType = HttpHeader("Content-Type", "application/vnd.mapbox-vector-tile") 95 | new HttpBody(data, Some(contentType)) 96 | } 97 | } 98 | 99 | class WMSService(context: ServerContext) extends HttpService(context) { 100 | 101 | val allowOrigin = HttpHeader("Access-Control-Allow-Origin", "*") 102 | val allowMethod = HttpHeader("Access-Control-Allow-Method", "GET") 103 | val crossDomainHeaders = HttpHeaders(allowOrigin, allowMethod) 104 | 105 | def handle = { 106 | case request @ Get on Root => { 107 | Callback.successful(request.ok("Simple Web Map Server!")) 108 | } 109 | 110 | case request @ Options on Root / layerName / Long(zoom) / Long(row) / fileName => { 111 | 112 | Callback.successful(request.ok("", crossDomainHeaders)) 113 | } 114 | 115 | case request @ Get on Root / layerName / Long(zoom) / Long(row) / fileName => { 116 | 117 | val index = fileName.indexOf(".pbf") 118 | if(index == -1){ 119 | Callback.successful(request.badRequest[String]("Wrong path format")) 120 | } else { 121 | val column = fileName.substring(0, index).toLong 122 | val code = encodeTile(layerName,(row, column, zoom)) 123 | 124 | val bytes = cache.get(code) 125 | if(bytes.isEmpty){ 126 | Callback.successful(request.notFound[String](s"The requested tile parameters is (${row}, ${column}, ${zoom})")) 127 | } else { 128 | implicit val encoder = new ArrayByteEncoder 129 | Callback.successful(request.ok(bytes.get, crossDomainHeaders)) 130 | } 131 | } 132 | 133 | } 134 | } 135 | } 136 | 137 | class WMSInitializer(worker: WorkerRef) extends Initializer(worker) { 138 | 139 | def onConnect = context => new WMSService(context) 140 | 141 | } 142 | 143 | implicit val actorSystem = ActorSystem() 144 | implicit val io = IOSystem() 145 | 146 | Server.start("wms", 9000){ worker => new WMSInitializer(worker) } 147 | -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/sql/GeometricUtils.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark.sql 2 | 3 | import com.vividsolutions.jts.geom.Geometry 4 | import com.vividsolutions.jts.io.{WKBReader, WKBWriter, WKTReader, WKTWriter} 5 | 6 | /** 7 | * Created by CodeMeow on 2017/5/11. 8 | */ 9 | 10 | // Ref https://github.com/locationtech/geomesa/blob/master/geomesa-utils/src/main/scala/org/locationtech/geomesa/utils/text/WKUtils.scala 11 | object WKTUtils { 12 | private val readerPool = new ThreadLocal[WKTReader]{ 13 | override def initialValue() = new WKTReader 14 | } 15 | 16 | private val writePool = new ThreadLocal[WKTWriter]{ 17 | override def initialValue() = new WKTWriter 18 | } 19 | 20 | def read(s: String): Geometry = readerPool.get.read(s) 21 | def write(g: Geometry): String = writePool.get.write(g) 22 | } 23 | 24 | object WKBUtils { 25 | private val readerPool = new ThreadLocal[WKBReader]{ 26 | override def initialValue() = new WKBReader 27 | } 28 | 29 | private val writePool = new ThreadLocal[WKBWriter]{ 30 | override def initialValue() = new WKBWriter 31 | } 32 | 33 | def read(b: Array[Byte]): Geometry = readerPool.get.read(b) 34 | def write(g: Geometry): Array[Byte] = writePool.get.write(g) 35 | } -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/sql/SQLGeometricExtensions.scala: -------------------------------------------------------------------------------- 1 | package org.apache.spark.sql 2 | 3 | import com.vividsolutions.jts.geom._ 4 | import org.apache.spark.sql.catalyst.InternalRow 5 | import org.apache.spark.sql.catalyst.expressions.GenericInternalRow 6 | import org.apache.spark.sql.types._ 7 | import org.geotools.geometry.jts.JTS 8 | 9 | import scala.reflect.ClassTag 10 | 11 | /** 12 | * Created by CodeMeow on 2017/5/11. 13 | */ 14 | 15 | 16 | object SQLGeometricExtensions { 17 | 18 | UDTRegistration.register(classOf[Point].getCanonicalName, classOf[PointUDT].getCanonicalName) 19 | UDTRegistration.register(classOf[MultiPoint].getCanonicalName, classOf[MultiPointUDT].getCanonicalName) 20 | UDTRegistration.register(classOf[LineString].getCanonicalName, classOf[LineStringUDT].getCanonicalName) 21 | UDTRegistration.register(classOf[MultiLineString].getCanonicalName, classOf[MultiLineStringUDT].getCanonicalName) 22 | UDTRegistration.register(classOf[Polygon].getCanonicalName, classOf[PolygonUDT].getCanonicalName) 23 | UDTRegistration.register(classOf[MultiPolygon].getCanonicalName, classOf[MultiPolygonUDT].getCanonicalName) 24 | UDTRegistration.register(classOf[Geometry].getCanonicalName, classOf[GeometryUDT].getCanonicalName) 25 | 26 | def registerExtensions(spark: SparkSession): Unit ={ 27 | SQLGeometricFunctions.registerFunctions(spark) 28 | } 29 | } 30 | 31 | // Ref https://github.com/locationtech/geomesa/blob/master/geomesa-spark/geomesa-spark-sql/src/main/scala/org/apache/spark/sql/SQLGeometricConstructorFunctions.scala 32 | object SQLGeometricFunctions { 33 | 34 | val ST_MakePoint: (Double, Double) => Point = (x: Double, y: Double) => WKTUtils.read(s"POINT($x $y)").asInstanceOf[Point] 35 | val ST_MakeBox2D: (Point, Point) => Geometry = (ll: Point, ur: Point) => JTS.toGeometry(new Envelope(ll.getX, ur.getX, ll.getY, ur.getY)) 36 | val ST_GeomFromWKB: Array[Byte] => Geometry = (array: Array[Byte]) => WKBUtils.read(array) 37 | val ST_Intersects: (Geometry, Geometry) => Boolean = (geom1: Geometry, geom2: Geometry) => geom1.intersects(geom2) 38 | 39 | def registerFunctions(spark: SparkSession): Unit = { 40 | spark.udf.register("ST_MakePoint", ST_MakePoint) 41 | spark.udf.register("ST_MakeBox2D", ST_MakeBox2D) 42 | spark.udf.register("ST_GeomFromWKB", ST_GeomFromWKB) 43 | spark.udf.register("ST_Intersects", ST_Intersects) 44 | } 45 | 46 | } 47 | 48 | // Ref https://github.com/locationtech/geomesa/blob/master/geomesa-spark/geomesa-spark-sql/src/main/scala/org/apache/spark/sql/SQLTypes.scala 49 | abstract class AbstractGeometryUDT[T >: Null <: Geometry](override val simpleString: String)(implicit cm: ClassTag[T]) extends UserDefinedType[T]{ 50 | override def serialize(obj: T): InternalRow = { 51 | new GenericInternalRow(Array[Any](WKBUtils.write(obj))) 52 | } 53 | 54 | override def sqlType: DataType = StructType( 55 | Seq( 56 | StructField("wkb", DataTypes.BinaryType) 57 | ) 58 | ) 59 | 60 | override def userClass: Class[T] = cm.runtimeClass.asInstanceOf[Class[T]] 61 | 62 | override def deserialize(datum: Any): T = { 63 | val ir = datum.asInstanceOf[InternalRow] 64 | WKBUtils.read(ir.getBinary(0)).asInstanceOf[T] 65 | } 66 | } 67 | 68 | private[spark] class PointUDT extends AbstractGeometryUDT[Point]("point") 69 | private[spark] class MultiPointUDT extends AbstractGeometryUDT[MultiPoint]("multipoint") 70 | private[spark] class LineStringUDT extends AbstractGeometryUDT[LineString]("linestring") 71 | private[spark] class MultiLineStringUDT extends AbstractGeometryUDT[MultiLineString]("multilinestring") 72 | private[spark] class PolygonUDT extends AbstractGeometryUDT[Polygon]("polygon") 73 | private[spark] class MultiPolygonUDT extends AbstractGeometryUDT[MultiPolygon]("multipolygon") 74 | 75 | private[spark] class GeometryUDT extends AbstractGeometryUDT[Geometry]("geometry"){ 76 | private[sql] override def acceptsType(dataType: DataType): Boolean = { 77 | super.acceptsType(dataType) || 78 | dataType.getClass == classOf[GeometryUDT] || 79 | dataType.getClass == classOf[PointUDT] || 80 | dataType.getClass == classOf[MultiPointUDT] || 81 | dataType.getClass == classOf[LineStringUDT] || 82 | dataType.getClass == classOf[MultiLineStringUDT] || 83 | dataType.getClass == classOf[PolygonUDT] || 84 | dataType.getClass == classOf[MultiPolygonUDT] 85 | } 86 | } -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/CRSUtils.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric 2 | 3 | /** 4 | * Created by CodeMeow on 2017/5/12. 5 | */ 6 | 7 | // Ref https://github.com/mapbox/mercantile/blob/master/mercantile/__init__.py 8 | object CRSUtils { 9 | 10 | val MAX_ZOOM: Long = 28 11 | val RADIUS: Double = 6378137.0 12 | 13 | // Returns the upper left (lon, lat) of tile 14 | def upperLeft(tile: (Long, Long, Long)): (Double, Double) ={ 15 | val (row, column, zoom) = tile 16 | val n = math.pow(2.0, zoom) 17 | val lon_deg = row / n * 360.0 - 180.0 18 | val lat_rad = math.atan(math.sinh(math.Pi * (1 - 2 * column / n))) 19 | val lat_deg = math.toDegrees(lat_rad) 20 | return (lon_deg, lat_deg) 21 | } 22 | 23 | // Returns the upper left Mercator coordinate (x, y) of tile 24 | def upperLeftMercator(tile: (Long, Long, Long)): (Double, Double) ={ 25 | val (lng, lat) = upperLeft(tile) 26 | lnglatToMercator(lng, lat) 27 | } 28 | 29 | // Returns the (lon, lat) bounding box of a tile 30 | def bounds(tile: (Long, Long, Long)): (Double, Double, Double, Double) ={ 31 | val (row, column, zoom) = tile 32 | val a = upperLeft(tile) 33 | val b = upperLeft((row + 1, column + 1, zoom)) 34 | return (a._1, b._2, b._1, a._2) 35 | } 36 | 37 | // Returns the Mercator coordinate (x, y) bounding box of a tile 38 | def boundsMercator(tile: (Long, Long, Long)): (Double, Double, Double, Double) ={ 39 | val (row, column, zoom) = tile 40 | val a = upperLeftMercator(row + 1, column, zoom) 41 | val b = upperLeftMercator((row, column + 1, zoom)) 42 | return (a._1, b._1, a._2, b._2) 43 | } 44 | 45 | // Returns the Spherical Mercator (x, y) in meters 46 | def lnglatToMercator(lng: Double, lat: Double): (Double, Double) ={ 47 | val x = RADIUS * math.toRadians(lng) 48 | val y = RADIUS * math.log(math.tan((math.Pi * 0.25) + (0.5 * math.toRadians(lat)))) 49 | return (x, y) 50 | } 51 | 52 | // Returns the Location (lon, lat) from Spherical Mercator (x, y) 53 | def mercatorTolnglat(x: Double, y: Double): (Double, Double) ={ 54 | val lng = math.toDegrees(x / RADIUS) 55 | val lat = math.toDegrees(math.atan(math.exp(y / RADIUS)) * 2.0 - math.Pi / 2) 56 | return (lng, lat) 57 | } 58 | 59 | // Returns the (row, column, zoom) tile of a location (lon, lat) 60 | def lnglatToTile(lng: Double, lat: Double, zoom: Long): (Long, Long, Long) ={ 61 | val lat_rad = math.toRadians(lat) 62 | val n = math.pow(2.0, zoom) 63 | val row = math.floor((lng + 180.0) / 360.0 * n).toLong 64 | val column = math.floor((1.0 - math.log(math.tan(lat_rad) + (1.0 / math.cos(lat_rad))) / math.Pi) / 2.0 * n).toLong 65 | return (row, column, zoom) 66 | } 67 | 68 | // Returns the (row, column, zoom) tile from Spherical Mercator (x, y) in special zoom level 69 | def mercatorToTile(x: Double, y: Double, zoom: Long): (Long, Long, Long) ={ 70 | val (lng, lat) = mercatorTolnglat(x, y) 71 | lnglatToTile(lng, lat, zoom) 72 | } 73 | 74 | // Returns the tiles included in the Mercator bbox (minX, minY, maxX, maxY) in special zoom level 75 | def mercatorToTiles(bbox: (Double, Double, Double, Double), zoom: Long): Seq[(Long, Long, Long)] ={ 76 | val ul = mercatorToTile(bbox._1, bbox._4, zoom) 77 | val lr_ = mercatorToTile(bbox._3, bbox._2, zoom) 78 | val lr = (lr_._1 + 1, lr_._2 + 1, zoom) 79 | 80 | (ul._1 to lr._1).flatMap(row => { 81 | (ul._2 to lr._2).map(column => { 82 | (row, column, zoom) 83 | }) 84 | }) 85 | } 86 | 87 | // Returns the children of an (row, column, zoom) tile 88 | def childrenTiles(tile: (Long, Long, Long)): Seq[(Long, Long, Long)] ={ 89 | val (row, column, zoom) = tile 90 | return Seq[(Long, Long, Long)]( 91 | (row * 2, column * 2, zoom + 1), 92 | (row * 2 + 1, column * 2, zoom + 1), 93 | (row * 2 + 1, column * 2 + 1, zoom + 1), 94 | (row * 2, column * 2 + 1, zoom + 1) 95 | ) 96 | } 97 | 98 | // Returns the smallest tile (row, column, zoom) containing the bbox (west, south, east, north) 99 | def boundingTile(bbox: (Double, Double, Double, Double)): (Long, Long, Long) ={ 100 | val (w, s, e, n) = bbox 101 | val tmin = lnglatToTile(w, s, 32) 102 | val tmax = lnglatToTile(e, n, 32) 103 | val cell = (tmin._1, tmin._2, tmax._1, tmax._2) 104 | val zoom = getBboxZoom(cell) 105 | if(zoom == 0) { 106 | return (0, 0, 0) 107 | } 108 | val row = rshift(cell._1, (32 - zoom)) 109 | val column = rshift(cell._2, (32 - zoom)) 110 | return (row, column, zoom) 111 | } 112 | 113 | def rshift(value: Long, n: Long): Long ={ 114 | return (value % 0x100000000L) >> n 115 | } 116 | 117 | def getBboxZoom(bbox: (Long, Long, Long, Long)): Long ={ 118 | for(zoom <- 0L to MAX_ZOOM){ 119 | val mask = 1 << (32 - (zoom + 1)) 120 | if(((bbox._1 & mask) != (bbox._3 & mask)) || ((bbox._2 & mask) != (bbox._4 & mask))){ 121 | return zoom 122 | } 123 | } 124 | return MAX_ZOOM 125 | } 126 | 127 | } 128 | -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/Feature.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric 2 | 3 | import com.vividsolutions.jts.geom.Geometry 4 | 5 | /** 6 | * Created by CodeMeow on 2017/5/13. 7 | */ 8 | case class Feature(id: Long, geom: Geometry, props: scala.collection.Map[String, String]) -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/GeometricUtils.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric 2 | 3 | import com.vividsolutions.jts.geom.{Envelope, Geometry} 4 | import org.apache.spark.rdd.RDD 5 | import org.geotools.geometry.jts.JTS 6 | 7 | /** 8 | * Created by CodeMeow on 2017/5/12. 9 | */ 10 | object GeometricUtils { 11 | 12 | def boundary(rdd: RDD[Geometry]): (Double, Double, Double, Double) ={ 13 | implicit val XMinComparator = new Ordering[Geometry]{ 14 | override def compare(geom1: Geometry, geom2: Geometry): Int ={ 15 | val x1 = geom1.getEnvelopeInternal.getMinX 16 | val x2 = geom2.getEnvelopeInternal.getMinX 17 | return if(x1 > x2) 1 else (if(x1 < x2) -1 else 0) 18 | } 19 | } 20 | implicit val XMaxComparator = new Ordering[Geometry]{ 21 | override def compare(geom1: Geometry, geom2: Geometry): Int ={ 22 | val x1 = geom1.getEnvelopeInternal.getMaxX 23 | val x2 = geom2.getEnvelopeInternal.getMaxX 24 | return if(x1 > x2) 1 else (if(x1 < x2) -1 else 0) 25 | } 26 | } 27 | implicit val YMinComparator = new Ordering[Geometry]{ 28 | override def compare(geom1: Geometry, geom2: Geometry): Int ={ 29 | val y1 = geom1.getEnvelopeInternal.getMinY 30 | val y2 = geom2.getEnvelopeInternal.getMinY 31 | return if(y1 > y2) 1 else (if(y1 < y2) -1 else 0) 32 | } 33 | } 34 | implicit val YMaxComparator = new Ordering[Geometry]{ 35 | override def compare(geom1: Geometry, geom2: Geometry): Int ={ 36 | val y1 = geom1.getEnvelopeInternal.getMaxY 37 | val y2 = geom2.getEnvelopeInternal.getMaxY 38 | return if(y1 > y2) 1 else (if(y1 < y2) -1 else 0) 39 | } 40 | } 41 | val minX: Double = rdd.min()(XMinComparator).getEnvelopeInternal.getMinX 42 | val maxX: Double = rdd.max()(XMaxComparator).getEnvelopeInternal.getMaxX 43 | val minY: Double = rdd.min()(YMinComparator).getEnvelopeInternal.getMinY 44 | val maxY: Double = rdd.max()(YMaxComparator).getEnvelopeInternal.getMaxY 45 | 46 | return (minX, minY, maxX, maxY) 47 | } 48 | 49 | def intersectedTiles(geom: Geometry, zoom: Long): Seq[(Long, Long, Long)] ={ 50 | val minX = geom.getEnvelopeInternal.getMinX 51 | val maxX = geom.getEnvelopeInternal.getMaxX 52 | val minY = geom.getEnvelopeInternal.getMinY 53 | val maxY = geom.getEnvelopeInternal.getMaxY 54 | val bbox = (minX, minY, maxX, maxY) 55 | 56 | val tiles = CRSUtils.mercatorToTiles(bbox, zoom) 57 | tiles.filter(tile => { 58 | val ll = CRSUtils.upperLeftMercator((tile._1, tile._2 + 1, tile._3)) 59 | val ur = CRSUtils.upperLeftMercator((tile._1 + 1, tile._2, tile._3)) 60 | val boundary = JTS.toGeometry(new Envelope(ll._1, ur._1, ll._2, ur._2)) 61 | boundary.intersects(geom) 62 | }) 63 | } 64 | 65 | def encodeTile(layerName: String, tile: (Long, Long, Long)): String ={ 66 | val (row, column, zoom) = tile 67 | val code = (row % 0x1000000L) << 40 | (column % 0x1000000L) << 16 | (zoom % 0x10000L) 68 | s"${layerName}:${code}" 69 | } 70 | 71 | def decodeTile(key: String): (String, (Long, Long, Long)) ={ 72 | val x = key.split(":").toSeq 73 | val layerName = x(0) 74 | val code = x(1).toLong 75 | val row = code >> 40 76 | val column = (code >> 16) % 0x1000000L 77 | val zoom = code % 0x10000L 78 | (layerName, (row, column, zoom)) 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/MvtBuilder.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric 2 | 3 | import com.vividsolutions.jts.geom.{Envelope, Geometry, GeometryFactory} 4 | import com.wdtinc.mapbox_vector_tile.VectorTile 5 | import com.wdtinc.mapbox_vector_tile.VectorTile.Tile 6 | import com.wdtinc.mapbox_vector_tile.VectorTile.Tile.Layer 7 | import com.wdtinc.mapbox_vector_tile.adapt.jts.{IGeometryFilter, JtsAdapter, UserDataIgnoreConverter} 8 | import com.wdtinc.mapbox_vector_tile.build.{MvtLayerBuild, MvtLayerParams, MvtLayerProps} 9 | 10 | /** 11 | * Created by CodeMeow on 2017/5/13. 12 | */ 13 | object MvtBuilder { 14 | 15 | def buildLayer(layerName: String, tile: (Long, Long, Long), features: TraversableOnce[Feature]): Layer ={ 16 | val bounds = CRSUtils.boundsMercator(tile) 17 | val tileEnvelope = new Envelope(bounds._1, bounds._2, bounds._3, bounds._4) 18 | 19 | val layerParams = new MvtLayerParams 20 | val layerBuilder = MvtLayerBuild.newLayerBuilder(layerName, layerParams) 21 | 22 | for(feature <- features){ 23 | val geomFactory = new GeometryFactory() 24 | val tileGeom = JtsAdapter.createTileGeom(feature.geom, tileEnvelope, geomFactory, layerParams, AcceptAllGeomFilter) 25 | 26 | val layerProps = new MvtLayerProps 27 | for((key, value) <- feature.props){ 28 | if(key == null || value == null){ 29 | } else { 30 | layerProps.addKey(key) 31 | layerProps.addValue(value) 32 | } 33 | } 34 | 35 | val f = JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, new UserDataIgnoreConverter) 36 | layerBuilder.addAllFeatures(f) 37 | } 38 | 39 | return layerBuilder.build() 40 | } 41 | 42 | def buildMvt(layers: Seq[Layer]): Tile ={ 43 | val tileBuilder = VectorTile.Tile.newBuilder() 44 | for(layer <- layers){ 45 | tileBuilder.addLayers(layer) 46 | } 47 | tileBuilder.build() 48 | } 49 | 50 | def buildMvt(layer: Layer): Tile ={ 51 | val tileBuilder = VectorTile.Tile.newBuilder() 52 | tileBuilder.addLayers(layer) 53 | tileBuilder.build() 54 | } 55 | 56 | } 57 | 58 | object AcceptAllGeomFilter extends IGeometryFilter{ 59 | override def accept(geometry: Geometry) ={ 60 | true 61 | } 62 | } -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/spark/Configurations.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric.spark 2 | 3 | 4 | import com.fasterxml.jackson.annotation.JsonProperty 5 | import com.fasterxml.jackson.databind.ObjectMapper 6 | import com.fasterxml.jackson.dataformat.yaml.YAMLFactory 7 | import com.fasterxml.jackson.module.scala.DefaultScalaModule 8 | import com.google.common.base.Preconditions 9 | import org.apache.hadoop.conf.Configuration 10 | import org.apache.hadoop.fs.{FileSystem, Path} 11 | 12 | 13 | 14 | /** 15 | * Created by CodeMeow on 2017/5/12. 16 | */ 17 | object Configurations { 18 | def fromFile(_path: String): MainConfiguration ={ 19 | val config = new Configuration 20 | val path = new Path(_path) 21 | val fs = FileSystem.get(path.toUri, config) 22 | val file = fs.open(path) 23 | val mapper = new ObjectMapper(new YAMLFactory()) 24 | mapper.registerModule(DefaultScalaModule) 25 | mapper.readValue(file, classOf[MainConfiguration]) 26 | } 27 | } 28 | 29 | class MainConfiguration ( 30 | @JsonProperty("appName") _appName: String, 31 | @JsonProperty("bbox") _bbox: (Double, Double, Double, Double), // lat1, lon1, lat2, lon2 32 | @JsonProperty("layers") _layers: Array[LayerConfiguration], 33 | @JsonProperty("sequenceFileDir") _sequenceFileDir: String 34 | ) extends Serializable{ 35 | val appName = Preconditions.checkNotNull(_appName, "appName cannot be null": Object) 36 | val bbox = _bbox 37 | val layers = Preconditions.checkNotNull(_layers, "layers cannot be null": Object) 38 | val sequenceFileDir = Preconditions.checkNotNull(_sequenceFileDir, "sequenceFileDir cannot be null": Object) 39 | } 40 | 41 | class LayerConfiguration ( 42 | @JsonProperty("layerName") _layerName: String, 43 | @JsonProperty("minZoom") _minZoom: Int, 44 | @JsonProperty("maxZoom") _maxZoom: Int, 45 | @JsonProperty("dataProvider") _dataProvider: String, 46 | @JsonProperty("kwargs") _kwargs: Map[String, Any] 47 | ) extends Serializable{ 48 | val layerName = Preconditions.checkNotNull(_layerName, "layerName cannot be null": Object) 49 | val minZoom = Preconditions.checkNotNull(_minZoom, "minZoom cannot be null": Object) 50 | val maxZoom = Preconditions.checkNotNull(_maxZoom, "maxZoom cannot be null": Object) 51 | val dataProvider = Preconditions.checkNotNull(_dataProvider, "dataProvider cannot be null": Object) 52 | val kwargs = Preconditions.checkNotNull(_kwargs, "kwargs cannot be null": Object) 53 | } -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/spark/VectorTileTask.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric.spark 2 | 3 | import org.apache.hadoop.io.compress.GzipCodec 4 | import org.apache.spark.sql._ 5 | import org.ieee.codemeow.geometric.{Feature, GeometricUtils, MvtBuilder} 6 | import org.ieee.codemeow.geometric.spark.data.AbstractDataProvider 7 | 8 | /** 9 | * Created by CodeMeow on 2017/5/13. 10 | */ 11 | object VectorTileTask { 12 | 13 | val usage = "Usage: configFile" 14 | 15 | private def checkArgs(args: Array[String]) = { 16 | assert(args != null && args.length >= 1, usage) 17 | } 18 | 19 | def main(args: Array[String]): Unit ={ 20 | checkArgs(args) 21 | 22 | // load application config 23 | val config: MainConfiguration = Configurations.fromFile(args(0)) 24 | 25 | // Ref http://spark.apache.org/docs/latest/sql-programming-guide.html#starting-point-sparksession 26 | val spark = SparkSession.builder().appName(config.appName).getOrCreate() 27 | 28 | // Ref http://stackoverflow.com/questions/38664972/why-is-unable-to-find-encoder-for-type-stored-in-a-dataset-when-creating-a-dat 29 | import spark.implicits._ 30 | // Ref http://stackoverflow.com/questions/36648128/how-to-store-custom-objects-in-dataset 31 | implicit val featureEncoder = Encoders.kryo[Feature] 32 | 33 | SQLGeometricExtensions.registerExtensions(spark) 34 | 35 | val layersCollection = config.layers.map(layer => { 36 | // classloader 37 | val constructor = Class.forName(layer.dataProvider).getConstructor(classOf[SparkSession], classOf[LayerConfiguration]) 38 | val dataSource = constructor.newInstance(spark, layer).asInstanceOf[AbstractDataProvider] 39 | 40 | val tileCodeWithBytesCollection = buildLayerRDD(spark, dataSource, layer) 41 | tileCodeWithBytesCollection 42 | }).reduce((ds1, ds2) => { 43 | ds1.union(ds2) 44 | }) 45 | 46 | layersCollection.rdd.saveAsSequenceFile(config.sequenceFileDir, Some(classOf[GzipCodec])) 47 | } 48 | 49 | def buildLayerRDD(spark: SparkSession, dataSource: AbstractDataProvider, layer: LayerConfiguration): Dataset[(String, Array[Byte])] ={ 50 | // Ref http://stackoverflow.com/questions/38664972/why-is-unable-to-find-encoder-for-type-stored-in-a-dataset-when-creating-a-dat 51 | import spark.implicits._ 52 | // Ref http://stackoverflow.com/questions/36648128/how-to-store-custom-objects-in-dataset 53 | //implicit val featureEncoder = Encoders.kryo[Feature] 54 | 55 | 56 | val layerMvtCollection = (layer.minZoom to layer.maxZoom).map(zoom => { 57 | val featureCollection = dataSource.getFeatures(layer.layerName, zoom) 58 | 59 | if(featureCollection.isEmpty){ 60 | 61 | spark.emptyDataset[((Long, Long, Long), Seq[Feature])] 62 | } else { 63 | val featureWithTilesCollection = featureCollection.get.map(feature => { 64 | (feature, GeometricUtils.intersectedTiles(feature.geom, zoom)) 65 | }) 66 | 67 | val tileWithFeatureCollection = featureWithTilesCollection.flatMap(tuple => { 68 | tuple._2.map(tile => { 69 | (tile, tuple._1) 70 | }) 71 | }) 72 | 73 | val featureGroupByTileCollection_ = tileWithFeatureCollection.groupByKey(tuple => tuple._1) 74 | 75 | val featureGroupByTileCollection = featureGroupByTileCollection_.mapGroups((tile, groups) => { 76 | (tile, groups.map(tuple => tuple._2).toSeq) 77 | }) 78 | 79 | featureGroupByTileCollection 80 | } 81 | 82 | }).reduce((ds1, ds2) => { 83 | ds1.union(ds2) 84 | }) 85 | 86 | val tileWithBytesCollection = layerMvtCollection.map(tuple => { 87 | val layer_ = MvtBuilder.buildLayer(layer.layerName, tuple._1, tuple._2) 88 | val b = MvtBuilder.buildMvt(layer_).toByteArray 89 | (tuple._1, b) 90 | }) 91 | 92 | val tileCodeWithBytesCollection = tileWithBytesCollection.map(tuple => { 93 | (GeometricUtils.encodeTile(layer.layerName, tuple._1), tuple._2) 94 | }) 95 | 96 | tileCodeWithBytesCollection 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/spark/data/AbstractDataProvider.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric.spark.data 2 | 3 | import org.apache.spark.sql.{Dataset, SparkSession} 4 | import org.ieee.codemeow.geometric.Feature 5 | import org.ieee.codemeow.geometric.spark.LayerConfiguration 6 | 7 | /** 8 | * Created by CodeMeow on 2017/5/16. 9 | */ 10 | abstract class AbstractDataProvider(_spark: SparkSession, _layer: LayerConfiguration) { 11 | 12 | val spark = _spark 13 | val layer = _layer 14 | 15 | def getFeatures(layerName: String, zoom: Long): Option[Dataset[Feature]] 16 | } 17 | -------------------------------------------------------------------------------- /src/main/scala/org/ieee/codemeow/geometric/spark/data/SQLDataProvider.scala: -------------------------------------------------------------------------------- 1 | package org.ieee.codemeow.geometric.spark.data 2 | 3 | import com.vividsolutions.jts.geom.Geometry 4 | import org.apache.spark.sql.{Dataset, Encoders, SparkSession} 5 | import org.ieee.codemeow.geometric.Feature 6 | import org.ieee.codemeow.geometric.spark.LayerConfiguration 7 | 8 | 9 | /** 10 | * Created by CodeMeow on 2017/5/13. 11 | */ 12 | class SQLDataProvider(_spark: SparkSession, _layer: LayerConfiguration) extends AbstractDataProvider(_spark, _layer){ 13 | 14 | val url = layer.kwargs.get("url").get.asInstanceOf[String] 15 | val dbtables = layer.kwargs.get("dbtables").get.asInstanceOf[Map[String, String]] 16 | val user = layer.kwargs.get("user").get.asInstanceOf[String] 17 | val password = layer.kwargs.get("password").get.asInstanceOf[String] 18 | val zoomConfig = layer.kwargs.get("zooms").get.asInstanceOf[Map[String, String]] 19 | 20 | // load all tables 21 | dbtables.foreach(tuple => { 22 | val sparkTableName = tuple._1 23 | val realTableName = tuple._2 24 | 25 | val mapDataFrame = spark.read.format("jdbc") 26 | .option("url", url) 27 | .option("user", user) 28 | .option("password", password) 29 | .option("dbtable", realTableName).load 30 | 31 | mapDataFrame.createOrReplaceTempView(sparkTableName) 32 | }) 33 | 34 | override def getFeatures(layerName: String, zoom: Long): Option[Dataset[Feature]] ={ 35 | // Ref http://stackoverflow.com/questions/38664972/why-is-unable-to-find-encoder-for-type-stored-in-a-dataset-when-creating-a-dat 36 | import spark.implicits._ 37 | // Ref http://stackoverflow.com/questions/36648128/how-to-store-custom-objects-in-dataset 38 | implicit val featureEncoder = Encoders.kryo[Feature] 39 | 40 | val natureSQL = zoomConfig.get(zoom.toString) 41 | if(natureSQL.isEmpty){ 42 | return None 43 | } 44 | 45 | val rawDF = spark.sql(natureSQL.get) 46 | val featureCollection = rawDF.map(row => { 47 | val id = row.getAs[Long]("__id__") 48 | val geom = row.getAs[Geometry]("__geometry__") 49 | val fields = row.schema.filter(field => { 50 | !Seq("__id__", "__geometry__").contains(field.name) 51 | }).map(field => field.name) 52 | val props = row.getValuesMap[String](fields) 53 | Feature(id, geom, props) 54 | }) 55 | Some(featureCollection) 56 | } 57 | } -------------------------------------------------------------------------------- /src/test/scala/samples/junit.scala: -------------------------------------------------------------------------------- 1 | package samples 2 | 3 | import org.junit._ 4 | import Assert._ 5 | 6 | @Test 7 | class AppTest { 8 | 9 | @Test 10 | def testOK() = assertTrue(true) 11 | 12 | // @Test 13 | // def testKO() = assertTrue(false) 14 | 15 | } 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/test/scala/samples/scalatest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2001-2009 Artima, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package samples 17 | 18 | /* 19 | ScalaTest facilitates different styles of testing by providing traits you can mix 20 | together to get the behavior and syntax you prefer. A few examples are 21 | included here. For more information, visit: 22 | 23 | http://www.scalatest.org/ 24 | 25 | One way to use ScalaTest is to help make JUnit or TestNG tests more 26 | clear and concise. Here's an example: 27 | */ 28 | import scala.collection.mutable.Stack 29 | import org.scalatest.Assertions 30 | import org.junit.Test 31 | 32 | class StackSuite extends Assertions { 33 | 34 | @Test def stackShouldPopValuesIinLastInFirstOutOrder() { 35 | val stack = new Stack[Int] 36 | stack.push(1) 37 | stack.push(2) 38 | assert(stack.pop() === 2) 39 | assert(stack.pop() === 1) 40 | } 41 | 42 | @Test def stackShouldThrowNoSuchElementExceptionIfAnEmptyStackIsPopped() { 43 | val emptyStack = new Stack[String] 44 | intercept[NoSuchElementException] { 45 | emptyStack.pop() 46 | } 47 | } 48 | } 49 | 50 | /* 51 | Here's an example of a FunSuite with ShouldMatchers mixed in: 52 | */ 53 | import org.scalatest.FunSuite 54 | import org.scalatest.matchers.ShouldMatchers 55 | 56 | import org.junit.runner.RunWith 57 | import org.scalatest.junit.JUnitRunner 58 | @RunWith(classOf[JUnitRunner]) 59 | class ListSuite extends FunSuite with ShouldMatchers { 60 | 61 | test("An empty list should be empty") { 62 | List() should be ('empty) 63 | Nil should be ('empty) 64 | } 65 | 66 | test("A non-empty list should not be empty") { 67 | List(1, 2, 3) should not be ('empty) 68 | List("fee", "fie", "foe", "fum") should not be ('empty) 69 | } 70 | 71 | test("A list's length should equal the number of elements it contains") { 72 | List() should have length (0) 73 | List(1, 2) should have length (2) 74 | List("fee", "fie", "foe", "fum") should have length (4) 75 | } 76 | } 77 | 78 | /* 79 | ScalaTest also supports the behavior-driven development style, in which you 80 | combine tests with text that specifies the behavior being tested. Here's 81 | an example whose text output when run looks like: 82 | 83 | A Map 84 | - should only contain keys and values that were added to it 85 | - should report its size as the number of key/value pairs it contains 86 | */ 87 | import org.scalatest.FunSpec 88 | import scala.collection.mutable.Stack 89 | 90 | class ExampleSpec extends FunSpec { 91 | 92 | describe("A Stack") { 93 | 94 | it("should pop values in last-in-first-out order") { 95 | val stack = new Stack[Int] 96 | stack.push(1) 97 | stack.push(2) 98 | assert(stack.pop() === 2) 99 | assert(stack.pop() === 1) 100 | } 101 | 102 | it("should throw NoSuchElementException if an empty stack is popped") { 103 | val emptyStack = new Stack[Int] 104 | intercept[NoSuchElementException] { 105 | emptyStack.pop() 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/test/scala/samples/specs.scala: -------------------------------------------------------------------------------- 1 | package samples 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.mutable._ 5 | import org.specs2.runner._ 6 | 7 | 8 | /** 9 | * Sample specification. 10 | * 11 | * This specification can be executed with: scala -cp ${package}.SpecsTest 12 | * Or using maven: mvn test 13 | * 14 | * For more information on how to write or run specifications, please visit: 15 | * http://etorreborre.github.com/specs2/guide/org.specs2.guide.Runners.html 16 | * 17 | */ 18 | @RunWith(classOf[JUnitRunner]) 19 | class MySpecTest extends Specification { 20 | "The 'Hello world' string" should { 21 | "contain 11 characters" in { 22 | "Hello world" must have size(11) 23 | } 24 | "start with 'Hello'" in { 25 | "Hello world" must startWith("Hello") 26 | } 27 | "end with 'world'" in { 28 | "Hello world" must endWith("world") 29 | } 30 | } 31 | } 32 | --------------------------------------------------------------------------------