├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── build.gradle ├── docker-compose.yml ├── docker └── Dockerfile ├── gradle.properties ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── libs └── legacy-geo-7.17.28.jar └── src ├── main ├── java │ └── org │ │ └── opendatasoft │ │ └── elasticsearch │ │ ├── .DS_Store │ │ ├── ingest │ │ └── GeoExtensionProcessor.java │ │ ├── plugin │ │ ├── GeoExtensionPlugin.java │ │ └── GeoUtils.java │ │ ├── script │ │ └── ScriptGeoSimplify.java │ │ └── search │ │ └── aggregations │ │ └── bucket │ │ └── geoshape │ │ ├── GeoShape.java │ │ ├── GeoShapeAggregator.java │ │ ├── GeoShapeAggregatorFactory.java │ │ ├── GeoShapeAggregatorSupplier.java │ │ ├── GeoShapeBuilder.java │ │ └── InternalGeoShape.java └── main.iml ├── test └── resources │ └── rest-api-spec │ └── test │ ├── .DS_Store │ └── GeoExtension │ └── .DS_Store └── yamlRestTest ├── java └── org │ └── opendatasoft │ └── elasticsearch │ └── RestApiYamlIT.java └── resources └── rest-api-spec └── test └── GeoExtension ├── 10_basic.yml ├── 20_geo_ingest_processor.yml ├── 30_simplify_script.yml ├── 40_geoshape_aggregation.yml └── 50_invalid_polygon.yml /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | .DS_Store 3 | .gradle/ 4 | .idea/ 5 | build/ 6 | .vscode/ 7 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### 7.17.28.0 2 | 3 | * Repackaging for Elasticsearch 7.17.28 4 | 5 | ### 7.17.6.1 6 | 7 | * Fix bbox on linestrings and points 8 | 9 | ### 7.17.6.0 10 | 11 | * Repackaging for ES 7.17.6 12 | 13 | ### 7.17.1.2 14 | 15 | * Simplify consistency: script is now using the same tolerance value as the agg one 16 | 17 | ### 7.17.1.1 18 | 19 | * Fix deduplication of points 20 | * Fix handling of GeometryCollections 21 | * Add tests 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Elasticsearch GeoShape Plugin 2 | 3 | 4 | This plugin can be used to index geo_shape objects in elasticsearch, then aggregate and/or script-simplify them. 5 | 6 | This is an `Ingest`, `Search` and `Script` plugin. 7 | 8 | 9 | ## Installation 10 | 11 | Current supported version is Elasticsearch 7.x (7.17.28). 12 | You can find past releases [here](https://github.com/opendatasoft/elasticsearch-plugin-geoshape/releases). 13 | 14 | The first 3 digits of the plugin version is the corresponding Elasticsearch version. The last digit is used for plugin versioning. 15 | 16 | To install it, launch this command in Elasticsearch directory replacing the url by the correct link for your Elasticsearch version (see table) 17 | `bin/elasticsearch-plugin install https://github.com/opendatasoft/elasticsearch-plugin-geoshape/releases/download/v7.17.28.0/elasticsearch-plugin-geoshape-7.17.28.0.zip"` 18 | 19 | 20 | ## Build 21 | 22 | Built with Java 17 and Gradle 8.10.2 (but you should use the packaged gradlew included in this repo anyway). 23 | 24 | 25 | ## Usage 26 | 27 | ### Ingest processor and indexing 28 | 29 | A new processor `geo_extension` adds custom fields to the desired geo_shape data object at ingest time. 30 | 31 | #### Params 32 | 33 | Processor name: `geo_extension`. 34 | 35 | |Name|Required|Default|Description| 36 | |----|--------|-------|-----------| 37 | | `field` | yes | - | The geo shape field to use. This parameter accepts wildcard to match multiple `geo_shape` fields 38 | | `path` | no | - | The field that contains the field to expand. When using wildcard in `field`, matching will be done under this path only 39 | | `keep_original_shape` | no | `true` | Keep the original unfixed shape in a `shape` field 40 | | `shape_field` | no | `shape` | Name of sub `shape` field 41 | | `fix_shape` | no | `true` | Fix invalid shape. For the moment it only fixes duplicate consecutive coordinates in polygon (https://github.com/elastic/elasticsearch/issues/14014) 42 | | `fixed_field` | no | `fixed_shape` | Name of sub `fixed_shape` field 43 | | `wkb` | no | `true` | Compute wkb from shape field 44 | | `wkb_field` | no | `wkb` | name of wkb subfield 45 | | `type` | no | `true` | Compute geo shape type (Polygon, point, LineString, ...) 46 | | `type_field` | no | `type` | name of type subfield 47 | | `area` | no | `true` | Compute area of shape 48 | | `area_field` | no | `area` | name of `area` subfield 49 | | `bbox` | no | `true` | Compute geo_point array containing topLeft and bottomRight points of shape envelope 50 | | `bbox_field` | no | `bbox` | name of `bbox` subfield 51 | | `centroid` | no | `true` | Compute geo_point representing shape centroid 52 | | `centroid_field` | no | `centroid` | name of `centroid` subfield 53 | | `hash` | no | `true` | Compute shape digest to perform exact request on shape (in other words: used as a primary key. we may want to use the wkt in the future?) 54 | | `hash_field` | no | `hash` | name of `hash` subfield 55 | 56 | 57 | #### Example 58 | ``` 59 | 60 | PUT _ingest/pipeline/geo_extension 61 | { 62 | "description": "Add extra geo fields to geo_shape objects.", 63 | "processors": [ 64 | { 65 | "geo_extension": { 66 | "field": "geoshape_*" 67 | } 68 | } 69 | ] 70 | } 71 | PUT main 72 | { 73 | "mappings": { 74 | "dynamic_templates": [ 75 | { 76 | "geoshapes": { 77 | "match": "geoshape_*", 78 | "mapping": { 79 | "properties": { 80 | "geoshape": {"type": "geo_shape"}, 81 | "hash": {"type": "keyword"}, 82 | "wkb": {"type": "binary", "doc_values": true}, 83 | "type": {"type": "keyword"}, 84 | "area": {"type": "half_float"}, 85 | "bbox": {"type": "geo_point"}, 86 | "centroid": {"type": "geo_point"} 87 | } 88 | } 89 | } 90 | } 91 | ] 92 | } 93 | } 94 | GET main/_mapping 95 | ``` 96 | 97 | Result: 98 | ``` 99 | { 100 | "main": { 101 | "mappings": { 102 | "_doc": { 103 | "dynamic_templates": [ 104 | { 105 | "geoshapes": { 106 | "match": "geoshape_*", 107 | "mapping": { 108 | "properties": { 109 | "geoshape": { 110 | "type": "geo_shape" 111 | }, 112 | "hash": { 113 | "type": "keyword" 114 | }, 115 | "wkb": { 116 | "type": "binary", 117 | "doc_values": true 118 | }, 119 | "type": { 120 | "type": "keyword" 121 | }, 122 | "area": { 123 | "type": "half_float" 124 | }, 125 | "bbox": { 126 | "type": "geo_point" 127 | }, 128 | "centroid": { 129 | "type": "geo_point" 130 | } 131 | } 132 | } 133 | } 134 | } 135 | ] 136 | } 137 | } 138 | } 139 | } 140 | ``` 141 | 142 | Document indexing with shape fixing: 143 | ``` 144 | POST main/_doc?pipeline=geo_extension 145 | { 146 | "geoshape_0": { 147 | "type": "Polygon", 148 | "coordinates": [ 149 | [ 150 | [ 151 | 1.6809082031249998, 152 | 49.05227025601607 153 | ], 154 | [ 155 | 2.021484375, 156 | 48.596592251456705 157 | ], 158 | [ 159 | 2.021484375, 160 | 48.596592251456705 161 | ], 162 | [ 163 | 3.262939453125, 164 | 48.922499263758255 165 | ], 166 | [ 167 | 2.779541015625, 168 | 49.196064000723794 169 | ], 170 | [ 171 | 2.0654296875, 172 | 49.23194729854559 173 | ], 174 | [ 175 | 1.6809082031249998, 176 | 49.05227025601607 177 | ] 178 | ] 179 | ] 180 | } 181 | } 182 | GET main/_search 183 | ``` 184 | 185 | Result: 186 | ``` 187 | "hits": [ 188 | { 189 | "_source": { 190 | "geoshape_0": { 191 | "area": 0.594432056845634, 192 | "centroid": { 193 | "lat": 48.95553463671871, 194 | "lon": 2.3829210191713015 195 | }, 196 | "bbox": [ 197 | { 198 | "lat": 48.596592251456705, 199 | "lon": 1.6809082031249998 200 | }, 201 | { 202 | "lat": 49.23194729854559, 203 | "lon": 3.262939453125 204 | } 205 | ], 206 | "type": "Polygon", 207 | "geoshape": { 208 | "coordinates": [ 209 | [ 210 | [ 211 | 1.6809082031249998, 212 | 49.05227025601607 213 | ], 214 | [ 215 | 2.021484375, 216 | 48.596592251456705 217 | ], 218 | [ 219 | 3.262939453125, 220 | 48.922499263758255 221 | ], 222 | [ 223 | 2.779541015625, 224 | 49.196064000723794 225 | ], 226 | [ 227 | 2.0654296875, 228 | 49.23194729854559 229 | ], 230 | [ 231 | 1.6809082031249998, 232 | 49.05227025601607 233 | ] 234 | ] 235 | ], 236 | "type": "Polygon" 237 | }, 238 | "hash": "-5012816342630707936", 239 | "wkb": "AAAAAAMAAAABAAAABkAALAAAAAAAQEhMXSKIhttAChqAAAAAAEBIdhR0tDaAQAY8gAAAAABASJkYoAuEDEAAhgAAAAAAQEidsHL20w4/+uT//////0BIhrDKsBJAQAAsAAAAAABASExdIoiG2w==" 240 | } 241 | } 242 | } 243 | ``` 244 | Note that the duplicated point has been deduplicated. 245 | 246 | 247 | 248 | ### Geoshape aggregation 249 | 250 | This aggregation creates a bucket for each input shape (based on the hash of its WKB representation) and compute a simplified version of the shape in the bucket. 251 | The simplification part is similar to what is done with the simplify script. 252 | The `size` parameter allows you to retain only the biggest (longer) N shapes. 253 | Moreover, compared to regular search results, results of an aggregation can be [cached by ElasticSearch](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations.html#agg-caches). 254 | 255 | 256 | 257 | #### Params 258 | 259 | - `field` (mandatory): the field used for aggregating. Must be of wkb type. E.g.: "geoshape_0.wkb". 260 | - `output_format`: the output_format in [`geojson`, `wkt`, `wkb`]. Default to `geojson`. 261 | - `simplify`: 262 | - `zoom`: the zoom level in range [0, 20]. 0 is the most simplified and 20 is the least. Default to 0. 263 | - `algorithm`: simplify algorithm in [`DOUGLAS_PEUCKER`, `TOPOLOGY_PRESERVING`]. Default to `DOUGLAS_PEUCKER`. 264 | - `size`: can be set to define how many buckets should be returned. See elasticsearch official terms aggregation documentation for more explanation. Buckets are ordered by the length (perimeter for polygons) of their shape, longer shapes first. 265 | - `shard_size`: can be used to minimize the extra work that comes with bigger requested `size`. See elasticsearch official terms aggregation documentation for more explanation. 266 | 267 | 268 | #### Example 269 | 270 | ``` 271 | GET main/_search?size=0 272 | { 273 | "aggs": { 274 | "geo_preview": { 275 | "geoshape": { 276 | "field": "geoshape_0.wkb", 277 | "output_format": "wkb", 278 | "simplify": { 279 | "zoom": 8, 280 | "algorithm": "douglas_peucker" 281 | }, 282 | "size": 10, 283 | "shard_size": 10 284 | } 285 | } 286 | } 287 | } 288 | ``` 289 | 290 | Result: 291 | ``` 292 | "aggregations": { 293 | "geo_preview": { 294 | "buckets": [ 295 | { 296 | "key": "AAAAAAMAAAABAAAABkAALAAAAAAAQEhMXSKIhts/+uT//////0BIhrDKsBJAQACGAAAAAABASJ2wcvbTDkAGPIAAAAAAQEiZGKALhAxAChqAAAAAAEBIdhR0tDaAQAAsAAAAAABASExdIoiG2w==", 297 | "digest": "-5012816342630707936", 298 | "type": "Polygon", 299 | "doc_count": 1 300 | } 301 | ] 302 | } 303 | } 304 | ``` 305 | 306 | 307 | 308 | 309 | ### Geoshape simplify script 310 | 311 | Search script for simplifying shapes dynamically. 312 | 313 | 314 | #### Script params 315 | 316 | - `field`: the field to apply the script to. 317 | - `zoom`: the zoom level in range [0, 20]. 0 is the most simplified and 20 is the least. Default to 0. 318 | - `algorithm`: simplify algorithm in [`DOUGLAS_PEUCKER`, `TOPOLOGY_PRESERVING`]. Default to `DOUGLAS_PEUCKER`. 319 | - `output_format`: the output_format in [`geojson`, `wkt`, `wkb`]. Default to `geojson`. 320 | 321 | 322 | #### Example 323 | 324 | ``` 325 | GET main/_search 326 | { 327 | "script_fields": { 328 | "simplified_shape": { 329 | "script": { 330 | "lang": "geo_extension_scripts", 331 | "source": "geo_simplify", 332 | "params": { 333 | "field": "geoshape_0", 334 | "zoom": 8, 335 | "output_format": "wkt" 336 | } 337 | } 338 | } 339 | } 340 | } 341 | ``` 342 | 343 | Result: 344 | ``` 345 | "hits": [ 346 | { 347 | "fields": { 348 | "simplified_shape": [ 349 | { 350 | "real_type": "Polygon", 351 | "geom": "POLYGON ((2.021484375 48.596592251456705, 1.6809082031249998 49.05227025601607, 2.0654296875 49.23194729854559, 2.779541015625 49.196064000723794, 3.262939453125 48.922499263758255, 2.021484375 48.596592251456705))", 352 | "type": "Polygon" 353 | } 354 | ] 355 | } 356 | } 357 | ``` 358 | 359 | ## Development Environment Setup 360 | 361 | Built with Java 17 and Gradle 8.10.2. 362 | 363 | Build the plugin using gradle: 364 | 365 | ```sh 366 | ./gradlew build 367 | ``` 368 | 369 | or 370 | ```sh 371 | ./gradlew assemble # (to avoid the test suite) 372 | ``` 373 | 374 | Then you can find the current version of the plugin at `elasticsearch-plugin-geoshape-7.17.z.d.zip` 375 | 376 | In case you have to upgrade Gradle, you can do it with `./gradlew wrapper --gradler-version x.y.z`. 377 | 378 | Then the following command will start a dockerized ES and will install the previously built plugin: 379 | 380 | ```sh 381 | docker compose up 382 | ``` 383 | 384 | Check the Elasticsearch instance at `localhost:9200` and the plugin version with `localhost:9200/_cat/plugins`. 385 | 386 | Please be careful during development: you'll need to manually rebuild the .zip using `./gradlew build` on each code 387 | change before running `docker-compose` up again. 388 | 389 | > NOTE: In `docker-compose.yml` you can uncomment the debug env and attach a REMOTE JVM on `*:5005` to debug the plugin. 390 | 391 | Note also that this plugin depends on some Types and Classes from the legacygeo elasticsearch module. In our Gradle script, you find: 392 | 393 | ``` 394 | compileOnly files('libs/legacy-geo-7.17.28.jar') 395 | ``` 396 | 397 | or a different version number depending on the current supported Elasticsearch version. 398 | 399 | If you're going to update this plugin with a new version, you also need to update this JAR file. To do so, it's necassary to build it from source. So you have to `git clone` the [elasticsearch source code](https://github.com/elastic/elasticsearch), `git checkout vX.Y.Z` with the wanted version and run a build for this specific module with: 400 | 401 | `./gradlew :modules:legacy-geo:assemble` with the same Java version. Then you find a JAR file in `modules/legacy-geo/build/distributions/`, e.g. `legacy-geo-7.17.28-SNAPSHOT.jar`. Just copy it into `./libs/legacy-geo-7.17.28.jar` and run the compilation of this plugin with gradle. 402 | 403 | Take also a look at this potential deprecation about legacy-geo https://github.com/elastic/elasticsearch/issues/96097 404 | 405 | ## License 406 | 407 | This software is under The MIT License (MIT). 408 | -------------------------------------------------------------------------------- /build.gradle: -------------------------------------------------------------------------------- 1 | buildscript { 2 | repositories { 3 | mavenCentral() 4 | 5 | } 6 | 7 | dependencies { 8 | classpath "org.elasticsearch.gradle:build-tools:${elastic_version}" 9 | } 10 | } 11 | 12 | 13 | repositories { 14 | mavenCentral() 15 | mavenLocal() 16 | maven { 17 | url "https://repo1.maven.org/maven2" 18 | } 19 | } 20 | 21 | 22 | group = 'org.elasticsearch.plugin' 23 | version = "${plugin_version}" 24 | 25 | apply plugin: 'java' 26 | apply plugin: 'idea' 27 | apply plugin: 'elasticsearch.esplugin' 28 | apply plugin: 'elasticsearch.yaml-rest-test' 29 | 30 | esplugin { 31 | name 'elasticsearch-plugin-geoshape' 32 | description 'Add extra geo capabilities to default elastic geo_shape data type.' 33 | classname 'org.opendatasoft.elasticsearch.plugin.GeoExtensionPlugin' 34 | licenseFile = rootProject.file('LICENSE') 35 | noticeFile = rootProject.file('README.md') 36 | } 37 | 38 | 39 | // exclude junit from implementation to resolve a version conflict (but not from tests) 40 | configurations { 41 | implementation { 42 | exclude(group: 'junit', module: 'junit') 43 | } 44 | } 45 | 46 | dependencies { 47 | implementation "org.elasticsearch:elasticsearch:${elastic_version}" 48 | 49 | // We ship the legacy geo archive with sources, 50 | // because the compilation stage needs to resolve symbols 51 | // However this archive is not included in the final plugin bundle, since 52 | // legacy geo and its dependencies (JTS) are already present at runtime 53 | // with ES (see the jar/manifest rule below) 54 | compileOnly files('libs/legacy-geo-7.17.28.jar') 55 | 56 | // jts.io.common is not part of ES and will be shipped with this plugin 57 | implementation group: 'org.locationtech.jts.io', name: 'jts-io-common', version: '1.15.0' 58 | 59 | yamlRestTestImplementation "org.elasticsearch.test:framework:${elastic_version}" 60 | yamlRestTestImplementation "org.apache.logging.log4j:log4j-core:2.17.1" 61 | } 62 | 63 | tasks.named("yamlRestTest").configure { 64 | systemProperty 'tests.security.manager', 'false' 65 | } 66 | 67 | // Since this plugin is meant to be loaded in an ES >=7.16 instance, 68 | // we can "link" it to existing classes through a class path without adding other archives 69 | // to the plugin bundle 70 | jar { 71 | manifest { 72 | attributes( 73 | "Class-Path": "../../modules/legacy-geo/legacy-geo-7.17.28.jar ../../modules/legacy-geo/jts-core-1.15.0.jar ../../modules/legacy-geo/spatial4j-0.7.jar") 74 | 75 | } 76 | } 77 | 78 | // Make sure the ES distribution used for rest tests is the "complete" variant 79 | // Otherwise weirds errors occur (like the geo_shape type is not handled) 80 | testClusters.configureEach { 81 | testDistribution = 'DEFAULT' 82 | // disable security to disable failing warnings 83 | setting 'xpack.security.enabled', 'false' 84 | } 85 | 86 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | elasticsearch-plugin-debug: 3 | build: 4 | context: . 5 | dockerfile: docker/Dockerfile 6 | target: elasticsearch-plugin-debug 7 | environment: 8 | - discovery.type=single-node 9 | # NO DEBUG 10 | - ES_JAVA_OPTS=-Xms512m -Xmx512m 11 | # DEBUG 12 | # - ES_JAVA_OPTS=-Xms512m -Xmx512m -agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005 13 | ports: 14 | - "9200:9200" 15 | - "5005:5005" # DEBUG 16 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM docker.elastic.co/elasticsearch/elasticsearch:7.17.28 AS elasticsearch-plugin-debug 2 | 3 | COPY /build/distributions/elasticsearch-plugin-geoshape-7.17.28.0.zip /tmp/elasticsearch-plugin-geoshape-7.17.28.0.zip 4 | RUN ./bin/elasticsearch-plugin install file:/tmp/elasticsearch-plugin-geoshape-7.17.28.0.zip 5 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | elastic_version = 7.17.28 2 | plugin_version = 7.17.28.0 3 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatasoft/elasticsearch-plugin-geoshape/ef245eb68b7528329fae618b50d5910235d7ae9e/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip 4 | networkTimeout=10000 5 | zipStoreBase=GRADLE_USER_HOME 6 | zipStorePath=wrapper/dists 7 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | # This is normally unused 84 | # shellcheck disable=SC2034 85 | APP_BASE_NAME=${0##*/} 86 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. 147 | # shellcheck disable=SC3045 148 | MAX_FD=$( ulimit -H -n ) || 149 | warn "Could not query maximum file descriptor limit" 150 | esac 151 | case $MAX_FD in #( 152 | '' | soft) :;; #( 153 | *) 154 | # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. 155 | # shellcheck disable=SC3045 156 | ulimit -n "$MAX_FD" || 157 | warn "Could not set maximum file descriptor limit to $MAX_FD" 158 | esac 159 | fi 160 | 161 | # Collect all arguments for the java command, stacking in reverse order: 162 | # * args from the command line 163 | # * the main class name 164 | # * -classpath 165 | # * -D...appname settings 166 | # * --module-path (only if needed) 167 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 168 | 169 | # For Cygwin or MSYS, switch paths to Windows format before running java 170 | if "$cygwin" || "$msys" ; then 171 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 172 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 173 | 174 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 175 | 176 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 177 | for arg do 178 | if 179 | case $arg in #( 180 | -*) false ;; # don't mess with options #( 181 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 182 | [ -e "$t" ] ;; #( 183 | *) false ;; 184 | esac 185 | then 186 | arg=$( cygpath --path --ignore --mixed "$arg" ) 187 | fi 188 | # Roll the args list around exactly as many times as the number of 189 | # args, so each arg winds up back in the position where it started, but 190 | # possibly modified. 191 | # 192 | # NB: a `for` loop captures its iteration list before it begins, so 193 | # changing the positional parameters here affects neither the number of 194 | # iterations, nor the values presented in `arg`. 195 | shift # remove old arg 196 | set -- "$@" "$arg" # push replacement arg 197 | done 198 | fi 199 | 200 | # Collect all arguments for the java command; 201 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 202 | # shell script including quotes and variable substitutions, so put them in 203 | # double quotes to make sure that they get re-expanded; and 204 | # * put everything else in single quotes, so that it's not re-expanded. 205 | 206 | set -- \ 207 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 208 | -classpath "$CLASSPATH" \ 209 | org.gradle.wrapper.GradleWrapperMain \ 210 | "$@" 211 | 212 | # Stop when "xargs" is not available. 213 | if ! command -v xargs >/dev/null 2>&1 214 | then 215 | die "xargs is not available" 216 | fi 217 | 218 | # Use "xargs" to parse quoted args. 219 | # 220 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 221 | # 222 | # In Bash we could simply go: 223 | # 224 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 225 | # set -- "${ARGS[@]}" "$@" 226 | # 227 | # but POSIX shell has neither arrays nor command substitution, so instead we 228 | # post-process each arg (as a line of input to sed) to backslash-escape any 229 | # character that might be a shell metacharacter, then use eval to reverse 230 | # that process (while maintaining the separation between arguments), and wrap 231 | # the whole thing up as a single "set" statement. 232 | # 233 | # This will of course break if any of these variables contains a newline or 234 | # an unmatched quote. 235 | # 236 | 237 | eval "set -- $( 238 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 239 | xargs -n1 | 240 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 241 | tr '\n' ' ' 242 | )" '"$@"' 243 | 244 | exec "$JAVACMD" "$@" 245 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | @rem This is normally unused 30 | set APP_BASE_NAME=%~n0 31 | set APP_HOME=%DIRNAME% 32 | 33 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 34 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 35 | 36 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 37 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 38 | 39 | @rem Find java.exe 40 | if defined JAVA_HOME goto findJavaFromJavaHome 41 | 42 | set JAVA_EXE=java.exe 43 | %JAVA_EXE% -version >NUL 2>&1 44 | if %ERRORLEVEL% equ 0 goto execute 45 | 46 | echo. 47 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 48 | echo. 49 | echo Please set the JAVA_HOME variable in your environment to match the 50 | echo location of your Java installation. 51 | 52 | goto fail 53 | 54 | :findJavaFromJavaHome 55 | set JAVA_HOME=%JAVA_HOME:"=% 56 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 57 | 58 | if exist "%JAVA_EXE%" goto execute 59 | 60 | echo. 61 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 62 | echo. 63 | echo Please set the JAVA_HOME variable in your environment to match the 64 | echo location of your Java installation. 65 | 66 | goto fail 67 | 68 | :execute 69 | @rem Setup the command line 70 | 71 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 72 | 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if %ERRORLEVEL% equ 0 goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | set EXIT_CODE=%ERRORLEVEL% 85 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 86 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 87 | exit /b %EXIT_CODE% 88 | 89 | :mainEnd 90 | if "%OS%"=="Windows_NT" endlocal 91 | 92 | :omega 93 | -------------------------------------------------------------------------------- /libs/legacy-geo-7.17.28.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatasoft/elasticsearch-plugin-geoshape/ef245eb68b7528329fae618b50d5910235d7ae9e/libs/legacy-geo-7.17.28.jar -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatasoft/elasticsearch-plugin-geoshape/ef245eb68b7528329fae618b50d5910235d7ae9e/src/main/java/org/opendatasoft/elasticsearch/.DS_Store -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/ingest/GeoExtensionProcessor.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.ingest; 2 | 3 | 4 | import org.apache.lucene.util.BytesRef; 5 | import org.elasticsearch.common.bytes.BytesReference; 6 | import org.elasticsearch.common.geo.GeoPoint; 7 | import org.elasticsearch.common.regex.Regex; 8 | import org.elasticsearch.ingest.AbstractProcessor; 9 | import org.elasticsearch.ingest.ConfigurationUtils; 10 | import org.elasticsearch.ingest.IngestDocument; 11 | import org.elasticsearch.ingest.Processor; 12 | import org.elasticsearch.legacygeo.XShapeCollection; 13 | import org.elasticsearch.legacygeo.builders.ShapeBuilder; 14 | import org.elasticsearch.legacygeo.builders.CoordinatesBuilder; 15 | import org.elasticsearch.legacygeo.parsers.ShapeParser; 16 | import org.elasticsearch.xcontent.DeprecationHandler; 17 | import org.elasticsearch.xcontent.NamedXContentRegistry; 18 | import org.elasticsearch.xcontent.XContentBuilder; 19 | import org.elasticsearch.xcontent.XContentParser; 20 | import org.elasticsearch.xcontent.json.JsonXContent; 21 | import org.locationtech.jts.geom.Coordinate; 22 | import org.locationtech.jts.geom.Geometry; 23 | import org.locationtech.jts.geom.GeometryFactory; 24 | import org.locationtech.jts.geom.PrecisionModel; 25 | import org.locationtech.jts.io.ParseException; 26 | import org.locationtech.jts.io.WKBWriter; 27 | import org.locationtech.jts.io.WKTReader; 28 | import org.locationtech.jts.io.WKTWriter; 29 | import org.locationtech.spatial4j.exception.InvalidShapeException; 30 | import org.locationtech.spatial4j.shape.Shape; 31 | import org.locationtech.spatial4j.shape.jts.JtsGeometry; 32 | import org.locationtech.spatial4j.shape.jts.JtsPoint; 33 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 34 | 35 | import java.io.IOException; 36 | import java.util.ArrayList; 37 | import java.util.Arrays; 38 | import java.util.List; 39 | import java.util.Map; 40 | 41 | 42 | public class GeoExtensionProcessor extends AbstractProcessor { 43 | public static final String TYPE = "geo_extension"; 44 | 45 | private final String field; 46 | private final String path; 47 | private final Boolean keepShape; 48 | private final String shapeField; 49 | private final String fixedField; 50 | private final String wkbField; 51 | private final String hashField; 52 | private final String typeField; 53 | private final String areaField; 54 | private final String bboxField; 55 | private final String centroidField; 56 | 57 | private final GeometryFactory geomFactory; 58 | private final WKTWriter wktWriter; 59 | private final WKTReader wktReader; 60 | 61 | private GeoExtensionProcessor(String tag, String description, String field, String path, Boolean keepShape, String shapeField, 62 | String fixedField, String wkbField, String hashField, String typeField, 63 | String areaField, String bboxField, String centroidField) { 64 | super(tag, description); 65 | this.field = field; 66 | this.path = path; 67 | this.keepShape = keepShape; 68 | this.shapeField = shapeField; 69 | this.fixedField = fixedField; 70 | this.wkbField = wkbField; 71 | this.hashField = hashField; 72 | this.typeField = typeField; 73 | this.areaField = areaField; 74 | this.bboxField = bboxField; 75 | this.centroidField = centroidField; 76 | 77 | PrecisionModel precisionModel = new PrecisionModel(PrecisionModel.FLOATING); 78 | this.geomFactory = new GeometryFactory(precisionModel, 0); 79 | 80 | this.wktWriter = new WKTWriter(); 81 | this.wktReader = new WKTReader(); 82 | } 83 | 84 | @SuppressWarnings("unchecked") 85 | private List getGeoShapeFieldsFromDoc(IngestDocument ingestDocument) { 86 | List fields = new ArrayList<>(); 87 | 88 | Map baseMap; 89 | if (path != null) { 90 | baseMap = ingestDocument.getFieldValue(this.path, Map.class); 91 | } else { 92 | baseMap = ingestDocument.getSourceAndMetadata(); 93 | } 94 | 95 | for (String fieldName : baseMap.keySet()) { 96 | if (Regex.simpleMatch(field, fieldName)) { 97 | if (path != null) { 98 | fieldName = path + "." + fieldName; 99 | } 100 | fields.add(fieldName); 101 | } 102 | } 103 | 104 | return fields; 105 | } 106 | 107 | private ShapeBuilder getShapeBuilderFromObject(Object object) throws IOException{ 108 | XContentBuilder contentBuilder = JsonXContent.contentBuilder().value(object); 109 | 110 | XContentParser parser = JsonXContent.jsonXContent.createParser( 111 | NamedXContentRegistry.EMPTY, DeprecationHandler.THROW_UNSUPPORTED_OPERATION, 112 | BytesReference.bytes(contentBuilder).streamInput() 113 | ); 114 | 115 | parser.nextToken(); 116 | return ShapeParser.parse(parser); 117 | } 118 | 119 | private class GeometryWithWkt { 120 | public Geometry geometry; 121 | public String wkt; 122 | } 123 | 124 | private GeometryWithWkt s4jToJts(Shape shape) { 125 | // Convert a Spatial4J shape into a JTSGeometry 126 | // A Spatial4J shape is basically either: 127 | // - a JtsPoint 128 | // - a JtsGeometry 129 | // - an XShapeCollection (multi-* and GeometryCollection) 130 | // There is a special case for MultiPoint since the WKTWriter of JTS 131 | // is not compliant with the wkt reader of ES. 132 | 133 | GeometryWithWkt geomWithWkt = new GeometryWithWkt(); 134 | Geometry geom = null; 135 | 136 | String altWKT = null; 137 | if (shape instanceof JtsPoint) { 138 | geom = ((JtsPoint) shape).getGeom(); 139 | } else if (shape instanceof JtsGeometry) { 140 | geom = ((JtsGeometry) shape).getGeom(); 141 | } else if (shape instanceof XShapeCollection) { 142 | XShapeCollection collection = (XShapeCollection) shape; 143 | ArrayList geoms = new ArrayList<>(collection.size()); 144 | ArrayList wkts = new ArrayList<>(collection.size()); 145 | 146 | boolean hasWkt = false; 147 | for (int i = 0; i < collection.size(); i++) { 148 | GeometryWithWkt g = s4jToJts(collection.get(i)); 149 | geoms.add(g.geometry); 150 | wkts.add(g.wkt); 151 | if (g.wkt != null) { 152 | hasWkt = true; 153 | } 154 | } 155 | 156 | if (hasWkt) { 157 | // if it has a WKT, it means it is a geometrycolleciton 158 | // containing a multipoint 159 | altWKT = "GEOMETRYCOLLECTION("; 160 | for (int i = 0; i < collection.size(); i++) { 161 | if (wkts.get(i) != null) { 162 | altWKT += wkts.get(i); 163 | } else { 164 | altWKT += wktWriter.write(geoms.get(i)); 165 | } 166 | if (i < collection.size() - 1) { 167 | altWKT += ","; 168 | } 169 | } 170 | altWKT += ")"; 171 | } 172 | geom = geomFactory.buildGeometry(geoms); 173 | 174 | if (geom.getGeometryType() == "MultiPoint") { 175 | // special case of multipoint where the WKT must be corrected 176 | // ES wants multipoint without extra parenthesis between points 177 | altWKT = wktWriter.write(geom).replace("((", "(").replace("))", ")").replace("), (", ", "); 178 | } 179 | } 180 | geomWithWkt.geometry = geom; 181 | geomWithWkt.wkt = altWKT; 182 | return geomWithWkt; 183 | } 184 | 185 | @SuppressWarnings("unchecked") 186 | @Override 187 | public IngestDocument execute(IngestDocument ingestDocument) throws IOException, ParseException { 188 | List geo_objects_list = getGeoShapeFieldsFromDoc(ingestDocument); 189 | for (String geoShapeField : geo_objects_list) { 190 | 191 | Object geoShapeObject = ingestDocument.getFieldValue(geoShapeField, Object.class); 192 | 193 | if (geoShapeObject == null) { 194 | continue; 195 | } 196 | 197 | // From an input Object that represents a GeoJSON we need to: 198 | // - store a WKT version of the geometry, that will also be indexed as a geometry by ES 199 | // - make sure this WKT does not have duplicated points, since ES refuse them 200 | // - make sure this WKT geometry is equal (or close enough) to the geometry indexed by ES (why ?). 201 | // It means if ES splits the geometry around the dateline, we want to do the same 202 | // - compute its area 203 | // - compute its WKB representation 204 | // - compute its centroid 205 | // 206 | // We currently do this by: 207 | // 1/ creating a Spatial4J shape from an Object (thanks to legacygeo.ShapeBuilder) 208 | // 2/ converting this Spatial4J shape to a JTSGeometry 209 | // 3/ call area(), wkbwriter(), centroid() and wktwriter on the JTSGeometry 210 | // 211 | // 2/ could be avoided but the S4J WKTWriter is of poor quality (e.g. a multipoint can be represented as a geometrycollection of points) 212 | // 1/ could be avoided if we find how to create a JTSGeometry from an Object 213 | 214 | ShapeBuilder shapeBuilder = getShapeBuilderFromObject(geoShapeObject); 215 | 216 | // buildS4J() will try to clean up and fix the shape. If it fails, an exception is raised 217 | // Included fixes: 218 | // - dateline warping (enforce lon in [-180,180]) 219 | Shape shape = null; 220 | try { 221 | try { 222 | shape = shapeBuilder.buildS4J(); 223 | } catch (InvalidShapeException e) { 224 | // buildS4J does not always deduplicate points 225 | shapeBuilder = GeoUtils.removeDuplicateCoordinates(shapeBuilder); 226 | shape = shapeBuilder.buildS4J(); 227 | } 228 | } 229 | catch (Throwable e) { 230 | // sometimes it is still not enough 231 | // e.g. when non-EPSG:4326 coordinates are input 232 | // buildS4J will try to warp date lines and may generate lots of small components 233 | // which could yield a GeometryCollection, which raises an AssertionError 234 | // So, we catch here both Error (AssertionError) and Exceptions 235 | throw new IllegalArgumentException("Unable to parse shape [" + shapeBuilder.toWKT() + "]: " + e.getMessage()); 236 | } 237 | 238 | GeometryWithWkt geometryWithWkt = s4jToJts(shape); 239 | Geometry geom = geometryWithWkt.geometry; 240 | String altWKT = geometryWithWkt.wkt; 241 | 242 | if (geom == null) { 243 | throw new IllegalArgumentException("Unable to parse shape [" + shapeBuilder.toWKT() + "]"); 244 | } 245 | 246 | ingestDocument.removeField(geoShapeField); 247 | 248 | if (keepShape) { 249 | ingestDocument.setFieldValue(geoShapeField + "." + shapeField, geoShapeObject); 250 | } 251 | 252 | if (fixedField != null) { 253 | ingestDocument.setFieldValue(geoShapeField + "." + fixedField, 254 | altWKT != null ? altWKT : wktWriter.write(geom)); 255 | } 256 | 257 | // compute and add extra geo sub-fields 258 | byte[] wkb = new WKBWriter().write(geom); // elastic will auto-encode this as b64 259 | 260 | if (hashField != null) ingestDocument.setFieldValue( 261 | geoShapeField + ".hash", String.valueOf(GeoUtils.getHashFromWKB(new BytesRef(wkb)))); 262 | if (wkbField != null) ingestDocument.setFieldValue( 263 | geoShapeField + "." + wkbField, wkb); 264 | if (typeField != null) ingestDocument.setFieldValue( 265 | geoShapeField + "." + typeField, geom.getGeometryType()); 266 | if (areaField != null) ingestDocument.setFieldValue( 267 | geoShapeField + "." + areaField, geom.getArea()); 268 | if (centroidField != null) ingestDocument.setFieldValue( 269 | geoShapeField + "." + centroidField, GeoUtils.getCentroidFromGeom(geom)); 270 | if (bboxField != null) { 271 | Coordinate[] coords = geom.getEnvelope().getCoordinates(); 272 | if (coords.length >= 4) { 273 | ingestDocument.setFieldValue( 274 | geoShapeField + "." + bboxField, 275 | GeoUtils.getBboxFromCoords(coords)); 276 | } else if (coords.length == 1) { 277 | GeoPoint point = new GeoPoint( 278 | org.elasticsearch.common.geo.GeoUtils.normalizeLat(coords[0].y), 279 | org.elasticsearch.common.geo.GeoUtils.normalizeLon(coords[0].x) 280 | ); 281 | ingestDocument.setFieldValue( 282 | geoShapeField + "." + bboxField, 283 | Arrays.asList(point, point)); 284 | } 285 | } 286 | } 287 | return ingestDocument; 288 | } 289 | 290 | @Override 291 | public String getType() { 292 | return TYPE; 293 | } 294 | 295 | public static final class Factory implements Processor.Factory { 296 | @Override 297 | public GeoExtensionProcessor create(Map registry, String processorTag, 298 | String description, 299 | Map config) { 300 | String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); 301 | String path = ConfigurationUtils.readOptionalStringProperty(TYPE, processorTag, config, "path"); 302 | 303 | boolean keep_shape = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "keep_original_shape", true); 304 | String shapeField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "shape_field", "shape"); 305 | 306 | boolean fix = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "fix_shape", true); 307 | String fixedField = null; 308 | if (fix) { 309 | fixedField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "fixed_field", "fixed_shape"); 310 | } 311 | 312 | boolean needWkb = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "wkb", true); 313 | String wkbField = null; 314 | if (needWkb) { 315 | wkbField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "wkb_field", "wkb"); 316 | } 317 | 318 | boolean needHash = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "hash", true); 319 | String hashField = null; 320 | if (needHash) { 321 | hashField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "hash_field", "hash"); 322 | } 323 | 324 | boolean needType = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "type", true); 325 | String typeField = null; 326 | if (needType) { 327 | typeField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "type_field", "type"); 328 | } 329 | 330 | boolean needArea = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "area", true); 331 | String areaField = null; 332 | if (needArea) { 333 | areaField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "area_field", "area"); 334 | } 335 | 336 | boolean needBbox = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "bbox", true); 337 | String bboxField = null; 338 | if (needBbox) { 339 | bboxField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "bbox_field", "bbox"); 340 | } 341 | 342 | boolean needCentroid = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "centroid", true); 343 | String centroidField = null; 344 | if (needCentroid) { 345 | centroidField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "centroid_field", "centroid"); 346 | } 347 | 348 | return new GeoExtensionProcessor( 349 | processorTag, 350 | description, 351 | field, 352 | path, 353 | keep_shape, 354 | shapeField, 355 | fixedField, 356 | wkbField, 357 | hashField, 358 | typeField, 359 | areaField, 360 | bboxField, 361 | centroidField 362 | ); 363 | } 364 | } 365 | } 366 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/plugin/GeoExtensionPlugin.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.plugin; 2 | 3 | import org.elasticsearch.common.settings.Settings; 4 | import org.elasticsearch.ingest.Processor; 5 | import org.elasticsearch.plugins.IngestPlugin; 6 | import org.elasticsearch.plugins.Plugin; 7 | import org.elasticsearch.plugins.ScriptPlugin; 8 | import org.elasticsearch.plugins.SearchPlugin; 9 | import org.elasticsearch.script.ScriptContext; 10 | import org.elasticsearch.script.ScriptEngine; 11 | import org.opendatasoft.elasticsearch.ingest.GeoExtensionProcessor; 12 | import org.opendatasoft.elasticsearch.script.ScriptGeoSimplify; 13 | import org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape.GeoShapeBuilder; 14 | import org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape.InternalGeoShape; 15 | 16 | import java.util.ArrayList; 17 | import java.util.Collection; 18 | import java.util.Collections; 19 | import java.util.Map; 20 | 21 | 22 | public class GeoExtensionPlugin extends Plugin implements IngestPlugin, ScriptPlugin, SearchPlugin { 23 | // Ingest plugin method 24 | @Override 25 | public Map getProcessors(Processor.Parameters parameters) { 26 | return Collections.singletonMap(GeoExtensionProcessor.TYPE, new GeoExtensionProcessor.Factory()); 27 | } 28 | 29 | // Script plugin method 30 | @Override 31 | public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { 32 | return new ScriptGeoSimplify(); 33 | } 34 | 35 | // Search plugin method 36 | @Override 37 | public ArrayList getAggregations() { 38 | ArrayList r = new ArrayList<>(); 39 | 40 | r.add( 41 | new SearchPlugin.AggregationSpec( 42 | GeoShapeBuilder.NAME, 43 | GeoShapeBuilder::new, 44 | GeoShapeBuilder::parse) 45 | .addResultReader(InternalGeoShape::new) 46 | .setAggregatorRegistrar(GeoShapeBuilder::registerAggregators) 47 | ); 48 | 49 | return r; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/plugin/GeoUtils.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.plugin; 2 | 3 | import org.apache.lucene.util.BytesRef; 4 | import org.elasticsearch.common.geo.GeoPoint; 5 | import org.elasticsearch.common.geo.Orientation; 6 | import org.elasticsearch.common.hash.MurmurHash3; 7 | import org.elasticsearch.geometry.Point; 8 | import org.elasticsearch.legacygeo.builders.GeometryCollectionBuilder; 9 | import org.elasticsearch.legacygeo.builders.LineStringBuilder; 10 | import org.elasticsearch.legacygeo.builders.MultiPolygonBuilder; 11 | import org.elasticsearch.legacygeo.builders.PolygonBuilder; 12 | import org.elasticsearch.legacygeo.builders.ShapeBuilder; 13 | import org.locationtech.jts.geom.Coordinate; 14 | import org.locationtech.jts.geom.Geometry; 15 | import org.locationtech.jts.geom.GeometryCollection; 16 | import org.locationtech.jts.geom.MultiPolygon; 17 | import org.locationtech.jts.geom.Polygon; 18 | import org.locationtech.jts.io.ParseException; 19 | import org.locationtech.jts.io.WKBReader; 20 | import org.locationtech.jts.io.WKBWriter; 21 | import org.locationtech.jts.io.WKTWriter; 22 | import org.locationtech.jts.io.geojson.GeoJsonWriter; 23 | import org.elasticsearch.geometry.Line; 24 | 25 | import java.util.ArrayList; 26 | import java.util.Arrays; 27 | import java.util.Collection; 28 | import java.util.HashMap; 29 | import java.util.Iterator; 30 | import java.util.List; 31 | import java.util.ListIterator; 32 | import java.util.Vector; 33 | 34 | public class GeoUtils { 35 | public enum OutputFormat { 36 | WKT, 37 | WKB, 38 | GEOJSON 39 | } 40 | 41 | public enum SimplifyAlgorithm { 42 | DOUGLAS_PEUCKER, 43 | TOPOLOGY_PRESERVING 44 | } 45 | 46 | public static long getHashFromWKB(BytesRef wkb) { 47 | return MurmurHash3.hash128(wkb.bytes, wkb.offset, wkb.length, 0, new MurmurHash3.Hash128()).h1; 48 | } 49 | 50 | public static List getBboxFromCoords(Coordinate[] coords) { 51 | GeoPoint topLeft = new GeoPoint( 52 | org.elasticsearch.common.geo.GeoUtils.normalizeLat(coords[0].y), 53 | org.elasticsearch.common.geo.GeoUtils.normalizeLon(coords[0].x) 54 | ); 55 | GeoPoint bottomRight = new GeoPoint( 56 | org.elasticsearch.common.geo.GeoUtils.normalizeLat(coords[2].y), 57 | org.elasticsearch.common.geo.GeoUtils.normalizeLon(coords[2].x) 58 | ); 59 | return Arrays.asList(topLeft, bottomRight); 60 | } 61 | 62 | public static GeoPoint getCentroidFromGeom(Geometry geom) { 63 | Geometry geom_centroid = geom.getCentroid(); 64 | return new GeoPoint(geom_centroid.getCoordinate().y, geom_centroid.getCoordinate().x); 65 | } 66 | 67 | // Return true if wkb is a point 68 | // http://en.wikipedia.org/wiki/Well-known_text#Well-known_binary 69 | public static boolean wkbIsPoint(byte[] wkb) { 70 | if (wkb.length < 5) { 71 | return false; 72 | } 73 | 74 | // Big endian or little endian shape representation 75 | if (wkb[0] == 0) { 76 | return wkb[1] == 0 && wkb[2] == 0 && wkb[3] == 0 && wkb[4] == 1; 77 | } else { 78 | return wkb[1] == 1 && wkb[2] == 0 && wkb[3] == 0 && wkb[4] == 0; 79 | } 80 | } 81 | 82 | public static double getMeterByPixel(int zoom, double lat) { 83 | return (org.elasticsearch.common.geo.GeoUtils.EARTH_EQUATOR / 256) * (Math.cos(Math.toRadians(lat)) / Math.pow(2, zoom)); 84 | } 85 | 86 | public static double getDecimalDegreeFromMeter(double meter) { 87 | return meter * 360 / org.elasticsearch.common.geo.GeoUtils.EARTH_EQUATOR; 88 | } 89 | 90 | public static double getDecimalDegreeFromMeter(double meter, double latitude) { 91 | return meter * 360 / (org.elasticsearch.common.geo.GeoUtils.EARTH_EQUATOR * Math.cos(Math.toRadians(latitude))); 92 | } 93 | 94 | public static double getToleranceFromZoom(int zoom) { 95 | /* 96 | This is a simplified formula for 97 | double meterByPixel = GeoUtils.getMeterByPixel(zoom, lat); 98 | double tol = GeoUtils.getDecimalDegreeFromMeter(meterByPixel, lat); 99 | */ 100 | return 360 / (256 * Math.pow(2, zoom)); 101 | } 102 | 103 | 104 | public static String exportWkbTo(BytesRef wkb, OutputFormat output_format, GeoJsonWriter geoJsonWriter) 105 | throws ParseException { 106 | switch (output_format) { 107 | case WKT: 108 | Geometry geom = new WKBReader().read(wkb.bytes); 109 | return new WKTWriter().write(geom); 110 | case WKB: 111 | return WKBWriter.toHex(wkb.bytes); 112 | default: 113 | Geometry geo = new WKBReader().read(wkb.bytes); 114 | return geoJsonWriter.write(geo); 115 | } 116 | } 117 | 118 | public static String exportGeoTo(Geometry geom, OutputFormat outputFormat, GeoJsonWriter geoJsonWriter) { 119 | switch (outputFormat) { 120 | case WKT: 121 | return new WKTWriter().write(geom); 122 | case WKB: 123 | return WKBWriter.toHex(new WKBWriter().write(geom)); 124 | default: 125 | return geoJsonWriter.write(geom); 126 | } 127 | } 128 | 129 | public static LineStringBuilder removeDuplicateCoordinates(LineStringBuilder builder) { 130 | Line line = (Line)builder.buildGeometry(); // no direct access to coordinates 131 | Vector newCoordinates = new Vector(); 132 | 133 | Point previous = null; 134 | for (int i = 0; i < line.length(); i++) { 135 | Point current = new Point(line.getX(i), line.getY(i)); 136 | if ((previous != null) && (previous.equals(current))) { 137 | continue; 138 | } 139 | newCoordinates.add(new Coordinate(current.getX(), current.getY())); 140 | previous = current; 141 | } 142 | return new LineStringBuilder(newCoordinates); 143 | } 144 | 145 | public static PolygonBuilder removeDuplicateCoordinates(PolygonBuilder builder) { 146 | LineStringBuilder exteriorBuilder = removeDuplicateCoordinates(builder.shell()); 147 | PolygonBuilder pb = new PolygonBuilder(exteriorBuilder, /* unused */Orientation.RIGHT, true); 148 | List holes = builder.holes(); 149 | for (int i = 0; i < holes.size(); i++) { 150 | pb.hole(removeDuplicateCoordinates(holes.get(i)), true); 151 | } 152 | return pb; 153 | } 154 | 155 | public static MultiPolygonBuilder removeDuplicateCoordinates(MultiPolygonBuilder builder) { 156 | MultiPolygonBuilder mpb = new MultiPolygonBuilder(); 157 | List polygons = builder.polygons(); 158 | for (int i = 0; i < polygons.size(); i++) { 159 | mpb.polygon(removeDuplicateCoordinates(polygons.get(i))); 160 | } 161 | return mpb; 162 | } 163 | 164 | public static GeometryCollectionBuilder removeDuplicateCoordinates(GeometryCollectionBuilder builder) { 165 | GeometryCollectionBuilder gcb = new GeometryCollectionBuilder(); 166 | for (int i = 0; i < builder.numShapes(); i++) { 167 | gcb.shape(removeDuplicateCoordinates(builder.getShapeAt(i))); 168 | } 169 | return gcb; 170 | } 171 | 172 | public static ShapeBuilder removeDuplicateCoordinates(ShapeBuilder builder){ 173 | if (builder instanceof LineStringBuilder) { 174 | return removeDuplicateCoordinates((LineStringBuilder)builder); 175 | } 176 | if (builder instanceof PolygonBuilder) { 177 | return removeDuplicateCoordinates((PolygonBuilder)builder); 178 | } 179 | if (builder instanceof MultiPolygonBuilder) { 180 | return removeDuplicateCoordinates((MultiPolygonBuilder)builder); 181 | } 182 | if (builder instanceof GeometryCollectionBuilder) { 183 | return removeDuplicateCoordinates((GeometryCollectionBuilder) builder); 184 | } 185 | return builder; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/script/ScriptGeoSimplify.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.script; 2 | 3 | import org.apache.lucene.index.LeafReaderContext; 4 | import org.apache.lucene.util.BytesRef; 5 | import org.elasticsearch.index.fielddata.ScriptDocValues; 6 | import org.elasticsearch.script.FieldScript; 7 | import org.elasticsearch.script.ScriptContext; 8 | import org.elasticsearch.script.ScriptEngine; 9 | import org.elasticsearch.script.ScriptException; 10 | import org.elasticsearch.search.lookup.SearchLookup; 11 | import org.locationtech.jts.geom.Geometry; 12 | import org.locationtech.jts.geom.GeometryFactory; 13 | import org.locationtech.jts.io.ParseException; 14 | import org.locationtech.jts.io.WKBReader; 15 | import org.locationtech.jts.io.geojson.GeoJsonWriter; 16 | import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; 17 | import org.locationtech.jts.simplify.TopologyPreservingSimplifier; 18 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 19 | 20 | import java.util.Collections; 21 | import java.util.HashMap; 22 | import java.util.Locale; 23 | import java.util.Map; 24 | import java.util.Set; 25 | 26 | 27 | public class ScriptGeoSimplify implements ScriptEngine { 28 | public static final ScriptContext CONTEXT = new ScriptContext<>("geo_simplify", GeoSearchLeafFactory.class); 29 | 30 | @Override 31 | public String getType() { 32 | return "geo_extension_scripts"; 33 | } 34 | 35 | @Override 36 | public T compile(String scriptName, String scriptSource, 37 | ScriptContext context, Map params) { 38 | if (!context.equals(FieldScript.CONTEXT)) { 39 | throw new IllegalArgumentException(getType() 40 | + " scripts cannot be used for context [" 41 | + context.name + "]"); 42 | } 43 | if ("geo_simplify".equals(scriptName)) { 44 | FieldScript.Factory factory = GeoSearchLeafFactory::new; 45 | return context.factoryClazz.cast(factory); 46 | } 47 | throw new IllegalArgumentException("Unknown script name " + scriptSource); 48 | } 49 | 50 | @Override 51 | public Set> getSupportedContexts() { 52 | return Collections.singleton(CONTEXT); 53 | } 54 | 55 | @Override 56 | public void close() { 57 | // optionally close resources 58 | } 59 | 60 | private static class GeoSearchLeafFactory implements FieldScript.LeafFactory { 61 | private final Map params; 62 | private final SearchLookup lookup; 63 | private final String field; 64 | private final int zoom; 65 | GeoUtils.OutputFormat output_format; 66 | GeoUtils.SimplifyAlgorithm algorithm; 67 | // private final int geojson_decimals; 68 | GeoJsonWriter geoJsonWriter; 69 | 70 | 71 | private Geometry getSimplifiedShape(Geometry geometry) { 72 | double lat = geometry.getCentroid().getCoordinate().y; 73 | double meterByPixel = GeoUtils.getMeterByPixel(zoom, lat); 74 | 75 | // double tolerance = 360 / (256 * Math.pow(zoom, 3)); 76 | double tolerance = GeoUtils.getDecimalDegreeFromMeter(meterByPixel, lat); 77 | if (algorithm == GeoUtils.SimplifyAlgorithm.TOPOLOGY_PRESERVING) 78 | return TopologyPreservingSimplifier.simplify(geometry, tolerance); 79 | else 80 | return DouglasPeuckerSimplifier.simplify(geometry, tolerance); 81 | } 82 | 83 | private GeoSearchLeafFactory( 84 | Map params, SearchLookup lookup) { 85 | 86 | if (params.isEmpty()) { 87 | throw new IllegalArgumentException("[params] field is mandatory"); 88 | 89 | } 90 | if (!params.containsKey("field")) { 91 | throw new IllegalArgumentException("Missing mandatory parameter [field]"); 92 | } 93 | if (!params.containsKey("zoom")) { 94 | throw new IllegalArgumentException("Missing mandatory parameter [zoom]"); 95 | } 96 | this.params = params; 97 | this.lookup = lookup; 98 | field = params.get("field").toString(); 99 | zoom = (int) params.get("zoom"); 100 | 101 | output_format = GeoUtils.OutputFormat.GEOJSON; 102 | if (params.containsKey("output_format")) { 103 | String string_output_format = params.get("output_format").toString(); 104 | if (string_output_format != null) 105 | output_format = GeoUtils.OutputFormat.valueOf(string_output_format.toUpperCase(Locale.getDefault())); 106 | } 107 | 108 | algorithm = GeoUtils.SimplifyAlgorithm.DOUGLAS_PEUCKER; 109 | if (params.containsKey("algorithm")) { 110 | String algorithm_string = params.get("algorithm").toString(); 111 | if (algorithm_string != null) 112 | algorithm = GeoUtils.SimplifyAlgorithm.valueOf(algorithm_string.toUpperCase(Locale.getDefault())); 113 | } 114 | 115 | // geojson_decimals = 20; 116 | geoJsonWriter = new GeoJsonWriter(); 117 | } 118 | 119 | @Override 120 | public FieldScript newInstance(LeafReaderContext context) { 121 | return new FieldScript(params, lookup, context) { 122 | @Override 123 | public Object execute() { 124 | Map resMap = new HashMap<>(); 125 | 126 | BytesRef wkb; 127 | try { 128 | ScriptDocValues values_list = getDoc().get(field); 129 | wkb = (BytesRef) values_list.get(0); 130 | } 131 | catch (Exception e) { 132 | return resMap; 133 | } 134 | 135 | GeometryFactory geometryFactory = new GeometryFactory(); 136 | try { 137 | Geometry geom = new WKBReader().read(wkb.bytes); 138 | String realType = geom.getGeometryType(); 139 | Geometry simplifiedGeom = getSimplifiedShape(geom); 140 | if (!simplifiedGeom.isEmpty()) { 141 | resMap.put("shape", GeoUtils.exportGeoTo(simplifiedGeom, output_format, geoJsonWriter)); 142 | resMap.put("type", simplifiedGeom.getGeometryType()); 143 | resMap.put("real_type", realType); 144 | } else { 145 | // If the simplified polygon is empty because it was too small, return a point 146 | resMap.put("shape", GeoUtils.exportGeoTo( 147 | geometryFactory.createPoint(geom.getCoordinate()), 148 | output_format, 149 | geoJsonWriter)); 150 | resMap.put("type", "SimplificationPoint"); 151 | } 152 | } catch (ParseException e) { 153 | throw new ScriptException("Can't parse WKB", e.getCause(), Collections.emptyList(), 154 | "geo_simplified", "geo_extension_scripts"); 155 | } 156 | 157 | return resMap; 158 | } 159 | 160 | }; 161 | } 162 | 163 | } 164 | 165 | } 166 | 167 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/GeoShape.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; 4 | 5 | import java.util.List; 6 | 7 | /** 8 | * An aggregation of geo_shape using wkb field. 9 | */ 10 | public interface GeoShape extends MultiBucketsAggregation { 11 | interface Bucket extends MultiBucketsAggregation.Bucket {} 12 | enum Algorithm { 13 | DOUGLAS_PEUCKER, 14 | TOPOLOGY_PRESERVING 15 | } 16 | 17 | @Override 18 | List getBuckets(); 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/GeoShapeAggregator.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.apache.lucene.index.LeafReaderContext; 4 | import org.apache.lucene.util.BytesRef; 5 | import org.apache.lucene.util.BytesRefBuilder; 6 | import org.elasticsearch.ElasticsearchException; 7 | import org.elasticsearch.common.io.stream.StreamInput; 8 | import org.elasticsearch.common.io.stream.StreamOutput; 9 | import org.elasticsearch.common.io.stream.Writeable; 10 | import org.elasticsearch.common.util.BytesRefHash; 11 | import org.elasticsearch.core.Releasables; 12 | import org.elasticsearch.index.fielddata.SortedBinaryDocValues; 13 | import org.elasticsearch.search.aggregations.Aggregator; 14 | import org.elasticsearch.search.aggregations.AggregatorFactories; 15 | import org.elasticsearch.search.aggregations.CardinalityUpperBound; 16 | import org.elasticsearch.search.aggregations.InternalAggregation; 17 | import org.elasticsearch.search.aggregations.LeafBucketCollector; 18 | import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; 19 | import org.elasticsearch.search.aggregations.bucket.BucketsAggregator; 20 | import org.elasticsearch.search.aggregations.support.AggregationContext; 21 | import org.elasticsearch.search.aggregations.support.ValuesSource; 22 | import org.elasticsearch.xcontent.ToXContentFragment; 23 | import org.elasticsearch.xcontent.XContentBuilder; 24 | import org.locationtech.jts.geom.Geometry; 25 | import org.locationtech.jts.geom.GeometryFactory; 26 | import org.locationtech.jts.io.ParseException; 27 | import org.locationtech.jts.io.WKBReader; 28 | import org.locationtech.jts.io.WKBWriter; 29 | import org.locationtech.jts.simplify.DouglasPeuckerSimplifier; 30 | import org.locationtech.jts.simplify.TopologyPreservingSimplifier; 31 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 32 | 33 | import java.io.IOException; 34 | import java.util.Arrays; 35 | import java.util.Map; 36 | import java.util.Objects; 37 | 38 | 39 | public class GeoShapeAggregator extends BucketsAggregator { 40 | private final ValuesSource valuesSource; 41 | private final BytesRefHash bucketOrds; 42 | private final BucketCountThresholds bucketCountThresholds; 43 | private GeoUtils.OutputFormat output_format; 44 | private boolean must_simplify; 45 | private int zoom; 46 | private GeoShape.Algorithm algorithm; 47 | 48 | private WKBReader wkbReader; 49 | private final GeometryFactory geometryFactory; 50 | 51 | 52 | public GeoShapeAggregator( 53 | String name, 54 | AggregatorFactories factories, 55 | AggregationContext context, 56 | ValuesSource valuesSource, 57 | GeoUtils.OutputFormat output_format, 58 | boolean must_simplify, 59 | int zoom, 60 | GeoShape.Algorithm algorithm, 61 | BucketCountThresholds bucketCountThresholds, 62 | Aggregator parent, 63 | CardinalityUpperBound cardinalityUpperBound, 64 | Map metaData 65 | ) throws IOException { 66 | super(name, factories, context, parent, cardinalityUpperBound, metaData); 67 | this.valuesSource = valuesSource; 68 | this.output_format = output_format; 69 | this.must_simplify = must_simplify; 70 | this.zoom = zoom; 71 | this.algorithm = algorithm; 72 | bucketOrds = new BytesRefHash(1, context.bigArrays()); 73 | this.bucketCountThresholds = bucketCountThresholds; 74 | 75 | this.wkbReader = new WKBReader(); 76 | this.geometryFactory = new GeometryFactory(); 77 | } 78 | 79 | /** 80 | * The collector collects the docs, including or not some score (depending of the including of a Scorer) in the 81 | * collect() process. 82 | * 83 | * The LeafBucketCollector is a "Per-leaf bucket collector". It collects docs for the account of buckets. 84 | */ 85 | @Override 86 | public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { 87 | if (valuesSource == null) { 88 | return LeafBucketCollector.NO_OP_COLLECTOR; 89 | } 90 | final SortedBinaryDocValues values = valuesSource.bytesValues(ctx); 91 | return new LeafBucketCollectorBase(sub, values) { 92 | final BytesRefBuilder previous = new BytesRefBuilder(); 93 | /** 94 | * Collect the given doc in the given bucket. 95 | * Called once for every document matching a query, with the unbased document number. 96 | */ 97 | @Override 98 | public void collect(int doc, long owningBucketOrdinal) throws IOException { 99 | assert owningBucketOrdinal == 0; 100 | if (values.advanceExact(doc)) { 101 | final int valuesCount = values.docValueCount(); 102 | previous.clear(); 103 | 104 | for (int i = 0; i < valuesCount; ++i) { 105 | final BytesRef bytesValue = values.nextValue(); 106 | if (previous.get().equals(bytesValue)) { 107 | continue; 108 | } 109 | long bucketOrdinal = bucketOrds.add(bytesValue); 110 | if (bucketOrdinal < 0) { // already seen 111 | bucketOrdinal = - 1 - bucketOrdinal; 112 | collectExistingBucket(sub, doc, bucketOrdinal); 113 | } else { 114 | collectBucket(sub, doc, bucketOrdinal); 115 | } 116 | previous.copyBytes(bytesValue); 117 | } 118 | } 119 | } 120 | }; 121 | } 122 | 123 | @Override 124 | public InternalAggregation[] buildAggregations(long[] owningBucketOrdinals) throws IOException { 125 | InternalGeoShape.InternalBucket[][] topBucketsPerOrd = new InternalGeoShape.InternalBucket[owningBucketOrdinals.length][]; 126 | InternalGeoShape[] results = new InternalGeoShape[owningBucketOrdinals.length]; 127 | 128 | for (int ordIdx = 0; ordIdx < owningBucketOrdinals.length; ordIdx++) { 129 | assert owningBucketOrdinals[ordIdx] == 0; 130 | 131 | final int size = (int) Math.min(bucketOrds.size(), bucketCountThresholds.getShardSize()); 132 | // We will insert buckets in a priority queue with a capacity of up to N=size elements 133 | InternalGeoShape.BucketPriorityQueue ordered = new InternalGeoShape.BucketPriorityQueue(size); 134 | 135 | InternalGeoShape.InternalBucket spare = null; 136 | for (int i = 0; i < bucketOrds.size(); i++) { 137 | if (spare == null) { 138 | spare = new InternalGeoShape.InternalBucket(new BytesRef(), null, null, 0, 0, null); 139 | } 140 | bucketOrds.get(i, spare.wkb); 141 | 142 | // FIXME: why do we need a deepCopy here ? 143 | spare.wkb = BytesRef.deepCopyOf(spare.wkb); 144 | spare.wkbHash = String.valueOf(GeoUtils.getHashFromWKB(spare.wkb)); 145 | 146 | if (GeoUtils.wkbIsPoint(spare.wkb.bytes)) { 147 | spare.perimeter = 0; 148 | spare.realType = "Point"; 149 | } else { 150 | Geometry geom; 151 | 152 | try { 153 | geom = wkbReader.read(spare.wkb.bytes); 154 | } catch (ParseException e) { 155 | continue; 156 | } 157 | 158 | spare.perimeter = geom.getLength(); 159 | spare.realType = geom.getGeometryType(); 160 | 161 | } 162 | 163 | spare.docCount = bucketDocCount(i); 164 | spare.bucketOrd = i; 165 | spare = ordered.insertWithOverflow(spare); 166 | } 167 | 168 | // Once we get the top N results, we can compute a simplification 169 | topBucketsPerOrd[ordIdx] = new InternalGeoShape.InternalBucket[ordered.size()]; 170 | for (int i = ordered.size() - 1; i >= 0; --i) { 171 | final InternalGeoShape.InternalBucket bucket = ordered.pop(); 172 | 173 | Geometry geom; 174 | try { 175 | geom = wkbReader.read(bucket.wkb.bytes); 176 | } catch (ParseException e) { 177 | continue; 178 | } 179 | if (must_simplify) { 180 | geom = simplifyGeoShape(geom); 181 | bucket.wkb = new BytesRef(new WKBWriter().write(geom)); 182 | bucket.perimeter = geom.getLength(); 183 | 184 | } 185 | 186 | topBucketsPerOrd[ordIdx][i] = bucket; 187 | } 188 | 189 | results[ordIdx] = new InternalGeoShape( 190 | name, 191 | Arrays.asList(topBucketsPerOrd[ordIdx]), 192 | output_format, 193 | bucketCountThresholds.getRequiredSize(), 194 | bucketCountThresholds.getShardSize(), 195 | metadata()); 196 | } 197 | 198 | // Build sub-aggregations 199 | buildSubAggsForAllBuckets( 200 | topBucketsPerOrd, 201 | b -> b.bucketOrd, 202 | (b, aggregations) -> b.subAggregations = aggregations 203 | ); 204 | return results; 205 | } 206 | 207 | @Override 208 | public InternalAggregation buildEmptyAggregation() { 209 | return new InternalGeoShape(name, null, output_format, bucketCountThresholds.getRequiredSize(), 210 | bucketCountThresholds.getShardSize(), metadata()); 211 | } 212 | 213 | private Geometry simplifyGeoShape(Geometry geom) { 214 | Geometry polygonSimplified = getSimplifiedShape(geom); 215 | if (polygonSimplified.isEmpty()) { 216 | polygonSimplified = this.geometryFactory.createPoint(geom.getCoordinate()); 217 | } 218 | return polygonSimplified; 219 | } 220 | 221 | private Geometry getSimplifiedShape(Geometry geometry) { 222 | double tol = GeoUtils.getToleranceFromZoom(zoom); 223 | 224 | switch (algorithm) { 225 | case TOPOLOGY_PRESERVING: 226 | return TopologyPreservingSimplifier.simplify(geometry, tol); 227 | default: 228 | return DouglasPeuckerSimplifier.simplify(geometry, tol); 229 | } 230 | } 231 | 232 | @Override 233 | protected void doClose() { 234 | Releasables.close(bucketOrds); 235 | } 236 | 237 | public static class BucketCountThresholds implements Writeable, ToXContentFragment { 238 | private int requiredSize; 239 | private int shardSize; 240 | 241 | public BucketCountThresholds(int requiredSize, int shardSize) { 242 | this.requiredSize = requiredSize; 243 | this.shardSize = shardSize; 244 | } 245 | 246 | /** 247 | * Read from a stream. 248 | */ 249 | public BucketCountThresholds(StreamInput in) throws IOException { 250 | requiredSize = in.readInt(); 251 | shardSize = in.readInt(); 252 | } 253 | 254 | @Override 255 | public void writeTo(StreamOutput out) throws IOException { 256 | out.writeInt(requiredSize); 257 | out.writeInt(shardSize); 258 | } 259 | 260 | public BucketCountThresholds(GeoShapeAggregator.BucketCountThresholds bucketCountThresholds) { 261 | this(bucketCountThresholds.requiredSize, bucketCountThresholds.shardSize); 262 | } 263 | 264 | public void ensureValidity() { 265 | // shard_size cannot be smaller than size as we need to at least fetch size entries from every shards in order to return size 266 | if (shardSize < requiredSize) { 267 | setShardSize(requiredSize); 268 | } 269 | 270 | if (requiredSize <= 0 || shardSize <= 0) { 271 | throw new ElasticsearchException("parameters [required_size] and [shard_size] must be >0 in geoshape aggregation."); 272 | } 273 | } 274 | 275 | public int getRequiredSize() { 276 | return requiredSize; 277 | } 278 | 279 | public void setRequiredSize(int requiredSize) { 280 | this.requiredSize = requiredSize; 281 | } 282 | 283 | public int getShardSize() { 284 | return shardSize; 285 | } 286 | 287 | public void setShardSize(int shardSize) { 288 | this.shardSize = shardSize; 289 | } 290 | 291 | @Override 292 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 293 | builder.field(GeoShapeBuilder.SIZE_FIELD.getPreferredName(), requiredSize); 294 | if (shardSize != -1) { 295 | builder.field(GeoShapeBuilder.SHARD_SIZE_FIELD.getPreferredName(), shardSize); 296 | } 297 | return builder; 298 | } 299 | 300 | @Override 301 | public int hashCode() { 302 | return Objects.hash(requiredSize, shardSize); 303 | } 304 | 305 | @Override 306 | public boolean equals(Object obj) { 307 | if (obj == null) { 308 | return false; 309 | } 310 | if (getClass() != obj.getClass()) { 311 | return false; 312 | } 313 | GeoShapeAggregator.BucketCountThresholds other = (GeoShapeAggregator.BucketCountThresholds) obj; 314 | return Objects.equals(requiredSize, other.requiredSize) 315 | && Objects.equals(shardSize, other.shardSize); 316 | } 317 | } 318 | 319 | } 320 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/GeoShapeAggregatorFactory.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.elasticsearch.search.aggregations.Aggregator; 4 | import org.elasticsearch.search.aggregations.AggregatorFactories; 5 | import org.elasticsearch.search.aggregations.AggregatorFactory; 6 | import org.elasticsearch.search.aggregations.CardinalityUpperBound; 7 | import org.elasticsearch.search.aggregations.InternalAggregation; 8 | import org.elasticsearch.search.aggregations.NonCollectingAggregator; 9 | import org.elasticsearch.search.aggregations.support.AggregationContext; 10 | import org.elasticsearch.search.aggregations.support.ValuesSource; 11 | import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; 12 | import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; 13 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 14 | 15 | import java.io.IOException; 16 | import java.util.ArrayList; 17 | import java.util.Map; 18 | 19 | 20 | class GeoShapeAggregatorFactory extends ValuesSourceAggregatorFactory { 21 | 22 | private GeoUtils.OutputFormat output_format; 23 | private boolean must_simplify; 24 | private int zoom; 25 | private GeoShape.Algorithm algorithm; 26 | private final GeoShapeAggregator.BucketCountThresholds bucketCountThresholds; 27 | 28 | GeoShapeAggregatorFactory(String name, 29 | ValuesSourceConfig config, 30 | GeoUtils.OutputFormat output_format, 31 | boolean must_simplify, 32 | int zoom, 33 | GeoShape.Algorithm algorithm, 34 | GeoShapeAggregator.BucketCountThresholds bucketCountThresholds, 35 | AggregationContext context, 36 | AggregatorFactory parent, 37 | AggregatorFactories.Builder subFactoriesBuilder, 38 | Map metaData 39 | ) throws IOException { 40 | super(name, config, context, parent, subFactoriesBuilder, metaData); 41 | this.output_format = output_format; 42 | this.must_simplify = must_simplify; 43 | this.zoom = zoom; 44 | this.algorithm = algorithm; 45 | this.bucketCountThresholds = bucketCountThresholds; 46 | } 47 | 48 | @Override 49 | protected Aggregator createUnmapped( 50 | Aggregator parent, 51 | Map metadata) throws IOException { 53 | final InternalAggregation aggregation = new InternalGeoShape(name, new ArrayList<>(), output_format, 54 | bucketCountThresholds.getRequiredSize(), bucketCountThresholds.getShardSize(), 55 | metadata); 56 | return new NonCollectingAggregator(name, context, parent, factories, metadata) { 57 | @Override 58 | public InternalAggregation buildEmptyAggregation() { 59 | return aggregation; 60 | } 61 | }; 62 | } 63 | 64 | @Override 65 | protected Aggregator doCreateInternal(Aggregator parent, CardinalityUpperBound cardinality, 66 | Map metadata) throws IOException { 67 | GeoShapeAggregator.BucketCountThresholds bucketCountThresholds = new 68 | GeoShapeAggregator.BucketCountThresholds(this.bucketCountThresholds); 69 | bucketCountThresholds.ensureValidity(); 70 | ValuesSource valuesSourceBytes = config.getValuesSource(); 71 | return new GeoShapeAggregator( 72 | name, 73 | factories, 74 | context, 75 | valuesSourceBytes, 76 | output_format, 77 | must_simplify, 78 | zoom, 79 | algorithm, 80 | bucketCountThresholds, 81 | parent, 82 | cardinality, 83 | metadata); 84 | } 85 | } 86 | 87 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/GeoShapeAggregatorSupplier.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.elasticsearch.search.aggregations.Aggregator; 4 | import org.elasticsearch.search.aggregations.AggregatorFactories; 5 | import org.elasticsearch.search.aggregations.CardinalityUpperBound; 6 | import org.elasticsearch.search.aggregations.support.AggregationContext; 7 | import org.elasticsearch.search.aggregations.support.ValuesSource; 8 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 9 | 10 | import java.io.IOException; 11 | import java.util.Map; 12 | 13 | @FunctionalInterface 14 | public interface GeoShapeAggregatorSupplier { 15 | Aggregator build( 16 | String name, 17 | AggregatorFactories factories, 18 | AggregationContext context, 19 | ValuesSource valuesSource, 20 | GeoUtils.OutputFormat output_format, 21 | boolean must_simplify, 22 | int zoom, 23 | GeoShape.Algorithm algorithm, 24 | GeoShapeAggregator.BucketCountThresholds bucketCountThresholds, 25 | Aggregator parent, 26 | CardinalityUpperBound cardinalityUpperBound, 27 | Map metadata) throws IOException; 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/GeoShapeBuilder.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.elasticsearch.Version; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | import org.elasticsearch.search.aggregations.AggregationBuilder; 7 | import org.elasticsearch.search.aggregations.AggregatorFactories; 8 | import org.elasticsearch.search.aggregations.AggregatorFactories.Builder; 9 | import org.elasticsearch.search.aggregations.AggregatorFactory; 10 | import org.elasticsearch.search.aggregations.support.AggregationContext; 11 | import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; 12 | import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; 13 | import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; 14 | import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; 15 | import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; 16 | import org.elasticsearch.search.aggregations.support.ValuesSourceType; 17 | import org.elasticsearch.xcontent.ObjectParser; 18 | import org.elasticsearch.xcontent.ParseField; 19 | import org.elasticsearch.xcontent.XContentBuilder; 20 | import org.elasticsearch.xcontent.XContentParser; 21 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 22 | 23 | import java.io.IOException; 24 | import java.util.Arrays; 25 | import java.util.Collections; 26 | import java.util.List; 27 | import java.util.Locale; 28 | import java.util.Map; 29 | import java.util.Objects; 30 | 31 | 32 | /** 33 | * The builder of the aggregatorFactory. Also implements the parsing of the request. 34 | */ 35 | public class GeoShapeBuilder extends ValuesSourceAggregationBuilder 36 | /*implements MultiBucketAggregationBuilder*/ { 37 | public static final String NAME = "geoshape"; 38 | 39 | public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = 40 | new ValuesSourceRegistry.RegistryKey<>(NAME, GeoShapeAggregatorSupplier.class); 41 | 42 | private static final ParseField OUTPUT_FORMAT_FIELD = new ParseField("output_format"); 43 | public static final ParseField SIMPLIFY_FIELD = new ParseField("simplify"); 44 | public static final ParseField SIZE_FIELD = new ParseField("size"); 45 | public static final ParseField SHARD_SIZE_FIELD = new ParseField("shard_size"); 46 | 47 | public static final GeoShapeAggregator.BucketCountThresholds DEFAULT_BUCKET_COUNT_THRESHOLDS = new 48 | GeoShapeAggregator.BucketCountThresholds(10, -1); 49 | private static final ObjectParser PARSER; 50 | static { 51 | PARSER = new ObjectParser<>(GeoShapeBuilder.NAME); 52 | ValuesSourceAggregationBuilder.declareFields(PARSER, true, true, false); 53 | PARSER.declareString(GeoShapeBuilder::output_format, OUTPUT_FORMAT_FIELD); 54 | PARSER.declareObjectArray(GeoShapeBuilder::simplify_keys, 55 | (p, c) -> SimplifyKeysParser.Parser.parseSimplifyParam(p), SIMPLIFY_FIELD); 56 | PARSER.declareInt(GeoShapeBuilder::size, SIZE_FIELD); 57 | PARSER.declareInt(GeoShapeBuilder::shardSize, SHARD_SIZE_FIELD); 58 | } 59 | 60 | public static GeoShapeBuilder parse(XContentParser parser, String aggregationName) throws IOException { 61 | return PARSER.parse(parser, new GeoShapeBuilder(aggregationName), null); 62 | } 63 | 64 | static class SimplifyKeysParser { 65 | static class Parser { 66 | static List parseSimplifyParam(XContentParser parser) throws IOException { 67 | XContentParser.Token token; 68 | int zoom = -1; 69 | String algorithm = null; 70 | 71 | String currentFieldName = null; 72 | while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { 73 | if (token == XContentParser.Token.FIELD_NAME) { 74 | currentFieldName = parser.currentName(); 75 | } else if (token == XContentParser.Token.VALUE_NUMBER) { 76 | if ("zoom".equals(currentFieldName)) { 77 | zoom = parser.intValue(); 78 | } 79 | } else if (token == XContentParser.Token.VALUE_STRING) { 80 | if ("algorithm".equals(currentFieldName)) { 81 | algorithm = parser.text(); 82 | } 83 | } 84 | } 85 | if ((zoom != -1) && (algorithm != null)) 86 | return Arrays.asList(zoom, algorithm); 87 | else 88 | return Collections.emptyList(); 89 | } 90 | } 91 | } 92 | 93 | public static final GeoUtils.OutputFormat DEFAULT_OUTPUT_FORMAT = GeoUtils.OutputFormat.GEOJSON; 94 | private boolean must_simplify = false; 95 | public static final int DEFAULT_ZOOM = 0; 96 | public static final GeoShape.Algorithm DEFAULT_ALGORITHM = GeoShape.Algorithm.DOUGLAS_PEUCKER; 97 | private GeoUtils.OutputFormat output_format = DEFAULT_OUTPUT_FORMAT; 98 | private int simplify_zoom = DEFAULT_ZOOM; 99 | private GeoShape.Algorithm simplify_algorithm = DEFAULT_ALGORITHM; 100 | private GeoShapeAggregator.BucketCountThresholds bucketCountThresholds = new GeoShapeAggregator.BucketCountThresholds( 101 | DEFAULT_BUCKET_COUNT_THRESHOLDS); 102 | 103 | 104 | private GeoShapeBuilder(String name) { 105 | super(name); 106 | } 107 | 108 | /** 109 | * Read from a stream 110 | * 111 | */ 112 | public GeoShapeBuilder(StreamInput in) throws IOException { 113 | super(in); 114 | bucketCountThresholds = new GeoShapeAggregator.BucketCountThresholds(in); 115 | must_simplify = in.readBoolean(); 116 | output_format = GeoUtils.OutputFormat.valueOf(in.readString()); 117 | simplify_zoom = in.readInt(); 118 | simplify_algorithm = GeoShape.Algorithm.valueOf(in.readString()); 119 | } 120 | 121 | /** 122 | * Write to a stream 123 | */ 124 | @Override 125 | protected void innerWriteTo(StreamOutput out) throws IOException { 126 | bucketCountThresholds.writeTo(out); 127 | out.writeBoolean(must_simplify); 128 | out.writeString(output_format.name()); 129 | out.writeInt(simplify_zoom); 130 | out.writeString(simplify_algorithm.name()); 131 | } 132 | 133 | private GeoShapeBuilder(GeoShapeBuilder clone, Builder factoriesBuilder, 134 | Map metaData) { 135 | super(clone, factoriesBuilder, metaData); 136 | output_format = clone.output_format; 137 | must_simplify = clone.must_simplify; 138 | simplify_zoom = clone.simplify_zoom; 139 | simplify_algorithm = clone.simplify_algorithm; 140 | this.bucketCountThresholds = new GeoShapeAggregator.BucketCountThresholds(clone.bucketCountThresholds); 141 | } 142 | 143 | @Override 144 | protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metaData) { 145 | return new GeoShapeBuilder(this, factoriesBuilder, metaData); 146 | } 147 | 148 | @Override 149 | public BucketCardinality bucketCardinality() { 150 | return null; 151 | } 152 | 153 | private GeoShapeBuilder output_format(String output_format) { 154 | this.output_format = GeoUtils.OutputFormat.valueOf(output_format.toUpperCase(Locale.getDefault())); 155 | return this; 156 | } 157 | 158 | @SuppressWarnings("unchecked") 159 | private GeoShapeBuilder simplify_keys(List simplify) { 160 | List simplify_keys = (List) simplify.get(0); 161 | if (!simplify_keys.isEmpty()) { 162 | this.must_simplify = true; 163 | this.simplify_zoom = (int) simplify_keys.get(0); 164 | this.simplify_algorithm = GeoShape.Algorithm.valueOf(((String) simplify_keys.get(1)).toUpperCase(Locale.getDefault())); 165 | } 166 | return this; 167 | } 168 | 169 | @Override 170 | protected boolean serializeTargetValueType(Version version) { 171 | return true; 172 | } 173 | 174 | @Override 175 | protected ValuesSourceRegistry.RegistryKey getRegistryKey() { 176 | return REGISTRY_KEY; 177 | } 178 | 179 | @Override 180 | protected ValuesSourceType defaultValueSourceType() { 181 | return CoreValuesSourceType.KEYWORD; 182 | } 183 | 184 | /** 185 | * Sets the size - indicating how many term buckets should be returned 186 | * (defaults to 10) 187 | */ 188 | public GeoShapeBuilder size(int size) { 189 | if (size <= 0) { 190 | throw new IllegalArgumentException("[size] must be greater than 0. Found [" + size + "] in [" + name + "]"); 191 | } 192 | bucketCountThresholds.setRequiredSize(size); 193 | return this; 194 | } 195 | 196 | /** 197 | * Sets the shard_size - indicating the number of term buckets each shard 198 | * will return to the coordinating node (the node that coordinates the 199 | * search execution). The higher the shard size is, the more accurate the 200 | * results are. 201 | */ 202 | public GeoShapeBuilder shardSize(int shardSize) { 203 | if (shardSize <= 0) { 204 | throw new IllegalArgumentException( 205 | "[shardSize] must be greater than 0. Found [" + shardSize + "] in [" + name + "]"); 206 | } 207 | bucketCountThresholds.setShardSize(shardSize); 208 | return this; 209 | } 210 | 211 | @Override 212 | protected ValuesSourceAggregatorFactory innerBuild( 213 | AggregationContext queryShardContext, 214 | ValuesSourceConfig config, 215 | AggregatorFactory parent, 216 | AggregatorFactories.Builder subFactoriesBuilder) throws IOException { 217 | return new GeoShapeAggregatorFactory( 218 | name, config, output_format, must_simplify, simplify_zoom, simplify_algorithm, 219 | bucketCountThresholds, queryShardContext, parent, subFactoriesBuilder, metadata); 220 | } 221 | 222 | @Override 223 | protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { 224 | builder.startObject(); 225 | 226 | if (!output_format.equals(DEFAULT_OUTPUT_FORMAT)) { 227 | builder.field(OUTPUT_FORMAT_FIELD.getPreferredName(), output_format); 228 | } 229 | 230 | return builder.endObject(); 231 | } 232 | 233 | /** 234 | * Used for caching requests, amongst other things. 235 | */ 236 | @Override 237 | public int hashCode() { 238 | return Objects.hash(super.hashCode(), output_format, must_simplify, simplify_zoom, simplify_algorithm, bucketCountThresholds); 239 | } 240 | 241 | @Override 242 | public boolean equals(Object obj) { 243 | if (this == obj) return true; 244 | if (obj == null || getClass() != obj.getClass()) return false; 245 | if (!super.equals(obj)) return false; 246 | 247 | GeoShapeBuilder other = (GeoShapeBuilder) obj; 248 | return Objects.equals(output_format, other.output_format) 249 | && Objects.equals(must_simplify, other.must_simplify) 250 | && Objects.equals(simplify_zoom, other.simplify_zoom) 251 | && Objects.equals(simplify_algorithm, other.simplify_algorithm) 252 | && Objects.equals(bucketCountThresholds, other.bucketCountThresholds); 253 | } 254 | 255 | @Override 256 | public String getType() { 257 | return NAME; 258 | } 259 | 260 | public static void registerAggregators(ValuesSourceRegistry.Builder builder) { 261 | builder.register( 262 | GeoShapeBuilder.REGISTRY_KEY, 263 | CoreValuesSourceType.KEYWORD, 264 | GeoShapeAggregator::new, 265 | true); 266 | } 267 | } 268 | 269 | -------------------------------------------------------------------------------- /src/main/java/org/opendatasoft/elasticsearch/search/aggregations/bucket/geoshape/InternalGeoShape.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch.search.aggregations.bucket.geoshape; 2 | 3 | import org.apache.lucene.util.BytesRef; 4 | import org.apache.lucene.util.PriorityQueue; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.util.LongObjectPagedHashMap; 8 | import org.elasticsearch.search.aggregations.Aggregation; 9 | import org.elasticsearch.search.aggregations.Aggregations; 10 | import org.elasticsearch.search.aggregations.InternalAggregation; 11 | import org.elasticsearch.search.aggregations.InternalAggregations; 12 | import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; 13 | import org.elasticsearch.search.aggregations.KeyComparable; 14 | import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; 15 | import org.elasticsearch.xcontent.XContentBuilder; 16 | import org.locationtech.jts.io.ParseException; 17 | import org.locationtech.jts.io.geojson.GeoJsonWriter; 18 | import org.opendatasoft.elasticsearch.plugin.GeoUtils; 19 | import org.opendatasoft.elasticsearch.plugin.GeoUtils.OutputFormat; 20 | 21 | import java.io.IOException; 22 | import java.util.ArrayList; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | import java.util.Map; 26 | import java.util.Objects; 27 | 28 | 29 | /** 30 | * An internal implementation of {@link InternalMultiBucketAggregation} which extends {@link Aggregation}. 31 | */ 32 | public class InternalGeoShape extends InternalMultiBucketAggregation implements GeoShape { 34 | 35 | /** 36 | * The bucket class of InternalGeoShape. 37 | * @see MultiBucketsAggregation.Bucket 38 | */ 39 | public static class InternalBucket extends InternalMultiBucketAggregation.InternalBucket implements 40 | GeoShape.Bucket, KeyComparable { 41 | 42 | protected BytesRef wkb; 43 | protected String wkbHash; 44 | protected String realType; 45 | protected double perimeter; 46 | long bucketOrd; 47 | protected long docCount; 48 | protected InternalAggregations subAggregations; 49 | 50 | public InternalBucket(BytesRef wkb, String wkbHash, String realType, double perimeter, 51 | long docCount, InternalAggregations subAggregations) { 52 | this.wkb = wkb; 53 | this.wkbHash = wkbHash; 54 | this.realType = realType; 55 | this.docCount = docCount; 56 | this.subAggregations = subAggregations; 57 | this.perimeter = perimeter; 58 | } 59 | 60 | /** 61 | * Read from a stream. 62 | */ 63 | public InternalBucket(StreamInput in) throws IOException { 64 | wkb = in.readBytesRef(); 65 | wkbHash = in.readString(); 66 | realType = in.readString(); 67 | perimeter = in.readDouble(); 68 | docCount = in.readLong(); 69 | subAggregations = InternalAggregations.readFrom(in); 70 | } 71 | 72 | /** 73 | * Write to a stream. 74 | */ 75 | @Override 76 | public void writeTo(StreamOutput out) throws IOException { 77 | out.writeBytesRef(wkb); 78 | out.writeString(wkbHash); 79 | out.writeString(realType); 80 | out.writeDouble(perimeter); 81 | out.writeLong(docCount); 82 | subAggregations.writeTo(out); 83 | } 84 | 85 | @Override 86 | public String getKey() { 87 | return wkb.toString(); 88 | } 89 | 90 | @Override 91 | public String getKeyAsString() { 92 | return wkb.utf8ToString(); 93 | } 94 | 95 | @Override 96 | public int compareKey(InternalGeoShape.InternalBucket other) { 97 | return wkb.compareTo(other.wkb); 98 | } 99 | 100 | private long getShapeHash() { 101 | return wkb.hashCode(); 102 | } 103 | 104 | private String getType() { 105 | return realType; 106 | } 107 | 108 | private int compareTo(InternalBucket other) { 109 | if (this.docCount > other.docCount) { 110 | return 1; 111 | } 112 | else if (this.docCount < other.docCount) { 113 | return -1; 114 | } 115 | else 116 | return 0; 117 | } 118 | 119 | @Override 120 | public long getDocCount() { 121 | return docCount; 122 | } 123 | 124 | @Override 125 | public Aggregations getAggregations() { 126 | return subAggregations; 127 | } 128 | 129 | @Override 130 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 131 | builder.startObject(); 132 | builder.field(CommonFields.DOC_COUNT.getPreferredName(), docCount); 133 | subAggregations.toXContentInternal(builder, params); 134 | builder.endObject(); 135 | return builder; 136 | } 137 | 138 | } 139 | 140 | private List buckets; 141 | private final int requiredSize; 142 | private final int shardSize; 143 | private OutputFormat output_format; 144 | private GeoJsonWriter geoJsonWriter; 145 | 146 | public InternalGeoShape( 147 | String name, 148 | List buckets, 149 | OutputFormat output_format, 150 | int requiredSize, 151 | int shardSize, 152 | Map metadata 153 | ) { 154 | super(name, metadata); 155 | this.buckets = buckets; 156 | this.output_format = output_format; 157 | this.requiredSize = requiredSize; 158 | this.shardSize = shardSize; 159 | geoJsonWriter = new GeoJsonWriter(); 160 | } 161 | 162 | /** 163 | * Read from a stream. 164 | */ 165 | public InternalGeoShape(StreamInput in) throws IOException { 166 | super(in); 167 | output_format = OutputFormat.valueOf(in.readString()); 168 | requiredSize = readSize(in); 169 | shardSize = readSize(in); 170 | this.buckets = in.readList(InternalBucket::new); 171 | } 172 | 173 | /** 174 | * Write to a stream. 175 | */ 176 | @Override 177 | protected void doWriteTo(StreamOutput out) throws IOException { 178 | out.writeString(output_format.name()); 179 | writeSize(requiredSize, out); 180 | writeSize(shardSize, out); 181 | out.writeList(buckets); 182 | } 183 | 184 | @Override 185 | public String getWriteableName() { 186 | return GeoShapeBuilder.NAME; 187 | } 188 | 189 | // protected int getShardSize() { 190 | // return shardSize; 191 | // } 192 | 193 | @Override 194 | public InternalGeoShape create(List buckets) { 195 | return new InternalGeoShape(this.name, buckets, output_format, 196 | requiredSize, shardSize, this.metadata); 197 | } 198 | 199 | @Override 200 | public InternalBucket createBucket(InternalAggregations aggregations, InternalBucket prototype) { 201 | return new InternalBucket(prototype.wkb, prototype.wkbHash, prototype.realType, 202 | prototype.perimeter, prototype.docCount, aggregations); 203 | } 204 | 205 | @Override 206 | public List getBuckets() { 207 | return buckets; 208 | } 209 | 210 | /** 211 | * Reduces the given aggregations to a single one and returns it. 212 | */ 213 | @Override 214 | public InternalGeoShape reduce(List aggregations, ReduceContext reduceContext) { 215 | LongObjectPagedHashMap> buckets = null; 216 | 217 | for (InternalAggregation aggregation : aggregations) { 218 | InternalGeoShape shape = (InternalGeoShape) aggregation; 219 | if (buckets == null) { 220 | buckets = new LongObjectPagedHashMap<>(shape.buckets.size(), reduceContext.bigArrays()); 221 | } 222 | 223 | for (InternalBucket bucket : shape.buckets) { 224 | List existingBuckets = buckets.get(bucket.getShapeHash()); 225 | if (existingBuckets == null) { 226 | existingBuckets = new ArrayList<>(aggregations.size()); 227 | buckets.put(bucket.getShapeHash(), existingBuckets); 228 | } 229 | existingBuckets.add(bucket); 230 | } 231 | } 232 | 233 | final int size = !reduceContext.isFinalReduce() ? (int) buckets.size() : Math.min(requiredSize, (int) buckets.size()); 234 | 235 | BucketPriorityQueue ordered = new BucketPriorityQueue(size); 236 | for (LongObjectPagedHashMap.Cursor> cursor : buckets) { 237 | List sameCellBuckets = cursor.value; 238 | ordered.insertWithOverflow(reduceBucket(sameCellBuckets, reduceContext)); 239 | } 240 | buckets.close(); 241 | InternalBucket[] list = new InternalBucket[ordered.size()]; 242 | for (int i = ordered.size() - 1; i >= 0; i--) { 243 | list[i] = ordered.pop(); 244 | } 245 | 246 | return new InternalGeoShape(getName(), Arrays.asList(list), output_format, requiredSize, shardSize, 247 | getMetadata()); 248 | } 249 | 250 | @Override 251 | public InternalBucket reduceBucket(List buckets, ReduceContext context) { 252 | List aggregationsList = new ArrayList<>(buckets.size()); 253 | InternalBucket reduced = null; 254 | for (InternalBucket bucket : buckets) { 255 | if (reduced == null) { 256 | reduced = bucket; 257 | } else { 258 | reduced.docCount += bucket.docCount; 259 | } 260 | aggregationsList.add(bucket.subAggregations); 261 | } 262 | reduced.subAggregations = InternalAggregations.reduce(aggregationsList, context); 263 | return reduced; 264 | } 265 | 266 | @Override 267 | public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { 268 | builder.startArray(CommonFields.BUCKETS.getPreferredName()); 269 | for (InternalBucket bucket : buckets) { 270 | builder.startObject(); 271 | try { 272 | builder.field(CommonFields.KEY.getPreferredName(), GeoUtils.exportWkbTo(bucket.wkb, output_format, geoJsonWriter)); 273 | builder.field("digest", bucket.wkbHash); 274 | builder.field("type", bucket.getType()); 275 | } catch (ParseException e) { 276 | continue; 277 | } 278 | builder.field(CommonFields.DOC_COUNT.getPreferredName(), bucket.getDocCount()); 279 | bucket.getAggregations().toXContentInternal(builder, params); 280 | builder.endObject(); 281 | } 282 | builder.endArray(); 283 | return builder; 284 | } 285 | 286 | @Override 287 | public int hashCode() { 288 | return Objects.hash(super.hashCode(), buckets, output_format, requiredSize, shardSize); 289 | } 290 | 291 | @Override 292 | public boolean equals(Object obj) { 293 | if (this == obj) return true; 294 | if (obj == null || getClass() != obj.getClass()) return false; 295 | if (!super.equals(obj)) return false; 296 | 297 | InternalGeoShape that = (InternalGeoShape) obj; 298 | return Objects.equals(buckets, that.buckets) 299 | && Objects.equals(output_format, that.output_format) 300 | && Objects.equals(requiredSize, that.requiredSize) 301 | && Objects.equals(shardSize, that.shardSize); 302 | } 303 | 304 | // The priority queue is used to retain the top N buckets (i.e. shapes) 305 | // Buckets are here ordered by area (!) then by hash 306 | static class BucketPriorityQueue extends PriorityQueue { 307 | 308 | BucketPriorityQueue(int size) { 309 | super(size); 310 | } 311 | 312 | @Override 313 | protected boolean lessThan(InternalBucket o1, InternalBucket o2) { 314 | 315 | double i = o2.perimeter - o1.perimeter; 316 | if (i == 0) { 317 | i = o2.compareTo(o1); 318 | if (i == 0) { 319 | i = System.identityHashCode(o2) - System.identityHashCode(o1); 320 | } 321 | } 322 | return i > 0; 323 | } 324 | } 325 | } 326 | -------------------------------------------------------------------------------- /src/main/main.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/test/resources/rest-api-spec/test/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatasoft/elasticsearch-plugin-geoshape/ef245eb68b7528329fae618b50d5910235d7ae9e/src/test/resources/rest-api-spec/test/.DS_Store -------------------------------------------------------------------------------- /src/test/resources/rest-api-spec/test/GeoExtension/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/opendatasoft/elasticsearch-plugin-geoshape/ef245eb68b7528329fae618b50d5910235d7ae9e/src/test/resources/rest-api-spec/test/GeoExtension/.DS_Store -------------------------------------------------------------------------------- /src/yamlRestTest/java/org/opendatasoft/elasticsearch/RestApiYamlIT.java: -------------------------------------------------------------------------------- 1 | package org.opendatasoft.elasticsearch; 2 | 3 | import com.carrotsearch.randomizedtesting.annotations.Name; 4 | import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; 5 | import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; 6 | import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; 7 | 8 | /* 9 | * Generic loader for yaml integration tests 10 | */ 11 | 12 | public class RestApiYamlIT extends ESClientYamlSuiteTestCase { 13 | public RestApiYamlIT (@Name("yaml") ClientYamlTestCandidate testCandidate) { 14 | super(testCandidate); 15 | } 16 | 17 | @ParametersFactory 18 | public static Iterable parameters() throws Exception { 19 | return ESClientYamlSuiteTestCase.createParameters(); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/GeoExtension/10_basic.yml: -------------------------------------------------------------------------------- 1 | "Geoshape plugin installed": 2 | - do: 3 | cluster.state: {} 4 | 5 | - set: {master_node: master} 6 | 7 | - do: 8 | nodes.info: {} 9 | 10 | - match: {nodes.$master.plugins.0.name: elasticsearch-plugin-geoshape} 11 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/GeoExtension/20_geo_ingest_processor.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Test geo extension pipeline and dynamic mapping": 3 | 4 | # Create the pipeline 5 | - do: 6 | ingest.put_pipeline: 7 | id: "geo_extension" 8 | body: > 9 | { 10 | "description": "Add extra geo fields to geo_shape fields.", 11 | "processors": [ 12 | { 13 | "geo_extension": { 14 | "field": "geo_shape_*" 15 | } 16 | } 17 | ] 18 | } 19 | - match: { acknowledged: true } 20 | 21 | - do: 22 | ingest.get_pipeline: 23 | id: "geo_extension" 24 | - match: { geo_extension.description: "Add extra geo fields to geo_shape fields." } 25 | 26 | 27 | # Test geo extension dynamic mapping 28 | - do: 29 | indices.create: 30 | index: test_index 31 | 32 | - do: 33 | indices.put_mapping: 34 | index: test_index 35 | body: 36 | dynamic_templates: [ 37 | { 38 | "geo_shapes": { 39 | "match": "geo_shape_*", 40 | "mapping": { 41 | "properties": { 42 | "shape": {"enabled": false}, 43 | "fixed_shape": {"type": "geo_shape"}, 44 | "hash": {"type": "keyword"}, 45 | "wkb": {"type": "binary", "doc_values": true}, 46 | "type": {"type": "keyword"}, 47 | "area": {"type": "half_float"}, 48 | "bbox": {"type": "geo_point"}, 49 | "centroid": {"type": "geo_point"} 50 | } 51 | } 52 | } 53 | } 54 | ] 55 | 56 | - do: 57 | indices.get_mapping: {} 58 | 59 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.match: "geo_shape_*"} 60 | 61 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.fixed_shape.type: "geo_shape"} 62 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.hash.type: "keyword"} 63 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.wkb.type: "binary"} 64 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.wkb.doc_values: true} 65 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.type.type: "keyword"} 66 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.area.type: "half_float"} 67 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.bbox.type: "geo_point"} 68 | - match: {test_index.mappings.dynamic_templates.0.geo_shapes.mapping.properties.centroid.type: "geo_point"} 69 | 70 | 71 | # Test that mapping is correct after a document POST 72 | - do: 73 | index: 74 | index: test_index 75 | pipeline: "geo_extension" 76 | body: { 77 | "id": 1, 78 | "geo_shape_0": { 79 | "type": "Polygon", 80 | "coordinates": [ 81 | [ 82 | [ 83 | 1.6809082031249998, 84 | 49.05227025601607 85 | ], 86 | [ 87 | 2.021484375, 88 | 48.596592251456705 89 | ], 90 | [ 91 | 2.021484375, 92 | 48.596592251456705 93 | ], 94 | [ 95 | 3.262939453125, 96 | 48.922499263758255 97 | ], 98 | [ 99 | 2.779541015625, 100 | 49.196064000723794 101 | ], 102 | [ 103 | 2.0654296875, 104 | 49.23194729854559 105 | ], 106 | [ 107 | 1.6809082031249998, 108 | 49.05227025601607 109 | ] 110 | ] 111 | ] 112 | } 113 | } 114 | 115 | - do: 116 | indices.refresh: {} 117 | 118 | - do: 119 | indices.get_mapping: {} 120 | 121 | - match: {test_index.mappings.properties.geo_shape_0.properties.fixed_shape.type: "geo_shape"} 122 | - match: {test_index.mappings.properties.geo_shape_0.properties.hash.type: "keyword"} 123 | - match: {test_index.mappings.properties.geo_shape_0.properties.wkb.type: "binary"} 124 | - match: {test_index.mappings.properties.geo_shape_0.properties.wkb.doc_values: true} 125 | - match: {test_index.mappings.properties.geo_shape_0.properties.type.type: "keyword"} 126 | - match: {test_index.mappings.properties.geo_shape_0.properties.area.type: "half_float"} 127 | - match: {test_index.mappings.properties.geo_shape_0.properties.bbox.type: "geo_point"} 128 | - match: {test_index.mappings.properties.geo_shape_0.properties.centroid.type: "geo_point"} 129 | 130 | 131 | # Test that the shape has been fixed 132 | - do: 133 | search: 134 | body: 135 | query: 136 | term: 137 | id: 1 138 | 139 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "POLYGON ((2.021484375 48.596592251456705, 3.262939453125 48.922499263758255, 2.779541015625 49.196064000723794, 2.0654296875 49.23194729854559, 1.6809082031249998 49.05227025601607, 2.021484375 48.596592251456705))" } 140 | 141 | # Test Point 142 | - do: 143 | index: 144 | index: test_index 145 | pipeline: "geo_extension" 146 | body: { 147 | "id": 2, 148 | "geo_shape_0": { 149 | "type": "Point", 150 | "coordinates": [ 151 | 110.74218749999999, 152 | -82.16644600847728 153 | ] 154 | } 155 | } 156 | 157 | - do: 158 | indices.refresh: {} 159 | 160 | - do: 161 | search: 162 | body: 163 | query: 164 | term: 165 | id: 2 166 | 167 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "POINT (110.74218749999999 -82.16644600847728)"} 168 | - match: {hits.hits.0._source.geo_shape_0.bbox.0.lat: -82.16644600847728} 169 | - match: {hits.hits.0._source.geo_shape_0.bbox.0.lon: 110.74218749999999} 170 | - match: {hits.hits.0._source.geo_shape_0.bbox.1.lat: -82.16644600847728} 171 | - match: {hits.hits.0._source.geo_shape_0.bbox.1.lon: 110.74218749999999} 172 | 173 | # Test Linestring 174 | - do: 175 | index: 176 | index: test_index 177 | pipeline: "geo_extension" 178 | body: { 179 | "id": 3, 180 | "geo_shape_0": { 181 | "type": "LineString", 182 | "coordinates": [ 183 | [ 184 | 110.74218749999999, 185 | -82.16644600847728 186 | ], 187 | [ 188 | 132.890625, 189 | -83.71554430601263 190 | ] 191 | ] 192 | } 193 | } 194 | 195 | - do: 196 | indices.refresh: {} 197 | 198 | - do: 199 | search: 200 | body: 201 | query: 202 | term: 203 | id: 3 204 | 205 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "LINESTRING (110.74218749999999 -82.16644600847728, 132.890625 -83.71554430601263)"} 206 | 207 | # Test MultiPoint 208 | - do: 209 | index: 210 | index: test_index 211 | pipeline: "geo_extension" 212 | body: { 213 | "id": 4, 214 | "geo_shape_0": { 215 | "type": "MultiPoint", 216 | "coordinates": [ 217 | [ 218 | 110.74218749999999, 219 | -82.16644600847728 220 | ], 221 | [ 222 | 132.890625, 223 | -83.71554430601263 224 | ] 225 | ] 226 | } 227 | } 228 | 229 | - do: 230 | indices.refresh: {} 231 | 232 | - do: 233 | search: 234 | body: 235 | query: 236 | term: 237 | id: 4 238 | 239 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "MULTIPOINT (110.74218749999999 -82.16644600847728, 132.890625 -83.71554430601263)"} 240 | 241 | # Test MultiLineString 242 | - do: 243 | index: 244 | index: test_index 245 | pipeline: "geo_extension" 246 | body: { 247 | "id": 5, 248 | "geo_shape_0": { 249 | "type": "MultiLineString", 250 | "coordinates": [ 251 | [[ 252 | 110.74218749999999, 253 | -82.16644600847728 254 | ], 255 | [ 256 | 132.890625, 257 | -83.71554430601263 258 | ]], 259 | [[ 260 | 132.890625, 261 | -83.71554430601263 262 | ], 263 | [ 264 | 140, 265 | -83.94227191521858 266 | ]] 267 | ] 268 | } 269 | } 270 | 271 | - do: 272 | indices.refresh: {} 273 | 274 | - do: 275 | search: 276 | body: 277 | query: 278 | term: 279 | id: 5 280 | 281 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "MULTILINESTRING ((110.74218749999999 -82.16644600847728, 132.890625 -83.71554430601263), (132.890625 -83.71554430601263, 140 -83.94227191521858))"} 282 | 283 | 284 | # Test MultiPolygon 285 | - do: 286 | index: 287 | index: test_index 288 | pipeline: "geo_extension" 289 | body: { 290 | "id": 6, 291 | "geo_shape_0": { 292 | "type": "MultiPolygon", 293 | "coordinates": [[ 294 | [ 295 | [ 296 | -2.26318359375, 297 | 48.125767833701666 298 | ], 299 | [ 300 | -1.8814086914062498, 301 | 48.156925112380684 302 | ], 303 | [ 304 | -1.9033813476562498, 305 | 48.31060120649363 306 | ], 307 | [ 308 | -2.26318359375, 309 | 48.125767833701666 310 | ] 311 | ] 312 | ],[ 313 | [ 314 | [ 315 | -1.78802490234375, 316 | 48.23930899024907 317 | ], 318 | [ 319 | -1.7660522460937498, 320 | 48.123934463666366 321 | ], 322 | [ 323 | -1.5985107421875, 324 | 48.1789071002632 325 | ], 326 | [ 327 | -1.78802490234375, 328 | 48.23930899024907 329 | ] 330 | ]] 331 | ] 332 | }} 333 | 334 | - do: 335 | indices.refresh: {} 336 | 337 | - do: 338 | search: 339 | body: 340 | query: 341 | term: 342 | id: 6 343 | 344 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "MULTIPOLYGON (((-1.8814086914062498 48.156925112380684, -1.9033813476562498 48.31060120649363, -2.26318359375 48.125767833701666, -1.8814086914062498 48.156925112380684)), ((-1.7660522460937498 48.123934463666366, -1.5985107421875 48.1789071002632, -1.78802490234375 48.23930899024907, -1.7660522460937498 48.123934463666366)))"} 345 | 346 | # Test GeometryCollection 347 | - do: 348 | index: 349 | index: test_index 350 | pipeline: "geo_extension" 351 | body: { 352 | "id": 7, 353 | "geo_shape_0": { 354 | "type": "GeometryCollection", 355 | "geometries": [ 356 | { 357 | "type": "Polygon", 358 | "coordinates": [ 359 | [ 360 | [ 361 | -123.11839233491114, 362 | 49.2402245918293 363 | ], 364 | [ 365 | -123.11875175091907, 366 | 49.24005998018907 367 | ], 368 | [ 369 | -123.11737309548549, 370 | 49.23887966363327 371 | ], 372 | [ 373 | -123.11703513714964, 374 | 49.23902676693859 375 | ], 376 | [ 377 | -123.11839233491114, 378 | 49.2402245918293 379 | ] 380 | ] 381 | ] 382 | },{ 383 | "type": "LineString", 384 | "coordinates": [ 385 | [ 386 | -123.11826024867999, 387 | 49.24043019142397 388 | ], 389 | [ 390 | -123.11673782884, 391 | 49.23906844767802 392 | ] 393 | ] 394 | } 395 | ] 396 | } 397 | } 398 | 399 | - do: 400 | indices.refresh: {} 401 | 402 | - do: 403 | search: 404 | body: 405 | query: 406 | term: 407 | id: 7 408 | 409 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "GEOMETRYCOLLECTION (POLYGON ((-123.11875175091907 49.24005998018907, -123.11737309548549 49.23887966363327, -123.11703513714964 49.23902676693859, -123.11839233491114 49.2402245918293, -123.11875175091907 49.24005998018907)), LINESTRING (-123.11826024867999 49.24043019142397, -123.11673782884 49.23906844767802))"} 410 | 411 | # Test shape accross dateline is warped correctly 412 | - do: 413 | index: 414 | index: test_index 415 | pipeline: "geo_extension" 416 | body: { 417 | "id": 8, 418 | "geo_shape_0": { 419 | "type": "Polygon", 420 | "coordinates": [ 421 | [ 422 | [ 423 | 110.74218749999999, 424 | -82.16644600847728 425 | ], 426 | [ 427 | 132.890625, 428 | -83.71554430601263 429 | ], 430 | [ 431 | 132.890625, 432 | -83.71554430601263 433 | ], 434 | [ 435 | 213.75, 436 | -83.94227191521858 437 | ], 438 | [ 439 | 110.74218749999999, 440 | -82.16644600847728 441 | ] 442 | ] 443 | ] 444 | } 445 | } 446 | 447 | - do: 448 | indices.refresh: {} 449 | 450 | 451 | # Test that the shape has been fixed 452 | - do: 453 | search: 454 | body: 455 | query: 456 | term: 457 | id: 8 458 | 459 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "MULTIPOLYGON (((180 -83.84763778268045, 180 -83.36043134509174, 110.74218749999999 -82.16644600847728, 132.890625 -83.71554430601263, 180 -83.84763778268045)), ((-180 -83.36043134509174, -180 -83.84763778268045, -146.25 -83.94227191521858, -180 -83.36043134509174)))" } 460 | 461 | # Test point deduplication (from Helpscout #22513 - vancouver domain) 462 | - do: 463 | index: 464 | index: test_index 465 | pipeline: "geo_extension" 466 | body: { 467 | "id": 9, 468 | "geo_shape_0": { 469 | "type": "Polygon", 470 | "coordinates": [[ 471 | [-123.05572027973177, 49.25832825652564], 472 | [-123.05565987253833, 49.25831694688045], 473 | [-123.05559934281857, 49.25830631249652], 474 | [-123.05551963301089, 49.25830671063574], 475 | [-123.05551521820138, 49.25865914517688], 476 | [-123.05572342764653, 49.25865850523312], 477 | [-123.05572027973177, 49.25832825652564], 478 | [-123.05572027973177, 49.25832825652564] 479 | ]] 480 | } 481 | } 482 | 483 | - do: 484 | indices.refresh: {} 485 | 486 | 487 | # Test that the shape has been fixed 488 | - do: 489 | search: 490 | body: 491 | query: 492 | term: 493 | id: 9 494 | 495 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "POLYGON ((-123.05565987253833 49.25831694688045, -123.05559934281857 49.25830631249652, -123.05551963301089 49.25830671063574, -123.05551521820138 49.25865914517688, -123.05572342764653 49.25865850523312, -123.05572027973177 49.25832825652564, -123.05565987253833 49.25831694688045))" } 496 | 497 | # Test point deduplication (from Helpscout #22451 - basel-stadt - kept in the original coordinate system) 498 | # This generates an assertion error (!) "GeometryCollection unsupported" 499 | - do: 500 | catch: /GeometryCollection unsupported/ 501 | index: 502 | index: test_index 503 | pipeline: "geo_extension" 504 | body: { 505 | "id": 10, 506 | "geo_shape_0": { 507 | "type": "Polygon", 508 | "coordinates": [[ 509 | [2611313.705, 1267399.002], 510 | [2611313.705, 1267399.002], 511 | [2611313.663, 1267399.0590000001], 512 | [2611264.938, 1267459.53], 513 | [2611264.723, 1267459.858], 514 | [2611264.576, 1267460.222], 515 | [2611264.503, 1267460.607], 516 | [2611264.506, 1267460.999], 517 | [2611264.586, 1267461.383], 518 | [2611264.74, 1267461.744], 519 | [2611264.96, 1267462.068], 520 | [2611265.24, 1267462.342], 521 | [2611265.568, 1267462.557], 522 | [2611265.932, 1267462.704], 523 | [2611266.3170000003, 1267462.777], 524 | [2611266.7090000003, 1267462.774], 525 | [2611267.093, 1267462.6940000001], 526 | [2611267.454, 1267462.54], 527 | [2611267.778, 1267462.32], 528 | [2611268.052, 1267462.04], 529 | [2611316.752, 1267401.601], 530 | [2611316.7970000003, 1267401.55], 531 | [2611316.867, 1267401.454], 532 | [2611316.867, 1267401.454], 533 | [2611316.937, 1267401.358], 534 | [2611317.136, 1267401.02], 535 | [2611317.265, 1267400.6500000001], 536 | [2611317.319, 1267400.262], 537 | [2611317.2970000003, 1267399.871], 538 | [2611317.199, 1267399.491], 539 | [2611317.028, 1267399.138], 540 | [2611316.792, 1267398.825], 541 | [2611316.499, 1267398.564], 542 | [2611316.161, 1267398.365], 543 | [2611315.791, 1267398.236], 544 | [2611315.403, 1267398.182], 545 | [2611315.012, 1267398.204], 546 | [2611314.632, 1267398.3020000001], 547 | [2611314.279, 1267398.473], 548 | [2611313.966, 1267398.709], 549 | [2611313.705, 1267399.002], 550 | [2611313.705, 1267399.002] 551 | ]] 552 | } 553 | } 554 | - match: {error.root_cause.0.type: "illegal_argument_exception"} 555 | 556 | # Test point deduplication (from Helpscout #22451 - basel-stadt) 557 | - do: 558 | index: 559 | index: test_index 560 | pipeline: "geo_extension" 561 | body: { 562 | "id": 10, 563 | "geo_shape_0": { 564 | "type": "Polygon", 565 | "coordinates": [[ 566 | [7.588936981310943, 47.55720863846534], 567 | [7.588936981310943, 47.55720863846534], 568 | [7.588936424737011, 47.55720915182582], 569 | [7.588290582941092, 47.55775384276131], 570 | [7.588287734663826, 47.557756796361765], 571 | [7.588285790758167, 47.557760072575384], 572 | [7.588284830563759, 47.55776353639435], 573 | [7.588284880300491, 47.557767061872795], 574 | [7.588285952876823, 47.55777051408822], 575 | [7.588288008056212, 47.55777375818632], 576 | [7.588290939196003, 47.557776668391696], 577 | [7.58829466625118, 47.55777912788855], 578 | [7.588299029559907, 47.557781055938285], 579 | [7.588303869460291, 47.55778237180224], 580 | [7.588308986507193, 47.55778302177403], 581 | [7.588314194642488, 47.557782988105124], 582 | [7.588319294546984, 47.55778226205775], 583 | [7.588324087002335, 47.55778087086895], 584 | [7.588328386202518, 47.557778886727185], 585 | [7.588332019577427, 47.55777636381657], 586 | [7.58897752987938, 47.55723196088119], 587 | [7.588978126463396, 47.55723150143134], 588 | [7.588979054061008, 47.55723063683655], 589 | [7.588979054061008, 47.55723063683655], 590 | [7.588979981658589, 47.5572297722418], 591 | [7.588982617038069, 47.55722672896098], 592 | [7.588984321579252, 47.5572233990813], 593 | [7.588985029206171, 47.557219908600395], 594 | [7.588984727012668, 47.55721639244084], 595 | [7.588983415353376, 47.55721297651448], 596 | [7.588981134491882, 47.55720980466902], 597 | [7.588977991046303, 47.55720699368544], 598 | [7.588974091609352, 47.55720465135093], 599 | [7.588969595867449, 47.55720286739679], 600 | [7.588964676742427, 47.557201713549794], 601 | [7.58895952036617, 47.557201234538525], 602 | [7.588954326055416, 47.55720143909957], 603 | [7.588949279815451, 47.557202326993135], 604 | [7.588944594147866, 47.55720387096396], 605 | [7.588940441569443, 47.557205998839706], 606 | [7.588936981310943, 47.55720863846534], 607 | [7.588936981310943, 47.55720863846534] 608 | ]] 609 | } 610 | } 611 | - do: 612 | indices.refresh: {} 613 | # Test that the shape has been fixed 614 | - do: 615 | search: 616 | body: 617 | query: 618 | term: 619 | id: 10 620 | 621 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "POLYGON ((7.588940441569443 47.557205998839706, 7.588944594147866 47.55720387096396, 7.588949279815451 47.557202326993135, 7.588954326055416 47.55720143909957, 7.58895952036617 47.557201234538525, 7.588964676742427 47.557201713549794, 7.588969595867449 47.55720286739679, 7.588974091609352 47.55720465135093, 7.588977991046303 47.55720699368544, 7.588981134491882 47.55720980466902, 7.588983415353376 47.55721297651448, 7.588984727012668 47.55721639244084, 7.588985029206171 47.557219908600395, 7.588984321579252 47.5572233990813, 7.588982617038069 47.55722672896098, 7.588979981658589 47.5572297722418, 7.588979054061008 47.55723063683655, 7.588978126463396 47.55723150143134, 7.58897752987938 47.55723196088119, 7.588332019577427 47.55777636381657, 7.588328386202518 47.557778886727185, 7.588324087002335 47.55778087086895, 7.588319294546984 47.55778226205775, 7.588314194642488 47.557782988105124, 7.588308986507193 47.55778302177403, 7.588303869460291 47.55778237180224, 7.588299029559907 47.557781055938285, 7.58829466625118 47.55777912788855, 7.588290939196003 47.557776668391696, 7.588288008056212 47.55777375818632, 7.588285952876823 47.55777051408822, 7.588284880300491 47.557767061872795, 7.588284830563759 47.55776353639435, 7.588285790758167 47.557760072575384, 7.588287734663826 47.557756796361765, 7.588290582941092 47.55775384276131, 7.588936424737011 47.55720915182582, 7.588936981310943 47.55720863846534, 7.588940441569443 47.557205998839706))"} 622 | 623 | # Test point deduplication on linestring 624 | # (points do not need to be deduplicated on linestrings) 625 | - do: 626 | index: 627 | index: test_index 628 | pipeline: "geo_extension" 629 | body: { 630 | "id": 11, 631 | "geo_shape_0": { 632 | "type": "LineString", 633 | "coordinates": [ 634 | [0.0, 0.0], 635 | [0.0, 0.0], 636 | [1.0, 0.0] 637 | ] 638 | } 639 | } 640 | 641 | - do: 642 | indices.refresh: {} 643 | 644 | 645 | # Test that the shape has been fixed 646 | - do: 647 | search: 648 | body: 649 | query: 650 | term: 651 | id: 11 652 | 653 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "LINESTRING (0 0, 0 0, 1 0)" } 654 | 655 | # Test point deduplication on multipolygon 656 | - do: 657 | index: 658 | index: test_index 659 | pipeline: "geo_extension" 660 | body: { 661 | "id": 12, 662 | "geo_shape_0": { 663 | "type": "MultiPolygon", 664 | "coordinates": [[[ 665 | [0.0, 0.0], 666 | [0.0, 0.0], 667 | [1.0, 0.0], [1.0, 1.0], [0.0, 1.0], [0.0, 0.0] 668 | ]]] 669 | } 670 | } 671 | 672 | - do: 673 | indices.refresh: {} 674 | 675 | 676 | # Test that the shape has been fixed 677 | - do: 678 | search: 679 | body: 680 | query: 681 | term: 682 | id: 12 683 | 684 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "POLYGON ((1 0, 1 1, 0 1, 0 0, 1 0))" } 685 | 686 | 687 | # Test point deduplication on geometrycollection 688 | - do: 689 | index: 690 | index: test_index 691 | pipeline: "geo_extension" 692 | body: { 693 | "id": 13, 694 | "geo_shape_0": { 695 | "type": "GeometryCollection", 696 | "geometries": [ 697 | { 698 | "type": "Polygon", 699 | "coordinates": [ 700 | [ 701 | [ 702 | 4.921875, 703 | 46.07323062540835 704 | ], 705 | [ 706 | 10.8984375, 707 | 46.31658418182218 708 | ], 709 | [ 710 | 10.8984375, 711 | 46.31658418182218 712 | ], 713 | [ 714 | 8.7890625, 715 | 47.989921667414194 716 | ], 717 | [ 718 | 4.921875, 719 | 46.07323062540835 720 | ] 721 | ] 722 | ] 723 | }, 724 | { 725 | "type": "Point", 726 | "coordinates": [ 727 | 5.09765625, 728 | 47.66538735632654 729 | ] 730 | } 731 | ] 732 | }} 733 | 734 | - do: 735 | indices.refresh: {} 736 | 737 | 738 | # Test that the shape has been fixed 739 | - do: 740 | search: 741 | body: 742 | query: 743 | term: 744 | id: 13 745 | 746 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "GEOMETRYCOLLECTION (POLYGON ((10.8984375 46.31658418182218, 8.7890625 47.989921667414194, 4.921875 46.07323062540835, 10.8984375 46.31658418182218)), POINT (5.09765625 47.66538735632654))" } 747 | 748 | 749 | # Test polygon invalidity check 750 | # This polygon is valid for OGR (at least under a double floating point precision model) 751 | # But is not valid for ES 752 | - do: 753 | catch: /Self-intersection at or near/ 754 | index: 755 | index: test_index 756 | pipeline: "geo_extension" 757 | body: { 758 | "id": 14, 759 | "geo_shape_0": { 760 | "coordinates": [[[-1,0],[-1,1],[1,1],[1,0], 761 | [0.00000000000000004,0],[0.5,0.5], 762 | [-0.5,0.5],[0,0],[-1,0]]], 763 | "type":"Polygon" 764 | } 765 | } 766 | 767 | - match: {error.type: "mapper_parsing_exception"} 768 | 769 | # Test geometrycollection of multi 770 | - do: 771 | index: 772 | index: test_index 773 | pipeline: "geo_extension" 774 | body: { 775 | "id": 15, 776 | "geo_shape_0": { 777 | "type": "GeometryCollection", 778 | "geometries": [ 779 | { 780 | "type": "Point", 781 | "coordinates": [4.0, 46.0] 782 | },{ 783 | "type": "MultiPoint", 784 | "coordinates": [ 785 | [ 786 | 4.921875, 787 | 46.07323062540835 788 | ], 789 | [ 790 | 10.8984375, 791 | 46.31658418182218 792 | ] 793 | ] 794 | } 795 | ] 796 | }} 797 | - do: 798 | indices.refresh: {} 799 | 800 | 801 | # Test that the shape has been fixed 802 | - do: 803 | search: 804 | body: 805 | query: 806 | term: 807 | id: 15 808 | 809 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "GEOMETRYCOLLECTION(POINT (4 46),MULTIPOINT (4.921875 46.07323062540835, 10.8984375 46.31658418182218))" } 810 | 811 | # Test geometrycollection of geometrycollection 812 | - do: 813 | index: 814 | index: test_index 815 | pipeline: "geo_extension" 816 | body: { 817 | "id": 16, 818 | "geo_shape_0": { 819 | "type": "GeometryCollection", 820 | "geometries": [ 821 | { 822 | "type": "Point", 823 | "coordinates": [ 4.0, 46.0 ] 824 | },{ 825 | "type": "GeometryCollection", 826 | "geometries": [ 827 | { 828 | "type": "Point", 829 | "coordinates": [ 3.0, 45.0 ] 830 | }, 831 | { 832 | "type": "MultiPoint", 833 | "coordinates": [ 834 | [ 835 | 4.921875, 836 | 46.07323062540835 837 | ], 838 | [ 839 | 10.8984375, 840 | 46.31658418182218 841 | ] 842 | ] 843 | } 844 | ] 845 | } 846 | ] 847 | } 848 | } 849 | - do: 850 | indices.refresh: {} 851 | 852 | 853 | # Test that the shape has been fixed 854 | - do: 855 | search: 856 | body: 857 | query: 858 | term: 859 | id: 16 860 | 861 | - match: {hits.hits.0._source.geo_shape_0.fixed_shape: "GEOMETRYCOLLECTION(POINT (4 46),GEOMETRYCOLLECTION(POINT (3 45),MULTIPOINT (4.921875 46.07323062540835, 10.8984375 46.31658418182218)))" } 862 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/GeoExtension/30_simplify_script.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Test simplification script": 3 | 4 | # Create the pipeline, index and mapping 5 | - do: 6 | ingest.put_pipeline: 7 | id: "geo_extension" 8 | body: > 9 | { 10 | "description": "Add extra geo fields to geo_shape fields.", 11 | "processors": [ 12 | { 13 | "geo_extension": { 14 | "field": "geo_shape_*" 15 | } 16 | } 17 | ] 18 | } 19 | - match: { acknowledged: true } 20 | 21 | - do: 22 | ingest.get_pipeline: 23 | id: "geo_extension" 24 | - match: { geo_extension.description: "Add extra geo fields to geo_shape fields." } 25 | 26 | - do: 27 | indices.create: 28 | index: test_index 29 | 30 | - do: 31 | indices.put_mapping: 32 | index: test_index 33 | body: 34 | dynamic_templates: [ 35 | { 36 | "geo_shapes": { 37 | "match": "geo_shape_*", 38 | "mapping": { 39 | "properties": { 40 | "shape": {"enabled": false}, 41 | "fixed_shape": {"type": "geo_shape"}, 42 | "hash": {"type": "keyword"}, 43 | "wkb": {"type": "binary", "doc_values": true}, 44 | "type": {"type": "keyword"}, 45 | "area": {"type": "half_float"}, 46 | "bbox": {"type": "geo_point"}, 47 | "centroid": {"type": "geo_point"} 48 | } 49 | } 50 | } 51 | } 52 | ] 53 | 54 | # Add documents 55 | - do: 56 | index: 57 | index: test_index 58 | pipeline: "geo_extension" 59 | body: { 60 | "id": 1, 61 | "geo_shape_0": { 62 | "type": "Polygon", 63 | "coordinates": [ 64 | [ 65 | [ 66 | -3.3858489990234375, 67 | 47.7442871774986 68 | ], 69 | [ 70 | -3.3889389038085938, 71 | 47.73770713305151 72 | ], 73 | [ 74 | -3.3777809143066406, 75 | 47.738515253481545 76 | ], 77 | [ 78 | -3.3858489990234375, 79 | 47.7442871774986 80 | ] 81 | ] 82 | ] 83 | } 84 | } 85 | 86 | - do: 87 | index: 88 | index: test_index 89 | pipeline: "geo_extension" 90 | body: { 91 | "id": 2, 92 | "geo_shape_0": { 93 | "type": "LineString", 94 | "coordinates": [ 95 | [ 96 | -3.3805704116821285, 97 | 47.75757459952785 98 | ], 99 | [ 100 | -3.3811068534851074, 101 | 47.757711639949754 102 | ], 103 | [ 104 | -3.3817613124847408, 105 | 47.75771524627175 106 | ], 107 | [ 108 | -3.3826088905334473, 109 | 47.75768278936459 110 | ], 111 | [ 112 | -3.3831185102462764, 113 | 47.75758181219062 114 | ] 115 | ] 116 | } 117 | } 118 | 119 | - do: 120 | index: 121 | index: test_index 122 | pipeline: "geo_extension" 123 | body: { 124 | "id": 3, 125 | "name": "Le BHV Marais - Paris", 126 | "geo_shape_0": {"type": "Polygon", "coordinates": [[[2.3525621, 48.857395999727686], [2.3525681, 48.85737839972767], [2.3525818, 48.8573607997277], [2.3526161, 48.85734039972769], [2.3526682, 48.857330099727704], [2.3527051, 48.85733319972769], [2.3527095, 48.85732729972771], [2.3527088, 48.85732179972772], [2.3527346, 48.8573134997277], [2.3528807, 48.85728029972772], [2.3530401, 48.857241799727724], [2.3531794, 48.85720819972773], [2.3532811, 48.85718359972775], [2.3534759, 48.85713659972777], [2.3536211, 48.857101599727756], [2.3537783, 48.85706359972777], [2.3538754, 48.85709369972776], [2.3538906, 48.85712099972776], [2.3539385, 48.857206799727734], [2.3540005, 48.857317999727705], [2.3540525, 48.857411199727686], [2.3540721, 48.857446399727685], [2.3540668, 48.85744779972767], [2.3540707, 48.857472599727664], [2.3540622, 48.857497599727665], [2.3540541, 48.857508299727655], [2.3540239, 48.857528399727634], [2.3540001, 48.85753509972766], [2.3540023, 48.85753889972765], [2.3539471, 48.857558899727636], [2.3538824, 48.85758219972763], [2.3538218, 48.85760409972764], [2.3537725, 48.85762189972764], [2.3537123, 48.85764369972763], [2.3536506, 48.85766599972763], [2.3536151, 48.85767879972762], [2.3535832, 48.85769029972763], [2.3535222, 48.8577123997276], [2.3534533, 48.8577372997276], [2.3533762, 48.85776509972759], [2.3532997, 48.8577927997276], [2.353232, 48.85781719972759], [2.3531528, 48.857845799727585], [2.3531009, 48.85786459972756], [2.3530659, 48.857872399727576], [2.3530474, 48.85787309972757], [2.3530175, 48.85787069972757], [2.3529784, 48.857856999727574], [2.3529624, 48.85784619972758], [2.3529494, 48.857833299727574], [2.3529439, 48.857834699727576], [2.3529125, 48.857800499727595], [2.3528479, 48.8577299997276], [2.3527546, 48.85762839972763], [2.3526549, 48.857519699727646], [2.3526422, 48.85750589972766], [2.3526216, 48.85748359972768], [2.3526279, 48.85747989972768], [2.352624, 48.857477099727674], [2.3526301, 48.85747449972769], [2.3526191, 48.85746599972768], [2.3525904, 48.857452299727655], [2.3525766, 48.85744089972767], [2.352564, 48.85741979972767], [2.3525621, 48.857395999727686]]]} 127 | } 128 | 129 | - do: 130 | indices.refresh: {} 131 | 132 | # Test polygon simplification with douglas peucker 133 | # With zoom=1, polygon is reduced to a point 134 | - do: 135 | search: 136 | body: 137 | query: 138 | term: 139 | id: 1 140 | script_fields: 141 | simplified: 142 | script: 143 | source: geo_simplify 144 | lang: geo_extension_scripts 145 | params: 146 | field: "geo_shape_0.wkb" 147 | zoom: 1 148 | algorithm: DOUGLAS_PEUCKER 149 | 150 | - match: {hits.hits.0.fields.simplified.0.shape: "{\"type\":\"Point\",\"coordinates\":[-3.3889389,47.73770713],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}" } 151 | 152 | # With topology preservation 153 | - do: 154 | search: 155 | body: 156 | query: 157 | term: 158 | id: 1 159 | script_fields: 160 | simplified: 161 | script: 162 | source: geo_simplify 163 | lang: geo_extension_scripts 164 | params: 165 | field: "geo_shape_0.wkb" 166 | zoom: 1 167 | algorithm: TOPOLOGY_PRESERVING 168 | 169 | - match: {hits.hits.0.fields.simplified.0.shape: "{\"type\":\"Polygon\",\"coordinates\":[[[-3.3889389,47.73770713],[-3.37778091,47.73851525],[-3.385849,47.74428718],[-3.3889389,47.73770713]]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"} 170 | 171 | # simplification of a linestring 172 | - do: 173 | search: 174 | body: 175 | query: 176 | term: 177 | id: 2 # linestring 178 | script_fields: 179 | simplified: 180 | script: 181 | source: geo_simplify 182 | lang: geo_extension_scripts 183 | params: 184 | field: "geo_shape_0.wkb" 185 | zoom: 1 186 | algorithm: DOUGLAS_PEUCKER 187 | 188 | - match: {hits.hits.0.fields.simplified.0.shape: "{\"type\":\"LineString\",\"coordinates\":[[-3.38057041,47.7575746],[-3.38311851,47.75758181]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"} 189 | 190 | # test with high zoom and TOPOLOGY_PRESERVING (exact same test as 40_geoshape_aggregation.yml one!) 191 | - do: 192 | search: 193 | body: 194 | query: 195 | term: 196 | id: 3 197 | script_fields: 198 | simplified: 199 | script: 200 | source: geo_simplify 201 | lang: geo_extension_scripts 202 | params: 203 | field: "geo_shape_0.wkb" 204 | zoom: 20 205 | algorithm: TOPOLOGY_PRESERVING 206 | 207 | # coordinates length must be 37. it's not possible to test this length because the shape is dumped as a text, so we're testing the full shape length here instead 208 | - length: {hits.hits.0.fields.simplified.0.shape: 935} 209 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/GeoExtension/40_geoshape_aggregation.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Test aggregation": 3 | 4 | # Create the pipeline, index and mapping 5 | - do: 6 | ingest.put_pipeline: 7 | id: "geo_extension" 8 | body: > 9 | { 10 | "description": "Add extra geo fields to geo_shape fields.", 11 | "processors": [ 12 | { 13 | "geo_extension": { 14 | "field": "geo_shape_*" 15 | } 16 | } 17 | ] 18 | } 19 | - match: { acknowledged: true } 20 | 21 | - do: 22 | ingest.get_pipeline: 23 | id: "geo_extension" 24 | - match: { geo_extension.description: "Add extra geo fields to geo_shape fields." } 25 | 26 | - do: 27 | indices.create: 28 | index: test_index 29 | 30 | - do: 31 | indices.put_mapping: 32 | index: test_index 33 | body: 34 | dynamic_templates: [ 35 | { 36 | "geo_shapes": { 37 | "match": "geo_shape_*", 38 | "mapping": { 39 | "properties": { 40 | "shape": {"enabled": false}, 41 | "fixed_shape": {"type": "geo_shape"}, 42 | "hash": {"type": "keyword"}, 43 | "wkb": {"type": "binary", "doc_values": true}, 44 | "type": {"type": "keyword"}, 45 | "area": {"type": "half_float"}, 46 | "bbox": {"type": "geo_point"}, 47 | "centroid": {"type": "geo_point"} 48 | } 49 | } 50 | } 51 | } 52 | ] 53 | 54 | # Add documents 55 | - do: 56 | index: 57 | index: test_index 58 | pipeline: "geo_extension" 59 | body: { 60 | "id": 1, 61 | "geo_shape_0": { 62 | "type": "Polygon", 63 | "coordinates": [ 64 | [ 65 | [ 66 | -3.3858489990234375, 67 | 47.7442871774986 68 | ], 69 | [ 70 | -3.3889389038085938, 71 | 47.73770713305151 72 | ], 73 | [ 74 | -3.3777809143066406, 75 | 47.738515253481545 76 | ], 77 | [ 78 | -3.3858489990234375, 79 | 47.7442871774986 80 | ] 81 | ] 82 | ] 83 | } 84 | } 85 | 86 | - do: 87 | index: 88 | index: test_index 89 | pipeline: "geo_extension" 90 | body: { 91 | "id": 2, 92 | "geo_shape_0": { 93 | "type": "LineString", 94 | "coordinates": [ 95 | [ 96 | -3.3805704116821285, 97 | 47.75757459952785 98 | ], 99 | [ 100 | -3.3811068534851074, 101 | 47.757711639949754 102 | ], 103 | [ 104 | -3.3817613124847408, 105 | 47.75771524627175 106 | ], 107 | [ 108 | -3.3826088905334473, 109 | 47.75768278936459 110 | ], 111 | [ 112 | -3.3831185102462764, 113 | 47.75758181219062 114 | ] 115 | ] 116 | } 117 | } 118 | 119 | - do: 120 | index: 121 | index: test_index 122 | pipeline: "geo_extension" 123 | body: { 124 | "id": 3, 125 | "name": "Le BHV Marais - Paris", 126 | "geo_shape_0": {"type": "Polygon", "coordinates": [[[2.3525621, 48.857395999727686], [2.3525681, 48.85737839972767], [2.3525818, 48.8573607997277], [2.3526161, 48.85734039972769], [2.3526682, 48.857330099727704], [2.3527051, 48.85733319972769], [2.3527095, 48.85732729972771], [2.3527088, 48.85732179972772], [2.3527346, 48.8573134997277], [2.3528807, 48.85728029972772], [2.3530401, 48.857241799727724], [2.3531794, 48.85720819972773], [2.3532811, 48.85718359972775], [2.3534759, 48.85713659972777], [2.3536211, 48.857101599727756], [2.3537783, 48.85706359972777], [2.3538754, 48.85709369972776], [2.3538906, 48.85712099972776], [2.3539385, 48.857206799727734], [2.3540005, 48.857317999727705], [2.3540525, 48.857411199727686], [2.3540721, 48.857446399727685], [2.3540668, 48.85744779972767], [2.3540707, 48.857472599727664], [2.3540622, 48.857497599727665], [2.3540541, 48.857508299727655], [2.3540239, 48.857528399727634], [2.3540001, 48.85753509972766], [2.3540023, 48.85753889972765], [2.3539471, 48.857558899727636], [2.3538824, 48.85758219972763], [2.3538218, 48.85760409972764], [2.3537725, 48.85762189972764], [2.3537123, 48.85764369972763], [2.3536506, 48.85766599972763], [2.3536151, 48.85767879972762], [2.3535832, 48.85769029972763], [2.3535222, 48.8577123997276], [2.3534533, 48.8577372997276], [2.3533762, 48.85776509972759], [2.3532997, 48.8577927997276], [2.353232, 48.85781719972759], [2.3531528, 48.857845799727585], [2.3531009, 48.85786459972756], [2.3530659, 48.857872399727576], [2.3530474, 48.85787309972757], [2.3530175, 48.85787069972757], [2.3529784, 48.857856999727574], [2.3529624, 48.85784619972758], [2.3529494, 48.857833299727574], [2.3529439, 48.857834699727576], [2.3529125, 48.857800499727595], [2.3528479, 48.8577299997276], [2.3527546, 48.85762839972763], [2.3526549, 48.857519699727646], [2.3526422, 48.85750589972766], [2.3526216, 48.85748359972768], [2.3526279, 48.85747989972768], [2.352624, 48.857477099727674], [2.3526301, 48.85747449972769], [2.3526191, 48.85746599972768], [2.3525904, 48.857452299727655], [2.3525766, 48.85744089972767], [2.352564, 48.85741979972767], [2.3525621, 48.857395999727686]]]} 127 | } 128 | 129 | - do: 130 | indices.refresh: {} 131 | 132 | # Test polygon simplification with douglas peucker 133 | # With zoom=1, polygon is reduced to a point 134 | - do: 135 | search: 136 | body: 137 | query: 138 | term: 139 | id: 1 140 | size: 0 141 | aggs: 142 | g: 143 | geoshape: 144 | field: "geo_shape_0.wkb" 145 | output_format: geojson 146 | simplify: 147 | zoom: 1 148 | algorithm: DOUGLAS_PEUCKER 149 | size: 4 150 | 151 | - match: {aggregations.g.buckets.0.key: "{\"type\":\"Point\",\"coordinates\":[-3.3889389,47.73770713],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}" } 152 | 153 | # With topology preservation 154 | - do: 155 | search: 156 | body: 157 | query: 158 | term: 159 | id: 1 160 | size: 0 161 | aggs: 162 | g: 163 | geoshape: 164 | field: "geo_shape_0.wkb" 165 | output_format: geojson 166 | simplify: 167 | zoom: 1 168 | algorithm: TOPOLOGY_PRESERVING 169 | size: 4 170 | 171 | - match: {aggregations.g.buckets.0.key: "{\"type\":\"Polygon\",\"coordinates\":[[[-3.3889389,47.73770713],[-3.37778091,47.73851525],[-3.385849,47.74428718],[-3.3889389,47.73770713]]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"} 172 | 173 | # Test size restriction will return the largest shape 174 | - do: 175 | search: 176 | body: 177 | size: 0 178 | aggs: 179 | g: 180 | geoshape: 181 | field: "geo_shape_0.wkb" 182 | output_format: geojson 183 | simplify: 184 | zoom: 1 185 | algorithm: TOPOLOGY_PRESERVING 186 | size: 1 # size restriction here 187 | 188 | - match: {aggregations.g.buckets.0.key: "{\"type\":\"Polygon\",\"coordinates\":[[[-3.3889389,47.73770713],[-3.37778091,47.73851525],[-3.385849,47.74428718],[-3.3889389,47.73770713]]],\"crs\":{\"type\":\"name\",\"properties\":{\"name\":\"EPSG:0\"}}}"} 189 | 190 | # test with high zoom and TOPOLOGY_PRESERVING (exact same test as 30_simplify_script.yml one!) 191 | - do: 192 | search: 193 | body: 194 | size: 0 195 | aggs: 196 | g: 197 | geoshape: 198 | field: "geo_shape_0.wkb" 199 | output_format: geojson 200 | simplify: 201 | zoom: 20 202 | algorithm: TOPOLOGY_PRESERVING 203 | 204 | # coordinates length must be 37. it's not possible to test this length because the shape is dumped as a text, so we're testing the full shape length here instead 205 | - length: {aggregations.g.buckets.1.key: 935} 206 | -------------------------------------------------------------------------------- /src/yamlRestTest/resources/rest-api-spec/test/GeoExtension/50_invalid_polygon.yml: -------------------------------------------------------------------------------- 1 | --- 2 | "Test aggregation": 3 | 4 | # Create the pipeline, index and mapping 5 | - do: 6 | ingest.put_pipeline: 7 | id: "geo_extension" 8 | body: > 9 | { 10 | "description": "Add extra geo fields to geo_shape fields.", 11 | "processors": [ 12 | { 13 | "geo_extension": { 14 | "field": "geo_shape_*" 15 | } 16 | } 17 | ] 18 | } 19 | - match: { acknowledged: true } 20 | 21 | - do: 22 | ingest.get_pipeline: 23 | id: "geo_extension" 24 | - match: { geo_extension.description: "Add extra geo fields to geo_shape fields." } 25 | 26 | - do: 27 | indices.create: 28 | index: test_index 29 | 30 | - do: 31 | indices.put_mapping: 32 | index: test_index 33 | body: 34 | dynamic_templates: [ 35 | { 36 | "geo_shapes": { 37 | "match": "geo_shape_*", 38 | "mapping": { 39 | "properties": { 40 | "shape": {"enabled": false}, 41 | "fixed_shape": {"type": "geo_shape"}, 42 | "hash": {"type": "keyword"}, 43 | "wkb": {"type": "binary", "doc_values": true}, 44 | "type": {"type": "keyword"}, 45 | "area": {"type": "half_float"}, 46 | "bbox": {"type": "geo_point"}, 47 | "centroid": {"type": "geo_point"} 48 | } 49 | } 50 | } 51 | } 52 | ] 53 | 54 | # Add documents 55 | # An invalid Polygon for Elastic 7.17.6 and Lucene 8.11.3 BUT valid for Lucene 8.11.3 56 | - do: 57 | index: 58 | index: test_index 59 | pipeline: "geo_extension" 60 | body: { 61 | "id": 1, 62 | "name": "invalid", 63 | "geo_shape_0": "POLYGON ((-88.3245325358123 41.9306419084828,-88.3243288475156 41.9308130944597,-88.3244513948451 41.930891654082,-88.3246174067624 41.930998076295,-88.3245448815692 41.9310557712027,-88.3239353718069 41.9313272600886,-88.3237355617867 41.9313362704162,-88.3237347670323 41.9311150951881,-88.3237340649402 41.931103661118,-88.3235660813522 41.9311112432041,-88.3234509652339 41.9311164377155,-88.3232353124097 41.9311261692953,-88.3232343331295 41.9313588701899,-88.323028772523 41.9313681383084,-88.3229999744274 41.930651995613,-88.3236147717043 41.9303655647412,-88.323780013667 41.929458561339,-88.3240657895016 41.9293998882959,-88.3243948640426 41.9293028003164,-88.324740490767 41.9301340399879,-88.3251305560187 41.9302766363048,-88.3248260581475 41.9308286995884,-88.3246595186817 41.9307227160738,-88.3245325358123 41.9306419084828),(-88.3245658060855 41.930351580587,-88.3246004191532 41.9302095159456,-88.3246375011905 41.9300573183932,-88.3243392233337 41.9300159738164,-88.3243011787553 41.9301696594472,-88.3242661951392 41.9303109843373,-88.3245658060855 41.930351580587),(-88.3245325358123 41.9306419084828,-88.3245478066552 41.9305086556331,-88.3245658060855 41.930351580587,-88.3242368660096 41.9303327977821,-88.3242200926128 41.9304905242189,-88.324206161464 41.9306215207536,-88.3245325358123 41.9306419084828),(-88.3236767661893 41.9307089429871,-88.3237008716322 41.930748885445,-88.323876104365 41.9306891087739,-88.324063438129 41.9306252050871,-88.3239244290607 41.930399373909,-88.3237349076233 41.9304653056436,-88.3235653339759 41.9305242981369,-88.3236767661893 41.9307089429871))" 64 | } 65 | --------------------------------------------------------------------------------