├── .gitignore ├── README.md ├── core-signatures.txt ├── pom.xml └── src ├── main ├── assemblies │ └── plugin.xml ├── java │ └── de │ │ └── spinscale │ │ └── elasticsearch │ │ ├── action │ │ └── suggest │ │ │ ├── refresh │ │ │ ├── ShardSuggestRefreshRequest.java │ │ │ ├── ShardSuggestRefreshResponse.java │ │ │ ├── SuggestRefreshAction.java │ │ │ ├── SuggestRefreshRequest.java │ │ │ ├── SuggestRefreshResponse.java │ │ │ └── TransportSuggestRefreshAction.java │ │ │ ├── statistics │ │ │ ├── FstStats.java │ │ │ ├── ShardSuggestStatisticsRequest.java │ │ │ ├── ShardSuggestStatisticsResponse.java │ │ │ ├── SuggestStatisticsAction.java │ │ │ ├── SuggestStatisticsRequest.java │ │ │ ├── SuggestStatisticsResponse.java │ │ │ └── TransportSuggestStatisticsAction.java │ │ │ └── suggest │ │ │ ├── ShardSuggestRequest.java │ │ │ ├── ShardSuggestResponse.java │ │ │ ├── SuggestAction.java │ │ │ ├── SuggestRequest.java │ │ │ ├── SuggestResponse.java │ │ │ └── TransportSuggestAction.java │ │ ├── client │ │ └── action │ │ │ └── suggest │ │ │ ├── SuggestRefreshRequestBuilder.java │ │ │ ├── SuggestRequestBuilder.java │ │ │ └── SuggestStatisticsRequestBuilder.java │ │ ├── module │ │ └── suggest │ │ │ └── ShardSuggestModule.java │ │ ├── plugin │ │ └── suggest │ │ │ └── SuggestPlugin.java │ │ ├── rest │ │ └── action │ │ │ └── suggest │ │ │ ├── RestRefreshSuggestAction.java │ │ │ ├── RestStatisticsAction.java │ │ │ └── RestSuggestAction.java │ │ └── service │ │ └── suggest │ │ ├── AbstractCacheLoaderSuggester.java │ │ ├── ShardSuggestService.java │ │ └── SuggestService.java └── resources │ └── es-plugin.properties └── test ├── java └── de │ └── spinscale │ └── elasticsearch │ └── module │ └── suggest │ └── test │ ├── AbstractSuggestTest.java │ ├── RestSuggestActionTest.java │ ├── SuggestBuildersTest.java │ ├── SuggestionQuery.java │ ├── TransportClientTest.java │ └── TransportSuggestActionTest.java └── resources ├── log4j.properties └── product.json /.gitignore: -------------------------------------------------------------------------------- 1 | .project 2 | .settings 3 | .classpath 4 | .gradle 5 | data 6 | build 7 | target 8 | *.swp 9 | *.iml 10 | .idea 11 | .local-execution-hints.log 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DO NOT USE THIS PLUGIN ANYMORE 2 | 3 | This plugin has been superceded by the [completion suggester](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters-completion.html) in Elasticsearch and is not developed further. There is an excellent [introductory blog post](https://www.elastic.co/blog/you-complete-me) available as well. 4 | 5 | **This plugin is not developed further than for Elasticsearch 1.3, which you should not use anymore!** 6 | 7 | # Suggester Plugin for Elasticsearch 8 | 9 | **Note**: If you only need prefix suggestions, please use the new `completion suggest` feature available since elasticsearch 0.90.3, which features blazing fast real time suggestions, uses the `AnalyzingSuggester` under the hood and will also support fuzzy mode in 0.90.4. 10 | 11 | This plugin uses the FSTSuggester, the AnalyzingSuggester or the FuzzySuggester from Lucene to create suggestions from a certain field for a specified term instead of returning the whole document data. 12 | 13 | Feel free to comment, improve and help - I am thankful for any insights, no matter whether you want to help with elasticsearch, lucene or my other flaws I will have done for sure. 14 | 15 | Oh and in case you have not read it above: 16 | 17 | In case you want to contact me, drop me a mail at alexander@reelsen.net 18 | 19 | ## Breaking changes for elasticsearch 0.90 20 | 21 | Because elasticsearch now comes with its own suggest API (not based on in-memory automatons per shard), big parts of this plugin needs to be changed. 22 | 23 | ### REST endpoints have been moved 24 | 25 | Both REST endpoints have been moved. The `/_suggest` endpoint now resides at `__suggest`. Refreshing has changed from `_suggestRefresh` to `__suggestRefresh`. 26 | I do not like this renaming either, but I have not yet got the ieda of a better name. 27 | I am totally open for better names. This is a WIP until elasticsearch 1.0 is released. 28 | 29 | ### Package names have been moved 30 | 31 | Everything is now in the `de.spinscale` package name space in order to avoid clashes. This means, if you are using the request builder classes, you will have to change your application. 32 | 33 | ## Installation 34 | 35 | If you do not want to work on the repository, just use the standard elasticsearch plugin command (inside your elasticsearch/bin directory) 36 | 37 | ``` 38 | bin/plugin -install de.spinscale/elasticsearch-plugin-suggest/0.90.5-0.9 39 | ``` 40 | 41 | ### Compatibility 42 | 43 | 44 | **Note**: Please make sure the plugin version matches with your elasticsearch version. Follow this compatibility matrix 45 | 46 | ---------------------------------------- 47 | | suggest plugin | Elasticsearch | 48 | ---------------------------------------- 49 | | 1.3.2-2.0.1 | 1.3.2 -> master | 50 | ---------------------------------------- 51 | | 1.0.1-2.0.0 | 1.0.1 | 52 | ---------------------------------------- 53 | | 0.90.12-1.1 | 0.90.12 | 54 | ---------------------------------------- 55 | | 0.90.7-1.0 | 0.90.7 | 56 | ---------------------------------------- 57 | | 0.90.5-0.9 | 0.90.5 | 58 | ---------------------------------------- 59 | | 0.90.3-0.8.* | 0.90.3 | 60 | ---------------------------------------- 61 | | 0.90.1-0.7 | 0.90.1 | 62 | ---------------------------------------- 63 | | 0.90.0-0.6.* | 0.90.0 | 64 | ---------------------------------------- 65 | | 0.20.5-0.5 | 0.20.5 -> 0.20.6 | 66 | ---------------------------------------- 67 | | 0.20.2-0.4 | 0.20.2 -> 0.20.4 | 68 | ---------------------------------------- 69 | | 0.19.12-0.2 | 0.19.12 | 70 | ---------------------------------------- 71 | | 0.19.11-0.1 | 0.19.11 | 72 | ---------------------------------------- 73 | 74 | 75 | ### Development 76 | 77 | If you want to work on the repository 78 | 79 | * Clone this repo with `git clone git://github.com/spinscale/elasticsearch-suggest-plugin.git` 80 | * Checkout the tag (find out via `git tag`) you want to build with (possibly master is not for your elasticsearch version) 81 | * Run: `mvn clean package -DskipTests=true` - this does not run any unit tests, as they take some time. If you want to run them, better run `mvn clean package` 82 | * Install the plugin: `/path/to/elasticsearch/bin/plugin -install elasticsearch-suggest -url file:///$PWD/target/releases/elasticsearch-suggest-$version.zip` 83 | 84 | Alternatively you can now use this plugin via maven and include it via the sonatype repo likes this in your pom.xml (or any other dependency manager) 85 | 86 | ``` 87 | 88 | 89 | Sonatype 90 | Sonatype 91 | http://oss.sonatype.org/content/repositories/releases/ 92 | 93 | 94 | 95 | 96 | 97 | de.spinscale 98 | elasticsearch-suggest-plugin 99 | 1.3.2-2.0.1 100 | 101 | ... 102 | 103 | ``` 104 | 105 | The maven repo can be visited at https://oss.sonatype.org/content/repositories/releases/de/spinscale/elasticsearch-plugin-suggest/ 106 | 107 | ## Usage 108 | 109 | ### FST based suggestions 110 | 111 | Fire up curl like this, in case you have a products index and the right fields - if not, read below how to setup a clean elasticsearch in order to support suggestions. 112 | 113 | ``` 114 | # curl -X POST 'localhost:9200/products1/product/__suggest?pretty=1' -d '{ "field": "ProductName.suggest", "term": "tischwäsche", "size": "10" }' 115 | { 116 | "suggest" : [ "tischwäsche", "tischwäsche 100", 117 | "tischwäsche aberdeen", "tischwäsche acryl", "tischwäsche ambiente", 118 | "tischwäsche aquarius", "tischwäsche atlanta", "tischwäsche atlas", 119 | "tischwäsche augsburg", "tischwäsche aus", "tischwäsche austria" ] 120 | } 121 | ``` 122 | 123 | As you can see, this queries the products index for the field `ProductName.suggest` with the specified term and size. 124 | 125 | You can also use HTTP GET for getting suggestions - even with the `callback` and the `source` parameters like in any normal elasticsearch search. 126 | 127 | You might want to check out the included unit test as well. I use a shingle filter in my examples, take a look at the files in `src/test/resources` directory. 128 | 129 | ### Full suggestions 130 | 131 | With Lucene 4 (and the upgrade to elasticsearch 0.90.0) two new suggesters were added, one of them the [AnalyzingSuggester](http://lucene.apache.org/core/4_3_0/suggest/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.html) and the [FuzzySuggester](http://lucene.apache.org/core/4_3_0/suggest/org/apache/lucene/search/suggest/analyzing/FuzzySuggester.html) based on the first one. Both have the great capability of returning the original form, but search on an analyzed one. Take this example (notice the search for a lowercase `b`, but getting back the original field name): 132 | 133 | ``` 134 | » curl -X POST localhost:9200/cars/car/__suggest -d '{ "field" : "name", "type": "full", "term" : "b", "analyzer" : "standard" }' 135 | 136 | {"suggestions":["BMW 320","BMW 525d"],"_shards":{"total":5,"successful":5,"failed":0}} 137 | ``` 138 | 139 | *Note*: If you use type `full` or type `fuzzy`, the `similarity` parameter will not have any effect. In addition, these parameters are supported only for `full` and `fuzzy`: 140 | 141 | * `analyzer`: 142 | * `index_analyzer`: 143 | * `search_analyzer`: 144 | 145 | This suggester can even ignore stopwords if configured appropriately - but only if you disable position increments for stopwords. Use this mapping and index settings when creating an index: 146 | 147 | ``` 148 | curl -X DELETE localhost:9200/cars 149 | curl -X PUT localhost:9200/cars -d '{ 150 | "mappings" : { 151 | "car" : { 152 | "properties" : { 153 | "name" : { 154 | "type" : "multi_field", 155 | "fields" : { 156 | "name": { "type": "string", "index": "not_analyzed" } 157 | } 158 | } 159 | } 160 | } 161 | }, 162 | "settings" : { 163 | "analysis" : { 164 | "analyzer" : { 165 | "suggest_analyzer_stopwords" : { 166 | "type" : "custom", 167 | "tokenizer" : "standard", 168 | "filter" : [ "standard", "lowercase", "stopword_no_position_increment" ] 169 | }, 170 | "suggest_analyzer_synonyms" : { 171 | "type" : "custom", 172 | "tokenizer" : "standard", 173 | "filter" : [ "standard", "lowercase", "my_synonyms" ] 174 | } 175 | }, 176 | "filter" : { 177 | "stopword_no_position_increment" : { 178 | "type" : "stop", 179 | "enable_position_increments" : false 180 | }, 181 | "my_synonyms" : { 182 | "type" : "synonym", 183 | "synonyms" : [ "jetta, bora" ] 184 | } 185 | } 186 | } 187 | } 188 | }' 189 | 190 | 191 | curl -X POST localhost:9200/cars/car -d '{ "name" : "The BMW ever" }' 192 | curl -X POST localhost:9200/cars/car -d '{ "name" : "BMW 320" }' 193 | curl -X POST localhost:9200/cars/car -d '{ "name" : "BMW 525d" }' 194 | curl -X POST localhost:9200/cars/car -d '{ "name" : "VW Jetta" }' 195 | curl -X POST localhost:9200/cars/car -d '{ "name" : "VW Bora" }' 196 | ``` 197 | 198 | Now when querying with a stopwords analyzer, you can even get back `The BMW ever` 199 | 200 | ``` 201 | » curl -X POST localhost:9200/cars/car/__suggest -d '{ "field" : "name", "type": "full", "term" : "b", "analyzer" : "suggest_analyzer_stopwords" }' 202 | {"suggestions":["BMW 320","BMW 525d","The BMW ever"],"_shards":{"total":5,"successful":5,"failed":0}} 203 | ``` 204 | 205 | Or you could use synonyms (FYI: jetta and bora were the same cars, but named different in USA and Europe, so a search should return both) 206 | 207 | ``` 208 | » curl -X POST localhost:9200/cars/car/__suggest -d '{ "field" : "name", "type": "full", "term" : "vw je", "analyzer" : "suggest_analyzer_synonyms" }' 209 | 210 | {"suggestions":["VW Bora","VW Jetta"],"_shards":{"total":5,"successful":5,"failed":0}} 211 | ``` 212 | 213 | ### Full fuzzy suggestions 214 | 215 | The FuzzySuggester uses LevenShtein distance to cater for typos. 216 | 217 | ``` 218 | » curl -X POST localhost:9200/cars/car/__suggest -d '{ "field" : "name", "type": "fuzzy", "term" : "bwm", "analyzer" : "standard" }' 219 | 220 | {"suggestions":["BMW 320","BMW 525d"],"_shards":{"total":5,"successful":5,"failed":0}} 221 | ``` 222 | 223 | ### Statistics 224 | 225 | The `FuzzySuggester` and the `AnalyzingSuggester` suggesters contain a method to find out their size, which is also exposed as an own endpoint, in case you want to monitor memory consumption of the in-memory structures. 226 | 227 | ``` 228 | » curl localhost:9200/__suggestStatistics 229 | {"_shards":{"total":2,"successful":2,"failed":0},"fstStats":{"cars-0":[{"analyzingsuggester-name-queryAnalyzer:suggest_analyzer_synonyms-indexAnalyzer:suggest_analyzer_synonyms":147},{"analyzingsuggester-name-queryAnalyzer:suggest_analyzer_stopwords-indexAnalyzer:suggest_analyzer_stopwords":126}]}} 230 | ``` 231 | 232 | ### Configuration 233 | 234 | Furthermore the suggest data is not updated, whenever you index a new product but every few minutes. The default is to update the index every 10 minutes, but you can change that in your elasticsearch.yml configuration: 235 | 236 | ``` 237 | suggest: 238 | refresh_interval: 600s 239 | ``` 240 | 241 | In this case the suggest indexes are refreshed every 10 minutes. This is also the default. You can use values like "10s", "10ms" or "10m" as with most other time based configuration settings in elasticsearch. 242 | 243 | If you want to deactivate automatic refresh completely, put this in your elasticsearch configuration 244 | 245 | ``` 246 | suggest: 247 | refresh_disabled: true 248 | ``` 249 | 250 | If you want to refresh your FST suggesters manually instead of waiting for 10 minutes just issue a POST request to the `/__suggestRefresh` URL. 251 | 252 | ``` 253 | # curl -X POST 'localhost:9200/__suggestRefresh' 254 | # curl -X POST 'localhost:9200/products/product/__suggestRefresh' 255 | # curl -X POST 'localhost:9200/products/product/__suggestRefresh' -d '{ "field" : "ProductName.suggest" }' 256 | ``` 257 | 258 | ## Usage from Java 259 | 260 | ``` 261 | SuggestRequest request = new SuggestRequest(index); 262 | request.term(term); 263 | request.field(field); 264 | request.size(size); 265 | request.similarity(similarity); 266 | 267 | SuggestResponse response = node.client().execute(SuggestAction.INSTANCE, request).actionGet(); 268 | ``` 269 | 270 | Refresh works like this - you can add an index and a field in the suggest refresh request as well, if you want to trigger it externally: 271 | 272 | ``` 273 | SuggestRefreshRequest refreshRequest = new SuggestRefreshRequest(); 274 | SuggestRefreshResponse response = node.client().execute(SuggestRefreshAction.INSTANCE, refreshRequest).actionGet(); 275 | ``` 276 | 277 | You can also use the included builders 278 | 279 | ``` 280 | List suggestions = new SuggestRequestBuilder(client) 281 | .field(field) 282 | .term(term) 283 | .size(size) 284 | .similarity(similarity) 285 | .execute().actionGet().suggestions(); 286 | ``` 287 | 288 | ``` 289 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(client); 290 | builder.execute().actionGet(); 291 | ``` 292 | 293 | ## Thanks 294 | 295 | * Shay ([@kimchy](http://twitter.com/kimchy)) for giving feedback 296 | * David ([@dadoonet](http://twitter.com/dadoonet)) for pushing me to get it into the maven repo 297 | * Adrien ([@jpountz](http://twitter.com/jpountz)) for helping me to understand the the AnalyzingSuggester details and having the idea for only creating the FST on the primary shard 298 | 299 | ## TODO 300 | 301 | * Find and verify the absence of the current resource leak (open deleted files after lots of merging) with the new architecture 302 | * Create the FST structure only on the primary shard and send it to the replica over the wire as byte array 303 | * Allow deletion of of fields in cache instead of refresh 304 | * Reenable the field refresh tests by checking statistics 305 | * Also expose the guava cache statistics in the endpoint 306 | * Create a testing rule that does start a node/cluster only once per test run, not per test. This costs so much time. 307 | 308 | ## Changelog 309 | 310 | * 2014-09-03: Version bump to 1.3.2, also created a 1.0, 1.1 and 1.2 branch 311 | * 2014-03-01: Version bump to 1.0.1, created a 0.90 branch 312 | * 2014-03-01: Version bump to 0.90.12, switched to randomized elasticsearch testing resulting in testing code cleanups and waaaaaaaay faster tests 313 | * 2013-12-10: Version bump to 0.90.7 314 | * 2013-08-13: Version bump to 0.90.3. Due to changes in Lucene 4.4, please check the tests to see that stopwords need to be handled on the request side if you use the fuzzy or full mode. 315 | * 2013-05-31: Removing usage of jdk7 only methods, version bump to 0.90.1 316 | * 2013-05-25: Changing suggest statistics format, fixing cache loading bug for analyzing/fuzzysuggester 317 | * 2013-05-12: Fix for trying to access a closed index reader in AnalyzingSuggesster (i.e. after refresh) 318 | * 2013-05-12: Documentation update 319 | * 2013-05-01: Added support for the fuzzy suggester 320 | * 2013-04-28: Added support for the analyzing suggester and stopwords 321 | * 2013-03-20: Moved to own package namespaces, changed REST endpoints in order to be compatible with elasticsearch 0.90 322 | * 2013-01-18: Support for HTTP GET, together with JSONP and the source parameter. 323 | * 2012-10-21: The REST urls can now be used without specifiying a type (which is unused at the moment anyway). You can use now the `$index/_suggest` and `$index/_suggestRefresh` urls 324 | * 2012-10-21: Allowing to set `suggest.refresh_disabled = true` in order to deactivate automatic refreshing of the suggest index 325 | * 2012-10-06: Shutting down the shard suggest service clean in case the instance is stopped or a shard is moved 326 | * 2012-10-03: Starting cluster nodes in parallel in tests where several nodes are created (big speedup) 327 | * 2012-10-03: Added tests for refreshing suggest in memory structures for one index or one field in an index only 328 | * 2012-10-03: Replaced gradle with maven 329 | * 2012-10-03: Updated to elasticsearch 0.19.10 330 | * 2012-10-03: You can use the plugin now with a TransportClient for the first time. Yay! 331 | * 2012-10-03: Using the FSTCompletionLookup now instead of the deprecated FSTLookup 332 | * 2012-10-03: Pretty much a core rewrite today (having tests is great, even if they run 10 minutes). The suggest service is now implemented as service on shard level - no more central Suggester structures. The whole implementation is much cleaner and adheres way better to the whole elasticsearch architecture instead of being cowboy coded together - at least that is what I think. 333 | * 2012-09-30: Updated to elasticsearch 0.19.9. Making TransportClients work again not spitting an exception on startup, when the module is in classpath. Updated this docs. 334 | * 2012-06-25: Trying to fix another resource leak, which did not eat up diskspace but still did not close all files 335 | * 2012-06-11: Fixing bad resoure leak due to not closing index reader properly - this lead to lots of deleted files, which still had open handles, thus taking up space 336 | * 2012-05-13: Updated to work with elasticsearch 0.19.3 337 | * 2012-03-07: Updated to work with elasticsearch 0.19.0 338 | * 2012-02-10: Created `SuggestRequestBuilder` and `SuggestRefreshRequestBuilder` classes - results in easy to use request classes (check the examples and tests) 339 | * 2011-12-29: The refresh interval can now be chosen as time based value like any other elasticsearch configuration 340 | * 2011-12-29: Instead of having all nodes sleeping the same time and updating the suggester asynchronously, the master node now triggers the update for all slaves 341 | * 2011-12-20: Added transport action (and REST action) to trigger reloading of all FST suggesters 342 | * 2011-12-11: Fixed the biggest issue: Searchers are released now and do not leak 343 | * 2011-12-11: Indexing is now done periodically 344 | * 2011-12-11: Found a way to get the injector from the node, so I can build my tests without using HTTP requests 345 | 346 | # HOWTO - the long version 347 | 348 | This HOWTO will help you to setup a clean elasticsearch installation with the correct index settings and mappings, so you can use the plugin as easy as possible. 349 | We will setup elasticsearch, index some products and query those for suggestions. 350 | 351 | Get elasticsearch, install it, get this plugin, install it. 352 | 353 | Add a suggest and a lowercase analyzer to your `elasticsearch/config/elasticsearch.yml` config file (or do it on index creation whatever you like) 354 | 355 | ``` 356 | index: 357 | analysis: 358 | analyzer: 359 | lowercase_analyzer: 360 | type: custom 361 | tokenizer: standard 362 | filter: [standard, lowercase] 363 | suggest_analyzer: 364 | type: custom 365 | tokenizer: standard 366 | filter: [standard, lowercase, shingle] 367 | ``` 368 | 369 | 370 | Start elasticsearch and create a mapping. You can either create it via configuration in a file or during index creation. We will create an index with a mapping now 371 | 372 | ``` 373 | curl -X PUT localhost:9200/products -d '{ 374 | "mappings" : { 375 | "product" : { 376 | "properties" : { 377 | "ProductId": { "type": "string", "index": "not_analyzed" }, 378 | "ProductName" : { 379 | "type" : "multi_field", 380 | "fields" : { 381 | "ProductName": { "type": "string", "index": "not_analyzed" }, 382 | "lowercase": { "type": "string", "analyzer": "lowercase_analyzer" }, 383 | "suggest" : { "type": "string", "analyzer": "suggest_analyzer" } 384 | } 385 | } 386 | } 387 | } 388 | } 389 | }' 390 | ``` 391 | 392 | Lets add some products 393 | 394 | ``` 395 | for i in 1 2 3 4 5 6 7 8 9 10 100 101 1000; do 396 | json=$(printf '{"ProductId": "%s", "ProductName": "%s" }', $i, "My Product $i") 397 | curl -X PUT localhost:9200/products/product/$i -d "$json" 398 | done 399 | ``` 400 | 401 | ## Queries 402 | 403 | Time to query and understand the different analyzers 404 | 405 | Queries the not analyzed field, returns 10 matches (default), always the full product name: 406 | 407 | ``` 408 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName", "term": "My" }' 409 | ``` 410 | 411 | Queries the not analyzed field, returns nothing (because lowercase): 412 | 413 | ``` 414 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName", "term": "my" }' 415 | ``` 416 | 417 | Queries the lowercase field, returns only the occuring word (which is pretty bad for suggests): 418 | 419 | ``` 420 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": 421 | "ProductName.lowercase", "term": "m" }' 422 | ``` 423 | 424 | Queries the suggest field, returns two words (this is the default length of the shingle filter), in this case "my" and "my product" 425 | 426 | ``` 427 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName.suggest", "term": "my" }' 428 | ``` 429 | 430 | Queries the suggest field, returns ten product names as we started with the second word + another one due to the shingle 431 | 432 | ``` 433 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName.suggest", "term": "product" }' 434 | ``` 435 | 436 | Queries the suggest field, returns all products with "product 1" in the shingle 437 | 438 | ``` 439 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName.suggest", "term": "product 1" }' 440 | ``` 441 | 442 | The same query as above, but limits the result set to two 443 | 444 | ``` 445 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName.suggest", "term": "product 1", "size": 2 }' 446 | ``` 447 | 448 | And last but not least, typo finding, the query without similarity parameter set returns nothing: 449 | 450 | ``` 451 | curl -X POST localhost:9200/products/product/_suggest -d '{ "field": "ProductName.suggest", "term": "proudct", similarity: 0.7 }' 452 | ``` 453 | 454 | The similarity is a float between 0.0 and 1.0 - if it is not specified 1.0 is used, which means it must match exactly. I've found 0.7 ok for cases, when two letters were exchanged, but mileage may very as I tested merely on german product names. 455 | 456 | With the tests I did, a shingle filter held the best results. Please check http://www.elasticsearch.org/guide/reference/index-modules/analysis/shingle-tokenfilter.html for more information about setup, like the default tokenization of two terms. 457 | 458 | Now test with your data, come up and improve this configuration. I am happy to hear about your specific configuration for successful suggestion queries. 459 | -------------------------------------------------------------------------------- /core-signatures.txt: -------------------------------------------------------------------------------- 1 | @defaultMessage spawns threads with vague names; use a custom thread factory and name threads so that you can tell (by its name) which executor it is associated with 2 | 3 | java.util.concurrent.Executors#newFixedThreadPool(int) 4 | java.util.concurrent.Executors#newSingleThreadExecutor() 5 | java.util.concurrent.Executors#newCachedThreadPool() 6 | java.util.concurrent.Executors#newSingleThreadScheduledExecutor() 7 | java.util.concurrent.Executors#newScheduledThreadPool(int) 8 | java.util.concurrent.Executors#defaultThreadFactory() 9 | java.util.concurrent.Executors#privilegedThreadFactory() 10 | 11 | java.lang.Character#codePointBefore(char[],int) @ Implicit start offset is error-prone when the char[] is a buffer and the first chars are random chars 12 | java.lang.Character#codePointAt(char[],int) @ Implicit end offset is error-prone when the char[] is a buffer and the last chars are random chars 13 | 14 | @defaultMessage Collections.sort dumps data into an array, sorts the array and reinserts data into the list, one should rather use Lucene's CollectionUtil sort methods which sort in place 15 | 16 | java.util.Collections#sort(java.util.List) 17 | java.util.Collections#sort(java.util.List,java.util.Comparator) 18 | 19 | java.io.StringReader#(java.lang.String) @ Use FastStringReader instead 20 | 21 | @defaultMessage Reference management is tricky, leave it to SearcherManager 22 | org.apache.lucene.index.IndexReader#decRef() 23 | org.apache.lucene.index.IndexReader#incRef() 24 | org.apache.lucene.index.IndexReader#tryIncRef() 25 | 26 | @defaultMessage QueryWrapperFilter is cachable by default - use Queries#wrap instead 27 | org.apache.lucene.search.QueryWrapperFilter#(org.apache.lucene.search.Query) 28 | 29 | @defaultMessage Because the filtercache doesn't take deletes into account FilteredQuery can't be used - use XFilteredQuery instead 30 | org.apache.lucene.search.FilteredQuery#(org.apache.lucene.search.Query,org.apache.lucene.search.Filter) 31 | org.apache.lucene.search.FilteredQuery#(org.apache.lucene.search.Query,org.apache.lucene.search.Filter,org.apache.lucene.search.FilteredQuery$FilterStrategy) 32 | 33 | @defaultMessage Pass the precision step from the mappings explicitly instead 34 | org.apache.lucene.search.NumericRangeQuery#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) 35 | org.apache.lucene.search.NumericRangeQuery#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) 36 | org.apache.lucene.search.NumericRangeQuery#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) 37 | org.apache.lucene.search.NumericRangeQuery#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) 38 | org.apache.lucene.search.NumericRangeFilter#newDoubleRange(java.lang.String,java.lang.Double,java.lang.Double,boolean,boolean) 39 | org.apache.lucene.search.NumericRangeFilter#newFloatRange(java.lang.String,java.lang.Float,java.lang.Float,boolean,boolean) 40 | org.apache.lucene.search.NumericRangeFilter#newIntRange(java.lang.String,java.lang.Integer,java.lang.Integer,boolean,boolean) 41 | org.apache.lucene.search.NumericRangeFilter#newLongRange(java.lang.String,java.lang.Long,java.lang.Long,boolean,boolean) 42 | 43 | @defaultMessage Only use wait / notify when really needed try to use concurrency primitives, latches or callbacks instead. 44 | java.lang.Object#wait() 45 | java.lang.Object#wait(long) 46 | java.lang.Object#wait(long,int) 47 | java.lang.Object#notify() 48 | java.lang.Object#notifyAll() 49 | 50 | @defaultMessage Beware of the behavior of this method on MIN_VALUE 51 | java.lang.Math#abs(int) 52 | java.lang.Math#abs(long) 53 | 54 | @defaultMessage Please do not try to stop the world 55 | java.lang.System#gc() 56 | 57 | @defaultMessage Use Channels.* methods to write to channels. Do not write directly. 58 | java.nio.channels.WritableByteChannel#write(java.nio.ByteBuffer) 59 | java.nio.channels.FileChannel#write(java.nio.ByteBuffer, long) 60 | java.nio.channels.GatheringByteChannel#write(java.nio.ByteBuffer[], int, int) 61 | java.nio.channels.GatheringByteChannel#write(java.nio.ByteBuffer[]) 62 | java.nio.channels.ReadableByteChannel#read(java.nio.ByteBuffer) 63 | java.nio.channels.ScatteringByteChannel#read(java.nio.ByteBuffer[]) 64 | java.nio.channels.ScatteringByteChannel#read(java.nio.ByteBuffer[], int, int) 65 | java.nio.channels.FileChannel#read(java.nio.ByteBuffer, long) 66 | 67 | @defaultMessage Use Lucene.parseLenient instead it strips off minor version 68 | org.apache.lucene.util.Version#parseLeniently(java.lang.String) 69 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | elasticsearch-plugin-suggest 5 | 4.0.0 6 | de.spinscale 7 | elasticsearch-plugin-suggest 8 | 1.3.2-2.0.2-SNAPSHOT 9 | jar 10 | Lucene FSTSuggester implementation for ElasticSearch 11 | 2011 12 | https://github.com/spinscale/elasticsearch-suggest-plugin/ 13 | 14 | scm:git:git://github.com/spinscale/elasticsearch-suggest-plugin.git 15 | scm:git:git@github.com:spinscale/elasticsearch-suggest-plugin.git 16 | https://github.com/spinscale/elasticsearch-suggest-plugin 17 | 18 | 19 | 20 | 1.3.2 21 | 4.9.0 22 | auto 23 | true 24 | onerror 25 | 26 | INFO 27 | 28 | 29 | 30 | 31 | org.apache.lucene 32 | lucene-test-framework 33 | ${lucene.version} 34 | test 35 | 36 | 37 | 38 | org.elasticsearch 39 | elasticsearch 40 | ${elasticsearch.version} 41 | compile 42 | 43 | 44 | 45 | org.apache.lucene 46 | lucene-misc 47 | ${lucene.version} 48 | compile 49 | 50 | 51 | 52 | org.elasticsearch 53 | elasticsearch 54 | ${elasticsearch.version} 55 | test 56 | test-jar 57 | 58 | 59 | 60 | log4j 61 | log4j 62 | 1.2.17 63 | test 64 | 65 | 66 | 67 | junit 68 | junit 69 | 4.11 70 | test 71 | 72 | 73 | 74 | commons-io 75 | commons-io 76 | 2.4 77 | test 78 | 79 | 80 | 81 | com.ning 82 | async-http-client 83 | 1.7.19 84 | test 85 | 86 | 87 | 88 | org.hamcrest 89 | hamcrest-core 90 | 1.2 91 | test 92 | 93 | 94 | 95 | org.hamcrest 96 | hamcrest-library 97 | 1.2 98 | test 99 | 100 | 101 | 102 | org.codehaus.jackson 103 | jackson-mapper-asl 104 | 1.9.12 105 | test 106 | 107 | 108 | 109 | 110 | org.slf4j 111 | slf4j-log4j12 112 | 1.6.1 113 | test 114 | 115 | 116 | 117 | commons-lang 118 | commons-lang 119 | 2.6 120 | test 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | org.apache.maven.plugins 129 | maven-compiler-plugin 130 | 2.3.2 131 | 132 | 1.7 133 | 1.7 134 | 135 | 136 | 137 | com.carrotsearch.randomizedtesting 138 | junit4-maven-plugin 139 | 2.0.15 140 | 141 | 142 | tests 143 | test 144 | 145 | junit4 146 | 147 | 148 | 20 149 | pipe,warn 150 | true 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | ${tests.jvms} 161 | 162 | 163 | 164 | 165 | 166 | ${tests.jvm.argline} 167 | 168 | -Xmx512m 169 | -Xss256k 170 | -XX:MaxDirectMemorySize=512m 171 | -Des.logger.prefix= 172 | 173 | ${tests.shuffle} 174 | ${tests.verbose} 175 | ${tests.seed} 176 | ${tests.failfast} 177 | 178 | 179 | ${tests.jvm.argline} 180 | ${tests.appendseed} 181 | ${tests.iters} 182 | ${tests.class} 183 | ${tests.method} 184 | ${tests.cluster_seed} 185 | ${tests.client.ratio} 186 | ${tests.enable_mock_modules} 187 | ${tests.assertion.disabled} 188 | ${es.logger.level} 189 | true 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | org.apache.maven.plugins 198 | maven-surefire-plugin 199 | 2.16 200 | 201 | true 202 | 203 | 204 | 205 | org.apache.maven.plugins 206 | maven-source-plugin 207 | 2.1.2 208 | 209 | 210 | attach-sources 211 | 212 | jar 213 | 214 | 215 | 216 | 217 | 218 | maven-assembly-plugin 219 | 2.3 220 | 221 | false 222 | ${project.build.directory}/releases/ 223 | 224 | ${basedir}/src/main/assemblies/plugin.xml 225 | 226 | 227 | 228 | 229 | package 230 | 231 | single 232 | 233 | 234 | 235 | 236 | 237 | org.eclipse.m2e 238 | lifecycle-mapping 239 | 1.0.0 240 | 241 | 242 | 243 | 244 | 245 | 246 | de.thetaphi 247 | forbiddenapis 248 | [1.0.0,) 249 | 250 | testCheck 251 | check 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | de.thetaphi 265 | forbiddenapis 266 | 1.6.1 267 | 268 | 269 | 270 | check-forbidden-apis 271 | 272 | 1.7 273 | 274 | true 275 | 276 | false 277 | 278 | de/spinscale/elasticsearch/service/suggest/ShardSuggestService.class 279 | 280 | 281 | 282 | jdk-unsafe 283 | jdk-deprecated 284 | jdk-system-out 285 | 286 | 287 | core-signatures.txt 288 | 289 | 290 | compile 291 | 292 | check 293 | 294 | 295 | 296 | check-forbidden-test-apis 297 | 298 | 1.7 299 | 300 | true 301 | 302 | false 303 | 304 | 305 | jdk-unsafe 306 | jdk-deprecated 307 | 308 | 309 | test-compile 310 | 311 | testCheck 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | org.sonatype.oss 322 | oss-parent 323 | 7 324 | 325 | 326 | 327 | 328 | The Apache Software License, Version 2.0 329 | http://www.apache.org/licenses/LICENSE-2.0.txt 330 | repo 331 | 332 | 333 | 334 | 335 | 336 | alr 337 | Alexander Reelsen 338 | alr@spinscale.de 339 | http://spinscale.github.com/ 340 | +1 341 | 342 | 343 | 344 | 345 | GitHub 346 | https://github.com/spinscale/elasticsearch-suggest-plugin/issues/ 347 | 348 | 349 | -------------------------------------------------------------------------------- /src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/ShardSuggestRefreshRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import java.io.IOException; 4 | 5 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; 6 | import org.elasticsearch.common.io.stream.StreamInput; 7 | import org.elasticsearch.common.io.stream.StreamOutput; 8 | 9 | 10 | public class ShardSuggestRefreshRequest extends BroadcastShardOperationRequest { 11 | 12 | private String field; 13 | 14 | public ShardSuggestRefreshRequest() {} 15 | 16 | public ShardSuggestRefreshRequest(String index, int shardId, SuggestRefreshRequest request) { 17 | super(index, shardId, request); 18 | field = request.field(); 19 | } 20 | 21 | public String field() { 22 | return field; 23 | } 24 | 25 | public void field(String field) { 26 | this.field = field; 27 | } 28 | 29 | @Override public void readFrom(StreamInput in) throws IOException { 30 | super.readFrom(in); 31 | field = in.readOptionalString(); 32 | } 33 | 34 | @Override public void writeTo(StreamOutput out) throws IOException { 35 | super.writeTo(out); 36 | out.writeOptionalString(field); 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return index() + " " + shardId() + " " + field; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/ShardSuggestRefreshResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationResponse; 4 | 5 | 6 | public class ShardSuggestRefreshResponse extends BroadcastShardOperationResponse { 7 | 8 | public ShardSuggestRefreshResponse() {} 9 | 10 | public ShardSuggestRefreshResponse(String index, int shardId) { 11 | super(index, shardId); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/SuggestRefreshAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRefreshRequestBuilder; 4 | import org.elasticsearch.action.ClientAction; 5 | import org.elasticsearch.client.Client; 6 | 7 | public class SuggestRefreshAction extends ClientAction { 8 | 9 | public static final SuggestRefreshAction INSTANCE = new SuggestRefreshAction(); 10 | public static final String NAME = "suggestRefresh-fst"; 11 | 12 | private SuggestRefreshAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public SuggestRefreshRequestBuilder newRequestBuilder(Client client) { 18 | return new SuggestRefreshRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public SuggestRefreshResponse newResponse() { 23 | return new SuggestRefreshResponse(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/SuggestRefreshRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import java.io.IOException; 4 | import java.util.Arrays; 5 | import java.util.Locale; 6 | 7 | import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; 8 | import org.elasticsearch.common.io.stream.StreamInput; 9 | import org.elasticsearch.common.io.stream.StreamOutput; 10 | 11 | 12 | public class SuggestRefreshRequest extends BroadcastOperationRequest { 13 | 14 | private String field; 15 | 16 | public SuggestRefreshRequest() {} 17 | 18 | public SuggestRefreshRequest(String... indices) { 19 | super(indices); 20 | } 21 | 22 | public String field() { 23 | return field; 24 | } 25 | 26 | public void field(String field) { 27 | this.field = field; 28 | } 29 | 30 | @Override 31 | public void readFrom(StreamInput in) throws IOException { 32 | super.readFrom(in); 33 | field = in.readOptionalString(); 34 | } 35 | 36 | @Override 37 | public void writeTo(StreamOutput out) throws IOException { 38 | super.writeTo(out); 39 | out.writeOptionalString(field); 40 | } 41 | 42 | @Override public String toString() { 43 | return String.format(Locale.ROOT, "[%s] field[%s]", Arrays.toString(indices), field); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/SuggestRefreshResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.elasticsearch.action.ShardOperationFailedException; 7 | import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; 8 | import org.elasticsearch.common.io.stream.StreamInput; 9 | import org.elasticsearch.common.io.stream.StreamOutput; 10 | import org.elasticsearch.common.xcontent.ToXContent; 11 | import org.elasticsearch.common.xcontent.XContentBuilder; 12 | 13 | 14 | public class SuggestRefreshResponse extends BroadcastOperationResponse implements ToXContent { 15 | 16 | public SuggestRefreshResponse() {} 17 | 18 | public SuggestRefreshResponse(int totalShards, int successfulShards, int failedShards, List shardFailures) { 19 | super(totalShards, successfulShards, failedShards, shardFailures); 20 | } 21 | 22 | @Override 23 | public void readFrom(StreamInput in) throws IOException { 24 | super.readFrom(in); 25 | } 26 | 27 | @Override 28 | public void writeTo(StreamOutput out) throws IOException { 29 | super.writeTo(out); 30 | } 31 | 32 | @Override 33 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 34 | return builder; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/refresh/TransportSuggestRefreshAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.refresh; 2 | 3 | import de.spinscale.elasticsearch.service.suggest.ShardSuggestService; 4 | import org.elasticsearch.ElasticsearchException; 5 | import org.elasticsearch.action.ShardOperationFailedException; 6 | import org.elasticsearch.action.support.DefaultShardOperationFailedException; 7 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; 8 | import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; 9 | import org.elasticsearch.cluster.ClusterService; 10 | import org.elasticsearch.cluster.ClusterState; 11 | import org.elasticsearch.cluster.block.ClusterBlockException; 12 | import org.elasticsearch.cluster.block.ClusterBlockLevel; 13 | import org.elasticsearch.cluster.routing.GroupShardsIterator; 14 | import org.elasticsearch.cluster.routing.ShardRouting; 15 | import org.elasticsearch.common.collect.Lists; 16 | import org.elasticsearch.common.inject.Inject; 17 | import org.elasticsearch.common.settings.Settings; 18 | import org.elasticsearch.index.service.IndexService; 19 | import org.elasticsearch.indices.IndicesService; 20 | import org.elasticsearch.threadpool.ThreadPool; 21 | import org.elasticsearch.transport.TransportService; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.atomic.AtomicReferenceArray; 25 | 26 | 27 | public class TransportSuggestRefreshAction extends TransportBroadcastOperationAction { 28 | 29 | private final IndicesService indicesService; 30 | 31 | @Inject 32 | public TransportSuggestRefreshAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, 33 | TransportService transportService, IndicesService indicesService) { 34 | super(settings, SuggestRefreshAction.NAME, threadPool, clusterService, transportService); 35 | this.indicesService = indicesService; 36 | } 37 | 38 | @Override 39 | protected String executor() { 40 | return ThreadPool.Names.INDEX; 41 | } 42 | 43 | @Override 44 | protected SuggestRefreshRequest newRequest() { 45 | return new SuggestRefreshRequest(); 46 | } 47 | 48 | @Override 49 | protected SuggestRefreshResponse newResponse(SuggestRefreshRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { 50 | int successfulShards = 0; 51 | int failedShards = 0; 52 | List shardFailures = Lists.newArrayList(); 53 | 54 | for (int i = 0; i < shardsResponses.length(); i++) { 55 | Object shardResponse = shardsResponses.get(i); 56 | if (shardResponse == null) { 57 | failedShards++; 58 | } else if (shardResponse instanceof BroadcastShardOperationFailedException) { 59 | failedShards++; 60 | shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); 61 | } else { 62 | successfulShards++; 63 | } 64 | } 65 | 66 | return new SuggestRefreshResponse(shardsResponses.length(), successfulShards, failedShards, shardFailures); 67 | } 68 | 69 | @Override 70 | protected ShardSuggestRefreshRequest newShardRequest() { 71 | return new ShardSuggestRefreshRequest(); 72 | } 73 | 74 | @Override 75 | protected ShardSuggestRefreshRequest newShardRequest(int numShards, ShardRouting shard, SuggestRefreshRequest request) { 76 | return new ShardSuggestRefreshRequest(shard.index(), shard.id(), request); 77 | } 78 | 79 | @Override 80 | protected ShardSuggestRefreshResponse newShardResponse() { 81 | return new ShardSuggestRefreshResponse(); 82 | } 83 | 84 | @Override 85 | protected ShardSuggestRefreshResponse shardOperation(ShardSuggestRefreshRequest request) throws ElasticsearchException { 86 | logger.trace("Entered TransportSuggestRefreshAction.shardOperation()"); 87 | IndexService indexService = indicesService.indexServiceSafe(request.index()); 88 | ShardSuggestService suggestShardService = indexService.shardInjectorSafe(request.shardId()).getInstance(ShardSuggestService.class); 89 | return suggestShardService.refresh(request); 90 | } 91 | 92 | @Override 93 | protected GroupShardsIterator shards(ClusterState clusterState, SuggestRefreshRequest request, String[] concreteIndices) { 94 | return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, null); 95 | } 96 | 97 | @Override 98 | protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestRefreshRequest request) { 99 | return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA); 100 | } 101 | 102 | @Override 103 | protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestRefreshRequest request, String[] concreteIndices) { 104 | return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA, concreteIndices); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/FstStats.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import de.spinscale.elasticsearch.service.suggest.ShardSuggestService; 4 | import org.elasticsearch.common.collect.Lists; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.common.io.stream.Streamable; 8 | import org.elasticsearch.common.xcontent.ToXContent; 9 | import org.elasticsearch.common.xcontent.XContentBuilder; 10 | import org.elasticsearch.index.shard.ShardId; 11 | 12 | import java.io.IOException; 13 | import java.io.Serializable; 14 | import java.util.List; 15 | 16 | public class FstStats implements Streamable, Serializable, ToXContent { 17 | 18 | private List stats = Lists.newArrayList(); 19 | 20 | public List getStats() { 21 | return stats; 22 | } 23 | 24 | @Override 25 | public void readFrom(StreamInput in) throws IOException { 26 | long size = in.readLong(); 27 | for (int i = 0 ; i < size; i++) { 28 | stats.add(FstIndexShardStats.readFstIndexShardStats(in)); 29 | } 30 | } 31 | 32 | @Override 33 | public void writeTo(StreamOutput out) throws IOException { 34 | out.writeLong(stats.size()); 35 | for (FstIndexShardStats fstIndexShardStats : stats) { 36 | fstIndexShardStats.writeTo(out); 37 | } 38 | } 39 | 40 | @Override 41 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 42 | builder.startArray("fstStats"); 43 | 44 | for (FstIndexShardStats fstIndexShardStats : stats) { 45 | fstIndexShardStats.toXContent(builder, params); 46 | } 47 | 48 | builder.endArray(); 49 | 50 | return builder; 51 | } 52 | 53 | 54 | public static class FstIndexShardStats implements Streamable, Serializable, ToXContent { 55 | 56 | private ShardId shardId; 57 | private String type; 58 | private ShardSuggestService.FieldType fieldType; 59 | private long sizeInBytes; 60 | 61 | public FstIndexShardStats() {} 62 | 63 | public FstIndexShardStats(ShardId shardId, String type, ShardSuggestService.FieldType fieldType, long sizeInBytes) { 64 | this.shardId = shardId; 65 | this.type = type; 66 | this.fieldType = fieldType; 67 | this.sizeInBytes = sizeInBytes; 68 | } 69 | 70 | public ShardId getShardId() { 71 | return shardId; 72 | } 73 | 74 | public String getType() { 75 | return type; 76 | } 77 | 78 | public ShardSuggestService.FieldType getFieldType() { 79 | return fieldType; 80 | } 81 | 82 | public long getSizeInBytes() { 83 | return sizeInBytes; 84 | } 85 | 86 | @Override 87 | public void readFrom(StreamInput in) throws IOException { 88 | type = in.readString(); 89 | sizeInBytes = in.readLong(); 90 | shardId = ShardId.readShardId(in); 91 | fieldType = new ShardSuggestService.FieldType(); 92 | fieldType.readFrom(in); 93 | 94 | } 95 | 96 | @Override 97 | public void writeTo(StreamOutput out) throws IOException { 98 | out.writeString(type); 99 | out.writeLong(sizeInBytes); 100 | shardId.writeTo(out); 101 | fieldType.writeTo(out); 102 | } 103 | 104 | @Override 105 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 106 | builder.startObject(); 107 | builder.field("index", shardId.getIndex()); 108 | builder.field("id", shardId.getId()); 109 | builder.field("sizeInBytes", sizeInBytes); 110 | builder.field("type", type); 111 | fieldType.toXContent(builder, params); 112 | builder.endObject(); 113 | return builder; 114 | } 115 | 116 | public static FstIndexShardStats readFstIndexShardStats(StreamInput in) throws IOException { 117 | FstIndexShardStats fstIndexShardStats = new FstIndexShardStats(); 118 | fstIndexShardStats.readFrom(in); 119 | return fstIndexShardStats; 120 | } 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/ShardSuggestStatisticsRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; 4 | 5 | public class ShardSuggestStatisticsRequest extends BroadcastShardOperationRequest { 6 | 7 | public ShardSuggestStatisticsRequest() { 8 | } 9 | 10 | public ShardSuggestStatisticsRequest(String index, int shardId, SuggestStatisticsRequest request) { 11 | super(index, shardId, request); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/ShardSuggestStatisticsResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationResponse; 4 | import org.elasticsearch.common.collect.Lists; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | import org.elasticsearch.index.shard.ShardId; 8 | 9 | import java.io.IOException; 10 | import java.util.List; 11 | 12 | public class ShardSuggestStatisticsResponse extends BroadcastShardOperationResponse { 13 | 14 | private List shardStats = Lists.newArrayList(); 15 | 16 | public ShardSuggestStatisticsResponse() {} 17 | 18 | public ShardSuggestStatisticsResponse(ShardId shardId) { 19 | super(shardId.getIndex(), shardId.id()); 20 | } 21 | 22 | @Override 23 | public void readFrom(StreamInput in) throws IOException { 24 | super.readFrom(in); 25 | long size = in.readLong(); 26 | if (size > 0) { 27 | for (int i = 0; i < size; i++) { 28 | FstStats.FstIndexShardStats fstIndexShardStats = FstStats.FstIndexShardStats.readFstIndexShardStats(in); 29 | shardStats.add(fstIndexShardStats); 30 | } 31 | } 32 | } 33 | 34 | @Override 35 | public void writeTo(StreamOutput out) throws IOException { 36 | super.writeTo(out); 37 | out.writeLong(shardStats.size()); 38 | for (FstStats.FstIndexShardStats fstIndexShardStats : shardStats) { 39 | fstIndexShardStats.writeTo(out); 40 | } 41 | } 42 | 43 | public List getFstIndexShardStats() { 44 | return shardStats; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/SuggestStatisticsAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import de.spinscale.elasticsearch.client.action.suggest.SuggestStatisticsRequestBuilder; 4 | import org.elasticsearch.action.ClientAction; 5 | import org.elasticsearch.client.Client; 6 | 7 | public class SuggestStatisticsAction extends ClientAction { 8 | 9 | public static final SuggestStatisticsAction INSTANCE = new SuggestStatisticsAction(); 10 | public static final String NAME = "suggestStatistics"; 11 | 12 | private SuggestStatisticsAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public SuggestStatisticsRequestBuilder newRequestBuilder(Client client) { 18 | return new SuggestStatisticsRequestBuilder(client); 19 | } 20 | 21 | @Override 22 | public SuggestStatisticsResponse newResponse() { 23 | return new SuggestStatisticsResponse(); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/SuggestStatisticsRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; 4 | 5 | public class SuggestStatisticsRequest extends BroadcastOperationRequest { 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/SuggestStatisticsResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import org.elasticsearch.action.ShardOperationFailedException; 4 | import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; 5 | import org.elasticsearch.common.Strings; 6 | import org.elasticsearch.common.collect.Maps; 7 | import org.elasticsearch.common.io.stream.StreamInput; 8 | import org.elasticsearch.common.io.stream.StreamOutput; 9 | import org.elasticsearch.common.xcontent.ToXContent; 10 | import org.elasticsearch.common.xcontent.XContentBuilder; 11 | 12 | import java.io.IOException; 13 | import java.util.List; 14 | import java.util.Map; 15 | 16 | import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastShardsHeader; 17 | 18 | public class SuggestStatisticsResponse extends BroadcastOperationResponse implements ToXContent { 19 | 20 | private FstStats fstStats = new FstStats(); 21 | 22 | public SuggestStatisticsResponse() {} 23 | 24 | public SuggestStatisticsResponse(int totalShards, int successfulShards, int failedShards, 25 | List successfulStatistics, 26 | List shardFailures) { 27 | super(totalShards, successfulShards, failedShards, shardFailures); 28 | 29 | for (ShardSuggestStatisticsResponse response : successfulStatistics) { 30 | if (response.getFstIndexShardStats() != null && response.getFstIndexShardStats().size() > 0) { 31 | fstStats.getStats().addAll(response.getFstIndexShardStats()); 32 | } 33 | } 34 | } 35 | 36 | @Override 37 | public void readFrom(StreamInput in) throws IOException { 38 | super.readFrom(in); 39 | fstStats = new FstStats(); 40 | fstStats.readFrom(in); 41 | } 42 | 43 | @Override 44 | public void writeTo(StreamOutput out) throws IOException { 45 | super.writeTo(out); 46 | fstStats.writeTo(out); 47 | } 48 | 49 | public FstStats fstStats() { 50 | return fstStats; 51 | } 52 | public FstStats getFstStats() { 53 | return fstStats; 54 | } 55 | 56 | @Override 57 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 58 | buildBroadcastShardsHeader(builder, this); 59 | fstStats.toXContent(builder, params); 60 | return builder; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/statistics/TransportSuggestStatisticsAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.statistics; 2 | 3 | import de.spinscale.elasticsearch.service.suggest.ShardSuggestService; 4 | import org.elasticsearch.ElasticsearchException; 5 | import org.elasticsearch.action.ShardOperationFailedException; 6 | import org.elasticsearch.action.support.DefaultShardOperationFailedException; 7 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; 8 | import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; 9 | import org.elasticsearch.cluster.ClusterService; 10 | import org.elasticsearch.cluster.ClusterState; 11 | import org.elasticsearch.cluster.block.ClusterBlockException; 12 | import org.elasticsearch.cluster.block.ClusterBlockLevel; 13 | import org.elasticsearch.cluster.routing.GroupShardsIterator; 14 | import org.elasticsearch.cluster.routing.ShardRouting; 15 | import org.elasticsearch.common.collect.Lists; 16 | import org.elasticsearch.common.inject.Inject; 17 | import org.elasticsearch.common.settings.Settings; 18 | import org.elasticsearch.index.service.IndexService; 19 | import org.elasticsearch.indices.IndicesService; 20 | import org.elasticsearch.threadpool.ThreadPool; 21 | import org.elasticsearch.transport.TransportService; 22 | 23 | import java.util.List; 24 | import java.util.concurrent.atomic.AtomicReferenceArray; 25 | 26 | public class TransportSuggestStatisticsAction extends TransportBroadcastOperationAction { 27 | 28 | private final IndicesService indicesService; 29 | 30 | @Inject 31 | public TransportSuggestStatisticsAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, 32 | TransportService transportService, IndicesService indicesService) { 33 | super(settings, SuggestStatisticsAction.NAME, threadPool, clusterService, transportService); 34 | this.indicesService = indicesService; 35 | } 36 | 37 | @Override 38 | protected String executor() { 39 | return ThreadPool.Names.MANAGEMENT; 40 | } 41 | 42 | @Override 43 | protected SuggestStatisticsRequest newRequest() { 44 | return new SuggestStatisticsRequest(); 45 | } 46 | 47 | @Override 48 | protected SuggestStatisticsResponse newResponse(SuggestStatisticsRequest request, AtomicReferenceArray shardsResponses, ClusterState clusterState) { 49 | int successfulShards = 0; 50 | int failedShards = 0; 51 | List shardFailures = Lists.newArrayList(); 52 | List successfulStatistics = Lists.newArrayList(); 53 | 54 | for (int i = 0; i < shardsResponses.length(); i++) { 55 | Object shardResponse = shardsResponses.get(i); 56 | if (shardResponse == null) { 57 | failedShards++; 58 | } else if (shardResponse instanceof BroadcastShardOperationFailedException) { 59 | failedShards++; 60 | shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); 61 | } else { 62 | successfulShards++; 63 | successfulStatistics.add((ShardSuggestStatisticsResponse)shardResponse); 64 | } 65 | } 66 | 67 | return new SuggestStatisticsResponse(shardsResponses.length(), successfulShards, failedShards, successfulStatistics, shardFailures); 68 | } 69 | 70 | @Override 71 | protected ShardSuggestStatisticsRequest newShardRequest() { 72 | return new ShardSuggestStatisticsRequest(); 73 | } 74 | 75 | @Override 76 | protected ShardSuggestStatisticsRequest newShardRequest(int numShards, ShardRouting shard, SuggestStatisticsRequest request) { 77 | return new ShardSuggestStatisticsRequest(shard.index(), shard.id(), request); 78 | } 79 | 80 | @Override 81 | protected ShardSuggestStatisticsResponse newShardResponse() { 82 | return new ShardSuggestStatisticsResponse(); 83 | } 84 | 85 | @Override 86 | protected ShardSuggestStatisticsResponse shardOperation(ShardSuggestStatisticsRequest request) throws ElasticsearchException { 87 | IndexService indexService = indicesService.indexServiceSafe(request.index()); 88 | ShardSuggestService suggestShardService = indexService.shardInjectorSafe(request.shardId()).getInstance(ShardSuggestService.class); 89 | return suggestShardService.getStatistics(); 90 | } 91 | 92 | @Override 93 | protected GroupShardsIterator shards(ClusterState clusterState, SuggestStatisticsRequest request, String[] concreteIndices) { 94 | return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, null); 95 | } 96 | 97 | @Override 98 | protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestStatisticsRequest request) { 99 | return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA); 100 | } 101 | 102 | @Override 103 | protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestStatisticsRequest request, String[] concreteIndices) { 104 | return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA, concreteIndices); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/ShardSuggestRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import java.io.IOException; 4 | 5 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest; 6 | import org.elasticsearch.common.Strings; 7 | import org.elasticsearch.common.io.stream.StreamInput; 8 | import org.elasticsearch.common.io.stream.StreamOutput; 9 | 10 | public class ShardSuggestRequest extends BroadcastShardOperationRequest { 11 | 12 | private int size = 10; 13 | private String field; 14 | private float similarity = 1.0f; 15 | private String term; 16 | private String[] types = Strings.EMPTY_ARRAY; 17 | private String suggestType = "fst"; 18 | private String queryAnalyzer; 19 | private String indexAnalyzer; 20 | private boolean preservePositionIncrements = true; 21 | 22 | public ShardSuggestRequest() {} 23 | 24 | public ShardSuggestRequest(String index, int shardId, SuggestRequest request) { 25 | super(index, shardId, request); 26 | size = request.size(); 27 | field = request.field(); 28 | term = request.term(); 29 | similarity = request.similarity(); 30 | types = request.types(); 31 | suggestType = request.suggestType(); 32 | queryAnalyzer = request.queryAnalyzer(); 33 | indexAnalyzer = request.indexAnalyzer(); 34 | preservePositionIncrements = request.preservePositionIncrements(); 35 | } 36 | 37 | public int size() { 38 | return size; 39 | } 40 | 41 | public void size(int size) { 42 | this.size = size; 43 | } 44 | 45 | public String field() { 46 | return field; 47 | } 48 | 49 | public void field(String field) { 50 | this.field = field; 51 | } 52 | 53 | public float similarity() { 54 | return similarity; 55 | } 56 | 57 | public void similarity(float similarity) { 58 | this.similarity = similarity; 59 | } 60 | 61 | public String term() { 62 | return term; 63 | } 64 | 65 | public void term(String term) { 66 | this.term = term; 67 | } 68 | 69 | public String suggestType() { 70 | return suggestType; 71 | } 72 | 73 | public void suggestType(String suggestType) { 74 | this.suggestType = suggestType; 75 | } 76 | 77 | public String[] types() { 78 | return types; 79 | } 80 | 81 | public String queryAnalyzer() { 82 | return queryAnalyzer; 83 | } 84 | 85 | public void queryAnalyzer(String queryAnalyzer) { 86 | this.queryAnalyzer = queryAnalyzer; 87 | } 88 | 89 | public String indexAnalyzer() { 90 | return indexAnalyzer; 91 | } 92 | 93 | public void indexAnalyzer(String indexAnalyzer) { 94 | this.indexAnalyzer = indexAnalyzer; 95 | } 96 | 97 | public boolean preservePositionIncrements() { 98 | return preservePositionIncrements; 99 | } 100 | 101 | public void preservePositionIncrements(boolean preservePositionIncrements) { 102 | this.preservePositionIncrements = preservePositionIncrements; 103 | } 104 | 105 | @Override public void readFrom(StreamInput in) throws IOException { 106 | super.readFrom(in); 107 | size = in.readVInt(); 108 | similarity = in.readFloat(); 109 | field = in.readString(); 110 | term = in.readString(); 111 | suggestType = in.readString(); 112 | queryAnalyzer = in.readOptionalString(); 113 | indexAnalyzer = in.readOptionalString(); 114 | types = in.readStringArray(); 115 | preservePositionIncrements = in.readBoolean(); 116 | } 117 | 118 | @Override public void writeTo(StreamOutput out) throws IOException { 119 | super.writeTo(out); 120 | out.writeVInt(size); 121 | out.writeFloat(similarity); 122 | out.writeString(field); 123 | out.writeString(term); 124 | out.writeString(suggestType); 125 | out.writeOptionalString(queryAnalyzer); 126 | out.writeOptionalString(indexAnalyzer); 127 | out.writeStringArrayNullable(types); 128 | out.writeBoolean(preservePositionIncrements); 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/ShardSuggestResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationResponse; 7 | import org.elasticsearch.common.collect.Lists; 8 | import org.elasticsearch.common.io.stream.StreamInput; 9 | import org.elasticsearch.common.io.stream.StreamOutput; 10 | 11 | public class ShardSuggestResponse extends BroadcastShardOperationResponse { 12 | 13 | private List suggestions; 14 | 15 | public ShardSuggestResponse() {} 16 | 17 | public ShardSuggestResponse(String index, int shardId, List suggestions) { 18 | super(index, shardId); 19 | this.suggestions = suggestions; 20 | } 21 | 22 | public List getSuggestions() { 23 | return Lists.newArrayList(suggestions); 24 | } 25 | 26 | public List suggestions() { 27 | return Lists.newArrayList(suggestions); 28 | } 29 | 30 | @Override public void readFrom(StreamInput in) throws IOException { 31 | super.readFrom(in); 32 | suggestions = (List) in.readGenericValue(); 33 | } 34 | 35 | @Override public void writeTo(StreamOutput out) throws IOException { 36 | super.writeTo(out); 37 | out.writeGenericValue(suggestions); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/SuggestAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRequestBuilder; 4 | import org.elasticsearch.action.ClientAction; 5 | import org.elasticsearch.client.Client; 6 | 7 | public class SuggestAction extends ClientAction { 8 | 9 | public static final SuggestAction INSTANCE = new SuggestAction(); 10 | public static final String NAME = "suggest-fst"; 11 | 12 | private SuggestAction() { 13 | super(NAME); 14 | } 15 | 16 | @Override 17 | public SuggestResponse newResponse() { 18 | return new SuggestResponse(); 19 | } 20 | 21 | @Override 22 | public SuggestRequestBuilder newRequestBuilder(Client client) { 23 | return new SuggestRequestBuilder(client); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/SuggestRequest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import java.io.IOException; 4 | import java.util.Arrays; 5 | import java.util.Locale; 6 | 7 | import org.elasticsearch.action.ActionRequestValidationException; 8 | import org.elasticsearch.action.ValidateActions; 9 | import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest; 10 | import org.elasticsearch.common.io.stream.StreamInput; 11 | import org.elasticsearch.common.io.stream.StreamOutput; 12 | 13 | public class SuggestRequest extends BroadcastOperationRequest { 14 | 15 | private String[] types = org.elasticsearch.common.Strings.EMPTY_ARRAY; 16 | 17 | private int size = 10; 18 | private String field; 19 | private float similarity = 1.0f; 20 | private String term; 21 | private String suggestType = "fst"; 22 | private String queryAnalyzer; 23 | private String indexAnalyzer; 24 | private boolean preservePositionIncrements = true; 25 | 26 | public SuggestRequest() { 27 | } 28 | 29 | public SuggestRequest(String... indices) { 30 | super(indices); 31 | } 32 | 33 | public int size() { 34 | return size; 35 | } 36 | 37 | public void size(int size) { 38 | this.size = size; 39 | } 40 | 41 | public String field() { 42 | return field; 43 | } 44 | 45 | public void field(String field) { 46 | this.field = field; 47 | } 48 | 49 | public float similarity() { 50 | return similarity; 51 | } 52 | 53 | public void similarity(float similarity) { 54 | this.similarity = similarity; 55 | } 56 | 57 | public String term() { 58 | return term; 59 | } 60 | 61 | public void term(String term) { 62 | this.term = term; 63 | } 64 | 65 | public void suggestType(String suggestType) { 66 | this.suggestType = suggestType; 67 | } 68 | 69 | public String suggestType() { 70 | return suggestType; 71 | } 72 | 73 | public String queryAnalyzer() { 74 | return queryAnalyzer; 75 | } 76 | 77 | public void queryAnalyzer(String queryAnalyzer) { 78 | this.queryAnalyzer = queryAnalyzer; 79 | } 80 | 81 | public String indexAnalyzer() { 82 | return indexAnalyzer; 83 | } 84 | 85 | public void indexAnalyzer(String indexAnalyzer) { 86 | this.indexAnalyzer = indexAnalyzer; 87 | } 88 | 89 | public void analyzer(String analyzer) { 90 | indexAnalyzer(analyzer); 91 | queryAnalyzer(analyzer); 92 | } 93 | 94 | @Override public ActionRequestValidationException validate() { 95 | ActionRequestValidationException validationException = super.validate(); 96 | if (field == null || field.length() == 0) { 97 | validationException = ValidateActions.addValidationError("No suggest field specified", validationException); 98 | } 99 | if (term == null || term.length() == 0) { 100 | validationException = ValidateActions.addValidationError("No query term specified", validationException); 101 | } 102 | return validationException; 103 | } 104 | 105 | String[] types() { 106 | return types; 107 | } 108 | 109 | public SuggestRequest types(String... types) { 110 | this.types = types; 111 | return this; 112 | } 113 | 114 | public SuggestRequest preservePositionIncrements(boolean preservePositionIncrements) { 115 | this.preservePositionIncrements = preservePositionIncrements; 116 | return this; 117 | } 118 | 119 | public boolean preservePositionIncrements() { 120 | return preservePositionIncrements; 121 | } 122 | 123 | @Override public void readFrom(StreamInput in) throws IOException { 124 | super.readFrom(in); 125 | size = in.readVInt(); 126 | similarity = in.readFloat(); 127 | field = in.readString(); 128 | term = in.readString(); 129 | suggestType = in.readString(); 130 | queryAnalyzer = in.readOptionalString(); 131 | indexAnalyzer = in.readOptionalString(); 132 | types = in.readStringArray(); 133 | } 134 | 135 | @Override public void writeTo(StreamOutput out) throws IOException { 136 | super.writeTo(out); 137 | out.writeVInt(size); 138 | out.writeFloat(similarity); 139 | out.writeString(field); 140 | out.writeString(term); 141 | out.writeString(suggestType); 142 | out.writeOptionalString(queryAnalyzer); 143 | out.writeOptionalString(indexAnalyzer); 144 | out.writeStringArray(types); 145 | } 146 | 147 | @Override public String toString() { 148 | return String.format(Locale.ROOT, "[%s] %s, field[%s], term[%s], size[%s], similarity[%s], suggestType[%s], indexAnalyzer[%s], queryAnalyzer[%s]", 149 | Arrays.toString(indices), Arrays.toString(types), field, term, size, similarity, suggestType, indexAnalyzer, queryAnalyzer); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/SuggestResponse.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import java.io.IOException; 4 | import java.util.List; 5 | 6 | import org.elasticsearch.action.ShardOperationFailedException; 7 | import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse; 8 | import org.elasticsearch.common.collect.Lists; 9 | import org.elasticsearch.common.io.stream.StreamInput; 10 | import org.elasticsearch.common.io.stream.StreamOutput; 11 | import org.elasticsearch.common.xcontent.ToXContent; 12 | import org.elasticsearch.common.xcontent.XContentBuilder; 13 | 14 | import static org.elasticsearch.rest.action.support.RestActions.buildBroadcastShardsHeader; 15 | 16 | public class SuggestResponse extends BroadcastOperationResponse implements ToXContent { 17 | 18 | private List suggestions; 19 | 20 | public SuggestResponse() { 21 | } 22 | 23 | public SuggestResponse(List suggestions, int totalShards, int successfulShards, int failedShards, List shardFailures) { 24 | super(totalShards, successfulShards, failedShards, shardFailures); 25 | this.suggestions = suggestions; 26 | } 27 | 28 | public List suggestions() { 29 | return Lists.newArrayList(suggestions); 30 | } 31 | 32 | public List getSuggestions() { 33 | return Lists.newArrayList(suggestions); 34 | } 35 | 36 | @Override public void readFrom(StreamInput in) throws IOException { 37 | super.readFrom(in); 38 | suggestions = (List) in.readGenericValue(); 39 | } 40 | 41 | @Override public void writeTo(StreamOutput out) throws IOException { 42 | super.writeTo(out); 43 | out.writeGenericValue(suggestions); 44 | } 45 | 46 | @Override 47 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 48 | buildBroadcastShardsHeader(builder, this); 49 | builder.field("suggestions", suggestions); 50 | return builder; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/action/suggest/suggest/TransportSuggestAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.action.suggest.suggest; 2 | 3 | import de.spinscale.elasticsearch.service.suggest.ShardSuggestService; 4 | import org.elasticsearch.ElasticsearchException; 5 | import org.elasticsearch.action.ShardOperationFailedException; 6 | import org.elasticsearch.action.support.DefaultShardOperationFailedException; 7 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; 8 | import org.elasticsearch.action.support.broadcast.TransportBroadcastOperationAction; 9 | import org.elasticsearch.cluster.ClusterService; 10 | import org.elasticsearch.cluster.ClusterState; 11 | import org.elasticsearch.cluster.block.ClusterBlockException; 12 | import org.elasticsearch.cluster.block.ClusterBlockLevel; 13 | import org.elasticsearch.cluster.routing.GroupShardsIterator; 14 | import org.elasticsearch.cluster.routing.ShardRouting; 15 | import org.elasticsearch.common.collect.ImmutableSortedSet; 16 | import org.elasticsearch.common.collect.Lists; 17 | import org.elasticsearch.common.inject.Inject; 18 | import org.elasticsearch.common.settings.Settings; 19 | import org.elasticsearch.index.service.IndexService; 20 | import org.elasticsearch.indices.IndicesService; 21 | import org.elasticsearch.threadpool.ThreadPool; 22 | import org.elasticsearch.transport.TransportService; 23 | 24 | import java.util.List; 25 | import java.util.concurrent.atomic.AtomicReferenceArray; 26 | 27 | import static org.elasticsearch.common.collect.Lists.newArrayList; 28 | 29 | public class TransportSuggestAction extends TransportBroadcastOperationAction { 30 | 31 | private final IndicesService indicesService; 32 | 33 | @Inject public TransportSuggestAction(Settings settings, ThreadPool threadPool, 34 | ClusterService clusterService, TransportService transportService, 35 | IndicesService indicesService) { 36 | super(settings, SuggestAction.NAME, threadPool, clusterService, transportService); 37 | this.indicesService = indicesService; 38 | } 39 | 40 | @Override 41 | protected String executor() { 42 | return ThreadPool.Names.SEARCH; 43 | } 44 | 45 | @Override 46 | protected SuggestRequest newRequest() { 47 | return new SuggestRequest(); 48 | } 49 | 50 | @Override 51 | protected SuggestResponse newResponse(SuggestRequest request, 52 | AtomicReferenceArray shardsResponses, ClusterState clusterState) { 53 | logger.trace("Entered TransportSuggestAction.newResponse()"); 54 | 55 | int successfulShards = 0; 56 | int failedShards = 0; 57 | List shardFailures = null; 58 | List items = Lists.newArrayList(); 59 | for (int i = 0; i < shardsResponses.length(); i++) { 60 | Object shardResponse = shardsResponses.get(i); 61 | if (shardResponse == null) { 62 | failedShards++; 63 | } else if (shardResponse instanceof BroadcastShardOperationFailedException) { 64 | failedShards++; 65 | if (shardFailures == null) { 66 | shardFailures = newArrayList(); 67 | } 68 | shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); 69 | } else if (shardResponse instanceof ShardSuggestResponse) { 70 | ShardSuggestResponse shardSuggestResponse = (ShardSuggestResponse) shardResponse; 71 | List shardItems = shardSuggestResponse.suggestions(); 72 | items.addAll(shardItems); 73 | successfulShards++; 74 | } else { 75 | successfulShards++; 76 | } 77 | } 78 | 79 | List resultItems = ImmutableSortedSet.copyOf(items).asList(); 80 | return new SuggestResponse(resultItems.subList(0, Math.min(resultItems.size(), request.size())), 81 | shardsResponses.length(), successfulShards, failedShards, shardFailures); 82 | } 83 | 84 | @Override 85 | protected ShardSuggestRequest newShardRequest() { 86 | return new ShardSuggestRequest(); 87 | } 88 | 89 | @Override 90 | protected ShardSuggestRequest newShardRequest(int numShards, ShardRouting shard, SuggestRequest request) { 91 | return new ShardSuggestRequest(shard.index(), shard.id(), request); 92 | } 93 | 94 | @Override 95 | protected ShardSuggestResponse newShardResponse() { 96 | return new ShardSuggestResponse(); 97 | } 98 | 99 | @Override 100 | protected ShardSuggestResponse shardOperation(ShardSuggestRequest request) throws ElasticsearchException { 101 | logger.trace("Entered TransportSuggestAction.shardOperation()"); 102 | IndexService indexService = indicesService.indexServiceSafe(request.index()); 103 | ShardSuggestService suggestShardService = indexService.shardInjectorSafe(request.shardId()).getInstance(ShardSuggestService.class); 104 | return suggestShardService.suggest(request); 105 | } 106 | 107 | @Override 108 | protected GroupShardsIterator shards(ClusterState clusterState, 109 | SuggestRequest request, String[] concreteIndices) { 110 | logger.trace("Entered TransportSuggestAction.shards()"); 111 | return clusterService.operationRouting().searchShards(clusterState, request.indices(), concreteIndices, null, null); 112 | } 113 | 114 | @Override 115 | protected ClusterBlockException checkGlobalBlock(ClusterState state, SuggestRequest request) { 116 | return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA); 117 | } 118 | 119 | @Override 120 | protected ClusterBlockException checkRequestBlock(ClusterState state, SuggestRequest request, String[] concreteIndices) { 121 | return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA, concreteIndices); 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/client/action/suggest/SuggestRefreshRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.client.action.suggest; 2 | 3 | import org.elasticsearch.action.ActionListener; 4 | import org.elasticsearch.action.ActionRequestBuilder; 5 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshAction; 6 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshRequest; 7 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshResponse; 8 | import org.elasticsearch.client.Client; 9 | 10 | public class SuggestRefreshRequestBuilder extends ActionRequestBuilder { 11 | 12 | public SuggestRefreshRequestBuilder(Client client) { 13 | super(client, new SuggestRefreshRequest()); 14 | } 15 | 16 | public SuggestRefreshRequestBuilder setIndices(String ... indices) { 17 | request.indices(indices); 18 | return this; 19 | } 20 | 21 | public SuggestRefreshRequestBuilder setField(String field) { 22 | request.field(field); 23 | return this; 24 | } 25 | 26 | @Override 27 | protected void doExecute(ActionListener listener) { 28 | client.execute(SuggestRefreshAction.INSTANCE, request, listener); 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/client/action/suggest/SuggestRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.client.action.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestAction; 4 | import org.elasticsearch.action.ActionListener; 5 | import org.elasticsearch.action.ActionRequestBuilder; 6 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestRequest; 7 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestResponse; 8 | import org.elasticsearch.client.Client; 9 | 10 | public class SuggestRequestBuilder extends ActionRequestBuilder { 11 | 12 | public SuggestRequestBuilder(Client client) { 13 | super(client, new SuggestRequest()); 14 | } 15 | 16 | @Override 17 | protected void doExecute(ActionListener listener) { 18 | client.execute(SuggestAction.INSTANCE, request, listener); 19 | } 20 | 21 | public SuggestRequestBuilder term(String term) { 22 | request.term(term); 23 | return this; 24 | } 25 | 26 | public SuggestRequestBuilder field(String field) { 27 | request.field(field); 28 | return this; 29 | } 30 | 31 | public SuggestRequestBuilder suggestType(String suggestType) { 32 | request.suggestType(suggestType); 33 | return this; 34 | } 35 | 36 | public SuggestRequestBuilder similarity(float similarity) { 37 | request.similarity(similarity); 38 | return this; 39 | } 40 | 41 | public SuggestRequestBuilder size(int size) { 42 | request.size(size); 43 | return this; 44 | } 45 | 46 | public SuggestRequestBuilder setIndices(String ... indices) { 47 | request.indices(indices); 48 | return this; 49 | } 50 | 51 | public SuggestRequestBuilder queryAnalyzer(String queryAnalyzer) { 52 | request.queryAnalyzer(queryAnalyzer); 53 | return this; 54 | } 55 | 56 | public SuggestRequestBuilder indexAnalyzer(String indexAnalyzer) { 57 | request.indexAnalyzer(indexAnalyzer); 58 | return this; 59 | } 60 | 61 | public SuggestRequestBuilder analyzer(String analyzer) { 62 | request.indexAnalyzer(analyzer); 63 | request.queryAnalyzer(analyzer); 64 | return this; 65 | } 66 | 67 | public SuggestRequestBuilder preservePositionIncrements(boolean preservePositionIncrements) { 68 | request.preservePositionIncrements(preservePositionIncrements); 69 | return this; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/client/action/suggest/SuggestStatisticsRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.client.action.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsAction; 4 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsRequest; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsResponse; 6 | import org.elasticsearch.action.ActionListener; 7 | import org.elasticsearch.action.ActionRequestBuilder; 8 | import org.elasticsearch.client.Client; 9 | 10 | public class SuggestStatisticsRequestBuilder extends ActionRequestBuilder { 11 | 12 | public SuggestStatisticsRequestBuilder(Client client) { 13 | super(client, new SuggestStatisticsRequest()); 14 | } 15 | 16 | @Override 17 | protected void doExecute(ActionListener listener) { 18 | client.execute(SuggestStatisticsAction.INSTANCE, request, listener); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/module/suggest/ShardSuggestModule.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest; 2 | 3 | import org.elasticsearch.common.inject.AbstractModule; 4 | import de.spinscale.elasticsearch.service.suggest.ShardSuggestService; 5 | 6 | 7 | public class ShardSuggestModule extends AbstractModule { 8 | 9 | @Override 10 | protected void configure() { 11 | bind(ShardSuggestService.class).asEagerSingleton(); 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/plugin/suggest/SuggestPlugin.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.plugin.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshAction; 4 | import de.spinscale.elasticsearch.action.suggest.refresh.TransportSuggestRefreshAction; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsAction; 6 | import de.spinscale.elasticsearch.action.suggest.statistics.TransportSuggestStatisticsAction; 7 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestAction; 8 | import de.spinscale.elasticsearch.action.suggest.suggest.TransportSuggestAction; 9 | import de.spinscale.elasticsearch.module.suggest.ShardSuggestModule; 10 | import de.spinscale.elasticsearch.rest.action.suggest.RestRefreshSuggestAction; 11 | import de.spinscale.elasticsearch.rest.action.suggest.RestStatisticsAction; 12 | import de.spinscale.elasticsearch.rest.action.suggest.RestSuggestAction; 13 | import de.spinscale.elasticsearch.service.suggest.SuggestService; 14 | import org.elasticsearch.ElasticsearchException; 15 | import org.elasticsearch.Version; 16 | import org.elasticsearch.action.ActionModule; 17 | import org.elasticsearch.common.collect.Lists; 18 | import org.elasticsearch.common.component.LifecycleComponent; 19 | import org.elasticsearch.common.inject.Inject; 20 | import org.elasticsearch.common.inject.Module; 21 | import org.elasticsearch.common.settings.Settings; 22 | import org.elasticsearch.plugins.AbstractPlugin; 23 | import org.elasticsearch.rest.RestModule; 24 | 25 | import java.util.Collection; 26 | import java.util.Locale; 27 | 28 | public class SuggestPlugin extends AbstractPlugin { 29 | 30 | private final boolean isClient; 31 | 32 | @Inject 33 | public SuggestPlugin(Settings settings) { 34 | this.isClient = settings.getAsBoolean("node.client", false); 35 | 36 | // Check if the plugin is newer than elasticsearch 37 | // First failure, if the versions dont match 38 | // Second failure: if the Version specified in before() does not yet exist, therefore catching Throwable 39 | try { 40 | if (Version.CURRENT.before(Version.V_1_2_4)) { 41 | throw new Exception(); 42 | } 43 | } catch (Throwable e) { 44 | String error = String.format(Locale.ROOT, "The elasticsearch suggest plugin needs a newer version of elasticsearch than %s", Version.CURRENT); 45 | throw new ElasticsearchException(error); 46 | } 47 | } 48 | 49 | @Override 50 | public String name() { 51 | return "suggest"; 52 | } 53 | 54 | @Override 55 | public String description() { 56 | return "Suggest Plugin"; 57 | } 58 | 59 | public void onModule(RestModule restModule) { 60 | restModule.addRestAction(RestSuggestAction.class); 61 | restModule.addRestAction(RestRefreshSuggestAction.class); 62 | restModule.addRestAction(RestStatisticsAction.class); 63 | } 64 | 65 | public void onModule(ActionModule actionModule) { 66 | actionModule.registerAction(SuggestAction.INSTANCE, TransportSuggestAction.class); 67 | actionModule.registerAction(SuggestRefreshAction.INSTANCE, TransportSuggestRefreshAction.class); 68 | actionModule.registerAction(SuggestStatisticsAction.INSTANCE, TransportSuggestStatisticsAction.class); 69 | } 70 | 71 | @SuppressWarnings("rawtypes") 72 | @Override public Collection> services() { 73 | Collection> services = Lists.newArrayList(); 74 | 75 | if (!isClient) { 76 | services.add(SuggestService.class); 77 | } 78 | return services; 79 | } 80 | 81 | @Override 82 | public Collection> shardModules() { 83 | Collection> modules = Lists.newArrayList(); 84 | modules.add(ShardSuggestModule.class); 85 | return modules; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/rest/action/suggest/RestRefreshSuggestAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.rest.action.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshAction; 4 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshRequest; 5 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshResponse; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.common.Strings; 8 | import org.elasticsearch.common.inject.Inject; 9 | import org.elasticsearch.common.settings.Settings; 10 | import org.elasticsearch.common.xcontent.XContentFactory; 11 | import org.elasticsearch.common.xcontent.XContentParser; 12 | import org.elasticsearch.common.xcontent.support.XContentMapValues; 13 | import org.elasticsearch.rest.*; 14 | import org.elasticsearch.rest.action.support.AcknowledgedRestListener; 15 | import org.elasticsearch.rest.action.support.RestToXContentListener; 16 | 17 | import java.io.IOException; 18 | import java.util.Map; 19 | 20 | import static org.elasticsearch.rest.RestRequest.Method.POST; 21 | 22 | public class RestRefreshSuggestAction extends BaseRestHandler { 23 | 24 | @Inject public RestRefreshSuggestAction(Settings settings, Client client, RestController controller) { 25 | super(settings, client); 26 | controller.registerHandler(POST, "/__suggestRefresh", this); 27 | controller.registerHandler(POST, "/{index}/__suggestRefresh", this); 28 | controller.registerHandler(POST, "/{index}/{type}/__suggestRefresh", this); 29 | } 30 | 31 | @Override 32 | public void handleRequest(final RestRequest request, final RestChannel channel, Client client) { 33 | final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); 34 | 35 | try { 36 | SuggestRefreshRequest suggestRefreshRequest = new SuggestRefreshRequest(indices); 37 | 38 | if (request.hasContent()) { 39 | XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content()); 40 | Map parserMap = parser.mapAndClose(); 41 | 42 | if (parserMap.containsKey("field")) { 43 | suggestRefreshRequest.field(XContentMapValues.nodeStringValue(parserMap.get("field"), "")); 44 | } 45 | } 46 | 47 | client.execute(SuggestRefreshAction.INSTANCE, suggestRefreshRequest, new RestToXContentListener(channel)); 48 | } catch (IOException e) { 49 | channel.sendResponse(new BytesRestResponse(RestStatus.BAD_REQUEST, "Could not extract field")); 50 | } 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/rest/action/suggest/RestStatisticsAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.rest.action.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsAction; 4 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsRequest; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsResponse; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.common.inject.Inject; 8 | import org.elasticsearch.common.settings.Settings; 9 | import org.elasticsearch.rest.BaseRestHandler; 10 | import org.elasticsearch.rest.RestChannel; 11 | import org.elasticsearch.rest.RestController; 12 | import org.elasticsearch.rest.RestRequest; 13 | import org.elasticsearch.rest.action.support.RestToXContentListener; 14 | 15 | import static org.elasticsearch.rest.RestRequest.Method.GET; 16 | 17 | public class RestStatisticsAction extends BaseRestHandler { 18 | 19 | @Inject 20 | public RestStatisticsAction(Settings settings, Client client, RestController controller) { 21 | super(settings, client); 22 | controller.registerHandler(GET, "/__suggestStatistics", this); 23 | } 24 | 25 | @Override 26 | public void handleRequest(final RestRequest request, final RestChannel channel, Client client) { 27 | client.execute(SuggestStatisticsAction.INSTANCE, new SuggestStatisticsRequest(), 28 | new RestToXContentListener(channel)); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/rest/action/suggest/RestSuggestAction.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.rest.action.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestAction; 4 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestRequest; 5 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestResponse; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.common.Strings; 8 | import org.elasticsearch.common.inject.Inject; 9 | import org.elasticsearch.common.settings.Settings; 10 | import org.elasticsearch.common.xcontent.XContentFactory; 11 | import org.elasticsearch.common.xcontent.XContentParser; 12 | import org.elasticsearch.common.xcontent.support.XContentMapValues; 13 | import org.elasticsearch.rest.*; 14 | import org.elasticsearch.rest.action.support.RestToXContentListener; 15 | 16 | import java.io.IOException; 17 | import java.util.Map; 18 | 19 | import static org.elasticsearch.rest.RestRequest.Method.GET; 20 | import static org.elasticsearch.rest.RestRequest.Method.POST; 21 | import static org.elasticsearch.rest.RestStatus.BAD_REQUEST; 22 | 23 | public class RestSuggestAction extends BaseRestHandler { 24 | 25 | @Inject 26 | public RestSuggestAction(Settings settings, Client client, RestController controller) { 27 | super(settings, client); 28 | controller.registerHandler(GET, "/{index}/__suggest", this); 29 | controller.registerHandler(GET, "/{index}/{type}/__suggest", this); 30 | controller.registerHandler(POST, "/{index}/__suggest", this); 31 | controller.registerHandler(POST, "/{index}/{type}/__suggest", this); 32 | } 33 | 34 | @Override 35 | public void handleRequest(final RestRequest request, final RestChannel channel, Client client) { 36 | final String[] indices = Strings.splitStringByCommaToArray(request.param("index")); 37 | 38 | try { 39 | Map parserMap = null; 40 | if (request.hasContent()) { 41 | XContentParser parser = XContentFactory.xContent(request.content()).createParser(request.content()); 42 | parserMap = parser.mapAndClose(); 43 | } else if (request.hasParam("source")) { 44 | String source = request.param("source"); 45 | XContentParser parser = XContentFactory.xContent(source).createParser(source); 46 | parserMap = parser.mapAndClose(); 47 | } else { 48 | channel.sendResponse(new BytesRestResponse(BAD_REQUEST, "Please provide body data or source parameter")); 49 | } 50 | 51 | SuggestRequest suggestRequest = new SuggestRequest(indices); 52 | suggestRequest.field(XContentMapValues.nodeStringValue(parserMap.get("field"), "")); 53 | suggestRequest.suggestType(XContentMapValues.nodeStringValue(parserMap.get("type"), "")); 54 | if (parserMap.containsKey("analyzer")) { 55 | suggestRequest.indexAnalyzer(XContentMapValues.nodeStringValue(parserMap.get("analyzer"), "")); 56 | suggestRequest.queryAnalyzer(XContentMapValues.nodeStringValue(parserMap.get("analyzer"), "")); 57 | } else { 58 | suggestRequest.indexAnalyzer(XContentMapValues.nodeStringValue(parserMap.get("indexAnalyzer"), "")); 59 | suggestRequest.queryAnalyzer(XContentMapValues.nodeStringValue(parserMap.get("queryAnalyzer"), "")); 60 | } 61 | suggestRequest.term(XContentMapValues.nodeStringValue(parserMap.get("term"), "")); 62 | suggestRequest.similarity(XContentMapValues.nodeFloatValue(parserMap.get("similarity"), 1.0f)); 63 | suggestRequest.size(XContentMapValues.nodeIntegerValue(parserMap.get("size"), 10)); 64 | 65 | client.execute(SuggestAction.INSTANCE, suggestRequest, new RestToXContentListener(channel)); 66 | } catch (IOException e) { 67 | try { 68 | channel.sendResponse(new BytesRestResponse(channel, RestStatus.BAD_REQUEST, e)); 69 | } catch (IOException e1) { 70 | logger.error("Failed to send failure response", e1); 71 | } 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/service/suggest/AbstractCacheLoaderSuggester.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.service.suggest; 2 | 3 | import org.apache.lucene.analysis.Analyzer; 4 | import org.apache.lucene.analysis.standard.StandardAnalyzer; 5 | import org.apache.lucene.search.spell.HighFrequencyDictionary; 6 | import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; 7 | import org.apache.lucene.search.suggest.analyzing.FuzzySuggester; 8 | import org.elasticsearch.ElasticsearchException; 9 | import org.elasticsearch.common.Strings; 10 | import org.elasticsearch.common.cache.CacheLoader; 11 | import org.elasticsearch.common.cache.LoadingCache; 12 | import org.elasticsearch.index.analysis.AnalysisService; 13 | import org.elasticsearch.index.analysis.NamedAnalyzer; 14 | import org.elasticsearch.index.mapper.FieldMapper; 15 | import org.elasticsearch.index.mapper.MapperService; 16 | 17 | public abstract class AbstractCacheLoaderSuggester extends CacheLoader { 18 | 19 | private MapperService mapperService; 20 | private AnalysisService analysisService; 21 | protected LoadingCache dictCache; 22 | 23 | public AbstractCacheLoaderSuggester(MapperService mapperService, AnalysisService analysisService, 24 | LoadingCache dictCache) { 25 | this.mapperService = mapperService; 26 | this.analysisService = analysisService; 27 | this.dictCache = dictCache; 28 | } 29 | 30 | @Override 31 | public T load(ShardSuggestService.FieldType fieldType) throws Exception { 32 | MapperService.SmartNameFieldMappers fieldMappers = mapperService.smartName(fieldType.field(), fieldType.types()); 33 | 34 | Analyzer queryAnalyzer = null; 35 | Analyzer indexAnalyzer = null; 36 | if (fieldMappers != null) { 37 | FieldMapper fieldMapper = mapperService.smartName(fieldType.field(), fieldType.types()).mapper(); 38 | 39 | queryAnalyzer = fieldMapper.searchAnalyzer(); 40 | if (Strings.hasLength(fieldType.indexAnalyzer())) { 41 | NamedAnalyzer namedAnalyzer = analysisService.analyzer(fieldType.queryAnalyzer()); 42 | if (namedAnalyzer == null) { 43 | throw new ElasticsearchException("Query analyzer[" + fieldType.queryAnalyzer() + "] does not exist."); 44 | } 45 | queryAnalyzer = namedAnalyzer.analyzer(); 46 | } 47 | 48 | indexAnalyzer = fieldMapper.searchAnalyzer(); 49 | if (Strings.hasLength(fieldType.indexAnalyzer())) { 50 | NamedAnalyzer namedAnalyzer = analysisService.analyzer(fieldType.indexAnalyzer()); 51 | if (namedAnalyzer == null) { 52 | throw new ElasticsearchException("Index analyzer[" + fieldType.indexAnalyzer() + "] does not exist."); 53 | } 54 | indexAnalyzer = namedAnalyzer.analyzer(); 55 | } 56 | } 57 | 58 | if (queryAnalyzer == null) { 59 | queryAnalyzer = new StandardAnalyzer(org.elasticsearch.Version.CURRENT.luceneVersion); 60 | } 61 | if (indexAnalyzer == null) { 62 | indexAnalyzer = new StandardAnalyzer(org.elasticsearch.Version.CURRENT.luceneVersion); 63 | } 64 | 65 | return getSuggester(indexAnalyzer, queryAnalyzer, fieldType); 66 | } 67 | 68 | public abstract T getSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, 69 | ShardSuggestService.FieldType fieldType) throws Exception; 70 | 71 | 72 | public static class CacheLoaderAnalyzingSuggester extends AbstractCacheLoaderSuggester { 73 | 74 | public CacheLoaderAnalyzingSuggester(MapperService mapperService, AnalysisService analysisService, LoadingCache dictCache) { 75 | super(mapperService, analysisService, dictCache); 76 | } 77 | 78 | @Override 79 | public AnalyzingSuggester getSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, 80 | ShardSuggestService.FieldType fieldType) throws Exception { 81 | AnalyzingSuggester analyzingSuggester = new AnalyzingSuggester(indexAnalyzer, queryAnalyzer, 82 | AnalyzingSuggester.EXACT_FIRST, 256, -1, fieldType.preservePositionIncrements()); 83 | analyzingSuggester.build(dictCache.getUnchecked(fieldType.field())); 84 | return analyzingSuggester; 85 | } 86 | } 87 | 88 | public static class CacheLoaderFuzzySuggester extends AbstractCacheLoaderSuggester { 89 | 90 | public CacheLoaderFuzzySuggester(MapperService mapperService, AnalysisService analysisService, LoadingCache dictCache) { 91 | super(mapperService, analysisService, dictCache); 92 | } 93 | 94 | @Override 95 | public FuzzySuggester getSuggester(Analyzer indexAnalyzer, Analyzer queryAnalyzer, 96 | ShardSuggestService.FieldType fieldType) throws Exception { 97 | FuzzySuggester fuzzySuggester = new FuzzySuggester(indexAnalyzer, queryAnalyzer, FuzzySuggester.EXACT_FIRST | FuzzySuggester.PRESERVE_SEP, 256, -1, 98 | fieldType.preservePositionIncrements(), FuzzySuggester.DEFAULT_MAX_EDITS, FuzzySuggester.DEFAULT_TRANSPOSITIONS, 99 | FuzzySuggester.DEFAULT_NON_FUZZY_PREFIX, FuzzySuggester.DEFAULT_MIN_FUZZY_LENGTH, FuzzySuggester.DEFAULT_UNICODE_AWARE); 100 | fuzzySuggester.build(dictCache.getUnchecked(fieldType.field())); 101 | return fuzzySuggester; 102 | } 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/service/suggest/ShardSuggestService.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.service.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.refresh.ShardSuggestRefreshRequest; 4 | import de.spinscale.elasticsearch.action.suggest.refresh.ShardSuggestRefreshResponse; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 6 | import de.spinscale.elasticsearch.action.suggest.statistics.ShardSuggestStatisticsResponse; 7 | import de.spinscale.elasticsearch.action.suggest.suggest.ShardSuggestRequest; 8 | import de.spinscale.elasticsearch.action.suggest.suggest.ShardSuggestResponse; 9 | import org.apache.lucene.analysis.core.WhitespaceAnalyzer; 10 | import org.apache.lucene.index.IndexReader; 11 | import org.apache.lucene.index.IndexWriterConfig; 12 | import org.apache.lucene.search.spell.HighFrequencyDictionary; 13 | import org.apache.lucene.search.spell.SpellChecker; 14 | import org.apache.lucene.search.suggest.Lookup.LookupResult; 15 | import org.apache.lucene.search.suggest.analyzing.AnalyzingSuggester; 16 | import org.apache.lucene.search.suggest.analyzing.FuzzySuggester; 17 | import org.apache.lucene.search.suggest.fst.FSTCompletionLookup; 18 | import org.apache.lucene.store.RAMDirectory; 19 | import org.apache.lucene.util.Version; 20 | import org.elasticsearch.ElasticsearchException; 21 | import org.elasticsearch.common.Strings; 22 | import org.elasticsearch.common.base.Function; 23 | import org.elasticsearch.common.base.Joiner; 24 | import org.elasticsearch.common.base.Objects; 25 | import org.elasticsearch.common.cache.CacheBuilder; 26 | import org.elasticsearch.common.cache.CacheLoader; 27 | import org.elasticsearch.common.cache.LoadingCache; 28 | import org.elasticsearch.common.collect.Collections2; 29 | import org.elasticsearch.common.collect.Lists; 30 | import org.elasticsearch.common.inject.Inject; 31 | import org.elasticsearch.common.inject.internal.ToStringBuilder; 32 | import org.elasticsearch.common.io.stream.StreamInput; 33 | import org.elasticsearch.common.io.stream.StreamOutput; 34 | import org.elasticsearch.common.io.stream.Streamable; 35 | import org.elasticsearch.common.settings.Settings; 36 | import org.elasticsearch.common.xcontent.ToXContent; 37 | import org.elasticsearch.common.xcontent.XContentBuilder; 38 | import org.elasticsearch.index.analysis.AnalysisService; 39 | import org.elasticsearch.index.engine.Engine; 40 | import org.elasticsearch.index.mapper.MapperService; 41 | import org.elasticsearch.index.settings.IndexSettings; 42 | import org.elasticsearch.index.shard.AbstractIndexShardComponent; 43 | import org.elasticsearch.index.shard.IndexShardState; 44 | import org.elasticsearch.index.shard.ShardId; 45 | import org.elasticsearch.index.shard.service.IndexShard; 46 | 47 | import java.io.IOException; 48 | import java.io.Serializable; 49 | import java.util.*; 50 | import java.util.concurrent.locks.ReentrantLock; 51 | 52 | public class ShardSuggestService extends AbstractIndexShardComponent { 53 | 54 | private final IndexShard indexShard; 55 | 56 | private final ReentrantLock lock = new ReentrantLock(); 57 | private IndexReader indexReader; 58 | private final LoadingCache lookupCache; 59 | private final LoadingCache analyzingSuggesterCache; 60 | private final LoadingCache fuzzySuggesterCache; 61 | private final LoadingCache dictCache; 62 | private final LoadingCache spellCheckerCache; 63 | private final LoadingCache ramDirectoryCache; 64 | 65 | @Inject 66 | public ShardSuggestService(ShardId shardId, @IndexSettings Settings indexSettings, IndexShard indexShard, 67 | final AnalysisService analysisService, final MapperService mapperService) { 68 | super(shardId, indexSettings); 69 | this.indexShard = indexShard; 70 | 71 | ramDirectoryCache = CacheBuilder.newBuilder().build( 72 | new CacheLoader() { 73 | @Override 74 | public RAMDirectory load(String field) throws Exception { 75 | return new RAMDirectory(); 76 | } 77 | } 78 | ); 79 | 80 | dictCache = CacheBuilder.newBuilder().build( 81 | new CacheLoader() { 82 | @Override 83 | public HighFrequencyDictionary load(String field) throws Exception { 84 | return new HighFrequencyDictionary(createOrGetIndexReader(), field, 0.00001f); 85 | } 86 | } 87 | ); 88 | 89 | spellCheckerCache = CacheBuilder.newBuilder().build( 90 | new CacheLoader() { 91 | @Override 92 | public SpellChecker load(String field) throws Exception { 93 | SpellChecker spellChecker = new SpellChecker(ramDirectoryCache.get(field)); 94 | IndexWriterConfig indexWriterConfig = new IndexWriterConfig(Version.LUCENE_44, new WhitespaceAnalyzer(Version.LUCENE_44)); 95 | spellChecker.indexDictionary(dictCache.getUnchecked(field), indexWriterConfig, false); 96 | return spellChecker; 97 | } 98 | } 99 | ); 100 | 101 | lookupCache = CacheBuilder.newBuilder().build( 102 | new CacheLoader() { 103 | @Override 104 | public FSTCompletionLookup load(String field) throws Exception { 105 | FSTCompletionLookup lookup = new FSTCompletionLookup(); 106 | lookup.build(dictCache.getUnchecked(field)); 107 | return lookup; 108 | } 109 | } 110 | ); 111 | 112 | analyzingSuggesterCache = CacheBuilder.newBuilder().build( 113 | new AbstractCacheLoaderSuggester.CacheLoaderAnalyzingSuggester(mapperService, analysisService, dictCache)); 114 | 115 | fuzzySuggesterCache = CacheBuilder.newBuilder().build( 116 | new AbstractCacheLoaderSuggester.CacheLoaderFuzzySuggester(mapperService, analysisService, dictCache)); 117 | } 118 | 119 | public ShardSuggestRefreshResponse refresh(ShardSuggestRefreshRequest shardSuggestRefreshRequest) { 120 | String field = shardSuggestRefreshRequest.field(); 121 | if (!Strings.hasLength(field)) { 122 | update(); 123 | } else { 124 | resetIndexReader(); 125 | 126 | HighFrequencyDictionary dict = dictCache.getIfPresent(field); 127 | if (dict != null) dictCache.refresh(field); 128 | 129 | RAMDirectory ramDirectory = ramDirectoryCache.getIfPresent(field); 130 | if (ramDirectory != null) { 131 | ramDirectory.close(); 132 | ramDirectoryCache.invalidate(field); 133 | } 134 | 135 | SpellChecker spellChecker = spellCheckerCache.getIfPresent(field); 136 | if (spellChecker != null) { 137 | spellCheckerCache.refresh(field); 138 | try { 139 | spellChecker.close(); 140 | } catch (IOException e) { 141 | logger.error("Could not close spellchecker in indexshard [{}] for field [{}]", e, indexShard, field); 142 | } 143 | } 144 | 145 | FSTCompletionLookup lookup = lookupCache.getIfPresent(field); 146 | if (lookup != null) lookupCache.refresh(field); 147 | 148 | for (FieldType fieldType : analyzingSuggesterCache.asMap().keySet()) { 149 | if (fieldType.field().equals(shardSuggestRefreshRequest.field())) { 150 | analyzingSuggesterCache.refresh(fieldType); 151 | } 152 | } 153 | 154 | for (FieldType fieldType : fuzzySuggesterCache.asMap().keySet()) { 155 | if (fieldType.field().equals(shardSuggestRefreshRequest.field())) { 156 | fuzzySuggesterCache.refresh(fieldType); 157 | } 158 | } 159 | } 160 | 161 | return new ShardSuggestRefreshResponse(shardId.index().name(), shardId.id()); 162 | } 163 | 164 | public void shutDown() { 165 | resetIndexReader(); 166 | dictCache.invalidateAll(); 167 | for (Map.Entry entry : spellCheckerCache.asMap().entrySet()) { 168 | try { 169 | ramDirectoryCache.getUnchecked(entry.getKey()).close(); 170 | entry.getValue().close(); 171 | } catch (IOException e) { 172 | logger.error("Could not close spellchecker in indexshard [{}] for field [{}]", e, indexShard, entry.getKey()); 173 | } 174 | } 175 | spellCheckerCache.invalidateAll(); 176 | ramDirectoryCache.invalidateAll(); 177 | lookupCache.invalidateAll(); 178 | analyzingSuggesterCache.invalidateAll(); 179 | fuzzySuggesterCache.invalidateAll(); 180 | } 181 | 182 | public void update() { 183 | resetIndexReader(); 184 | 185 | for (String field : dictCache.asMap().keySet()) { 186 | dictCache.refresh(field); 187 | } 188 | 189 | try { 190 | for (String field : spellCheckerCache.asMap().keySet()) { 191 | SpellChecker oldSpellchecker = spellCheckerCache.getUnchecked(field); 192 | RAMDirectory oldRamDirectory = ramDirectoryCache.getUnchecked(field); 193 | ramDirectoryCache.refresh(field); 194 | spellCheckerCache.refresh(field); 195 | oldRamDirectory.close(); 196 | oldSpellchecker.close(); 197 | } 198 | } catch (IOException e ) { 199 | logger.error("Error refreshing spell checker cache [{}]", e, shardId); 200 | } 201 | 202 | for (String field : lookupCache.asMap().keySet()) { 203 | lookupCache.refresh(field); 204 | } 205 | 206 | for (FieldType fieldType : analyzingSuggesterCache.asMap().keySet()) { 207 | analyzingSuggesterCache.refresh(fieldType); 208 | } 209 | 210 | for (FieldType fieldType : fuzzySuggesterCache.asMap().keySet()) { 211 | fuzzySuggesterCache.refresh(fieldType); 212 | } 213 | } 214 | 215 | public ShardSuggestResponse suggest(ShardSuggestRequest shardSuggestRequest) { 216 | List suggestions; 217 | try { 218 | suggestions = Lists.newArrayList(getSuggestions(shardSuggestRequest)); 219 | } catch (IOException e) { 220 | throw new ElasticsearchException("Error getting suggestions", e); 221 | } 222 | return new ShardSuggestResponse(shardId.index().name(), shardId.id(), suggestions); 223 | } 224 | 225 | private Collection getSimilarSuggestions(ShardSuggestRequest shardSuggestRequest) { 226 | String field = shardSuggestRequest.field(); 227 | String term = shardSuggestRequest.term(); 228 | Integer limit = shardSuggestRequest.size(); 229 | Float similarity = shardSuggestRequest.similarity(); 230 | 231 | try { 232 | String[] suggestSimilar = spellCheckerCache.getUnchecked(field).suggestSimilar(term, limit, similarity); 233 | return Arrays.asList(suggestSimilar); 234 | } catch (IOException e) { 235 | logger.error("Error getting spellchecker suggestions for shard [{}] field [{}] term [{}] limit [{}] similarity [{}]", e, shardId, field, term, limit, similarity); 236 | } 237 | 238 | return Collections.emptyList(); 239 | } 240 | 241 | private Collection getSuggestions(ShardSuggestRequest shardSuggestRequest) throws IOException { 242 | List lookupResults = Lists.newArrayList(); 243 | if ("full".equals(shardSuggestRequest.suggestType())) { 244 | AnalyzingSuggester analyzingSuggester = analyzingSuggesterCache.getUnchecked(new FieldType(shardSuggestRequest)); 245 | lookupResults.addAll(analyzingSuggester.lookup(shardSuggestRequest.term(), false, shardSuggestRequest.size())); 246 | } else if ("fuzzy".equals(shardSuggestRequest.suggestType())) { 247 | lookupResults.addAll(fuzzySuggesterCache.getUnchecked(new FieldType(shardSuggestRequest)) 248 | .lookup(shardSuggestRequest.term(), false, shardSuggestRequest.size())); 249 | 250 | } else { 251 | lookupResults.addAll(lookupCache.getUnchecked(shardSuggestRequest.field()) 252 | .lookup(shardSuggestRequest.term(), true, shardSuggestRequest.size() + 1)); 253 | Collection suggestions = Collections2.transform(lookupResults, new LookupResultToStringFunction()); 254 | 255 | float similarity = shardSuggestRequest.similarity(); 256 | if (similarity < 1.0f && suggestions.size() < shardSuggestRequest.size()) { 257 | suggestions = Lists.newArrayList(suggestions); 258 | suggestions.addAll(getSimilarSuggestions(shardSuggestRequest)); 259 | } 260 | 261 | return suggestions; 262 | } 263 | 264 | return Collections2.transform(lookupResults, new LookupResultToStringFunction()); 265 | } 266 | 267 | private class LookupResultToStringFunction implements Function { 268 | @Override 269 | public String apply(LookupResult result) { 270 | return result.key.toString(); 271 | } 272 | } 273 | 274 | public void resetIndexReader() { 275 | IndexReader currentIndexReader = null; 276 | if (indexShard.state() == IndexShardState.STARTED) { 277 | try (Engine.Searcher currentIndexSearcher = indexShard.acquireSearcher( "suggest" )) { 278 | currentIndexReader = currentIndexSearcher.reader(); 279 | } 280 | } 281 | 282 | // if this index reader is not used in the current index searcher, we need to decrease the old refcount 283 | if (indexReader != null && indexReader.getRefCount() > 0 && !indexReader.equals(currentIndexReader)) { 284 | try { 285 | indexReader.decRef(); 286 | } catch (IOException e) { 287 | logger.error("Error decreasing indexreader ref count [{}] of shard [{}]", e, indexReader.getRefCount(), shardId); 288 | } 289 | } 290 | 291 | indexReader = null; 292 | } 293 | 294 | public ShardSuggestStatisticsResponse getStatistics() { 295 | ShardSuggestStatisticsResponse shardSuggestStatisticsResponse = new ShardSuggestStatisticsResponse(shardId()); 296 | 297 | for (FieldType fieldType : analyzingSuggesterCache.asMap().keySet()) { 298 | long sizeInBytes = analyzingSuggesterCache.getIfPresent(fieldType).ramBytesUsed(); 299 | FstStats.FstIndexShardStats fstIndexShardStats = new FstStats.FstIndexShardStats(shardId, "analyzingsuggester", fieldType, sizeInBytes); 300 | shardSuggestStatisticsResponse.getFstIndexShardStats().add(fstIndexShardStats); 301 | } 302 | 303 | for (FieldType fieldType : fuzzySuggesterCache.asMap().keySet()) { 304 | long sizeInBytes = fuzzySuggesterCache.getIfPresent(fieldType).ramBytesUsed(); 305 | FstStats.FstIndexShardStats fstIndexShardStats = new FstStats.FstIndexShardStats(shardId, "fuzzysuggester", fieldType, sizeInBytes); 306 | shardSuggestStatisticsResponse.getFstIndexShardStats().add(fstIndexShardStats); 307 | } 308 | 309 | return shardSuggestStatisticsResponse; 310 | } 311 | 312 | // this does not look thread safe and nice... 313 | private IndexReader createOrGetIndexReader() { 314 | try { 315 | if (indexReader == null) { 316 | lock.lock(); 317 | if (indexReader == null) { 318 | try (Engine.Searcher indexSearcher = indexShard.acquireSearcher( "suggest" )) { 319 | indexReader = indexSearcher.reader(); 320 | // logger.info("1 shard {} : ref count {}", shardId, indexReader.getRefCount()); 321 | } 322 | 323 | // If an indexreader closes, we have to refresh all our data structures! 324 | indexReader.addReaderClosedListener(new IndexReader.ReaderClosedListener() { 325 | @Override 326 | public void onClose(IndexReader reader) { 327 | update(); 328 | } 329 | }); 330 | } 331 | } 332 | } finally { 333 | if (lock.isLocked()) { 334 | lock.unlock(); 335 | } 336 | } 337 | 338 | return indexReader; 339 | } 340 | 341 | public static class FieldType implements Streamable, Serializable, ToXContent { 342 | 343 | private String field; 344 | private List types = Lists.newArrayList(); 345 | private String queryAnalyzer; 346 | private String indexAnalyzer; 347 | private boolean preservePositionIncrements = true; 348 | 349 | public FieldType() {} 350 | 351 | public FieldType(ShardSuggestRequest shardSuggestRequest) { 352 | this.field = shardSuggestRequest.field(); 353 | this.types = Arrays.asList(shardSuggestRequest.types()); 354 | this.queryAnalyzer = shardSuggestRequest.queryAnalyzer(); 355 | this.indexAnalyzer = shardSuggestRequest.indexAnalyzer(); 356 | this.preservePositionIncrements = shardSuggestRequest.preservePositionIncrements(); 357 | } 358 | 359 | public String field() { 360 | return field; 361 | } 362 | 363 | public String[] types() { 364 | return types.toArray(new String[]{}); 365 | } 366 | 367 | public String queryAnalyzer() { 368 | return queryAnalyzer; 369 | } 370 | 371 | public String indexAnalyzer() { 372 | return indexAnalyzer; 373 | } 374 | 375 | public boolean preservePositionIncrements() { 376 | return preservePositionIncrements; 377 | } 378 | 379 | @Override 380 | public boolean equals(Object obj) { 381 | if (obj == null) { 382 | return false; 383 | } 384 | if (getClass() != obj.getClass()) { 385 | return false; 386 | } 387 | final FieldType other = (FieldType) obj; 388 | 389 | return Objects.equal(this.field(), other.field()) 390 | && Objects.equal(this.queryAnalyzer(), other.queryAnalyzer()) 391 | && Objects.equal(this.indexAnalyzer(), other.indexAnalyzer()) 392 | && Objects.equal(this.types, other.types) 393 | && Objects.equal(this.preservePositionIncrements(), other.preservePositionIncrements()); 394 | } 395 | 396 | @Override 397 | public int hashCode() { 398 | int hashCode = this.field().hashCode(); 399 | hashCode += this.types.hashCode(); 400 | if (this.queryAnalyzer != null) hashCode += this.queryAnalyzer.hashCode(); 401 | if (this.indexAnalyzer != null) hashCode += this.indexAnalyzer.hashCode(); 402 | hashCode += Boolean.valueOf(preservePositionIncrements).hashCode(); 403 | 404 | return hashCode; 405 | } 406 | 407 | @Override 408 | public String toString() { 409 | ToStringBuilder toStringBuilder = new ToStringBuilder(this.getClass()) 410 | .add("field", this.field()); 411 | 412 | toStringBuilder.add("preservePositionIncrements", this.preservePositionIncrements); 413 | if (queryAnalyzer != null && queryAnalyzer.equals(indexAnalyzer)) { 414 | toStringBuilder.add("analyzer", this.queryAnalyzer); 415 | } else { 416 | if (queryAnalyzer != null) { 417 | toStringBuilder.add("queryAnalyzer", queryAnalyzer); 418 | } 419 | if (indexAnalyzer != null) { 420 | toStringBuilder.add("indexAnalyzer", indexAnalyzer); 421 | } 422 | } 423 | 424 | if (types.size() > 0) { 425 | toStringBuilder.add("types", Joiner.on("-").join(types)); 426 | } 427 | 428 | return toStringBuilder.toString(); 429 | } 430 | 431 | @Override 432 | public void readFrom(StreamInput in) throws IOException { 433 | field = in.readString(); 434 | queryAnalyzer = in.readOptionalString(); 435 | indexAnalyzer = in.readOptionalString(); 436 | types = (List) in.readGenericValue(); 437 | preservePositionIncrements = in.readBoolean(); 438 | } 439 | 440 | @Override 441 | public void writeTo(StreamOutput out) throws IOException { 442 | out.writeString(field); 443 | out.writeOptionalString(queryAnalyzer); 444 | out.writeOptionalString(indexAnalyzer); 445 | out.writeGenericValue(types); 446 | out.writeBoolean(preservePositionIncrements); 447 | } 448 | 449 | @Override 450 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 451 | //builder.startObject(field); 452 | builder.field("field", field); 453 | if (queryAnalyzer != null && queryAnalyzer.equals(indexAnalyzer)) { 454 | builder.field("analyzer", this.queryAnalyzer); 455 | } else { 456 | if (queryAnalyzer != null) builder.field("queryAnalyzer", queryAnalyzer); 457 | if (indexAnalyzer != null) builder.field("indexAnalyzer", indexAnalyzer); 458 | } 459 | if (!preservePositionIncrements) builder.field("preservePositionIncrements", preservePositionIncrements); 460 | if (types.size() > 0) builder.field("types", types()); 461 | //builder.endObject(); 462 | return builder; 463 | } 464 | } 465 | } 466 | -------------------------------------------------------------------------------- /src/main/java/de/spinscale/elasticsearch/service/suggest/SuggestService.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.service.suggest; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshRequest; 4 | import de.spinscale.elasticsearch.action.suggest.refresh.TransportSuggestRefreshAction; 5 | import org.elasticsearch.ElasticsearchException; 6 | import org.elasticsearch.cluster.ClusterService; 7 | import org.elasticsearch.cluster.node.DiscoveryNode; 8 | import org.elasticsearch.common.Nullable; 9 | import org.elasticsearch.common.StopWatch; 10 | import org.elasticsearch.common.component.AbstractLifecycleComponent; 11 | import org.elasticsearch.common.component.Lifecycle; 12 | import org.elasticsearch.common.inject.Inject; 13 | import org.elasticsearch.common.settings.Settings; 14 | import org.elasticsearch.common.unit.TimeValue; 15 | import org.elasticsearch.common.util.concurrent.EsExecutors; 16 | import org.elasticsearch.index.service.IndexService; 17 | import org.elasticsearch.index.shard.ShardId; 18 | import org.elasticsearch.index.shard.service.IndexShard; 19 | import org.elasticsearch.indices.IndicesLifecycle; 20 | import org.elasticsearch.indices.IndicesService; 21 | 22 | import java.util.Iterator; 23 | 24 | public class SuggestService extends AbstractLifecycleComponent { 25 | 26 | private final TimeValue suggestRefreshInterval; 27 | private final boolean suggestRefreshDisabled; 28 | private volatile Thread suggestUpdaterThread; 29 | private volatile boolean closed; 30 | private final TransportSuggestRefreshAction suggestRefreshAction; 31 | private final ClusterService clusterService; 32 | private final IndicesService indicesService; 33 | 34 | @Inject public SuggestService(Settings settings, TransportSuggestRefreshAction suggestRefreshAction, 35 | ClusterService clusterService, IndicesService indicesService) { 36 | super(settings); 37 | this.suggestRefreshAction = suggestRefreshAction; 38 | this.clusterService = clusterService; 39 | this.indicesService = indicesService; 40 | suggestRefreshDisabled = settings.getAsBoolean("suggest.refresh_disabled", false); 41 | suggestRefreshInterval = settings.getAsTime("suggest.refresh_interval", TimeValue.timeValueMinutes(10)); 42 | } 43 | 44 | @Override 45 | protected void doStart() throws ElasticsearchException { 46 | if (suggestRefreshDisabled) { 47 | logger.info("Suggest component started with out refreshing automatically"); 48 | } else { 49 | suggestUpdaterThread = EsExecutors.daemonThreadFactory(settings, "suggest_updater").newThread(new SuggestUpdaterThread()); 50 | suggestUpdaterThread.start(); 51 | logger.info("Suggest component started with refresh interval [{}]", suggestRefreshInterval); 52 | } 53 | 54 | // When the instance is shut down or the index is deleted 55 | indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() { 56 | @Override 57 | public void beforeIndexClosed(IndexService indexService) { 58 | for (Iterator shardServiceIterator = indexService.iterator(); shardServiceIterator.hasNext(); ) { 59 | IndexShard indexShard = shardServiceIterator.next(); 60 | ShardSuggestService suggestShardService = indexService.shardInjectorSafe(indexShard.shardId().id()).getInstance(ShardSuggestService.class); 61 | suggestShardService.shutDown(); 62 | } 63 | } 64 | }); 65 | 66 | // when the shard is deleted (or moved to another cluster instance) 67 | // using this in the above case fails, because i cannot get the indexService anymore at beforeIndexShardClosed() 68 | indicesService.indicesLifecycle().addListener(new IndicesLifecycle.Listener() { 69 | @Override 70 | public void beforeIndexShardClosed(ShardId shardId, @Nullable IndexShard indexShard) { 71 | IndexService indexService = indicesService.indexService(shardId.index().name()); 72 | if (indexService != null) { 73 | ShardSuggestService suggestShardService = indexService.shardInjectorSafe(shardId.id()).getInstance(ShardSuggestService.class); 74 | suggestShardService.shutDown(); 75 | } 76 | } 77 | }); 78 | } 79 | 80 | @Override 81 | protected void doClose() throws ElasticsearchException { 82 | if (closed) { 83 | return; 84 | } 85 | closed = true; 86 | 87 | if (suggestUpdaterThread != null) { 88 | suggestUpdaterThread.interrupt(); 89 | } 90 | logger.info("Suggest component stopped"); 91 | } 92 | 93 | @Override 94 | protected void doStop() throws ElasticsearchException {} 95 | 96 | public class SuggestUpdaterThread implements Runnable { 97 | @Override 98 | public void run() { 99 | while (!closed) { 100 | DiscoveryNode node = clusterService.localNode(); 101 | boolean isClusterStarted = clusterService.lifecycleState().equals(Lifecycle.State.STARTED); 102 | 103 | if (isClusterStarted && node != null && node.isMasterNode()) { 104 | StopWatch sw = new StopWatch().start(); 105 | suggestRefreshAction.execute(new SuggestRefreshRequest()).actionGet(); 106 | logger.info("Suggest update took [{}], next update in [{}]", sw.stop().totalTime(), suggestRefreshInterval); 107 | } else { 108 | if (node != null) { 109 | logger.debug("[{}]/[{}] is not master node, not triggering update", node.getId(), node.getName()); 110 | } 111 | } 112 | 113 | try { 114 | Thread.sleep(suggestRefreshInterval.millis()); 115 | } catch (InterruptedException e1) { 116 | continue; 117 | } 118 | } 119 | } 120 | } 121 | } 122 | -------------------------------------------------------------------------------- /src/main/resources/es-plugin.properties: -------------------------------------------------------------------------------- 1 | plugin=de.spinscale.elasticsearch.plugin.suggest.SuggestPlugin 2 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/AbstractSuggestTest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 4 | import de.spinscale.elasticsearch.plugin.suggest.SuggestPlugin; 5 | import org.apache.commons.io.IOUtils; 6 | import org.apache.commons.lang.RandomStringUtils; 7 | import org.elasticsearch.action.admin.indices.create.CreateIndexResponse; 8 | import org.elasticsearch.action.admin.indices.settings.put.UpdateSettingsResponse; 9 | import org.elasticsearch.action.bulk.BulkRequest; 10 | import org.elasticsearch.action.bulk.BulkResponse; 11 | import org.elasticsearch.action.index.IndexRequest; 12 | import org.elasticsearch.common.Strings; 13 | import org.elasticsearch.common.collect.Lists; 14 | import org.elasticsearch.common.collect.Maps; 15 | import org.elasticsearch.common.settings.Settings; 16 | import org.elasticsearch.test.ElasticsearchIntegrationTest; 17 | import org.elasticsearch.test.junit.annotations.TestLogging; 18 | import org.junit.Before; 19 | import org.junit.Ignore; 20 | import org.junit.Test; 21 | 22 | import java.io.IOException; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | import java.util.Locale; 26 | import java.util.Map; 27 | 28 | import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; 29 | import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; 30 | import static org.hamcrest.Matchers.*; 31 | 32 | @TestLogging("_root:info") 33 | @Ignore 34 | public abstract class AbstractSuggestTest extends ElasticsearchIntegrationTest { 35 | 36 | public final String index = randomAsciiOfLength(10).toLowerCase(Locale.ROOT); 37 | public final String type = randomAsciiOfLength(10).toLowerCase(Locale.ROOT); 38 | 39 | @Before 40 | public void createIndex() throws Exception { 41 | createIndexWithProductsMapping(index); 42 | } 43 | 44 | abstract public List getSuggestions(SuggestionQuery suggestionQuery) throws Exception; 45 | abstract public void refreshAllSuggesters() throws Exception; 46 | abstract public void refreshIndexSuggesters(String index) throws Exception; 47 | abstract public void refreshFieldSuggesters(String index, String field) throws Exception; 48 | abstract public FstStats getStatistics() throws Exception; 49 | 50 | @Override 51 | protected Settings nodeSettings(int nodeOrdinal) { 52 | return settingsBuilder() 53 | .put(super.nodeSettings(nodeOrdinal)) 54 | .put("plugin.types", SuggestPlugin.class.getName()) 55 | .build(); 56 | } 57 | 58 | @Test 59 | public void testThatSimpleSuggestionWorks() throws Exception { 60 | List> products = createProducts("ProductName", "foo", "foob", "foobar", "boof"); 61 | indexProducts(products); 62 | 63 | List suggestions = getSuggestions("ProductName.suggest", "foo", 10); 64 | assertSuggestions(suggestions, "foo", "foob", "foobar"); 65 | } 66 | 67 | @Test 68 | public void testThatAllFieldSuggestionsWorks() throws Exception { 69 | List> products = createProducts("ProductName", "foo", "foob", "foobar", "boof"); 70 | indexProducts(products); 71 | 72 | List suggestions = getSuggestions("_all", "foo", 10); 73 | assertSuggestions(suggestions, "foo", "foob", "foobar"); 74 | } 75 | 76 | @Test 77 | public void testThatSimpleSuggestionShouldSupportLimit() throws Exception { 78 | List> products = createProducts("ProductName", "foo", "fooba", "foobar"); 79 | indexProducts(products); 80 | 81 | List suggestions = getSuggestions("ProductName.suggest", "foo", 2); 82 | assertSuggestions(suggestions, "foo", "fooba"); 83 | } 84 | 85 | @Test 86 | public void testThatSimpleSuggestionShouldSupportLimitWithConcreteWord() throws Exception { 87 | List> products = createProducts("ProductName", "foo", "fooba", "foobar"); 88 | indexProducts(products); 89 | 90 | List suggestions = getSuggestions("ProductName.suggest", "foo", 2); 91 | assertSuggestions(suggestions, "foo", "fooba"); 92 | } 93 | 94 | @Test 95 | public void testThatSuggestionShouldNotContainDuplicates() throws Exception { 96 | List> products = createProducts("ProductName", "foo", "foo", "foob"); 97 | indexProducts(products); 98 | 99 | List suggestions = getSuggestions("ProductName.suggest", "foo", 10); 100 | assertSuggestions(suggestions, "foo", "foob"); 101 | } 102 | 103 | @Test 104 | public void testThatSuggestionShouldWorkOnDifferentFields() throws Exception { 105 | List> products = createProducts("ProductName", "Kochjacke Pute", "Kochjacke Henne", "Kochjacke Hahn"); 106 | products.get(0).put("Description", "Kochjacke Pute"); 107 | products.get(1).put("Description", "Kochjacke Henne"); 108 | products.get(2).put("Description", "Kochjacke Hahn"); 109 | indexProducts(products); 110 | 111 | List suggestions = getSuggestions("ProductName.suggest", "kochjacke", 10); 112 | assertSuggestions(suggestions, "kochjacke", "kochjacke hahn", "kochjacke henne", "kochjacke pute"); 113 | 114 | suggestions = getSuggestions("Description", "Kochjacke", 10); 115 | assertSuggestions(suggestions, "Kochjacke Hahn", "Kochjacke Henne", "Kochjacke Pute"); 116 | } 117 | 118 | @Test 119 | public void testThatSuggestionShouldWorkWithWhitespaces() throws Exception { 120 | List> products = createProducts("ProductName", "Kochjacke Paul", "Kochjacke Pauline", 121 | "Kochjacke Paulinea"); 122 | indexProducts(products); 123 | 124 | List suggestions = getSuggestions("ProductName.suggest", "kochja", 10); 125 | assertSuggestions(suggestions, "kochjacke", "kochjacke paul", "kochjacke pauline", "kochjacke paulinea"); 126 | 127 | suggestions = getSuggestions("ProductName.suggest", "kochjacke ", 10); 128 | assertSuggestions(suggestions, "kochjacke paul", "kochjacke pauline", "kochjacke paulinea"); 129 | 130 | suggestions = getSuggestions("ProductName.suggest", "kochjacke pauline", 10); 131 | assertSuggestions(suggestions, "kochjacke pauline", "kochjacke paulinea"); 132 | } 133 | 134 | @Test 135 | public void testThatSuggestionWithShingleWorksAfterUpdate() throws Exception { 136 | List> products = createProducts("ProductName", "Kochjacke Paul", "Kochjacke Pauline", 137 | "Kochjacke Paulinator"); 138 | indexProducts(products); 139 | 140 | List suggestions = getSuggestions("ProductName.suggest", "kochjacke", 10); 141 | assertSuggestions(suggestions, "kochjacke", "kochjacke paul", "kochjacke paulinator", "kochjacke pauline"); 142 | 143 | products = createProducts(1); 144 | products.get(0).put("ProductName", "Kochjacke PaulinPanzer"); 145 | indexProducts(products); 146 | refresh(); 147 | refreshAllSuggesters(); 148 | 149 | suggestions = getSuggestions("ProductName.suggest", "kochjacke paulin", 10); 150 | assertSuggestions(suggestions, "kochjacke paulinator", "kochjacke pauline", "kochjacke paulinpanzer"); 151 | 152 | cleanIndex(); 153 | refreshAllSuggesters(); 154 | suggestions = getSuggestions("ProductName.suggest", "kochjacke paulin", 10); 155 | assertThat(suggestions.size(), is(0)); 156 | } 157 | 158 | @Test 159 | public void testThatSuggestionWorksWithSimilarity() throws Exception { 160 | List> products = createProducts("ProductName", "kochjacke bla", "kochjacke blubb", 161 | "kochjacke blibb", "kochjacke paul"); 162 | indexProducts(products); 163 | 164 | List suggestions = getSuggestions("ProductName.suggest", "kochajcke", 10, 0.75f); 165 | assertThat(suggestions, hasSize(1)); 166 | assertThat(suggestions, contains("kochjacke")); 167 | } 168 | 169 | @Ignore("This test is useless in this setup, as it may return better/more data than expected and therefore fails") 170 | @Test 171 | public void testThatRefreshingPerIndexWorks() throws Exception { 172 | // having routing ensures that all the data is written into one shard 173 | // this ensures that when adding the second product it is added to the same shard 174 | // if it is not added to the same shard, refreshing might not work as expected 175 | // in case the data is added to a shard where there was no data in before, it is added immediately to the 176 | // suggestions, this means more results than expected might be returned in the last line of this test 177 | createIndexWithProductsMapping("secondproductsindex"); 178 | 179 | List> products = createProducts("ProductName", "autoreifen", "autorad"); 180 | indexProducts(products, index, "someRoutingKey"); 181 | indexProducts(products, "secondproductsindex", "someRoutingKey"); 182 | 183 | // get suggestions from both indexes to create fst structures 184 | SuggestionQuery productsQuery = new SuggestionQuery(index, type, "ProductName.suggest", "auto"); 185 | SuggestionQuery secondProductsIndexQuery = new SuggestionQuery("secondproductsindex", type, "ProductName.suggest", "auto"); 186 | getSuggestions(productsQuery); 187 | getSuggestions(secondProductsIndexQuery); 188 | 189 | // index another product 190 | List> newProducts = createProducts("ProductName", "automatik"); 191 | indexProducts(newProducts, index, "someRoutingKey"); 192 | indexProducts(newProducts, "secondproductsindex", "someRoutingKey"); 193 | 194 | refreshIndexSuggesters(index); 195 | 196 | assertSuggestions(productsQuery, "automatik", "autorad", "autoreifen"); 197 | assertSuggestions(secondProductsIndexQuery, "autorad", "autoreifen"); 198 | } 199 | 200 | @Ignore("This test is useless in this setup, as it may return better/more data than expected and therefore fails") 201 | @Test 202 | public void testThatRefreshingPerIndexFieldWorks() throws Exception { 203 | // having routing ensures that all the data is written into one shard 204 | // this ensures that when adding the second product it is added to the same shard 205 | // if it is not added to the same shard, refreshing might not work as expected 206 | // in case the data is added to a shard where there was no data in before, it is added immediately to the 207 | // suggestions, this means more results than expected might be returned in the last line of this test 208 | List> products = createProducts("ProductName", "autoreifen", "autorad"); 209 | indexProducts(products, index, "someRoutingKey"); 210 | 211 | SuggestionQuery suggestionQuery = new SuggestionQuery(index, type, "ProductName.suggest", "auto"); 212 | SuggestionQuery lowerCaseQuery = new SuggestionQuery(index, type, "ProductName.lowercase", "auto"); 213 | getSuggestions(suggestionQuery); 214 | getSuggestions(lowerCaseQuery); 215 | 216 | List> newProducts = createProducts("ProductName", "automatik"); 217 | indexProducts(newProducts, index, "someRoutingKey"); 218 | 219 | refreshFieldSuggesters(index, "ProductName.suggest"); 220 | 221 | assertSuggestions(suggestionQuery, "automatik", "autorad", "autoreifen"); 222 | assertSuggestions(lowerCaseQuery, "autorad", "autoreifen"); 223 | } 224 | 225 | @Test 226 | public void testThatAnalyzingSuggesterWorks() throws Exception { 227 | List> products = createProducts("ProductName", "BMW 318", "BMW 528", "BMW M3", 228 | "the BMW 320", "VW Jetta"); 229 | indexProducts(products); 230 | 231 | SuggestionQuery query = new SuggestionQuery(index, type, "ProductName.keyword", "b") 232 | .suggestType("full").analyzer("simple").size(10); 233 | List suggestions = getSuggestions(query); 234 | 235 | assertSuggestions(suggestions, "BMW 318", "BMW 528", "BMW M3"); 236 | } 237 | 238 | @Test 239 | public void testThatAnalyzingSuggesterSupportsStopWords() throws Exception { 240 | List> products = createProducts("ProductName", "BMW 318", "BMW 528", "BMW M3", 241 | "the BMW 320", "VW Jetta"); 242 | indexProducts(products); 243 | 244 | SuggestionQuery query = new SuggestionQuery(index, type, "ProductName.keyword", "b") 245 | .suggestType("full").indexAnalyzer("stop").queryAnalyzer("stop") 246 | .preservePositionIncrements(false).size(10); 247 | List suggestions = getSuggestions(query); 248 | 249 | assertSuggestions(suggestions, "BMW 318", "BMW 528", "BMW M3", "the BMW 320"); 250 | } 251 | 252 | @Test 253 | public void testThatFuzzySuggesterWorks() throws Exception { 254 | List> products = createProducts("ProductName", "BMW 318", "BMW 528", "BMW M3", 255 | "the BMW 320", "VW Jetta"); 256 | indexProducts(products); 257 | 258 | SuggestionQuery query = new SuggestionQuery(index, type, "ProductName.keyword", "bwm") 259 | .suggestType("fuzzy").analyzer("standard").size(10); 260 | List suggestions = getSuggestions(query); 261 | 262 | assertSuggestions(suggestions, "BMW 318", "BMW 528", "BMW M3"); 263 | } 264 | 265 | @Test 266 | public void testThatFlushForcesReloadingOfAllFieldsWithoutErrors() throws Exception { 267 | List> products = createProducts("ProductName", "BMW 318"); 268 | indexProducts(products); 269 | 270 | SuggestionQuery query = new SuggestionQuery(index, type, "ProductName.keyword", "bwm").suggestType("full"); 271 | getSuggestions(query); 272 | 273 | // add data to index and flush 274 | // indexProducts(createProducts("ProductName", "BMW 320"), node); 275 | client().prepareIndex(index, type, "foo").setSource(createProducts(1).get(0)).execute().actionGet(); 276 | client().admin().indices().prepareFlush(index).execute().actionGet(); 277 | getSuggestions(query); 278 | } 279 | 280 | @Test 281 | public void gettingStatisticsShouldWork() throws Exception { 282 | // needed to make sure that we hit the already queried shards for stats, the other are empty 283 | Settings settings = settingsBuilder() 284 | .put("index.number_of_replicas", 0) 285 | .build(); 286 | UpdateSettingsResponse response = client().admin().indices().prepareUpdateSettings(index).setSettings(settings).get(); 287 | assertThat(response.isAcknowledged(), is(true)); 288 | 289 | List> products = createProducts("ProductName", 290 | "BMW 318", "BMW 528", "BMW M3", "the BMW 320", "VW Jetta"); 291 | indexProducts(products); 292 | 293 | FstStats emptyFstStats = getStatistics(); 294 | assertThat(emptyFstStats.getStats(), hasSize(0)); 295 | assertThat(getFstSizeSum(emptyFstStats), equalTo(0L)); 296 | 297 | SuggestionQuery query = new SuggestionQuery(index, type, "ProductName.keyword", "b") 298 | .suggestType("full").analyzer("stop").size(10); 299 | List suggestions = getSuggestions(query); 300 | assertSuggestions(suggestions, "BMW 318", "BMW 528", "BMW M3", "the BMW 320"); 301 | 302 | FstStats filledFstStats = getStatistics(); 303 | assertThat(filledFstStats.getStats(), hasSize(greaterThanOrEqualTo(1))); 304 | 305 | List allStats = Lists.newArrayList(filledFstStats.getStats()); 306 | assertThat(allStats.get(0).getShardId().id(), greaterThanOrEqualTo(0)); 307 | assertThat(getFstSizeSum(filledFstStats), greaterThan(0L)); 308 | } 309 | 310 | private long getFstSizeSum(FstStats fstStats) { 311 | long totalFstSize = 0; 312 | 313 | for (FstStats.FstIndexShardStats stats : fstStats.getStats()) { 314 | totalFstSize += stats.getSizeInBytes(); 315 | } 316 | 317 | return totalFstSize; 318 | } 319 | 320 | // @Test 321 | // public void performanceTest() throws Exception { 322 | // List> products = createProducts(60000); 323 | // indexProducts(products); 324 | // 325 | // System.out.println(measureSuggestTime("a")); 326 | // System.out.println(measureSuggestTime("aa")); 327 | // System.out.println(measureSuggestTime("aaa")); 328 | // System.out.println(measureSuggestTime("aaab")); 329 | // System.out.println(measureSuggestTime("aaabc")); 330 | // System.out.println(measureSuggestTime("aaabcd")); 331 | // } 332 | // 333 | // private long measureSuggestTime(String search) throws Exception { 334 | // long start = System.currentTimeMillis(); 335 | // getSuggestions("ProductName.suggest", "aaa", 10); 336 | // long end = System.currentTimeMillis(); 337 | // 338 | // return end - start; 339 | // } 340 | 341 | private List getSuggestions(String field, String term, Integer size) throws Exception { 342 | return getSuggestions(new SuggestionQuery(index, type, field, term).size(size)); 343 | } 344 | 345 | private List getSuggestions(String field, String term, Integer size, Float similarity) throws Exception { 346 | return getSuggestions(new SuggestionQuery(index, type, field, term).size(size).similarity(similarity)); 347 | } 348 | 349 | private void assertSuggestions(List suggestions, String ... terms) { 350 | String msg = String.format(Locale.ROOT, "%s should have size %s, content %s", suggestions, terms.length, Arrays.asList(terms)); 351 | assertThat(msg, suggestions, hasSize(terms.length)); 352 | assertThat("Suggestions are: " + suggestions, suggestions, contains(terms)); 353 | } 354 | 355 | private void assertSuggestions(SuggestionQuery query, String ... terms) throws Exception { 356 | List suggestions = getSuggestions(query); 357 | String assertionError = String.format(Locale.ROOT, "%s for query %s should be %s", suggestions, query, Arrays.asList(terms)); 358 | assertThat(assertionError, suggestions, hasSize(terms.length)); 359 | assertThat("Suggestions are: " + suggestions, suggestions, contains(terms)); 360 | } 361 | 362 | private void cleanIndex() { 363 | client().prepareDeleteByQuery(index).setTypes("product").setQuery(matchAllQuery()).get(); 364 | } 365 | 366 | 367 | private void createIndexWithProductsMapping(String indexName) throws IOException { 368 | String settingsData = IOUtils.toString(this.getClass().getResourceAsStream("/product.json")); 369 | CreateIndexResponse createIndexResponse = client().admin().indices().prepareCreate(indexName) 370 | .setSource(settingsData).execute().actionGet(); 371 | assertThat(createIndexResponse.isAcknowledged(), is(true)); 372 | 373 | client().admin().cluster().prepareHealth(indexName).setWaitForGreenStatus().execute().actionGet(); 374 | } 375 | 376 | protected void indexProducts(List> products) throws Exception { 377 | indexProducts(products, index); 378 | } 379 | 380 | private void indexProducts(List> products, String index) throws Exception { 381 | indexProducts(products, index, null); 382 | } 383 | 384 | private void indexProducts(List> products, String index, String routing) throws Exception { 385 | long currentCount = getCurrentDocumentCount(index); 386 | BulkRequest bulkRequest = new BulkRequest(); 387 | for (Map product : products) { 388 | IndexRequest indexRequest = new IndexRequest(index, "product", (String)product.get("ProductId")); 389 | indexRequest.source(product); 390 | if (Strings.hasLength(routing)) { 391 | indexRequest.routing(routing); 392 | } 393 | bulkRequest.add(indexRequest); 394 | } 395 | bulkRequest.refresh(true); 396 | BulkResponse response = client().bulk(bulkRequest).actionGet(); 397 | if (response.hasFailures()) { 398 | fail("Error in creating products: " + response.buildFailureMessage()); 399 | } 400 | 401 | assertDocumentCountAfterIndexing(index, products.size() + currentCount); 402 | } 403 | 404 | protected List> createProducts(int count) { 405 | List> products = Lists.newArrayList(); 406 | 407 | for (int i = 0 ; i < count; i++) { 408 | Map product = Maps.newHashMap(); 409 | product.put("ProductName", RandomStringUtils.randomAlphabetic(10)); 410 | product.put("ProductId", i + "_" + RandomStringUtils.randomAlphabetic(10)); 411 | products.add(product); 412 | } 413 | 414 | return products; 415 | } 416 | 417 | private List> createProducts(String fieldName, String ... fields) { 418 | List> products = createProducts(fields.length); 419 | 420 | for (int i = 0 ; i < fields.length ; i++) { 421 | products.get(i).put(fieldName, fields[i]); 422 | } 423 | 424 | return products; 425 | } 426 | 427 | private void assertDocumentCountAfterIndexing(String index, long expectedDocumentCount) throws Exception { 428 | assertThat(getCurrentDocumentCount(index), is(expectedDocumentCount)); 429 | } 430 | 431 | private long getCurrentDocumentCount(String index) { 432 | return client().prepareCount(index).setQuery(matchAllQuery()).execute().actionGet(2000).getCount(); 433 | } 434 | 435 | } 436 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/RestSuggestActionTest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import com.ning.http.client.AsyncHttpClient; 4 | import com.ning.http.client.Response; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 6 | import org.codehaus.jackson.JsonNode; 7 | import org.codehaus.jackson.map.ObjectMapper; 8 | import org.codehaus.jackson.node.ArrayNode; 9 | import org.elasticsearch.action.admin.cluster.node.info.NodesInfoResponse; 10 | import org.elasticsearch.common.Strings; 11 | import org.elasticsearch.common.collect.Lists; 12 | import org.elasticsearch.common.settings.Settings; 13 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 14 | import org.elasticsearch.common.xcontent.XContentParser; 15 | import org.elasticsearch.common.xcontent.json.JsonXContent; 16 | import org.elasticsearch.index.shard.ShardId; 17 | import org.elasticsearch.test.ElasticsearchIntegrationTest; 18 | import org.junit.After; 19 | import org.junit.Before; 20 | import org.junit.Test; 21 | 22 | import java.io.IOException; 23 | import java.net.URLEncoder; 24 | import java.util.Iterator; 25 | import java.util.List; 26 | import java.util.Locale; 27 | import java.util.Map; 28 | 29 | import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; 30 | import static org.hamcrest.Matchers.*; 31 | 32 | @ElasticsearchIntegrationTest.ClusterScope(scope = ElasticsearchIntegrationTest.Scope.SUITE, transportClientRatio = 0.0) 33 | public class RestSuggestActionTest extends AbstractSuggestTest { 34 | 35 | private final AsyncHttpClient httpClient = new AsyncHttpClient(); 36 | private int port; 37 | 38 | @Before 39 | public void setPort() { 40 | NodesInfoResponse response = client().admin().cluster().prepareNodesInfo().setHttp(true).execute().actionGet(); 41 | port = ((InetSocketTransportAddress) response.getNodes()[0].getHttp().address().boundAddress()).address().getPort(); 42 | } 43 | 44 | @After 45 | public void closeHttpClient() { 46 | httpClient.close(); 47 | } 48 | 49 | @Override 50 | protected Settings nodeSettings(int nodeOrdinal) { 51 | return settingsBuilder().put(super.nodeSettings(nodeOrdinal)) 52 | .put("http.jsonp.enable", true) 53 | .build(); 54 | } 55 | 56 | @Test 57 | public void testThatSuggestionsShouldWorkWithCallbackAndGetRequestParameter() throws Exception { 58 | List> products = createProducts(4); 59 | products.get(0).put("ProductName", "foo"); 60 | products.get(1).put("ProductName", "foob"); 61 | products.get(2).put("ProductName", "foobar"); 62 | 63 | indexProducts(products); 64 | refreshAllSuggesters(); 65 | 66 | String json = String.format(Locale.ROOT, "{ \"field\": \"%s\", \"term\": \"%s\" }", "ProductName.suggest", "foobar"); 67 | String query = URLEncoder.encode(json, "UTF8"); 68 | String queryString = "callback=mycallback&source=" + query; 69 | String response = httpClient.prepareGet("http://localhost:" + port + "/"+ index +"/product/__suggest?" + queryString). 70 | execute().get().getResponseBody(); 71 | assertThat(response, startsWith("mycallback({\"_shards\":{\"total\"")); 72 | assertThat(response, endsWith("\"suggestions\":[\"foobar\"]});")); 73 | } 74 | 75 | @Override 76 | public List getSuggestions(SuggestionQuery suggestionQuery) throws Exception { 77 | String json = createJSONQuery(suggestionQuery); 78 | 79 | String url = "http://localhost:" + port + "/" + suggestionQuery.index + "/" + suggestionQuery.type + "/__suggest"; 80 | Response r = httpClient.preparePost(url).setBody(json).execute().get(); 81 | assertThat(r.getStatusCode(), is(200)); 82 | assertThatResponseHasNoShardFailures(r); 83 | 84 | // System.out.println("REQ : " + json); 85 | // System.out.println("RESP: " + r.getResponseBody()); 86 | 87 | return getSuggestionsFromResponse(r.getResponseBody()); 88 | } 89 | 90 | private void assertThatResponseHasNoShardFailures(Response r) throws IOException { 91 | XContentParser parser = JsonXContent.jsonXContent.createParser(r.getResponseBody()); 92 | Map jsonResponse = parser.mapAndClose(); 93 | assertThat(jsonResponse, hasKey("_shards")); 94 | Map shardResponse = (Map) jsonResponse.get("_shards"); 95 | assertThat(shardResponse, not(hasKey("failures"))); 96 | } 97 | 98 | @Override 99 | public void refreshAllSuggesters() throws Exception { 100 | Response r = httpClient.preparePost("http://localhost:" + port + "/__suggestRefresh").execute().get(); 101 | assertThat(r.getStatusCode(), is(200)); 102 | } 103 | 104 | @Override 105 | public void refreshIndexSuggesters(String index) throws Exception { 106 | Response r = httpClient.preparePost("http://localhost:" + port + "/"+ index + "/product/__suggestRefresh").execute().get(); 107 | assertThat(r.getStatusCode(), is(200)); 108 | } 109 | 110 | @Override 111 | public void refreshFieldSuggesters(String index, String field) throws Exception { 112 | String jsonBody = String.format(Locale.ROOT, "{ \"field\": \"%s\" } ", field); 113 | 114 | Response r = httpClient.preparePost("http://localhost:"+ port +"/" + index + "/product/__suggestRefresh").setBody(jsonBody).execute().get(); 115 | assertThat(r.getStatusCode(), is(200)); 116 | } 117 | 118 | @Override 119 | public FstStats getStatistics() throws Exception { 120 | List stats = Lists.newArrayList(); 121 | 122 | Response r = httpClient.prepareGet("http://localhost:" + port + "/__suggestStatistics").execute().get(); 123 | assertThat(r.getStatusCode(), is(200)); 124 | System.out.println(r.getResponseBody()); 125 | 126 | ObjectMapper objectMapper = new ObjectMapper(); 127 | JsonNode rootObj = objectMapper.readTree(r.getResponseBody()); 128 | FstStats fstStats = new FstStats(); 129 | ArrayNode jsonFstStats = (ArrayNode) rootObj.get("fstStats"); 130 | Iterator nodesIterator = jsonFstStats.iterator(); 131 | 132 | while (nodesIterator.hasNext()) { 133 | JsonNode fstStatsNodeEntry = nodesIterator.next(); 134 | 135 | if (fstStatsNodeEntry.isObject()) { 136 | ShardId shardId = new ShardId(fstStatsNodeEntry.get("index").asText(), fstStatsNodeEntry.get("id").asInt()); 137 | FstStats.FstIndexShardStats fstIndexShardStats = new FstStats.FstIndexShardStats(shardId, null, null, fstStatsNodeEntry.get("sizeInBytes").getLongValue()); 138 | stats.add(fstIndexShardStats); 139 | } 140 | 141 | fstStats.getStats().addAll(stats); 142 | 143 | } 144 | 145 | return fstStats; 146 | } 147 | 148 | private String createJSONQuery(SuggestionQuery suggestionQuery) { 149 | StringBuilder query = new StringBuilder("{"); 150 | query.append(String.format(Locale.ROOT, "\"field\": \"%s\", ", suggestionQuery.field)); 151 | query.append(String.format(Locale.ROOT, "\"term\": \"%s\"", suggestionQuery.term)); 152 | if (suggestionQuery.size != null) { 153 | query.append(String.format(Locale.ROOT, ", \"size\": \"%s\"", suggestionQuery.size)); 154 | } 155 | if (suggestionQuery.suggestType != null) { 156 | query.append(String.format(Locale.ROOT, ", \"type\": \"%s\"", suggestionQuery.suggestType)); 157 | } 158 | if (suggestionQuery.similarity != null && suggestionQuery.similarity > 0.0 && suggestionQuery.similarity < 1.0) { 159 | query.append(String.format(Locale.ROOT, ", \"similarity\": \"%s\"", suggestionQuery.similarity)); 160 | } 161 | if (Strings.hasLength(suggestionQuery.indexAnalyzer)) { 162 | query.append(String.format(Locale.ROOT, ", \"indexAnalyzer\": \"%s\"", suggestionQuery.indexAnalyzer)); 163 | } 164 | if (Strings.hasLength(suggestionQuery.queryAnalyzer)) { 165 | query.append(String.format(Locale.ROOT, ", \"queryAnalyzer\": \"%s\"", suggestionQuery.queryAnalyzer)); 166 | } 167 | if (Strings.hasLength(suggestionQuery.analyzer)) { 168 | query.append(String.format(Locale.ROOT, ", \"analyzer\": \"%s\"", suggestionQuery.analyzer)); 169 | } 170 | query.append(String.format(Locale.ROOT, ", \"preserve_position_increments\": \"%s\"", suggestionQuery.preservePositionIncrements)); 171 | query.append("}"); 172 | 173 | return query.toString(); 174 | } 175 | 176 | @SuppressWarnings("unchecked") 177 | private List getSuggestionsFromResponse(String response) throws IOException { 178 | XContentParser parser = JsonXContent.jsonXContent.createParser(response); 179 | Map jsonResponse = parser.map(); 180 | assertThat(jsonResponse, hasKey("suggestions")); 181 | return (List) jsonResponse.get("suggestions"); 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/SuggestBuildersTest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 4 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestResponse; 5 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRefreshRequestBuilder; 6 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRequestBuilder; 7 | import de.spinscale.elasticsearch.client.action.suggest.SuggestStatisticsRequestBuilder; 8 | import org.elasticsearch.common.Strings; 9 | 10 | import java.util.List; 11 | 12 | import static org.hamcrest.Matchers.emptyArray; 13 | import static org.hamcrest.Matchers.is; 14 | 15 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; 16 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; 17 | 18 | @ClusterScope(scope = Scope.SUITE, transportClientRatio = 0.0) 19 | public class SuggestBuildersTest extends AbstractSuggestTest { 20 | 21 | @Override 22 | public List getSuggestions(SuggestionQuery suggestionQuery) throws Exception { 23 | SuggestRequestBuilder builder = new SuggestRequestBuilder(client()) 24 | .setIndices(suggestionQuery.index) 25 | .field(suggestionQuery.field) 26 | .term(suggestionQuery.term); 27 | 28 | if (suggestionQuery.size != null) { 29 | builder.size(suggestionQuery.size); 30 | } 31 | if (suggestionQuery.similarity != null && suggestionQuery.similarity > 0.0 && suggestionQuery.similarity < 1.0) { 32 | builder.similarity(suggestionQuery.similarity); 33 | } 34 | if (suggestionQuery.suggestType != null) { 35 | builder.suggestType(suggestionQuery.suggestType); 36 | } 37 | if (Strings.hasLength(suggestionQuery.indexAnalyzer)) { 38 | builder.indexAnalyzer(suggestionQuery.indexAnalyzer); 39 | } 40 | if (Strings.hasLength(suggestionQuery.queryAnalyzer)) { 41 | builder.queryAnalyzer(suggestionQuery.queryAnalyzer); 42 | } 43 | if (Strings.hasLength(suggestionQuery.analyzer)) { 44 | builder.analyzer(suggestionQuery.analyzer); 45 | } 46 | builder.preservePositionIncrements(suggestionQuery.preservePositionIncrements); 47 | 48 | SuggestResponse suggestResponse = builder.execute().actionGet(); 49 | assertThat(suggestResponse.getShardFailures(), is(emptyArray())); 50 | 51 | return suggestResponse.suggestions(); 52 | } 53 | 54 | @Override 55 | public void refreshAllSuggesters() throws Exception { 56 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(client()); 57 | builder.execute().actionGet(); 58 | } 59 | 60 | @Override 61 | public void refreshIndexSuggesters(String index) throws Exception { 62 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(client()).setIndices(index); 63 | builder.execute().actionGet(); 64 | } 65 | 66 | @Override 67 | public void refreshFieldSuggesters(String index, String field) throws Exception { 68 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(client()).setIndices(index).setField(field); 69 | builder.execute().actionGet(); 70 | } 71 | 72 | @Override 73 | public FstStats getStatistics() throws Exception { 74 | SuggestStatisticsRequestBuilder builder = new SuggestStatisticsRequestBuilder(client()); 75 | return builder.execute().actionGet().fstStats(); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/SuggestionQuery.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import java.util.Locale; 4 | 5 | public class SuggestionQuery { 6 | 7 | public final String index; 8 | public final String type; 9 | public final String field; 10 | public final String term; 11 | public String suggestType; 12 | public String indexAnalyzer; 13 | public String queryAnalyzer; 14 | public Integer size; 15 | public Float similarity; 16 | public String analyzer; 17 | public boolean preservePositionIncrements = true; 18 | 19 | public SuggestionQuery(String index, String type, String field, String term) { 20 | this.index = index; 21 | this.type = type; 22 | this.field = field; 23 | this.term = term; 24 | } 25 | 26 | public SuggestionQuery size(Integer size) { 27 | this.size = size; 28 | return this; 29 | } 30 | 31 | public SuggestionQuery similarity(Float similarity) { 32 | this.similarity = similarity; 33 | return this; 34 | } 35 | 36 | public SuggestionQuery suggestType(String suggestType) { 37 | this.suggestType = suggestType; 38 | return this; 39 | } 40 | 41 | public SuggestionQuery analyzer(String analyzer) { 42 | this.analyzer = analyzer; 43 | return this; 44 | } 45 | 46 | public SuggestionQuery indexAnalyzer(String indexAnalyzer) { 47 | this.indexAnalyzer = indexAnalyzer; 48 | return this; 49 | } 50 | 51 | public SuggestionQuery queryAnalyzer(String queryAnalyzer) { 52 | this.queryAnalyzer = queryAnalyzer; 53 | return this; 54 | } 55 | 56 | @Override 57 | public String toString() { 58 | StringBuilder sb = new StringBuilder(); 59 | sb.append(String.format(Locale.ROOT, "Index [%s] type [%s] field [%s] term [%s]", index, type, field, term)); 60 | if (size != null) sb.append(String.format(Locale.ROOT, " size[%s]", size)); 61 | if (similarity != null) sb.append(String.format(Locale.ROOT, " similarity[%s]", similarity)); 62 | if (suggestType != null) sb.append(String.format(Locale.ROOT, " suggestType[%s]", suggestType)); 63 | if (analyzer != null) sb.append(String.format(Locale.ROOT, " analyzer[%s]", analyzer)); 64 | if (indexAnalyzer!= null) sb.append(String.format(Locale.ROOT, " indexAnalyzer[%s]", indexAnalyzer)); 65 | if (queryAnalyzer != null) sb.append(String.format(Locale.ROOT, " queryAnalyzer[%s]", queryAnalyzer)); 66 | sb.append(String.format(Locale.ROOT, " preservePositionIncrements[%s]", preservePositionIncrements)); 67 | return sb.toString(); 68 | } 69 | 70 | public SuggestionQuery preservePositionIncrements(boolean preservePositionIncrements) { 71 | this.preservePositionIncrements = preservePositionIncrements; 72 | return this; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/TransportClientTest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 4 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestResponse; 5 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRefreshRequestBuilder; 6 | import de.spinscale.elasticsearch.client.action.suggest.SuggestRequestBuilder; 7 | import de.spinscale.elasticsearch.client.action.suggest.SuggestStatisticsRequestBuilder; 8 | import org.elasticsearch.client.transport.TransportClient; 9 | import org.elasticsearch.common.Strings; 10 | import org.elasticsearch.common.settings.Settings; 11 | import org.elasticsearch.transport.Transport; 12 | import org.junit.After; 13 | 14 | import java.util.List; 15 | 16 | import static org.elasticsearch.common.settings.ImmutableSettings.settingsBuilder; 17 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; 18 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; 19 | import static org.hamcrest.Matchers.emptyArray; 20 | import static org.hamcrest.Matchers.is; 21 | 22 | @ClusterScope(scope=Scope.SUITE, transportClientRatio = 0.0) 23 | public class TransportClientTest extends AbstractSuggestTest { 24 | 25 | // TODO: Remove this class, once we can set transport client settings programmatically and then remove the transportclient ratio from all tests 26 | 27 | private TransportClient transportClient; 28 | 29 | private TransportClient getTransportClient() { 30 | if (transportClient == null) { 31 | transportClient = new TransportClient(settingsBuilder() 32 | .put("cluster.name", internalCluster().getClusterName()) 33 | .put("name", "programmatic_transport_client") 34 | .put("client.transport.nodes_sampler_interval", "1s") 35 | .build()); 36 | Transport transport = internalCluster().getDataNodeInstance(Transport.class); 37 | transportClient.addTransportAddress(transport.boundAddress().publishAddress()); 38 | } 39 | 40 | return transportClient; 41 | } 42 | 43 | @After 44 | public void closeTransportClient() { 45 | if (transportClient != null) { 46 | transportClient.close(); 47 | } 48 | } 49 | 50 | @Override 51 | protected Settings nodeSettings(int nodeOrdinal) { 52 | return settingsBuilder().put(super.nodeSettings(nodeOrdinal)) 53 | .put("node.mode", "network") 54 | .build(); 55 | } 56 | 57 | @Override 58 | public List getSuggestions(SuggestionQuery suggestionQuery) throws Exception { 59 | SuggestRequestBuilder builder = new SuggestRequestBuilder(getTransportClient()) 60 | .setIndices(suggestionQuery.index) 61 | .field(suggestionQuery.field) 62 | .term(suggestionQuery.term); 63 | 64 | if (suggestionQuery.size != null) { 65 | builder.size(suggestionQuery.size); 66 | } 67 | if (suggestionQuery.similarity != null && suggestionQuery.similarity > 0.0 && suggestionQuery.similarity < 1.0) { 68 | builder.similarity(suggestionQuery.similarity); 69 | } 70 | if (suggestionQuery.suggestType != null) { 71 | builder.suggestType(suggestionQuery.suggestType); 72 | } 73 | if (Strings.hasLength(suggestionQuery.queryAnalyzer)) { 74 | builder.queryAnalyzer(suggestionQuery.queryAnalyzer); 75 | } 76 | if (Strings.hasLength(suggestionQuery.indexAnalyzer)) { 77 | builder.indexAnalyzer(suggestionQuery.indexAnalyzer); 78 | } 79 | if (Strings.hasLength(suggestionQuery.analyzer)) { 80 | builder.analyzer(suggestionQuery.analyzer); 81 | } 82 | builder.preservePositionIncrements(suggestionQuery.preservePositionIncrements); 83 | 84 | SuggestResponse suggestResponse = builder.execute().actionGet(); 85 | assertThat(suggestResponse.getShardFailures(), is(emptyArray())); 86 | 87 | return suggestResponse.suggestions(); 88 | } 89 | 90 | @Override 91 | public void refreshAllSuggesters() throws Exception { 92 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(getTransportClient()); 93 | builder.execute().actionGet(); 94 | } 95 | 96 | @Override 97 | public void refreshIndexSuggesters(String index) throws Exception { 98 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(getTransportClient()).setIndices(index); 99 | builder.execute().actionGet(); 100 | } 101 | 102 | @Override 103 | public void refreshFieldSuggesters(String index, String field) throws Exception { 104 | SuggestRefreshRequestBuilder builder = new SuggestRefreshRequestBuilder(getTransportClient()).setIndices(index).setField(field); 105 | builder.execute().actionGet(); 106 | } 107 | 108 | @Override 109 | public FstStats getStatistics() throws Exception { 110 | SuggestStatisticsRequestBuilder builder = new SuggestStatisticsRequestBuilder(getTransportClient()); 111 | return builder.execute().actionGet().fstStats(); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/de/spinscale/elasticsearch/module/suggest/test/TransportSuggestActionTest.java: -------------------------------------------------------------------------------- 1 | package de.spinscale.elasticsearch.module.suggest.test; 2 | 3 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshAction; 4 | import de.spinscale.elasticsearch.action.suggest.refresh.SuggestRefreshRequest; 5 | import de.spinscale.elasticsearch.action.suggest.statistics.FstStats; 6 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsAction; 7 | import de.spinscale.elasticsearch.action.suggest.statistics.SuggestStatisticsRequest; 8 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestAction; 9 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestRequest; 10 | import de.spinscale.elasticsearch.action.suggest.suggest.SuggestResponse; 11 | import org.elasticsearch.common.Strings; 12 | 13 | import java.util.List; 14 | 15 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.ClusterScope; 16 | import static org.elasticsearch.test.ElasticsearchIntegrationTest.Scope; 17 | import static org.hamcrest.Matchers.emptyArray; 18 | import static org.hamcrest.Matchers.is; 19 | 20 | @ClusterScope(scope = Scope.SUITE, transportClientRatio = 0.0) 21 | public class TransportSuggestActionTest extends AbstractSuggestTest { 22 | 23 | @Override 24 | public List getSuggestions(SuggestionQuery suggestionQuery) throws Exception { 25 | SuggestRequest request = new SuggestRequest(suggestionQuery.index); 26 | 27 | request.term(suggestionQuery.term); 28 | request.field(suggestionQuery.field); 29 | 30 | if (suggestionQuery.size != null) { 31 | request.size(suggestionQuery.size); 32 | } 33 | if (suggestionQuery.similarity != null && suggestionQuery.similarity > 0.0 && suggestionQuery.similarity < 1.0) { 34 | request.similarity(suggestionQuery.similarity); 35 | } 36 | if (suggestionQuery.suggestType != null) { 37 | request.suggestType(suggestionQuery.suggestType); 38 | } 39 | if (Strings.hasLength(suggestionQuery.indexAnalyzer)) { 40 | request.indexAnalyzer(suggestionQuery.indexAnalyzer); 41 | } 42 | if (Strings.hasLength(suggestionQuery.queryAnalyzer)) { 43 | request.queryAnalyzer(suggestionQuery.queryAnalyzer); 44 | } 45 | if (Strings.hasLength(suggestionQuery.analyzer)) { 46 | request.analyzer(suggestionQuery.analyzer); 47 | } 48 | 49 | request.preservePositionIncrements(suggestionQuery.preservePositionIncrements); 50 | 51 | SuggestResponse suggestResponse = client().execute(SuggestAction.INSTANCE, request).actionGet(); 52 | assertThat(suggestResponse.getShardFailures(), is(emptyArray())); 53 | 54 | return suggestResponse.suggestions(); 55 | } 56 | 57 | @Override 58 | public void refreshAllSuggesters() throws Exception { 59 | SuggestRefreshRequest refreshRequest = new SuggestRefreshRequest(); 60 | client().execute(SuggestRefreshAction.INSTANCE, refreshRequest).actionGet(); 61 | } 62 | 63 | @Override 64 | public void refreshIndexSuggesters(String index) throws Exception { 65 | SuggestRefreshRequest refreshRequest = new SuggestRefreshRequest(index); 66 | client().execute(SuggestRefreshAction.INSTANCE, refreshRequest).actionGet(); 67 | } 68 | 69 | @Override 70 | public void refreshFieldSuggesters(String index, String field) throws Exception { 71 | SuggestRefreshRequest refreshRequest = new SuggestRefreshRequest(index); 72 | refreshRequest.field(field); 73 | client().execute(SuggestRefreshAction.INSTANCE, refreshRequest).actionGet(); 74 | } 75 | 76 | @Override 77 | public FstStats getStatistics() throws Exception { 78 | SuggestStatisticsRequest suggestStatisticsRequest = new SuggestStatisticsRequest(); 79 | return client().execute(SuggestStatisticsAction.INSTANCE, suggestStatisticsRequest).actionGet().fstStats(); 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/test/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | es.logger.level=INFO 2 | log4j.rootLogger=${es.logger.level}, out 3 | 4 | log4j.logger.org.apache.http=INFO, out 5 | log4j.additivity.org.apache.http=false 6 | 7 | log4j.appender.out=org.apache.log4j.ConsoleAppender 8 | log4j.appender.out.layout=org.apache.log4j.PatternLayout 9 | log4j.appender.out.layout.conversionPattern=[%d{ISO8601}][%-5p][%-25c] %m%n 10 | -------------------------------------------------------------------------------- /src/test/resources/product.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings" : { 3 | "product": { 4 | "properties": { 5 | "ProductId": { "type": "string", "index": "not_analyzed" }, 6 | "Description": { "type": "string", "index": "not_analyzed" }, 7 | 8 | "ProductName" : { 9 | "type" : "multi_field", 10 | "fields" : { 11 | "ProductName": { "type": "string", "index": "not_analyzed", "store": "yes" }, 12 | "standard": { "type": "string" }, 13 | "suggest" : { "type": "string", "analyzer": "suggest_analyzer" }, 14 | "lowercase" : { "type": "string", "analyzer": "lowercase_analyzer" }, 15 | "keyword" : { "type": "string", "analyzer": "keyword" } 16 | } 17 | } 18 | } 19 | } 20 | }, 21 | "settings" : { 22 | "analysis": { 23 | "analyzer" : { 24 | "lowercase_analyzer" : { 25 | "type" : "custom", 26 | "tokenizer" : "standard", 27 | "filter" : [ "standard", "lowercase" ] 28 | }, 29 | "suggest_analyzer" : { 30 | "type" : "custom", 31 | "tokenizer" : "standard", 32 | "filter" : [ "standard", "lowercase", "shingle" ] 33 | }, 34 | "keyword_lowercase_analyzer" : { 35 | "type" : "custom", 36 | "tokenizer" : "keyword", 37 | "filter" : [ "lowercase" ] 38 | } 39 | } 40 | } 41 | } 42 | } 43 | --------------------------------------------------------------------------------