├── .gitignore ├── LICENSE ├── README.md ├── chapter_03 ├── analyzers │ └── analyzers.json ├── attachment │ └── attachment_mapping.json ├── child │ └── child_mapping.json ├── geopoint │ ├── geopoint_mapping.json │ └── values.json ├── geoshape │ └── geoshape_mapping.json ├── ip │ └── ip_field.json ├── meta │ └── meta_mapping.json ├── multi_field │ └── multi_field.json └── special │ └── geopoint_mapping.json ├── chapter_04 ├── 01_create │ └── create.sh ├── 02_delete_an_index │ └── delete.sh ├── 03_open_close_index │ └── open_close.sh ├── delete_mapping.sh ├── get_mapping.sh ├── index_a_document.sh ├── populate.sh ├── populate_parent_child.sh ├── put_mapping.sh ├── reindex.sh ├── rolling_index.sh ├── shrink.sh └── update.sh ├── chapter_05 ├── count.sh ├── delete_by_query.sh ├── document.json ├── inner_hits.sh ├── match_query.sh ├── populate_query.sh ├── profile.sh ├── rescore.sh ├── scroll.sh ├── search.sh ├── search_after.sh ├── sort.sh ├── term_query_filter.sh └── update_by_query.sh ├── chapter_06 ├── common_terms_query.sh ├── exists_query.sh ├── function_score_query.sh ├── ids_query.sh ├── match_phrase_prefix_query.sh ├── match_phrase_query.sh ├── match_query_query.sh ├── missing_query.sh ├── missing_query_query.sh ├── multi_match_query.sh ├── prefix_query.sh ├── query_string_query.sh ├── range_query.sh ├── regexp_query.sh ├── simple_query_string_query.sh ├── span_containing_query.sh ├── span_first_query.sh ├── span_multi_query.sh ├── span_not_query.sh ├── span_or_query.sh ├── span_within_query.sh ├── template_query.sh ├── term2_query.sh ├── term_query.sh ├── terms_query.sh └── wildcard_query.sh ├── chapter_07 ├── geo.json ├── geo_bbox_query.sh ├── geo_distance_query.sh ├── geo_polygon_query.sh ├── has_child_query.sh ├── has_child_query_sort.sh ├── has_parent_query.sh ├── mapping.json ├── nested_query.sh ├── populate_geo.sh └── populate_relations.sh ├── chapter_08 ├── executing_an_aggregation.sh ├── executing_children_aggregations.sh ├── executing_date_histogram_aggregations.sh ├── executing_filter_aggregation.sh ├── executing_filters_aggregation.sh ├── executing_geo_bounds_aggregation.sh ├── executing_geo_centroid_aggregation.sh ├── executing_geo_distance_aggregations.sh ├── executing_global_aggregations.sh ├── executing_matrix_stats.sh ├── executing_missing_aggregations.sh ├── executing_range_aggregations.sh ├── executing_significant_terms_aggregations.sh ├── executing_stat_aggregations.sh ├── executing_terms_aggregation.sh ├── executing_top_hit_aggregations.sh ├── fix │ └── executing_histogram_aggregations.sh ├── geo_bounds_aggregation.sh ├── populate.sh ├── populate_aggregations.sh └── populate_parent_child.sh ├── chapter_09 ├── .DS_Store ├── aggregation_script.sh ├── call_indexed_script.sh ├── complex_script.sh ├── field_script.sh ├── filter_script.sh ├── indexed_script.sh ├── painless_script_score.json ├── painless_script_score.sh ├── populate_scripting.sh ├── sort_ramdom_script.sh ├── sort_script.sh └── update_script.sh ├── chapter_13 ├── attachment_processor.sh ├── builtin_processors.sh ├── delete_pipeline.sh ├── geoip_processor.sh ├── get_pipeline.sh ├── grok_processor.sh ├── put_pipeline.sh └── simulate_pipeline.sh ├── chapter_14 ├── http_es_client │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── packtpub │ │ │ └── App.java │ │ └── test │ │ └── java │ │ └── com │ │ └── packtpub │ │ └── AppTest.java ├── http_java_client │ ├── pom.xml │ └── src │ │ ├── main │ │ └── java │ │ │ └── com │ │ │ └── packtpub │ │ │ ├── App.java │ │ │ └── MyRequestRetryHandler.java │ │ └── test │ │ └── java │ │ └── com │ │ └── packtpub │ │ └── AppTest.java └── nativeclient │ ├── pom.xml │ └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── packtpub │ │ │ ├── AggregationExample.java │ │ │ ├── BulkOperations.java │ │ │ ├── DocumentOperations.java │ │ │ ├── IndicesOperations.java │ │ │ ├── MappingOperations.java │ │ │ ├── NativeClient.java │ │ │ ├── QueryCreation.java │ │ │ ├── QueryExample.java │ │ │ ├── QueryHelper.java │ │ │ └── ScrollQueryExample.java │ └── resources │ │ └── log4j2.properties │ └── test │ └── java │ └── com │ └── packtpub │ └── AppTest.java ├── chapter_15 └── elastic4s_sample │ ├── build.sbt │ ├── pom.xml_ │ ├── project │ ├── build.properties │ └── plugins.sbt │ └── src │ └── main │ ├── resources │ └── log4j2.properties │ └── scala │ └── com │ └── packtpub │ ├── AggregationExample.scala │ ├── ClientSample.scala │ ├── DocumentExample.scala │ ├── ElasticSearchClientTrait.scala │ ├── IndicesExample.scala │ ├── MappingExample.scala │ └── QueryExample.scala ├── chapter_16 ├── __init__.py ├── aggregation.py ├── client_creation.py ├── document_management.py ├── indices_management.py ├── mapping_management.py ├── searching.py └── utils.py ├── chapter_17 ├── analysis_plugin │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── main │ │ ├── assemblies │ │ └── plugin.xml │ │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ ├── index │ │ │ └── analysis │ │ │ │ └── CustomEnglishAnalyzerProvider.java │ │ │ └── plugin │ │ │ └── analysis │ │ │ └── AnalysisPlugin.java │ │ └── resources │ │ └── plugin-descriptor.properties ├── ingest_plugin │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── main │ │ ├── assemblies │ │ └── plugin.xml │ │ ├── java │ │ ├── com │ │ │ └── packtpub │ │ │ │ └── ingest │ │ │ │ └── InitialProcessor.java │ │ └── org │ │ │ └── elasticsearch │ │ │ └── plugin │ │ │ └── ingest │ │ │ └── MyIngestPlugin.java │ │ └── resources │ │ └── plugin-descriptor.properties ├── rest_plugin │ ├── .gitignore │ ├── pom.xml │ └── src │ │ └── main │ │ ├── assemblies │ │ └── plugin.xml │ │ ├── java │ │ └── org │ │ │ └── elasticsearch │ │ │ ├── action │ │ │ └── simple │ │ │ │ ├── ShardSimpleResponse.java │ │ │ │ ├── SimpleAction.java │ │ │ │ ├── SimpleRequest.java │ │ │ │ ├── SimpleRequestBuilder.java │ │ │ │ ├── SimpleResponse.java │ │ │ │ └── TransportSimpleAction.java │ │ │ ├── plugin │ │ │ └── simple │ │ │ │ └── RestPlugin.java │ │ │ └── rest │ │ │ └── RestSimpleAction.java │ │ └── resources │ │ └── plugin-descriptor.properties └── simple_plugin │ ├── .gitignore │ ├── pom.xml │ └── src │ └── main │ ├── assemblies │ └── plugin.xml │ ├── java │ └── org │ │ └── elasticsearch │ │ └── plugin │ │ └── simple │ │ └── SimplePlugin.java │ └── resources │ └── plugin-descriptor.properties ├── chapter_18 ├── download_es_hadoop.sh ├── download_spark.sh ├── es.pig └── install_pig_spark_macosx.sh └── test.json /.gitignore: -------------------------------------------------------------------------------- 1 | # pypi 2 | /build 3 | /dist 4 | 5 | # do not wants 6 | *.py[c|o] 7 | *.egg-info 8 | *.egg 9 | .*.swp 10 | .DS_Store 11 | *.zip 12 | 13 | # IDEs 14 | .idea 15 | .project 16 | 17 | target/ 18 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016, Paro Alberto Maria Angelo 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | 25 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | elasticsearch-cookbook-third-edition 2 | ================================ 3 | 4 | PacktPub ElasticSearch CookBook code repository 5 | 6 | 7 | The code on MacOSX, Linux, FreeBSD and other Unix based systems everything should be installed in a default installation. 8 | 9 | For windows users 10 | ----------------- 11 | 12 | I suggest to install a "Sane shell environment on Windows". Instructions are available on: 13 | http://blog.ruilopes.com/post/2143557964/sane-shell-environment-on-windows 14 | 15 | For downloading curl for Windows: 16 | http://www.confusedbycode.com/curl/#downloads 17 | 18 | Generally a bash shell and curl must be installed. 19 | 20 | These are very common tools in ElasticSearch community. 21 | 22 | -------------------------------------------------------------------------------- /chapter_03/analyzers/analyzers.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": { 3 | "type": "string", 4 | "index": "analyzed", 5 | "index_analyzer": "standard", 6 | "search_analyzer": "simple" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /chapter_03/attachment/attachment_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "email": { 3 | "properties": { 4 | "sender": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "document": { 15 | "type": "attachment", 16 | "fields": { 17 | "file": { 18 | "store": "yes", 19 | "index": "analyzed" 20 | }, 21 | "date": { 22 | "store": "yes" 23 | }, 24 | "author": { 25 | "store": "yes" 26 | }, 27 | "keywords": { 28 | "store": "yes" 29 | }, 30 | "content_type": { 31 | "store": "yes" 32 | }, 33 | "title": { 34 | "store": "yes" 35 | } 36 | } 37 | } 38 | } 39 | } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /chapter_03/child/child_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "properties": { 4 | "id": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "customer_id": { 15 | "type": "string", 16 | "store": "yes", 17 | "index": "not_analyzed" 18 | }, 19 | "sent": { 20 | "type": "boolean", 21 | "store": "no", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | }, 26 | "item": { 27 | "_parent": { 28 | "type": "order" 29 | }, 30 | "type": "nested", 31 | "properties": { 32 | "name": { 33 | "type": "string", 34 | "store": "no", 35 | "index": "analyzed" 36 | }, 37 | "quantity": { 38 | "type": "integer", 39 | "store": "no", 40 | "index": "not_analyzed" 41 | }, 42 | "vat": { 43 | "type": "double", 44 | "store": "no", 45 | "index": "not_analyzed" 46 | } 47 | } 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /chapter_03/geopoint/geopoint_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "properties": { 4 | "id": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "customer_id": { 15 | "type": "string", 16 | "store": "yes", 17 | "index": "not_analyzed" 18 | }, 19 | "customer_ip": { 20 | "type": "ip", 21 | "store": "yes", 22 | "index": "yes" 23 | }, 24 | "customer_location": { 25 | "type": "geo_point", 26 | "store": "yes" 27 | }, 28 | "sent": { 29 | "type": "boolean", 30 | "store": "no", 31 | "index": "not_analyzed" 32 | } 33 | } 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /chapter_03/geopoint/values.json: -------------------------------------------------------------------------------- 1 | { 2 | "customer_location": { 3 | "lat": 45.61752, 4 | "lon": 9.08363 5 | }, 6 | 7 | "customer_location": "45.61752,9.08363", 8 | "customer_location": "u0n7w8qmrfj", 9 | "customer_location": [9.08363, 45.61752] 10 | } 11 | -------------------------------------------------------------------------------- /chapter_03/geoshape/geoshape_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "properties": { 4 | "id": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "customer_id": { 15 | "type": "string", 16 | "store": "yes", 17 | "index": "not_analyzed" 18 | }, 19 | "customer_ip": { 20 | "type": "ip", 21 | "store": "yes", 22 | "index": "yes" 23 | }, 24 | "customer_location": { 25 | "type": "geo_shape", 26 | "tree": "quadtree", 27 | "precision": "1m" 28 | }, 29 | "sent": { 30 | "type": "boolean", 31 | "store": "no", 32 | "index": "not_analyzed" 33 | } 34 | } 35 | } 36 | } 37 | 38 | -------------------------------------------------------------------------------- /chapter_03/ip/ip_field.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "properties": { 4 | "id": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "customer_id": { 15 | "type": "string", 16 | "store": "yes", 17 | "index": "not_analyzed" 18 | }, 19 | "customer_ip": { 20 | "type": "ip", 21 | "store": "yes", 22 | "index": "yes" 23 | }, 24 | "sent": { 25 | "type": "boolean", 26 | "store": "no", 27 | "index": "not_analyzed" 28 | } 29 | } 30 | }, 31 | "item": { 32 | "_parent": { 33 | "type": "order" 34 | }, 35 | "type": "nested", 36 | "properties": { 37 | "name": { 38 | "type": "multi_field", 39 | "fields": { 40 | "name": { 41 | "type": "string", 42 | "index": "not_analyzed" 43 | }, 44 | "tk": { 45 | "type": "string", 46 | "index": "analyzed" 47 | }, 48 | "code": { 49 | "type": "string", 50 | "index": "analyzed", 51 | "analyzer": "code_analyzer" 52 | } 53 | } 54 | }, 55 | "quantity": { 56 | "type": "integer", 57 | "store": "no", 58 | "index": "not_analyzed" 59 | }, 60 | "vat": { 61 | "type": "double", 62 | "store": "no", 63 | "index": "not_analyzed" 64 | } 65 | } 66 | } 67 | } 68 | 69 | -------------------------------------------------------------------------------- /chapter_03/meta/meta_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "_meta": { 4 | "attr1": ["value1", "value2"], 5 | "attr2": { 6 | "attr3": "value3" 7 | } 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /chapter_03/multi_field/multi_field.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "properties": { 4 | "id": { 5 | "type": "string", 6 | "store": "yes", 7 | "index": "not_analyzed" 8 | }, 9 | "date": { 10 | "type": "date", 11 | "store": "no", 12 | "index": "not_analyzed" 13 | }, 14 | "customer_id": { 15 | "type": "string", 16 | "store": "yes", 17 | "index": "not_analyzed" 18 | }, 19 | "sent": { 20 | "type": "boolean", 21 | "store": "no", 22 | "index": "not_analyzed" 23 | } 24 | } 25 | }, 26 | "item": { 27 | "_parent": { 28 | "type": "order" 29 | }, 30 | "type": "nested", 31 | "properties": { 32 | "name": { 33 | "type": "multi_field", 34 | "fields": { 35 | "name": { 36 | "type": "string", 37 | "index": "not_analyzed" 38 | }, 39 | "tk": { 40 | "type": "string", 41 | "index": "analyzed" 42 | }, 43 | "code": { 44 | "type": "string", 45 | "index": "analyzed", 46 | "analyzer": "code_analyzer" 47 | } 48 | } 49 | }, 50 | "quantity": { 51 | "type": "integer", 52 | "store": "no", 53 | "index": "not_analyzed" 54 | }, 55 | "vat": { 56 | "type": "double", 57 | "store": "no", 58 | "index": "not_analyzed" 59 | } 60 | } 61 | } 62 | } 63 | 64 | -------------------------------------------------------------------------------- /chapter_03/special/geopoint_mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "order": { 3 | "_uid": { 4 | "store": "yes" 5 | }, 6 | "_id": { 7 | "path": "order_id" 8 | }, 9 | "_type": { 10 | "store": "yes" 11 | }, 12 | "_source": { 13 | "store": "yes" 14 | }, 15 | "_all": { 16 | "enable": false 17 | }, 18 | "_analyzer": { 19 | "path": "analyzer_field" 20 | }, 21 | "_boost": { 22 | "null_value": 1.0 23 | }, 24 | "_routing": { 25 | "path": "customer_id", 26 | "required": true 27 | }, 28 | "_index": { 29 | "enabled": true 30 | }, 31 | "_size": { 32 | "enabled": true, 33 | "store": "yes" 34 | }, 35 | "_timestamp": { 36 | "enabled": true, 37 | "store": "yes", 38 | "path": "date" 39 | }, 40 | "_ttl": { 41 | "enabled": true, 42 | "default": "3y" 43 | }, 44 | "properties": { 45 | "order_id": { 46 | "type": "string", 47 | "store": "yes", 48 | "index": "not_analyzed" 49 | }, 50 | "date": { 51 | "type": "date", 52 | "store": "no", 53 | "index": "not_analyzed" 54 | }, 55 | "analyzer_field": { 56 | "type": "string", 57 | "store": "yes", 58 | "index": "not_analyzed" 59 | }, 60 | "customer_id": { 61 | "type": "string", 62 | "store": "yes", 63 | "index": "not_analyzed" 64 | }, 65 | "customer_ip": { 66 | "type": "ip", 67 | "store": "yes", 68 | "index": "yes" 69 | }, 70 | "customer_location": { 71 | "type": "geo_point", 72 | "store": "yes" 73 | }, 74 | "sent": { 75 | "type": "boolean", 76 | "store": "no", 77 | "index": "not_analyzed" 78 | } 79 | } 80 | } 81 | } 82 | 83 | -------------------------------------------------------------------------------- /chapter_04/01_create/create.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT http://127.0.0.1:9200/myindex -d '{ 2 | "settings" : { 3 | "index" : { 4 | "number_of_shards" : 2, 5 | "number_of_replicas" : 1 6 | } 7 | } 8 | }' 9 | -------------------------------------------------------------------------------- /chapter_04/02_delete_an_index/delete.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST http://127.0.0.1:9200/myindex/_close 2 | curl -XPOST http://127.0.0.1:9200/myindex/_open 3 | -------------------------------------------------------------------------------- /chapter_04/03_open_close_index/open_close.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE http://127.0.0.1:9200/myindex 2 | -------------------------------------------------------------------------------- /chapter_04/delete_mapping.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://localhost:9200/myindex/order/' -------------------------------------------------------------------------------- /chapter_04/get_mapping.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://localhost:9200/myindex/order/_mapping' -------------------------------------------------------------------------------- /chapter_04/index_a_document.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://localhost:9200/myindex/order/2qLrAfPVQvCRMe7Ku8r0Tw' -d '{ 2 | "id" : "1234", 3 | "date" : "2013-06-07T12:14:54", 4 | "customer_id" : "customer1", 5 | "sent" : true, 6 | "in_stock_items" : 0, 7 | "items":[ 8 | {"name":"item1", "quantity":3, "vat":20.0}, 9 | {"name":"item2", "quantity":2, "vat":20.0}, 10 | {"name":"item3", "quantity":1, "vat":10.0} 11 | ] 12 | }' 13 | -------------------------------------------------------------------------------- /chapter_04/populate.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/myindex?pretty=true' 2 | curl -XPUT 'http://127.0.0.1:9200/myindex?pretty=true' -d '{"mappings": {"test-type": {"properties": {"pos": {"type": "integer", "store": "yes"}, "uuid": {"store": "yes", "type": "keyword"}, "parsedtext": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "name": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "title": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}}}}}' 3 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/1?pretty=true' -d '{"position": 1, "parsedtext": "Joe Testere nice guy", "name": "Joe Tester", "uuid": "11111", "price":4.0}' 4 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/3?pretty=true' -d '{"position": 3, "parsedtext": "Bill is not\n nice guy", "name": "Bill Clinton", "uuid": "33333", "price":6.0}' 5 | curl -XPOST 'http://127.0.0.1:9200/myindex/_refresh?pretty=true' 6 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 7 | 8 | 9 | -------------------------------------------------------------------------------- /chapter_04/populate_parent_child.sh: -------------------------------------------------------------------------------- 1 | #curl -XHEAD 'http://127.0.0.1:9200/myindex?pretty=true' 2 | curl -XDELETE 'http://127.0.0.1:9200/myindex?pretty=true' 3 | curl -XPUT 'http://127.0.0.1:9200/myindex?pretty=true' -d '{"mappings": {"test-type": {"properties": {"pos": {"type": "integer", "store": "yes"}, "uuid": {"store": "yes", "type": "keyword"}, "parsedtext": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "name": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "title": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}}}, "test-type2": {"_parent": {"type": "test-type"}}}}' 4 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/1?pretty=true' -d '{"position": 1, "parsedtext": "Joe Testere nice guy", "name": "Joe Tester", "uuid": "11111", "price":4.0}' 5 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type2/1?parent=1&pretty=true' -d '{"name": "data1", "value": "value1"}' 6 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/2?pretty=true' -d '{"position": 2, "parsedtext": "Bill Testere nice guy", "name": "Bill Baloney", "uuid": "22222", "price":5.0}' 7 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type2/2?parent=2&pretty=true' -d '{"name": "data2", "value": "value2"}' 8 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/3?pretty=true' -d '{"position": 3, "parsedtext": "Bill is not\n nice guy", "name": "Bill Clinton", "uuid": "33333", "price":6.0}' 9 | curl -XPOST 'http://127.0.0.1:9200/myindex/_refresh?pretty=true' 10 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 11 | 12 | 13 | -------------------------------------------------------------------------------- /chapter_04/put_mapping.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT 'http://localhost:9200/myindex/order/_mapping' -d '{ 2 | "order" : { 3 | "properties" : { 4 | "id" : {"type" : "string", "store" : "yes" , "index":"not_analyzed"}, 5 | "date" : {"type" : "date", "store" : "no" , "index":"not_analyzed"}, 6 | "customer_id" : {"type" : "string", "store" : "yes" , "index":"not_analyzed"}, 7 | "sent" : {"type" : "boolean", "index":"not_analyzed"}, 8 | "name" : {"type" : "string", "index":"analyzed"}, 9 | "quantity" : {"type" : "integer", "index":"not_analyzed"}, 10 | "vat" : {"type" : "double", "index":"no"} 11 | } 12 | } 13 | }' 14 | -------------------------------------------------------------------------------- /chapter_04/reindex.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST "http://127.0.0.1:9200/_reindex?pretty" -d '{ 2 | "source": { 3 | "index": "myindex" 4 | }, 5 | "dest": { 6 | "index": "myindex2" 7 | } 8 | }' 9 | 10 | -------------------------------------------------------------------------------- /chapter_04/rolling_index.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT 'http://127.0.0.1:9200/mylogs-000001' -d ' 2 | { 3 | "aliases": { 4 | "logs_write": {} 5 | } 6 | }' 7 | 8 | curl -XPOST 'http://127.0.0.1:9200/logs_write/_rollover' -d ' 9 | { 10 | "conditions": { 11 | "max_age": "7d", 12 | "max_docs": 100000 13 | }, 14 | "settings": { 15 | "index.number_of_shards": 3 16 | } 17 | }' 18 | -------------------------------------------------------------------------------- /chapter_04/shrink.sh: -------------------------------------------------------------------------------- 1 | curl -XGET ''http://localhost:9200/_nodes?pretty' 2 | 3 | curl -XGET 'http://localhost:9200/_cluster/health?pretty' 4 | 5 | curl -XPUT 'http://localhost:9200/myindex/_settings' -d ' 6 | { 7 | "settings": { 8 | "index.routing.allocation.require._name": "Gin Genie", 9 | "index.blocks.write": true 10 | } 11 | }' 12 | 13 | 14 | curl -XPOST 'http://localhost:9200/myindex/_shrink/reduced_index' -d ' 15 | { 16 | "settings": { 17 | "index.number_of_replicas": 1, 18 | "index.number_of_shards": 1, 19 | "index.codec": "best_compression" 20 | }, 21 | "aliases": { 22 | "my_search_indices": {} 23 | } 24 | }' 25 | 26 | curl -XGET 'http://localhost:9200/_cluster/health?wait_for_status=yellow' 27 | 28 | -------------------------------------------------------------------------------- /chapter_04/update.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://localhost:9200/myindex/order/2qLrAfPVQvCRMe7Ku8r0Tw/_update?pretty' -d '{ 2 | "script" : { 3 | "inline":"ctx._source.in_stock_items += params.count", 4 | "params" : { 5 | "count" : 4 6 | } 7 | } 8 | }' 9 | -------------------------------------------------------------------------------- /chapter_05/count.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_count?pretty=true' -d '{"match_all":{}}' 2 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_count?pretty=true&q=uuid:11111' 3 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type2/_count?pretty=true' 4 | 5 | -------------------------------------------------------------------------------- /chapter_05/delete_by_query.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/test-index/test-type/_query?pretty=true' -d '{"match_all":{}}' 2 | curl -XDELETE 'http://127.0.0.1:9200/test-index/test-type/_query?pretty=true&q=uuid:11111' 3 | 4 | 5 | -------------------------------------------------------------------------------- /chapter_05/document.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Paul", 3 | "address": { 4 | "city": "Sydney", 5 | "street": "Opera House Road", 6 | "number": "44" 7 | } 8 | } -------------------------------------------------------------------------------- /chapter_05/inner_hits.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty' -d '{ 2 | "query": { 3 | "has_child" : { 4 | "type" : "test-type2", 5 | "query" : { 6 | "term" : { 7 | "value" : "value1" 8 | } 9 | }, 10 | "inner_hits":{} 11 | } 12 | } 13 | } 14 | }' 15 | 16 | -------------------------------------------------------------------------------- /chapter_05/match_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "match" : { 4 | "parsedtext" : "nice guy", 5 | "operator": "and" 6 | } 7 | } 8 | }' 9 | 10 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 11 | "query": { 12 | "match_phrase" : { 13 | "parsedtext" : "nice guy" 14 | } 15 | } 16 | }' 17 | -------------------------------------------------------------------------------- /chapter_05/populate_query.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/test-index?pretty=true' 2 | curl -XPUT 'http://127.0.0.1:9200/test-index?pretty=true' -d '{ 3 | "mappings": { 4 | "test-type": { 5 | "properties": { 6 | "pos": { 7 | "type": "integer", 8 | "store": "yes" 9 | }, 10 | "uuid": { 11 | "store": "yes", 12 | "type": "keyword" 13 | }, 14 | "parsedtext": { 15 | "term_vector": "with_positions_offsets", 16 | "store": "yes", 17 | "type": "text" 18 | }, 19 | "name": { 20 | "term_vector": "with_positions_offsets", 21 | "store": "yes", 22 | "fielddata": true, 23 | "type": "text", 24 | "fields": { 25 | "raw": { 26 | "type": "keyword" 27 | } 28 | } 29 | }, 30 | "title": { 31 | "term_vector": "with_positions_offsets", 32 | "store": "yes", 33 | "type": "text", 34 | "fielddata": true, 35 | "fields": { 36 | "raw": { 37 | "type": "keyword" 38 | } 39 | } 40 | } 41 | } 42 | }, 43 | "test-type2": { 44 | "_parent": { 45 | "type": "test-type" 46 | } 47 | } 48 | } 49 | }' 50 | #curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/_mapping?pretty=true' -d '{"test-type2": {"_parent": {"type": "test-type"}}}' 51 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/1?pretty=true' -d '{"position": 1, "parsedtext": "Joe Testere nice guy", "name": "Joe Tester", "uuid": "11111", "price":4.0}' 52 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/1?parent=1&pretty=true' -d '{"name": "data1", "value": "value1", "amount":10}' 53 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/2?pretty=true' -d '{"position": 2, "parsedtext": "Bill Testere nice guy", "name": "Bill Baloney", "uuid": "22222", "price":5.0}' 54 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/2?parent=2&pretty=true' -d '{"name": "data2", "value": "value2", "amount":20}' 55 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/3?pretty=true' -d '{"position": 3, "parsedtext": "Bill is not\n nice guy", "name": "Bill Clinton", "uuid": "33333", "price":6.0}' 56 | curl -XPOST 'http://127.0.0.1:9200/test-index/_refresh?pretty=true' 57 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 58 | 59 | 60 | -------------------------------------------------------------------------------- /chapter_05/profile.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty' -d '{ 2 | "profile": true, 3 | "query": { 4 | "term": { 5 | "uuid": "11111" 6 | } 7 | } 8 | }' 9 | 10 | -------------------------------------------------------------------------------- /chapter_05/rescore.sh: -------------------------------------------------------------------------------- 1 | curl -s -XPOST 'localhost:9200/_search' -d '{ 2 | "query" : { 3 | "match" : { 4 | "parsedtext" : { 5 | "operator" : "or", 6 | "query" : "nice guy joe", 7 | "type" : "boolean" 8 | } 9 | } 10 | }, 11 | "rescore" : { 12 | "window_size" : 50, 13 | "query" : { 14 | "rescore_query" : { 15 | "match" : { 16 | "parsedtext" : { 17 | "query" : "joe nice guy", 18 | "type" : "phrase", 19 | "slop" : 2 20 | } 21 | } 22 | }, 23 | "query_weight" : 0.7, 24 | "rescore_query_weight" : 1.2 25 | } 26 | } 27 | } 28 | ' 29 | -------------------------------------------------------------------------------- /chapter_05/scroll.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&scroll=10m&size=1' -d ' { 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "sort": [ 6 | "_doc" 7 | ] 8 | }' 9 | 10 | 11 | curl -XDELETE localhost:9200/_search/scroll -d ' 12 | { 13 | "scroll_id" : ["DnF1ZXJ5VGhlbkZldGNoBQAA..."] 14 | }' -------------------------------------------------------------------------------- /chapter_05/search.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{"query":{"match_all":{}}}' 2 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&q=uuid:11111' 3 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type2/_search?pretty=true' 4 | 5 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&df=uuid&q=11111' 6 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&q=parsedtext:joe&fields=name' 7 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&q=parsedtext:joe&explain=true' 8 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&sort=name:desc' -------------------------------------------------------------------------------- /chapter_05/search_after.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d ' 2 | { 3 | "size": 1, 4 | "query": { 5 | "match_all" : {} 6 | }, 7 | "sort": [ 8 | {"price": "asc"}, 9 | {"_uid": "desc"} 10 | ] 11 | }' 12 | 13 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d ' 14 | { 15 | "size": 1, 16 | "query": { 17 | "match_all" : {} 18 | }, 19 | "search_after": [4.0, "test-type#1"], 20 | "sort": [ 21 | {"price": "asc"}, 22 | {"_uid": "desc"} 23 | ] 24 | }' -------------------------------------------------------------------------------- /chapter_05/sort.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d ' { 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "sort": [ 6 | { 7 | "price": { 8 | "order": "asc", 9 | "mode": "avg", 10 | "unmapped_type" : "double", 11 | "missing": "_last" 12 | } 13 | }, 14 | "_score" 15 | ] 16 | }' -------------------------------------------------------------------------------- /chapter_05/term_query_filter.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "term": { 4 | "uuid": "33333" 5 | } 6 | } 7 | }' 8 | 9 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 10 | "query": { 11 | "filtered": { 12 | "filter": { 13 | "term": { 14 | "uuid": "33333" 15 | } 16 | }, 17 | "query": { 18 | "match_all": {} 19 | } 20 | } 21 | } 22 | }' 23 | 24 | -------------------------------------------------------------------------------- /chapter_05/update_by_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_update_by_query?pretty' -d '{"query":{"match_all":{}}}' 2 | 3 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_update_by_query?pretty' -d ' 4 | { 5 | "script": { 6 | "inline": "ctx._source.hit=4", 7 | "lang": "painless" 8 | }, 9 | "query":{"match_all":{}} 10 | } 11 | ' 12 | 13 | -------------------------------------------------------------------------------- /chapter_06/common_terms_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "common": { 4 | "parsedtext": { 5 | "query": "a nice guy", 6 | "cutoff_frequency": 0.001 7 | } 8 | } 9 | } 10 | }' -------------------------------------------------------------------------------- /chapter_06/exists_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "exists": { 4 | "field":"parsedtext" 5 | } 6 | } 7 | }' 8 | 9 | -------------------------------------------------------------------------------- /chapter_06/function_score_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'localhost:9200/test-index/test-type/_search?pretty' -d '{ 2 | "query": { 3 | "function_score": { 4 | "query": { 5 | "query_string": { 6 | "query": "bill" 7 | } 8 | }, 9 | "functions": [{ 10 | "linear": { 11 | "position": { 12 | "origin": "0", 13 | "scale": "20" 14 | } 15 | } 16 | }], 17 | "score_mode": "multiply" 18 | } 19 | } 20 | }' 21 | 22 | -------------------------------------------------------------------------------- /chapter_06/ids_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "ids" : { 4 | "type" : "test-type", 5 | "values" : ["1", "2", "3"] 6 | } 7 | } 8 | } 9 | }' 10 | 11 | 12 | -------------------------------------------------------------------------------- /chapter_06/match_phrase_prefix_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "match_phrase_prefix" : { 4 | "parsedtext" : "nice gu" 5 | } 6 | } 7 | }' 8 | 9 | -------------------------------------------------------------------------------- /chapter_06/match_phrase_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "match_phrase" : { 4 | "parsedtext" : "nice guy" 5 | } 6 | } 7 | }' 8 | 9 | 10 | -------------------------------------------------------------------------------- /chapter_06/match_query_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "match" : { 4 | "parsedtext" : { 5 | "query": "nice guy", 6 | "operator": "and" 7 | } 8 | } 9 | } 10 | }' 11 | -------------------------------------------------------------------------------- /chapter_06/missing_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "bool": { 4 | "must_not": { 5 | "exists": { 6 | "field": "parsedtext" 7 | } 8 | } 9 | } 10 | } 11 | }' 12 | -------------------------------------------------------------------------------- /chapter_06/missing_query_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "bool": { 4 | "must_not": { 5 | "exists": { 6 | "field": "parsedtext" 7 | } 8 | } 9 | } 10 | } 11 | }' 12 | 13 | 14 | -------------------------------------------------------------------------------- /chapter_06/multi_match_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "multi_match" : { 4 | "fields":["parsedtext", "name"] 5 | "query": "nice guy", 6 | "operator": "and" 7 | } 8 | } 9 | }' 10 | -------------------------------------------------------------------------------- /chapter_06/prefix_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "prefix": { 4 | "uuid": "222" 5 | } 6 | } 7 | }' 8 | 9 | 10 | -------------------------------------------------------------------------------- /chapter_06/query_string_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "query_string": { 4 | "query": "\"nice guy\" -parsedtext:not price:{ * TO 5 } ", 5 | "fields": [ 6 | "parsedtext^5" 7 | ], 8 | "default_operator": "and" 9 | } 10 | } 11 | }' -------------------------------------------------------------------------------- /chapter_06/range_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "range" : { 4 | "position" : { 5 | "from" : 3, 6 | "to" : 4, 7 | "include_lower" : true, 8 | "include_upper" : false 9 | } 10 | } 11 | } 12 | }' 13 | 14 | 15 | -------------------------------------------------------------------------------- /chapter_06/regexp_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "regexp": { 4 | "parsedtext": { 5 | "value": "j.*", 6 | "flags" : "INTERSECTION|COMPLEMENT|EMPTY" 7 | } 8 | } 9 | } 10 | }' 11 | 12 | -------------------------------------------------------------------------------- /chapter_06/simple_query_string_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "simple_query_string": { 4 | "query": "\"nice guy\" -not", 5 | "fields": [ 6 | "parsedtext^5", 7 | "_all" 8 | ], 9 | "default_operator": "and" 10 | } 11 | } 12 | }' -------------------------------------------------------------------------------- /chapter_06/span_containing_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_containing" : { 4 | "little" : { 5 | "span_term" : { "parsedtext" : "nice" } 6 | }, 7 | "big" : { 8 | "span_near" : { 9 | "clauses" : [ 10 | { "span_term" : { "parsedtext" : "not" } }, 11 | { "span_term" : { "parsedtext" : "guy" } } 12 | ], 13 | "slop" : 5, 14 | "in_order" : true 15 | } 16 | } 17 | } 18 | } 19 | }' 20 | -------------------------------------------------------------------------------- /chapter_06/span_first_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_first" : { 4 | "match" : { 5 | "span_term" : { "parsedtext" : "joe" } 6 | }, 7 | "end" : 5 8 | } 9 | } 10 | }' 11 | 12 | -------------------------------------------------------------------------------- /chapter_06/span_multi_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_multi":{ 4 | "match":{ 5 | "prefix" : { "parsedtext" : { "value" : "jo" } } 6 | } 7 | } 8 | } 9 | }' 10 | 11 | 12 | -------------------------------------------------------------------------------- /chapter_06/span_not_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_not" : { 4 | "include" : { 5 | "span_term" : { "parsedtext" : "nice" } 6 | }, 7 | "exclude" : { 8 | "span_near" : { 9 | "clauses" : [ 10 | { "span_term" : { "parsedtext" : "not" } }, 11 | { "span_term" : { "parsedtext" : "nice" } } 12 | ], 13 | "slop" : 0, 14 | "in_order" : true 15 | } 16 | } 17 | } 18 | } 19 | }' 20 | -------------------------------------------------------------------------------- /chapter_06/span_or_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_or" : { 4 | "clauses" : [ 5 | { "span_term" : { "parsedtext" : "nice" } }, 6 | { "span_term" : { "parsedtext" : "cool" } }, 7 | { "span_term" : { "parsedtext" : "wonderful"}} 8 | ] 9 | } 10 | } 11 | }' 12 | 13 | 14 | -------------------------------------------------------------------------------- /chapter_06/span_within_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "span_within" : { 4 | "little" : { 5 | "span_term" : { "parsedtext" : "nice" } 6 | }, 7 | "big" : { 8 | "span_near" : { 9 | "clauses" : [ 10 | { "span_term" : { "parsedtext" : "is" } }, 11 | { "span_term" : { "parsedtext" : "not" } } 12 | ], 13 | "slop" : 1, 14 | "in_order" : true 15 | } 16 | } 17 | } 18 | } 19 | }' 20 | 21 | -------------------------------------------------------------------------------- /chapter_06/template_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/_search/template?pretty' -d ' 2 | { 3 | "inline": { 4 | "query": { 5 | "term": { 6 | "uuid": "{{value}}" 7 | } 8 | } 9 | }, 10 | "params": { 11 | "value": "22222" 12 | } 13 | }' 14 | 15 | 16 | curl -XPOST 'http://127.0.0.1:9200/_search/template/myTemplate' -d ' 17 | { 18 | "template": { 19 | "query": { 20 | "term": { 21 | "uuid": "{{value}}" 22 | } 23 | } 24 | } 25 | }' 26 | 27 | 28 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 29 | "query": 30 | { 31 | "template": { 32 | "id": "myTemplate", 33 | "params": { 34 | "value": "22222" 35 | } 36 | }} 37 | }' 38 | 39 | curl -XPOST 'http://127.0.0.1:9200/_render/template?pretty' -d '{ 40 | "id": "myTemplate", 41 | "params": { 42 | "value": "22222" 43 | } 44 | }' 45 | 46 | -------------------------------------------------------------------------------- /chapter_06/term2_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "bool": { 4 | "filter": { 5 | "term": { 6 | "uuid": "33333" 7 | } 8 | } 9 | } 10 | } 11 | }' 12 | 13 | 14 | -------------------------------------------------------------------------------- /chapter_06/term_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "term": { 4 | "uuid": "33333" 5 | } 6 | } 7 | }' 8 | 9 | 10 | -------------------------------------------------------------------------------- /chapter_06/terms_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "terms": { 4 | "uuid": ["33333", "32222"] 5 | } 6 | } 7 | }' 8 | 9 | -------------------------------------------------------------------------------- /chapter_06/wildcard_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "wildcard": { 4 | "uuid": "22?2*" 5 | } 6 | } 7 | }' 8 | 9 | 10 | -------------------------------------------------------------------------------- /chapter_07/geo.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "filtered": { 4 | "filter": { 5 | "geo_polygon": { 6 | "pin.location": { 7 | "points": [ 8 | { 9 | "lat": 50, 10 | "lon": -30 11 | }, 12 | { 13 | "lat": 30, 14 | "lon": -80 15 | }, 16 | { 17 | "lat": 80, 18 | "lon": -90 19 | } 20 | ] 21 | } 22 | } 23 | }, 24 | "query": { 25 | "match_all": {} 26 | } 27 | } 28 | } 29 | } -------------------------------------------------------------------------------- /chapter_07/geo_bbox_query.sh: -------------------------------------------------------------------------------- 1 | curl -XGET http://127.0.0.1:9200/test-mindex/_search -d '{ 2 | "query": { 3 | "geo_bounding_box": { 4 | "pin.location": { 5 | "bottom_right": { 6 | "lat": 40.03, 7 | "lon": 72.0 8 | }, 9 | "top_left": { 10 | "lat": 40.717, 11 | "lon": 70.99 12 | } 13 | } 14 | } 15 | } 16 | 17 | }' 18 | -------------------------------------------------------------------------------- /chapter_07/geo_distance_query.sh: -------------------------------------------------------------------------------- 1 | curl -XGET http://127.0.0.1:9200/test-mindex/_search -d '{ 2 | "query": { 3 | "geo_distance": { 4 | "pin.location": { 5 | "lat": 40, 6 | "lon": 70 7 | }, 8 | "distance": "200km", 9 | "optimize_bbox": "memory" 10 | } 11 | } 12 | } 13 | }' 14 | -------------------------------------------------------------------------------- /chapter_07/geo_polygon_query.sh: -------------------------------------------------------------------------------- 1 | curl -XGET http://127.0.0.1:9200/test-mindex/_search -d '{ 2 | "query": { 3 | "geo_polygon": { 4 | "pin.location": { 5 | "points": [ 6 | { 7 | "lat": 50, 8 | "lon": -30 9 | }, 10 | { 11 | "lat": 30, 12 | "lon": -80 13 | }, 14 | { 15 | "lat": 80, 16 | "lon": -90 17 | } 18 | ] 19 | } 20 | } 21 | } 22 | }' 23 | -------------------------------------------------------------------------------- /chapter_07/has_child_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "has_child" : { 4 | "type" : "test-type2", 5 | "query" : { 6 | "term" : { 7 | "value" : "value1" 8 | } 9 | } 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_07/has_child_query_sort.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "has_child" : { 4 | "type" : "test-type2", 5 | "score_mode" : "max", 6 | "query" : { 7 | "function_score" : { 8 | "script_score": { 9 | "script": "doc[\"amount\"].value" 10 | } 11 | } 12 | } 13 | } 14 | } 15 | } 16 | }' 17 | -------------------------------------------------------------------------------- /chapter_07/has_parent_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type2/_search?pretty=true' -d '{ 2 | "query": { 3 | "has_parent" : { 4 | "type" : "test-type", 5 | "query" : { 6 | "term" : { 7 | "parsedtext" : "joe" 8 | } 9 | } 10 | } 11 | } 12 | } 13 | }' 14 | -------------------------------------------------------------------------------- /chapter_07/mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "mappings": { 3 | "test-type": { 4 | "properties": { 5 | "versions": { 6 | "type": "nested", 7 | "properties": { 8 | "color": { 9 | "store": "yes", 10 | "type": "keyword" 11 | }, 12 | "size": { 13 | "store": "yes", 14 | "type": "integer" 15 | } 16 | } 17 | }, 18 | "pos": { 19 | "type": "integer", 20 | "store": "yes" 21 | }, 22 | "uuid": { 23 | "store": "yes", 24 | "type": "keyword" 25 | }, 26 | "parsedtext": { 27 | "term_vector": "with_positions_offsets", 28 | "store": "yes", 29 | "type": "text" 30 | }, 31 | "name": { 32 | "term_vector": "with_positions_offsets", 33 | "store": "yes", 34 | "fielddata": true, 35 | "type": "text", 36 | "fields": { 37 | "raw": { 38 | "type": "keyword" 39 | } 40 | } 41 | }, 42 | "title": { 43 | "term_vector": "with_positions_offsets", 44 | "store": "yes", 45 | "type": "text", 46 | "fielddata": true, 47 | "fields": { 48 | "raw": { 49 | "type": "keyword" 50 | } 51 | } 52 | } 53 | } 54 | }, 55 | "test-type2": { 56 | "_parent": { 57 | "type": "test-type" 58 | } 59 | } 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /chapter_07/nested_query.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true' -d '{ 2 | "query": { 3 | "nested" : { 4 | "path" : "versions", 5 | "score_mode" : "avg", 6 | "query" : { 7 | "bool" : { 8 | "must" : [ 9 | { "term" : {"versions.color" : "blue"} }, 10 | { "range" : {"versions.size" : {"gt" : 10}} } 11 | ] 12 | } 13 | } 14 | } 15 | } 16 | }' 17 | 18 | -------------------------------------------------------------------------------- /chapter_07/populate_geo.sh: -------------------------------------------------------------------------------- 1 | curl -XHEAD 'http://127.0.0.1:9200/test-index?pretty=true' 2 | curl -XDELETE 'http://127.0.0.1:9200/test-index?pretty=true' 3 | curl -XHEAD 'http://127.0.0.1:9200/test-mindex?pretty=true' 4 | curl -XPUT 'http://127.0.0.1:9200/test-mindex?pretty=true' 5 | curl -XPUT 'http://127.0.0.1:9200/test-mindex/test-type/_mapping?pretty=true' -d '{"test-type": {"properties": {"pin": {"properties": {"location": {"type": "geo_point"}}}}}}' 6 | curl -XPUT 'http://127.0.0.1:9200/test-mindex/test-type/1?pretty=true' -d '{"pin": {"location": {"lat": 40.12, "lon": -71.34}}}' 7 | curl -XPUT 'http://127.0.0.1:9200/test-mindex/test-type/2?pretty=true' -d '{"pin": {"location": {"lat": 40.12, "lon": 71.34}}}' 8 | curl -XPOST 'http://127.0.0.1:9200/test-mindex/_refresh?pretty=true' 9 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 10 | -------------------------------------------------------------------------------- /chapter_07/populate_relations.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/test-index?pretty=true' 2 | #curl -H "Accept: application/json" -H "X-HTTP-Method-Override: PUT" -XPUT 'http://127.0.0.1:9200/test-index/_mapping?pretty=true' -d @mapping.json 3 | 4 | curl -XPUT 'http://127.0.0.1:9200/test-index?pretty=true' -d @mapping.json 5 | 6 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/_mapping?pretty=true' -d '{"test-type2": {"_parent": {"type": "test-type"}}}' 7 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/1?pretty=true' -d '{"position": 1, "parsedtext": "Joe Testere nice guy", "name": "Joe Tester", "uuid": "11111", "price":4.0, "versions":[{"color":"yellow", "size":5},{"color":"blue", "size":15}]}' 8 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/1?parent=1&pretty=true' -d '{"name": "data1", "value": "value1", "amount":10}' 9 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/2?pretty=true' -d '{"position": 2, "parsedtext": "Bill Testere nice guy", "name": "Bill Baloney", "uuid": "22222", "price":5.0, "versions":[{"color":"red", "size":2},{"color":"blue", "size":10}]}' 10 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type2/2?parent=2&pretty=true' -d '{"name": "data2", "value": "value2", "amount":20}' 11 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/3?pretty=true' -d '{"position": 3, "parsedtext": "Bill is not\n nice guy", "name": "Bill Clinton", "uuid": "33333", "price":6.0, "versions":[{"color":"red", "size":2}]}' 12 | curl -XPOST 'http://127.0.0.1:9200/test-index/_refresh?pretty=true' 13 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 14 | 15 | 16 | -------------------------------------------------------------------------------- /chapter_08/executing_an_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST "http://127.0.0.1:9200/test-index/_search?size=0&pretty" -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "tag": { 8 | "terms": { 9 | "field": "tag", 10 | "size": 10 11 | } 12 | } 13 | } 14 | }' 15 | 16 | -------------------------------------------------------------------------------- /chapter_08/executing_children_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST "http://127.0.0.1:9200/myindex/test-type/_search?size=0&pretty" -d '{ 2 | "aggs": { 3 | "uuid": { 4 | "terms": { 5 | "field": "uuid", 6 | "size": 10 7 | }, 8 | "aggs": { 9 | "to-children": { 10 | "children": { 11 | "type": "test-type2" 12 | }, 13 | "aggs": { 14 | "top-values": { 15 | "terms": { 16 | "field": "value.keyword", 17 | "size": 10 18 | } 19 | } 20 | } 21 | } 22 | } 23 | } 24 | } 25 | }' 26 | 27 | -------------------------------------------------------------------------------- /chapter_08/executing_date_histogram_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?from=0&pretty=true&size=0' -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "date_year": { 8 | "date_histogram": { 9 | "field": "date", 10 | "interval": "year" 11 | } 12 | }, 13 | "date_quarter": { 14 | "date_histogram": { 15 | "field": "date", 16 | "interval": "quarter", 17 | "time_zone": "+01:00" 18 | } 19 | } 20 | 21 | 22 | } 23 | }' 24 | -------------------------------------------------------------------------------- /chapter_08/executing_filter_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?size=0&pretty' -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "ullam_docs": { 8 | "filter" : { 9 | "term" : { "tag" : "ullam" } 10 | } 11 | }, 12 | "age37_docs": { 13 | "filter" : { 14 | "term" : { "age" : 37 } 15 | } 16 | } 17 | } 18 | }' 19 | 20 | -------------------------------------------------------------------------------- /chapter_08/executing_filters_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?size=0&pretty' -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "expensive_docs": { 8 | "filters": { 9 | "other_bucket": true, 10 | "other_bucket_key": "other_documents", 11 | "filters": { 12 | "2016_over_50": { 13 | "bool": { 14 | "must": [ 15 | { 16 | "range": { 17 | "date": { 18 | "gte": "2016-01-01" 19 | } 20 | } 21 | }, 22 | { 23 | "range": { 24 | "price": { 25 | "gte": 50 26 | } 27 | } 28 | } 29 | ] 30 | } 31 | }, 32 | "previous_2016_over_50": { 33 | "bool": { 34 | "must": [ 35 | { 36 | "range": { 37 | "date": { 38 | "lt": "2016-01-01" 39 | } 40 | } 41 | }, 42 | { 43 | "range": { 44 | "price": { 45 | "gte": 50 46 | } 47 | } 48 | } 49 | ] 50 | } 51 | } 52 | } 53 | } 54 | } 55 | } 56 | }' 57 | 58 | -------------------------------------------------------------------------------- /chapter_08/executing_geo_bounds_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty&size=0' -d ' { 2 | "query" : { 3 | "match_all" : {} 4 | }, 5 | "aggs" : { 6 | "box" : { 7 | "geo_bounds" : { 8 | "field":"position", 9 | "wrap_longitude" : true 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_08/executing_geo_centroid_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty&size=0' -d ' { 2 | "query" : { 3 | "match_all" : {} 4 | }, 5 | "aggs" : { 6 | "centroid" : { 7 | "geo_centroid" : { 8 | "field":"position" 9 | } 10 | } 11 | } 12 | }' -------------------------------------------------------------------------------- /chapter_08/executing_geo_distance_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d ' { 2 | "query" : { 3 | "match_all" : {} 4 | }, 5 | "aggs" : { 6 | "position" : { 7 | "geo_distance" : { 8 | "field":"position", 9 | "origin" : { 10 | "lat": 83.76, 11 | "lon": -81.20 12 | }, 13 | "ranges" : [ 14 | { "to" : 10 }, 15 | { "from" : 10, "to" : 20 }, 16 | { "from" : 20, "to" : 50 }, 17 | { "from" : 50, "to" : 100 }, 18 | { "from" : 100 } 19 | ] 20 | } 21 | } 22 | } 23 | }' 24 | -------------------------------------------------------------------------------- /chapter_08/executing_global_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?size=0&pretty=true' -d ' 2 | { 3 | "query": { 4 | "term" : { "tag" : "ullam" } 5 | }, 6 | "aggregations": { 7 | "query_age_avg": { 8 | "avg" : { 9 | "field" : "age" 10 | } 11 | }, 12 | "all_persons":{ 13 | "global": {}, 14 | "aggs":{ 15 | "age_global_avg": { 16 | "avg" : { 17 | "field" : "age" 18 | } 19 | } 20 | } 21 | } 22 | } 23 | }' 24 | -------------------------------------------------------------------------------- /chapter_08/executing_matrix_stats.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?size=0&pretty' -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "matrixstats": { 8 | "matrix_stats": { 9 | "fields": ["age", "price"] 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_08/executing_missing_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?size=0&pretty' -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "missin_code": { 8 | "missing" : { 9 | "field" : "code" 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_08/executing_range_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "aggs": { 6 | "prices": { 7 | "range": { 8 | "field": "price", 9 | "ranges": [ 10 | {"to": 10}, 11 | {"from": 10,"to": 20}, 12 | {"from": 20,"to": 100}, 13 | {"from": 100} 14 | ] 15 | } 16 | }, 17 | "ages": { 18 | "range": { 19 | "field": "age", 20 | "ranges": [ 21 | {"to": 25}, 22 | {"from": 25,"to": 50}, 23 | {"from": 50,"to": 75}, 24 | {"from": 75} 25 | ] 26 | } 27 | }, 28 | "range": { 29 | "range": { 30 | "field": "date", 31 | "ranges": [ 32 | {"from": "2012-01-01","to": "2012-07-01"}, 33 | {"from": "2012-07-01","to": "2012-12-31"}, 34 | {"from": "2013-01-01","to": "2013-12-31"} 35 | ] 36 | } 37 | } 38 | } 39 | }' 40 | 41 | -------------------------------------------------------------------------------- /chapter_08/executing_significant_terms_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 2 | "query" : { 3 | "terms" : {"tag" : [ "ullam", "in", "ex" ]} 4 | }, 5 | "aggs": { 6 | "significant_tags": { 7 | "significant_terms": { 8 | "field": "tag" 9 | } 10 | } 11 | } 12 | }' 13 | 14 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 15 | "query" : { 16 | "terms" : {"tag" : [ "ullam", "in", "ex" ]} 17 | }, 18 | "aggs": { 19 | "significant_tags": { 20 | "terms": { 21 | "field": "tag" 22 | } 23 | } 24 | } 25 | }' 26 | 27 | -------------------------------------------------------------------------------- /chapter_08/executing_stat_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST "http://127.0.0.1:9200/test-index/_search?size=0&pretty" -d ' 2 | { 3 | "query": { 4 | "match_all": {} 5 | }, 6 | "aggs": { 7 | "age_stats": { 8 | "extended_stats": { 9 | "field": "age" 10 | } 11 | } 12 | } 13 | }' 14 | -------------------------------------------------------------------------------- /chapter_08/executing_terms_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "aggs": { 6 | "tag": { 7 | "terms": { 8 | "field": "tag", 9 | "size": 10 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_08/executing_top_hit_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "size": 0, 6 | "aggs": { 7 | "tags": { 8 | "terms": { 9 | "field": "tag", 10 | "size": 2 11 | }, 12 | "aggs": { 13 | "top_tag_hits": { 14 | "top_hits": { 15 | "sort": [ 16 | { 17 | "age": { 18 | "order": "desc" 19 | } 20 | } 21 | ], 22 | "_source": { 23 | "includes": [ 24 | "name" 25 | ] 26 | }, 27 | "size": 1 28 | } 29 | } 30 | } 31 | } 32 | } 33 | }' 34 | 35 | -------------------------------------------------------------------------------- /chapter_08/fix/executing_histogram_aggregations.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=0' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "aggs": { 6 | "age" : { 7 | "histogram" : { 8 | "field" : "age", 9 | "interval" : 5 10 | } 11 | }, 12 | "price" : { 13 | "histogram" : { 14 | "field" : "price", 15 | "interval" : 10.0 16 | } 17 | } 18 | } 19 | }' 20 | 21 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=0' -d '{ 22 | "query": { 23 | "match_all": {} 24 | }, 25 | "aggs": { 26 | "age" : { 27 | "histogram" : { 28 | "field" : "age", 29 | "script": "_value*3", 30 | "interval" : 5 31 | } 32 | } 33 | } 34 | }' 35 | 36 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=0' -d '{ 37 | "query": { 38 | "match_all": {} 39 | }, 40 | "aggs": { 41 | "age" : { 42 | "histogram" : { 43 | "script": "doc['age'].value", 44 | "interval" : 5 45 | } 46 | } 47 | } 48 | }' 49 | -------------------------------------------------------------------------------- /chapter_08/geo_bounds_aggregation.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?pretty&size=0' -d ' { 2 | "query" : { 3 | "match_all" : {} 4 | }, 5 | "aggs" : { 6 | "box" : { 7 | "geo_bounds" : { 8 | "field":"position", 9 | "wrap_longitude" : true 10 | } 11 | } 12 | } 13 | }' 14 | 15 | -------------------------------------------------------------------------------- /chapter_08/populate.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/test-index?pretty=true' 2 | curl -XPUT 'http://127.0.0.1:9200/test-index?pretty=true' 3 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/_mapping?pretty=true' -d '{"test-type": {"properties": {"name": {"index": "analyzed", "term_vector": "with_positions_offsets", "boost": 1.0, "store": "yes", "type": "string"}, "title": {"index": "analyzed", "term_vector": "with_positions_offsets", "boost": 1.0, "store": "yes", "type": "string"}, "parsedtext": {"index": "analyzed", "term_vector": "with_positions_offsets", "boost": 1.0, "store": "yes", "type": "string"}, "tag": {"type": "string", "store": "yes"}, "date": {"type": "date", "store": "yes"}, "position": {"type": "integer", "store": "yes"}, "uuid": {"index": "not_analyzed", "boost": 1.0, "store": "yes", "type": "string"}}}}' 4 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/1?pretty=true' -d '{"name": "Joe Tester", "parsedtext": "Joe Testere nice guy", "tag": "foo", "date": "2011-05-16T00:00:00", "position": 1, "uuid": "11111"}' 5 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/2?pretty=true' -d '{"name": " Bill Baloney", "parsedtext": "Bill Testere nice guy", "tag": "foo", "date": "2011-04-16T00:00:00", "position": 2, "uuid": "22222"}' 6 | curl -XPUT 'http://127.0.0.1:9200/test-index/test-type/3?pretty=true' -d '{"name": "Bill Clinton", "parsedtext": "Bill is not nice guy", "tag": "bar", "date": "2011-04-28T00:00:00", "position": 3, "uuid": "33333"}' 7 | curl -XPOST 'http://127.0.0.1:9200/test-index/_refresh?pretty=true' 8 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 9 | -------------------------------------------------------------------------------- /chapter_08/populate_parent_child.sh: -------------------------------------------------------------------------------- 1 | #curl -XHEAD 'http://127.0.0.1:9200/myindex?pretty=true' 2 | curl -XDELETE 'http://127.0.0.1:9200/myindex?pretty=true' 3 | curl -XPUT 'http://127.0.0.1:9200/myindex?pretty=true' -d '{"mappings": {"test-type": {"properties": {"pos": {"type": "integer", "store": "yes"}, "uuid": {"store": "yes", "type": "keyword"}, "parsedtext": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "name": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}, "title": {"term_vector": "with_positions_offsets", "store": "yes", "type": "text"}}}, "test-type2": {"_parent": {"type": "test-type"}}}}' 4 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/1?pretty=true' -d '{"position": 1, "parsedtext": "Joe Testere nice guy", "name": "Joe Tester", "uuid": "11111", "price":4.0}' 5 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type2/1?parent=1&pretty=true' -d '{"name": "data1", "value": "value1"}' 6 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/2?pretty=true' -d '{"position": 2, "parsedtext": "Bill Testere nice guy", "name": "Bill Baloney", "uuid": "22222", "price":5.0}' 7 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type2/2?parent=2&pretty=true' -d '{"name": "data2", "value": "value2"}' 8 | curl -XPUT 'http://127.0.0.1:9200/myindex/test-type/3?pretty=true' -d '{"position": 3, "parsedtext": "Bill is not\n nice guy", "name": "Bill Clinton", "uuid": "33333", "price":6.0}' 9 | curl -XPOST 'http://127.0.0.1:9200/myindex/_refresh?pretty=true' 10 | curl -XGET 'http://127.0.0.1:9200/_cluster/health?wait_for_status=green&timeout=0s&pretty=true' 11 | 12 | 13 | -------------------------------------------------------------------------------- /chapter_09/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aparo/elasticsearch-cookbook-third-edition/3599aab581e1e537b35a65ee82ae8edd2c8102c3/chapter_09/.DS_Store -------------------------------------------------------------------------------- /chapter_09/aggregation_script.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d ' { 2 | "aggs": { 3 | "my_value": { 4 | "sum": { 5 | "script": { 6 | "inline": "doc[\"price\"].value * doc[\"price\"].value" 7 | } 8 | } 9 | } 10 | } 11 | }' 12 | 13 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 14 | "aggs": { 15 | "my_value": { 16 | "sum": { 17 | "field":"price", 18 | "script": { 19 | "inline": "_value * _value" 20 | } 21 | } 22 | } 23 | } 24 | }' 25 | 26 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?pretty=true&size=0' -d '{ 27 | "aggs": { 28 | "my_value": { 29 | "terms": { 30 | "field":"tag", 31 | "size":5, 32 | "script": { 33 | "inline": "if(params.replace.containsKey(_value.toUpperCase())) { params.replace[_value.toUpperCase()] } else { _value.toUpperCase() }", 34 | "params":{"replace":{"PORRO":"Result1", "LABORUM":"Result2"}} 35 | } 36 | } 37 | } 38 | } 39 | }' 40 | -------------------------------------------------------------------------------- /chapter_09/call_indexed_script.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "sort": { 6 | "_script": { 7 | "script":{ 8 | "id": "my_script", 9 | "lang": "groovy", 10 | "params": { 11 | "factor": 1.1 12 | } 13 | }, 14 | "type": "number", 15 | "order": "asc" 16 | } 17 | } 18 | }' 19 | -------------------------------------------------------------------------------- /chapter_09/complex_script.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/9/_update?&pretty=true' -d ' 2 | { 3 | "script": { 4 | "inline": "ctx.op = \"none\";\n if(ctx._source.containsValue(\"tags\")){\n for(def item : params.new_tags){\n if(!ctx._source.tags.contains(item)){\n ctx._source.tags.add(item);\n ctx.op = \"index\";\n }\n }\n }else{ ctx._source.tags=params.new_tags; ctx.op = \"index\" }\n if(ctx._source.containsValue(\"labels\")){\n for(def item : params.new_labels){\n if(!ctx._source.labels.contains(item)){\n ctx._source.labels.add(item);\n ctx.op = \"index\"\n }\n }\n }else{\n ctx._source.labels=params.new_labels;\n ctx.op = \"index\"}", 5 | "params": { 6 | "new_tags": [ 7 | "cool", 8 | "nice" 9 | ], 10 | "new_labels": [ 11 | "red", 12 | "blue", 13 | "green" 14 | ] 15 | } 16 | } 17 | }' 18 | -------------------------------------------------------------------------------- /chapter_09/field_script.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "script_fields" : { 6 | "my_calc_field" : { 7 | "script" : "doc[\"name\"].value + \" -- \" + doc[\"description\"].value" 8 | }, 9 | "my_calc_field2" : { 10 | "script" : "doc[\"price\"].value * discount", 11 | "params" : { 12 | "discount" : 0.8 13 | } 14 | } 15 | } 16 | }' 17 | -------------------------------------------------------------------------------- /chapter_09/filter_script.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 2 | "query": { 3 | "bool": { 4 | "filter": { 5 | "script": { 6 | "script": { 7 | "inline": "doc[\"age\"].value > params.param1", 8 | "params": { 9 | "param1": 80 10 | } 11 | } 12 | } 13 | } 14 | } 15 | } 16 | }' 17 | 18 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 19 | "query": { 20 | "script": { 21 | "script": { 22 | "inline": "doc[\"age\"].value > param1", 23 | "lang": "javascript", 24 | "params": { 25 | "param1": 80 26 | } 27 | } 28 | } 29 | } 30 | }' 31 | 32 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 33 | "query": { 34 | "filtered": { 35 | "filter": { 36 | "script": { 37 | "script": "doc[\"age\"].value > param1", 38 | "lang":"python", 39 | "params" : { 40 | "param1" : 80 41 | } 42 | } 43 | }, 44 | "query": { 45 | "match_all": {} 46 | } 47 | } 48 | } 49 | }' -------------------------------------------------------------------------------- /chapter_09/indexed_script.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST localhost:9200/_scripts/groovy/my_script -d '{ 2 | "script":"doc[\"price\"].value * factor" 3 | }' -------------------------------------------------------------------------------- /chapter_09/painless_script_score.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "function_score": { 4 | "script_score": { 5 | "script": { 6 | "lang": "painless", 7 | "inline": "doc['price'].value * 1.2" 8 | } 9 | } 10 | } 11 | } 12 | } -------------------------------------------------------------------------------- /chapter_09/painless_script_score.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty&size=2' -d @painless_script_score.json 2 | -------------------------------------------------------------------------------- /chapter_09/sort_ramdom_script.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty&size=3' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "sort": { 6 | "_script": { 7 | "script": { 8 | "inline": "Math.random()" 9 | }, 10 | "type": "number", 11 | "order": "asc" 12 | } 13 | } 14 | }' 15 | -------------------------------------------------------------------------------- /chapter_09/sort_script.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/test-index/test-type/_search?&pretty=true&size=3' -d '{ 2 | "query": { 3 | "match_all": {} 4 | }, 5 | "sort": { 6 | "_script": { 7 | "script": { 8 | "inline": "Math.sqrt(doc[\"price\"].value * params.factor)", 9 | "params": { 10 | "factor": 1.1 11 | } 12 | }, 13 | "type": "number", 14 | "order": "asc" 15 | } 16 | } 17 | }' 18 | -------------------------------------------------------------------------------- /chapter_09/update_script.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/9/_update?&pretty=true' -d '{ 2 | "script" : { 3 | "inline":"ctx._source.tag.add(params.tag)", 4 | "params" : { 5 | "tag" : "cool" 6 | } 7 | } 8 | }' 9 | 10 | 11 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/9/_update?&pretty=true' -d '{ 12 | "script" : "ctx._source.tag += tag", 13 | "lang":"js", 14 | "params" : { 15 | "tag" : "cool" 16 | } 17 | }' 18 | 19 | 20 | curl -XPOST 'http://127.0.0.1:9200/test-index/test-type/8/_update?&pretty=true' -d '{ 21 | "script" : "ctx[\"_source\"][\"tag\"] = list(ctx[\"_source\"][\"tag\"]) + [tag]", 22 | "lang":"python", 23 | "params" : { 24 | "tag" : "cool" 25 | } 26 | }' 27 | 28 | -------------------------------------------------------------------------------- /chapter_13/attachment_processor.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT 'http://127.0.0.1:9200/_ingest/pipeline/attachment' -d '{ 2 | "description" : "Extract data from an attachment via Tika", 3 | "processors" : [ 4 | { 5 | "attachment" : { 6 | "field" : "data" 7 | } 8 | } 9 | ], 10 | "version":1 11 | }' 12 | 13 | curl -XPUT 'http://127.0.0.1:9200/my_index/my_type/my_id?pipeline=attachment' -d '{ 14 | "data": "e1xydGYxXGFuc2kNCkxvcmVtIGlwc3VtIGRvbG9yIHNpdCBhbWV0DQpccGFyIH0=" 15 | }' 16 | 17 | curl -XGET 'http://127.0.0.1:9200/my_index/my_type/my_id?pretty' 18 | -------------------------------------------------------------------------------- /chapter_13/builtin_processors.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/_ingest/pipeline/_simulate?pretty' -d '{ 2 | "pipeline": { 3 | "description": "Testing some build-processors", 4 | "processors": [ 5 | { 6 | "dot_expander": { 7 | "field": "extfield.innerfield" 8 | } 9 | }, 10 | { 11 | "remove": { 12 | "field": "unwanted" 13 | } 14 | }, 15 | { 16 | "trim": { 17 | "field": "message" 18 | } 19 | }, 20 | { 21 | "set": { 22 | "field": "tokens", 23 | "value": "{{message}}" 24 | } 25 | }, 26 | { 27 | "split": { 28 | "field": "tokens", 29 | "separator": "\\s+" 30 | } 31 | }, 32 | { 33 | "sort": { 34 | "field": "tokens", 35 | "order": "desc" 36 | } 37 | }, 38 | { 39 | "convert": { 40 | "field": "mynumbertext", 41 | "target_field": "mynumber", 42 | "type": "integer" 43 | } 44 | } 45 | ] 46 | }, 47 | "docs": [ 48 | { 49 | "_index": "index", 50 | "_type": "type", 51 | "_id": "1", 52 | "_source": { 53 | "extfield.innerfield": "booo", 54 | "unwanted": 32243, 55 | "message": " 155.2.124.3 GET /index.html 15442 0.038 ", 56 | "mynumbertext": "3123" 57 | } 58 | } 59 | ] 60 | }' 61 | -------------------------------------------------------------------------------- /chapter_13/delete_pipeline.sh: -------------------------------------------------------------------------------- 1 | curl -XDELETE 'http://127.0.0.1:9200/_ingest/pipeline/add-user-john' 2 | -------------------------------------------------------------------------------- /chapter_13/geoip_processor.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT 'http://127.0.0.1:9200/_ingest/pipeline/geoip' -d '{ 2 | "description" : "Extract geopoint from an IP", 3 | "processors" : [ 4 | { 5 | "geoip" : { 6 | "field" : "ip" 7 | } 8 | } 9 | ], 10 | "version":1 11 | }' 12 | 13 | curl -XPUT 'http://127.0.0.1:9200/my_index/my_type/my_id?pipeline=geoip' -d '{ 14 | "ip": "8.8.8.8" 15 | }' 16 | 17 | curl -XGET 'http://127.0.0.1:9200/my_index/my_type/my_id?pretty' 18 | 19 | -------------------------------------------------------------------------------- /chapter_13/get_pipeline.sh: -------------------------------------------------------------------------------- 1 | curl -XGET 'http://127.0.0.1:9200/_ingest/pipeline/add-user-john?pretty' 2 | 3 | curl -XGET 'http://127.0.0.1:9200/_ingest/pipeline/add-user-john?filter_path=*.version&pretty' 4 | -------------------------------------------------------------------------------- /chapter_13/grok_processor.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/_ingest/pipeline/_simulate?pretty' -d '{ 2 | "pipeline": { 3 | "description": "Testing grok pattern", 4 | "processors": [ 5 | { 6 | "grok": { 7 | "field": "message", 8 | "patterns": [ 9 | "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}" 10 | ] 11 | } 12 | } 13 | ] 14 | }, 15 | "docs": [ 16 | { 17 | "_index": "index", 18 | "_type": "type", 19 | "_id": "1", 20 | "_source": { 21 | "message": "155.2.124.3 GET /index.html 15442 0.038" 22 | } 23 | } 24 | ] 25 | }' 26 | 27 | curl -XPOST 'http://127.0.0.1:9200/_ingest/pipeline/_simulate?pretty' -d '{ 28 | "pipeline": { 29 | "description" : "custom grok pattern", 30 | "processors": [ 31 | { 32 | "grok": { 33 | "field": "message", 34 | "patterns": ["my favorite color is %{COLOR:color}"], 35 | "pattern_definitions" : { 36 | "COLOR" : "RED|GREEN|BLUE" 37 | } 38 | } 39 | } 40 | ] 41 | }, 42 | "docs":[ 43 | { 44 | "_source": { 45 | "message": "my favorite color is RED" 46 | } 47 | }, 48 | { 49 | "_source": { 50 | "message": "happy fail!!" 51 | } 52 | } 53 | ] 54 | }' -------------------------------------------------------------------------------- /chapter_13/put_pipeline.sh: -------------------------------------------------------------------------------- 1 | curl -XPUT 'http://127.0.0.1:9200/_ingest/pipeline/add-user-john' -d '{ 2 | "description" : "Add user john field", 3 | "processors" : [ 4 | { 5 | "set" : { 6 | "field": "user", 7 | "value": "john" 8 | } 9 | } 10 | ], 11 | "version":1 12 | }' 13 | -------------------------------------------------------------------------------- /chapter_13/simulate_pipeline.sh: -------------------------------------------------------------------------------- 1 | curl -XPOST 'http://127.0.0.1:9200/_ingest/pipeline/_simulate?pretty' -d '{ 2 | "pipeline": { 3 | "description": "Add user john field", 4 | "processors": [ 5 | { 6 | "set": { 7 | "field": "user", 8 | "value": "john" 9 | } 10 | }, 11 | { 12 | "set": { 13 | "field": "job", 14 | "value": 10 15 | } 16 | } 17 | ], 18 | "version": 1 19 | }, 20 | "docs": [ 21 | { 22 | "_index": "index", 23 | "_type": "type", 24 | "_id": "1", 25 | "_source": { 26 | "name": "docs1" 27 | } 28 | }, 29 | { 30 | "_index": "index", 31 | "_type": "type", 32 | "_id": "2", 33 | "_source": { 34 | "name": "docs2" 35 | } 36 | } 37 | ] 38 | }' 39 | 40 | curl -XPOST 'http://127.0.0.1:9200/_ingest/pipeline/_simulate?verbose&pretty' -d '{ 41 | "pipeline": { 42 | "description": "Add user john field", 43 | "processors": [ 44 | { 45 | "set": { 46 | "field": "user", 47 | "value": "john" 48 | } 49 | }, 50 | { 51 | "set": { 52 | "field": "job", 53 | "value": 10 54 | } 55 | } 56 | ], 57 | "version": 1 58 | }, 59 | "docs": [ 60 | { 61 | "_index": "index", 62 | "_type": "type", 63 | "_id": "1", 64 | "_source": { 65 | "name": "docs1" 66 | } 67 | }, 68 | { 69 | "_index": "index", 70 | "_type": "type", 71 | "_id": "2", 72 | "_source": { 73 | "name": "docs2" 74 | } 75 | } 76 | ] 77 | }' 78 | -------------------------------------------------------------------------------- /chapter_14/http_es_client/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.packtpub 5 | http_es_client 6 | jar 7 | 1.0-SNAPSHOT 8 | httpclient 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 4.12 15 | test 16 | 17 | 18 | org.elasticsearch.client 19 | rest 20 | 5.1.1 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /chapter_14/http_es_client/src/main/java/com/packtpub/App.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.apache.http.HttpEntity; 4 | import org.apache.http.HttpHost; 5 | import org.apache.http.HttpStatus; 6 | import org.apache.http.util.EntityUtils; 7 | import org.elasticsearch.client.Response; 8 | import org.elasticsearch.client.RestClient; 9 | 10 | import java.io.IOException; 11 | 12 | public class App { 13 | 14 | public static void main(String[] args) { 15 | RestClient client = RestClient.builder( 16 | new HttpHost("localhost", 9200, "http")).build(); 17 | 18 | try { 19 | Response response = client.performRequest("GET", "/test-index/test-type/1"); 20 | 21 | if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { 22 | System.err.println("Method failed: " + response.getStatusLine()); 23 | } else { 24 | HttpEntity entity = response.getEntity(); 25 | String responseBody = EntityUtils.toString(entity); 26 | System.out.println(responseBody); 27 | } 28 | 29 | } catch (IOException e) { 30 | System.err.println("Fatal transport error: " + e.getMessage()); 31 | e.printStackTrace(); 32 | } finally { 33 | // Release the connection. 34 | try { 35 | client.close(); 36 | } catch (IOException e) { 37 | e.printStackTrace(); 38 | } 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /chapter_14/http_es_client/src/test/java/com/packtpub/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /chapter_14/http_java_client/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.packtpub 5 | httpjavaclient 6 | jar 7 | 1.0-SNAPSHOT 8 | httpclient 9 | http://maven.apache.org 10 | 11 | 12 | junit 13 | junit 14 | 4.12 15 | test 16 | 17 | 18 | org.apache.httpcomponents 19 | httpclient 20 | 4.5.2 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /chapter_14/http_java_client/src/main/java/com/packtpub/App.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.apache.http.*; 4 | import org.apache.http.client.methods.CloseableHttpResponse; 5 | import org.apache.http.client.methods.HttpGet; 6 | import org.apache.http.impl.client.CloseableHttpClient; 7 | import org.apache.http.impl.client.HttpClients; 8 | import org.apache.http.util.EntityUtils; 9 | 10 | import java.io.*; 11 | 12 | public class App { 13 | 14 | private static String wsUrl = "http://127.0.0.1:9200"; 15 | 16 | public static void main(String[] args) { 17 | CloseableHttpClient client = HttpClients.custom() 18 | .setRetryHandler(new MyRequestRetryHandler()).build(); 19 | 20 | HttpGet method = new HttpGet(wsUrl+"/test-index/test-type/1"); 21 | // Execute the method. 22 | 23 | try { 24 | CloseableHttpResponse response = client.execute(method); 25 | 26 | if (response.getStatusLine().getStatusCode() != HttpStatus.SC_OK) { 27 | System.err.println("Method failed: " + response.getStatusLine()); 28 | }else{ 29 | HttpEntity entity = response.getEntity(); 30 | String responseBody = EntityUtils.toString(entity); 31 | System.out.println(responseBody); 32 | } 33 | 34 | } catch (IOException e) { 35 | System.err.println("Fatal transport error: " + e.getMessage()); 36 | e.printStackTrace(); 37 | } finally { 38 | // Release the connection. 39 | method.releaseConnection(); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /chapter_14/http_java_client/src/main/java/com/packtpub/MyRequestRetryHandler.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.apache.http.HttpEntityEnclosingRequest; 4 | import org.apache.http.HttpRequest; 5 | import org.apache.http.client.HttpRequestRetryHandler; 6 | import org.apache.http.client.protocol.HttpClientContext; 7 | import org.apache.http.conn.ConnectTimeoutException; 8 | import org.apache.http.protocol.HttpContext; 9 | 10 | import javax.net.ssl.SSLException; 11 | import java.io.IOException; 12 | import java.io.InterruptedIOException; 13 | import java.net.UnknownHostException; 14 | 15 | public class MyRequestRetryHandler implements HttpRequestRetryHandler { 16 | @Override 17 | public boolean retryRequest( 18 | IOException exception, 19 | int executionCount, 20 | HttpContext context) { 21 | if (executionCount >= 3) { 22 | // Do not retry if over max retry count 23 | return false; 24 | } 25 | if (exception instanceof InterruptedIOException) { 26 | // Timeout 27 | return false; 28 | } 29 | if (exception instanceof UnknownHostException) { 30 | // Unknown host 31 | return false; 32 | } 33 | if (exception instanceof ConnectTimeoutException) { 34 | // Connection refused 35 | return false; 36 | } 37 | if (exception instanceof SSLException) { 38 | // SSL handshake exception 39 | return false; 40 | } 41 | HttpClientContext clientContext = HttpClientContext.adapt(context); 42 | HttpRequest request = clientContext.getRequest(); 43 | return !(request instanceof HttpEntityEnclosingRequest); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /chapter_14/http_java_client/src/test/java/com/packtpub/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.packtpub 5 | nativeclient 6 | jar 7 | 1.0-SNAPSHOT 8 | nativeclient 9 | http://maven.apache.org 10 | 11 | 12 | org.elasticsearch.client 13 | transport 14 | 5.1.1 15 | 16 | 17 | org.elasticsearch.plugin 18 | transport-netty4-client 19 | 5.1.1 20 | 21 | 22 | org.apache.logging.log4j 23 | log4j-api 24 | 2.6.2 25 | 26 | 27 | org.apache.logging.log4j 28 | log4j-core 29 | 2.6.2 30 | 31 | 32 | junit 33 | junit 34 | 4.12 35 | test 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/AggregationExample.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.elasticsearch.client.Client; 5 | import java.net.UnknownHostException; 6 | import org.elasticsearch.search.aggregations.AggregationBuilder; 7 | import org.elasticsearch.search.aggregations.bucket.terms.Terms; 8 | import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStats; 9 | import org.elasticsearch.search.aggregations.metrics.stats.extended.ExtendedStatsAggregationBuilder; 10 | import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; 11 | import static org.elasticsearch.search.aggregations.AggregationBuilders.extendedStats; 12 | import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; 13 | 14 | public class AggregationExample { 15 | public static void main(String[] args) throws UnknownHostException { 16 | String index = "mytest"; 17 | String type = "mytype"; 18 | QueryHelper qh = new QueryHelper(); 19 | qh.populateData(index, type); 20 | Client client = qh.getClient(); 21 | AggregationBuilder aggsBuilder = terms("tag").field("tag"); 22 | ExtendedStatsAggregationBuilder aggsBuilder2 = extendedStats("number1").field("number1"); 23 | 24 | SearchResponse response = client.prepareSearch(index).setTypes(type).setSize(0) 25 | .setQuery(matchAllQuery()).addAggregation(aggsBuilder). 26 | addAggregation(aggsBuilder2) 27 | .execute().actionGet(); 28 | if (response.status().getStatus() == 200) { 29 | System.out.println("Matched number of documents: " + response.getHits().totalHits()); 30 | Terms termsAggs = response.getAggregations().get("tag"); 31 | System.out.println("Aggregation name: " + termsAggs.getName()); 32 | System.out.println("Aggregation total: " + termsAggs.getBuckets().size()); 33 | for (Terms.Bucket entry : termsAggs.getBuckets()) { 34 | System.out.println(" - " + entry.getKey() + " " + entry.getDocCount()); 35 | } 36 | 37 | ExtendedStats extStats = response.getAggregations().get("number1"); 38 | System.out.println("Aggregation name: " + extStats.getName()); 39 | System.out.println("Count: " + extStats.getCount()); 40 | System.out.println("Min: " + extStats.getMin()); 41 | System.out.println("Max: " + extStats.getMax()); 42 | System.out.println("Standard Deviation: " + extStats.getStdDeviation()); 43 | System.out.println("Sum of Squares: " + extStats.getSumOfSquares()); 44 | System.out.println("Variance: " + extStats.getVariance()); 45 | 46 | } 47 | qh.dropIndex(index); 48 | 49 | //we need to close the client to free resources 50 | client.close(); 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/BulkOperations.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.common.xcontent.XContentFactory; 6 | import org.elasticsearch.script.Script; 7 | 8 | import java.io.IOException; 9 | import java.net.UnknownHostException; 10 | 11 | public class BulkOperations { 12 | 13 | 14 | public static void main(String[] args) throws UnknownHostException { 15 | String index = "mytest"; 16 | String type = "mytype"; 17 | Client client = NativeClient.createTransportClient(); 18 | IndicesOperations io = new IndicesOperations(client); 19 | if (io.checkIndexExists(index)) 20 | io.deleteIndex(index); 21 | 22 | try { 23 | client.admin().indices().prepareCreate(index) 24 | .addMapping(type, XContentFactory.jsonBuilder() 25 | .startObject() 26 | .startObject(type) 27 | .startObject("properties") 28 | .startObject("position").field("type", "integer").field("store", "yes").endObject() 29 | .endObject() 30 | .endObject() 31 | .endObject()) 32 | .execute().actionGet(); 33 | } catch (IOException e) { 34 | System.out.println("Unable to create mapping"); 35 | } 36 | 37 | BulkRequestBuilder bulker = client.prepareBulk(); 38 | for (Integer i = 1; i <= 1000; i++) { 39 | bulker.add(client.prepareIndex(index, type, i.toString()).setSource("position", i.toString())); 40 | } 41 | System.out.println("Number of actions for index: " + bulker.numberOfActions()); 42 | bulker.execute().actionGet(); 43 | 44 | bulker = client.prepareBulk(); 45 | for (Integer i = 1; i <= 1000; i++) { 46 | bulker.add(client.prepareUpdate(index, type, i.toString()).setScript(new Script("ctx._source.position += 2"))); 47 | } 48 | System.out.println("Number of actions for update: " + bulker.numberOfActions()); 49 | bulker.execute().actionGet(); 50 | 51 | bulker = client.prepareBulk(); 52 | for (Integer i = 1; i <= 1000; i++) { 53 | bulker.add(client.prepareDelete(index, type, i.toString())); 54 | } 55 | System.out.println("Number of actions for delete: " + bulker.numberOfActions()); 56 | bulker.execute().actionGet(); 57 | 58 | io.deleteIndex(index); 59 | 60 | //we need to close the client to free resources 61 | client.close(); 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/DocumentOperations.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.delete.DeleteResponse; 4 | import org.elasticsearch.action.get.GetResponse; 5 | import org.elasticsearch.action.index.IndexResponse; 6 | import org.elasticsearch.action.update.UpdateResponse; 7 | import org.elasticsearch.client.Client; 8 | import org.elasticsearch.common.xcontent.XContentFactory; 9 | import org.elasticsearch.script.Script; 10 | 11 | import java.io.IOException; 12 | import java.net.UnknownHostException; 13 | 14 | public class DocumentOperations { 15 | 16 | 17 | public static void main(String[] args) throws UnknownHostException { 18 | String index = "mytest"; 19 | String type = "mytype"; 20 | Client client = NativeClient.createTransportClient(); 21 | IndicesOperations io = new IndicesOperations(client); 22 | if (io.checkIndexExists(index)) 23 | io.deleteIndex(index); 24 | 25 | try { 26 | client.admin().indices().prepareCreate(index) 27 | .addMapping(type, XContentFactory.jsonBuilder() 28 | .startObject() 29 | .startObject(type) 30 | .startObject("properties") 31 | .startObject("text").field("type", "text").field("store", "yes").endObject() 32 | .endObject() 33 | .endObject() 34 | .endObject()) 35 | .execute().actionGet(); 36 | } catch (IOException e) { 37 | System.out.println("Unable to create mapping"); 38 | } 39 | 40 | IndexResponse ir = client.prepareIndex(index, type, "2").setSource("text", "unicorn").execute().actionGet(); 41 | System.out.println("Version: " + ir.getVersion()); 42 | GetResponse gr = client.prepareGet(index, type, "2").execute().actionGet(); 43 | System.out.println("Version: " + gr.getVersion()); 44 | 45 | UpdateResponse ur = client.prepareUpdate(index, type, "2").setScript(new Script("ctx._source.text = 'v2'")).execute().actionGet(); 46 | System.out.println("Version: " + ur.getVersion()); 47 | 48 | DeleteResponse dr = client.prepareDelete(index, type, "2").execute().actionGet(); 49 | io.deleteIndex(index); 50 | 51 | //we need to close the client to free resources 52 | client.close(); 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/IndicesOperations.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.admin.indices.exists.indices.IndicesExistsResponse; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.node.NodeValidationException; 6 | 7 | import java.io.IOException; 8 | import java.net.UnknownHostException; 9 | 10 | public class IndicesOperations { 11 | private final Client client; 12 | 13 | public IndicesOperations(Client client) { 14 | this.client = client; 15 | } 16 | 17 | public boolean checkIndexExists(String name){ 18 | IndicesExistsResponse response=client.admin().indices().prepareExists(name).execute().actionGet(); 19 | return response.isExists(); 20 | } 21 | 22 | public void createIndex(String name){ 23 | client.admin().indices().prepareCreate(name).execute().actionGet(); 24 | 25 | } 26 | 27 | public void deleteIndex(String name){ 28 | client.admin().indices().prepareDelete(name).execute().actionGet(); 29 | } 30 | 31 | public void closeIndex(String name){ 32 | client.admin().indices().prepareClose(name).execute().actionGet(); 33 | } 34 | 35 | public void openIndex(String name){ 36 | client.admin().indices().prepareOpen(name).execute().actionGet(); 37 | } 38 | 39 | public void putMapping(String index, String typeName, String source){ 40 | client.admin().indices().preparePutMapping(index).setType(typeName).setSource(source).execute().actionGet(); 41 | } 42 | 43 | 44 | public static void main( String[] args ) throws InterruptedException, IOException, NodeValidationException { 45 | NativeClient nativeClient=new NativeClient(); 46 | Client client =nativeClient.getClient(); 47 | IndicesOperations io=new IndicesOperations(client); 48 | String myIndex = "test"; 49 | if(io.checkIndexExists(myIndex)) 50 | io.deleteIndex(myIndex); 51 | io.createIndex(myIndex); 52 | Thread.sleep(1000); 53 | io.closeIndex(myIndex); 54 | io.openIndex(myIndex); 55 | io.deleteIndex(myIndex); 56 | 57 | //we need to close the client to free resources 58 | nativeClient.close(); 59 | 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/MappingOperations.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.admin.indices.mapping.put.PutMappingResponse; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.common.xcontent.XContentBuilder; 6 | 7 | import java.io.IOException; 8 | import java.net.UnknownHostException; 9 | 10 | import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; 11 | 12 | public class MappingOperations { 13 | 14 | public static void main( String[] args ) throws UnknownHostException { 15 | String index="mytest"; 16 | String type="mytype"; 17 | Client client =NativeClient.createTransportClient(); 18 | IndicesOperations io=new IndicesOperations(client); 19 | if(io.checkIndexExists(index)) 20 | io.deleteIndex(index); 21 | io.createIndex(index); 22 | 23 | XContentBuilder builder = null; 24 | try { 25 | builder = jsonBuilder(). 26 | startObject(). 27 | field("type1"). 28 | startObject(). 29 | field("properties"). 30 | startObject(). 31 | field("nested1"). 32 | startObject(). 33 | field("type"). 34 | value("nested"). 35 | endObject(). 36 | endObject(). 37 | endObject(). 38 | endObject(); 39 | PutMappingResponse response=client.admin().indices().preparePutMapping(index).setType(type).setSource(builder).execute().actionGet(); 40 | if(!response.isAcknowledged()){ 41 | System.out.println("Something strange happens"); 42 | } 43 | } catch (IOException e) { 44 | System.out.println("Unable to create mapping"); 45 | } 46 | 47 | io.deleteIndex(index); 48 | 49 | //we need to close the client to free resources 50 | client.close(); 51 | 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/NativeClient.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.client.Client; 4 | import org.elasticsearch.client.transport.TransportClient; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.elasticsearch.common.transport.InetSocketTransportAddress; 7 | import org.elasticsearch.node.Node; 8 | import org.elasticsearch.node.NodeValidationException; 9 | import org.elasticsearch.node.internal.InternalSettingsPreparer; 10 | import org.elasticsearch.plugins.Plugin; 11 | import org.elasticsearch.transport.Netty4Plugin; 12 | import org.elasticsearch.transport.client.PreBuiltTransportClient; 13 | 14 | import java.io.IOException; 15 | import java.net.InetAddress; 16 | import java.net.UnknownHostException; 17 | import java.util.Collections; 18 | import java.util.List; 19 | 20 | public class NativeClient { 21 | public static Client createTransportClient() throws UnknownHostException { 22 | final Settings settings = Settings.builder() 23 | .put("client.transport.sniff", true) 24 | .put("cluster.name", "elasticsearch").build(); 25 | TransportClient client = new PreBuiltTransportClient(settings); 26 | 27 | return client 28 | .addTransportAddress(new InetSocketTransportAddress(InetAddress.getByName("127.0.0.1"), 9300)); 29 | } 30 | 31 | private final Node node; 32 | private final Client client; 33 | 34 | private static class PluginNode extends Node { 35 | public PluginNode(Settings preparedSettings, List> plugins) { 36 | super(InternalSettingsPreparer.prepareEnvironment(preparedSettings, null), plugins); 37 | } 38 | } 39 | 40 | 41 | public NativeClient() throws NodeValidationException { 42 | final Settings settings = Settings.builder() 43 | .put("path.home", "/tmp") 44 | .put("client.transport.sniff", true) 45 | .put("cluster.name", "elasticsearch") 46 | .put("node.data", false) 47 | .put("node.master", false) 48 | .put("node.ingest", false).build(); 49 | 50 | node = new PluginNode(settings, Collections.>singletonList(Netty4Plugin.class)); 51 | node.start(); 52 | client = node.client(); 53 | } 54 | 55 | public Client getClient() { 56 | return client; 57 | } 58 | 59 | public void close() throws IOException { 60 | client.close(); 61 | node.close(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/QueryCreation.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 4 | import org.elasticsearch.action.search.SearchResponse; 5 | import org.elasticsearch.client.Client; 6 | import org.elasticsearch.common.xcontent.XContentFactory; 7 | import org.elasticsearch.index.query.BoolQueryBuilder; 8 | import org.elasticsearch.index.query.RangeQueryBuilder; 9 | import org.elasticsearch.index.query.TermQueryBuilder; 10 | 11 | import java.io.IOException; 12 | import java.net.UnknownHostException; 13 | 14 | import static org.elasticsearch.index.query.QueryBuilders.*; 15 | 16 | public class QueryCreation { 17 | 18 | public static void main(String[] args) throws UnknownHostException { 19 | String index = "mytest"; 20 | String type = "mytype"; 21 | Client client = NativeClient.createTransportClient(); 22 | IndicesOperations io = new IndicesOperations(client); 23 | if (io.checkIndexExists(index)) 24 | io.deleteIndex(index); 25 | 26 | try { 27 | client.admin().indices().prepareCreate(index) 28 | .addMapping(type, XContentFactory.jsonBuilder() 29 | .startObject() 30 | .startObject(type) 31 | .startObject("properties") 32 | .startObject("text").field("type", "integer").field("store", "yes").endObject() 33 | .startObject("number1").field("type", "integer").field("store", "yes").endObject() 34 | .startObject("number2").field("type", "integer").field("store", "yes").endObject() 35 | .endObject() 36 | .endObject() 37 | .endObject()) 38 | .execute().actionGet(); 39 | } catch (IOException e) { 40 | System.out.println("Unable to create mapping"); 41 | } 42 | 43 | BulkRequestBuilder bulker = client.prepareBulk(); 44 | for (Integer i = 1; i < 1000; i++) { 45 | bulker.add(client.prepareIndex(index, type, i.toString()).setSource("text", i.toString(), "number1", i + 1, "number2", i % 2)); 46 | } 47 | bulker.execute().actionGet(); 48 | 49 | client.admin().indices().prepareRefresh(index).execute().actionGet(); 50 | 51 | TermQueryBuilder filter = termQuery("number2", 1); 52 | RangeQueryBuilder range = rangeQuery("number1").gt(500); 53 | BoolQueryBuilder query = boolQuery().must(range).filter(filter); 54 | 55 | SearchResponse response = client.prepareSearch(index).setTypes(type).setQuery(query).execute().actionGet(); 56 | System.out.println("Matched records of elements: " + response.getHits().getTotalHits()); 57 | 58 | io.deleteIndex(index); 59 | 60 | //we need to close the client to free resources 61 | client.close(); 62 | 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/QueryExample.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.index.query.QueryBuilder; 6 | import org.elasticsearch.search.SearchHit; 7 | import org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder; 8 | 9 | import java.net.UnknownHostException; 10 | 11 | import static org.elasticsearch.index.query.QueryBuilders.*; 12 | 13 | public class QueryExample { 14 | public static void main(String[] args) throws UnknownHostException { 15 | String index = "mytest"; 16 | String type = "mytype"; 17 | QueryHelper qh = new QueryHelper(); 18 | qh.populateData(index, type); 19 | Client client = qh.getClient(); 20 | 21 | QueryBuilder query = boolQuery().must(rangeQuery("number1").gte(500)).filter(termQuery("number2", 1)); 22 | 23 | SearchResponse response = client.prepareSearch(index).setTypes(type) 24 | .setQuery(query).highlighter(new HighlightBuilder().field("name")) 25 | .execute().actionGet(); 26 | if (response.status().getStatus() == 200) { 27 | System.out.println("Matched number of documents: " + response.getHits().totalHits()); 28 | System.out.println("Maximum score: " + response.getHits().maxScore()); 29 | 30 | for (SearchHit hit : response.getHits().getHits()) { 31 | System.out.println("hit: " + hit.getIndex() + ":" + hit.getType() + ":" + hit.getId()); 32 | } 33 | } 34 | qh.dropIndex(index); 35 | 36 | //we need to close the client to free resources 37 | client.close(); 38 | 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/QueryHelper.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.bulk.BulkRequestBuilder; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.common.Priority; 6 | import org.elasticsearch.common.xcontent.XContentFactory; 7 | 8 | import java.io.IOException; 9 | import java.net.UnknownHostException; 10 | import java.util.Random; 11 | 12 | public class QueryHelper { 13 | private final Client client; 14 | private final IndicesOperations io; 15 | 16 | public QueryHelper() throws UnknownHostException { 17 | this.client = NativeClient.createTransportClient(); 18 | io = new IndicesOperations(client); 19 | } 20 | 21 | private String[] tags = new String[]{"nice", "cool", "bad", "amazing"}; 22 | 23 | private String getTag() { 24 | return tags[new Random().nextInt(tags.length)]; 25 | } 26 | 27 | public void populateData(String index, String type) { 28 | if (io.checkIndexExists(index)) 29 | io.deleteIndex(index); 30 | 31 | try { 32 | client.admin().indices().prepareCreate(index) 33 | .addMapping(type, XContentFactory.jsonBuilder() 34 | .startObject() 35 | .startObject(type) 36 | .startObject("properties") 37 | .startObject("name") 38 | .field("type", "text") 39 | .field("term_vector", "with_positions_offsets") 40 | .field("store", "yes") 41 | .endObject() 42 | .startObject("tag") 43 | .field("type", "keyword") 44 | .endObject() 45 | .endObject() 46 | .endObject() 47 | .endObject()) 48 | .execute().actionGet(); 49 | } catch (IOException e) { 50 | System.out.println("Unable to create mapping"); 51 | } 52 | client.admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).setWaitForGreenStatus().execute().actionGet(); 53 | 54 | BulkRequestBuilder bulker = client.prepareBulk(); 55 | for (Integer i = 1; i <= 1000; i++) { 56 | bulker.add(client.prepareIndex(index, type, i.toString()).setSource("text", i.toString(), "number1", i + 1, "number2", i % 2, "tag", getTag())); 57 | } 58 | bulker.execute().actionGet(); 59 | 60 | client.admin().indices().prepareRefresh(index).execute().actionGet(); 61 | 62 | } 63 | 64 | public void dropIndex(String index) { 65 | io.deleteIndex(index); 66 | } 67 | 68 | public Client getClient() { 69 | return client; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/java/com/packtpub/ScrollQueryExample.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import org.elasticsearch.action.search.SearchResponse; 4 | import org.elasticsearch.client.Client; 5 | import org.elasticsearch.common.unit.TimeValue; 6 | import org.elasticsearch.index.query.QueryBuilder; 7 | import org.elasticsearch.search.SearchHit; 8 | 9 | import java.net.UnknownHostException; 10 | 11 | import static org.elasticsearch.index.query.QueryBuilders.*; 12 | 13 | public class ScrollQueryExample { 14 | public static void main(String[] args) throws UnknownHostException { 15 | String index = "mytest"; 16 | String type = "mytype"; 17 | QueryHelper qh = new QueryHelper(); 18 | qh.populateData(index, type); 19 | Client client = qh.getClient(); 20 | 21 | QueryBuilder query = boolQuery().must(rangeQuery("number1").gte(500)).filter(termQuery("number2", 1)); 22 | 23 | SearchResponse response = client.prepareSearch(index).setTypes(type).setSize(30) 24 | .setQuery(query).setScroll(TimeValue.timeValueMinutes(2)) 25 | .execute().actionGet(); 26 | 27 | do { 28 | for (SearchHit hit : response.getHits().getHits()) { 29 | System.out.println("hit: " + hit.getIndex() + ":" + hit.getType() + ":" + hit.getId()); 30 | } 31 | response = client.prepareSearchScroll(response.getScrollId()).setScroll(TimeValue.timeValueMinutes(2)).execute().actionGet(); 32 | } while (response.getHits().getHits().length != 0); // Zero hits mark the end of the scroll and the while loop. 33 | 34 | 35 | qh.dropIndex(index); 36 | 37 | //we need to close the client to free resources 38 | client.close(); 39 | 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | appender.console.type = Console 2 | appender.console.name = console 3 | appender.console.layout.type = PatternLayout 4 | 5 | rootLogger.level = info 6 | rootLogger.appenderRef.console.ref = console -------------------------------------------------------------------------------- /chapter_14/nativeclient/src/test/java/com/packtpub/AppTest.java: -------------------------------------------------------------------------------- 1 | package com.packtpub; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase 12 | { 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest( String testName ) 19 | { 20 | super( testName ); 21 | } 22 | 23 | /** 24 | * @return the suite of tests being tested 25 | */ 26 | public static Test suite() 27 | { 28 | return new TestSuite( AppTest.class ); 29 | } 30 | 31 | /** 32 | * Rigourous Test :-) 33 | */ 34 | public void testApp() 35 | { 36 | assertTrue( true ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/build.sbt: -------------------------------------------------------------------------------- 1 | organization := "com.packtpub" 2 | 3 | name := """elastic4s-sample""" 4 | 5 | version := "0.0.1" 6 | 7 | scalaVersion := "2.11.8" 8 | 9 | scalacOptions := Seq("-unchecked", "-deprecation", "-encoding", "utf8") 10 | 11 | libraryDependencies ++= { 12 | val elastic4sV = "5.0.0" 13 | val scalaTestV = "3.0.0" 14 | val Log4jVersion = "2.6.2" 15 | Seq( 16 | "com.sksamuel.elastic4s" %% "elastic4s-core" % elastic4sV, 17 | "com.sksamuel.elastic4s" %% "elastic4s-circe" % elastic4sV, 18 | "org.apache.logging.log4j" % "log4j-api" % Log4jVersion, 19 | "org.apache.logging.log4j" % "log4j-core" % Log4jVersion, 20 | "org.apache.logging.log4j" % "log4j-1.2-api" % Log4jVersion, 21 | "org.scalatest" %% "scalatest" % scalaTestV % "test" 22 | 23 | ) 24 | } 25 | 26 | resolvers ++= Seq( 27 | Resolver.sonatypeRepo("releases"), 28 | Resolver.jcenterRepo 29 | ) 30 | 31 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/pom.xml_: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | com.packtpub 5 | elastic4s 6 | jar 7 | 1.0-SNAPSHOT 8 | elastic4s-sample 9 | http://maven.apache.org 10 | 11 | 12 | com.sksamuel.elastic4s 13 | elastic4s-core_2.11 14 | 5.0.0 15 | 16 | 17 | junit 18 | junit 19 | 4.12 20 | test 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.12 2 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/project/plugins.sbt: -------------------------------------------------------------------------------- 1 | addSbtPlugin("io.get-coursier" % "sbt-coursier" % "1.0.0-M14-6") 2 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/resources/log4j2.properties: -------------------------------------------------------------------------------- 1 | appender.console.type = Console 2 | appender.console.name = console 3 | appender.console.layout.type = PatternLayout 4 | 5 | rootLogger.level = info 6 | rootLogger.appenderRef.console.ref = console -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/AggregationExample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.ElasticDsl._ 4 | import org.elasticsearch.search.aggregations.metrics.geocentroid.InternalGeoCentroid 5 | import org.elasticsearch.search.aggregations.metrics.stats.extended.InternalExtendedStats 6 | import scala.collection.JavaConversions._ 7 | 8 | object AggregationExample extends App with ElasticSearchClientTrait { 9 | val indexName = "myindex" 10 | val typeName = "mytype" 11 | ensureIndexMapping(indexName, typeName) 12 | populateSampleData(indexName, typeName, 1000) 13 | 14 | val resp = client.execute { 15 | search(indexName / typeName) size 0 aggregations ( 16 | termsAggregation("tag") field "tag" size 100 subAggregations( 17 | extendedStatsAggregation("price") field "price" 18 | , extendedStatsAggregation("size") field "size" 19 | ,geoBoundsAggregation("centroid") field "location" 20 | )) 21 | }.await 22 | 23 | val tagsAgg = resp.aggregations.stringTermsResult("tag") 24 | 25 | println(s"Result Hits: ${resp.size}") 26 | println(s"number of tags: ${tagsAgg.getBuckets.size()}") 27 | println(s"max price of first tag ${tagsAgg.getBuckets.head.getKey}: ${tagsAgg.getBuckets.head.getAggregations.get[InternalExtendedStats]("price").value("max")}") 28 | println(s"min size of first tag ${tagsAgg.getBuckets.head.getKey}: ${tagsAgg.getBuckets.head.getAggregations.get[InternalExtendedStats]("size").value("min")}") 29 | //println(s"center of first tag ${tagsAgg.getBuckets.head.getKey}: ${tagsAgg.getBuckets.head.getAggregations.get[InternalGeoCentroid]("centroid").centroid()}") 30 | 31 | client.execute(deleteIndex(indexName)).await 32 | 33 | client.close() 34 | } 35 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/ClientSample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.{ElasticClient, ElasticsearchClientUri} 4 | import com.sksamuel.elastic4s.ElasticDsl._ 5 | 6 | object ClientSample extends App { 7 | val uri = ElasticsearchClientUri("elasticsearch://127.0.0.1:9300?cluster.name=elasticsearch") 8 | val client = ElasticClient.transport(uri) 9 | 10 | // await is a helper method to make this operation synchronous instead of async 11 | // You would normally avoid doing this in a real program as it will block your thread 12 | client.execute { indexInto("bands" / "artists") fields "name"->"coldplay" }.await 13 | 14 | // we need to wait until the index operation has been flushed by the server. 15 | // this is an important point - when the index future completes, that doesn't mean that the doc 16 | // is necessarily searchable. It simply means the server has processed your request and the doc is 17 | // queued to be flushed to the indexes. Elasticsearch is eventually consistent. 18 | // For this demo, we'll simply wait for 2 seconds (default refresh interval is 1 second). 19 | Thread.sleep(2000) 20 | 21 | // now we can search for the document we indexed earlier 22 | val resp = client.execute { search("bands" / "artists") query "coldplay" }.await 23 | println(resp) 24 | 25 | 26 | } 27 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/DocumentExample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.ElasticDsl._ 4 | import com.sksamuel.elastic4s.circe._ 5 | 6 | object DocumentExample extends App with ElasticSearchClientTrait{ 7 | val indexName="myindex" 8 | val typeName="mytype" 9 | 10 | ensureIndexMapping(indexName, typeName) 11 | 12 | client.execute { 13 | indexInto(indexName / typeName) id "0" fields ( 14 | "name" -> "brown", 15 | "tag" -> List("nice", "simple") 16 | ) 17 | }.await 18 | 19 | val bwn=client.execute { 20 | get("0") from indexName / typeName 21 | }.await 22 | 23 | println(bwn.sourceAsString) 24 | 25 | client.execute { 26 | update("0").in(indexName / typeName).script("ctx._source.name = 'red'") 27 | }.await 28 | 29 | val red=client.execute { 30 | get("0") from indexName / typeName 31 | }.await 32 | 33 | println(red.sourceAsString) 34 | 35 | client.execute { 36 | delete("0") from indexName / typeName 37 | }.await 38 | 39 | case class Place(id: Int, name: String) 40 | case class Cafe(name: String, place: Place) 41 | 42 | import io.circe.generic.auto._ 43 | import com.sksamuel.elastic4s.Indexable 44 | implicitly[Indexable[Cafe]] 45 | 46 | val cafe=Cafe("nespresso", Place(20,"Milan")) 47 | 48 | client.execute { 49 | indexInto(indexName / typeName).id(cafe.name).source(cafe) 50 | }.await 51 | 52 | 53 | 54 | client.execute(deleteIndex(indexName)).await 55 | 56 | client.close() 57 | } 58 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/ElasticSearchClientTrait.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.ElasticDsl._ 4 | import com.sksamuel.elastic4s.{ElasticClient, ElasticsearchClientUri} 5 | import com.sksamuel.elastic4s.indexes._ 6 | import org.elasticsearch.cluster.health.ClusterHealthStatus 7 | 8 | trait ElasticSearchClientTrait { 9 | val client:ElasticClient ={ 10 | val uri = ElasticsearchClientUri("elasticsearch://127.0.0.1:9300?cluster.name=elasticsearch") 11 | ElasticClient.transport(uri) 12 | } 13 | 14 | def ensureIndexMapping(indexName:String, mappingName:String): Unit ={ 15 | if(client.execute{ indexExists(indexName)}.await.isExists){ 16 | client.execute{ deleteIndex(indexName)}.await 17 | } 18 | 19 | client.execute{ 20 | createIndex(indexName) shards 1 replicas 0 mappings ( 21 | mapping(mappingName).as ( 22 | textField("name") termVector "with_positions_offsets" stored true, 23 | longField("size"), 24 | doubleField("price"), 25 | geopointField("location"), 26 | keywordField("tag") stored true 27 | ) 28 | ) 29 | }.await 30 | 31 | client.execute{ 32 | clusterHealth waitForStatus ClusterHealthStatus.YELLOW 33 | } 34 | 35 | } 36 | 37 | def populateSampleData(indexName:String, mappingName:String, size:Int=1000): Unit ={ 38 | import scala.util.Random 39 | val tags=List("cool", "nice", "bad", "awesome", "good") 40 | client.execute{ 41 | bulk(0.to(size).map{ 42 | i=> 43 | indexInto(indexName, mappingName) id i fields( 44 | "name" -> s"name_${i}", 45 | "size" -> (i%10)*8, 46 | "price" -> (i%10) * 1.2, 47 | "location" -> List(30.0 * Random.nextDouble(), 30.0* Random.nextDouble()), 48 | "tag" -> Random.shuffle(tags).take(3) 49 | ) 50 | }) 51 | }.await 52 | 53 | Thread.sleep(2000) 54 | 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/IndicesExample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | import com.sksamuel.elastic4s.ElasticDsl._ 3 | 4 | object IndicesExample extends App with ElasticSearchClientTrait{ 5 | val indexName="test" 6 | if(client.execute{ indexExists(indexName)}.await.isExists){ 7 | client.execute{ deleteIndex(indexName)}.await 8 | } 9 | 10 | client.execute{ 11 | createIndex(indexName) shards 1 replicas 0 mappings ( 12 | mapping("mytype") as ( 13 | textField("name").termVector("with_positions_offsets").stored(true), 14 | keywordField("tag") 15 | ) 16 | ) 17 | }.await 18 | 19 | Thread.sleep(2000) 20 | client.execute(closeIndex(indexName)).await 21 | client.execute(openIndex(indexName)).await 22 | client.execute(deleteIndex(indexName)).await 23 | 24 | client.close() 25 | } 26 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/MappingExample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.ElasticDsl._ 4 | 5 | object MappingExample extends App with ElasticSearchClientTrait{ 6 | val indexName="myindex" 7 | if(client.execute{ indexExists(indexName)}.await.isExists){ 8 | client.execute{ deleteIndex(indexName)}.await 9 | } 10 | 11 | client.execute{ 12 | createIndex(indexName) shards 1 replicas 0 mappings ( 13 | mapping("mytype")as ( 14 | textField("name").termVector("with_positions_offsets").stored(true) 15 | ) 16 | ) 17 | }.await 18 | Thread.sleep(2000) 19 | 20 | client.execute{ 21 | putMapping(indexName / "mytype").as( 22 | keywordField("tag") 23 | ) 24 | }.await 25 | 26 | val myMapping=client.execute{ 27 | getMapping(indexName / "mytype") 28 | }.await 29 | 30 | val tagMapping=myMapping.fieldFor(indexName / "mytype", "tag") 31 | println(tagMapping) 32 | 33 | client.execute(deleteIndex(indexName)).await 34 | 35 | client.close() 36 | } 37 | -------------------------------------------------------------------------------- /chapter_15/elastic4s_sample/src/main/scala/com/packtpub/QueryExample.scala: -------------------------------------------------------------------------------- 1 | package com.packtpub 2 | 3 | import com.sksamuel.elastic4s.ElasticDsl._ 4 | import com.sksamuel.elastic4s.circe._ 5 | import com.sksamuel.elastic4s.Indexable 6 | import io.circe.generic.auto._ 7 | 8 | object QueryExample extends App with ElasticSearchClientTrait { 9 | val indexName = "myindex" 10 | val typeName = "mytype" 11 | 12 | case class Place(id: Int, name: String) 13 | case class Cafe(name: String, place: Place) 14 | 15 | implicitly[Indexable[Cafe]] 16 | 17 | ensureIndexMapping(indexName, typeName) 18 | 19 | client.execute { 20 | bulk( 21 | indexInto(indexName / typeName).id("0").source(Cafe("nespresso", Place(20, "Milan"))), 22 | indexInto(indexName / typeName).id("1").source(Cafe("java", Place(60, "Rome"))), 23 | indexInto(indexName / typeName).id("2").source(Cafe("nespresso", Place(70, "Paris"))), 24 | indexInto(indexName / typeName).id("3").source(Cafe("java", Place(80, "Chicago"))), 25 | indexInto(indexName / typeName).id("4").source(Cafe("nespresso", Place(10, "London"))), 26 | indexInto(indexName / typeName).id("5").source(Cafe("java", Place(60, "Milan"))), 27 | indexInto(indexName / typeName).id("6").source(Cafe("nespresso", Place(25, "Rome"))), 28 | indexInto(indexName / typeName).id("7").source(Cafe("java", Place(56, "Paris"))), 29 | indexInto(indexName / typeName).id("8").source(Cafe("nespresso", Place(23, "Chicago"))), 30 | indexInto(indexName / typeName).id("9").source(Cafe("java", Place(89, "London"))) 31 | ) 32 | }.await 33 | 34 | Thread.sleep(2000) 35 | 36 | val resp = client.execute { 37 | search(indexName / typeName).bool(must(termQuery("name", "java"), rangeQuery("place.id").gte(80))) 38 | }.await 39 | 40 | println(resp.size) 41 | 42 | println(resp.to[Cafe].toList) 43 | 44 | 45 | 46 | //client.execute(deleteIndex(indexName)).await 47 | 48 | client.close() 49 | } 50 | -------------------------------------------------------------------------------- /chapter_16/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'alberto' 2 | -------------------------------------------------------------------------------- /chapter_16/aggregation.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | from pprint import pprint 3 | 4 | es = elasticsearch.Elasticsearch() 5 | index_name = "my_index" 6 | type_name = "my_type" 7 | 8 | if es.indices.exists(index_name): 9 | es.indices.delete(index_name) 10 | 11 | from utils import create_and_add_mapping, populate 12 | 13 | create_and_add_mapping(es, index_name, type_name) 14 | populate(es, index_name, type_name) 15 | 16 | results = es.search(index_name, type_name, 17 | { 18 | "query": {"match_all": {}}, 19 | "aggs": { 20 | "pterms": {"terms": {"field": "name", "size": 10}} 21 | } 22 | }) 23 | pprint(results) 24 | 25 | results = es.search(index_name, type_name, 26 | { 27 | "query": {"match_all": {}}, 28 | "aggs": { 29 | "date_histo": {"date_histogram": {"field": "date", "interval": "month"}} 30 | } 31 | }) 32 | pprint(results) 33 | 34 | es.indices.delete(index_name) 35 | -------------------------------------------------------------------------------- /chapter_16/client_creation.py: -------------------------------------------------------------------------------- 1 | __author__ = 'alberto' 2 | 3 | import elasticsearch 4 | 5 | # client using all the defaults: localhost:9200 and http urllib3 transport 6 | es = elasticsearch.Elasticsearch() 7 | 8 | # client using localhost:9200 and http requests transport 9 | from elasticsearch.connection import RequestsHttpConnection 10 | 11 | es = elasticsearch.Elasticsearch(sniff_on_start=True, connection_class=RequestsHttpConnection) 12 | 13 | # client using two nodes 14 | es = elasticsearch.Elasticsearch(["search1:9200", "search2:9200"]) 15 | 16 | # client using a node with sniffing 17 | es = elasticsearch.Elasticsearch("search1:9200", sniff_on_start=True) 18 | 19 | -------------------------------------------------------------------------------- /chapter_16/document_management.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | from datetime import datetime 3 | 4 | es = elasticsearch.Elasticsearch() 5 | 6 | index_name = "my_index" 7 | type_name = "my_type" 8 | 9 | from utils import create_and_add_mapping 10 | 11 | 12 | if es.indices.exists(index_name): 13 | es.indices.delete(index_name) 14 | 15 | 16 | create_and_add_mapping(es, index_name, type_name) 17 | 18 | es.index(index=index_name, doc_type=type_name, id=1, 19 | body={"name": "Joe Tester", "parsedtext": "Joe Testere nice guy", "uuid": "11111", "position": 1, 20 | "date": datetime(2013, 12, 8)}) 21 | es.index(index=index_name, doc_type=type_name + "2", id=1, body={"name": "data1", "value": "value1"}, parent=1) 22 | es.index(index=index_name, doc_type=type_name, id=2, 23 | body={"name": "Bill Baloney", "parsedtext": "Bill Testere nice guy", "uuid": "22222", "position": 2, 24 | "date": datetime(2013, 12, 8)}) 25 | es.index(index=index_name, doc_type=type_name + "2", id=2, body={"name": "data2", "value": "value2"}, parent=2) 26 | es.index(index=index_name, doc_type=type_name, id=3, body={"name": "Bill Clinton", "parsedtext": """Bill is not 27 | nice guy""", "uuid": "33333", "position": 3, "date": datetime(2013, 12, 8)}) 28 | 29 | es.update(index=index_name, doc_type=type_name, id=2, body={"script": 'ctx._source.position += 1'}) 30 | 31 | document=es.get(index=index_name, doc_type=type_name, id=2) 32 | print(document) 33 | 34 | es.delete(index=index_name, doc_type=type_name, id=3) 35 | 36 | from elasticsearch.helpers import bulk 37 | bulk(es, [ 38 | {"_index":index_name, "_type":type_name, "_id":"1", "source":{"name": "Joe Tester", "parsedtext": "Joe Testere nice guy", "uuid": "11111", "position": 1, 39 | "date": datetime(2013, 12, 8)}}, 40 | 41 | {"_index": index_name, "_type": type_name, "_id": "1", 42 | "source": {"name": "Bill Baloney", "parsedtext": "Bill Testere nice guy", "uuid": "22222", "position": 2, 43 | "date": datetime(2013, 12, 8)}} 44 | ]) 45 | 46 | es.indices.delete(index_name) 47 | -------------------------------------------------------------------------------- /chapter_16/indices_management.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | 3 | es = elasticsearch.Elasticsearch() 4 | 5 | index_name = "my_index" 6 | 7 | if es.indices.exists(index_name): 8 | es.indices.delete(index_name) 9 | 10 | es.indices.create(index_name) 11 | 12 | es.cluster.health(wait_for_status="yellow") 13 | 14 | es.indices.close(index_name) 15 | 16 | es.indices.open(index_name) 17 | 18 | es.cluster.health(wait_for_status="yellow") 19 | 20 | es.indices.forcemerge(index_name) 21 | 22 | es.indices.delete(index_name) 23 | -------------------------------------------------------------------------------- /chapter_16/mapping_management.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | 3 | es = elasticsearch.Elasticsearch() 4 | 5 | index_name = "my_index" 6 | type_name = "my_type" 7 | 8 | if es.indices.exists(index_name): 9 | es.indices.delete(index_name) 10 | 11 | es.indices.create(index_name) 12 | es.cluster.health(wait_for_status="yellow") 13 | 14 | es.indices.put_mapping(index=index_name, doc_type=type_name, body={type_name:{"properties": { 15 | "uuid": {"type": "keyword", "store": "true"}, 16 | "title": {"type": "text", "store": "true", "term_vector": "with_positions_offsets"}, 17 | "parsedtext": { "type": "text", "store": "true", "term_vector": "with_positions_offsets"}, 18 | "nested": {"type": "nested", "properties": {"num": {"type": "integer", "store": "true"}, 19 | "name": {"type": "keyword", "store": "true"}, 20 | "value": {"type": "keyword", "store": "true"}}}, 21 | "date": {"type": "date", "store": "true"}, 22 | "position": {"type": "integer", "store": "true"}, 23 | "name": {"type": "text", "store": "true", "term_vector": "with_positions_offsets"}}}}) 24 | 25 | mappings = es.indices.get_mapping(index_name, type_name) 26 | 27 | print(mappings) 28 | 29 | es.indices.delete(index_name) -------------------------------------------------------------------------------- /chapter_16/searching.py: -------------------------------------------------------------------------------- 1 | import elasticsearch 2 | from pprint import pprint 3 | 4 | es = elasticsearch.Elasticsearch() 5 | index_name = "my_index" 6 | type_name = "my_type" 7 | 8 | if es.indices.exists(index_name): 9 | es.indices.delete(index_name) 10 | 11 | from utils import create_and_add_mapping, populate 12 | 13 | create_and_add_mapping(es, index_name, type_name) 14 | populate(es, index_name, type_name) 15 | 16 | results = es.search(index_name, type_name, {"query": {"match_all": {}}}) 17 | pprint(results) 18 | 19 | results = es.search(index_name, type_name, { 20 | "query": { 21 | "term": {"name": {"boost": 3.0, "value": "joe"}}} 22 | }) 23 | pprint(results) 24 | 25 | results = es.search(index_name, type_name, {"query": { 26 | "bool": { 27 | "filter": { 28 | "bool": { 29 | "should": [ 30 | {"term": {"position": 1}}, 31 | {"term": {"position": 2}}]} 32 | }}}}) 33 | pprint(results) 34 | 35 | es.indices.delete(index_name) 36 | -------------------------------------------------------------------------------- /chapter_16/utils.py: -------------------------------------------------------------------------------- 1 | __author__ = 'alberto' 2 | from datetime import datetime 3 | 4 | 5 | def create_and_add_mapping(connection, index_name, type_name): 6 | try: 7 | connection.indices.create(index_name) 8 | except: 9 | # we skip exception if index already exists 10 | pass 11 | connection.cluster.health(wait_for_status="yellow") 12 | 13 | type2 = type_name + "2" 14 | connection.indices.put_mapping(index=index_name, doc_type=type2, body={type2: { 15 | "_parent": { 16 | "type": "my_type" 17 | }, 18 | "_routing": { 19 | "required": True 20 | }, 21 | "properties": { 22 | "name": {"type": "text", "store": "true", "term_vector": "with_positions_offsets", "fielddata": "true"}, 23 | "value": { 24 | "type": "text" 25 | } 26 | } 27 | }}) 28 | 29 | connection.indices.put_mapping(index=index_name, doc_type=type_name, body={type_name:{"properties": { 30 | "uuid": {"type": "keyword", "store": "true"}, 31 | "title": {"type": "text", "store": "true", "term_vector": "with_positions_offsets"}, 32 | "parsedtext": { "type": "text", "store": "true", "term_vector": "with_positions_offsets", "fielddata": "true"}, 33 | "nested": {"type": "nested", "properties": {"num": {"type": "integer", "store": "true"}, 34 | "name": {"type": "keyword", "store": "true"}, 35 | "value": {"type": "keyword", "store": "true"}}}, 36 | "date": {"type": "date", "store": "true"}, 37 | "position": {"type": "integer", "store": "true"}, 38 | "name": {"type": "text", "store": "yes", "term_vector": "with_positions_offsets", "fielddata": "true"}}}}) 39 | 40 | 41 | 42 | def populate(connection, index_name, type_name): 43 | connection.index(index=index_name, doc_type=type_name, id=1, 44 | body={"name": "Joe Tester", "parsedtext": "Joe Testere nice guy", "uuid": "11111", 45 | "position": 1, 46 | "date": datetime(2013, 12, 8)}) 47 | connection.index(index=index_name, doc_type=type_name + "2", id=1, body={"name": "data1", "value": "value1"}, 48 | parent=1) 49 | connection.index(index=index_name, doc_type=type_name, id=2, 50 | body={"name": "Bill Baloney", "parsedtext": "Bill Testere nice guy", "uuid": "22222", 51 | "position": 2, 52 | "date": datetime(2013, 12, 8)}) 53 | connection.index(index=index_name, doc_type=type_name + "2", id=2, body={"name": "data2", "value": "value2"}, 54 | parent=2) 55 | connection.index(index=index_name, doc_type=type_name, id=3, body={"name": "Bill Clinton", "parsedtext": """Bill is not 56 | nice guy""", "uuid": "33333", "position": 3, "date": datetime(2013, 12, 8)}) 57 | 58 | connection.indices.refresh(index_name) -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.settings 9 | /.classpath 10 | /.project 11 | -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | elasticsearch-simple-plugin 6 | 4.0.0 7 | com.packtpub 8 | analysis-plugin 9 | 0.0.2-SNAPSHOT 10 | jar 11 | An analysis plugin for ElasticSearch 12 | 2014 13 | 14 | 15 | The Apache Software License, Version 2.0 16 | http://www.apache.org/licenses/LICENSE-2.0.txt 17 | repo 18 | 19 | 20 | 21 | 22 | org.sonatype.oss 23 | oss-parent 24 | 7 25 | 26 | 27 | 28 | 5.1.1 29 | 1.8 30 | ${project.basedir}/src/main/assemblies/plugin.xml 31 | simple-plugin 32 | org.elasticsearch.plugin.simple.SimplePlugin 33 | true 34 | false 35 | true 36 | 37 | 38 | 39 | 40 | sonatype 41 | http://oss.sonatype.org/content/repositories/releases/ 42 | 43 | 44 | 45 | 46 | 47 | org.elasticsearch 48 | elasticsearch 49 | ${elasticsearch.version} 50 | compile 51 | 52 | 53 | 54 | 55 | org.apache.httpcomponents 56 | httpclient 57 | 4.5.2 58 | 59 | 60 | 61 | org.apache.logging.log4j 62 | log4j-api 63 | 2.3 64 | 65 | 66 | 67 | org.hamcrest 68 | hamcrest-core 69 | 1.3 70 | test 71 | 72 | 73 | 74 | org.hamcrest 75 | hamcrest-library 76 | 1.3 77 | test 78 | 79 | 80 | junit 81 | junit 82 | 4.12 83 | test 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-compiler-plugin 93 | 3.5.1 94 | 95 | ${maven.compiler.target} 96 | ${maven.compiler.target} 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.11 103 | 104 | 105 | **/*Tests.java 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-source-plugin 112 | 2.1.2 113 | 114 | 115 | attach-sources 116 | 117 | jar 118 | 119 | 120 | 121 | 122 | 123 | maven-assembly-plugin 124 | 125 | 126 | false 127 | ${project.build.directory}/releases/ 128 | 129 | ${basedir}/src/main/assemblies/plugin.xml 130 | 131 | 132 | 133 | fully.qualified.MainClass 134 | 135 | 136 | 137 | 138 | 139 | package 140 | 141 | single 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | / 19 | true 20 | true 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/src/main/java/org/elasticsearch/index/analysis/CustomEnglishAnalyzerProvider.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.index.analysis; 2 | 3 | import org.apache.lucene.analysis.en.EnglishAnalyzer; 4 | import org.apache.lucene.analysis.util.CharArraySet; 5 | import org.elasticsearch.common.settings.Settings; 6 | import org.elasticsearch.env.Environment; 7 | import org.elasticsearch.index.IndexSettings; 8 | 9 | public class CustomEnglishAnalyzerProvider extends AbstractIndexAnalyzerProvider { 10 | public static String NAME = "custom_english"; 11 | 12 | private final EnglishAnalyzer analyzer; 13 | 14 | public CustomEnglishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings, boolean useSmart) { 15 | super(indexSettings, name, settings); 16 | 17 | analyzer = new EnglishAnalyzer( 18 | Analysis.parseStopWords(env, settings, EnglishAnalyzer.getDefaultStopSet(), true), 19 | Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)); 20 | } 21 | 22 | public static CustomEnglishAnalyzerProvider getCustomEnglishAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { 23 | return new CustomEnglishAnalyzerProvider(indexSettings, env, name, settings, true); 24 | } 25 | 26 | @Override 27 | public EnglishAnalyzer get() { 28 | return this.analyzer; 29 | } 30 | } -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/src/main/java/org/elasticsearch/plugin/analysis/AnalysisPlugin.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.plugin.analysis; 2 | 3 | import org.apache.lucene.analysis.Analyzer; 4 | import org.elasticsearch.index.analysis.AnalyzerProvider; 5 | import org.elasticsearch.index.analysis.CustomEnglishAnalyzerProvider; 6 | import org.elasticsearch.indices.analysis.AnalysisModule; 7 | import org.elasticsearch.plugins.Plugin; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | public class AnalysisPlugin extends Plugin implements org.elasticsearch.plugins.AnalysisPlugin { 13 | @Override 14 | public Map>> getAnalyzers() { 15 | Map>> analyzers = new HashMap(); 16 | analyzers.put(CustomEnglishAnalyzerProvider.NAME, CustomEnglishAnalyzerProvider::getCustomEnglishAnalyzerProvider); 17 | return analyzers; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /chapter_17/analysis_plugin/src/main/resources/plugin-descriptor.properties: -------------------------------------------------------------------------------- 1 | # Elasticsearch plugin descriptor file 2 | # This file must exist as 'plugin-descriptor.properties' at 3 | # the root directory of all plugins. 4 | # 5 | # A plugin can be 'site', 'jvm', or both. 6 | # 7 | ### example site plugin for "foo": 8 | # 9 | # foo.zip <-- zip file for the plugin, with this structure: 10 | # _site/ <-- the contents that will be served 11 | # plugin-descriptor.properties <-- example contents below: 12 | # 13 | # site=true 14 | # description=My cool plugin 15 | # version=1.0 16 | # 17 | ### example jvm plugin for "foo" 18 | # 19 | # foo.zip <-- zip file for the plugin, with this structure: 20 | # .jar <-- classes, resources, dependencies 21 | # .jar <-- any number of jars 22 | # plugin-descriptor.properties <-- example contents below: 23 | # 24 | # jvm=true 25 | # classname=foo.bar.BazPlugin 26 | # description=My cool plugin 27 | # version=2.0.0-rc1 28 | # elasticsearch.version=2.0 29 | # java.version=1.7 30 | # 31 | ### mandatory elements for all plugins: 32 | # 33 | # 'description': simple summary of the plugin 34 | description=${project.description} 35 | # 36 | # 'version': plugin's version 37 | version=${project.version} 38 | # 39 | # 'name': the plugin name 40 | name=${elasticsearch.plugin.name} 41 | 42 | ### mandatory elements for site plugins: 43 | # 44 | # 'site': set to true to indicate contents of the _site/ 45 | # directory in the root of the plugin should be served. 46 | site=${elasticsearch.plugin.site} 47 | # 48 | ### mandatory elements for jvm plugins : 49 | # 50 | # 'jvm': true if the 'classname' class should be loaded 51 | # from jar files in the root directory of the plugin. 52 | # Note that only jar files in the root directory are 53 | # added to the classpath for the plugin! If you need 54 | # other resources, package them into a resources jar. 55 | jvm=${elasticsearch.plugin.jvm} 56 | # 57 | # 'classname': the name of the class to load, fully-qualified. 58 | classname=${elasticsearch.plugin.classname} 59 | # 60 | # 'java.version' version of java the code is built against 61 | # use the system property java.specification.version 62 | # version string must be a sequence of nonnegative decimal integers 63 | # separated by "."'s and may have leading zeros 64 | java.version=${maven.compiler.target} 65 | # 66 | # 'elasticsearch.version' version of elasticsearch compiled against 67 | # You will have to release a new version of the plugin for each new 68 | # elasticsearch release. This version is checked when the plugin 69 | # is loaded so Elasticsearch will refuse to start in the presence of 70 | # plugins with the incorrect elasticsearch.version. 71 | elasticsearch.version=${elasticsearch.version} 72 | # 73 | ### deprecated elements for jvm plugins : 74 | # 75 | # 'isolated': true if the plugin should have its own classloader. 76 | # passing false is deprecated, and only intended to support plugins 77 | # that have hard dependencies against each other. If this is 78 | # not specified, then the plugin is isolated by default. 79 | isolated=${elasticsearch.plugin.isolated} 80 | # -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.settings 9 | /.classpath 10 | /.project 11 | -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | elasticsearch-simple-plugin 6 | 4.0.0 7 | com.packtpub 8 | myingest-plugin 9 | ${elasticsearch.version} 10 | jar 11 | A simple ingest plugin for ElasticSearch 12 | 2016 13 | 14 | 15 | The Apache Software License, Version 2.0 16 | http://www.apache.org/licenses/LICENSE-2.0.txt 17 | repo 18 | 19 | 20 | 21 | 22 | org.sonatype.oss 23 | oss-parent 24 | 7 25 | 26 | 27 | 28 | 5.1.1 29 | 1.8 30 | ${project.basedir}/src/main/assemblies/plugin.xml 31 | myingest-plugin 32 | org.elasticsearch.plugin.ingest.MyIngestPlugin 33 | true 34 | false 35 | true 36 | 37 | 38 | 39 | 40 | sonatype 41 | http://oss.sonatype.org/content/repositories/releases/ 42 | 43 | 44 | 45 | 46 | 47 | org.elasticsearch 48 | elasticsearch 49 | ${elasticsearch.version} 50 | compile 51 | 52 | 53 | 54 | 55 | org.apache.httpcomponents 56 | httpclient 57 | 4.5.2 58 | 59 | 60 | 61 | org.apache.logging.log4j 62 | log4j-api 63 | 2.3 64 | 65 | 66 | 67 | org.hamcrest 68 | hamcrest-core 69 | 1.3 70 | test 71 | 72 | 73 | 74 | org.hamcrest 75 | hamcrest-library 76 | 1.3 77 | test 78 | 79 | 80 | junit 81 | junit 82 | 4.12 83 | test 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-compiler-plugin 93 | 3.5.1 94 | 95 | ${maven.compiler.target} 96 | ${maven.compiler.target} 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.11 103 | 104 | 105 | **/*Tests.java 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-source-plugin 112 | 2.1.2 113 | 114 | 115 | attach-sources 116 | 117 | jar 118 | 119 | 120 | 121 | 122 | 123 | maven-assembly-plugin 124 | 125 | false 126 | ${project.build.directory}/releases/ 127 | 128 | ${basedir}/src/main/assemblies/plugin.xml 129 | 130 | 131 | 132 | fully.qualified.MainClass 133 | 134 | 135 | 136 | 137 | 138 | package 139 | 140 | single 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | simple-plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/src/main/java/com/packtpub/ingest/InitialProcessor.java: -------------------------------------------------------------------------------- 1 | package com.packtpub.ingest; 2 | 3 | import org.elasticsearch.ingest.AbstractProcessor; 4 | import org.elasticsearch.ingest.ConfigurationUtils; 5 | import org.elasticsearch.ingest.IngestDocument; 6 | import org.elasticsearch.ingest.Processor; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * Processor that allows to create a new field with the lowercase first letter of another one. Will throw exception if the field is not present. 12 | */ 13 | public final class InitialProcessor extends AbstractProcessor { 14 | 15 | public static final String TYPE = "initial"; 16 | 17 | private final String field; 18 | private final String targetField; 19 | private final boolean ignoreMissing; 20 | 21 | InitialProcessor(String tag, String field, String targetField, boolean ignoreMissing) { 22 | super(tag); 23 | this.field = field; 24 | this.targetField = targetField; 25 | this.ignoreMissing = ignoreMissing; 26 | } 27 | 28 | String getField() { 29 | return field; 30 | } 31 | 32 | String getTargetField() { 33 | return targetField; 34 | } 35 | 36 | boolean isIgnoreMissing() { 37 | return ignoreMissing; 38 | } 39 | 40 | @Override 41 | public void execute(IngestDocument document) { 42 | if (document.hasField(field, true) == false) { 43 | if (ignoreMissing) { 44 | return; 45 | } else { 46 | throw new IllegalArgumentException("field [" + field + "] doesn't exist"); 47 | } 48 | } 49 | // We fail here if the target field point to an array slot that is out of range. 50 | // If we didn't do this then we would fail if we set the value in the target_field 51 | // and then on failure processors would not see that value we tried to rename as we already 52 | // removed it. 53 | if (document.hasField(targetField, true)) { 54 | throw new IllegalArgumentException("field [" + targetField + "] already exists"); 55 | } 56 | 57 | Object value = document.getFieldValue(field, Object.class); 58 | if( value!=null && value instanceof String ) { 59 | String myValue=value.toString().trim(); 60 | if(myValue.length()>1){ 61 | try { 62 | document.setFieldValue(targetField, myValue.substring(0,1).toLowerCase()); 63 | } catch (Exception e) { 64 | // setting the value back to the original field shouldn't as we just fetched it from that field: 65 | document.setFieldValue(field, value); 66 | throw e; 67 | } 68 | } 69 | } 70 | } 71 | 72 | @Override 73 | public String getType() { 74 | return TYPE; 75 | } 76 | 77 | public static final class Factory implements Processor.Factory { 78 | @Override 79 | public InitialProcessor create(Map registry, String processorTag, 80 | Map config) throws Exception { 81 | String field = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "field"); 82 | String targetField = ConfigurationUtils.readStringProperty(TYPE, processorTag, config, "target_field"); 83 | boolean ignoreMissing = ConfigurationUtils.readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); 84 | return new InitialProcessor(processorTag, field, targetField, ignoreMissing); 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/src/main/java/org/elasticsearch/plugin/ingest/MyIngestPlugin.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.plugin.ingest; 2 | 3 | 4 | import com.packtpub.ingest.InitialProcessor; 5 | import org.elasticsearch.ingest.Processor; 6 | import org.elasticsearch.plugins.IngestPlugin; 7 | import org.elasticsearch.plugins.Plugin; 8 | 9 | import java.util.Collections; 10 | import java.util.Map; 11 | 12 | public class MyIngestPlugin extends Plugin implements IngestPlugin { 13 | @Override 14 | public Map getProcessors(Processor.Parameters parameters) { 15 | return Collections.singletonMap(InitialProcessor.TYPE, 16 | (factories, tag, config) -> new InitialProcessor.Factory().create(factories, tag, config)); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /chapter_17/ingest_plugin/src/main/resources/plugin-descriptor.properties: -------------------------------------------------------------------------------- 1 | # Elasticsearch plugin descriptor file 2 | # This file must exist as 'plugin-descriptor.properties' at 3 | # the root directory of all plugins. 4 | # 5 | # A plugin can be 'site', 'jvm', or both. 6 | # 7 | ### example site plugin for "foo": 8 | # 9 | # foo.zip <-- zip file for the plugin, with this structure: 10 | # _site/ <-- the contents that will be served 11 | # plugin-descriptor.properties <-- example contents below: 12 | # 13 | # site=true 14 | # description=My cool plugin 15 | # version=1.0 16 | # 17 | ### example jvm plugin for "foo" 18 | # 19 | # foo.zip <-- zip file for the plugin, with this structure: 20 | # .jar <-- classes, resources, dependencies 21 | # .jar <-- any number of jars 22 | # plugin-descriptor.properties <-- example contents below: 23 | # 24 | # jvm=true 25 | # classname=foo.bar.BazPlugin 26 | # description=My cool plugin 27 | # version=2.0.0-rc1 28 | # elasticsearch.version=2.0 29 | # java.version=1.7 30 | # 31 | ### mandatory elements for all plugins: 32 | # 33 | # 'description': simple summary of the plugin 34 | description=${project.description} 35 | # 36 | # 'version': plugin's version 37 | version=${project.version} 38 | # 39 | # 'name': the plugin name 40 | name=${elasticsearch.plugin.name} 41 | 42 | ### mandatory elements for site plugins: 43 | # 44 | # 'site': set to true to indicate contents of the _site/ 45 | # directory in the root of the plugin should be served. 46 | site=${elasticsearch.plugin.site} 47 | # 48 | ### mandatory elements for jvm plugins : 49 | # 50 | # 'jvm': true if the 'classname' class should be loaded 51 | # from jar files in the root directory of the plugin. 52 | # Note that only jar files in the root directory are 53 | # added to the classpath for the plugin! If you need 54 | # other resources, package them into a resources jar. 55 | jvm=${elasticsearch.plugin.jvm} 56 | # 57 | # 'classname': the name of the class to load, fully-qualified. 58 | classname=${elasticsearch.plugin.classname} 59 | # 60 | # 'java.version' version of java the code is built against 61 | # use the system property java.specification.version 62 | # version string must be a sequence of nonnegative decimal integers 63 | # separated by "."'s and may have leading zeros 64 | java.version=${maven.compiler.target} 65 | # 66 | # 'elasticsearch.version' version of elasticsearch compiled against 67 | # You will have to release a new version of the plugin for each new 68 | # elasticsearch release. This version is checked when the plugin 69 | # is loaded so Elasticsearch will refuse to start in the presence of 70 | # plugins with the incorrect elasticsearch.version. 71 | elasticsearch.version=${elasticsearch.version} 72 | # 73 | ### deprecated elements for jvm plugins : 74 | # 75 | # 'isolated': true if the plugin should have its own classloader. 76 | # passing false is deprecated, and only intended to support plugins 77 | # that have hard dependencies against each other. If this is 78 | # not specified, then the plugin is isolated by default. 79 | isolated=${elasticsearch.plugin.isolated} 80 | # -------------------------------------------------------------------------------- /chapter_17/rest_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.settings 9 | /.classpath 10 | /.project 11 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | elasticsearch-simple-plugin 6 | 4.0.0 7 | com.packtpub 8 | elasticsearch-rest-plugin 9 | ${elasticsearch.version} 10 | jar 11 | A REST plugin for ElasticSearch 12 | 2016 13 | 14 | 15 | The Apache Software License, Version 2.0 16 | http://www.apache.org/licenses/LICENSE-2.0.txt 17 | repo 18 | 19 | 20 | 21 | 22 | org.sonatype.oss 23 | oss-parent 24 | 7 25 | 26 | 27 | 28 | 5.1.1 29 | 1.8 30 | ${project.basedir}/src/main/assemblies/plugin.xml 31 | simple-plugin 32 | org.elasticsearch.plugin.simple.RestPlugin 33 | true 34 | false 35 | true 36 | 37 | 38 | 39 | 40 | sonatype 41 | http://oss.sonatype.org/content/repositories/releases/ 42 | 43 | 44 | 45 | 46 | 47 | 48 | org.elasticsearch 49 | elasticsearch 50 | ${elasticsearch.version} 51 | compile 52 | 53 | 54 | 55 | 56 | org.apache.httpcomponents 57 | httpclient 58 | 4.5.2 59 | test 60 | 61 | 62 | 63 | org.apache.logging.log4j 64 | log4j-api 65 | 2.3 66 | test 67 | 68 | 69 | 70 | org.hamcrest 71 | hamcrest-core 72 | 1.3 73 | test 74 | 75 | 76 | 77 | org.hamcrest 78 | hamcrest-library 79 | 1.3 80 | test 81 | 82 | 83 | junit 84 | junit 85 | 4.12 86 | test 87 | 88 | 89 | 90 | 91 | 92 | 93 | org.apache.maven.plugins 94 | maven-compiler-plugin 95 | 3.5.1 96 | 97 | ${maven.compiler.target} 98 | ${maven.compiler.target} 99 | 100 | 101 | 102 | org.apache.maven.plugins 103 | maven-surefire-plugin 104 | 2.11 105 | 106 | 107 | **/*Tests.java 108 | 109 | 110 | 111 | 112 | org.apache.maven.plugins 113 | maven-source-plugin 114 | 2.1.2 115 | 116 | 117 | attach-sources 118 | 119 | jar 120 | 121 | 122 | 123 | 124 | 125 | maven-assembly-plugin 126 | 127 | false 128 | ${project.build.directory}/releases/ 129 | 130 | ${basedir}/src/main/assemblies/plugin.xml 131 | 132 | 133 | 134 | fully.qualified.MainClass 135 | 136 | 137 | 138 | 139 | 140 | package 141 | 142 | single 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | analysis-ik-release 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | ${project.basedir}/config 11 | /config 12 | 13 | 14 | 15 | 16 | 17 | ${project.basedir}/src/main/resources/plugin-descriptor.properties 18 | 19 | true 20 | 21 | 22 | 23 | 24 | / 25 | true 26 | true 27 | 28 | org.elasticsearch:elasticsearch 29 | 30 | 31 | 32 | / 33 | true 34 | true 35 | 36 | org.apache.httpcomponents:httpclient 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/ShardSimpleResponse.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.cluster.routing.ShardRouting; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | import org.elasticsearch.common.io.stream.Streamable; 7 | import org.elasticsearch.common.xcontent.ToXContent; 8 | import org.elasticsearch.common.xcontent.XContentBuilder; 9 | 10 | import java.io.IOException; 11 | import java.util.HashSet; 12 | import java.util.Set; 13 | 14 | class ShardSimpleResponse implements Streamable, ToXContent { 15 | 16 | private Set simple; 17 | private ShardRouting shardRouting; 18 | 19 | ShardSimpleResponse() { 20 | } 21 | 22 | public ShardSimpleResponse(ShardRouting routing, Set simple) { 23 | this.shardRouting = routing; 24 | this.simple = simple; 25 | } 26 | 27 | public Set getTermList() { 28 | return simple; 29 | } 30 | 31 | @Override 32 | public void readFrom(StreamInput in) throws IOException { 33 | this.shardRouting = new ShardRouting(in); 34 | int n = in.readInt(); 35 | simple = new HashSet(n); 36 | for (int i = 0; i < n; i++) { 37 | simple.add(in.readString()); 38 | } 39 | } 40 | 41 | @Override 42 | public void writeTo(StreamOutput out) throws IOException { 43 | this.shardRouting.writeTo(out); 44 | out.writeInt(simple.size()); 45 | for (String t : simple) { 46 | out.writeString(t); 47 | } 48 | } 49 | 50 | public static ShardSimpleResponse readShardResult(StreamInput in) throws IOException { 51 | ShardSimpleResponse result = new ShardSimpleResponse(); 52 | result.readFrom(in); 53 | return result; 54 | } 55 | 56 | 57 | @Override 58 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { 59 | builder.startObject(ShardSimpleResponse.Fields.ROUTING) 60 | .field(ShardSimpleResponse.Fields.STATE, shardRouting.state()) 61 | .field(ShardSimpleResponse.Fields.PRIMARY, shardRouting.primary()) 62 | .field(ShardSimpleResponse.Fields.NODE, shardRouting.currentNodeId()) 63 | .field(ShardSimpleResponse.Fields.RELOCATING_NODE, shardRouting.relocatingNodeId()) 64 | .endObject(); 65 | 66 | builder.startArray(Fields.TERMS); 67 | for (String t : simple) { 68 | builder.value(t); 69 | } 70 | builder.endArray(); 71 | return builder; 72 | } 73 | 74 | static final class Fields { 75 | static final String ROUTING = "routing"; 76 | static final String STATE = "state"; 77 | static final String PRIMARY = "primary"; 78 | static final String NODE = "node"; 79 | static final String RELOCATING_NODE = "relocating_node"; 80 | static final String TERMS = "terms"; 81 | } 82 | 83 | } -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/SimpleAction.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.action.Action; 4 | import org.elasticsearch.client.ElasticsearchClient; 5 | 6 | public class SimpleAction extends Action { 7 | 8 | public static final SimpleAction INSTANCE = new SimpleAction(); 9 | public static final String NAME = "custom:indices/simple"; 10 | 11 | private SimpleAction() { 12 | super(NAME); 13 | } 14 | 15 | @Override 16 | public SimpleResponse newResponse() { 17 | return new SimpleResponse(); 18 | } 19 | 20 | @Override 21 | public SimpleRequestBuilder newRequestBuilder(ElasticsearchClient elasticsearchClient) { 22 | return new SimpleRequestBuilder(elasticsearchClient, this); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/SimpleRequest.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastRequest; 4 | import org.elasticsearch.common.io.stream.StreamInput; 5 | import org.elasticsearch.common.io.stream.StreamOutput; 6 | 7 | import java.io.IOException; 8 | 9 | public class SimpleRequest extends BroadcastRequest { 10 | 11 | private String field; 12 | 13 | SimpleRequest() { 14 | } 15 | 16 | public SimpleRequest(String... indices) { 17 | super(indices); 18 | } 19 | 20 | public void setField(String field) { 21 | this.field = field; 22 | } 23 | 24 | public String getField() { 25 | return field; 26 | } 27 | 28 | @Override 29 | public void readFrom(StreamInput in) throws IOException { 30 | super.readFrom(in); 31 | field = in.readString(); 32 | } 33 | 34 | @Override 35 | public void writeTo(StreamOutput out) throws IOException { 36 | super.writeTo(out); 37 | out.writeString(field); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/SimpleRequestBuilder.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; 4 | import org.elasticsearch.client.ElasticsearchClient; 5 | 6 | /** 7 | * A request to get simples of one or more indices. 8 | */ 9 | public class SimpleRequestBuilder extends BroadcastOperationRequestBuilder { 10 | 11 | public SimpleRequestBuilder(ElasticsearchClient client, SimpleAction action) { 12 | super(client, action, new SimpleRequest()); 13 | } 14 | 15 | } 16 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/SimpleResponse.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.action.ShardOperationFailedException; 4 | import org.elasticsearch.action.support.broadcast.BroadcastResponse; 5 | import org.elasticsearch.common.io.stream.StreamInput; 6 | import org.elasticsearch.common.io.stream.StreamOutput; 7 | 8 | import java.io.IOException; 9 | import java.util.HashSet; 10 | import java.util.List; 11 | import java.util.Set; 12 | 13 | /** 14 | * A response for simple action. 15 | */ 16 | public class SimpleResponse extends BroadcastResponse { 17 | 18 | private Set simple; 19 | 20 | SimpleResponse() { 21 | } 22 | 23 | SimpleResponse(int totalShards, int successfulShards, int failedShards, List shardFailures, Set simple) { 24 | super(totalShards, successfulShards, failedShards, shardFailures); 25 | this.simple = simple; 26 | } 27 | 28 | public Set getSimple() { 29 | return simple; 30 | } 31 | 32 | @Override 33 | public void readFrom(StreamInput in) throws IOException { 34 | super.readFrom(in); 35 | int n = in.readInt(); 36 | simple = new HashSet(); 37 | for (int i = 0; i < n; i++) { 38 | simple.add(in.readString()); 39 | } 40 | } 41 | 42 | @Override 43 | public void writeTo(StreamOutput out) throws IOException { 44 | super.writeTo(out); 45 | out.writeInt(simple.size()); 46 | for (String t : simple) { 47 | out.writeString(t); 48 | } 49 | } 50 | } -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/action/simple/TransportSimpleAction.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.action.simple; 2 | 3 | import org.elasticsearch.action.ShardOperationFailedException; 4 | import org.elasticsearch.action.support.ActionFilters; 5 | import org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction; 6 | import org.elasticsearch.cluster.ClusterState; 7 | import org.elasticsearch.cluster.block.ClusterBlockException; 8 | import org.elasticsearch.cluster.block.ClusterBlockLevel; 9 | import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; 10 | import org.elasticsearch.cluster.routing.ShardRouting; 11 | import org.elasticsearch.cluster.routing.ShardsIterator; 12 | import org.elasticsearch.cluster.service.ClusterService; 13 | import org.elasticsearch.common.inject.Inject; 14 | import org.elasticsearch.common.io.stream.StreamInput; 15 | import org.elasticsearch.common.settings.Settings; 16 | import org.elasticsearch.index.IndexService; 17 | import org.elasticsearch.index.shard.IndexShard; 18 | import org.elasticsearch.indices.IndicesService; 19 | import org.elasticsearch.threadpool.ThreadPool; 20 | import org.elasticsearch.transport.TransportService; 21 | 22 | import java.io.IOException; 23 | import java.util.HashSet; 24 | import java.util.List; 25 | import java.util.Set; 26 | 27 | /** 28 | * Simple index/indices action. 29 | */ 30 | public class TransportSimpleAction 31 | extends TransportBroadcastByNodeAction { 32 | 33 | private final IndicesService indicesService; 34 | 35 | @Inject 36 | public TransportSimpleAction(Settings settings, ThreadPool threadPool, ClusterService clusterService, 37 | TransportService transportService, IndicesService indicesService, 38 | ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver) { 39 | super(settings, SimpleAction.NAME, threadPool, clusterService, transportService, actionFilters, 40 | indexNameExpressionResolver, SimpleRequest::new, ThreadPool.Names.SEARCH); 41 | this.indicesService = indicesService; 42 | } 43 | 44 | 45 | @Override 46 | protected SimpleResponse newResponse(SimpleRequest request, int totalShards, int successfulShards, int failedShards, List shardSimpleResponses, List shardFailures, ClusterState clusterState) { 47 | Set simple = new HashSet(); 48 | for (ShardSimpleResponse shardSimpleResponse : shardSimpleResponses) { 49 | simple.addAll(shardSimpleResponse.getTermList()); 50 | } 51 | 52 | return new SimpleResponse(totalShards, successfulShards, failedShards, shardFailures, simple); 53 | } 54 | 55 | @Override 56 | protected ShardSimpleResponse shardOperation(SimpleRequest request, ShardRouting shardRouting) throws IOException { 57 | IndexService indexService = indicesService.indexServiceSafe(shardRouting.shardId().getIndex()); 58 | IndexShard indexShard = indexService.getShard(shardRouting.shardId().id()); 59 | indexShard.store().directory(); 60 | Set set = new HashSet(); 61 | set.add(request.getField() + "_" + shardRouting.shardId()); 62 | return new ShardSimpleResponse(shardRouting, set); 63 | } 64 | 65 | 66 | @Override 67 | protected ShardSimpleResponse readShardResult(StreamInput in) throws IOException { 68 | return ShardSimpleResponse.readShardResult(in); 69 | } 70 | 71 | @Override 72 | protected SimpleRequest readRequestFrom(StreamInput in) throws IOException { 73 | SimpleRequest request = new SimpleRequest(); 74 | request.readFrom(in); 75 | return request; 76 | } 77 | 78 | @Override 79 | protected ShardsIterator shards(ClusterState clusterState, SimpleRequest request, String[] concreteIndices) { 80 | return clusterState.routingTable().allShards(concreteIndices); 81 | } 82 | 83 | @Override 84 | protected ClusterBlockException checkGlobalBlock(ClusterState state, SimpleRequest request) { 85 | return state.blocks().globalBlockedException(ClusterBlockLevel.METADATA_READ); 86 | } 87 | 88 | @Override 89 | protected ClusterBlockException checkRequestBlock(ClusterState state, SimpleRequest request, String[] concreteIndices) { 90 | return state.blocks().indicesBlockedException(ClusterBlockLevel.METADATA_READ, concreteIndices); 91 | } 92 | } -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/plugin/simple/RestPlugin.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.plugin.simple; 2 | 3 | import org.elasticsearch.action.ActionRequest; 4 | import org.elasticsearch.action.ActionResponse; 5 | import org.elasticsearch.action.simple.SimpleAction; 6 | import org.elasticsearch.action.simple.TransportSimpleAction; 7 | import org.elasticsearch.plugins.ActionPlugin; 8 | import org.elasticsearch.plugins.Plugin; 9 | import org.elasticsearch.rest.RestHandler; 10 | import org.elasticsearch.rest.RestSimpleAction; 11 | 12 | import java.util.List; 13 | 14 | import static java.util.Collections.singletonList; 15 | 16 | public class RestPlugin extends Plugin implements ActionPlugin { 17 | 18 | @Override 19 | public List> getRestHandlers() { 20 | return singletonList(RestSimpleAction.class); 21 | } 22 | 23 | @Override 24 | public List, ? extends ActionResponse>> getActions() { 25 | return singletonList(new ActionHandler<>(SimpleAction.INSTANCE, TransportSimpleAction.class)); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/java/org/elasticsearch/rest/RestSimpleAction.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.rest; 2 | 3 | import org.elasticsearch.action.simple.SimpleAction; 4 | import org.elasticsearch.action.simple.SimpleRequest; 5 | import org.elasticsearch.action.simple.SimpleResponse; 6 | import org.elasticsearch.client.Client; 7 | import org.elasticsearch.client.node.NodeClient; 8 | import org.elasticsearch.common.Strings; 9 | import org.elasticsearch.common.inject.Inject; 10 | import org.elasticsearch.common.settings.Settings; 11 | import org.elasticsearch.common.xcontent.XContentBuilder; 12 | import org.elasticsearch.rest.action.RestBuilderListener; 13 | 14 | import java.io.IOException; 15 | 16 | import static org.elasticsearch.rest.RestRequest.Method.GET; 17 | import static org.elasticsearch.rest.RestRequest.Method.POST; 18 | import static org.elasticsearch.rest.RestStatus.OK; 19 | 20 | 21 | public class RestSimpleAction extends BaseRestHandler { 22 | @Inject 23 | public RestSimpleAction(Settings settings, Client client, RestController controller) { 24 | super(settings); 25 | controller.registerHandler(POST, "/_simple", this); 26 | controller.registerHandler(POST, "/{index}/_simple", this); 27 | controller.registerHandler(POST, "/_simple/{field}", this); 28 | controller.registerHandler(GET, "/_simple", this); 29 | controller.registerHandler(GET, "/{index}/_simple", this); 30 | controller.registerHandler(GET, "/_simple/{field}", this); 31 | } 32 | 33 | @Override 34 | protected RestChannelConsumer prepareRequest(RestRequest request, NodeClient client) throws IOException { 35 | final SimpleRequest simpleRequest = new SimpleRequest(Strings.splitStringByCommaToArray(request.param("index"))); 36 | simpleRequest.setField(request.param("field")); 37 | return channel -> client.execute(SimpleAction.INSTANCE, simpleRequest, new RestBuilderListener(channel){ 38 | @Override 39 | public RestResponse buildResponse(SimpleResponse simpleResponse, XContentBuilder builder) throws Exception { 40 | try { 41 | builder.startObject(); 42 | builder.field("ok", true); 43 | builder.array("terms", simpleResponse.getSimple().toArray()); 44 | builder.endObject(); 45 | 46 | } catch (Exception e) { 47 | onFailure(e); 48 | } 49 | return new BytesRestResponse(OK, builder); 50 | } 51 | }); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /chapter_17/rest_plugin/src/main/resources/plugin-descriptor.properties: -------------------------------------------------------------------------------- 1 | # Elasticsearch plugin descriptor file 2 | # This file must exist as 'plugin-descriptor.properties' at 3 | # the root directory of all plugins. 4 | # 5 | # A plugin can be 'site', 'jvm', or both. 6 | # 7 | ### example site plugin for "foo": 8 | # 9 | # foo.zip <-- zip file for the plugin, with this structure: 10 | # _site/ <-- the contents that will be served 11 | # plugin-descriptor.properties <-- example contents below: 12 | # 13 | # site=true 14 | # description=My cool plugin 15 | # version=1.0 16 | # 17 | ### example jvm plugin for "foo" 18 | # 19 | # foo.zip <-- zip file for the plugin, with this structure: 20 | # .jar <-- classes, resources, dependencies 21 | # .jar <-- any number of jars 22 | # plugin-descriptor.properties <-- example contents below: 23 | # 24 | # jvm=true 25 | # classname=foo.bar.BazPlugin 26 | # description=My cool plugin 27 | # version=2.0.0-rc1 28 | # elasticsearch.version=2.0 29 | # java.version=1.7 30 | # 31 | ### mandatory elements for all plugins: 32 | # 33 | # 'description': simple summary of the plugin 34 | description=${project.description} 35 | # 36 | # 'version': plugin's version 37 | version=${project.version} 38 | # 39 | # 'name': the plugin name 40 | name=${elasticsearch.plugin.name} 41 | 42 | ### mandatory elements for site plugins: 43 | # 44 | # 'site': set to true to indicate contents of the _site/ 45 | # directory in the root of the plugin should be served. 46 | site=${elasticsearch.plugin.site} 47 | # 48 | ### mandatory elements for jvm plugins : 49 | # 50 | # 'jvm': true if the 'classname' class should be loaded 51 | # from jar files in the root directory of the plugin. 52 | # Note that only jar files in the root directory are 53 | # added to the classpath for the plugin! If you need 54 | # other resources, package them into a resources jar. 55 | jvm=${elasticsearch.plugin.jvm} 56 | # 57 | # 'classname': the name of the class to load, fully-qualified. 58 | classname=${elasticsearch.plugin.classname} 59 | # 60 | # 'java.version' version of java the code is built against 61 | # use the system property java.specification.version 62 | # version string must be a sequence of nonnegative decimal integers 63 | # separated by "."'s and may have leading zeros 64 | java.version=${maven.compiler.target} 65 | # 66 | # 'elasticsearch.version' version of elasticsearch compiled against 67 | # You will have to release a new version of the plugin for each new 68 | # elasticsearch release. This version is checked when the plugin 69 | # is loaded so Elasticsearch will refuse to start in the presence of 70 | # plugins with the incorrect elasticsearch.version. 71 | elasticsearch.version=${elasticsearch.version} 72 | # 73 | ### deprecated elements for jvm plugins : 74 | # 75 | # 'isolated': true if the plugin should have its own classloader. 76 | # passing false is deprecated, and only intended to support plugins 77 | # that have hard dependencies against each other. If this is 78 | # not specified, then the plugin is isolated by default. 79 | isolated=${elasticsearch.plugin.isolated} 80 | # -------------------------------------------------------------------------------- /chapter_17/simple_plugin/.gitignore: -------------------------------------------------------------------------------- 1 | /data 2 | /work 3 | /logs 4 | /.idea 5 | /target 6 | .DS_Store 7 | *.iml 8 | /.settings 9 | /.classpath 10 | /.project 11 | -------------------------------------------------------------------------------- /chapter_17/simple_plugin/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | elasticsearch-simple-plugin 6 | 4.0.0 7 | com.packtpub 8 | simple-plugin 9 | ${elasticsearch.version} 10 | jar 11 | A simple plugin for ElasticSearch 12 | 2016 13 | 14 | 15 | The Apache Software License, Version 2.0 16 | http://www.apache.org/licenses/LICENSE-2.0.txt 17 | repo 18 | 19 | 20 | 21 | 22 | org.sonatype.oss 23 | oss-parent 24 | 7 25 | 26 | 27 | 28 | 5.1.1 29 | 1.8 30 | ${project.basedir}/src/main/assemblies/plugin.xml 31 | simple-plugin 32 | org.elasticsearch.plugin.simple.SimplePlugin 33 | true 34 | false 35 | true 36 | 37 | 38 | 39 | 40 | sonatype 41 | http://oss.sonatype.org/content/repositories/releases/ 42 | 43 | 44 | 45 | 46 | 47 | org.elasticsearch 48 | elasticsearch 49 | ${elasticsearch.version} 50 | compile 51 | 52 | 53 | 54 | 55 | org.apache.httpcomponents 56 | httpclient 57 | 4.5.2 58 | 59 | 60 | 61 | org.apache.logging.log4j 62 | log4j-api 63 | 2.3 64 | 65 | 66 | 67 | org.hamcrest 68 | hamcrest-core 69 | 1.3 70 | test 71 | 72 | 73 | 74 | org.hamcrest 75 | hamcrest-library 76 | 1.3 77 | test 78 | 79 | 80 | junit 81 | junit 82 | 4.12 83 | test 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | org.apache.maven.plugins 92 | maven-compiler-plugin 93 | 3.5.1 94 | 95 | ${maven.compiler.target} 96 | ${maven.compiler.target} 97 | 98 | 99 | 100 | org.apache.maven.plugins 101 | maven-surefire-plugin 102 | 2.11 103 | 104 | 105 | **/*Tests.java 106 | 107 | 108 | 109 | 110 | org.apache.maven.plugins 111 | maven-source-plugin 112 | 2.1.2 113 | 114 | 115 | attach-sources 116 | 117 | jar 118 | 119 | 120 | 121 | 122 | 123 | maven-assembly-plugin 124 | 125 | false 126 | ${project.build.directory}/releases/ 127 | 128 | ${basedir}/src/main/assemblies/plugin.xml 129 | 130 | 131 | 132 | fully.qualified.MainClass 133 | 134 | 135 | 136 | 137 | 138 | package 139 | 140 | single 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | -------------------------------------------------------------------------------- /chapter_17/simple_plugin/src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | simple-plugin 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /chapter_17/simple_plugin/src/main/java/org/elasticsearch/plugin/simple/SimplePlugin.java: -------------------------------------------------------------------------------- 1 | package org.elasticsearch.plugin.simple; 2 | 3 | 4 | import org.elasticsearch.plugins.Plugin; 5 | 6 | public class SimplePlugin extends Plugin { 7 | 8 | } 9 | -------------------------------------------------------------------------------- /chapter_17/simple_plugin/src/main/resources/plugin-descriptor.properties: -------------------------------------------------------------------------------- 1 | # Elasticsearch plugin descriptor file 2 | # This file must exist as 'plugin-descriptor.properties' at 3 | # the root directory of all plugins. 4 | # 5 | # A plugin can be 'site', 'jvm', or both. 6 | # 7 | ### example site plugin for "foo": 8 | # 9 | # foo.zip <-- zip file for the plugin, with this structure: 10 | # _site/ <-- the contents that will be served 11 | # plugin-descriptor.properties <-- example contents below: 12 | # 13 | # site=true 14 | # description=My cool plugin 15 | # version=1.0 16 | # 17 | ### example jvm plugin for "foo" 18 | # 19 | # foo.zip <-- zip file for the plugin, with this structure: 20 | # .jar <-- classes, resources, dependencies 21 | # .jar <-- any number of jars 22 | # plugin-descriptor.properties <-- example contents below: 23 | # 24 | # jvm=true 25 | # classname=foo.bar.BazPlugin 26 | # description=My cool plugin 27 | # version=2.0.0-rc1 28 | # elasticsearch.version=2.0 29 | # java.version=1.7 30 | # 31 | ### mandatory elements for all plugins: 32 | # 33 | # 'description': simple summary of the plugin 34 | description=${project.description} 35 | # 36 | # 'version': plugin's version 37 | version=${project.version} 38 | # 39 | # 'name': the plugin name 40 | name=${elasticsearch.plugin.name} 41 | 42 | ### mandatory elements for site plugins: 43 | # 44 | # 'site': set to true to indicate contents of the _site/ 45 | # directory in the root of the plugin should be served. 46 | site=${elasticsearch.plugin.site} 47 | # 48 | ### mandatory elements for jvm plugins : 49 | # 50 | # 'jvm': true if the 'classname' class should be loaded 51 | # from jar files in the root directory of the plugin. 52 | # Note that only jar files in the root directory are 53 | # added to the classpath for the plugin! If you need 54 | # other resources, package them into a resources jar. 55 | jvm=${elasticsearch.plugin.jvm} 56 | # 57 | # 'classname': the name of the class to load, fully-qualified. 58 | classname=${elasticsearch.plugin.classname} 59 | # 60 | # 'java.version' version of java the code is built against 61 | # use the system property java.specification.version 62 | # version string must be a sequence of nonnegative decimal integers 63 | # separated by "."'s and may have leading zeros 64 | java.version=${maven.compiler.target} 65 | # 66 | # 'elasticsearch.version' version of elasticsearch compiled against 67 | # You will have to release a new version of the plugin for each new 68 | # elasticsearch release. This version is checked when the plugin 69 | # is loaded so Elasticsearch will refuse to start in the presence of 70 | # plugins with the incorrect elasticsearch.version. 71 | elasticsearch.version=${elasticsearch.version} 72 | # 73 | ### deprecated elements for jvm plugins : 74 | # 75 | # 'isolated': true if the plugin should have its own classloader. 76 | # passing false is deprecated, and only intended to support plugins 77 | # that have hard dependencies against each other. If this is 78 | # not specified, then the plugin is isolated by default. 79 | isolated=${elasticsearch.plugin.isolated} 80 | # -------------------------------------------------------------------------------- /chapter_18/download_es_hadoop.sh: -------------------------------------------------------------------------------- 1 | wget http://download.elastic.co/hadoop/elasticsearch-hadoop-5.1.1.zip 2 | unzip elasticsearch-hadoop-5.1.1.zip 3 | 4 | 5 | -------------------------------------------------------------------------------- /chapter_18/download_spark.sh: -------------------------------------------------------------------------------- 1 | wget http://d3kbcqa49mib13.cloudfront.net/spark-2.1.0-bin-hadoop2.7.tgz 2 | tar xfvz spark-2.1.0-bin-hadoop2.7.tgz 3 | 4 | -------------------------------------------------------------------------------- /chapter_18/es.pig: -------------------------------------------------------------------------------- 1 | REGISTER /Users/alberto/elasticsearch/elasticsearch-hadoop-5.1.1/dist/elasticsearch-hadoop-pig-5.1.1.jar; 2 | 3 | SET pig.noSplitCombination TRUE; 4 | 5 | DEFINE EsStorage org.elasticsearch.hadoop.pig.EsStorage(); 6 | 7 | -- launch the Map/Reduce job with 5 reducers 8 | SET default_parallel 5; 9 | 10 | --load the GB.txt file 11 | geonames= LOAD 'GB.txt' using PigStorage('\t') AS (geonameid:int,name:chararray,asciiname:chararray,alternatenames:chararray,latitude:double,longitude:double,feature_class:chararray,feature_code:chararray,country_code:chararray,cc2:chararray,admin1_code:chararray,admin2_code:chararray,admin3_code:chararray,admin4_code:chararray,population:int,elevation:int,dem:chararray,timezone:chararray,modification_date:chararray); 12 | 13 | 14 | STORE geonames INTO 'geoname/gb' USING EsStorage(); -------------------------------------------------------------------------------- /chapter_18/install_pig_spark_macosx.sh: -------------------------------------------------------------------------------- 1 | brew install pig 2 | brew install apache-spark 3 | 4 | -------------------------------------------------------------------------------- /test.json: -------------------------------------------------------------------------------- 1 | { 2 | "query": { 3 | "filtered": { 4 | "filter": { 5 | "script": { 6 | "script": "doc[\"age\"].value > 80" 7 | } 8 | }, 9 | "query": { 10 | "match_all": {} 11 | } 12 | } 13 | } 14 | } --------------------------------------------------------------------------------