18 |
--------------------------------------------------------------------------------
/mapping/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | es-demo-parent
7 | io.mincong
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | es-demo-mapping
13 | Elasticsearch Demos - Mapping
14 |
15 |
16 |
17 | org.elasticsearch.test
18 | framework
19 | test
20 |
21 |
22 | org.elasticsearch.client
23 | elasticsearch-rest-high-level-client
24 | ${elasticsearch.version}
25 |
26 |
27 | org.apache.logging.log4j
28 | log4j-core
29 | test
30 |
31 |
32 | org.assertj
33 | assertj-core
34 | test
35 |
36 |
37 |
38 |
39 |
40 |
41 | io.fabric8
42 | docker-maven-plugin
43 |
44 |
45 |
46 |
47 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/DynamicMappingTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.Map;
4 | import org.assertj.core.api.Assertions;
5 | import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
6 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
7 | import org.elasticsearch.common.xcontent.XContentType;
8 | import org.elasticsearch.rest.RestStatus;
9 | import org.elasticsearch.test.ESSingleNodeTestCase;
10 | import org.junit.Test;
11 |
12 | /**
13 | * Test dynamic mapping in Elasticsearch.
14 | *
15 | * Fields and mapping types do not need to be defined before being used. Thanks to dynamic
16 | * mapping, new field names will be added automatically, just by indexing a document. New fields can
17 | * be added both to the top-level mapping type, and to inner object and nested fields.
18 | *
19 | * @author Mincong Huang
20 | */
21 | public class DynamicMappingTest extends ESSingleNodeTestCase {
22 |
23 | @Test
24 | public void typeText() {
25 | index("{\"aText\": \"Hello world\"}");
26 |
27 | var mappingResponse =
28 | client()
29 | .admin()
30 | .indices()
31 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("aText"))
32 | .actionGet();
33 | @SuppressWarnings("unchecked")
34 | var messageField =
35 | (Map)
36 | mappingResponse.fieldMappings("my_index", "_doc", "aText").sourceAsMap().get("aText");
37 | var fields = Map.of("keyword", Map.of("type", "keyword", "ignore_above", 256));
38 | Assertions.assertThat(messageField)
39 | .hasSize(2)
40 | .containsEntry("type", "text")
41 | .containsEntry("fields", fields);
42 | }
43 |
44 | @Test
45 | public void typeDate() {
46 | index("{\"aDate\": \"2020-04-04T16:00:00\"}");
47 |
48 | var mappingResponse =
49 | client()
50 | .admin()
51 | .indices()
52 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("aDate"))
53 | .actionGet();
54 | @SuppressWarnings("unchecked")
55 | var messageField =
56 | (Map)
57 | mappingResponse.fieldMappings("my_index", "_doc", "aDate").sourceAsMap().get("aDate");
58 | Assertions.assertThat(messageField).hasSize(1).containsEntry("type", "date");
59 | }
60 |
61 | @Test
62 | public void typeLong() {
63 | index("{\"aLong\": 123}");
64 |
65 | var mappingResponse =
66 | client()
67 | .admin()
68 | .indices()
69 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("aLong"))
70 | .actionGet();
71 | @SuppressWarnings("unchecked")
72 | var messageField =
73 | (Map)
74 | mappingResponse.fieldMappings("my_index", "_doc", "aLong").sourceAsMap().get("aLong");
75 | Assertions.assertThat(messageField).hasSize(1).containsEntry("type", "long");
76 | }
77 |
78 | @Test
79 | public void typeDouble() {
80 | index("{\"aFloat\": 123.4}");
81 |
82 | var mappingResponse =
83 | client()
84 | .admin()
85 | .indices()
86 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("aFloat"))
87 | .actionGet();
88 | @SuppressWarnings("unchecked")
89 | var messageField =
90 | (Map)
91 | mappingResponse.fieldMappings("my_index", "_doc", "aFloat").sourceAsMap().get("aFloat");
92 | Assertions.assertThat(messageField).hasSize(1).containsEntry("type", "float");
93 | }
94 |
95 | @Test
96 | public void typeBoolean() {
97 | index("{\"aBoolean\": true}");
98 |
99 | var mappingResponse =
100 | client()
101 | .admin()
102 | .indices()
103 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("aBoolean"))
104 | .actionGet();
105 | @SuppressWarnings("unchecked")
106 | var messageField =
107 | (Map)
108 | mappingResponse
109 | .fieldMappings("my_index", "_doc", "aBoolean")
110 | .sourceAsMap()
111 | .get("aBoolean");
112 | Assertions.assertThat(messageField).hasSize(1).containsEntry("type", "boolean");
113 | }
114 |
115 | private void index(String source) {
116 | var indexResponse =
117 | client()
118 | .prepareIndex()
119 | .setIndex("my_index")
120 | .setSource(source, XContentType.JSON)
121 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
122 | .execute()
123 | .actionGet();
124 | Assertions.assertThat(indexResponse.status()).isEqualTo(RestStatus.CREATED);
125 | }
126 | }
127 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/ExplicitMappingTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.Map;
4 | import org.assertj.core.api.Assertions;
5 | import org.elasticsearch.action.admin.indices.create.CreateIndexRequest;
6 | import org.elasticsearch.action.admin.indices.mapping.get.GetFieldMappingsRequest;
7 | import org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest;
8 | import org.elasticsearch.common.xcontent.XContentType;
9 | import org.elasticsearch.test.ESSingleNodeTestCase;
10 | import org.junit.Test;
11 |
12 | /**
13 | * Test explicit mapping in Elasticsearch.
14 | *
15 | * You know more about your data than Elasticsearch can guess, so while dynamic mapping can be
16 | * useful to get started, at some point you will want to specify your own explicit mappings.
17 | *
18 | * @author Mincong Huang
19 | */
20 | public class ExplicitMappingTest extends ESSingleNodeTestCase {
21 |
22 | @Test
23 | public void createIndexWithExplictMapping() {
24 | var create =
25 | new CreateIndexRequest("my_index")
26 | .mapping(
27 | "_doc", "{\"properties\":{\"message\":{\"type\":\"text\"}}}", XContentType.JSON);
28 | var response = client().admin().indices().create(create).actionGet();
29 | Assertions.assertThat(response.isAcknowledged()).isTrue();
30 |
31 | var mappingResponse =
32 | client()
33 | .admin()
34 | .indices()
35 | .getFieldMappings(new GetFieldMappingsRequest().indices("my_index").fields("message"))
36 | .actionGet();
37 | @SuppressWarnings("unchecked")
38 | var messageField =
39 | (Map)
40 | mappingResponse
41 | .fieldMappings("my_index", "_doc", "message")
42 | .sourceAsMap()
43 | .get("message");
44 | Assertions.assertThat(messageField).hasSize(1).containsEntry("type", "text");
45 | }
46 |
47 | @Test
48 | public void addFieldToExistingMapping() {
49 | createIndexWithExplictMapping();
50 |
51 | var putMapping =
52 | new PutMappingRequest("my_index")
53 | .type("_doc")
54 | .source("{\"properties\":{\"description\":{\"type\":\"text\"}}}", XContentType.JSON);
55 | var response = client().admin().indices().putMapping(putMapping).actionGet();
56 | Assertions.assertThat(response.isAcknowledged()).isTrue();
57 |
58 | var mappingResponse =
59 | client()
60 | .admin()
61 | .indices()
62 | .getFieldMappings(
63 | new GetFieldMappingsRequest().indices("my_index").fields("description"))
64 | .actionGet();
65 | @SuppressWarnings("unchecked")
66 | var descriptionField =
67 | (Map)
68 | mappingResponse
69 | .fieldMappings("my_index", "_doc", "description")
70 | .sourceAsMap()
71 | .get("description");
72 | Assertions.assertThat(descriptionField).hasSize(1).containsEntry("type", "text");
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/IndexStatsTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import org.assertj.core.api.Assertions;
4 | import org.elasticsearch.action.index.IndexRequest;
5 | import org.elasticsearch.action.index.IndexResponse;
6 | import org.elasticsearch.common.xcontent.XContentType;
7 | import org.elasticsearch.rest.RestStatus;
8 | import org.elasticsearch.test.ESSingleNodeTestCase;
9 | import org.junit.Before;
10 | import org.junit.Test;
11 |
12 | public class IndexStatsTest extends ESSingleNodeTestCase {
13 |
14 | @Override
15 | @Before
16 | public void setUp() throws Exception {
17 | super.setUp();
18 |
19 | IndexRequest idxRequest =
20 | new IndexRequest("msg").source("{\"msg\":\"Hello world!\"}", XContentType.JSON);
21 | IndexResponse idxResponse = client().index(idxRequest).actionGet();
22 | assertEquals("msg", idxResponse.getIndex());
23 | assertEquals(RestStatus.CREATED, idxResponse.status());
24 | }
25 |
26 | @Test
27 | public void itShouldReturnEmptyStats() {
28 | var response = client().admin().indices().prepareStats().clear().get();
29 | Assertions.assertThat(response.getIndices()).containsOnlyKeys("msg");
30 | var index = response.getIndex("msg");
31 | // all stats are null, some examples:
32 | Assertions.assertThat(index.getPrimaries().completion).isNull();
33 | Assertions.assertThat(index.getPrimaries().docs).isNull();
34 | Assertions.assertThat(index.getPrimaries().fieldData).isNull();
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/IndexTemplateTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.List;
4 | import java.util.Map;
5 | import org.assertj.core.api.Assertions;
6 | import org.elasticsearch.common.xcontent.XContentType;
7 | import org.elasticsearch.rest.RestStatus;
8 | import org.elasticsearch.test.ESSingleNodeTestCase;
9 | import org.junit.Test;
10 |
11 | public class IndexTemplateTest extends ESSingleNodeTestCase {
12 |
13 | @Test
14 | public void createTemplate() {
15 | // Given a user template to create
16 |
17 | // When creating it
18 | client()
19 | .admin()
20 | .indices()
21 | .preparePutTemplate("user_template")
22 | .setPatterns(List.of("user*"))
23 | .addMapping(
24 | "_doc",
25 | json("{'properties': {'name': {'type': 'keyword'}, 'age': {'type': 'long'}}}"),
26 | XContentType.JSON)
27 | .get();
28 |
29 | // Then the creation is successful
30 | var response = client().admin().indices().prepareGetTemplates("user_template").get();
31 | var metadata = response.getIndexTemplates().get(0);
32 | Assertions.assertThat(metadata.getName()).isEqualTo("user_template");
33 | }
34 |
35 | /**
36 | * This test demonstrates how index template is used when a new index is created and it matches
37 | * the expression of the index template.
38 | */
39 | @Test
40 | public void createIndex() {
41 | // Given an existing template
42 | client()
43 | .admin()
44 | .indices()
45 | .preparePutTemplate("user_template")
46 | .setPatterns(List.of("user*"))
47 | .addMapping(
48 | "_doc",
49 | json("{'properties': {'name': {'type': 'keyword'}, 'age': {'type': 'long'}}}"),
50 | XContentType.JSON)
51 | .get();
52 |
53 | // When creating an index matching this template
54 | var indexResponse =
55 | client()
56 | .prepareIndex()
57 | .setIndex("user_fr")
58 | .setSource(json("{'name': 'First Last', 'age': 30}"), XContentType.JSON)
59 | .execute()
60 | .actionGet();
61 |
62 | // Then the index is created
63 | Assertions.assertThat(indexResponse.status()).isEqualTo(RestStatus.CREATED);
64 |
65 | // And the mappings are correct
66 | var mappingResponse = client().admin().indices().prepareGetMappings("user_fr").get();
67 | @SuppressWarnings("unchecked")
68 | var properties =
69 | (Map>)
70 | mappingResponse.mappings().get("user_fr").get("_doc").sourceAsMap().get("properties");
71 | Assertions.assertThat(properties)
72 | .hasSize(2)
73 | .containsEntry("age", Map.of("type", "long"))
74 | .containsEntry("name", Map.of("type", "keyword"));
75 | }
76 |
77 | String json(String singleQuoted) {
78 | return singleQuoted.replace("'", "\"");
79 | }
80 | }
81 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/StackOverflow60500157IT.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.concurrent.CountDownLatch;
4 | import java.util.concurrent.TimeUnit;
5 | import java.util.concurrent.atomic.AtomicReference;
6 | import org.apache.http.HttpHost;
7 | import org.assertj.core.api.Assertions;
8 | import org.elasticsearch.action.ActionListener;
9 | import org.elasticsearch.action.ActionRequestValidationException;
10 | import org.elasticsearch.action.support.master.AcknowledgedResponse;
11 | import org.elasticsearch.client.RequestOptions;
12 | import org.elasticsearch.client.RestClient;
13 | import org.elasticsearch.client.RestHighLevelClient;
14 | import org.elasticsearch.client.indices.CreateIndexRequest;
15 | import org.elasticsearch.common.xcontent.XContentType;
16 | import org.elasticsearch.test.rest.ESRestTestCase;
17 | import org.junit.*;
18 |
19 | /**
20 | * Put mapping with Elastic Search's High level REST JAVA client asynchronously - deprecated error
21 | *
22 | * https://stackoverflow.com/questions/60500157
23 | *
24 | * @author Mincong Huang
25 | */
26 | public class StackOverflow60500157IT extends ESRestTestCase {
27 |
28 | @BeforeClass
29 | public static void setUpBeforeClass() {
30 | System.setProperty("tests.rest.cluster", "localhost:19200");
31 | }
32 |
33 | @AfterClass
34 | public static void tearDownAfterClass() {
35 | System.clearProperty("tests.rest.cluster");
36 | }
37 |
38 | private RestHighLevelClient client;
39 |
40 | @Before
41 | @Override
42 | public void setUp() throws Exception {
43 | super.setUp();
44 |
45 | var builder = RestClient.builder(new HttpHost("localhost", 19200, "http"));
46 | client = new RestHighLevelClient(builder);
47 |
48 | var createRequest = new CreateIndexRequest("contacts");
49 | var response = client.indices().create(createRequest, RequestOptions.DEFAULT);
50 | Assertions.assertThat(response.isAcknowledged()).isTrue();
51 | }
52 |
53 | @After
54 | public void tearDown() throws Exception {
55 | client.close();
56 | super.tearDown();
57 | }
58 |
59 | @Test
60 | public void oldRequest() {
61 | var source =
62 | "{\"properties\":{\"list_id\":{\"type\":\"integer\"},\"contact_id\":{\"type\":\"integer\"}}}";
63 | var request =
64 | new org.elasticsearch.action.admin.indices.mapping.put.PutMappingRequest("contacts")
65 | .source(source, XContentType.JSON);
66 | Assertions.assertThatExceptionOfType(ActionRequestValidationException.class)
67 | .isThrownBy(
68 | () -> {
69 | @SuppressWarnings("deprecation")
70 | var response = client.indices().putMapping(request, RequestOptions.DEFAULT);
71 | })
72 | .withMessageContaining("mapping type is missing");
73 | }
74 |
75 | @Test
76 | public void newRequest() throws Exception {
77 | var source =
78 | "{\"properties\":{\"list_id\":{\"type\":\"integer\"},\"contact_id\":{\"type\":\"integer\"}}}";
79 | var request =
80 | new org.elasticsearch.client.indices.PutMappingRequest("contacts")
81 | .source(source, XContentType.JSON);
82 |
83 | var response = client.indices().putMapping(request, RequestOptions.DEFAULT);
84 | Assertions.assertThat(response.isAcknowledged()).isTrue();
85 | }
86 |
87 | @Test
88 | public void newRequestAsync() throws Exception {
89 | var latch = new CountDownLatch(1);
90 | var source =
91 | "{\"properties\":{\"list_id\":{\"type\":\"integer\"},\"contact_id\":{\"type\":\"integer\"}}}";
92 | var request =
93 | new org.elasticsearch.client.indices.PutMappingRequest("contacts")
94 | .source(source, XContentType.JSON);
95 |
96 | var response = new AtomicReference();
97 | client
98 | .indices()
99 | .putMappingAsync(
100 | request,
101 | RequestOptions.DEFAULT,
102 | new ActionListener<>() {
103 | @Override
104 | public void onResponse(AcknowledgedResponse r) {
105 | response.set(r);
106 | latch.countDown();
107 | }
108 |
109 | @Override
110 | public void onFailure(Exception e) {
111 | latch.countDown();
112 | }
113 | });
114 | latch.await(3, TimeUnit.SECONDS);
115 | Assertions.assertThat(response.get().isAcknowledged()).isTrue();
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/mapping/src/test/java/io/mincong/elasticsearch/StackOverflow60667649IT.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.Map;
4 | import org.apache.http.HttpHost;
5 | import org.assertj.core.api.Assertions;
6 | import org.elasticsearch.client.RequestOptions;
7 | import org.elasticsearch.client.RestClient;
8 | import org.elasticsearch.client.RestHighLevelClient;
9 | import org.elasticsearch.client.indices.CreateIndexRequest;
10 | import org.elasticsearch.client.indices.GetFieldMappingsRequest;
11 | import org.elasticsearch.client.indices.PutMappingRequest;
12 | import org.elasticsearch.common.xcontent.XContentBuilder;
13 | import org.elasticsearch.common.xcontent.XContentFactory;
14 | import org.elasticsearch.common.xcontent.XContentType;
15 | import org.elasticsearch.test.rest.ESRestTestCase;
16 | import org.junit.*;
17 |
18 | /**
19 | * Put mapping in ElasticSearch by Java API
20 | *
21 | * https://stackoverflow.com/questions/60667649
22 | *
23 | * @author Mincong Huang
24 | */
25 | public class StackOverflow60667649IT extends ESRestTestCase {
26 |
27 | @BeforeClass
28 | public static void setUpBeforeClass() {
29 | System.setProperty("tests.rest.cluster", "localhost:19200");
30 | }
31 |
32 | @AfterClass
33 | public static void tearDownAfterClass() {
34 | System.clearProperty("tests.rest.cluster");
35 | }
36 |
37 | private RestHighLevelClient restClient;
38 |
39 | @Before
40 | @Override
41 | public void setUp() throws Exception {
42 | super.setUp();
43 |
44 | var builder = RestClient.builder(new HttpHost("localhost", 19200, "http"));
45 | restClient = new RestHighLevelClient(builder);
46 |
47 | var createRequest = new CreateIndexRequest("my_index");
48 | var response = restClient.indices().create(createRequest, RequestOptions.DEFAULT);
49 | Assertions.assertThat(response.isAcknowledged()).isTrue();
50 | }
51 |
52 | @After
53 | public void tearDown() throws Exception {
54 | restClient.close();
55 | super.tearDown();
56 | }
57 |
58 | @Test
59 | public void javaSource() throws Exception {
60 | XContentBuilder builder = XContentFactory.jsonBuilder();
61 | builder.startObject();
62 | builder.startObject("properties");
63 | builder.startObject("mje-test-location");
64 | builder.field("type", "geo_point");
65 | builder.endObject();
66 | builder.endObject();
67 | builder.endObject();
68 |
69 | var putMapping = new PutMappingRequest("my_index").source(builder);
70 | var putResponse = restClient.indices().putMapping(putMapping, RequestOptions.DEFAULT);
71 | Assertions.assertThat(putResponse.isAcknowledged()).isTrue();
72 |
73 | var getFieldMapping =
74 | new GetFieldMappingsRequest().indices("my_index").fields("mje-test-location");
75 | var mappingResponse =
76 | restClient.indices().getFieldMapping(getFieldMapping, RequestOptions.DEFAULT);
77 |
78 | @SuppressWarnings("unchecked")
79 | var field =
80 | (Map)
81 | mappingResponse
82 | .fieldMappings("my_index", "mje-test-location")
83 | .sourceAsMap()
84 | .get("mje-test-location");
85 | Assertions.assertThat(field).hasSize(1).containsEntry("type", "geo_point");
86 | }
87 |
88 | @Test
89 | public void stringSource() throws Exception {
90 | var putMapping =
91 | new PutMappingRequest("my_index")
92 | .source(
93 | "{\"properties\":{\"mje-test-location\":{\"type\":\"geo_point\"}}}",
94 | XContentType.JSON);
95 | var putResponse = restClient.indices().putMapping(putMapping, RequestOptions.DEFAULT);
96 | Assertions.assertThat(putResponse.isAcknowledged()).isTrue();
97 |
98 | var getFieldMapping =
99 | new GetFieldMappingsRequest().indices("my_index").fields("mje-test-location");
100 | var mappingResponse =
101 | restClient.indices().getFieldMapping(getFieldMapping, RequestOptions.DEFAULT);
102 |
103 | @SuppressWarnings("unchecked")
104 | var field =
105 | (Map)
106 | mappingResponse
107 | .fieldMappings("my_index", "mje-test-location")
108 | .sourceAsMap()
109 | .get("mje-test-location");
110 | Assertions.assertThat(field).hasSize(1).containsEntry("type", "geo_point");
111 | }
112 | }
113 |
--------------------------------------------------------------------------------
/notebooks/README.md:
--------------------------------------------------------------------------------
1 | # Notebooks
2 |
3 | Different notebooks for Elasticsearch.
4 |
--------------------------------------------------------------------------------
/notebooks/partial-update.md:
--------------------------------------------------------------------------------
1 | # Partial Update Document
2 |
3 | Can Elasticsearch handle partial update for documents?
4 |
5 | ## Start Elasticsearch
6 |
7 | ```
8 | docker run \
9 | --rm \
10 | -p 9200:9200 \
11 | -p 9300:9300 \
12 | -e "discovery.type=single-node" \
13 | -e "cluster.name=es-docker-cluster" \
14 | docker.elastic.co/elasticsearch/elasticsearch:7.14.0
15 | ```
16 |
17 | ## Create new document
18 |
19 | Request:
20 |
21 | ```sh
22 | curl -X PUT http://localhost:9200/my_index/_doc/1?pretty \
23 | -H 'Content-Type: application/json' \
24 | -d '{"key1": "value1"}'
25 | ```
26 |
27 | Response:
28 |
29 | ```json
30 | {
31 | "_index" : "my_index",
32 | "_type" : "_doc",
33 | "_id" : "1",
34 | "_version" : 1,
35 | "result" : "created",
36 | "_shards" : {
37 | "total" : 2,
38 | "successful" : 1,
39 | "failed" : 0
40 | },
41 | "_seq_no" : 0,
42 | "_primary_term" : 1
43 | }
44 | ```
45 |
46 | ## Update existing document
47 |
48 | Request:
49 |
50 | ```sh
51 | curl -X POST 'http://localhost:9200/my_index/_update/1?pretty' \
52 | -H 'Content-Type: application/json' \
53 | -d '{
54 | "doc": {
55 | "key2": "value2"
56 | }
57 | }'
58 | ```
59 |
60 | Response:
61 |
62 | ```json
63 | {
64 | "_index" : "my_index",
65 | "_type" : "_doc",
66 | "_id" : "1",
67 | "_version" : 2,
68 | "result" : "updated",
69 | "_shards" : {
70 | "total" : 2,
71 | "successful" : 1,
72 | "failed" : 0
73 | },
74 | "_seq_no" : 1,
75 | "_primary_term" : 1
76 | }
77 | ```
78 |
79 | Result:
80 |
81 | ```
82 | curl 'http://localhost:9200/my_index/_doc/1?pretty'
83 | ```
84 |
85 | ```js
86 | {
87 | "_index" : "my_index",
88 | "_type" : "_doc",
89 | "_id" : "1",
90 | "_version" : 2,
91 | "_seq_no" : 1,
92 | "_primary_term" : 1,
93 | "found" : true,
94 | "_source" : {
95 | "key1" : "value1", // both key1 and key2 are available
96 | "key2" : "value2"
97 | }
98 | }
99 | ```
--------------------------------------------------------------------------------
/ops/gc.md:
--------------------------------------------------------------------------------
1 | # GC
2 |
3 |
4 |
--------------------------------------------------------------------------------
/scripts/start-elasticsearch.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 |
3 | # How can I get the source directory of a Bash script from within the script itself?
4 | # https://stackoverflow.com/questions/59895/
5 | current_dir="$(cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd)"
6 |
7 | docker run \
8 | --rm \
9 | -p 9200:9200 \
10 | -p 9300:9300 \
11 | -e "discovery.type=single-node" \
12 | -e "cluster.name=es-docker-cluster" \
13 | -v "${current_dir}/../demo-dvf/src/main/resources/config/custom.elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml" \
14 | docker.elastic.co/elasticsearch/elasticsearch:7.12.0
15 |
--------------------------------------------------------------------------------
/scripts/upgrade-es-version.sh:
--------------------------------------------------------------------------------
1 | #!/bin/bash
2 | #
3 | # Usage:
4 | #
5 | # scripts/upgrade-es-version.sh
6 | #
7 | # Sample upgrading from 7.8.0 to 7.10.0:
8 | #
9 | # scripts/upgrade-es-version.sh 7.8.0 7.10.0
10 | #
11 | old_version="$1"
12 | new_version="$2"
13 |
14 | if [[ -z $old_version || -z $new_version ]]
15 | then
16 | echo "Missing argument(s). Usage:"
17 | echo
18 | echo " upgrade-es-version.sh 7.8.0 7.10.0"
19 | echo
20 | exit 1
21 | fi
22 |
23 | # Update configuration files
24 | filepaths=($(rg --files-with-matches --glob "**/*.{xml,yml}" CURRENT_ES_VERSION))
25 | for filepath in "${filepaths[@]}"
26 | do
27 | sed -i '' -e "s/${old_version}/${new_version}/g" $filepath
28 | echo "✅ ${filepath}"
29 | done
30 |
31 | # Update README
32 | start=$(grep -n MANAGED_BLOCK_RUN_ES_START README.md | cut -f 1 -d :)
33 | end=$(grep -n MANAGED_BLOCK_RUN_ES_END README.md | cut -f 1 -d :)
34 | sed -i '' "${start},${end}s/${old_version}/${new_version}/g" README.md
35 | echo "✅ README.md"
36 |
37 | echo "Finished."
38 |
--------------------------------------------------------------------------------
/search/README.md:
--------------------------------------------------------------------------------
1 | # Search
2 |
3 | Search APIs ()
4 |
--------------------------------------------------------------------------------
/search/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | es-demo-parent
7 | io.mincong
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | es-demo-search
13 | Elasticsearch Demos - Search
14 |
15 |
16 |
17 | org.elasticsearch.test
18 | framework
19 | test
20 |
21 |
22 | org.apache.logging.log4j
23 | log4j-core
24 | test
25 |
26 |
27 | org.assertj
28 | assertj-core
29 | test
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/search/src/test/java/io/mincong/elasticsearch/BlogDisqus4852306721Test.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.Map;
4 | import org.assertj.core.api.Assertions;
5 | import org.elasticsearch.action.index.IndexRequest;
6 | import org.elasticsearch.action.search.SearchRequest;
7 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
8 | import org.elasticsearch.common.xcontent.XContentType;
9 | import org.elasticsearch.index.query.QueryBuilders;
10 | import org.elasticsearch.rest.RestStatus;
11 | import org.elasticsearch.search.SearchHit;
12 | import org.elasticsearch.search.builder.SearchSourceBuilder;
13 | import org.elasticsearch.test.ESSingleNodeTestCase;
14 | import org.junit.Before;
15 | import org.junit.Test;
16 |
17 | /**
18 | * Tests boolean query via Search API.
19 | *
20 | * Comment: https://mincong.io/2019/11/24/essinglenodetestcase/#comment-4852306721
21 | *
22 | * @author Mincong Huang
23 | */
24 | public class BlogDisqus4852306721Test extends ESSingleNodeTestCase {
25 |
26 | @Override
27 | @Before
28 | public void setUp() throws Exception {
29 | super.setUp();
30 |
31 | var bulkResponse =
32 | client()
33 | .prepareBulk()
34 | .add(
35 | new IndexRequest("transactions")
36 | .id("account1.tx1")
37 | .source(
38 | Map.of(
39 | "transactionDate", "2020-03-19T00:00:00",
40 | "accountId", "1",
41 | "amount", -10.0),
42 | XContentType.JSON))
43 | .add(
44 | new IndexRequest("transactions")
45 | .id("account1.tx2")
46 | .source(
47 | Map.of(
48 | "transactionDate", "2020-03-20T00:00:00",
49 | "accountId", "1",
50 | "amount", -20.0),
51 | XContentType.JSON))
52 | .add(
53 | new IndexRequest("transactions")
54 | .id("account2.tx3")
55 | .source(
56 | Map.of(
57 | "transactionDate", "2020-03-21T00:00:00",
58 | "accountId", "2",
59 | "amount", -30.0),
60 | XContentType.JSON))
61 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
62 | .execute()
63 | .actionGet();
64 |
65 | assertEquals(RestStatus.OK, bulkResponse.status());
66 | for (var r : bulkResponse.getItems()) {
67 | assertEquals(RestStatus.CREATED, r.status());
68 | }
69 | }
70 |
71 | @Test
72 | public void booleanQuery() {
73 | var sourceBuilder =
74 | QueryBuilders.boolQuery()
75 | .must(QueryBuilders.rangeQuery("transactionDate").gte("2020-03-20").lte("2020-03-28"))
76 | .must(QueryBuilders.matchQuery("accountId", "1"));
77 |
78 | var request = new SearchRequest().source(new SearchSourceBuilder().query(sourceBuilder));
79 | var response = client().search(request).actionGet();
80 |
81 | Assertions.assertThat(response.getHits().getHits())
82 | .hasSize(1)
83 | .extracting(SearchHit::getId)
84 | .containsExactly("account1.tx2");
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/search/src/test/java/io/mincong/elasticsearch/SearchScrollTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.ArrayList;
4 | import java.util.Map;
5 | import org.elasticsearch.action.index.IndexRequest;
6 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
7 | import org.elasticsearch.common.unit.TimeValue;
8 | import org.elasticsearch.test.ESSingleNodeTestCase;
9 | import org.junit.Before;
10 | import org.junit.Test;
11 |
12 | /**
13 | * Test Elasticsearch Search Scroll API
14 | *
15 | * @author Mincong Huang
16 | * @see Elasticsearch
18 | * REST API: Request Body Search - Scroll
19 | */
20 | public class SearchScrollTest extends ESSingleNodeTestCase {
21 |
22 | @Override
23 | @Before
24 | public void setUp() throws Exception {
25 | super.setUp();
26 | var bulkRequest = client().prepareBulk();
27 | for (int i = 0; i < 300; i++) {
28 | bulkRequest.add(new IndexRequest("my_index").id(String.valueOf(i)).source(Map.of()));
29 | }
30 | bulkRequest.setRefreshPolicy(RefreshPolicy.IMMEDIATE).execute().actionGet();
31 | }
32 |
33 | @Test
34 | public void scroll() {
35 | var results = new ArrayList();
36 |
37 | // first request
38 | var searchResponse =
39 | client()
40 | .prepareSearch()
41 | .setIndices("my_index")
42 | .setSize(100)
43 | .setScroll(TimeValue.timeValueMinutes(1))
44 | .execute()
45 | .actionGet();
46 | for (var hit : searchResponse.getHits()) {
47 | results.add(hit.getId());
48 | }
49 | logger.info(
50 | "results={} ({} new), scrollId={}",
51 | results.size(),
52 | results.size(),
53 | searchResponse.getScrollId());
54 |
55 | // more requests
56 | var scrollId = searchResponse.getScrollId();
57 | var hasNext = !results.isEmpty();
58 | while (hasNext) {
59 | var resp =
60 | client()
61 | .prepareSearchScroll(scrollId)
62 | .setScroll(TimeValue.timeValueMinutes(1))
63 | .execute()
64 | .actionGet();
65 | var newResults = new ArrayList();
66 | for (var hit : resp.getHits()) {
67 | newResults.add(hit.getId());
68 | }
69 | results.addAll(newResults);
70 | logger.info(
71 | "results={} ({} new), scrollId={}",
72 | results.size(),
73 | newResults.size(),
74 | resp.getScrollId());
75 |
76 | hasNext = !newResults.isEmpty();
77 | scrollId = resp.getScrollId();
78 | }
79 |
80 | assertEquals(300, results.size());
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/search/src/test/java/io/mincong/elasticsearch/SearchTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.Map;
4 | import org.elasticsearch.action.bulk.BulkItemResponse;
5 | import org.elasticsearch.action.bulk.BulkResponse;
6 | import org.elasticsearch.action.index.IndexRequest;
7 | import org.elasticsearch.action.search.SearchResponse;
8 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
9 | import org.elasticsearch.common.xcontent.XContentType;
10 | import org.elasticsearch.index.query.QueryBuilders;
11 | import org.elasticsearch.rest.RestStatus;
12 | import org.elasticsearch.search.SearchHits;
13 | import org.elasticsearch.test.ESSingleNodeTestCase;
14 | import org.junit.Before;
15 | import org.junit.Test;
16 |
17 | /**
18 | * Tests Search APIs.
19 | *
20 | * @author Mincong Huang
21 | * @see Search
23 | * APIs | Java REST Client | Elastic
24 | */
25 | public class SearchTest extends ESSingleNodeTestCase {
26 |
27 | @Override
28 | @Before
29 | public void setUp() throws Exception {
30 | super.setUp();
31 |
32 | BulkResponse response =
33 | client()
34 | .prepareBulk()
35 | .add(
36 | new IndexRequest("users")
37 | .id("sansa")
38 | .source(
39 | Map.of(
40 | "firstName", "Sansa",
41 | "lastName", "Stark",
42 | "gender", "female",
43 | "house", "House Stark"),
44 | XContentType.JSON))
45 | .add(
46 | new IndexRequest("users")
47 | .id("arya")
48 | .source(
49 | Map.of(
50 | "firstName", "Arya",
51 | "lastName", "Stark",
52 | "gender", "female",
53 | "house", "House Stark"),
54 | XContentType.JSON))
55 | .add(
56 | new IndexRequest("users")
57 | .id("tyrion")
58 | .source(
59 | Map.of(
60 | "firstName",
61 | "Tyrion",
62 | "lastName",
63 | "Lannister",
64 | "gender",
65 | "male",
66 | "house",
67 | "House Lannister"),
68 | XContentType.JSON))
69 | .add(
70 | new IndexRequest("users")
71 | .id("jaime")
72 | .source(
73 | Map.of(
74 | "firstName", "Jaime",
75 | "lastName", "Lannister",
76 | "gender", "male",
77 | "house", "House Lannister"),
78 | XContentType.JSON))
79 | .add(
80 | new IndexRequest("users")
81 | .id("cersei")
82 | .source(
83 | Map.of(
84 | "firstName", "Cersei",
85 | "lastName", "Lannister",
86 | "gender", "female",
87 | "house", "House Lannister"),
88 | XContentType.JSON))
89 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
90 | .execute()
91 | .actionGet();
92 |
93 | assertEquals(RestStatus.OK, response.status());
94 | for (BulkItemResponse r : response.getItems()) {
95 | assertEquals(RestStatus.CREATED, r.status());
96 | }
97 | }
98 |
99 | /**
100 | * Search API.
101 | *
102 | * @see Search
104 | * API | Java REST Client | Elastic
105 | */
106 | @Test
107 | public void searchApi_termQueryQuery() {
108 | SearchResponse response =
109 | client()
110 | .prepareSearch("users")
111 | .setQuery(QueryBuilders.termQuery("lastName", "stark"))
112 | .get();
113 |
114 | SearchHits hits = response.getHits();
115 | assertEquals(2L, hits.getTotalHits().value);
116 | assertEquals("sansa", hits.getHits()[0].getId());
117 | assertEquals("arya", hits.getHits()[1].getId());
118 | }
119 |
120 | @Test
121 | public void searchApi_allMatchQuery() {
122 | SearchResponse response =
123 | client() //
124 | .prepareSearch("users")
125 | .setQuery(QueryBuilders.matchAllQuery())
126 | .get();
127 |
128 | SearchHits hits = response.getHits();
129 | assertEquals(5L, hits.getTotalHits().value);
130 | }
131 |
132 | @Test
133 | public void searchApi_matchPhraseQuery() {
134 | SearchResponse response =
135 | client() //
136 | .prepareSearch("users")
137 | .setQuery(QueryBuilders.matchPhraseQuery("house", "House Lannister"))
138 | .get();
139 |
140 | SearchHits hits = response.getHits();
141 | assertEquals(3L, hits.getTotalHits().value);
142 | }
143 | }
144 |
--------------------------------------------------------------------------------
/snapshot/README.md:
--------------------------------------------------------------------------------
1 | # Snapshot and Restore
2 |
3 | Snapshot and restore in Elasticsearch.
4 |
5 | ## Prerequisite
6 |
7 | ```sh
8 | #
9 | # Start docker image:
10 | # - Use single-node discovery mode to bypass the bootstrap checks
11 | # - Use /tmp as the root path for the snapshot repositories
12 | # - Publish port 9200 to communicate with docker image
13 | #
14 | docker run \
15 | -e discovery.type=single-node \
16 | -e path.repo=/tmp \
17 | -p 9200:9200 \
18 | docker.elastic.co/elasticsearch/elasticsearch:7.6.2
19 | ```
20 |
21 | ## APIs
22 |
23 | API | Method | Description
24 | :--- | :---: | :---
25 | `/_snapshot/` | GET | List snapshot repositories.
26 | `/_snapshot/_status` | GET | Get all currently running snapshots with detailed status information.
27 | `/_snapshot/{repo}` | GET | Get settings of a snapshot repository.
28 | `/_snapshot/{repo}` | PUT | Add a new snapshot repository or edit the settings of an existing repository.
29 | `/_snapshot/{repo}/_all` | GET | List all snapshots inside the given repository.
30 | `/_snapshot/{repo}/_current` | GET | List all currently running snapshots inside the given repository.
31 | `/_snapshot/{repo}/_status` | GET | Get all currently running snapshots of the given repository with detailed status information.
32 | `/_snapshot/{repo}/{snapshot}` | GET | Get information about a single snapshot or multiple snapshots (using separator "," or wildcard expression "\*")
33 | `/_snapshot/{repo}/{snapshot}` | DELETE | Deletes one or multiple snapshots.
34 | `/_snapshot/{repo}/{snapshot}/_restore` | POST | Restore a snapshot of a cluster.
35 | `/_snapshot/{repo}/{snapshot}/_status` | GET | Get a detailed description of the current state for each shard partitipcating in the snapshot.
36 | `/_cat/snapshots/{repo}` | GET | List snapshots of a repository.
37 | `/_cat/recovery` | GET | List all the recoveries including snapshot recoveries, including restores
38 |
39 | ### Create Snapshot Repository
40 |
41 | Create a new snapshot repository `fs_backup` in local file-system for backup
42 | purpose:
43 |
44 | ```
45 | PUT /_snapshot/{repository}
46 | {
47 | "type": "fs",
48 | "settings": {
49 | "location": "my_backup_location"
50 | }
51 | }
52 | ```
53 |
54 | ```sh
55 | curl -X PUT localhost:9200/_snapshot/fs_backup \
56 | -H 'Content-Type: application/json' \
57 | -d '
58 | {
59 | "type": "fs",
60 | "settings": {
61 | "location": "/tmp"
62 | }
63 | }'
64 | # {"acknowledged":true}
65 | ```
66 |
67 | Other repository backends are available in these official plugins:
68 |
69 | - [repository-s3](https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository-s3.html)
70 | for S3 repository support
71 | - [repository-hdfs](https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository-hdfs.html)
72 | for HDFS repository support in Hadoop environments
73 | - [repository-azure](https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository-azure.html)
74 | for Azure storage repositories
75 | - [repository-gcs](https://www.elastic.co/guide/en/elasticsearch/plugins/7.7/repository-gcs.html)
76 | for Google Cloud Storage repositories
77 |
78 | ### Get Snapshot Repositories
79 |
80 | Retrieve information about all registered snapshot repositories
81 |
82 | ```
83 | GET /_snapshot
84 | ```
85 | ```
86 | GET /_snapshot/_all
87 | ```
88 | ```sh
89 | curl localhost:9200/_snapshot?pretty
90 | # {
91 | # "fs_backup" : {
92 | # "type" : "fs",
93 | # "settings" : {
94 | # "location" : "/tmp"
95 | # }
96 | # }
97 | # }
98 | ```
99 |
100 | ### Get Snapshot Repository
101 |
102 | Retrieve information about one snapshot repository.
103 |
104 | ```
105 | GET /_snapshot/{repository}
106 | ```
107 |
108 | ### Get Snapshots
109 |
110 | Retrieve information about all snapshots inside one snapshot repository.
111 |
112 | ```
113 | GET /_snapshot/{repository}/_all
114 | ```
115 |
116 | ## References
117 |
118 | - Elastic, "Snapshot and restore | Elasticsearch Reference \[7.6\]", _Elastic_, 2020.
119 |
120 |
--------------------------------------------------------------------------------
/snapshot/pom.xml:
--------------------------------------------------------------------------------
1 |
2 |
5 |
6 | es-demo-parent
7 | io.mincong
8 | 1.0-SNAPSHOT
9 |
10 | 4.0.0
11 |
12 | es-demo-snapshot
13 | Elasticsearch Demos - Snapshot
14 |
15 |
16 |
17 | org.elasticsearch.test
18 | framework
19 | test
20 |
21 |
22 | org.apache.logging.log4j
23 | log4j-core
24 | test
25 |
26 |
27 | org.assertj
28 | assertj-core
29 | test
30 |
31 |
32 |
33 |
--------------------------------------------------------------------------------
/snapshot/repository.md:
--------------------------------------------------------------------------------
1 | # Snapshot Repository
2 |
3 | Snapshot repository understanding in Elasticsearch.
4 |
5 | ## Prerequisite
6 |
7 | See
8 |
9 | ## Index
10 |
11 | There is one index "transactions" in the cluster:
12 |
13 | ```sh
14 | curl -s localhost:9200/_cat/indices
15 | ```
16 |
17 | ```
18 | yellow open transactions P59v6s2PS-GnaCrXgPv2Zg 1 1 10000 0 2.5mb 2.5mb
19 | ```
20 |
21 | According to the settings of this index, we can see that
22 | `P59v6s2PS-GnaCrXgPv2Zg` is the UUID of the index:
23 |
24 | ```sh
25 | curl -s localhost:9200/transactions/_settings | jq
26 | ```
27 |
28 | ```json
29 | {
30 | "transactions": {
31 | "settings": {
32 | "index": {
33 | "routing": {
34 | "allocation": {
35 | "include": {
36 | "_tier_preference": "data_content"
37 | }
38 | }
39 | },
40 | "number_of_shards": "1",
41 | "provided_name": "transactions",
42 | "creation_date": "1610896271859",
43 | "number_of_replicas": "1",
44 | "uuid": "P59v6s2PS-GnaCrXgPv2Zg",
45 | "version": {
46 | "created": "7100199"
47 | }
48 | }
49 | }
50 | }
51 | }
52 | ```
53 |
54 | ## Snapshots
55 |
56 | ```sh
57 | curl -s localhost:9200/_snapshot/dvf/_all | jq
58 | ```
59 |
60 | ```json
61 | {
62 | "snapshots": [
63 | {
64 | "snapshot": "transactions.2021-01-10",
65 | "uuid": "DsseRXnjTwC_tcV0-VgAww",
66 | "version_id": 7100199,
67 | "version": "7.10.1",
68 | "indices": [
69 | "transactions"
70 | ],
71 | "data_streams": [],
72 | "include_global_state": false,
73 | "state": "SUCCESS",
74 | "start_time": "2021-01-17T15:11:28.179Z",
75 | "start_time_in_millis": 1610896288179,
76 | "end_time": "2021-01-17T15:11:28.981Z",
77 | "end_time_in_millis": 1610896288981,
78 | "duration_in_millis": 802,
79 | "failures": [],
80 | "shards": {
81 | "total": 1,
82 | "failed": 0,
83 | "successful": 1
84 | }
85 | }
86 | ]
87 | }
88 | ```
89 |
90 | ## Repository
91 |
92 | _What is the structure of the snapshot repository?_
93 |
94 | By inspecting the directory of the repository "dvf", we can see the content as
95 | below:
96 |
97 | ```sh
98 | #
99 | # > pwd
100 | # /Users/minconghuang/es-backup/demo-dvf/dvf
101 | #
102 | tree .
103 | .
104 | ├── index-4
105 | ├── index.latest
106 | ├── indices
107 | │ └── toVoOSewT8eO7PbggT7SaA
108 | │ ├── 0
109 | │ │ ├── __7wvBjFfGSouvbPEOI53iMg
110 | │ │ ├── __AsxBUR80T3u6_HCXfAoUjg
111 | │ │ ├── __EjbTOvH4SDOx782QF8L3Ag
112 | │ │ ├── __I3G4iKJ8QvmG44-mX-qw_w
113 | │ │ ├── __KfAc01JpQHW2p9qMTMkeYA
114 | │ │ ├── __Lu_uYoC2RqSYF1Wv5GEchQ
115 | │ │ ├── __NOunPv39SU-IcZHUPjfFSw
116 | │ │ ├── __PhszAfkJRqyfCQme7HXTlw
117 | │ │ ├── __PwGnRH2uRj-E8o5h18l4jA
118 | │ │ ├── __T27uxGqgQoOj_0p86PDyuA
119 | │ │ ├── __YN69iWCtRqqzjFJ9Bgk68w
120 | │ │ ├── __Ywb0Yr67TH-S4tOHjnaoVQ
121 | │ │ ├── __a0QKSHNlT8W-HouU8hVDUw
122 | │ │ ├── __dXx6hK70Q8CuhvqtmOZx4Q
123 | │ │ ├── __fbcWE1PUSXuYUjzvq0hcqA
124 | │ │ ├── __ik7tosUWSSeHFq0Zli8vkA
125 | │ │ ├── index-qXJ7Ux1WSH6w8jELeKiwPA
126 | │ │ └── snap-DsseRXnjTwC_tcV0-VgAww.dat
127 | │ └── meta-l2TmEHcBch0uJWW8rNjp.dat
128 | ├── meta-DsseRXnjTwC_tcV0-VgAww.dat
129 | └── snap-DsseRXnjTwC_tcV0-VgAww.dat
130 |
131 | 3 directories, 23 files
132 | ```
133 |
134 | File `index-4` contains the names of all the snapshots in the repository.
135 |
136 | ```
137 | cat index-4 | jq
138 | ```
139 |
140 | ```js
141 | {
142 | "snapshots": [
143 | {
144 | "name": "transactions.2021-01-10",
145 | "uuid": "DsseRXnjTwC_tcV0-VgAww",
146 | "state": 1,
147 | "index_metadata_lookup": {
148 | /*
149 | * Index/metadata lookup table, where the key is the index ID and the
150 | * value is ???
151 | */
152 | "toVoOSewT8eO7PbggT7SaA": "umyNu_9iRE65F5RYKcS21A-_na_-1-3-1"
153 | },
154 | "version": "7.10.1"
155 | }
156 | ],
157 | "indices": {
158 | "transactions": {
159 | /*
160 | * This ID is for index "transactions" in the snapshot repository.
161 | *
162 | * Questions:
163 | * - Do we have the timestamps related to this index?
164 | */
165 | "id": "toVoOSewT8eO7PbggT7SaA",
166 | "snapshots": [
167 | "DsseRXnjTwC_tcV0-VgAww"
168 | ],
169 | "shard_generations": [
170 | "qXJ7Ux1WSH6w8jELeKiwPA"
171 | ]
172 | }
173 | },
174 | "min_version": "7.9.0",
175 | "index_metadata_identifiers": {
176 | /*
177 | * l2TmEHcBch0uJWW8rNjp is the ID of the metadata inside the snapshot
178 | * transactions.2021-01-10 (DsseRXnjTwC_tcV0-VgAww). The relative path
179 | * inside the repository dvf for this file is:
180 | *
181 | * indices/toVoOSewT8eO7PbggT7SaA/meta-l2TmEHcBch0uJWW8rNjp.dat
182 | *
183 | */
184 | "umyNu_9iRE65F5RYKcS21A-_na_-1-3-1": "l2TmEHcBch0uJWW8rNjp"
185 | }
186 | }
187 | ```
188 |
189 | ## Next Steps
190 |
191 | How to go further?
192 |
193 | - Find or implement a tool to translate SMILE JSON to normal JSON then inspect
194 | the JSON content. We can do that using Jackson "jackson-dataformat-smile", see
195 |
196 |
197 | ## References
198 |
199 | - Konrad Beiske, "Snapshot And Restore", Elasticsearch, 2014.
200 |
201 |
--------------------------------------------------------------------------------
/snapshot/src/test/java/io/mincong/elasticsearch/ConcurrentSnapshotDeletionTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.concurrent.ExecutionException;
4 | import org.assertj.core.api.Assertions;
5 | import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction;
6 | import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
7 | import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction;
8 | import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequestBuilder;
9 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
10 | import org.elasticsearch.cluster.metadata.RepositoryMetadata;
11 | import org.elasticsearch.common.settings.Settings;
12 | import org.elasticsearch.common.xcontent.XContentType;
13 | import org.elasticsearch.rest.RestStatus;
14 | import org.elasticsearch.test.ESSingleNodeTestCase;
15 | import org.junit.Before;
16 | import org.junit.Test;
17 |
18 | /**
19 | * Concurrent snapshot deletion in Elasticsearch.
20 | *
21 | * @author Mincong Huang
22 | */
23 | public class ConcurrentSnapshotDeletionTest extends ESSingleNodeTestCase {
24 |
25 | @Override
26 | @Before
27 | public void setUp() throws Exception {
28 | super.setUp();
29 | insertDocs();
30 | createRepo();
31 | createSnapshots();
32 | }
33 |
34 | private void insertDocs() {
35 | client()
36 | .prepareIndex()
37 | .setIndex("users")
38 | .setId("user1")
39 | .setSource("{\"name\":\"Tom\"}", XContentType.JSON)
40 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
41 | .execute();
42 |
43 | client()
44 | .prepareIndex()
45 | .setIndex("companies")
46 | .setId("elastic")
47 | .setSource("{\"name\":\"Elastic\"}", XContentType.JSON)
48 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
49 | .execute();
50 | }
51 |
52 | public void createRepo() {
53 | // create repository "snapshotRepository" before creating snapshot
54 | var repoStr = client().settings().get("path.repo");
55 | var settings =
56 | Settings.builder() //
57 | .put("location", repoStr)
58 | .put("compress", true)
59 | .build();
60 | var acknowledgedResponse =
61 | client()
62 | .admin()
63 | .cluster()
64 | .preparePutRepository("snapshotRepository")
65 | .setType("fs")
66 | .setSettings(settings)
67 | .setVerify(true)
68 | .get();
69 | Assertions.assertThat(acknowledgedResponse.isAcknowledged()).isTrue();
70 | var repositories =
71 | client()
72 | .admin()
73 | .cluster()
74 | .prepareGetRepositories("snapshotRepository")
75 | .get()
76 | .repositories();
77 | Assertions.assertThat(repositories)
78 | .extracting(RepositoryMetadata::name)
79 | .containsExactly("snapshotRepository");
80 | }
81 |
82 | private void createSnapshots() throws Exception {
83 | var requestU =
84 | new CreateSnapshotRequestBuilder(client(), CreateSnapshotAction.INSTANCE)
85 | .setIndices("users")
86 | .setSnapshot("users-snapshot")
87 | .setRepository("snapshotRepository")
88 | .setWaitForCompletion(true)
89 | .request();
90 |
91 | var requestC =
92 | new CreateSnapshotRequestBuilder(client(), CreateSnapshotAction.INSTANCE)
93 | .setIndices("companies")
94 | .setSnapshot("companies-snapshot")
95 | .setRepository("snapshotRepository")
96 | .setWaitForCompletion(true)
97 | .request();
98 |
99 | var responseU = client().admin().cluster().createSnapshot(requestU).get();
100 | var infoU = responseU.getSnapshotInfo();
101 | Assertions.assertThat(infoU.failedShards()).isZero();
102 | Assertions.assertThat(infoU.successfulShards()).isGreaterThan(0);
103 | Assertions.assertThat(infoU.indices()).containsExactly("users");
104 | Assertions.assertThat(infoU.status()).isEqualTo(RestStatus.OK);
105 |
106 | var responseC = client().admin().cluster().createSnapshot(requestC).get();
107 | var infoC = responseC.getSnapshotInfo();
108 | Assertions.assertThat(infoC.failedShards()).isZero();
109 | Assertions.assertThat(infoC.successfulShards()).isGreaterThan(0);
110 | Assertions.assertThat(infoC.indices()).containsExactly("companies");
111 | Assertions.assertThat(infoC.status()).isEqualTo(RestStatus.OK);
112 | }
113 |
114 | @Test
115 | public void removeSnapshot() throws ExecutionException, InterruptedException {
116 | var requestU =
117 | new DeleteSnapshotRequestBuilder(client(), DeleteSnapshotAction.INSTANCE)
118 | .setSnapshots("users-snapshot")
119 | .setRepository("snapshotRepository")
120 | .request();
121 | var requestC =
122 | new DeleteSnapshotRequestBuilder(client(), DeleteSnapshotAction.INSTANCE)
123 | .setSnapshots("companies-snapshot")
124 | .setRepository("snapshotRepository")
125 | .request();
126 |
127 | var responseU = client().admin().cluster().deleteSnapshot(requestU).get();
128 | var responseC = client().admin().cluster().deleteSnapshot(requestC).get();
129 | Assertions.assertThat(responseU.isAcknowledged()).isTrue();
130 | Assertions.assertThat(responseC.isAcknowledged()).isTrue();
131 | }
132 | }
133 |
--------------------------------------------------------------------------------
/snapshot/src/test/java/io/mincong/elasticsearch/SnapshotStateDemoTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import org.assertj.core.api.Assertions;
4 | import org.elasticsearch.action.admin.cluster.state.ClusterStateRequest;
5 | import org.elasticsearch.action.admin.cluster.state.ClusterStateResponse;
6 | import org.elasticsearch.cluster.RestoreInProgress;
7 | import org.elasticsearch.cluster.SnapshotDeletionsInProgress;
8 | import org.elasticsearch.cluster.SnapshotsInProgress;
9 | import org.elasticsearch.test.ESSingleNodeTestCase;
10 | import org.junit.Test;
11 |
12 | /**
13 | * Retrieves snapshot operations progress via cluster state: snapshots-in-progress,
14 | * restore-in-progress, snapshot-deletions-in-progress.
15 | *
16 | * These tests are not real tests, they are written for demo purpose.
17 | *
18 | * @author Mincong Huang
19 | */
20 | public class SnapshotStateDemoTest extends ESSingleNodeTestCase {
21 |
22 | @Test
23 | public void methodPrepareState() {
24 | ClusterStateResponse r =
25 | client()
26 | .admin() //
27 | .cluster()
28 | .prepareState()
29 | .clear()
30 | .setCustoms(true)
31 | .get();
32 |
33 | RestoreInProgress restore = r.getState().custom(RestoreInProgress.TYPE);
34 | SnapshotsInProgress snapshots = r.getState().custom(SnapshotsInProgress.TYPE);
35 | SnapshotDeletionsInProgress deletions = r.getState().custom(SnapshotDeletionsInProgress.TYPE);
36 |
37 | Assertions.assertThat(restore).isNull();
38 | Assertions.assertThat(snapshots).isNull();
39 | Assertions.assertThat(deletions).isNull();
40 | }
41 |
42 | @Test
43 | public void methodClusterStateRequest() {
44 | ClusterStateRequest request = new ClusterStateRequest().clear().customs(true);
45 | ClusterStateResponse r =
46 | client()
47 | .admin() //
48 | .cluster()
49 | .state(request)
50 | .actionGet();
51 |
52 | RestoreInProgress restore = r.getState().custom(RestoreInProgress.TYPE);
53 | SnapshotsInProgress snapshots = r.getState().custom(SnapshotsInProgress.TYPE);
54 | SnapshotDeletionsInProgress deletions = r.getState().custom(SnapshotDeletionsInProgress.TYPE);
55 |
56 | Assertions.assertThat(restore).isNull();
57 | Assertions.assertThat(snapshots).isNull();
58 | Assertions.assertThat(deletions).isNull();
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/snapshot/src/test/java/io/mincong/elasticsearch/SnapshotTest.java:
--------------------------------------------------------------------------------
1 | package io.mincong.elasticsearch;
2 |
3 | import java.util.concurrent.ExecutionException;
4 | import org.assertj.core.api.Assertions;
5 | import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotAction;
6 | import org.elasticsearch.action.admin.cluster.snapshots.create.CreateSnapshotRequestBuilder;
7 | import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotAction;
8 | import org.elasticsearch.action.admin.cluster.snapshots.delete.DeleteSnapshotRequestBuilder;
9 | import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotAction;
10 | import org.elasticsearch.action.admin.cluster.snapshots.restore.RestoreSnapshotRequestBuilder;
11 | import org.elasticsearch.action.admin.indices.close.CloseIndexResponse.IndexResult;
12 | import org.elasticsearch.action.support.WriteRequest.RefreshPolicy;
13 | import org.elasticsearch.cluster.metadata.RepositoryMetadata;
14 | import org.elasticsearch.common.settings.Settings;
15 | import org.elasticsearch.common.xcontent.XContentType;
16 | import org.elasticsearch.index.Index;
17 | import org.elasticsearch.rest.RestStatus;
18 | import org.elasticsearch.test.ESSingleNodeTestCase;
19 | import org.junit.After;
20 | import org.junit.Before;
21 | import org.junit.Test;
22 |
23 | /**
24 | * Snapshot and restore in Elasticsearch.
25 | *
26 | * @author Mincong Huang
27 | */
28 | public class SnapshotTest extends ESSingleNodeTestCase {
29 |
30 | @Override
31 | @Before
32 | public void setUp() throws Exception {
33 | super.setUp();
34 |
35 | client()
36 | .prepareIndex()
37 | .setIndex("users")
38 | .setId("user1")
39 | .setSource("{\"name\":\"Tom\"}", XContentType.JSON)
40 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
41 | .execute();
42 |
43 | client()
44 | .prepareIndex()
45 | .setIndex("companies")
46 | .setId("elastic")
47 | .setSource("{\"name\":\"Elastic\"}", XContentType.JSON)
48 | .setRefreshPolicy(RefreshPolicy.IMMEDIATE)
49 | .execute();
50 | }
51 |
52 | @Override
53 | @After
54 | public void tearDown() throws Exception {
55 | removeSnapshot();
56 | super.tearDown();
57 | }
58 |
59 | @Test
60 | public void createSnapshot() throws Exception {
61 | // create repository "snapshotRepository" before creating snapshot
62 | var repoStr = client().settings().get("path.repo");
63 | var settings =
64 | Settings.builder() //
65 | .put("location", repoStr)
66 | .put("compress", true)
67 | .build();
68 | var acknowledgedResponse =
69 | client()
70 | .admin()
71 | .cluster()
72 | .preparePutRepository("snapshotRepository")
73 | .setType("fs")
74 | .setSettings(settings)
75 | .setVerify(true)
76 | .get();
77 | Assertions.assertThat(acknowledgedResponse.isAcknowledged()).isTrue();
78 | var repositories =
79 | client()
80 | .admin()
81 | .cluster()
82 | .prepareGetRepositories("snapshotRepository")
83 | .get()
84 | .repositories();
85 | Assertions.assertThat(repositories)
86 | .extracting(RepositoryMetadata::name)
87 | .containsExactly("snapshotRepository");
88 |
89 | // create snapshot using "snapshotRepository"
90 | var createSnapshotRequest =
91 | new CreateSnapshotRequestBuilder(client(), CreateSnapshotAction.INSTANCE)
92 | .setIndices("users", "companies")
93 | .setSnapshot("snapshot1")
94 | .setRepository("snapshotRepository")
95 | .setWaitForCompletion(true)
96 | .request();
97 |
98 | var createSnapshotResponse =
99 | client().admin().cluster().createSnapshot(createSnapshotRequest).get();
100 | var snapshotInfo = createSnapshotResponse.getSnapshotInfo();
101 | Assertions.assertThat(snapshotInfo.failedShards()).isZero();
102 | Assertions.assertThat(snapshotInfo.successfulShards()).isGreaterThan(0);
103 | Assertions.assertThat(snapshotInfo.indices()).containsExactlyInAnyOrder("users", "companies");
104 | Assertions.assertThat(snapshotInfo.status()).isEqualTo(RestStatus.OK);
105 | }
106 |
107 | @Test
108 | public void restoreSnapshot() throws Exception {
109 | createSnapshot();
110 |
111 | /*
112 | * The restore operation can be performed on a functioning cluster.
113 | * However, an existing index can be only restored if it’s closed
114 | * and has the same number of shards as the index in the snapshot.
115 | */
116 | var closeIndexResponse = client().admin().indices().prepareClose("users", "companies").get();
117 | Assertions.assertThat(closeIndexResponse.getIndices())
118 | .extracting(IndexResult::getIndex)
119 | .extracting(Index::getName)
120 | .containsExactlyInAnyOrder("users", "companies");
121 |
122 | // restore snapshot using "snapshotRepository"
123 | var restoreSnapshotRequest =
124 | new RestoreSnapshotRequestBuilder(client(), RestoreSnapshotAction.INSTANCE)
125 | .setIndices("users", "companies")
126 | .setSnapshot("snapshot1")
127 | .setRepository("snapshotRepository")
128 | .setWaitForCompletion(true)
129 | .request();
130 |
131 | var restoreSnapshotResponse =
132 | client().admin().cluster().restoreSnapshot(restoreSnapshotRequest).get();
133 | var restoreInfo = restoreSnapshotResponse.getRestoreInfo();
134 | Assertions.assertThat(restoreInfo.failedShards()).isZero();
135 | Assertions.assertThat(restoreInfo.successfulShards()).isGreaterThan(0);
136 | Assertions.assertThat(restoreInfo.indices()).containsExactlyInAnyOrder("users", "companies");
137 | Assertions.assertThat(restoreInfo.status()).isEqualTo(RestStatus.OK);
138 | }
139 |
140 | private void removeSnapshot() throws ExecutionException, InterruptedException {
141 | var deleteSnapshot =
142 | new DeleteSnapshotRequestBuilder(client(), DeleteSnapshotAction.INSTANCE)
143 | .setSnapshots("snapshot1")
144 | .setRepository("snapshotRepository")
145 | .request();
146 |
147 | var acknowledgeResponse = client().admin().cluster().deleteSnapshot(deleteSnapshot).get();
148 | Assertions.assertThat(acknowledgeResponse.isAcknowledged()).isTrue();
149 | }
150 | }
151 |
--------------------------------------------------------------------------------