(numResponses);
54 | for (int i = 0; i < numResponses; i++) {
55 | responses.add(ShardExportResponse.readNew(in));
56 | }
57 | }
58 |
59 | @Override
60 | public void writeTo(StreamOutput out) throws IOException {
61 | super.writeTo(out);
62 | out.writeVLong(totalExported);
63 | out.writeVInt(responses.size());
64 | for (ShardExportResponse response : responses) {
65 | response.writeTo(out);
66 | }
67 | }
68 |
69 | @Override
70 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
71 | builder.startObject();
72 | builder.startArray("exports");
73 | for (ShardExportResponse r : this.responses) {
74 | r.toXContent(builder, params);
75 | }
76 | builder.endArray();
77 | builder.field("totalExported", totalExported);
78 | buildBroadcastShardsHeader(builder, this);
79 | builder.endObject();
80 | return builder;
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/ShardExportRequest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export;
2 |
3 | import org.elasticsearch.action.support.broadcast.BroadcastShardOperationRequest;
4 | import org.elasticsearch.common.Nullable;
5 | import org.elasticsearch.common.Strings;
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.io.stream.StreamInput;
8 | import org.elasticsearch.common.io.stream.StreamOutput;
9 |
10 | import java.io.IOException;
11 |
12 |
13 | /**
14 | * Internal export request executed directly against a specific index shard.
15 | */
16 | class ShardExportRequest extends BroadcastShardOperationRequest {
17 |
18 | private BytesReference source;
19 |
20 | private String[] types = Strings.EMPTY_ARRAY;
21 |
22 | @Nullable
23 | private String[] filteringAliases;
24 |
25 | ShardExportRequest() {
26 |
27 | }
28 |
29 | public ShardExportRequest(String index, int shardId, @Nullable String[] filteringAliases, ExportRequest request) {
30 | super(index, shardId, request);
31 | this.source = request.source();
32 | this.types = request.types();
33 | this.filteringAliases = filteringAliases;
34 | }
35 |
36 | public BytesReference source() {
37 | return source;
38 | }
39 |
40 | public String[] types() {
41 | return this.types;
42 | }
43 |
44 | public String[] filteringAliases() {
45 | return filteringAliases;
46 | }
47 |
48 | @Override
49 | public void readFrom(StreamInput in) throws IOException {
50 | super.readFrom(in);
51 | source = in.readBytesReference();
52 | int typesSize = in.readVInt();
53 | if (typesSize > 0) {
54 | types = new String[typesSize];
55 | for (int i = 0; i < typesSize; i++) {
56 | types[i] = in.readString();
57 | }
58 | }
59 | int aliasesSize = in.readVInt();
60 | if (aliasesSize > 0) {
61 | filteringAliases = new String[aliasesSize];
62 | for (int i = 0; i < aliasesSize; i++) {
63 | filteringAliases[i] = in.readString();
64 | }
65 | }
66 | }
67 |
68 | @Override
69 | public void writeTo(StreamOutput out) throws IOException {
70 | super.writeTo(out);
71 | out.writeBytesReference(source);
72 |
73 | out.writeVInt(types.length);
74 | for (String type : types) {
75 | out.writeString(type);
76 | }
77 | if (filteringAliases != null) {
78 | out.writeVInt(filteringAliases.length);
79 | for (String alias : filteringAliases) {
80 | out.writeString(alias);
81 | }
82 | } else {
83 | out.writeVInt(0);
84 | }
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/TransportExportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export;
2 |
3 | import crate.elasticsearch.action.export.parser.ExportParser;
4 | import crate.elasticsearch.export.Exporter;
5 | import org.elasticsearch.cache.recycler.CacheRecycler;
6 | import org.elasticsearch.cluster.ClusterService;
7 | import org.elasticsearch.common.inject.Inject;
8 | import org.elasticsearch.common.settings.Settings;
9 | import org.elasticsearch.env.NodeEnvironment;
10 | import org.elasticsearch.indices.IndicesService;
11 | import org.elasticsearch.script.ScriptService;
12 | import org.elasticsearch.threadpool.ThreadPool;
13 | import org.elasticsearch.transport.TransportService;
14 |
15 |
16 | /**
17 | *
18 | */
19 | public class TransportExportAction extends AbstractTransportExportAction {
20 |
21 | @Inject
22 | public TransportExportAction(Settings settings, ThreadPool threadPool, ClusterService clusterService,
23 | TransportService transportService, IndicesService indicesService,
24 | ScriptService scriptService, CacheRecycler cacheRecycler,
25 | ExportParser exportParser, Exporter exporter, NodeEnvironment nodeEnv) {
26 | super(settings, threadPool, clusterService, transportService, indicesService, scriptService,
27 | cacheRecycler, exportParser, exporter, nodeEnv);
28 | }
29 |
30 | @Override
31 | protected String transportAction() {
32 | return ExportAction.NAME;
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportCompressionParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 | import org.elasticsearch.search.SearchParseElement;
5 | import org.elasticsearch.search.SearchParseException;
6 | import org.elasticsearch.search.internal.SearchContext;
7 |
8 | import crate.elasticsearch.action.export.ExportContext;
9 |
10 | public class ExportCompressionParseElement implements SearchParseElement {
11 |
12 | @Override
13 | public void parse(XContentParser parser, SearchContext context)
14 | throws Exception {
15 | XContentParser.Token token = parser.currentToken();
16 | if (token.isValue()) {
17 | String lower = parser.text().toLowerCase();
18 | if (lower.equals("gzip")) {
19 | ((ExportContext) context).compression(true);
20 | } else if (!lower.isEmpty()) {
21 | throw new SearchParseException(context,
22 | "Compression format '" + lower + "' unknown or not supported.");
23 | }
24 | }
25 | }
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportForceOverwriteParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import crate.elasticsearch.action.export.ExportContext;
4 | import org.elasticsearch.common.xcontent.XContentParser;
5 | import org.elasticsearch.search.SearchParseElement;
6 | import org.elasticsearch.search.internal.SearchContext;
7 |
8 | /**
9 | * Parser for token ``force_overwrite``. Make sense if output_file was defined.
10 | */
11 | public class ExportForceOverwriteParseElement implements SearchParseElement {
12 |
13 | @Override
14 | public void parse(XContentParser parser, SearchContext context) throws Exception {
15 | XContentParser.Token token = parser.currentToken();
16 | if (token.isValue()) {
17 | ((ExportContext)context).forceOverride(parser.booleanValue());
18 | }
19 | }
20 | }
21 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportMappingsParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 | import org.elasticsearch.search.SearchParseElement;
5 | import org.elasticsearch.search.internal.SearchContext;
6 |
7 | import crate.elasticsearch.action.export.ExportContext;
8 |
9 | /**
10 | * Parser for token ``mappings``. Makes sense if output_file was defined.
11 | */
12 | public class ExportMappingsParseElement implements SearchParseElement {
13 |
14 | @Override
15 | public void parse(XContentParser parser, SearchContext context)
16 | throws Exception {
17 | XContentParser.Token token = parser.currentToken();
18 | if (token.isValue()) {
19 | ((ExportContext)context).mappings(parser.booleanValue());
20 | }
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportOutputCmdParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import crate.elasticsearch.action.export.ExportContext;
4 | import org.elasticsearch.common.xcontent.XContentParser;
5 | import org.elasticsearch.search.SearchParseElement;
6 | import org.elasticsearch.search.internal.SearchContext;
7 |
8 | import java.util.ArrayList;
9 | import java.util.List;
10 |
11 | /**
12 | * Parser for token ``output_cmd``. The value of the token might be a String
13 | * containing the command or an array containing the command and all
14 | * arguments as seperated parts.
15 | *
16 | *
17 | * "output_cmd": "gzip > /tmp/out"
18 | *
19 | * or
20 | *
21 | * "output_cmd": ["gzip", ">" "/tmp/out"]
22 | *
23 | */
24 | public class ExportOutputCmdParseElement implements SearchParseElement {
25 |
26 | @Override
27 | public void parse(XContentParser parser, SearchContext context) throws Exception {
28 | XContentParser.Token token = parser.currentToken();
29 | if (token.isValue()) {
30 | ((ExportContext) context).outputCmd(parser.text());
31 | } else if (token == XContentParser.Token.START_ARRAY) {
32 | List cmds = new ArrayList(4);
33 | while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
34 | cmds.add(parser.text());
35 | }
36 | ((ExportContext) context).outputCmdArray(cmds);
37 | }
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportOutputFileParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import crate.elasticsearch.action.export.ExportContext;
4 | import org.elasticsearch.common.xcontent.XContentParser;
5 | import org.elasticsearch.search.SearchParseElement;
6 | import org.elasticsearch.search.internal.SearchContext;
7 |
8 | /**
9 | * Parser for token ``output_file``. The value of the token must be a String.
10 | *
11 | *
12 | * "output_file": "/tmp/out"
13 | *
14 | */
15 | public class ExportOutputFileParseElement implements SearchParseElement {
16 |
17 | @Override
18 | public void parse(XContentParser parser, SearchContext context) throws Exception {
19 | XContentParser.Token token = parser.currentToken();
20 | if (token.isValue()) {
21 | ((ExportContext) context).outputFile(parser.text());
22 | }
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.collect.ImmutableMap;
8 | import org.elasticsearch.common.inject.Inject;
9 | import org.elasticsearch.common.xcontent.XContentFactory;
10 | import org.elasticsearch.common.xcontent.XContentHelper;
11 | import org.elasticsearch.common.xcontent.XContentParser;
12 | import org.elasticsearch.search.SearchParseElement;
13 | import org.elasticsearch.search.SearchParseException;
14 | import org.elasticsearch.search.fetch.FetchPhase;
15 | import org.elasticsearch.search.fetch.FieldsParseElement;
16 | import org.elasticsearch.search.fetch.explain.ExplainParseElement;
17 | import org.elasticsearch.search.query.QueryPhase;
18 |
19 | import crate.elasticsearch.action.export.ExportContext;
20 |
21 | /**
22 | * Parser for payload given to _export action.
23 | */
24 | public class ExportParser implements IExportParser {
25 |
26 | private final ImmutableMap elementParsers;
27 |
28 | @Inject
29 | public ExportParser(QueryPhase queryPhase, FetchPhase fetchPhase) {
30 | Map elementParsers = new HashMap();
31 | elementParsers.putAll(queryPhase.parseElements());
32 | elementParsers.put("fields", new FieldsParseElement());
33 | elementParsers.put("output_cmd", new ExportOutputCmdParseElement());
34 | elementParsers.put("output_file", new ExportOutputFileParseElement());
35 | elementParsers.put("force_overwrite", new ExportForceOverwriteParseElement());
36 | elementParsers.put("compression", new ExportCompressionParseElement());
37 | elementParsers.put("explain", new ExplainParseElement());
38 | elementParsers.put("mappings", new ExportMappingsParseElement());
39 | elementParsers.put("settings", new ExportSettingsParseElement());
40 | this.elementParsers = ImmutableMap.copyOf(elementParsers);
41 | }
42 |
43 | /**
44 | * validate given payload
45 | *
46 | * @param context
47 | */
48 | private void validate(ExportContext context) {
49 | if (!context.hasFieldNames()) {
50 | throw new SearchParseException(context, "No export fields defined");
51 | }
52 | for (String field : context.fieldNames()) {
53 | if (context.mapperService().name(field) == null && !field.equals("_version")) {
54 | throw new SearchParseException(context, "Export field [" + field + "] does not exist in the mapping");
55 | }
56 | }
57 | if (context.outputFile() != null) {
58 | if (context.outputCmdArray() != null || context.outputCmd() != null) {
59 | throw new SearchParseException(context, "Concurrent definition of 'output_cmd' and 'output_file'");
60 | }
61 | } else if (context.outputCmdArray() == null && context.outputCmd() == null) {
62 | throw new SearchParseException(context, "'output_cmd' or 'output_file' has not been defined");
63 | } else if (context.outputFile() == null && context.settings()) {
64 | throw new SearchParseException(context, "Parameter 'settings' requires usage of 'output_file'");
65 | } else if (context.outputFile() == null && context.mappings()) {
66 | throw new SearchParseException(context, "Parameter 'mappings' requires usage of 'output_file'");
67 | }
68 | }
69 |
70 | /**
71 | * Main method of this class to parse given payload of _export action
72 | *
73 | * @param context
74 | * @param source
75 | * @throws SearchParseException
76 | */
77 | public void parseSource(ExportContext context, BytesReference source) throws SearchParseException {
78 | XContentParser parser = null;
79 | try {
80 | if (source != null && source.length() != 0) {
81 | parser = XContentFactory.xContent(source).createParser(source);
82 | XContentParser.Token token;
83 | while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
84 | if (token == XContentParser.Token.FIELD_NAME) {
85 | String fieldName = parser.currentName();
86 | parser.nextToken();
87 | SearchParseElement element = elementParsers.get(fieldName);
88 | if (element == null) {
89 | throw new SearchParseException(context, "No parser for element [" + fieldName + "]");
90 | }
91 | element.parse(parser, context);
92 | } else if (token == null) {
93 | break;
94 | }
95 | }
96 | }
97 | validate(context);
98 | } catch (Exception e) {
99 | String sSource = "_na_";
100 | try {
101 | sSource = XContentHelper.convertToJson(source, false);
102 | } catch (Throwable e1) {
103 | // ignore
104 | }
105 | throw new SearchParseException(context, "Failed to parse source [" + sSource + "]", e);
106 | } finally {
107 | if (parser != null) {
108 | parser.close();
109 | }
110 | }
111 | }
112 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/ExportSettingsParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 | import org.elasticsearch.search.SearchParseElement;
5 | import org.elasticsearch.search.internal.SearchContext;
6 |
7 | import crate.elasticsearch.action.export.ExportContext;
8 |
9 | /**
10 | * Parser for token ``settings``. Makes sense if output_file was defined.
11 | */
12 | public class ExportSettingsParseElement implements SearchParseElement {
13 |
14 | @Override
15 | public void parse(XContentParser parser, SearchContext context)
16 | throws Exception {
17 | XContentParser.Token token = parser.currentToken();
18 | if (token.isValue()) {
19 | ((ExportContext)context).settings(parser.booleanValue());
20 | }
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/export/parser/IExportParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.export.parser;
2 |
3 | import crate.elasticsearch.action.export.ExportContext;
4 | import org.elasticsearch.common.bytes.BytesReference;
5 | import org.elasticsearch.search.SearchParseException;
6 |
7 | /**
8 | * Interface for source parsers
9 | */
10 | public interface IExportParser {
11 |
12 | public void parseSource(ExportContext context, BytesReference source) throws SearchParseException;
13 | }
14 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/AbstractTransportImportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import crate.elasticsearch.action.import_.parser.IImportParser;
4 | import crate.elasticsearch.import_.Importer;
5 | import org.elasticsearch.ElasticSearchException;
6 | import org.elasticsearch.action.FailedNodeException;
7 | import org.elasticsearch.action.support.nodes.TransportNodesOperationAction;
8 | import org.elasticsearch.cluster.ClusterName;
9 | import org.elasticsearch.cluster.ClusterService;
10 | import org.elasticsearch.common.bytes.BytesReference;
11 | import org.elasticsearch.common.inject.Inject;
12 | import org.elasticsearch.common.settings.Settings;
13 | import org.elasticsearch.env.NodeEnvironment;
14 | import org.elasticsearch.threadpool.ThreadPool;
15 | import org.elasticsearch.transport.TransportService;
16 |
17 | import java.io.File;
18 | import java.util.ArrayList;
19 | import java.util.List;
20 | import java.util.concurrent.atomic.AtomicReferenceArray;
21 |
22 | import static org.elasticsearch.common.collect.Lists.newArrayList;
23 |
24 | public abstract class AbstractTransportImportAction extends TransportNodesOperationAction{
25 |
26 | private IImportParser importParser;
27 |
28 | private Importer importer;
29 |
30 | private String nodePath = "";
31 |
32 | @Inject
33 | public AbstractTransportImportAction(Settings settings, ClusterName clusterName,
34 | ThreadPool threadPool, ClusterService clusterService,
35 | TransportService transportService, IImportParser importParser, Importer importer, NodeEnvironment nodeEnv) {
36 | super(settings, clusterName, threadPool, clusterService, transportService);
37 | this.importParser = importParser;
38 | this.importer = importer;
39 |
40 | File[] paths = nodeEnv.nodeDataLocations();
41 | if (paths.length > 0) {
42 | nodePath = paths[0].getAbsolutePath();
43 | }
44 | }
45 |
46 | @Override
47 | protected String executor() {
48 | return ThreadPool.Names.INDEX;
49 | }
50 |
51 | @Override
52 | protected ImportRequest newRequest() {
53 | return new ImportRequest();
54 | }
55 |
56 | @Override
57 | protected ImportResponse newResponse(ImportRequest request,
58 | AtomicReferenceArray nodesResponses) {
59 | int total = nodesResponses.length();
60 | int successfulNodes = 0;
61 | int failedNodes = 0;
62 | List nodeFailures = null;
63 | List responses = new ArrayList();
64 | for (int i=0; i < total; i++) {
65 | Object nodeResponse = nodesResponses.get(i);
66 | if (nodeResponse == null) {
67 | failedNodes++;
68 | } else if (nodeResponse instanceof FailedNodeException) {
69 | failedNodes++;
70 | if (nodeFailures == null) {
71 | nodeFailures = newArrayList();
72 | }
73 | nodeFailures.add((FailedNodeException) nodeResponse);
74 | } else if (nodeResponse instanceof Exception) {
75 | ((Exception) nodeResponse).getMessage();
76 | } else {
77 | responses.add((NodeImportResponse) nodeResponse);
78 | successfulNodes++;
79 | }
80 | }
81 | return new ImportResponse(responses, total, successfulNodes, failedNodes, nodeFailures);
82 | }
83 |
84 | /**
85 | * This method is called on non primary nodes
86 | */
87 | @Override
88 | protected NodeImportRequest newNodeRequest() {
89 | return new NodeImportRequest();
90 | }
91 |
92 | /**
93 | * This method is called on primary node for every node
94 | */
95 | @Override
96 | protected NodeImportRequest newNodeRequest(String nodeId,
97 | ImportRequest request) {
98 | return new NodeImportRequest(nodeId, request);
99 | }
100 |
101 | /**
102 | * This method is called on primary node for non-primary nodes
103 | */
104 | @Override
105 | protected NodeImportResponse newNodeResponse() {
106 | return new NodeImportResponse();
107 | }
108 |
109 | @Override
110 | protected NodeImportResponse nodeOperation(NodeImportRequest request)
111 | throws ElasticSearchException {
112 | ImportContext context = new ImportContext(nodePath);
113 |
114 | BytesReference source = request.source();
115 | importParser.parseSource(context, source);
116 | Importer.Result result = importer.execute(context, request);
117 | return new NodeImportResponse(clusterService.state().nodes().localNode(), result);
118 | }
119 |
120 | @Override
121 | protected boolean accumulateExceptions() {
122 | return true;
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/ImportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import crate.elasticsearch.client.action.import_.ImportRequestBuilder;
4 | import org.elasticsearch.action.Action;
5 | import org.elasticsearch.client.Client;
6 |
7 | public class ImportAction extends Action {
8 |
9 | public static final ImportAction INSTANCE = new ImportAction();
10 | public static final String NAME = "el-crate-import";
11 |
12 | private ImportAction() {
13 | super(NAME);
14 | }
15 |
16 | @Override
17 | public ImportResponse newResponse() {
18 | return new ImportResponse();
19 | }
20 |
21 | @Override
22 | public ImportRequestBuilder newRequestBuilder(Client client) {
23 | return new ImportRequestBuilder(client);
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/ImportContext.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import java.io.File;
4 | import java.util.regex.Pattern;
5 |
6 | public class ImportContext {
7 |
8 | private String nodePath;
9 | private boolean compression;
10 | private String directory;
11 | private Pattern file_pattern;
12 | private boolean mappings = false;
13 | private boolean settings = false;
14 |
15 | public ImportContext(String nodePath) {
16 | this.nodePath = nodePath;
17 | }
18 |
19 | public boolean compression() {
20 | return compression;
21 | }
22 |
23 | public void compression(boolean compression) {
24 | this.compression = compression;
25 | }
26 |
27 | public String directory() {
28 | return directory;
29 | }
30 |
31 | public void directory(String directory) {
32 | File file = new File(directory);
33 | if (!file.isAbsolute() && nodePath != null) {
34 | file = new File(nodePath, directory);
35 | directory = file.getAbsolutePath();
36 | }
37 | this.directory = directory;
38 | }
39 |
40 | public Pattern file_pattern() {
41 | return file_pattern;
42 | }
43 |
44 | public void file_pattern(Pattern file_pattern) {
45 | this.file_pattern = file_pattern;
46 | }
47 |
48 | public boolean mappings() {
49 | return mappings;
50 | }
51 |
52 | public void mappings(boolean mappings) {
53 | this.mappings = mappings;
54 | }
55 |
56 | public boolean settings() {
57 | return settings;
58 | }
59 |
60 | public void settings(boolean settings) {
61 | this.settings = settings;
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/ImportRequest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import org.elasticsearch.action.support.nodes.NodesOperationRequest;
4 | import org.elasticsearch.common.Required;
5 | import org.elasticsearch.common.bytes.BytesArray;
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.io.stream.StreamInput;
8 | import org.elasticsearch.common.io.stream.StreamOutput;
9 |
10 | import java.io.IOException;
11 |
12 | public class ImportRequest extends NodesOperationRequest {
13 |
14 | private BytesReference source;
15 |
16 | private String type;
17 | private String index;
18 |
19 | /**
20 | * Constructs a new import request against the provided nodes. No nodes provided
21 | * means it will run against all nodes.
22 | */
23 | public ImportRequest(String... nodes) {
24 | super(nodes);
25 | }
26 |
27 | /**
28 | * The query source to execute.
29 | * @return
30 | */
31 | public BytesReference source() {
32 | return source;
33 | }
34 |
35 | @Required
36 | public ImportRequest source(String source) {
37 | return this.source(new BytesArray(source), false);
38 | }
39 |
40 | @Required
41 | public ImportRequest source(BytesReference source, boolean unsafe) {
42 | this.source = source;
43 | return this;
44 | }
45 |
46 | public String type() {
47 | return this.type;
48 | }
49 |
50 | public void type(String type) {
51 | this.type = type;
52 | }
53 |
54 | public String index() {
55 | return this.index ;
56 | }
57 |
58 | public void index(String index) {
59 | this.index = index;
60 | }
61 |
62 | @Override
63 | public void readFrom(StreamInput in) throws IOException {
64 | super.readFrom(in);
65 | index = in.readOptionalString();
66 | type = in.readOptionalString();
67 | source = in.readBytesReference();
68 | }
69 |
70 | @Override
71 | public void writeTo(StreamOutput out) throws IOException {
72 | super.writeTo(out);
73 | out.writeOptionalString(index);
74 | out.writeOptionalString(type);
75 | out.writeBytesReference(source);
76 | }
77 |
78 | }
79 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/ImportResponse.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import java.io.IOException;
4 | import java.util.ArrayList;
5 | import java.util.List;
6 |
7 | import org.elasticsearch.action.FailedNodeException;
8 | import org.elasticsearch.action.support.nodes.NodesOperationResponse;
9 | import org.elasticsearch.common.io.stream.StreamInput;
10 | import org.elasticsearch.common.io.stream.StreamOutput;
11 | import org.elasticsearch.common.xcontent.ToXContent;
12 | import org.elasticsearch.common.xcontent.XContentBuilder;
13 |
14 | public class ImportResponse extends NodesOperationResponse implements ToXContent {
15 |
16 | private List responses;
17 | private List nodeFailures;
18 |
19 | public ImportResponse() {
20 | }
21 |
22 | public ImportResponse(List responses, int total,
23 | int successfulNodes, int failedNodes, List nodeFailures) {
24 | this.responses = responses;
25 | this.nodeFailures = nodeFailures;
26 | }
27 |
28 | public List getResponses() {
29 | return responses;
30 | }
31 |
32 | @Override
33 | public XContentBuilder toXContent(XContentBuilder builder, Params params)
34 | throws IOException {
35 | builder.startObject();
36 | builder.startArray("imports");
37 | for (NodeImportResponse r : this.responses) {
38 | r.toXContent(builder, params);
39 | }
40 | builder.endArray();
41 | if (nodeFailures != null && nodeFailures.size() > 0) {
42 | builder.startArray("failures");
43 | for (FailedNodeException failure : nodeFailures) {
44 | builder.startObject();
45 | builder.field("node_id", failure.nodeId());
46 | builder.field("reason", failure.getDetailedMessage());
47 | builder.endObject();
48 | }
49 | builder.endArray();
50 | }
51 | builder.endObject();
52 | return builder;
53 | }
54 |
55 | @Override
56 | public void readFrom(StreamInput in) throws IOException {
57 | super.readFrom(in);
58 | int responsesCount = in.readInt();
59 | this.responses = new ArrayList(responsesCount);
60 | for (int i = 0; i < responsesCount; i++) {
61 | responses.add(NodeImportResponse.readNew(in));
62 | }
63 | int failuresCount = in.readInt();
64 | this.nodeFailures = new ArrayList(failuresCount);
65 | for (int i = 0; i < failuresCount; i++) {
66 | String nodeId = in.readString();
67 | String msg = in.readOptionalString();
68 | FailedNodeException e = new FailedNodeException(nodeId, msg, null);
69 | nodeFailures.add(e);
70 | }
71 | }
72 |
73 | @Override
74 | public void writeTo(StreamOutput out) throws IOException {
75 | super.writeTo(out);
76 | out.writeInt(responses.size());
77 | for (NodeImportResponse response : responses) {
78 | response.writeTo(out);
79 | }
80 | out.writeInt(nodeFailures.size());
81 | for (FailedNodeException e : nodeFailures) {
82 | out.writeString(e.nodeId());
83 | out.writeOptionalString(e.getMessage());
84 | }
85 | }
86 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/NodeImportRequest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import java.io.IOException;
4 |
5 | import org.elasticsearch.action.support.nodes.NodeOperationRequest;
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.io.stream.StreamInput;
8 | import org.elasticsearch.common.io.stream.StreamOutput;
9 |
10 | public class NodeImportRequest extends NodeOperationRequest {
11 |
12 | public static final int DEFAULT_BULK_SIZE = 10000;
13 |
14 | private BytesReference source;
15 | private String index;
16 | private String type;
17 |
18 | NodeImportRequest() {
19 | }
20 |
21 | public NodeImportRequest(String nodeId, ImportRequest request) {
22 | super(request, nodeId);
23 | this.source = request.source();
24 | this.index = request.index();
25 | this.type = request.type();
26 | }
27 |
28 | @Override
29 | public void readFrom(StreamInput in) throws IOException {
30 | super.readFrom(in);
31 | source = in.readBytesReference();
32 | index = in.readOptionalString();
33 | type = in.readOptionalString();
34 | }
35 |
36 | @Override
37 | public void writeTo(StreamOutput out) throws IOException {
38 | super.writeTo(out);
39 | out.writeBytesReference(source);
40 | out.writeOptionalString(index);
41 | out.writeOptionalString(type);
42 | }
43 |
44 | public BytesReference source() {
45 | return source;
46 | }
47 |
48 | public String index() {
49 | return index;
50 | }
51 |
52 | public String type() {
53 | return type;
54 | }
55 |
56 | public int bulkSize() {
57 | return DEFAULT_BULK_SIZE;
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/NodeImportResponse.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import java.io.IOException;
4 |
5 | import org.elasticsearch.action.support.nodes.NodeOperationResponse;
6 | import org.elasticsearch.cluster.node.DiscoveryNode;
7 | import org.elasticsearch.common.io.stream.StreamInput;
8 | import org.elasticsearch.common.io.stream.StreamOutput;
9 | import org.elasticsearch.common.xcontent.ToXContent;
10 | import org.elasticsearch.common.xcontent.XContentBuilder;
11 | import org.elasticsearch.common.xcontent.XContentBuilderString;
12 |
13 | import crate.elasticsearch.import_.Importer;
14 |
15 | public class NodeImportResponse extends NodeOperationResponse implements ToXContent {
16 |
17 | private Importer.Result result;
18 |
19 | NodeImportResponse() {
20 | }
21 |
22 | public NodeImportResponse(DiscoveryNode discoveryNode, Importer.Result result) {
23 | super(discoveryNode);
24 | this.result = result;
25 | }
26 |
27 | @Override
28 | public XContentBuilder toXContent(XContentBuilder builder, Params params)
29 | throws IOException {
30 | builder.startObject();
31 | builder.field(Fields.NODE_ID, this.getNode().id());
32 | builder.field(Fields.TOOK, result.took);
33 | builder.startArray(Fields.IMPORTED_FILES);
34 | for (Importer.ImportCounts counts : result.importCounts) {
35 | builder.startObject();
36 | builder.field(Fields.FILE_NAME, counts.fileName);
37 | builder.field(Fields.SUCCESSES, counts.successes);
38 | builder.field(Fields.FAILURES, counts.failures);
39 | if (counts.invalid > 0) {
40 | builder.field(Fields.INVALIDATED, counts.invalid);
41 | }
42 | builder.endObject();
43 | }
44 | builder.endArray();
45 | builder.endObject();
46 | return builder;
47 | }
48 |
49 | @Override
50 | public void readFrom(StreamInput in) throws IOException {
51 | super.readFrom(in);
52 | result = new Importer.Result();
53 | result.took = in.readLong();
54 | int fileCount = in.readInt();
55 | for (int i = 0; i < fileCount; i++) {
56 | Importer.ImportCounts counts = new Importer.ImportCounts();
57 | counts.fileName = in.readString();
58 | counts.successes = in.readInt();
59 | counts.failures = in.readInt();
60 | counts.invalid = in.readInt();
61 | result.importCounts.add(counts);
62 | }
63 | }
64 |
65 | @Override
66 | public void writeTo(StreamOutput out) throws IOException {
67 | super.writeTo(out);
68 | out.writeLong(result.took);
69 | out.writeInt(result.importCounts.size());
70 | for (Importer.ImportCounts counts : result.importCounts) {
71 | out.writeString(counts.fileName);
72 | out.writeInt(counts.successes);
73 | out.writeInt(counts.failures);
74 | out.writeInt(counts.invalid);
75 | }
76 | }
77 |
78 | public static NodeImportResponse readNew(StreamInput in) throws IOException {
79 | NodeImportResponse response = new NodeImportResponse();
80 | response.readFrom(in);
81 | return response;
82 | }
83 |
84 | static final class Fields {
85 | static final XContentBuilderString NODE_ID = new XContentBuilderString("node_id");
86 | static final XContentBuilderString TOOK = new XContentBuilderString("took");
87 | static final XContentBuilderString IMPORTED_FILES = new XContentBuilderString("imported_files");
88 | static final XContentBuilderString FILE_NAME = new XContentBuilderString("file_name");
89 | static final XContentBuilderString SUCCESSES = new XContentBuilderString("successes");
90 | static final XContentBuilderString FAILURES = new XContentBuilderString("failures");
91 | static final XContentBuilderString INVALIDATED = new XContentBuilderString("invalidated");
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/TransportImportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_;
2 |
3 | import crate.elasticsearch.action.import_.parser.ImportParser;
4 | import crate.elasticsearch.import_.Importer;
5 | import org.elasticsearch.cluster.ClusterName;
6 | import org.elasticsearch.cluster.ClusterService;
7 | import org.elasticsearch.common.inject.Inject;
8 | import org.elasticsearch.common.settings.Settings;
9 | import org.elasticsearch.env.NodeEnvironment;
10 | import org.elasticsearch.threadpool.ThreadPool;
11 | import org.elasticsearch.transport.TransportService;
12 |
13 | /**
14 | *
15 | */
16 | public class TransportImportAction extends AbstractTransportImportAction {
17 |
18 | @Inject
19 | public TransportImportAction(Settings settings, ClusterName clusterName,
20 | ThreadPool threadPool, ClusterService clusterService,
21 | TransportService transportService, ImportParser importParser, Importer importer, NodeEnvironment nodeEnv) {
22 | super(settings, clusterName, threadPool, clusterService, transportService, importParser, importer, nodeEnv);
23 | }
24 |
25 | @Override
26 | protected String transportAction() {
27 | return ImportAction.NAME;
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/DirectoryParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | public class DirectoryParseElement implements ImportParseElement {
8 |
9 | @Override
10 | public void parse(XContentParser parser, ImportContext context)
11 | throws Exception {
12 | XContentParser.Token token = parser.currentToken();
13 | if (token.isValue()) {
14 | context.directory(parser.text());
15 | }
16 | }
17 |
18 | }
19 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/FilePatternParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import java.util.regex.Pattern;
4 |
5 | import org.elasticsearch.common.xcontent.XContentParser;
6 |
7 | import crate.elasticsearch.action.import_.ImportContext;
8 |
9 | public class FilePatternParseElement implements ImportParseElement {
10 |
11 | @Override
12 | public void parse(XContentParser parser, ImportContext context)
13 | throws Exception {
14 | XContentParser.Token token = parser.currentToken();
15 | if (token.isValue()) {
16 | Pattern p = Pattern.compile(parser.text());
17 | context.file_pattern(p);
18 | }
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/IImportParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import crate.elasticsearch.action.import_.ImportContext;
4 | import org.elasticsearch.common.bytes.BytesReference;
5 |
6 | /**
7 | *
8 | */
9 | public interface IImportParser {
10 | public void parseSource(ImportContext context, BytesReference source) throws ImportParseException;
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportCompressionParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | public class ImportCompressionParseElement implements ImportParseElement {
8 |
9 | @Override
10 | public void parse(XContentParser parser, ImportContext context)
11 | throws Exception {
12 | XContentParser.Token token = parser.currentToken();
13 | if (token.isValue()) {
14 | String lower = parser.text().toLowerCase();
15 | if (lower.equals("gzip")) {
16 | ((ImportContext) context).compression(true);
17 | } else if (!lower.isEmpty()) {
18 | throw new ImportParseException(context,
19 | "Compression format '" + lower + "' unknown or not supported.");
20 | }
21 | }
22 | }
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportMappingsParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | /**
8 | * Parser for token ``mappings``.
9 | */
10 | public class ImportMappingsParseElement implements ImportParseElement {
11 |
12 | @Override
13 | public void parse(XContentParser parser, ImportContext context)
14 | throws Exception {
15 | XContentParser.Token token = parser.currentToken();
16 | if (token.isValue()) {
17 | ((ImportContext)context).mappings(parser.booleanValue());
18 | }
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | public interface ImportParseElement {
8 |
9 | void parse(XContentParser parser, ImportContext context) throws Exception;
10 |
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportParseException.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.ElasticSearchException;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | public class ImportParseException extends ElasticSearchException {
8 |
9 | private static final long serialVersionUID = 910205724931139923L;
10 |
11 | public ImportParseException(ImportContext context, String msg) {
12 | super("Parse Failure [" + msg + "]");
13 | }
14 |
15 | public ImportParseException(ImportContext context, String msg, Throwable cause) {
16 | super("Parse Failure [" + msg + "]", cause);
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.collect.ImmutableMap;
8 | import org.elasticsearch.common.xcontent.XContentFactory;
9 | import org.elasticsearch.common.xcontent.XContentHelper;
10 | import org.elasticsearch.common.xcontent.XContentParser;
11 | import org.elasticsearch.search.SearchParseException;
12 |
13 | import crate.elasticsearch.action.import_.ImportContext;
14 |
15 | public class ImportParser implements IImportParser {
16 |
17 | private final ImmutableMap elementParsers;
18 |
19 | public ImportParser() {
20 | Map elementParsers = new HashMap();
21 | elementParsers.put("directory", new DirectoryParseElement());
22 | elementParsers.put("compression", new ImportCompressionParseElement());
23 | elementParsers.put("file_pattern", new FilePatternParseElement());
24 | elementParsers.put("mappings", new ImportMappingsParseElement());
25 | elementParsers.put("settings", new ImportSettingsParseElement());
26 | this.elementParsers = ImmutableMap.copyOf(elementParsers);
27 | }
28 |
29 | /**
30 | * Main method of this class to parse given payload of _export action
31 | *
32 | * @param context
33 | * @param source
34 | * @throws SearchParseException
35 | */
36 | public void parseSource(ImportContext context, BytesReference source) throws ImportParseException {
37 | XContentParser parser = null;
38 | try {
39 | if (source != null && source.length() != 0) {
40 | parser = XContentFactory.xContent(source).createParser(source);
41 | XContentParser.Token token;
42 | while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
43 | if (token == XContentParser.Token.FIELD_NAME) {
44 | String fieldName = parser.currentName();
45 | parser.nextToken();
46 | ImportParseElement element = elementParsers.get(fieldName);
47 | if (element == null) {
48 | throw new ImportParseException(context, "No parser for element [" + fieldName + "]");
49 | }
50 | element.parse(parser, context);
51 | } else if (token == null) {
52 | break;
53 | }
54 | }
55 | }
56 | validate(context);
57 | } catch (Exception e) {
58 | String sSource = "_na_";
59 | try {
60 | sSource = XContentHelper.convertToJson(source, false);
61 | } catch (Throwable e1) {
62 | // ignore
63 | }
64 | throw new ImportParseException(context, "Failed to parse source [" + sSource + "]", e);
65 | } finally {
66 | if (parser != null) {
67 | parser.close();
68 | }
69 | }
70 | }
71 |
72 | /**
73 | * validate given payload
74 | *
75 | * @param context
76 | */
77 | private void validate(ImportContext context) {
78 | if (context.directory() == null || context.directory().isEmpty()) {
79 | throw new ImportParseException(context, "No directory defined");
80 | }
81 | }
82 | }
83 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/import_/parser/ImportSettingsParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.import_.parser;
2 |
3 | import org.elasticsearch.common.xcontent.XContentParser;
4 |
5 | import crate.elasticsearch.action.import_.ImportContext;
6 |
7 | /**
8 | * Parser for token ``settings``.
9 | */
10 | public class ImportSettingsParseElement implements ImportParseElement {
11 |
12 | @Override
13 | public void parse(XContentParser parser, ImportContext context)
14 | throws Exception {
15 | XContentParser.Token token = parser.currentToken();
16 | if (token.isValue()) {
17 | ((ImportContext)context).settings(parser.booleanValue());
18 | }
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/reindex/ReindexAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.reindex;
2 |
3 | import org.elasticsearch.action.Action;
4 | import org.elasticsearch.client.Client;
5 |
6 | import crate.elasticsearch.action.searchinto.SearchIntoRequest;
7 | import crate.elasticsearch.action.searchinto.SearchIntoResponse;
8 | import crate.elasticsearch.client.action.searchinto.SearchIntoRequestBuilder;
9 |
10 | public class ReindexAction extends Action{
11 |
12 | public static final ReindexAction INSTANCE = new ReindexAction();
13 | public static final String NAME = "el-crate-reindex";
14 |
15 | protected ReindexAction() {
16 | super(NAME);
17 | }
18 |
19 | @Override
20 | public SearchIntoRequestBuilder newRequestBuilder(Client client) {
21 | return new SearchIntoRequestBuilder(client);
22 | }
23 |
24 | @Override
25 | public SearchIntoResponse newResponse() {
26 | return new SearchIntoResponse();
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/reindex/ReindexParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.reindex;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import org.elasticsearch.common.bytes.BytesReference;
7 | import org.elasticsearch.common.collect.ImmutableMap;
8 | import org.elasticsearch.common.inject.Inject;
9 | import org.elasticsearch.search.SearchParseElement;
10 | import org.elasticsearch.search.SearchParseException;
11 | import org.elasticsearch.search.fetch.FetchPhase;
12 | import org.elasticsearch.search.fetch.explain.ExplainParseElement;
13 | import org.elasticsearch.search.query.QueryPhase;
14 |
15 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
16 | import crate.elasticsearch.action.searchinto.parser.AbstractSearchIntoParser;
17 | import crate.elasticsearch.action.searchinto.parser.ISearchIntoParser;
18 |
19 | /**
20 | * Parser for pay load given to _reindex action.
21 | */
22 | public class ReindexParser extends AbstractSearchIntoParser implements ISearchIntoParser {
23 |
24 | private final ImmutableMap elementParsers;
25 |
26 | @Inject
27 | public ReindexParser(QueryPhase queryPhase, FetchPhase fetchPhase) {
28 | Map elementParsers = new HashMap();
30 | elementParsers.putAll(queryPhase.parseElements());
31 | elementParsers.put("explain", new ExplainParseElement());
32 | this.elementParsers = ImmutableMap.copyOf(elementParsers);
33 | }
34 |
35 | @Override
36 | protected ImmutableMap getElementParsers() {
37 | return elementParsers;
38 | }
39 |
40 | @Override
41 | public void parseSource(SearchIntoContext context, BytesReference source)
42 | throws SearchParseException {
43 | context.fieldNames().add("_id");
44 | context.fieldNames().add("_source");
45 | context.outputNames().put("_id", "_id");
46 | context.outputNames().put("_source", "_source");
47 | super.parseSource(context, source);
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/reindex/TransportReindexAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.reindex;
2 |
3 | import org.elasticsearch.cache.recycler.CacheRecycler;
4 | import org.elasticsearch.cluster.ClusterService;
5 | import org.elasticsearch.common.inject.Inject;
6 | import org.elasticsearch.common.settings.Settings;
7 | import org.elasticsearch.indices.IndicesService;
8 | import org.elasticsearch.script.ScriptService;
9 | import org.elasticsearch.threadpool.ThreadPool;
10 | import org.elasticsearch.transport.TransportService;
11 |
12 | import crate.elasticsearch.action.searchinto.AbstractTransportSearchIntoAction;
13 | import crate.elasticsearch.searchinto.Writer;
14 |
15 | public class TransportReindexAction extends AbstractTransportSearchIntoAction {
16 |
17 | @Inject
18 | public TransportReindexAction(Settings settings, ThreadPool threadPool,
19 | ClusterService clusterService, TransportService transportService, CacheRecycler cacheRecycler,
20 | IndicesService indicesService, ScriptService scriptService,
21 | ReindexParser parser, Writer writer) {
22 | super(settings, threadPool, clusterService, transportService, cacheRecycler, indicesService,
23 | scriptService, parser, writer);
24 | }
25 |
26 | @Override
27 | protected String transportAction() {
28 | return ReindexAction.NAME;
29 | }
30 |
31 | }
32 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/restore/RestoreAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.restore;
2 |
3 | import crate.elasticsearch.action.import_.ImportRequest;
4 | import crate.elasticsearch.action.import_.ImportResponse;
5 | import crate.elasticsearch.client.action.import_.ImportRequestBuilder;
6 | import org.elasticsearch.action.Action;
7 | import org.elasticsearch.client.Client;
8 |
9 |
10 | /**
11 | *
12 | */
13 | public class RestoreAction extends Action {
14 |
15 | public static final RestoreAction INSTANCE = new RestoreAction();
16 | public static final String NAME = "el-crate-restore";
17 |
18 | private RestoreAction() {
19 | super(NAME);
20 | }
21 |
22 | @Override
23 | public ImportResponse newResponse() {
24 | return new ImportResponse();
25 | }
26 |
27 | @Override
28 | public ImportRequestBuilder newRequestBuilder(Client client) {
29 | return new ImportRequestBuilder(client);
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/restore/TransportRestoreAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.restore;
2 |
3 | import crate.elasticsearch.action.import_.AbstractTransportImportAction;
4 | import crate.elasticsearch.action.restore.parser.RestoreParser;
5 | import crate.elasticsearch.import_.Importer;
6 | import org.elasticsearch.cluster.ClusterName;
7 | import org.elasticsearch.cluster.ClusterService;
8 | import org.elasticsearch.common.inject.Inject;
9 | import org.elasticsearch.common.settings.Settings;
10 | import org.elasticsearch.env.NodeEnvironment;
11 | import org.elasticsearch.threadpool.ThreadPool;
12 | import org.elasticsearch.transport.TransportService;
13 |
14 | /**
15 | *
16 | */
17 | public class TransportRestoreAction extends AbstractTransportImportAction {
18 |
19 | @Inject
20 | public TransportRestoreAction(Settings settings, ClusterName clusterName,
21 | ThreadPool threadPool, ClusterService clusterService,
22 | TransportService transportService, RestoreParser restoreParser, Importer importer, NodeEnvironment nodeEnv) {
23 | super(settings, clusterName, threadPool, clusterService, transportService, restoreParser, importer, nodeEnv);
24 | }
25 |
26 | @Override
27 | protected String transportAction() {
28 | return RestoreAction.NAME;
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/restore/parser/RestoreParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.restore.parser;
2 |
3 | import crate.elasticsearch.action.dump.parser.DumpParser;
4 | import crate.elasticsearch.action.import_.ImportContext;
5 | import crate.elasticsearch.action.import_.parser.DirectoryParseElement;
6 | import crate.elasticsearch.action.import_.parser.IImportParser;
7 | import crate.elasticsearch.action.import_.parser.ImportParseElement;
8 | import crate.elasticsearch.action.import_.parser.ImportParseException;
9 | import org.elasticsearch.common.bytes.BytesReference;
10 | import org.elasticsearch.common.collect.ImmutableMap;
11 | import org.elasticsearch.common.xcontent.XContentFactory;
12 | import org.elasticsearch.common.xcontent.XContentHelper;
13 | import org.elasticsearch.common.xcontent.XContentParser;
14 |
15 | import java.util.HashMap;
16 | import java.util.Map;
17 | import java.util.regex.Pattern;
18 |
19 | public class RestoreParser implements IImportParser {
20 |
21 | private final ImmutableMap elementParsers;
22 |
23 | public static final String FILE_PATTERN = ".*_.*_.*\\.json\\.gz";
24 |
25 | public RestoreParser() {
26 | Map elementParsers = new HashMap();
27 | elementParsers.put("directory", new DirectoryParseElement());
28 | this.elementParsers = ImmutableMap.copyOf(elementParsers);
29 | }
30 |
31 | /**
32 | * Main method of this class to parse given payload of _restore action
33 | *
34 | * @param context
35 | * @param source
36 | * @throws org.elasticsearch.search.SearchParseException
37 | */
38 | public void parseSource(ImportContext context, BytesReference source) throws ImportParseException {
39 | XContentParser parser = null;
40 | this.setDefaults(context);
41 | context.settings(true);
42 | context.mappings(true);
43 | try {
44 | if (source != null && source.length() != 0) {
45 | parser = XContentFactory.xContent(source).createParser(source);
46 | XContentParser.Token token;
47 | while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
48 | if (token == XContentParser.Token.FIELD_NAME) {
49 | String fieldName = parser.currentName();
50 | parser.nextToken();
51 | ImportParseElement element = elementParsers.get(fieldName);
52 | if (element == null) {
53 | throw new ImportParseException(context, "No parser for element [" + fieldName + "]");
54 | }
55 | element.parse(parser, context);
56 | } else if (token == null) {
57 | break;
58 | }
59 | }
60 | }
61 | if (context.directory() == null) {
62 | context.directory(DumpParser.DEFAULT_DIR);
63 | }
64 | } catch (Exception e) {
65 | String sSource = "_na_";
66 | try {
67 | sSource = XContentHelper.convertToJson(source, false);
68 | } catch (Throwable e1) {
69 | // ignore
70 | }
71 | throw new ImportParseException(context, "Failed to parse source [" + sSource + "]", e);
72 | } finally {
73 | if (parser != null) {
74 | parser.close();
75 | }
76 | }
77 | }
78 |
79 | /**
80 | * Set restore specific default values to the context like compression and file_pattern
81 | *
82 | * @param context
83 | */
84 | private void setDefaults(ImportContext context) {
85 | context.compression(true);
86 | Pattern p = Pattern.compile(FILE_PATTERN);
87 | context.file_pattern(p);
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 |
4 | import crate.elasticsearch.client.action.searchinto.SearchIntoRequestBuilder;
5 | import org.elasticsearch.action.Action;
6 | import org.elasticsearch.client.Client;
7 |
8 |
9 | /**
10 | *
11 | */
12 | public class SearchIntoAction extends Action {
14 |
15 | public static final SearchIntoAction INSTANCE = new SearchIntoAction();
16 | public static final String NAME = "el-crate-searchinto";
17 |
18 | private SearchIntoAction() {
19 | super(NAME);
20 | }
21 |
22 | @Override
23 | public SearchIntoResponse newResponse() {
24 | return new SearchIntoResponse();
25 | }
26 |
27 | @Override
28 | public SearchIntoRequestBuilder newRequestBuilder(Client client) {
29 | return new SearchIntoRequestBuilder(client);
30 | }
31 |
32 | }
33 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoContext.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 | import org.elasticsearch.cache.recycler.CacheRecycler;
4 | import org.elasticsearch.common.collect.ImmutableList;
5 | import org.elasticsearch.common.collect.Lists;
6 | import org.elasticsearch.common.transport.InetSocketTransportAddress;
7 | import org.elasticsearch.index.engine.Engine;
8 | import org.elasticsearch.index.service.IndexService;
9 | import org.elasticsearch.index.shard.service.IndexShard;
10 | import org.elasticsearch.script.ScriptService;
11 | import org.elasticsearch.search.SearchShardTarget;
12 | import org.elasticsearch.search.internal.SearchContext;
13 | import org.elasticsearch.search.internal.ShardSearchRequest;
14 |
15 | import java.util.HashMap;
16 | import java.util.List;
17 | import java.util.Map;
18 |
19 | /**
20 | * Container class for inout specific informations.
21 | */
22 | public class SearchIntoContext extends SearchContext {
23 |
24 | // currently we only support index targets
25 | private String targetType = "index";
26 |
27 | private List targetNodes;
28 |
29 |
30 | public Map outputNames() {
31 | return outputNames;
32 | }
33 |
34 | private final Map outputNames = new HashMap();
36 |
37 | public SearchIntoContext(long id, ShardSearchRequest request,
38 | SearchShardTarget shardTarget, Engine.Searcher engineSearcher,
39 | IndexService indexService, IndexShard indexShard,
40 | ScriptService scriptService, CacheRecycler cacheRecycler) {
41 | super(id, request, shardTarget, engineSearcher, indexService,
42 | indexShard, scriptService, cacheRecycler);
43 | }
44 |
45 | public String targetType() {
46 | // this is currently the only type supported
47 | return targetType;
48 | }
49 |
50 | public List targetNodes() {
51 | if (targetNodes == null) {
52 | targetNodes = Lists.newArrayList();
53 | }
54 | return targetNodes;
55 | }
56 |
57 | public void emptyTargetNodes() {
58 | this.targetNodes = ImmutableList.of();
59 | }
60 |
61 | }
62 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoRequest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 | import org.elasticsearch.action.ActionRequestValidationException;
4 | import org.elasticsearch.action.support.broadcast.BroadcastOperationRequest;
5 | import org.elasticsearch.common.Nullable;
6 | import org.elasticsearch.common.Required;
7 | import org.elasticsearch.common.Strings;
8 | import org.elasticsearch.common.bytes.BytesArray;
9 | import org.elasticsearch.common.bytes.BytesReference;
10 | import org.elasticsearch.common.io.stream.StreamInput;
11 | import org.elasticsearch.common.io.stream.StreamOutput;
12 | import org.elasticsearch.common.xcontent.XContentHelper;
13 |
14 | import java.io.IOException;
15 | import java.util.Arrays;
16 |
17 |
18 | public class SearchIntoRequest extends
19 | BroadcastOperationRequest {
20 |
21 | @Nullable
22 | protected String routing;
23 |
24 | @Nullable
25 | private String preference;
26 |
27 | private BytesReference source;
28 | private boolean querySourceUnsafe;
29 |
30 | private String[] types = Strings.EMPTY_ARRAY;
31 |
32 | SearchIntoRequest() {
33 | }
34 |
35 | /**
36 | * Constructs a new inout request against the provided indices. No
37 | * indices provided means it will
38 | * run against all indices.
39 | */
40 | public SearchIntoRequest(String... indices) {
41 | super(indices);
42 | }
43 |
44 | @Override
45 | public ActionRequestValidationException validate() {
46 | ActionRequestValidationException validationException = super
47 | .validate();
48 | return validationException;
49 | }
50 |
51 | @Override
52 | protected void beforeStart() {
53 | if (querySourceUnsafe) {
54 | source = source.copyBytesArray();
55 | querySourceUnsafe = false;
56 | }
57 | }
58 |
59 | /**
60 | * The query source to execute.
61 | */
62 | public BytesReference source() {
63 | return source;
64 | }
65 |
66 | @Required
67 | public SearchIntoRequest source(String source) {
68 | return this.source(new BytesArray(source), false);
69 | }
70 |
71 |
72 | @Required
73 | public SearchIntoRequest source(BytesReference source, boolean unsafe) {
74 | this.source = source;
75 | this.querySourceUnsafe = unsafe;
76 | return this;
77 | }
78 |
79 | /**
80 | * The types of documents the query will run against. Defaults to all
81 | * types.
82 | */
83 | String[] types() {
84 | return this.types;
85 | }
86 |
87 | /**
88 | * The types of documents the query will run against. Defaults to all
89 | * types.
90 | */
91 | public SearchIntoRequest types(String... types) {
92 | this.types = types;
93 | return this;
94 | }
95 |
96 | /**
97 | * A comma separated list of routing values to control the shards the
98 | * search will be executed on.
99 | */
100 | public String routing() {
101 | return this.routing;
102 | }
103 |
104 | /**
105 | * A comma separated list of routing values to control the shards the
106 | * search will be executed on.
107 | */
108 | public SearchIntoRequest routing(String routing) {
109 | this.routing = routing;
110 | return this;
111 | }
112 |
113 | /**
114 | * The routing values to control the shards that the search will be
115 | * executed on.
116 | */
117 | public SearchIntoRequest routing(String... routings) {
118 | this.routing = Strings.arrayToCommaDelimitedString(routings);
119 | return this;
120 | }
121 |
122 | public SearchIntoRequest preference(String preference) {
123 | this.preference = preference;
124 | return this;
125 | }
126 |
127 | public String preference() {
128 | return this.preference;
129 | }
130 |
131 | @Override
132 | public void readFrom(StreamInput in) throws IOException {
133 | super.readFrom(in);
134 | routing = in.readOptionalString();
135 | preference = in.readOptionalString();
136 | querySourceUnsafe = false;
137 | source = in.readBytesReference();
138 | types = in.readStringArray();
139 | }
140 |
141 | @Override
142 | public void writeTo(StreamOutput out) throws IOException {
143 | super.writeTo(out);
144 | out.writeOptionalString(routing);
145 | out.writeOptionalString(preference);
146 | out.writeBytesReference(source);
147 | out.writeStringArray(types);
148 | }
149 |
150 | @Override
151 | public String toString() {
152 | String sSource = "_na_";
153 | try {
154 | sSource = XContentHelper.convertToJson(source, false);
155 | } catch (Exception e) {
156 | // ignore
157 | }
158 | return "[" + Arrays.toString(indices) + "]" + Arrays.toString(
159 | types) + ", querySource[" + sSource + "]";
160 | }
161 | }
162 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/SearchIntoResponse.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 |
4 | import org.elasticsearch.action.ShardOperationFailedException;
5 | import org.elasticsearch.action.support.broadcast.BroadcastOperationResponse;
6 | import org.elasticsearch.common.io.stream.StreamInput;
7 | import org.elasticsearch.common.io.stream.StreamOutput;
8 | import org.elasticsearch.common.xcontent.ToXContent;
9 | import org.elasticsearch.common.xcontent.XContentBuilder;
10 |
11 | import java.io.IOException;
12 | import java.util.ArrayList;
13 | import java.util.List;
14 |
15 | import static org.elasticsearch.rest.action.support.RestActions
16 | .buildBroadcastShardsHeader;
17 |
18 | public class SearchIntoResponse extends BroadcastOperationResponse
19 | implements ToXContent {
20 |
21 |
22 | private List responses;
23 |
24 | private long totalWrites;
25 | private long failedWrites;
26 | private long succeededWrites;
27 |
28 | public SearchIntoResponse(List responses,
29 | int totalShards, int successfulShards, int failedShards,
30 | List shardFailures) {
31 | super(totalShards, successfulShards, failedShards, shardFailures);
32 | this.responses = responses;
33 | for (ShardSearchIntoResponse r : this.responses) {
34 | totalWrites += r.getTotalWrites();
35 | succeededWrites += r.getSucceededWrites();
36 | failedWrites += r.getFailedWrites();
37 | }
38 | }
39 |
40 | public SearchIntoResponse() {
41 |
42 | }
43 |
44 | public long getTotalWrites() {
45 | return totalWrites;
46 | }
47 |
48 |
49 | public List getResponses() {
50 | return responses;
51 | }
52 |
53 | @Override
54 | public void readFrom(StreamInput in) throws IOException {
55 | super.readFrom(in);
56 | totalWrites = in.readVLong();
57 | succeededWrites = in.readVLong();
58 | failedWrites = in.readVLong();
59 | int numResponses = in.readVInt();
60 | responses = new ArrayList(numResponses);
61 | for (int i = 0; i < numResponses; i++) {
62 | responses.add(ShardSearchIntoResponse.readNew(in));
63 | }
64 | }
65 |
66 | @Override
67 | public void writeTo(StreamOutput out) throws IOException {
68 | super.writeTo(out);
69 | out.writeVLong(totalWrites);
70 | out.writeVLong(succeededWrites);
71 | out.writeVLong(failedWrites);
72 | out.writeVInt(responses.size());
73 | for (ShardSearchIntoResponse response : responses) {
74 | response.writeTo(out);
75 | }
76 | }
77 |
78 | @Override
79 | public XContentBuilder toXContent(XContentBuilder builder,
80 | Params params) throws IOException {
81 | builder.startObject();
82 | builder.startArray("writes");
83 | for (ShardSearchIntoResponse r : this.responses) {
84 | r.toXContent(builder, params);
85 | }
86 | builder.endArray();
87 | builder.field("total", totalWrites);
88 | builder.field("succeeded", succeededWrites);
89 | builder.field("failed", failedWrites);
90 | buildBroadcastShardsHeader(builder, this);
91 | builder.endObject();
92 | return builder;
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/ShardSearchIntoRequest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 | import org.elasticsearch.action.support.broadcast
4 | .BroadcastShardOperationRequest;
5 | import org.elasticsearch.common.Nullable;
6 | import org.elasticsearch.common.Strings;
7 | import org.elasticsearch.common.bytes.BytesReference;
8 | import org.elasticsearch.common.io.stream.StreamInput;
9 | import org.elasticsearch.common.io.stream.StreamOutput;
10 |
11 | import java.io.IOException;
12 |
13 |
14 | /**
15 | * Internal inout request executed directly against a specific index shard.
16 | */
17 | class ShardSearchIntoRequest extends BroadcastShardOperationRequest {
18 |
19 | private BytesReference source;
20 |
21 | private String[] types = Strings.EMPTY_ARRAY;
22 |
23 | @Nullable
24 | private String[] filteringAliases;
25 |
26 | ShardSearchIntoRequest() {
27 |
28 | }
29 |
30 | public ShardSearchIntoRequest(String index, int shardId,
31 | @Nullable String[] filteringAliases, SearchIntoRequest request) {
32 | super(index, shardId, request);
33 | this.source = request.source();
34 | this.types = request.types();
35 | this.filteringAliases = filteringAliases;
36 | }
37 |
38 | public BytesReference source() {
39 | return source;
40 | }
41 |
42 | public String[] types() {
43 | return this.types;
44 | }
45 |
46 | public String[] filteringAliases() {
47 | return filteringAliases;
48 | }
49 |
50 | @Override
51 | public void readFrom(StreamInput in) throws IOException {
52 | super.readFrom(in);
53 | source = in.readBytesReference();
54 | int typesSize = in.readVInt();
55 | if (typesSize > 0) {
56 | types = new String[typesSize];
57 | for (int i = 0; i < typesSize; i++) {
58 | types[i] = in.readString();
59 | }
60 | }
61 | int aliasesSize = in.readVInt();
62 | if (aliasesSize > 0) {
63 | filteringAliases = new String[aliasesSize];
64 | for (int i = 0; i < aliasesSize; i++) {
65 | filteringAliases[i] = in.readString();
66 | }
67 | }
68 | }
69 |
70 | @Override
71 | public void writeTo(StreamOutput out) throws IOException {
72 | super.writeTo(out);
73 | out.writeBytesReference(source);
74 |
75 | out.writeVInt(types.length);
76 | for (String type : types) {
77 | out.writeString(type);
78 | }
79 | if (filteringAliases != null) {
80 | out.writeVInt(filteringAliases.length);
81 | for (String alias : filteringAliases) {
82 | out.writeString(alias);
83 | }
84 | } else {
85 | out.writeVInt(0);
86 | }
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/ShardSearchIntoResponse.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 | import crate.elasticsearch.searchinto.WriterResult;
4 | import org.elasticsearch.action.support.broadcast
5 | .BroadcastShardOperationResponse;
6 | import org.elasticsearch.common.io.stream.StreamInput;
7 | import org.elasticsearch.common.io.stream.StreamOutput;
8 | import org.elasticsearch.common.text.Text;
9 | import org.elasticsearch.common.xcontent.ToXContent;
10 | import org.elasticsearch.common.xcontent.XContentBuilder;
11 |
12 | import java.io.IOException;
13 | import java.util.List;
14 |
15 | /**
16 | * Internal searchinto response of a shard searchinto request executed
17 | * directly against a specific shard.
18 | */
19 | class ShardSearchIntoResponse extends BroadcastShardOperationResponse
20 | implements ToXContent {
21 |
22 | private WriterResult result;
23 | private List cmdArray;
24 | private boolean dryRun = false;
25 | private Text node;
26 |
27 | private long totalWrites;
28 | private long failedWrites;
29 | private long succeededWrites;
30 |
31 |
32 | ShardSearchIntoResponse() {
33 | }
34 |
35 |
36 | public ShardSearchIntoResponse(Text node, String index, int shardId,
37 | WriterResult result) {
38 | super(index, shardId);
39 | this.node = node;
40 | this.result = result;
41 | }
42 |
43 | /**
44 | * Constructor for dry runs. Does not contain any execution infos
45 | */
46 | public ShardSearchIntoResponse(Text node, String index, int shardId) {
47 | this.node = node;
48 | this.dryRun = true;
49 | }
50 |
51 | public long getTotalWrites() {
52 | return totalWrites;
53 | }
54 |
55 | long getFailedWrites() {
56 | return failedWrites;
57 | }
58 |
59 | long getSucceededWrites() {
60 | return succeededWrites;
61 | }
62 |
63 | public boolean dryRun() {
64 | return dryRun;
65 | }
66 |
67 | public Text getNode() {
68 | return node;
69 | }
70 |
71 | public static ShardSearchIntoResponse readNew(StreamInput in) throws
72 | IOException {
73 | ShardSearchIntoResponse response = new ShardSearchIntoResponse();
74 | response.readFrom(in);
75 | return response;
76 | }
77 |
78 |
79 | @Override
80 | public void readFrom(StreamInput in) throws IOException {
81 | super.readFrom(in);
82 | node = in.readOptionalText();
83 | dryRun = in.readBoolean();
84 | if (in.readBoolean()) {
85 | result = WriterResult.readNew(in);
86 | }
87 | }
88 |
89 | @Override
90 | public void writeTo(StreamOutput out) throws IOException {
91 | super.writeTo(out);
92 | out.writeOptionalText(node);
93 | out.writeBoolean(dryRun);
94 | if (result != null) {
95 | out.writeBoolean(true);
96 | result.writeTo(out);
97 | } else {
98 | out.writeBoolean(false);
99 | }
100 | }
101 |
102 | @Override
103 | public XContentBuilder toXContent(XContentBuilder builder,
104 | Params params) throws IOException {
105 | builder.startObject();
106 | builder.field("index", getIndex());
107 | builder.field("shard", getShardId());
108 | if (node != null) {
109 | builder.field("node", node);
110 | }
111 | result.toXContent(builder, params);
112 | builder.endObject();
113 | return builder;
114 | }
115 | }
116 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/TransportSearchIntoAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto;
2 |
3 | import org.elasticsearch.cache.recycler.CacheRecycler;
4 | import org.elasticsearch.cluster.ClusterService;
5 | import org.elasticsearch.common.inject.Inject;
6 | import org.elasticsearch.common.settings.Settings;
7 | import org.elasticsearch.indices.IndicesService;
8 | import org.elasticsearch.script.ScriptService;
9 | import org.elasticsearch.threadpool.ThreadPool;
10 | import org.elasticsearch.transport.TransportService;
11 |
12 | import crate.elasticsearch.action.searchinto.parser.SearchIntoParser;
13 | import crate.elasticsearch.searchinto.Writer;
14 |
15 |
16 | /**
17 | *
18 | */
19 | public class TransportSearchIntoAction extends AbstractTransportSearchIntoAction {
20 |
21 | @Inject
22 | public TransportSearchIntoAction(Settings settings,
23 | ThreadPool threadPool, ClusterService clusterService,
24 | TransportService transportService, CacheRecycler cacheRecycler,
25 | IndicesService indicesService, ScriptService scriptService,
26 | SearchIntoParser parser, Writer writer) {
27 | super(settings, threadPool, clusterService, transportService, cacheRecycler, indicesService,
28 | scriptService, parser, writer);
29 | }
30 |
31 | @Override
32 | protected String transportAction() {
33 | return SearchIntoAction.NAME;
34 | }
35 |
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/parser/AbstractSearchIntoParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto.parser;
2 |
3 | import org.elasticsearch.common.bytes.BytesReference;
4 | import org.elasticsearch.common.collect.ImmutableMap;
5 | import org.elasticsearch.common.xcontent.XContentFactory;
6 | import org.elasticsearch.common.xcontent.XContentHelper;
7 | import org.elasticsearch.common.xcontent.XContentParser;
8 | import org.elasticsearch.search.SearchParseElement;
9 | import org.elasticsearch.search.SearchParseException;
10 |
11 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
12 |
13 | public abstract class AbstractSearchIntoParser implements ISearchIntoParser {
14 |
15 | /**
16 | * Main method of this class to parse given payload of _search_into action
17 | *
18 | * @param context
19 | * @param source
20 | * @throws org.elasticsearch.search.SearchParseException
21 | *
22 | */
23 | public void parseSource(SearchIntoContext context,
24 | BytesReference source) throws SearchParseException {
25 | XContentParser parser = null;
26 | try {
27 | if (source != null) {
28 | parser = XContentFactory.xContent(source).createParser(source);
29 | XContentParser.Token token;
30 | while ((token = parser.nextToken()) != XContentParser.Token
31 | .END_OBJECT) {
32 | if (token == XContentParser.Token.FIELD_NAME) {
33 | String fieldName = parser.currentName();
34 | parser.nextToken();
35 | SearchParseElement element = getElementParsers().get(
36 | fieldName);
37 | if (element == null) {
38 | throw new SearchParseException(context,
39 | "No parser for element [" + fieldName +
40 | "]");
41 | }
42 | element.parse(parser, context);
43 | } else if (token == null) {
44 | break;
45 | }
46 | }
47 | }
48 | validate(context);
49 | } catch (Exception e) {
50 | String sSource = "_na_";
51 | try {
52 | sSource = XContentHelper.convertToJson(source, false);
53 | } catch (Throwable e1) {
54 | // ignore
55 | }
56 | throw new SearchParseException(context,
57 | "Failed to parse source [" + sSource + "]", e);
58 | } finally {
59 | if (parser != null) {
60 | parser.close();
61 | }
62 | }
63 | }
64 |
65 | /**
66 | * Get the element parser map
67 | * @return
68 | */
69 | protected abstract ImmutableMap getElementParsers();
70 |
71 | /**
72 | * Validate the pay load of the search-into context.
73 | * @param context
74 | */
75 | protected void validate(SearchIntoContext context) {
76 | if (context.hasFieldNames() && context.fieldNames().contains("_source")) {
77 | String index = context.mapperService().index().getName();
78 | for (String type : context.mapperService().types()) {
79 | if (!context.mapperService().documentMapper(type).sourceMapper().enabled()) {
80 | throw new SearchParseException(context,
81 | "The _source field of index " + index + " and type " + type + " is not stored.");
82 | }
83 | }
84 | }
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/parser/ISearchIntoParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto.parser;
2 |
3 | import org.elasticsearch.common.bytes.BytesReference;
4 |
5 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
6 |
7 | /**
8 | * Interface for search into parsers.
9 | *
10 | * Known implementations: @SearchIntoParser, @ReindexParser
11 | */
12 | public interface ISearchIntoParser {
13 |
14 | /**
15 | * Main method of this class to parse given payload of _search_into action
16 | *
17 | * @param context
18 | * @param source
19 | * @throws org.elasticsearch.search.SearchParseException
20 | *
21 | */
22 | void parseSource(SearchIntoContext context, BytesReference source);
23 |
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/parser/InvalidNodeAddressException.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto.parser;
2 |
3 | import org.elasticsearch.search.SearchContextException;
4 | import org.elasticsearch.search.internal.SearchContext;
5 |
6 | /**
7 | * An exception indicating a given node address used for TransportClient was invalid
8 | */
9 | public class InvalidNodeAddressException extends SearchContextException {
10 |
11 | public InvalidNodeAddressException(SearchContext context, String msg) {
12 | super(context, "Invalid Address [" + msg + "]");
13 | }
14 |
15 | public InvalidNodeAddressException(SearchContext context, String msg, Throwable t) {
16 | super(context, "Invalid Address [" + msg + "]", t);
17 | }
18 | }
19 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/parser/SearchIntoParser.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto.parser;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import org.elasticsearch.common.collect.ImmutableMap;
7 | import org.elasticsearch.common.inject.Inject;
8 | import org.elasticsearch.index.mapper.FieldMapper;
9 | import org.elasticsearch.search.SearchParseElement;
10 | import org.elasticsearch.search.SearchParseException;
11 | import org.elasticsearch.search.fetch.FetchPhase;
12 | import org.elasticsearch.search.fetch.explain.ExplainParseElement;
13 | import org.elasticsearch.search.query.QueryPhase;
14 |
15 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
16 |
17 | /**
18 | * Parser for payload given to _search_into action.
19 | */
20 | public class SearchIntoParser extends AbstractSearchIntoParser implements ISearchIntoParser {
21 |
22 | private final ImmutableMap elementParsers;
23 |
24 | @Inject
25 | public SearchIntoParser(QueryPhase queryPhase, FetchPhase fetchPhase) {
26 | Map elementParsers = new HashMap();
28 | elementParsers.putAll(queryPhase.parseElements());
29 | elementParsers.put("fields", new FieldsParseElement());
30 | elementParsers.put("targetNodes", new TargetNodesParseElement());
31 | elementParsers.put("explain", new ExplainParseElement());
32 | this.elementParsers = ImmutableMap.copyOf(elementParsers);
33 | }
34 |
35 | @Override
36 | protected void validate(SearchIntoContext context) {
37 | if (!context.hasFieldNames()) {
38 | throw new SearchParseException(context, "No fields defined");
39 | }
40 |
41 | for (String field : context.fieldNames()) {
42 | FieldMapper mapper = context.mapperService().smartNameFieldMapper(
43 | field);
44 | if (mapper == null && !field.equals(
45 | "_version") && !field.startsWith(
46 | FieldsParseElement.SCRIPT_FIELD_PREFIX)) {
47 | throw new SearchParseException(context,
48 | "SearchInto field [" + field + "] does not exist in " +
49 | "the mapping");
50 | }
51 | }
52 | super.validate(context);
53 | }
54 |
55 | @Override
56 | protected ImmutableMap getElementParsers() {
57 | return elementParsers;
58 | }
59 |
60 |
61 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/action/searchinto/parser/TargetNodesParseElement.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.action.searchinto.parser;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
4 | import org.elasticsearch.common.transport.InetSocketTransportAddress;
5 | import org.elasticsearch.common.xcontent.XContentParser;
6 | import org.elasticsearch.search.SearchParseElement;
7 | import org.elasticsearch.search.internal.SearchContext;
8 |
9 | import java.util.regex.Matcher;
10 | import java.util.regex.Pattern;
11 |
12 | /**
13 | * parses the targetNode field which looks like
14 | *
15 | * "targetNode": ["host:9300", "host:9301"]
16 | *
17 | * or
18 | *
19 | * "targetNode": "host:9300"
20 | *
21 | */
22 | public class TargetNodesParseElement implements SearchParseElement {
23 |
24 | private Pattern PATTERN = Pattern.compile("^\\s*(.*?):(\\d+)\\s*$");
25 |
26 | @Override
27 | public void parse(XContentParser parser, SearchContext context) throws Exception {
28 | XContentParser.Token token = parser.currentToken();
29 | if (token == XContentParser.Token.START_ARRAY) {
30 | boolean added = false;
31 | SearchIntoContext ctx = (SearchIntoContext)context;
32 | while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
33 | addAddress(ctx, parser.text());
34 | }
35 | if (!added) {
36 | ctx.emptyTargetNodes();
37 | }
38 | } else if (token == XContentParser.Token.VALUE_STRING) {
39 | addAddress((SearchIntoContext)context, parser.text());
40 | }
41 | }
42 |
43 | private void addAddress(SearchIntoContext context, String nodeAddress) {
44 | Matcher m = PATTERN.matcher(nodeAddress);
45 | if (m.matches()) {
46 | String host = m.group(1);
47 | int port = Integer.parseInt(m.group(2));
48 | InetSocketTransportAddress isa = new InetSocketTransportAddress(host, port);
49 | context.targetNodes().add(isa);
50 | } else {
51 | throw new InvalidNodeAddressException(context, nodeAddress);
52 | }
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/client/action/export/ExportRequestBuilder.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.client.action.export;
2 |
3 | import crate.elasticsearch.action.export.ExportAction;
4 | import crate.elasticsearch.action.export.ExportRequest;
5 | import crate.elasticsearch.action.export.ExportResponse;
6 | import org.elasticsearch.action.ActionListener;
7 | import org.elasticsearch.action.ActionRequestBuilder;
8 | import org.elasticsearch.client.Client;
9 | import org.elasticsearch.client.internal.InternalClient;
10 |
11 | public class ExportRequestBuilder extends ActionRequestBuilder {
12 |
13 | public ExportRequestBuilder(Client client) {
14 | super((InternalClient) client, new ExportRequest());
15 | }
16 |
17 | @Override
18 | protected void doExecute(ActionListener listener) {
19 | ((Client)client).execute(ExportAction.INSTANCE, request, listener);
20 | }
21 |
22 | public ExportRequestBuilder setIndices(String ... indices) {
23 | request.indices(indices);
24 | return this;
25 | }
26 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/client/action/import_/ImportRequestBuilder.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.client.action.import_;
2 |
3 | import org.elasticsearch.action.ActionListener;
4 | import org.elasticsearch.action.ActionRequestBuilder;
5 | import org.elasticsearch.client.Client;
6 | import org.elasticsearch.client.internal.InternalClient;
7 |
8 | import crate.elasticsearch.action.import_.ImportAction;
9 | import crate.elasticsearch.action.import_.ImportRequest;
10 | import crate.elasticsearch.action.import_.ImportResponse;
11 |
12 | public class ImportRequestBuilder extends ActionRequestBuilder {
13 |
14 | public ImportRequestBuilder(Client client) {
15 | super((InternalClient) client, new ImportRequest());
16 | }
17 |
18 | @Override
19 | protected void doExecute(ActionListener listener) {
20 | ((Client) client).execute(ImportAction.INSTANCE, request, listener);
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/client/action/searchinto/SearchIntoRequestBuilder.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.client.action.searchinto;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoAction;
4 | import crate.elasticsearch.action.searchinto.SearchIntoRequest;
5 | import crate.elasticsearch.action.searchinto.SearchIntoResponse;
6 | import org.elasticsearch.action.ActionListener;
7 | import org.elasticsearch.action.ActionRequestBuilder;
8 | import org.elasticsearch.client.Client;
9 | import org.elasticsearch.client.internal.InternalClient;
10 |
11 | public class SearchIntoRequestBuilder extends
12 | ActionRequestBuilder {
14 |
15 | public SearchIntoRequestBuilder(Client client) {
16 | super((InternalClient) client, new SearchIntoRequest());
17 | }
18 |
19 | @Override
20 | protected void doExecute(ActionListener listener) {
21 | ((Client) client).execute(SearchIntoAction.INSTANCE, request,
22 | listener);
23 | }
24 |
25 | public SearchIntoRequestBuilder setIndices(String... indices) {
26 | request.indices(indices);
27 | return this;
28 | }
29 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/ExportException.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import org.elasticsearch.search.SearchContextException;
4 | import org.elasticsearch.search.internal.SearchContext;
5 |
6 | /**
7 | * Created with IntelliJ IDEA.
8 | * User: bd
9 | * Date: 9.4.13
10 | * Time: 14:19
11 | * To change this template use File | Settings | File Templates.
12 | */
13 | public class ExportException extends SearchContextException {
14 |
15 | public ExportException(SearchContext context, String msg) {
16 | super(context, msg);
17 | }
18 |
19 | public ExportException(SearchContext context, String msg, Throwable t) {
20 | super(context, "Export Failed [" + msg + "]", t);
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/ExportFields.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import org.elasticsearch.common.bytes.BytesReference;
4 | import org.elasticsearch.common.xcontent.*;
5 | import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
6 | import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
7 | import org.elasticsearch.search.SearchHitField;
8 | import org.elasticsearch.search.internal.InternalSearchHit;
9 |
10 | import java.io.IOException;
11 | import java.util.ArrayList;
12 | import java.util.List;
13 |
14 | public class ExportFields implements ToXContent {
15 |
16 | private final List fields;
17 | private InternalSearchHit hit;
18 | private final List fieldExtractors;
19 |
20 | static final class Fields {
21 | static final XContentBuilderString _SOURCE = new XContentBuilderString("_source");
22 | static final XContentBuilderString _TYPE = new XContentBuilderString("_type");
23 | static final XContentBuilderString _INDEX = new XContentBuilderString("_index");
24 | static final XContentBuilderString _ID = new XContentBuilderString("_id");
25 | static final XContentBuilderString _VERSION = new XContentBuilderString("_version");
26 | static final XContentBuilderString _TIMESTAMP = new XContentBuilderString(TimestampFieldMapper.NAME);
27 | static final XContentBuilderString _TTL = new XContentBuilderString(TTLFieldMapper.NAME);
28 | }
29 |
30 | abstract class FieldExtractor implements ToXContent {
31 |
32 |
33 | }
34 |
35 |
36 | class SourceFieldExtractor extends FieldExtractor {
37 |
38 | @Override
39 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
40 | BytesReference source = hit.sourceRef();
41 | XContentType contentType = XContentFactory.xContentType(source);
42 | XContentParser parser = XContentFactory.xContent(contentType).createParser(source);
43 | try {
44 | parser.nextToken();
45 | builder.field(Fields._SOURCE);
46 | builder.copyCurrentStructure(parser);
47 | } finally {
48 | parser.close();
49 | }
50 | return builder;
51 | }
52 | }
53 |
54 | class HitFieldExtractor extends FieldExtractor {
55 |
56 | private final String fieldName;
57 |
58 | public HitFieldExtractor(String fieldName) {
59 | this.fieldName = fieldName;
60 | }
61 |
62 | @Override
63 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
64 | SearchHitField field = hit.getFields().get(fieldName);
65 | if (field != null && !field.values().isEmpty()) {
66 | if (field.values().size() == 1) {
67 | builder.field(field.name(), field.values().get(0));
68 | } else {
69 | builder.field(field.name());
70 | builder.startArray();
71 | for (Object value : field.values()) {
72 | builder.value(value);
73 | }
74 | builder.endArray();
75 | }
76 | }
77 | return builder;
78 | }
79 | }
80 |
81 | public void hit(InternalSearchHit hit) {
82 | this.hit = hit;
83 | }
84 |
85 | public ExportFields(List fields) {
86 | this.fields = fields;
87 | this.fieldExtractors = getFieldExtractors();
88 | }
89 |
90 | private List getFieldExtractors() {
91 | List extractors = new ArrayList(fields.size());
92 | for (String fn : fields) {
93 | FieldExtractor fc = null;
94 | if (fn.startsWith("_")) {
95 | if (fn.equals("_source")) {
96 | fc = new SourceFieldExtractor();
97 | } else if (fn.equals("_id")) {
98 | fc = new FieldExtractor() {
99 | @Override
100 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
101 | return builder.field(Fields._ID, hit.getId());
102 | }
103 | };
104 | } else if (fn.equals("_version")) {
105 | fc = new FieldExtractor() {
106 | @Override
107 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
108 | return builder.field(Fields._VERSION, hit.getVersion());
109 | }
110 | };
111 | } else if (fn.equals("_index")) {
112 | fc = new FieldExtractor() {
113 | @Override
114 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
115 | return builder.field(Fields._INDEX, hit.getIndex());
116 | }
117 | };
118 | } else if (fn.equals("_type")) {
119 | fc = new FieldExtractor() {
120 | @Override
121 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
122 | return builder.field(Fields._TYPE, hit.getType());
123 | }
124 | };
125 | } else {
126 | fc = new HitFieldExtractor(fn);
127 | }
128 | } else {
129 | fc = new HitFieldExtractor(fn);
130 | }
131 | extractors.add(fc);
132 | }
133 | return extractors;
134 | }
135 |
136 | @Override
137 | public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
138 | builder.startObject();
139 | for (FieldExtractor fc : fieldExtractors) {
140 | fc.toXContent(builder, params);
141 | }
142 | builder.endObject();
143 | return builder;
144 | }
145 | }
146 |
147 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/Output.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import java.io.OutputStream;
4 |
5 | public abstract class Output {
6 |
7 | private Result result;
8 |
9 | public class Result {
10 | public int exit;
11 | public String stdErr;
12 | public String stdOut;
13 | }
14 |
15 | public abstract void open() throws java.io.IOException;
16 |
17 | public abstract void close() throws java.io.IOException;
18 |
19 | public abstract OutputStream getOutputStream();
20 |
21 | public Result result() {
22 | return result;
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/OutputCommand.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import java.io.IOException;
4 | import java.io.OutputStream;
5 | import java.util.List;
6 | import java.util.zip.GZIPOutputStream;
7 |
8 |
9 | /**
10 | * Start an OS Command as a process and push strings to the process'
11 | * standard in. Get standard out and standard error messages when
12 | * process has finished.
13 | */
14 | public class OutputCommand extends Output {
15 |
16 | private static final int BUFFER_LEN = 8192;
17 |
18 | private final ProcessBuilder builder;
19 | private final boolean compression;
20 | private Process process;
21 | private Result result;
22 | private StreamConsumer outputConsumer, errorConsumer;
23 | private OutputStream os;
24 |
25 | /**
26 | * Initialize the process builder with a single command.
27 | * @param command
28 | */
29 | public OutputCommand(String command, boolean compression) {
30 | builder = new ProcessBuilder(command);
31 | this.compression = compression;
32 | }
33 |
34 | /**
35 | * Initialize the process with a command list.
36 | * @param cmdArray
37 | */
38 | public OutputCommand(List cmdArray, boolean compression) {
39 | builder = new ProcessBuilder(cmdArray);
40 | this.compression = compression;
41 | }
42 |
43 | /**
44 | * Start the process and prepare writing to it's standard in.
45 | *
46 | * @throws IOException
47 | */
48 | public void open() throws IOException {
49 | process = builder.start();
50 | outputConsumer = new StreamConsumer(process.getInputStream(),
51 | BUFFER_LEN);
52 | errorConsumer = new StreamConsumer(process.getErrorStream(),
53 | BUFFER_LEN);
54 | os = process.getOutputStream();
55 | if (compression) {
56 | os = new GZIPOutputStream(os);
57 | }
58 | }
59 |
60 | /**
61 | * Get the output stream to write to the process' standard in.
62 | */
63 | public OutputStream getOutputStream() {
64 | return os;
65 | }
66 |
67 | /**
68 | * Stop writing to the process' standard in and wait until the
69 | * process is finished and close all resources.
70 | *
71 | * @throws IOException
72 | */
73 | public void close() throws IOException {
74 | if (process != null) {
75 | os.flush();
76 | os.close();
77 | result = new Result();
78 | try {
79 | result.exit = process.waitFor();
80 | } catch (InterruptedException e) {
81 | result.exit = process.exitValue();
82 | }
83 | outputConsumer.waitFor();
84 | result.stdOut = outputConsumer.getBufferedOutput();
85 | errorConsumer.waitFor();
86 | result.stdErr = errorConsumer.getBufferedOutput();
87 | }
88 | }
89 |
90 | public Result result() {
91 | return result;
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/OutputFile.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import java.io.File;
4 | import java.io.FileOutputStream;
5 | import java.io.IOException;
6 | import java.io.OutputStream;
7 | import java.util.zip.GZIPOutputStream;
8 |
9 | public class OutputFile extends Output {
10 |
11 | private Result result;
12 | private final String path;
13 | private OutputStream os;
14 | private final boolean overwrite;
15 | private final boolean compression;
16 |
17 | public OutputFile(String path, boolean overwrite, boolean compression) {
18 | this.path = path;
19 | this.overwrite = overwrite;
20 | this.compression = compression;
21 | }
22 |
23 | @Override
24 | public void open() throws IOException {
25 | File outFile = new File(path);
26 | if (!overwrite && outFile.exists()){
27 | throw new IOException("File exists: " + path);
28 | }
29 | os = new FileOutputStream(outFile);
30 | if (compression) {
31 | os = new GZIPOutputStream(os);
32 | }
33 | }
34 |
35 | @Override
36 | public void close() throws IOException {
37 | result = new Result();
38 | if (os != null) {
39 | os.close();
40 | result.exit = 0;
41 | } else {
42 | result.exit = 1;
43 | }
44 | os = null;
45 | }
46 |
47 | @Override
48 | public OutputStream getOutputStream() {
49 | return os;
50 | }
51 |
52 | @Override
53 | public Result result() {
54 | return result;
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/export/StreamConsumer.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import java.io.BufferedReader;
4 | import java.io.IOException;
5 | import java.io.InputStream;
6 | import java.io.InputStreamReader;
7 |
8 | /**
9 | * A stream consumer to consume the output of an input stream.
10 | */
11 | public class StreamConsumer {
12 |
13 | private final StreamConsumerImpl impl;
14 | private Thread thread;
15 |
16 | /**
17 | * Constructor.
18 | *
19 | * @param inputStream the input stream to consume
20 | * @param bufferSize the buffer size to keep (first x bytes)
21 | */
22 | public StreamConsumer(InputStream inputStream, int bufferSize) {
23 | impl = new StreamConsumerImpl(inputStream, bufferSize);
24 | thread = new Thread(impl);
25 | thread.start();
26 | }
27 |
28 | /**
29 | * Get the buffered output of the stream (first x bytes defined
30 | * in buffer size)
31 | * @return
32 | */
33 | public String getBufferedOutput() {
34 | return impl.getOutput();
35 | }
36 |
37 | /**
38 | * Wait for the stream to finish.
39 | */
40 | public void waitFor() {
41 | try {
42 | thread.join();
43 | } catch (InterruptedException e) {
44 | e.printStackTrace();
45 | }
46 | }
47 |
48 | /**
49 | * Implementation class for thread.
50 | */
51 | private final class StreamConsumerImpl implements Runnable {
52 |
53 | private final int bufferSize;
54 | private final StringBuffer collectedOutput = new StringBuffer();
55 | private InputStream inputStream;
56 |
57 | private StreamConsumerImpl(InputStream inputStream, int bufferSize) {
58 | this.bufferSize = bufferSize;
59 | this.inputStream = inputStream;
60 | }
61 |
62 | public void run() {
63 | BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(
64 | inputStream));
65 | String line;
66 | try {
67 | do {
68 | line = bufferedReader.readLine();
69 | if (line != null && collectedOutput.length() < bufferSize) {
70 | collectedOutput.append(line + "\n");
71 | }
72 | } while (line != null);
73 | } catch (IOException e) {
74 | e.printStackTrace();
75 | } finally {
76 | try {
77 | inputStream.close();
78 | } catch (IOException e) {
79 | e.printStackTrace();
80 | }
81 | }
82 | }
83 |
84 | private String getOutput() {
85 | return collectedOutput.toString();
86 | }
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/import_/ImportBulkListener.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.import_;
2 |
3 | import java.util.concurrent.ExecutionException;
4 | import java.util.concurrent.atomic.AtomicLong;
5 |
6 | import org.elasticsearch.action.bulk.BulkItemResponse;
7 | import org.elasticsearch.action.bulk.BulkProcessor;
8 | import org.elasticsearch.action.bulk.BulkRequest;
9 | import org.elasticsearch.action.bulk.BulkResponse;
10 | import org.elasticsearch.common.util.concurrent.BaseFuture;
11 |
12 | import crate.elasticsearch.import_.Importer.ImportCounts;
13 |
14 | public class ImportBulkListener extends BaseFuture implements BulkProcessor.Listener {
15 |
16 | private AtomicLong bulksInProgress = new AtomicLong();
17 | private ImportCounts counts = new ImportCounts();
18 |
19 | public ImportBulkListener(String fileName) {
20 | counts.fileName = fileName;
21 | }
22 |
23 | @Override
24 | public ImportBulkListener get() throws InterruptedException,
25 | ExecutionException {
26 | if (bulksInProgress.get() == 0) {
27 | return this;
28 | }
29 | return super.get();
30 | }
31 |
32 | public void addFailure() {
33 | counts.failures++;
34 | }
35 |
36 | public ImportCounts importCounts() {
37 | return counts;
38 | }
39 |
40 | @Override
41 | public void beforeBulk(long executionId, BulkRequest request) {
42 | bulksInProgress.incrementAndGet();
43 | }
44 |
45 | @Override
46 | public void afterBulk(long executionId, BulkRequest request,
47 | BulkResponse response) {
48 | bulksInProgress.decrementAndGet();
49 | if (response.hasFailures()) {
50 | for (BulkItemResponse item : response.getItems()) {
51 | if (item.isFailed()) {
52 | counts.failures++;
53 | } else {
54 | counts.successes++;
55 | }
56 | }
57 | } else {
58 | counts.successes += response.getItems().length;
59 | }
60 | checkRelease();
61 | }
62 |
63 | @Override
64 | public void afterBulk(long executionId, BulkRequest request,
65 | Throwable failure) {
66 | bulksInProgress.decrementAndGet();
67 | counts.failures += request.requests().size();
68 | failure.printStackTrace();
69 | checkRelease();
70 | }
71 |
72 | private void checkRelease() {
73 | if (bulksInProgress.get() == 0) {
74 | this.set(this);
75 | }
76 | }
77 |
78 | public void addInvalid() {
79 | counts.invalid++;
80 | }
81 |
82 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/dump/DumpModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.dump;
2 |
3 |
4 | import crate.elasticsearch.action.dump.DumpAction;
5 | import crate.elasticsearch.action.dump.TransportDumpAction;
6 | import crate.elasticsearch.action.dump.parser.DumpParser;
7 | import org.elasticsearch.action.GenericAction;
8 | import org.elasticsearch.action.support.TransportAction;
9 | import org.elasticsearch.common.inject.AbstractModule;
10 | import org.elasticsearch.common.inject.multibindings.MapBinder;
11 |
12 | public class DumpModule extends AbstractModule {
13 |
14 | @Override
15 | protected void configure() {
16 | bind(TransportDumpAction.class).asEagerSingleton();
17 |
18 | bind(DumpParser.class).asEagerSingleton();
19 |
20 | MapBinder transportActionsBinder = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
21 |
22 | transportActionsBinder.addBinding(DumpAction.INSTANCE).to(TransportDumpAction.class).asEagerSingleton();
23 |
24 | MapBinder actionsBinder = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
25 | actionsBinder.addBinding(DumpAction.NAME).toInstance(DumpAction.INSTANCE);
26 | }
27 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/export/ExportModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.export;
2 |
3 |
4 | import crate.elasticsearch.action.export.AbstractTransportExportAction;
5 | import crate.elasticsearch.action.export.TransportExportAction;
6 | import crate.elasticsearch.action.export.parser.IExportParser;
7 | import org.elasticsearch.action.GenericAction;
8 | import org.elasticsearch.action.support.TransportAction;
9 | import org.elasticsearch.common.inject.AbstractModule;
10 | import org.elasticsearch.common.inject.multibindings.MapBinder;
11 |
12 | import crate.elasticsearch.action.export.ExportAction;
13 | import crate.elasticsearch.action.export.parser.ExportParser;
14 | import crate.elasticsearch.export.Exporter;
15 |
16 | public class ExportModule extends AbstractModule {
17 |
18 | @Override
19 | protected void configure() {
20 | bind(TransportExportAction.class).asEagerSingleton();
21 |
22 | bind(ExportParser.class).asEagerSingleton();
23 | bind(Exporter.class).asEagerSingleton();
24 |
25 | MapBinder transportActionsBinder = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
26 |
27 | transportActionsBinder.addBinding(ExportAction.INSTANCE).to(TransportExportAction.class).asEagerSingleton();
28 |
29 | MapBinder actionsBinder = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
30 | actionsBinder.addBinding(ExportAction.NAME).toInstance(ExportAction.INSTANCE);
31 | }
32 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/import_/ImportModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.import_;
2 |
3 | import crate.elasticsearch.action.import_.ImportAction;
4 | import crate.elasticsearch.action.import_.TransportImportAction;
5 | import crate.elasticsearch.action.import_.parser.ImportParser;
6 | import crate.elasticsearch.import_.Importer;
7 | import org.elasticsearch.action.GenericAction;
8 | import org.elasticsearch.action.support.TransportAction;
9 | import org.elasticsearch.common.inject.AbstractModule;
10 | import org.elasticsearch.common.inject.multibindings.MapBinder;
11 |
12 | public class ImportModule extends AbstractModule {
13 |
14 | @Override
15 | protected void configure() {
16 | bind(TransportImportAction.class).asEagerSingleton();
17 |
18 | bind(ImportParser.class).asEagerSingleton();
19 | bind(Importer.class).asEagerSingleton();
20 |
21 | MapBinder transportActionsBinder = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
22 | transportActionsBinder.addBinding(ImportAction.INSTANCE).to(TransportImportAction.class).asEagerSingleton();
23 |
24 | MapBinder actionsBinder = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
25 | actionsBinder.addBinding(ImportAction.NAME).toInstance(ImportAction.INSTANCE);
26 |
27 | }
28 |
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/reindex/ReindexModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.reindex;
2 |
3 | import org.elasticsearch.action.GenericAction;
4 | import org.elasticsearch.action.support.TransportAction;
5 | import org.elasticsearch.common.inject.AbstractModule;
6 | import org.elasticsearch.common.inject.multibindings.MapBinder;
7 |
8 | import crate.elasticsearch.action.reindex.ReindexAction;
9 | import crate.elasticsearch.action.reindex.ReindexParser;
10 | import crate.elasticsearch.action.reindex.TransportReindexAction;
11 |
12 | public class ReindexModule extends AbstractModule {
13 |
14 | @Override
15 | protected void configure() {
16 | bind(TransportReindexAction.class).asEagerSingleton();
17 |
18 | bind(ReindexParser.class).asEagerSingleton();
19 |
20 | MapBinder transportActionsBinder =
21 | MapBinder.newMapBinder(
22 | binder(), GenericAction.class, TransportAction.class);
23 |
24 | transportActionsBinder.addBinding(ReindexAction.INSTANCE).to(
25 | TransportReindexAction.class).asEagerSingleton();
26 |
27 | MapBinder actionsBinder = MapBinder
28 | .newMapBinder(binder(), String.class, GenericAction.class);
29 | actionsBinder.addBinding(ReindexAction.NAME).toInstance(
30 | ReindexAction.INSTANCE);
31 |
32 | }
33 |
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/restore/RestoreModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.restore;
2 |
3 | import crate.elasticsearch.action.restore.RestoreAction;
4 | import crate.elasticsearch.action.restore.TransportRestoreAction;
5 | import crate.elasticsearch.action.restore.parser.RestoreParser;
6 | import org.elasticsearch.action.GenericAction;
7 | import org.elasticsearch.action.support.TransportAction;
8 | import org.elasticsearch.common.inject.AbstractModule;
9 | import org.elasticsearch.common.inject.multibindings.MapBinder;
10 |
11 | public class RestoreModule extends AbstractModule {
12 |
13 | @Override
14 | protected void configure() {
15 | bind(TransportRestoreAction.class).asEagerSingleton();
16 |
17 | bind(RestoreParser.class).asEagerSingleton();
18 |
19 | MapBinder transportActionsBinder = MapBinder.newMapBinder(binder(), GenericAction.class, TransportAction.class);
20 | transportActionsBinder.addBinding(RestoreAction.INSTANCE).to(TransportRestoreAction.class).asEagerSingleton();
21 |
22 | MapBinder actionsBinder = MapBinder.newMapBinder(binder(), String.class, GenericAction.class);
23 | actionsBinder.addBinding(RestoreAction.NAME).toInstance(RestoreAction.INSTANCE);
24 |
25 | }
26 |
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/module/searchinto/SearchIntoModule.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.searchinto;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoAction;
4 | import crate.elasticsearch.action.searchinto.TransportSearchIntoAction;
5 | import crate.elasticsearch.action.searchinto.parser.SearchIntoParser;
6 | import crate.elasticsearch.searchinto.BulkWriterCollector;
7 | import crate.elasticsearch.searchinto.WriterCollectorFactory;
8 | import org.elasticsearch.action.GenericAction;
9 | import org.elasticsearch.action.support.TransportAction;
10 | import org.elasticsearch.common.inject.AbstractModule;
11 | import org.elasticsearch.common.inject.assistedinject.FactoryProvider;
12 | import org.elasticsearch.common.inject.multibindings.MapBinder;
13 |
14 | public class SearchIntoModule extends AbstractModule {
15 |
16 |
17 | @Override
18 | protected void configure() {
19 | bind(TransportSearchIntoAction.class).asEagerSingleton();
20 |
21 | bind(SearchIntoParser.class).asEagerSingleton();
22 |
23 | MapBinder transportActionsBinder =
24 | MapBinder.newMapBinder(
25 | binder(), GenericAction.class, TransportAction.class);
26 |
27 | transportActionsBinder.addBinding(SearchIntoAction.INSTANCE).to(
28 | TransportSearchIntoAction.class).asEagerSingleton();
29 |
30 | MapBinder actionsBinder = MapBinder
31 | .newMapBinder(binder(), String.class, GenericAction.class);
32 | actionsBinder.addBinding(SearchIntoAction.NAME).toInstance(
33 | SearchIntoAction.INSTANCE);
34 |
35 |
36 | MapBinder collectorBinder
37 | = MapBinder.newMapBinder(binder(),
38 | String.class, WriterCollectorFactory.class);
39 |
40 | collectorBinder.addBinding(BulkWriterCollector.NAME).toProvider(
41 | FactoryProvider
42 | .newFactory(WriterCollectorFactory.class,
43 | BulkWriterCollector.class));
44 |
45 |
46 | }
47 | }
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/plugin/inout/InOutPlugin.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.plugin.inout;
2 |
3 | import java.util.Collection;
4 |
5 | import org.elasticsearch.common.collect.Lists;
6 | import org.elasticsearch.common.inject.Module;
7 | import org.elasticsearch.common.settings.Settings;
8 | import org.elasticsearch.plugins.AbstractPlugin;
9 | import org.elasticsearch.rest.RestModule;
10 |
11 | import crate.elasticsearch.module.dump.DumpModule;
12 | import crate.elasticsearch.module.export.ExportModule;
13 | import crate.elasticsearch.module.import_.ImportModule;
14 | import crate.elasticsearch.module.reindex.ReindexModule;
15 | import crate.elasticsearch.module.restore.RestoreModule;
16 | import crate.elasticsearch.module.searchinto.SearchIntoModule;
17 | import crate.elasticsearch.rest.action.admin.dump.RestDumpAction;
18 | import crate.elasticsearch.rest.action.admin.export.RestExportAction;
19 | import crate.elasticsearch.rest.action.admin.import_.RestImportAction;
20 | import crate.elasticsearch.rest.action.admin.reindex.RestReindexAction;
21 | import crate.elasticsearch.rest.action.admin.restore.RestRestoreAction;
22 | import crate.elasticsearch.rest.action.admin.searchinto.RestSearchIntoAction;
23 |
24 | public class InOutPlugin extends AbstractPlugin {
25 |
26 | private final Settings settings;
27 |
28 | public InOutPlugin(Settings settings) {
29 | this.settings = settings;
30 | }
31 |
32 | public String name() {
33 | return "inout";
34 | }
35 |
36 | public String description() {
37 | return "InOut plugin";
38 | }
39 |
40 | public void onModule(RestModule restModule) {
41 | restModule.addRestAction(RestExportAction.class);
42 | restModule.addRestAction(RestImportAction.class);
43 | restModule.addRestAction(RestSearchIntoAction.class);
44 | restModule.addRestAction(RestDumpAction.class);
45 | restModule.addRestAction(RestRestoreAction.class);
46 | restModule.addRestAction(RestReindexAction.class);
47 | }
48 |
49 | @Override
50 | public Collection> modules() {
51 | Collection> modules = Lists.newArrayList();
52 | if (!settings.getAsBoolean("node.client", false)) {
53 | modules.add(ExportModule.class);
54 | modules.add(ImportModule.class);
55 | modules.add(SearchIntoModule.class);
56 | modules.add(DumpModule.class);
57 | modules.add(RestoreModule.class);
58 | modules.add(ReindexModule.class);
59 | }
60 | return modules;
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/rest/action/admin/dump/RestDumpAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.rest.action.admin.dump;
2 |
3 | import static org.elasticsearch.rest.RestRequest.Method.POST;
4 |
5 | import org.elasticsearch.action.Action;
6 | import org.elasticsearch.client.Client;
7 | import org.elasticsearch.common.inject.Inject;
8 | import org.elasticsearch.common.settings.Settings;
9 | import org.elasticsearch.rest.RestController;
10 |
11 | import crate.elasticsearch.action.dump.DumpAction;
12 | import crate.elasticsearch.action.export.ExportRequest;
13 | import crate.elasticsearch.action.export.ExportResponse;
14 | import crate.elasticsearch.client.action.export.ExportRequestBuilder;
15 | import crate.elasticsearch.rest.action.admin.export.RestExportAction;
16 |
17 | /**
18 | * Rest handler for _dump endpoint
19 | */
20 | public class RestDumpAction extends RestExportAction {
21 |
22 | @Inject
23 | public RestDumpAction(Settings settings, Client client, RestController controller) {
24 | super(settings, client, controller);
25 | }
26 |
27 | @Override
28 | protected Action action() {
29 | return DumpAction.INSTANCE;
30 | }
31 |
32 | @Override
33 | protected void registerHandlers(RestController controller) {
34 | controller.registerHandler(POST, "/_dump", this);
35 | controller.registerHandler(POST, "/{index}/_dump", this);
36 | controller.registerHandler(POST, "/{index}/{type}/_dump", this);
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/rest/action/admin/export/RestExportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.rest.action.admin.export;
2 |
3 | import static org.elasticsearch.rest.RestRequest.Method.POST;
4 | import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
5 | import static org.elasticsearch.rest.RestStatus.OK;
6 | import static org.elasticsearch.rest.action.support.RestActions.splitTypes;
7 |
8 | import java.io.IOException;
9 |
10 | import org.elasticsearch.action.Action;
11 | import org.elasticsearch.action.ActionListener;
12 | import org.elasticsearch.action.support.IgnoreIndices;
13 | import org.elasticsearch.action.support.broadcast.BroadcastOperationThreading;
14 | import org.elasticsearch.client.Client;
15 | import org.elasticsearch.common.bytes.BytesReference;
16 | import org.elasticsearch.common.inject.Inject;
17 | import org.elasticsearch.common.settings.Settings;
18 | import org.elasticsearch.common.xcontent.XContentBuilder;
19 | import org.elasticsearch.rest.BaseRestHandler;
20 | import org.elasticsearch.rest.RestChannel;
21 | import org.elasticsearch.rest.RestController;
22 | import org.elasticsearch.rest.RestRequest;
23 | import org.elasticsearch.rest.XContentRestResponse;
24 | import org.elasticsearch.rest.XContentThrowableRestResponse;
25 | import org.elasticsearch.rest.action.support.RestActions;
26 | import org.elasticsearch.rest.action.support.RestXContentBuilder;
27 |
28 | import crate.elasticsearch.action.export.ExportAction;
29 | import crate.elasticsearch.action.export.ExportRequest;
30 | import crate.elasticsearch.action.export.ExportResponse;
31 | import crate.elasticsearch.client.action.export.ExportRequestBuilder;
32 |
33 | /**
34 | *
35 | */
36 | public class RestExportAction extends BaseRestHandler {
37 |
38 | @Inject
39 | public RestExportAction(Settings settings, Client client, RestController controller) {
40 | super(settings, client);
41 | registerHandlers(controller);
42 | }
43 |
44 | protected void registerHandlers(RestController controller) {
45 | controller.registerHandler(POST, "/_export", this);
46 | controller.registerHandler(POST, "/{index}/_export", this);
47 | controller.registerHandler(POST, "/{index}/{type}/_export", this);
48 | }
49 |
50 | protected Action action() {
51 | return ExportAction.INSTANCE;
52 | }
53 |
54 | public void handleRequest(final RestRequest request, final RestChannel channel) {
55 | ExportRequest exportRequest = new ExportRequest(RestActions.splitIndices(request.param("index")));
56 |
57 | if (request.hasParam("ignore_indices")) {
58 | exportRequest.ignoreIndices(IgnoreIndices.fromString(request.param("ignore_indices")));
59 | }
60 | exportRequest.listenerThreaded(false);
61 | try {
62 | BroadcastOperationThreading operationThreading = BroadcastOperationThreading.fromString(request.param("operation_threading"), BroadcastOperationThreading.SINGLE_THREAD);
63 | if (operationThreading == BroadcastOperationThreading.NO_THREADS) {
64 | // since we don't spawn, don't allow no_threads, but change it to a single thread
65 | operationThreading = BroadcastOperationThreading.SINGLE_THREAD;
66 | }
67 | exportRequest.operationThreading(operationThreading);
68 | if (request.hasContent()) {
69 | exportRequest.source(request.content(), request.contentUnsafe());
70 | } else {
71 | String source = request.param("source");
72 | if (source != null) {
73 | exportRequest.source(source);
74 | } else {
75 | BytesReference querySource = RestActions.parseQuerySource(request);
76 | if (querySource != null) {
77 | exportRequest.source(querySource, false);
78 | }
79 | }
80 | }
81 | exportRequest.routing(request.param("routing"));
82 | exportRequest.types(splitTypes(request.param("type")));
83 | exportRequest.preference(request.param("preference", "_primary"));
84 | } catch (Exception e) {
85 | try {
86 | XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
87 | channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject()));
88 | } catch (IOException e1) {
89 | logger.error("Failed to send failure response", e1);
90 | }
91 | return;
92 | }
93 |
94 | client.execute(action(), exportRequest, new ActionListener() {
95 |
96 | public void onResponse(ExportResponse response) {
97 | try {
98 | XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
99 | response.toXContent(builder, request);
100 | channel.sendResponse(new XContentRestResponse(request, OK, builder));
101 | } catch (Exception e) {
102 | onFailure(e);
103 | }
104 | }
105 |
106 | public void onFailure(Throwable e) {
107 | try {
108 | channel.sendResponse(new XContentThrowableRestResponse(request, e));
109 | } catch (IOException e1) {
110 | logger.error("Failed to send failure response", e1);
111 | }
112 | }
113 | });
114 |
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/rest/action/admin/import_/RestImportAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.rest.action.admin.import_;
2 |
3 | import crate.elasticsearch.action.import_.ImportAction;
4 | import crate.elasticsearch.action.import_.ImportRequest;
5 | import crate.elasticsearch.action.import_.ImportResponse;
6 | import org.elasticsearch.action.Action;
7 | import org.elasticsearch.action.ActionListener;
8 | import org.elasticsearch.client.Client;
9 | import org.elasticsearch.common.bytes.BytesReference;
10 | import org.elasticsearch.common.inject.Inject;
11 | import org.elasticsearch.common.settings.Settings;
12 | import org.elasticsearch.common.xcontent.XContentBuilder;
13 | import org.elasticsearch.rest.*;
14 | import org.elasticsearch.rest.action.support.RestActions;
15 | import org.elasticsearch.rest.action.support.RestXContentBuilder;
16 |
17 | import java.io.IOException;
18 |
19 | import static org.elasticsearch.rest.RestRequest.Method.POST;
20 | import static org.elasticsearch.rest.RestStatus.BAD_REQUEST;
21 | import static org.elasticsearch.rest.RestStatus.OK;
22 |
23 | public class RestImportAction extends BaseRestHandler {
24 |
25 | @Inject
26 | public RestImportAction(Settings settings, Client client, RestController controller) {
27 | super(settings, client);
28 | registerHandlers(controller);
29 | }
30 |
31 | protected void registerHandlers(RestController controller) {
32 | controller.registerHandler(POST, "/_import", this);
33 | controller.registerHandler(POST, "/{index}/_import", this);
34 | controller.registerHandler(POST, "/{index}/{type}/_import", this);
35 | }
36 |
37 |
38 | protected Action action() {
39 | return ImportAction.INSTANCE;
40 | }
41 |
42 | public void handleRequest(final RestRequest request, final RestChannel channel) {
43 | ImportRequest importRequest = new ImportRequest();
44 | importRequest.listenerThreaded(false);
45 | try {
46 | if (request.hasContent()) {
47 | importRequest.source(request.content(), request.contentUnsafe());
48 | } else {
49 | String source = request.param("source");
50 | if (source != null) {
51 | importRequest.source(source);
52 | } else {
53 | BytesReference querySource = RestActions.parseQuerySource(request);
54 | if (querySource != null) {
55 | importRequest.source(querySource, false);
56 | }
57 | }
58 | }
59 | importRequest.index(request.param("index"));
60 | importRequest.type(request.param("type"));
61 | } catch (Exception e) {
62 | try {
63 | XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
64 | channel.sendResponse(new XContentRestResponse(request, BAD_REQUEST, builder.startObject().field("error", e.getMessage()).endObject()));
65 | } catch (IOException e1) {
66 | logger.error("Failed to send failure response", e1);
67 | }
68 | return;
69 | }
70 |
71 |
72 | client.execute(action(), importRequest, new ActionListener() {
73 |
74 | public void onResponse(ImportResponse response) {
75 | try {
76 | XContentBuilder builder = RestXContentBuilder.restContentBuilder(request);
77 | response.toXContent(builder, request);
78 | channel.sendResponse(new XContentRestResponse(request, OK, builder));
79 | } catch (Exception e) {
80 | onFailure(e);
81 | }
82 | }
83 |
84 | public void onFailure(Throwable e) {
85 | try {
86 | channel.sendResponse(new XContentThrowableRestResponse(request, e));
87 | } catch (IOException e1) {
88 | logger.error("Failed to send failure response", e1);
89 | }
90 | }
91 | });
92 | }
93 | }
94 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/rest/action/admin/reindex/RestReindexAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.rest.action.admin.reindex;
2 |
3 | import static org.elasticsearch.rest.RestRequest.Method.POST;
4 |
5 | import org.elasticsearch.action.Action;
6 | import org.elasticsearch.client.Client;
7 | import org.elasticsearch.common.inject.Inject;
8 | import org.elasticsearch.common.settings.Settings;
9 | import org.elasticsearch.rest.RestController;
10 |
11 | import crate.elasticsearch.action.reindex.ReindexAction;
12 | import crate.elasticsearch.action.searchinto.SearchIntoRequest;
13 | import crate.elasticsearch.action.searchinto.SearchIntoResponse;
14 | import crate.elasticsearch.client.action.searchinto.SearchIntoRequestBuilder;
15 | import crate.elasticsearch.rest.action.admin.searchinto.RestSearchIntoAction;
16 |
17 | /**
18 | * Rest action for the _reindex end points. Does the _search_into action to
19 | * the own index with the defined fields _id and _source.
20 | *
21 | * Does a re-index to either all indexes, a specified index or a specific type of
22 | * a specified index.
23 | */
24 | public class RestReindexAction extends RestSearchIntoAction {
25 |
26 | @Inject
27 | public RestReindexAction(Settings settings, Client client, RestController controller) {
28 | super(settings, client, controller);
29 | }
30 |
31 | @Override
32 | protected void registerHandlers(RestController controller) {
33 | controller.registerHandler(POST, "/_reindex", this);
34 | controller.registerHandler(POST, "/{index}/_reindex", this);
35 | controller.registerHandler(POST, "/{index}/{type}/_reindex", this);
36 | }
37 |
38 | @Override
39 | protected Action action() {
40 | return ReindexAction.INSTANCE;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/rest/action/admin/restore/RestRestoreAction.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.rest.action.admin.restore;
2 |
3 | import crate.elasticsearch.action.dump.DumpAction;
4 | import crate.elasticsearch.action.restore.RestoreAction;
5 | import crate.elasticsearch.rest.action.admin.export.RestExportAction;
6 | import crate.elasticsearch.rest.action.admin.import_.RestImportAction;
7 | import org.elasticsearch.action.Action;
8 | import org.elasticsearch.client.Client;
9 | import org.elasticsearch.common.inject.Inject;
10 | import org.elasticsearch.common.settings.Settings;
11 | import org.elasticsearch.rest.RestController;
12 |
13 | import static org.elasticsearch.rest.RestRequest.Method.POST;
14 |
15 | /**
16 | * Rest handler for _restore endpoint
17 | */
18 | public class RestRestoreAction extends RestImportAction {
19 |
20 | @Inject
21 | public RestRestoreAction(Settings settings, Client client, RestController controller) {
22 | super(settings, client, controller);
23 | }
24 |
25 | @Override
26 | protected Action action() {
27 | return RestoreAction.INSTANCE;
28 | }
29 |
30 | @Override
31 | protected void registerHandlers(RestController controller) {
32 | controller.registerHandler(POST, "/_restore", this);
33 | controller.registerHandler(POST, "/{index}/_restore", this);
34 | controller.registerHandler(POST, "/{index}/{type}/_restore", this);
35 | }
36 |
37 | }
38 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/Writer.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
4 | import org.apache.lucene.search.Query;
5 | import org.elasticsearch.common.inject.Inject;
6 | import org.elasticsearch.common.logging.ESLogger;
7 | import org.elasticsearch.common.logging.Loggers;
8 |
9 | import java.io.IOException;
10 | import java.util.Map;
11 |
12 | public class Writer {
13 |
14 | private static final ESLogger logger = Loggers.getLogger(Writer.class);
15 |
16 | private final Map collectors;
17 |
18 | @Inject
19 | public Writer(Map collectors) {
20 | this.collectors = collectors;
21 | }
22 |
23 |
24 | public WriterResult execute(SearchIntoContext context) throws
25 | WriterException {
26 | logger.info("writing {}/{}", context.shardTarget().index(),
27 | context.shardTarget().getShardId());
28 | Query query = context.query();
29 | context.version(true);
30 | WriterCollector wc = collectors.get(context.targetType()).create(
31 | context);
32 | wc.open();
33 | try {
34 | context.searcher().search(query, wc);
35 | } catch (IOException e) {
36 | throw new WriterException(context, "Failed to write docs", e);
37 | }
38 | wc.close();
39 | WriterResult res = wc.getResult();
40 | logger.info("exported {} docs from {}/{}", res.getTotalWrites(),
41 | context.shardTarget().index(),
42 | context.shardTarget().getShardId());
43 | return res;
44 |
45 |
46 | }
47 |
48 | }
49 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/WriterCollectorFactory.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
4 |
5 | public interface WriterCollectorFactory {
6 |
7 | WriterCollector create(SearchIntoContext context);
8 |
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/WriterException.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto;
2 |
3 | import org.elasticsearch.search.SearchContextException;
4 | import org.elasticsearch.search.internal.SearchContext;
5 |
6 | public class WriterException extends SearchContextException {
7 |
8 | public WriterException(SearchContext context, String msg) {
9 | super(context, msg);
10 | }
11 |
12 | public WriterException(SearchContext context, String msg, Throwable t) {
13 | super(context, "Write Failed [" + msg + "]", t);
14 | }
15 | }
16 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/WriterResult.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto;
2 |
3 | import org.elasticsearch.common.io.stream.StreamInput;
4 | import org.elasticsearch.common.io.stream.StreamOutput;
5 | import org.elasticsearch.common.io.stream.Streamable;
6 | import org.elasticsearch.common.xcontent.ToXContent;
7 | import org.elasticsearch.common.xcontent.XContentBuilder;
8 |
9 | import java.io.IOException;
10 |
11 | public class WriterResult implements ToXContent, Streamable {
12 |
13 | private long totalWrites;
14 | private long failedWrites;
15 | private long succeededWrites;
16 |
17 | public void setTotalWrites(long totalWrites) {
18 | this.totalWrites = totalWrites;
19 | }
20 |
21 | public void setFailedWrites(long failedWrites) {
22 | this.failedWrites = failedWrites;
23 | }
24 |
25 | public void setSucceededWrites(long succeededWrites) {
26 | this.succeededWrites = succeededWrites;
27 | }
28 |
29 | public long getTotalWrites() {
30 | return totalWrites;
31 | }
32 |
33 | public long getFailedWrites() {
34 | return failedWrites;
35 | }
36 |
37 | public long getSucceededWrites() {
38 | return succeededWrites;
39 | }
40 |
41 | @Override
42 | public void readFrom(StreamInput in) throws IOException {
43 | totalWrites = in.readVLong();
44 | succeededWrites = in.readVLong();
45 | failedWrites = in.readVLong();
46 | }
47 |
48 | @Override
49 | public void writeTo(StreamOutput out) throws IOException {
50 | out.writeVLong(totalWrites);
51 | out.writeVLong(succeededWrites);
52 | out.writeVLong(failedWrites);
53 | }
54 |
55 | @Override
56 | public XContentBuilder toXContent(XContentBuilder builder,
57 | Params params) throws IOException {
58 | builder.field("total", totalWrites);
59 | builder.field("succeeded", succeededWrites);
60 | builder.field("failed", failedWrites);
61 | return builder;
62 | }
63 |
64 | public static WriterResult readNew(StreamInput in) throws IOException {
65 | WriterResult r = new WriterResult();
66 | r.readFrom(in);
67 | return r;
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/mapping/FieldReader.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto.mapping;
2 |
3 | import org.elasticsearch.common.collect.ImmutableMap;
4 | import org.elasticsearch.common.collect.MapBuilder;
5 | import org.elasticsearch.index.mapper.internal.*;
6 | import org.elasticsearch.search.SearchHit;
7 | import org.elasticsearch.search.SearchHitField;
8 |
9 | import java.util.Map;
10 |
11 | public class FieldReader {
12 |
13 | private final String name;
14 | private HitReader reader;
15 | private SearchHit hit;
16 |
17 | private final static ImmutableMap readers;
18 |
19 | static {
20 | readers = MapBuilder.newMapBuilder().put(
21 | SourceFieldMapper.NAME, new HitReader>() {
22 | @Override
23 | public Map read(SearchHit hit) {
24 | return hit.getSource();
25 | }
26 | }).put(IndexFieldMapper.NAME, new HitReader() {
27 | @Override
28 | public String read(SearchHit hit) {
29 | return hit.index();
30 | }
31 | }).put(TypeFieldMapper.NAME, new HitReader() {
32 | @Override
33 | public String read(SearchHit hit) {
34 | return hit.type();
35 | }
36 | }).put(IdFieldMapper.NAME, new HitReader() {
37 | @Override
38 | public String read(SearchHit hit) {
39 | return hit.id();
40 | }
41 | }).put(TimestampFieldMapper.NAME, new HitReader() {
42 | @Override
43 | public Long read(SearchHit hit) {
44 | SearchHitField field = hit.getFields().get(
45 | TimestampFieldMapper.NAME);
46 | if (field != null && !field.values().isEmpty()) {
47 | return field.value();
48 | }
49 | return null;
50 | }
51 | }).put(TTLFieldMapper.NAME, new HitReader() {
52 | @Override
53 | public Long read(SearchHit hit) {
54 | SearchHitField field = hit.getFields().get(
55 | TTLFieldMapper.NAME);
56 | if (field != null && !field.values().isEmpty()) {
57 | return field.value();
58 | }
59 | return null;
60 | }
61 | }).put("_version", new HitReader() {
62 | @Override
63 | public Long read(SearchHit hit) {
64 | return hit.getVersion();
65 | }
66 | }).immutableMap();
67 | }
68 |
69 | static abstract class HitReader {
70 | public abstract T read(SearchHit hit);
71 | }
72 |
73 | static class HitFieldReader extends HitReader {
74 |
75 | private final String name;
76 |
77 | HitFieldReader(String name) {
78 | this.name = name;
79 | }
80 |
81 | @Override
82 | public Object read(SearchHit hit) {
83 | SearchHitField field = hit.getFields().get(name);
84 | if (field != null && !field.values().isEmpty()) {
85 | if (field.values().size() == 1) {
86 | return field.values().get(0);
87 | } else {
88 | return field.values();
89 | }
90 | }
91 | return null;
92 | }
93 | }
94 |
95 | public FieldReader(String name) {
96 | this.name = name;
97 | initReader();
98 | }
99 |
100 |
101 | private void initReader() {
102 | if (name.startsWith("_")) {
103 | reader = readers.get(name);
104 | }
105 | if (reader == null) {
106 | reader = new HitFieldReader(name);
107 | }
108 | }
109 |
110 | public void setHit(SearchHit hit) {
111 | this.hit = hit;
112 | }
113 |
114 | public Object getValue() {
115 | return reader.read(hit);
116 | }
117 |
118 | }
119 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/mapping/FieldWriter.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto.mapping;
2 |
3 | import java.util.HashMap;
4 | import java.util.Map;
5 |
6 | import org.elasticsearch.ElasticSearchException;
7 | import org.elasticsearch.common.collect.ImmutableMap;
8 | import org.elasticsearch.common.collect.MapBuilder;
9 | import org.elasticsearch.index.mapper.internal.IdFieldMapper;
10 | import org.elasticsearch.index.mapper.internal.IndexFieldMapper;
11 | import org.elasticsearch.index.mapper.internal.SourceFieldMapper;
12 | import org.elasticsearch.index.mapper.internal.TTLFieldMapper;
13 | import org.elasticsearch.index.mapper.internal.TimestampFieldMapper;
14 | import org.elasticsearch.index.mapper.internal.TypeFieldMapper;
15 |
16 | public class FieldWriter {
17 |
18 | private final String name;
19 | protected Object value;
20 | private BuilderWriter writer;
21 |
22 | private final static ImmutableMap writers;
23 |
24 | static {
25 | writers = MapBuilder.newMapBuilder().put(
26 | SourceFieldMapper.NAME, new BuilderWriter() {
27 | @Override
28 | public void write(IndexRequestBuilder builder, Object value) {
29 | builder.source = (Map) value;
30 | }
31 | }).put(IndexFieldMapper.NAME, new BuilderWriter() {
32 | @Override
33 | public void write(IndexRequestBuilder builder, Object value) {
34 | builder.request.index(value.toString());
35 | }
36 | }).put(IdFieldMapper.NAME, new BuilderWriter() {
37 | @Override
38 | public void write(IndexRequestBuilder builder, Object value) {
39 | builder.request.id(value.toString());
40 | }
41 | }).put(TypeFieldMapper.NAME, new BuilderWriter() {
42 | @Override
43 | public void write(IndexRequestBuilder builder, Object value) {
44 | builder.request.type(value.toString());
45 | }
46 | }).put(TimestampFieldMapper.NAME, new BuilderWriter() {
47 | @Override
48 | public void write(IndexRequestBuilder builder, Object value) {
49 | builder.request.timestamp(value.toString());
50 | }
51 | }).put(TTLFieldMapper.NAME, new BuilderWriter() {
52 | @Override
53 | public void write(IndexRequestBuilder builder, Object value) {
54 | builder.request.ttl((Long) value);
55 | }
56 | }).put("_version", new BuilderWriter() {
57 | @Override
58 | public void write(IndexRequestBuilder builder, Object value) {
59 | builder.request.version((Long) value);
60 | }
61 | }).immutableMap();
62 | }
63 |
64 | static abstract class BuilderWriter {
65 | public abstract void write(IndexRequestBuilder builder, Object value);
66 | }
67 |
68 | class SourceObjectWriter extends BuilderWriter {
69 |
70 | private final String name;
71 |
72 | SourceObjectWriter(String name) {
73 | this.name = name;
74 | }
75 |
76 | /**
77 | * Method to recursively create a nested object
78 | */
79 | private void writeMap(Map root, Object value, String part) {
80 | if (part.contains(".")) {
81 | String[] parts = part.split("\\.", 2);
82 | Object o = root.get(parts[0]);
83 | if (o == null) {
84 | o = new HashMap();
85 | } else if (!(o instanceof Map)) {
86 | throw new ElasticSearchException("Error on rewriting objects: Mixed objects and values");
87 | }
88 | Map sub = (Map) o;
89 | writeMap(sub, value, parts[1]);
90 | root.put(parts[0], sub);
91 | } else {
92 | if (((Map) root).get(part) instanceof Map) {
93 | throw new ElasticSearchException("Error on rewriting objects: Mixed objects and values");
94 | }
95 | root.put(part, value);
96 | }
97 | }
98 |
99 | @Override
100 | public void write(IndexRequestBuilder builder, Object value) {
101 | if (value != null) {
102 | writeMap(builder.source, value, name);
103 | }
104 | }
105 | }
106 |
107 | public FieldWriter(String name) {
108 | this.name = name;
109 | initWriter();
110 | }
111 |
112 | private void initWriter() {
113 | if (name.startsWith("_")) {
114 | writer = writers.get(name);
115 | }
116 | if (writer == null) {
117 | writer = new SourceObjectWriter(name);
118 | }
119 | }
120 |
121 | public void setValue(Object value) {
122 | this.value = value;
123 | }
124 |
125 | public IndexRequestBuilder toRequestBuilder(IndexRequestBuilder builder) {
126 | writer.write(builder, value);
127 | return builder;
128 | }
129 |
130 |
131 | }
132 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/mapping/IndexRequestBuilder.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto.mapping;
2 |
3 | import org.elasticsearch.action.index.IndexRequest;
4 |
5 | import java.util.HashMap;
6 | import java.util.Map;
7 |
8 |
9 | public class IndexRequestBuilder {
10 |
11 | Map source = new HashMap();
12 | final IndexRequest request = new IndexRequest();
13 |
14 | final Map meta = new HashMap();
15 |
16 | public IndexRequest build() {
17 | request.source(source);
18 | return request;
19 | }
20 |
21 | }
22 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/mapping/MappedFields.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto.mapping;
2 |
3 | import crate.elasticsearch.action.searchinto.SearchIntoContext;
4 | import org.elasticsearch.action.index.IndexRequest;
5 | import org.elasticsearch.search.SearchHit;
6 |
7 | import java.util.ArrayList;
8 | import java.util.List;
9 | import java.util.Map;
10 |
11 | public class MappedFields {
12 |
13 | private final SearchIntoContext context;
14 | private SearchHit hit;
15 | private final List outputMappings;
16 |
17 | public MappedFields(SearchIntoContext context) {
18 | this.context = context;
19 | this.outputMappings = getOutputMappings();
20 | }
21 |
22 | public void hit(SearchHit hit) {
23 | this.hit = hit;
24 | }
25 |
26 | private List getOutputMappings() {
27 | List oms = new ArrayList(
28 | context.outputNames().size());
29 | boolean indexDefined = false;
30 | boolean typeDefined = false;
31 | for (Map.Entry e : context.outputNames().entrySet()) {
32 | String srcName = e.getKey();
33 | String trgName = e.getValue();
34 | assert (trgName != null);
35 | if (trgName.equals("_index")) {
36 | indexDefined = true;
37 | } else if (trgName.equals("_type")) {
38 | typeDefined = true;
39 | }
40 | OutputMapping om = new OutputMapping(srcName, trgName);
41 | oms.add(om);
42 | }
43 | if (!indexDefined) {
44 | oms.add(new OutputMapping("_index", "_index"));
45 | }
46 | if (!typeDefined) {
47 | oms.add(new OutputMapping("_type", "_type"));
48 | }
49 |
50 | return oms;
51 | }
52 |
53 | public IndexRequest newIndexRequest() {
54 | IndexRequestBuilder builder = new IndexRequestBuilder();
55 | for (OutputMapping om : outputMappings) {
56 | om.setHit(hit);
57 | builder = om.toRequestBuilder(builder);
58 | }
59 | return builder.build();
60 |
61 | }
62 | }
63 |
64 |
--------------------------------------------------------------------------------
/src/main/java/crate/elasticsearch/searchinto/mapping/OutputMapping.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.searchinto.mapping;
2 |
3 | import org.elasticsearch.search.SearchHit;
4 |
5 | public class OutputMapping {
6 |
7 | private final String srcName;
8 | private final String trgName;
9 | private final String srcLiteral;
10 | private final FieldReader reader;
11 | private final FieldWriter writer;
12 | private SearchHit hit;
13 |
14 |
15 | private static String getLiteral(String candidate) {
16 | if (candidate != null && candidate.length() > 2) {
17 | if ((candidate.startsWith("'") && candidate.endsWith(
18 | "'")) || (candidate.startsWith("\"") && candidate.endsWith(
19 | "\""))) {
20 | return candidate.substring(1, candidate.length() - 1);
21 | }
22 | }
23 | return null;
24 | }
25 |
26 | public OutputMapping(String srcName, String trgName) {
27 | this.srcName = srcName;
28 | this.trgName = trgName;
29 | srcLiteral = getLiteral(srcName);
30 | if (srcLiteral == null) {
31 | this.reader = new FieldReader(srcName);
32 | } else {
33 | this.reader = null;
34 | }
35 | this.writer = new FieldWriter(trgName);
36 | }
37 |
38 | public void setHit(SearchHit hit) {
39 | this.hit = hit;
40 | }
41 |
42 | public IndexRequestBuilder toRequestBuilder(IndexRequestBuilder builder) {
43 | if (srcLiteral != null) {
44 | writer.setValue(srcLiteral);
45 | } else {
46 | reader.setHit(hit);
47 | writer.setValue(reader.getValue());
48 |
49 | }
50 | writer.toRequestBuilder(builder);
51 | return builder;
52 | }
53 |
54 | }
55 |
--------------------------------------------------------------------------------
/src/main/resources/es-plugin.properties:
--------------------------------------------------------------------------------
1 | plugin=crate.elasticsearch.plugin.inout.InOutPlugin
--------------------------------------------------------------------------------
/src/main/resources/essetup/data/test_a.json:
--------------------------------------------------------------------------------
1 | { "index" : { "_index" : "users", "_type" : "d", "_id" : "1" } }
2 | { "name": "car"}
3 | { "index" : { "_index" : "users", "_type" : "d", "_id" : "2" } }
4 | { "name": "bike"}
5 | { "index" : { "_index" : "users", "_type" : "d", "_id" : "3" } }
6 | { "name": "train"}
7 | { "index" : { "_index" : "users", "_type" : "d", "_id" : "4" } }
8 | { "name": "bus"}
9 |
--------------------------------------------------------------------------------
/src/main/resources/essetup/mappings/test_a.json:
--------------------------------------------------------------------------------
1 | {
2 | "d": {
3 | "properties": {
4 | "name": {
5 | "type": "string",
6 | "index": "not_analyzed",
7 | "store": true
8 | }
9 | }
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/src/main/resources/essetup/settings/test_a.json:
--------------------------------------------------------------------------------
1 | {
2 | "index": {
3 | "number_of_shards": 2,
4 | "number_of_replicas": 0
5 | }
6 | }
7 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_1/import_1.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"102","_source":{"name":"102"},"_index":"test","_type":"d"}
2 | {"_version":1,"_id":"103","_source":{"name":"103"},"_index":"test","_type":"d"}
3 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_2/import_2.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"202","_source":{"name":"202"},"_index":"test","_type":"d"}
2 | {"_version":1,"_id":"203","_source":{"name":"203"},"_index":"test"}
3 | {"_version":1,"_id":"204","_source":{"name":"204"},"_type":"d"}
4 | {"_version":1,"_id":"205","_source":{"name":"205"}}
5 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_3/import_3.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"302","_source":{"name":"302"},"_index":"test","_type":"d"}
2 |
3 | {"_version":1,"_id":"303","_source":{"name":"303"},"_index":"test","_type":"d"}
4 | {"_version":1,"_id":"304","_source":{"name":"304
5 | {"_version":1,"_id":"306","_source":{"name":"305"},"_index":"test","_type":"d"}
6 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_4/import_4.json:
--------------------------------------------------------------------------------
1 | {"_version":7,"_id":"402","_source":{"name":"402","otherField":7.5},"_index":"test","_type":"d","_routing":"the_routing","_timestamp":1367329785380,"_ttl":1867329687097}
2 | {"_version":1,"_id":"403","_source":{"name":"403","otherField":6.5},"_index":"test","_type":"d","_ttl":1867329687097}
3 | {"_version":1,"_id":"404","_source":{"name":"404","otherField":5.5},"_index":"test","_type":"d","_ttl":867329687097}
4 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_5/import_5_a.json:
--------------------------------------------------------------------------------
1 | {"_id":"501","_source":{"name":"501"},"_index":"test","_type":"d"}
2 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_5/import_5_b.json:
--------------------------------------------------------------------------------
1 | {"_id":"511","_source":{"name":"511"},"_index":"test","_type":"d"}
2 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_6/import_6.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"501","_source":{"name":"501-a"},"_index":"test","_type":"d"}
2 | {"_version":1,"_id":"501","_source":{"name":"501-b"},"_index":"test","_type":"d"}
3 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_7/import_7.json.gz:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/crate/elasticsearch-inout-plugin/a3bb7f6eeb62793c44e97dde666a1bf413c9daa9/src/main/resources/importdata/import_7/import_7.json.gz
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_8/index_test_1.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"802","_source":{"name":"802"},"_index":"test","_type":"d"}
2 | {"_version":1,"_id":"803","_source":{"name":"803"},"_index":"test","_type":"d"}
3 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_8/index_users_1.json:
--------------------------------------------------------------------------------
1 | {"_version":1,"_id":"811","_source":{"name":"811"},"_index":"users","_type":"d"}
2 | {"_version":1,"_id":"812","_source":{"name":"812"},"_index":"users","_type":"d"}
3 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_8/mapping_1.json:
--------------------------------------------------------------------------------
1 | {"test":{"d":{"properties":{"name":{"type":"string"}}}}}
2 | {"users":{"d":{"properties":{"name":{"type":"string"}}}}}
3 |
4 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_1.json:
--------------------------------------------------------------------------------
1 | {"_id":"1","_index":"index1","_type":"a","_source":{"name":"one"}}
2 | {"_id":"2","_index":"index1","_type":"b","_source":{"name":"two"}}
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_1.json.mapping:
--------------------------------------------------------------------------------
1 | {
2 | "index1" : {
3 | "1" : {
4 | "_timestamp" : {
5 | "enabled" : true,
6 | "store" : true
7 | },
8 | "_ttl" : {
9 | "enabled" : true,
10 | "default" : 86400000
11 | },
12 | "properties" : {
13 | "name" : {
14 | "type" : "string",
15 | "store" : true
16 | }
17 | }
18 | },
19 | "2" : {
20 | "properties" : {
21 | "name" : {
22 | "type" : "string"
23 | }
24 | }
25 | }
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_1.json.settings:
--------------------------------------------------------------------------------
1 | {
2 | "index1" : {
3 | "settings" : {
4 | "index.number_of_replicas" : "1",
5 | "index.number_of_shards" : "2",
6 | "index.version.created" : "900052"
7 | }
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_2.json:
--------------------------------------------------------------------------------
1 | {"_id":"3","_index":"index2","_type":"a","_source":{"name":"three"}}
2 | {"_id":"4","_index":"index2","_type":"b","_source":{"name":"four"}}
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_2.json.mapping:
--------------------------------------------------------------------------------
1 | {
2 | "index1" : {
3 | "1" : {
4 | "_timestamp" : {
5 | "enabled" : true,
6 | "store" : true
7 | },
8 | "_ttl" : {
9 | "enabled" : true,
10 | "default" : 86400000
11 | },
12 | "properties" : {
13 | "name" : {
14 | "type" : "string",
15 | "store" : true
16 | }
17 | }
18 | },
19 | "2" : {
20 | "properties" : {
21 | "name" : {
22 | "type" : "string"
23 | }
24 | }
25 | }
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/resources/importdata/import_9/index1_2.json.settings:
--------------------------------------------------------------------------------
1 | {
2 | "index1" : {
3 | "settings" : {
4 | "index.number_of_replicas" : "1",
5 | "index.number_of_shards" : "2",
6 | "index.version.created" : "900052"
7 | }
8 | }
9 | }
10 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/client/transport/TransportClientTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.client.transport;
2 |
3 | import org.elasticsearch.client.transport.TransportClient;
4 | import org.elasticsearch.common.settings.ImmutableSettings;
5 | import org.elasticsearch.common.settings.Settings;
6 | import org.junit.Test;
7 |
8 | import static org.junit.Assert.assertNotNull;
9 |
10 | /**
11 | * Test to make sure TransportClient works correctly with InOut plugin
12 | */
13 | public class TransportClientTest {
14 |
15 |
16 | /**
17 | * Instantiate a TransportClient to make sure dependency injection works correctly
18 | */
19 | @Test
20 | public void testTransportClient() {
21 |
22 | /**
23 | * InOut plugin modules must not be loaded for TransportClient instances
24 | */
25 | TransportClient client = new TransportClient();
26 | assertNotNull(client);
27 |
28 | /**
29 | * Internally, this get determined by the settings flag node.client which is set to true in case of
30 | * a TransportClient object. Thought the setting was given to the TransportClient with node.client = false
31 | * the constructor of TransportClient overwrites it to node.client = true
32 | */
33 | Settings settings = ImmutableSettings.settingsBuilder()
34 | .put("node.client", false)
35 | .build();
36 |
37 | client = null;
38 | client = new TransportClient(settings);
39 | assertNotNull(client);
40 |
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/doctests/DocTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.doctests;
2 |
3 | import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
4 | import static com.github.tlrx.elasticsearch.test.EsSetup.deleteAll;
5 | import static com.github.tlrx.elasticsearch.test.EsSetup.fromClassPath;
6 | import junit.framework.TestCase;
7 |
8 | import org.elasticsearch.common.settings.ImmutableSettings;
9 | import org.elasticsearch.common.settings.Settings;
10 | import org.junit.After;
11 | import org.junit.Before;
12 | import org.python.core.Py;
13 | import org.python.core.PyArray;
14 | import org.python.core.PyList;
15 | import org.python.core.PyString;
16 | import org.python.core.PySystemState;
17 | import org.python.util.PythonInterpreter;
18 |
19 | public class DocTest extends TestCase {
20 |
21 | StoreEsSetup esSetup, esSetup2;
22 |
23 | private static final String PY_TEST = "src/test/python/tests.py";
24 |
25 | private PythonInterpreter interp;
26 | private PySystemState sys;
27 |
28 | private void resetInterpreter() {
29 | interp = new PythonInterpreter(null, new PySystemState());
30 | sys = Py.getSystemState();
31 | }
32 |
33 | private void execFile(String filePath, String... arguments) {
34 | interp.cleanup();
35 | interp.set("__file__", filePath);
36 | sys.argv = new PyList(new PyString[]{new PyString(filePath)});
37 | sys.argv.extend(new PyArray(PyString.class, arguments));
38 | interp.execfile(filePath);
39 | }
40 |
41 | private void execDocFile(String name) {
42 | execFile(PY_TEST, name);
43 | }
44 |
45 | @Before
46 | public void setUp() {
47 | if (interp == null) {
48 | resetInterpreter();
49 | }
50 | Settings s1 = ImmutableSettings.settingsBuilder()
51 | .put("cluster.name", "a")
52 | .put("node.local", false)
53 | .build();
54 |
55 | esSetup = new StoreEsSetup(s1);
56 | esSetup.execute(deleteAll(), createIndex("users").withSettings(
57 | fromClassPath("essetup/settings/test_a.json")).withMapping("d",
58 | fromClassPath("essetup/mappings/test_a.json")).withData(
59 | fromClassPath("essetup/data/test_a.json")));
60 | esSetup.client().admin().indices().prepareRefresh("users").execute().actionGet();
61 |
62 | Settings s2 = ImmutableSettings.settingsBuilder()
63 | .put("cluster.name", "b")
64 | .put("node.local", false)
65 | .build();
66 | esSetup2 = new StoreEsSetup(s2);
67 | esSetup2.execute(deleteAll());
68 | }
69 |
70 | @After
71 | public void tearDown() {
72 | esSetup.terminate();
73 | if (esSetup2 != null) {
74 | esSetup2.terminate();
75 | }
76 | }
77 |
78 | public void testSearchInto() throws Exception {
79 | execDocFile("search_into.rst");
80 | }
81 |
82 | public void testReindex() throws Exception {
83 | execDocFile("reindex.rst");
84 | }
85 |
86 | }
87 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/doctests/StoreEsSetup.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.doctests;
2 |
3 | import org.elasticsearch.common.settings.Settings;
4 |
5 | import com.github.tlrx.elasticsearch.test.EsSetup;
6 |
7 | public class StoreEsSetup extends EsSetup {
8 |
9 | public StoreEsSetup() {
10 | super(new StoreLocalClientProvider());
11 | }
12 |
13 | public StoreEsSetup(Settings settings) {
14 | super(new StoreLocalClientProvider(settings));
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/doctests/StoreLocalClientProvider.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.doctests;
2 |
3 | import org.elasticsearch.common.network.NetworkUtils;
4 | import org.elasticsearch.common.settings.ImmutableSettings;
5 | import org.elasticsearch.common.settings.Settings;
6 |
7 | import com.github.tlrx.elasticsearch.test.provider.ClientProvider;
8 | import com.github.tlrx.elasticsearch.test.provider.LocalClientProvider;
9 |
10 | public class StoreLocalClientProvider extends LocalClientProvider implements ClientProvider {
11 |
12 | private Settings settings;
13 |
14 | public StoreLocalClientProvider() {
15 | super();
16 | }
17 |
18 | public StoreLocalClientProvider(Settings settings) {
19 | super(settings);
20 | this.settings = settings;
21 | }
22 |
23 | protected Settings buildNodeSettings() {
24 | // Build settings
25 | ImmutableSettings.Builder builder = ImmutableSettings.settingsBuilder()
26 | .put("node.name", "node-test-" + System.currentTimeMillis())
27 | .put("node.data", true)
28 | .put("cluster.name", "cluster-test-" + NetworkUtils.getLocalAddress().getHostName())
29 | .put("path.data", "./target/elasticsearch-test/data")
30 | .put("path.work", "./target/elasticsearch-test/work")
31 | .put("path.logs", "./target/elasticsearch-test/logs")
32 | .put("index.number_of_shards", "1")
33 | .put("index.number_of_replicas", "0")
34 | .put("cluster.routing.schedule", "50ms")
35 | .put("node.local", true);
36 |
37 | if (settings != null) {
38 | builder.put(settings);
39 | }
40 |
41 | return builder.build();
42 | }
43 |
44 | }
45 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/export/StreamConsumerTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.export;
2 |
3 | import org.junit.Test;
4 |
5 | import java.io.ByteArrayInputStream;
6 |
7 | import static org.junit.Assert.assertEquals;
8 |
9 | /**
10 | * Unit test for the @StreamConsumer class.
11 | */
12 | public class StreamConsumerTest {
13 |
14 | @Test
15 | public void test() {
16 | // Prepare an input stream with more than 8 bytes.
17 | String tmp = "one\ntwo\nthree\n";
18 | ByteArrayInputStream inputStream = new ByteArrayInputStream(tmp.getBytes());
19 |
20 | // Initialize a consumer. Remember the first 8 bytes.
21 | StreamConsumer consumer = new StreamConsumer(inputStream, 8);
22 |
23 | // Immediately the consumer does not get any output yet.
24 | String output = consumer.getBufferedOutput();
25 | assertEquals("", output);
26 |
27 | // Wait for the stream to finish.
28 | consumer.waitFor();
29 |
30 | // The output delivers the first 8 bytes of the stream.
31 | output = consumer.getBufferedOutput();
32 | assertEquals("one\ntwo\n", output);
33 |
34 | // The input stream has no bytes left. The rest of the output
35 | // is consumed.
36 | assertEquals(0, inputStream.available());
37 | }
38 |
39 | }
40 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/module/AbstractRestActionTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module;
2 |
3 | import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
4 | import static com.github.tlrx.elasticsearch.test.EsSetup.deleteAll;
5 | import static com.github.tlrx.elasticsearch.test.EsSetup.fromClassPath;
6 |
7 | import java.io.BufferedReader;
8 | import java.io.File;
9 | import java.io.FileInputStream;
10 | import java.io.FileNotFoundException;
11 | import java.io.IOException;
12 | import java.io.InputStreamReader;
13 | import java.util.ArrayList;
14 | import java.util.List;
15 | import java.util.Map;
16 | import java.util.zip.GZIPInputStream;
17 |
18 | import junit.framework.TestCase;
19 |
20 | import org.elasticsearch.common.xcontent.ToXContent;
21 | import org.elasticsearch.common.xcontent.XContentBuilder;
22 | import org.elasticsearch.common.xcontent.XContentFactory;
23 | import org.elasticsearch.common.xcontent.XContentType;
24 | import org.junit.After;
25 | import org.junit.Before;
26 |
27 | import com.github.tlrx.elasticsearch.test.EsSetup;
28 |
29 | /**
30 | * Abstract base class for the plugin's rest action tests. Sets up the client
31 | * and delivers some base functionality needed for all tests.
32 | */
33 | public abstract class AbstractRestActionTest extends TestCase {
34 |
35 | protected EsSetup esSetup, esSetup2;
36 |
37 | @Before
38 | public void setUp() {
39 | esSetup = new EsSetup();
40 | esSetup.execute(deleteAll(), createIndex("users").withSettings(
41 | fromClassPath("essetup/settings/test_a.json")).withMapping("d",
42 | fromClassPath("essetup/mappings/test_a.json")).withData(
43 | fromClassPath("essetup/data/test_a.json")));
44 | esSetup.client().admin().indices().prepareRefresh("users").execute();
45 | }
46 |
47 | @After
48 | public void tearDown() {
49 | esSetup.terminate();
50 | if (esSetup2 != null) {
51 | esSetup2.terminate();
52 | }
53 | }
54 |
55 | /**
56 | * Convert an XContent object to a Java map
57 | * @param toXContent
58 | * @return
59 | * @throws IOException
60 | */
61 | public static Map toMap(ToXContent toXContent) throws IOException {
62 | XContentBuilder builder = XContentFactory.contentBuilder(XContentType.JSON);
63 | toXContent.toXContent(builder, ToXContent.EMPTY_PARAMS);
64 | return XContentFactory.xContent(XContentType.JSON).createParser(
65 | builder.string()).mapOrderedAndClose();
66 | }
67 |
68 | /**
69 | * Set up a second node and wait for green status
70 | */
71 | protected void setUpSecondNode() {
72 | esSetup2 = new EsSetup();
73 | esSetup2.execute(deleteAll());
74 | esSetup2.client().admin().cluster().prepareHealth().setWaitForGreenStatus().execute().actionGet();
75 | }
76 |
77 | /**
78 | * Get a list of lines from a gzipped file.
79 | * Test fails if file not found or IO exception happens.
80 | *
81 | * @param filename the file name to read
82 | * @return a list of strings
83 | */
84 | protected List readLinesFromGZIP(String filename) {
85 | BufferedReader reader = null;
86 | try {
87 | reader = new BufferedReader(new InputStreamReader(
88 | new GZIPInputStream(new FileInputStream(new File(filename)))));
89 | } catch (FileNotFoundException e) {
90 | e.printStackTrace();
91 | fail("File not found");
92 | } catch (IOException e) {
93 | e.printStackTrace();
94 | fail("IO Excsption while reading ZIP stream");
95 | }
96 | return readLines(filename, reader);
97 | }
98 |
99 | protected List readLines(String filename, BufferedReader reader) {
100 | List lines = new ArrayList();
101 | try {
102 | String line;
103 | while ((line = reader.readLine()) != null) {
104 | lines.add(line);
105 | }
106 | } catch (IOException e) {
107 | e.printStackTrace();
108 | fail("IO Exception occured while reading file");
109 | }
110 | return lines;
111 | }
112 |
113 |
114 | }
115 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/module/reindex/test/RestReindexActionTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.reindex.test;
2 |
3 | import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
4 | import static com.github.tlrx.elasticsearch.test.EsSetup.index;
5 |
6 | import java.io.IOException;
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | import org.junit.Test;
11 |
12 | import crate.elasticsearch.action.reindex.ReindexAction;
13 | import crate.elasticsearch.action.searchinto.SearchIntoRequest;
14 | import crate.elasticsearch.action.searchinto.SearchIntoResponse;
15 | import crate.elasticsearch.module.AbstractRestActionTest;
16 |
17 | public class RestReindexActionTest extends AbstractRestActionTest {
18 |
19 | @Test
20 | public void testSearchIntoWithoutSource() {
21 | esSetup.execute(createIndex("test").withMapping("a",
22 | "{\"a\":{\"_source\": {\"enabled\": false}}}"));
23 | esSetup.execute(index("test", "a", "1").withSource("{\"name\": \"John\"}"));
24 | SearchIntoRequest request = new SearchIntoRequest("test");
25 | SearchIntoResponse res = esSetup.client().execute(ReindexAction.INSTANCE, request).actionGet();
26 | assertEquals(1, res.getFailedShards());
27 | assertTrue(res.getShardFailures()[0].reason().contains("Parse Failure [The _source field of index test and type a is not stored.]"));
28 | }
29 |
30 | private static List> get(SearchIntoResponse resp, String key) {
31 | Map res = null;
32 | try {
33 | res = toMap(resp);
34 | } catch (IOException e) {
35 | e.printStackTrace();
36 | return null;
37 | }
38 | return (List>) res.get(key);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/test/java/crate/elasticsearch/module/searchinto/test/RestSearchIntoActionTest.java:
--------------------------------------------------------------------------------
1 | package crate.elasticsearch.module.searchinto.test;
2 |
3 | import static com.github.tlrx.elasticsearch.test.EsSetup.createIndex;
4 | import static com.github.tlrx.elasticsearch.test.EsSetup.index;
5 |
6 | import java.io.IOException;
7 | import java.util.List;
8 | import java.util.Map;
9 |
10 | import org.elasticsearch.action.get.GetRequestBuilder;
11 | import org.elasticsearch.action.get.GetResponse;
12 | import org.junit.Test;
13 |
14 | import crate.elasticsearch.action.searchinto.SearchIntoAction;
15 | import crate.elasticsearch.action.searchinto.SearchIntoRequest;
16 | import crate.elasticsearch.action.searchinto.SearchIntoResponse;
17 | import crate.elasticsearch.module.AbstractRestActionTest;
18 |
19 | public class RestSearchIntoActionTest extends AbstractRestActionTest {
20 |
21 | @Test
22 | public void testSearchIntoWithoutSource() {
23 | esSetup.execute(createIndex("test").withMapping("a",
24 | "{\"a\":{\"_source\": {\"enabled\": false}}}"));
25 | esSetup.execute(index("test", "a", "1").withSource("{\"name\": \"John\"}"));
26 | SearchIntoRequest request = new SearchIntoRequest("test");
27 | request.source("{\"fields\": [\"_id\", \"_source\", [\"_index\", \"'newindex'\"]]}");
28 | SearchIntoResponse res = esSetup.client().execute(SearchIntoAction.INSTANCE, request).actionGet();
29 | assertEquals(1, res.getFailedShards());
30 | assertTrue(res.getShardFailures()[0].reason().contains("Parse Failure [The _source field of index test and type a is not stored.]"));
31 | }
32 |
33 | @Test
34 | public void testNestedObjectsRewriting() {
35 | prepareNested();
36 | SearchIntoRequest request = new SearchIntoRequest("test");
37 | request.source("{\"fields\": [\"_id\", [\"x.city\", \"_source.city\"], [\"x.surname\", \"_source.name.surname\"], [\"x.name\", \"_source.name.name\"], [\"_index\", \"'newindex'\"]]}");
38 | SearchIntoResponse res = esSetup.client().execute(SearchIntoAction.INSTANCE, request).actionGet();
39 |
40 | GetRequestBuilder rb = new GetRequestBuilder(esSetup.client(), "newindex");
41 | GetResponse getRes = rb.setType("a").setId("1").execute().actionGet();
42 | assertTrue(getRes.isExists());
43 | assertEquals("{\"x\":{\"name\":\"Doe\",\"surname\":\"John\",\"city\":\"Dornbirn\"}}", getRes.getSourceAsString());
44 | }
45 |
46 | @Test
47 | public void testNestedObjectsRewritingMixed1() {
48 | prepareNested();
49 | SearchIntoRequest request = new SearchIntoRequest("test");
50 | request.source("{\"fields\": [\"_id\", [\"x\", \"_source.city\"], [\"x.surname\", \"_source.name.surname\"], [\"x.name\", \"_source.name.name\"], [\"_index\", \"'newindex'\"]]}");
51 | SearchIntoResponse res = esSetup.client().execute(SearchIntoAction.INSTANCE, request).actionGet();
52 | assertTrue(res.getShardFailures()[0].reason().contains("Error on rewriting objects: Mixed objects and values]"));
53 | }
54 |
55 | @Test
56 | public void testNestedObjectsRewritingMixed2() {
57 | prepareNested();
58 | SearchIntoRequest request = new SearchIntoRequest("test");
59 | request.source("{\"fields\": [\"_id\", [\"x.surname.bad\", \"_source.city\"], [\"x.surname\", \"_source.name.surname\"], [\"x.name\", \"_source.name.name\"], [\"_index\", \"'newindex'\"]]}");
60 | SearchIntoResponse res = esSetup.client().execute(SearchIntoAction.INSTANCE, request).actionGet();
61 | assertTrue(res.getShardFailures()[0].reason().contains("Error on rewriting objects: Mixed objects and values]"));
62 | }
63 |
64 | @Test
65 | public void testNestedObjectsRewritingMixed3() {
66 | prepareNested();
67 | SearchIntoRequest request = new SearchIntoRequest("test");
68 | request.source("{\"fields\": [\"_id\", [\"x.surname\", \"_source.city\"], [\"x.surname.bad\", \"_source.name.surname\"], [\"x.name\", \"_source.name.name\"], [\"_index\", \"'newindex'\"]]}");
69 | SearchIntoResponse res = esSetup.client().execute(SearchIntoAction.INSTANCE, request).actionGet();
70 | assertTrue(res.getShardFailures()[0].reason().contains("Error on rewriting objects: Mixed objects and values]"));
71 | }
72 |
73 |
74 | private void prepareNested() {
75 | esSetup.execute(createIndex("test").withMapping("a",
76 | "{\"a\": {\"properties\": {\"name\": {\"properties\": {\"surname\":{\"type\":\"string\"}, \"name\": {\"type\":\"string\"}}}, \"city\": {\"type\": \"string\"}}}}"));
77 | esSetup.execute(index("test", "a", "1").withSource(
78 | "{\"city\": \"Dornbirn\", \"name\": {\"surname\": \"John\", \"name\": \"Doe\"}}"));
79 | }
80 |
81 | private static List> get(SearchIntoResponse resp, String key) {
82 | Map res = null;
83 | try {
84 | res = toMap(resp);
85 | } catch (IOException e) {
86 | e.printStackTrace();
87 | return null;
88 | }
89 | return (List>) res.get(key);
90 | }
91 |
92 | }
93 |
--------------------------------------------------------------------------------
/src/test/python/reindex.rst:
--------------------------------------------------------------------------------
1 | =======
2 | Reindex
3 | =======
4 |
5 | Via the ``_reindex`` endpoint it is possible to reindex one or all indexes
6 | with a given query.
7 |
8 | Reindex an existing Index
9 | =========================
10 |
11 | Create an index 'test' with a custom analyzer with the stop word 'guy'::
12 |
13 | >>> put("/test", {"settings": {"index": {"number_of_shards":1,
14 | ... "number_of_replicas":0,
15 | ... "analysis": {"analyzer": {"myan": {"type": "stop", "stopwords": ["guy"]}}}}}})
16 | {"ok":true,"acknowledged":true}
17 |
18 | Create the mapping for type 'a' and use the custom analyzer as index analyzer
19 | and use a simple search analyzer::
20 |
21 | >>> post("/test/a/_mapping", {"a": {"properties": {"name": {"type": "string", "index_analyzer": "myan", "search_analyzer": "simple", "store": "yes"}}}})
22 | {"ok":true,"acknowledged":true}
23 |
24 | Add a document::
25 |
26 | >>> post("/test/a/1", {"name": "a nice guy"})
27 | {"ok":true,"_index":"test","_type":"a","_id":"1","_version":1}
28 | >>> refresh()
29 | {...}
30 |
31 | Querying for a non stop word term delivers a result::
32 |
33 | >>> post("/test/a/_search?pretty", {"query": {"text": {"name": "nice"}}})
34 | {
35 | ...
36 | "hits" : {
37 | "total" : 1,
38 | ...
39 | }
40 |
41 | Querying for a stop word delivers no results::
42 |
43 | >>> post("/test/a/_search?pretty", {"query": {"text": {"name": "guy"}}})
44 | {
45 | ...
46 | "hits" : {
47 | "total" : 0,
48 | ...
49 | }
50 |
51 | Now update the stop words configuration. To update settings the index has to
52 | be closed first and then reopened::
53 |
54 | >>> post("/test/_close", {})
55 | {"ok":true,"acknowledged":true}
56 | >>> put("/test/_settings", {"analysis": {"analyzer": {"myan": {"type": "stop", "stopwords": ["nice"]}}}})
57 | {"ok":true}
58 | >>> post("/test/_open", {})
59 | {"ok":true,"acknowledged":true}
60 | >>> refresh()
61 | {...}
62 |
63 | As the index has not been reindexed yet, the query for 'nice' still delivers
64 | a result::
65 |
66 | >>> post("/test/a/_search?pretty", {"query": {"text": {"name": "nice"}}})
67 | {
68 | ...
69 | "hits" : {
70 | "total" : 1,
71 | ...
72 | }
73 |
74 | Now do a reindex on the index 'test'::
75 |
76 | >>> post("/test/_reindex", {})
77 | {"writes":[..."succeeded":1...],"total":0,"succeeded":0,"failed":0,"_shards":{"total":1,"successful":1,"failed":0}}
78 | >>> refresh()
79 | {...}
80 |
81 | No more result when querying for the new stop word 'nice'::
82 |
83 | >>> post("/test/a/_search?pretty", {"query": {"text": {"name": "nice"}}})
84 | {
85 | ...
86 | "hits" : {
87 | "total" : 0,
88 | ...
89 | }
90 |
91 | The removed stop word 'guy' now delivers a result::
92 |
93 | >>> post("/test/a/_search?pretty", {"query": {"text": {"name": "guy"}}})
94 | {
95 | ...
96 | "hits" : {
97 | "total" : 1,
98 | ...
99 | }
100 |
--------------------------------------------------------------------------------
/src/test/python/search_into.rst:
--------------------------------------------------------------------------------
1 | ===========
2 | Search Into
3 | ===========
4 |
5 | Via the ``_search_into`` endpoint it is possible to put the result of
6 | a given query directly into an index. It is inspired from the `Select
7 | Into SQL-Statement
8 | `__
9 |
10 | Here are some common use-cases that might be fulfilled by using this
11 | endpoint.
12 |
13 |
14 | Copy an existing index
15 | ======================
16 |
17 | To copy a whole index, the ``_index`` field source needs to be defined as a
18 | string literal like this::
19 |
20 | >>> fields = ["_id", "_source", ["_index", "'newindex'"]]
21 |
22 | There are two special things happening here. When defining a field as an
23 | array of 2 items, the first item is considered as the target-field name and
24 | the second item is considered as the source value. The above example
25 | defines a string literal (denoted by the surrounding apostrophes) as
26 | the value of the ``_index`` field.
27 |
28 | >>> post("/users/_search_into?pretty=1", {"fields":fields})
29 | {
30 | "writes" : [ {
31 | "index" : "users",
32 | "shard" : 0,
33 | "node" : "...",
34 | "total" : 2,
35 | "succeeded" : 2,
36 | "failed" : 0
37 | }, {
38 | "index" : "users",
39 | "shard" : 1,
40 | "node" : "...",
41 | "total" : 2,
42 | "succeeded" : 2,
43 | "failed" : 0
44 | }...
45 |
46 | Now a new index with the same content was created::
47 |
48 |
49 | >>> post("/newindex/_flush")
50 | {"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
51 |
52 | >>> post("/newindex/_refresh")
53 | {"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
54 |
55 |
56 | >>> get("/newindex/_search?pretty=1")
57 | {..."hits" : {
58 | "total" : 4,
59 | ...{
60 | "_index" : "newindex",
61 | "_type" : "d",
62 | "_id" : "1",
63 | "_score" : 1.0, "_source" : {"name":"car"}
64 | }...
65 |
66 |
67 | Copy an existing index into another cluster
68 | ===========================================
69 |
70 | To copy one index to another cluster set the option ``targetNodes`` in
71 | the payload. The value of the option might be given as a single value in
72 | the format : or as a list of such values.
73 |
74 | Initially, the second cluster does not contain any indizes::
75 |
76 | >>> node2.get("/_status?pretty=1")
77 | {
78 | ...
79 | "indices" : { }
80 | }
81 |
82 | >>> payload = {"fields":["_id", "_source"],
83 | ... "targetNodes":"localhost:9301"}
84 |
85 | >>> post("/users/_search_into?pretty=1", payload)
86 | {
87 | "writes" : [ {
88 | "index" : "users",
89 | "shard" : 0,
90 | "node" : "...",
91 | "total" : 2,
92 | "succeeded" : 2,
93 | "failed" : 0
94 | }, {
95 | "index" : "users",
96 | "shard" : 1,
97 | "node" : "...",
98 | "total" : 2,
99 | "succeeded" : 2,
100 | "failed" : 0
101 | } ],
102 | "total" : 0,
103 | "succeeded" : 0,
104 | "failed" : 0,
105 | "_shards" : {
106 | "total" : 2,
107 | "successful" : 2,
108 | "failed" : 0
109 | }
110 | }
111 |
112 | Now the new index with the same content was created in the other
113 | cluster::
114 |
115 | >>> node2.post("/users/_flush")
116 | {"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
117 |
118 | >>> node2.post("/users/_refresh")
119 | {"ok":true,"_shards":{"total":1,"successful":1,"failed":0}}
120 |
121 | >>> node2.get("/users/_search?pretty=1")
122 | {..."hits" : {
123 | "total" : 4,
124 | ...{
125 | "_index" : "users",
126 | "_type" : "d",
127 | "_id" : "1",
128 | "_score" : 1.0, "_source" : {"name":"car"}
129 | }...
130 |
--------------------------------------------------------------------------------
/src/test/python/tests.py:
--------------------------------------------------------------------------------
1 | import doctest
2 | import unittest
3 | import sys
4 | import json
5 | import urllib2
6 |
7 |
8 | class Endpoint(object):
9 | def __init__(self, base='http://localhost:9200'):
10 | self.base = base
11 |
12 | def _req(self, path):
13 | url = self.base + path
14 | return urllib2.Request(url)
15 |
16 |
17 | def get(self, path):
18 | req = self._req(path)
19 | print urllib2.urlopen(req).read()
20 |
21 | def post(self, path, data=''):
22 | req = self._req(path)
23 | body = json.dumps(data)
24 | req.add_data(body)
25 | print urllib2.urlopen(req).read()
26 |
27 | def put(self, path, data=''):
28 | req = self._req(path)
29 | body = json.dumps(data)
30 | req.add_data(body)
31 | req.get_method = lambda: 'PUT'
32 | print urllib2.urlopen(req).read()
33 |
34 | def refresh(self):
35 | self.post("/_flush")
36 | self.post("/_refresh")
37 |
38 |
39 | def setUp(test):
40 | ep = Endpoint()
41 | test.globs['get'] = ep.get
42 | test.globs['post'] = ep.post
43 | test.globs['put'] = ep.put
44 | test.globs['refresh'] = ep.refresh
45 | ep2 = Endpoint('http://localhost:9201')
46 | test.globs['node2'] = ep2
47 |
48 |
49 | def test_suite(fname):
50 | s = unittest.TestSuite((
51 | doctest.DocFileSuite(
52 | fname, setUp=setUp,
53 | optionflags=doctest.NORMALIZE_WHITESPACE | doctest.ELLIPSIS)
54 | ))
55 | return s
56 |
57 |
58 | if __name__ == '__main__':
59 | runner = unittest.TextTestRunner(failfast=True)
60 | res = runner.run(test_suite(sys.argv[1]))
61 | if res.failures or res.errors:
62 | sys.exit(1)
63 |
64 |
--------------------------------------------------------------------------------