├── .gitignore ├── README.textile ├── pom.xml └── src └── main ├── assemblies └── plugin.xml ├── java ├── co │ └── diji │ │ ├── rest │ │ ├── SolrSearchHandlerRestAction.java │ │ └── SolrUpdateHandlerRestAction.java │ │ ├── solr │ │ ├── SolrResponseWriter.java │ │ └── XMLWriter.java │ │ └── utils │ │ └── QueryStringDecoder.java └── org │ └── elasticsearch │ └── plugin │ └── diji │ └── MockSolrPlugin.java └── resources └── es-plugin.properties /.gitignore: -------------------------------------------------------------------------------- 1 | target 2 | .settings 3 | .classpath 4 | .project 5 | .idea 6 | -------------------------------------------------------------------------------- /README.textile: -------------------------------------------------------------------------------- 1 | h1. ElasticSearch Mock Solr Plugin 2 | 3 | |_.Mock Solr Plugin|_.elasticsearch|_.Lucene/Solr| 4 | |master|0.20.2 -> 0.20.X|3.6.2| 5 | |1.1.4|0.20.2 -> 0.20.X|3.6.2| 6 | |1.1.3|0.19.3 -> 0.20.1|3.6.0| 7 | |1.1.2|0.19.0 -> 0.19.2|3.5.0| 8 | |1.1.1|0.18.6 -> 0.18.7|3.5.0| 9 | |1.1.0|0.18.0 -> 0.18.5|3.5.0| 10 | 11 | h2. Use Solr clients/tools with ElasticSearch 12 | 13 | This plugin will allow you to use tools that were built to 14 | interact with Solr with ElasticSearch. 15 | 16 | The idea for this plugin came when I wanted to use Nutch with 17 | ElasticSearch. Instead of extending Nutch itself, 18 | I thought it would be nice to use any Solr clients with 19 | ElasticSearch. Some projects we can now use are 20 | Nutch, Apache ManifoldCF, and any tool using SolrJ. It 21 | should be possible to use non-java tools that write to 22 | Solr using the XML update and request handlers as well. 23 | 24 | h3. Supported Solr features 25 | 26 | * Update handlers 27 | ** XML Update Handler (ie. /update) 28 | ** JavaBin Update Handler (ie. /update/javabin) 29 | * Search handler (ie. /select) 30 | ** Basic lucene queries using the q paramter 31 | ** start, rows, and fl parameters 32 | ** sorting 33 | ** filter queries (fq parameters) 34 | ** hit highlighting (hl, hl.fl, hl.snippets, hl.fragsize, hl.simple.pre, hl.simple.post) 35 | ** faceting (facet, facet.field, facet.query, facet.sort, facet.limit) 36 | * XML and JavaBin request and response formats 37 | 38 | h3. How do you build this plugin? 39 | 40 | Use maven to build the package 41 | 42 |

 43 | mvn package
 44 |

45 | 46 | Then install the plugin 47 | 48 |

 49 | # if you've built it locally
 50 | $ES_HOME/bin/plugin -url file:./target/releases/elasticsearch-mocksolrplugin-*.zip -install mocksolrplugin
 51 |

52 | 53 | h3. How to use this plugin. 54 | 55 | Just point your Solr client/tool to your ElasticSearch instance and appending 56 | /_solr to the url. 57 | 58 | http://localhost:9200/${index}/${type}/_solr 59 | 60 | ${index} - the ES index you want to index/search against. Default "solr". 61 | ${type} - the ES type you want to index/search against. Default "docs". 62 | 63 | Example paths: 64 |

 65 | // Will search/index against index "solr" and type "docs"
 66 | http://localhost:9200/_solr
 67 | 
 68 | // Will search/index against index "testindex" and type "docs"
 69 | http://localhost:9200/testindex/_solr
 70 | 
 71 | // Will search/index against index "testindex" and type "testtype"
 72 | http://localhost:9200/testindex/testtype/_solr
 73 |

74 | 75 | Use the client/tool as you would with Solr. 76 | 77 | h3. Example SolrJ Indexing 78 | 79 |

 80 |     CommonsHttpSolrServer server = new CommonsHttpSolrServer("http://localhost:9200/testindex/testtype/_solr");
 81 |     server.setRequestWriter(new BinaryRequestWriter());
 82 |     // we support both xml and SolrBin response writers
 83 |     //server.setParser(new XMLResponseParser());
 84 |     
 85 |     SolrInputDocument doc1 = new SolrInputDocument();
 86 |     doc1.addField( "id", "id1", 1.0f );
 87 |     doc1.addField( "name", "doc1", 1.0f );
 88 |     doc1.addField( "price", 10 );
 89 | 
 90 |     SolrInputDocument doc2 = new SolrInputDocument();
 91 |     doc2.addField( "id", "id2", 1.0f );
 92 |     doc2.addField( "name", "doc2", 1.0f );
 93 |     doc2.addField( "price", 20 );
 94 |     
 95 |     Collection docs = new ArrayList();
 96 |     docs.add( doc1 );
 97 |     docs.add( doc2 );
 98 |     
 99 |     server.add( docs );
100 |     server.commit();
101 | 
102 |     // deletes work as well
103 |     //server.deleteById("id2");
104 |     //server.commit();
105 |

106 | 107 | Perform a search and verify the documents were indexed. 108 | 109 | h3. Example SolrJ Searching 110 | 111 |

112 |     CommonsHttpSolrServer server = new CommonsHttpSolrServer("http://localhost:9200/testindex/testtype/_solr");
113 | 
114 |     String qstr = "id:[* TO *]";
115 |     SolrQuery query = new SolrQuery();
116 |     query.setQuery(qstr);
117 | 
118 |     QueryResponse response = server.query(query);
119 |     for (SolrDocument doc : response.getResults()) {
120 |         for (String field : doc.getFieldNames()) {
121 |             System.out.println(field + " = " + doc.getFieldValue(field));
122 |         }
123 |         System.out.println();
124 |     }
125 |

126 | 127 | 128 | h3. Example using Nutch 129 | 130 | At a minimum, use the following type mapping for ElasticSearch. 131 | 132 |

133 | curl -XPUT 'http://localhost:9200/testindex'
134 | curl -XPUT 'http://localhost:9200/testindex/testtype/_mapping' -d '{
135 |     "testtype" : {
136 |         "properties" : {
137 |             "id" : {
138 |                 "type" : "string",
139 |                 "store": "yes"
140 |             },
141 |             "digest" : {
142 |                 "type" : "string",
143 |                 "store" : "yes",
144 |                 "index" : "no"
145 |             },
146 |             "boost" : {
147 |                 "type" : "float",
148 |                 "store" : "yes",
149 |                 "index" : "no"
150 |             },
151 |             "tstamp" : {
152 |                 "type" : "date",
153 |                 "store" : "yes",
154 |                 "index" : "no"
155 |             }
156 |         }
157 |     }
158 | }'
159 |

160 | 161 | Follow the nutch tutorial at http://wiki.apache.org/nutch/NutchTutorial 162 | * Follow steps 1 though 3.1 163 | * For step 3.1 use: 164 | 165 |

166 | bin/nutch crawl urls -solr http://localhost:9200/testindex/testtype/_solr -depth 3 -topN 5
167 |

168 | 169 | h3. Notes 170 | 171 | ElasticSearch does not require a schema and all the data you send to Solr will be indexed by default. You 172 | Can use the ElasticSearch PUT Mapping API to define your field types, what should be stored, analyzed, etc. 173 | All data that is indexed via the mock XML Update Handler will most likely be detected by ElasticSearch as 174 | strings, thus it is a good idea to mimic your Solr schema with an ElasticSearch type mapping. 175 | 176 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | elasticsearch-mocksolrplugin 4 | 4.0.0 5 | co.diji 6 | elasticsearch-mocksolrplugin 7 | 1.1.5-SNAPSHOT 8 | jar 9 | Use Solr clients/tools with ElasticSearch 10 | 2011 11 | 12 | 13 | The Apache Software License, Version 2.0 14 | http://www.apache.org/licenses/LICENSE-2.0.txt 15 | repo 16 | 17 | 18 | 19 | scm:git:git@github.com:mattweber/elasticsearch-mocksolrplugin.git 20 | scm:git:git@github.com:mattweber/elasticsearch-mocksolrplugin.git 21 | https://github.com/mattweber/elasticsearch-mocksolrplugin 22 | 23 | 24 | 25 | org.sonatype.oss 26 | oss-parent 27 | 7 28 | 29 | 30 | 31 | 0.20.5 32 | 3.6.2 33 | 34 | 35 | 36 | 37 | oss.sonatype.org 38 | OSS Sonatype 39 | http://oss.sonatype.org/content/repositories/releases/ 40 | 41 | 42 | 43 | 44 | 45 | org.elasticsearch 46 | elasticsearch 47 | ${elasticsearch.version} 48 | compile 49 | 50 | 51 | 52 | org.apache.solr 53 | solr-solrj 54 | ${solr.version} 55 | compile 56 | 57 | 58 | commons-logging 59 | commons-logging 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | org.apache.maven.plugins 69 | maven-compiler-plugin 70 | 2.3.2 71 | 72 | 1.6 73 | 1.6 74 | 75 | 76 | 77 | org.apache.maven.plugins 78 | maven-surefire-plugin 79 | 2.11 80 | 81 | 82 | **/*Tests.java 83 | 84 | 85 | 86 | 87 | org.apache.maven.plugins 88 | maven-source-plugin 89 | 2.1.2 90 | 91 | 92 | attach-sources 93 | 94 | jar 95 | 96 | 97 | 98 | 99 | 100 | maven-assembly-plugin 101 | 102 | ${project.build.directory}/releases/ 103 | 104 | ${basedir}/src/main/assemblies/plugin.xml 105 | 106 | 107 | 108 | 109 | package 110 | 111 | single 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /src/main/assemblies/plugin.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | zip 6 | 7 | false 8 | 9 | 10 | / 11 | true 12 | true 13 | 14 | org.elasticsearch:elasticsearch 15 | 16 | 17 | 18 | / 19 | true 20 | true 21 | 22 | org.apache.solr:solr-solrj 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /src/main/java/co/diji/rest/SolrSearchHandlerRestAction.java: -------------------------------------------------------------------------------- 1 | package co.diji.rest; 2 | 3 | import co.diji.solr.SolrResponseWriter; 4 | import co.diji.utils.QueryStringDecoder; 5 | import org.apache.solr.common.SolrDocument; 6 | import org.apache.solr.common.SolrDocumentList; 7 | import org.apache.solr.common.util.NamedList; 8 | import org.apache.solr.common.util.SimpleOrderedMap; 9 | import org.elasticsearch.action.ActionListener; 10 | import org.elasticsearch.action.search.SearchRequest; 11 | import org.elasticsearch.action.search.SearchResponse; 12 | import org.elasticsearch.client.Client; 13 | import org.elasticsearch.common.Strings; 14 | import org.elasticsearch.common.inject.Inject; 15 | import org.elasticsearch.common.joda.time.format.DateTimeFormatter; 16 | import org.elasticsearch.common.joda.time.format.ISODateTimeFormat; 17 | import org.elasticsearch.common.settings.Settings; 18 | import org.elasticsearch.index.query.AndFilterBuilder; 19 | import org.elasticsearch.index.query.FilterBuilder; 20 | import org.elasticsearch.index.query.QueryBuilder; 21 | import org.elasticsearch.index.query.QueryBuilders; 22 | import org.elasticsearch.rest.*; 23 | import org.elasticsearch.rest.action.support.RestActions; 24 | import org.elasticsearch.search.SearchHit; 25 | import org.elasticsearch.search.SearchHitField; 26 | import org.elasticsearch.search.SearchHits; 27 | import org.elasticsearch.search.builder.SearchSourceBuilder; 28 | import org.elasticsearch.search.facet.Facet; 29 | import org.elasticsearch.search.facet.query.QueryFacet; 30 | import org.elasticsearch.search.facet.query.QueryFacetBuilder; 31 | import org.elasticsearch.search.facet.terms.TermsFacet; 32 | import org.elasticsearch.search.facet.terms.TermsFacetBuilder; 33 | import org.elasticsearch.search.highlight.HighlightBuilder; 34 | import org.elasticsearch.search.highlight.HighlightField; 35 | import org.elasticsearch.search.sort.SortOrder; 36 | 37 | import java.io.IOException; 38 | import java.util.Iterator; 39 | import java.util.List; 40 | import java.util.Map; 41 | import java.util.regex.Pattern; 42 | 43 | import static org.elasticsearch.index.query.FilterBuilders.andFilter; 44 | import static org.elasticsearch.index.query.FilterBuilders.queryFilter; 45 | 46 | public class SolrSearchHandlerRestAction extends BaseRestHandler { 47 | 48 | // handles solr response formats 49 | private final SolrResponseWriter solrResponseWriter = new SolrResponseWriter(); 50 | 51 | // regex and date format to detect ISO8601 date formats 52 | private final Pattern datePattern = Pattern.compile("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?Z");; 53 | private final DateTimeFormatter dateFormat = ISODateTimeFormat.dateOptionalTimeParser(); 54 | 55 | /** 56 | * Rest actions that mocks the Solr search handler 57 | * 58 | * @param settings ES settings 59 | * @param client ES client 60 | * @param restController ES rest controller 61 | */ 62 | @Inject 63 | public SolrSearchHandlerRestAction(Settings settings, Client client, RestController restController) { 64 | super(settings, client); 65 | 66 | // register search handler 67 | // specifying and index and type is optional 68 | restController.registerHandler(RestRequest.Method.GET, "/_solr/select", this); 69 | restController.registerHandler(RestRequest.Method.GET, "/{index}/_solr/select", this); 70 | restController.registerHandler(RestRequest.Method.GET, "/{index}/{type}/_solr/select", this); 71 | } 72 | 73 | /** 74 | * Parse uri parameters. 75 | * 76 | * ES request.param does not support multiple parameters with the same name yet. This 77 | * is needed for parameters such as fq in Solr. This will not be needed once a fix is 78 | * in ES. https://github.com/elasticsearch/elasticsearch/issues/1544 79 | * 80 | * @param uri The uri to parse 81 | * @return a map of parameters, each parameter value is a list of strings. 82 | */ 83 | private Map> parseUriParams(String uri) { 84 | // use netty query string decoder 85 | QueryStringDecoder decoder = new QueryStringDecoder(uri); 86 | return decoder.getParameters(); 87 | } 88 | 89 | /* 90 | * (non-Javadoc) 91 | * 92 | * @see 93 | * org.elasticsearch.rest.RestHandler#handleRequest(org.elasticsearch.rest.RestRequest, org.elasticsearch.rest.RestChannel) 94 | */ 95 | public void handleRequest(final RestRequest request, final RestChannel channel) { 96 | // Get the parameters 97 | final Map> params = parseUriParams(request.uri()); 98 | 99 | // generate the search request 100 | SearchRequest searchRequest = getSearchRequest(params, request); 101 | searchRequest.listenerThreaded(false); 102 | 103 | // execute the search 104 | client.search(searchRequest, new ActionListener() { 105 | @Override 106 | public void onResponse(SearchResponse response) { 107 | try { 108 | // write response 109 | solrResponseWriter.writeResponse(createSearchResponse(params, request, response), request, channel); 110 | } catch (Exception e) { 111 | onFailure(e); 112 | } 113 | } 114 | 115 | @Override 116 | public void onFailure(Throwable e) { 117 | try { 118 | logger.error("Error processing executing search", e); 119 | channel.sendResponse(new XContentThrowableRestResponse(request, e)); 120 | } catch (IOException e1) { 121 | logger.error("Failed to send failure response", e1); 122 | } 123 | } 124 | }); 125 | } 126 | 127 | /** 128 | * Generates an ES SearchRequest based on the Solr Input Parameters 129 | * 130 | * @param request the ES RestRequest 131 | * @return the generated ES SearchRequest 132 | */ 133 | private SearchRequest getSearchRequest(Map> params, RestRequest request) { 134 | // get solr search parameters 135 | String q = request.param("q"); 136 | int start = request.paramAsInt("start", 0); 137 | int rows = request.paramAsInt("rows", 10); 138 | String fl = request.param("fl"); 139 | String sort = request.param("sort"); 140 | List fqs = params.get("fq"); 141 | boolean hl = request.paramAsBoolean("hl", false); 142 | boolean facet = request.paramAsBoolean("facet", false); 143 | boolean qDsl = request.paramAsBoolean("q.dsl", false); 144 | boolean fqDsl = request.paramAsBoolean("fq.dsl", false); 145 | 146 | // build the query 147 | SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); 148 | if (q != null) { 149 | QueryBuilder queryBuilder; 150 | if (qDsl) { 151 | queryBuilder = QueryBuilders.wrapperQuery(q); 152 | } else { 153 | queryBuilder = QueryBuilders.queryString(q); 154 | } 155 | searchSourceBuilder.query(queryBuilder); 156 | } 157 | 158 | searchSourceBuilder.from(start); 159 | searchSourceBuilder.size(rows); 160 | 161 | // parse fl into individual fields 162 | // solr supports separating by comma or spaces 163 | if (fl != null) { 164 | if (!Strings.hasText(fl)) { 165 | searchSourceBuilder.noFields(); 166 | } else { 167 | searchSourceBuilder.fields(fl.split("\\s|,")); 168 | } 169 | } 170 | 171 | // handle sorting 172 | if (sort != null) { 173 | String[] sorts = Strings.splitStringByCommaToArray(sort); 174 | for (int i = 0; i < sorts.length; i++) { 175 | String sortStr = sorts[i].trim(); 176 | int delimiter = sortStr.lastIndexOf(" "); 177 | if (delimiter != -1) { 178 | String sortField = sortStr.substring(0, delimiter); 179 | if ("score".equals(sortField)) { 180 | sortField = "_score"; 181 | } 182 | String reverse = sortStr.substring(delimiter + 1); 183 | if ("asc".equals(reverse)) { 184 | searchSourceBuilder.sort(sortField, SortOrder.ASC); 185 | } else if ("desc".equals(reverse)) { 186 | searchSourceBuilder.sort(sortField, SortOrder.DESC); 187 | } 188 | } else { 189 | searchSourceBuilder.sort(sortStr); 190 | } 191 | } 192 | } else { 193 | // default sort by descending score 194 | searchSourceBuilder.sort("_score", SortOrder.DESC); 195 | } 196 | 197 | // handler filters 198 | if (fqs != null && !fqs.isEmpty()) { 199 | FilterBuilder filterBuilder = null; 200 | 201 | // if there is more than one filter specified build 202 | // an and filter of query filters, otherwise just 203 | // build a single query filter. 204 | if (fqs.size() > 1) { 205 | AndFilterBuilder fqAnd = andFilter(); 206 | for (String fq : fqs) { 207 | QueryBuilder queryBuilder = fqDsl ? QueryBuilders.wrapperQuery(fq) : QueryBuilders.queryString(fq); 208 | fqAnd.add(queryFilter(queryBuilder)); 209 | } 210 | filterBuilder = fqAnd; 211 | } else { 212 | QueryBuilder queryBuilder = fqDsl ? QueryBuilders.wrapperQuery(fqs.get(0)) : QueryBuilders.queryString(fqs.get(0)); 213 | filterBuilder = queryFilter(queryBuilder); 214 | } 215 | 216 | searchSourceBuilder.filter(filterBuilder); 217 | } 218 | 219 | // handle highlighting 220 | if (hl) { 221 | // get supported highlighting parameters if they exist 222 | String hlfl = request.param("hl.fl"); 223 | int hlsnippets = request.paramAsInt("hl.snippets", 1); 224 | int hlfragsize = request.paramAsInt("hl.fragsize", 100); 225 | String hlsimplepre = request.param("hl.simple.pre"); 226 | String hlsimplepost = request.param("hl.simple.post"); 227 | 228 | HighlightBuilder highlightBuilder = new HighlightBuilder(); 229 | if (hlfl == null) { 230 | // run against default _all field 231 | highlightBuilder.field("_all", hlfragsize, hlsnippets); 232 | } else { 233 | String[] hlfls = hlfl.split("\\s|,"); 234 | for (String hlField : hlfls) { 235 | // skip wildcarded fields 236 | if (!hlField.contains("*")) { 237 | highlightBuilder.field(hlField, hlfragsize, hlsnippets); 238 | } 239 | } 240 | } 241 | 242 | // pre tags 243 | if (hlsimplepre != null) { 244 | highlightBuilder.preTags(hlsimplepre); 245 | } 246 | 247 | // post tags 248 | if (hlsimplepost != null) { 249 | highlightBuilder.postTags(hlsimplepost); 250 | } 251 | 252 | searchSourceBuilder.highlight(highlightBuilder); 253 | 254 | } 255 | 256 | // handle faceting 257 | if (facet) { 258 | // get supported facet parameters if they exist 259 | List facetFields = params.get("facet.field"); 260 | String facetSort = request.param("facet.sort"); 261 | int facetLimit = request.paramAsInt("facet.limit", 100); 262 | 263 | List facetQueries = params.get("facet.query"); 264 | 265 | if (facetFields != null && !facetFields.isEmpty()) { 266 | for (String facetField : facetFields) { 267 | TermsFacetBuilder termsFacetBuilder = new TermsFacetBuilder(facetField); 268 | termsFacetBuilder.size(facetLimit); 269 | termsFacetBuilder.field(facetField); 270 | 271 | if (facetSort != null && facetSort.equals("index")) { 272 | termsFacetBuilder.order(TermsFacet.ComparatorType.TERM); 273 | } else { 274 | termsFacetBuilder.order(TermsFacet.ComparatorType.COUNT); 275 | } 276 | 277 | searchSourceBuilder.facet(termsFacetBuilder); 278 | } 279 | } 280 | 281 | if (facetQueries != null && !facetQueries.isEmpty()) { 282 | for (String facetQuery : facetQueries) { 283 | QueryFacetBuilder queryFacetBuilder = new QueryFacetBuilder(facetQuery); 284 | queryFacetBuilder.query(QueryBuilders.queryString(facetQuery)); 285 | searchSourceBuilder.facet(queryFacetBuilder); 286 | } 287 | } 288 | } 289 | 290 | // get index and type we want to search against 291 | final String index = request.hasParam("index") ? request.param("index") : "solr"; 292 | final String type = request.hasParam("type") ? request.param("type") : "docs"; 293 | 294 | // Build the search Request 295 | String[] indices = RestActions.splitIndices(index); 296 | SearchRequest searchRequest = new SearchRequest(indices); 297 | searchRequest.extraSource(searchSourceBuilder); 298 | searchRequest.types(RestActions.splitTypes(type)); 299 | 300 | return searchRequest; 301 | } 302 | 303 | /** 304 | * Converts the search response into a NamedList that the Solr Response Writer can use. 305 | * 306 | * @param request the ES RestRequest 307 | * @param response the ES SearchResponse 308 | * @return a NamedList of the response 309 | */ 310 | private NamedList