├── .project
├── README.md
├── apache-real-time
    ├── flume.conf
    ├── morphline.conf
    ├── schema.xml
    └── solrconfig.xml
├── email-search
    ├── email-schema.xml
    ├── flume.config
    ├── indexer-config.xml
    ├── ks-indexer-email-morphlines.conf
    ├── morphlines.conf
    └── schema.xml
└── ocr
    ├── IdmpExtraction.scala
    ├── indexer-config.xml
    ├── morphlines.conf
    └── schema.xml


/.project:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0" encoding="UTF-8"?>
 2 | <projectDescription>
 3 | 	<name>cloudera-search</name>
 4 | 	<comment></comment>
 5 | 	<projects>
 6 | 	</projects>
 7 | 	<buildSpec>
 8 | 	</buildSpec>
 9 | 	<natures>
10 | 	</natures>
11 | </projectDescription>
12 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | cloudera-search
2 | ===============
3 | This repository has all you need to set up Cloudera Search in Environment
4 | 
5 | The directories are broken down by Use Cases
6 | 


--------------------------------------------------------------------------------
/apache-real-time/flume.conf:
--------------------------------------------------------------------------------
 1 | # Please paste flume.conf here. Example:
 2 | 
 3 | # Sources, channels, and sinks are defined per
 4 | # agent name, in this case 'tier1'.
 5 | tier1.sources  = source1
 6 | tier1.channels = channel1
 7 | tier1.sinks    = sink1
 8 | 
 9 | # For each source, channel, and sink, set
10 | # standard properties.
11 | 
12 | 
13 | 
14 | # Syslog Source Configuration
15 | tier1.sources.source1.type     = syslogtcp
16 | # the hostname that Flume Syslog source will be running on
17 | tier1.sources.source1.host     = localhost
18 | # the port that Flume Syslog source will listen on
19 | tier1.sources.source1.port     = 5040
20 | tier1.sources.source1.channels = channel1
21 | 
22 | 
23 | # Solr Sink configuration
24 | tier1.sinks.sink1.type          = org.apache.flume.sink.solr.morphline.MorphlineSolrSink
25 | tier1.sinks.sink1.morphlineFile = /tmp/morphline.conf
26 | tier1.sinks.sink1.channel       = channel1
27 | 
28 | 
29 | 
30 | # Kafka Channel Configuration
31 | tier1.channels.channel1.type                = org.apache.flume.channel.kafka.KafkaChannel
32 | tier1.channels.channel1.capacity            = 10000
33 | tier1.channels.channel1.transactionCapacity = 1000
34 | tier1.channels.channel1.brokerList          = kafkaf-2:9092,kafkaf-3:9092
35 | tier1.channels.channel1.topic               = channel1
36 | tier1.channels.channel1.zookeeperConnect    = kafkaf-1:2181
37 | tier1.channels.channel1.groupId = flume2
38 | 


--------------------------------------------------------------------------------
/apache-real-time/morphline.conf:
--------------------------------------------------------------------------------
  1 | # Specify server locations in a SOLR_LOCATOR variable;
  2 | # used later in variable substitutions
  3 | # Change the zkHost to point to your own Zookeeper quorum
  4 | SOLR_LOCATOR : {
  5 |     # Name of solr collection
  6 |     collection : accessCollection
  7 |     # ZooKeeper ensemble
  8 |     zkHost : "localhost:2181/solr"
  9 | }
 10 | 
 11 | # Specify an array of one or more morphlines, each of which defines an ETL
 12 | # transformation chain. A morphline consists of one or more (potentially
 13 | # nested) commands. A morphline is a way to consume records (e.g. Flume events,
 14 | # HDFS files or blocks), turn them into a stream of records, and pipe the stream
 15 | # of records through a set of easily configurable transformations on it's way to
 16 | # Solr (or a MapReduceIndexerTool RecordWriter that feeds via a Reducer into Solr).
 17 | morphlines : [
 18 | {
 19 |     # Name used to identify a morphline. E.g. used if there are multiple morphlines in a
 20 |     # morphline config file
 21 |     id : morphline1
 22 |     # Import all morphline commands in these java packages and their subpackages.
 23 |     # Other commands that may be present on the classpath are not visible to this morphline.
 24 |     importCommands : ["org.kitesdk.**", "org.apache.solr.**"]
 25 |     commands : [
 26 |     {
 27 |         ## Read the email stream and break it up into individual messages.
 28 |         ## The beginning of a message is marked by regex clause below
 29 |         ## The reason we use this command is that one event can have multiple
 30 |         ## messages
 31 |         readCSV {
 32 | 	    separator:  " "
 33 |             columns:  [client_ip,C1,C2,time,dummy1,request,code,bytes,referer,user_agent,C3]
 34 | 	    ignoreFirstLine : false
 35 |             quoteChar : "\""
 36 |             commentPrefix : ""
 37 |             trim : true
 38 |             charset : UTF-8
 39 |         }
 40 |     }
 41 |     {
 42 | 	split { 
 43 | 		inputField : request
 44 | 		outputFields : [method, url, protocol]          
 45 | 		separator : " "        
 46 | 		isRegex : false      
 47 | 		#separator : """\s*,\s*"""        
 48 | 		#  #isRegex : true      
 49 | 		addEmptyStrings : false
 50 | 		trim : true          
 51 |           }
 52 |     }
 53 |      {
 54 | 	split { 
 55 | 		inputField : url 
 56 | 		outputFields : ["", app, subapp]          
 57 | 		separator : "\/"        
 58 | 		isRegex : false      
 59 | 		#separator : """\s*,\s*"""        
 60 | 		#  #isRegex : true      
 61 | 		addEmptyStrings : false
 62 | 		trim : true          
 63 |           }
 64 |     }
 65 |     {
 66 | 	userAgent {
 67 | 		inputField : user_agent
 68 | 		outputFields : {
 69 | 			user_agent_family : "@{ua_family}"
 70 | 			user_agent_major  : "@{ua_major}"
 71 | 			device_family     : "@{device_family}"
 72 | 			os_family         : "@{os_family}"
 73 | 			os_major	  : "@{os_major}"
 74 | 		}          
 75 | 	}
 76 |     }
 77 |     {
 78 | 	#Extract GEO information
 79 | 	geoIP {
 80 |             inputField : client_ip
 81 |             database : "/tmp/GeoLite2-City.mmdb"
 82 | 	}
 83 |      }
 84 |      {
 85 | 
 86 | 	# extract parts of the geolocation info from the Jackson JsonNode Java 
 87 | 	# # object contained in the _attachment_body field and store the parts in
 88 | 	# # the given record output fields:      
 89 | 	extractJsonPaths {
 90 | 	flatten : false
 91 | 	paths : { 
 92 | 		country_code : /country/iso_code
 93 | 		country_name : /country/names/en
 94 |                 region_code  : /continent/code
 95 | 		#"/subdivisions[]/names/en" : "/subdivisions[]/names/en"     
 96 | 		#"/subdivisions[]/iso_code" : "/subdivisions[]/iso_code"     
 97 | 		city : /city/names/en
 98 | 		#/postal/code : /postal/code
 99 | 		latitude : /location/latitude
100 | 		longitude : /location/longitude
101 | 		#/location/latitude_longitude : /location/latitude_longitude
102 | 		#/location/longitude_latitude : /location/longitude_latitude
103 | 		} 
104 | 	}
105 |       }
106 |       #{logInfo { format : "BODY : {}", args : ["@{}"] } }
107 |     # add Unique ID, in case our message_id field from above is not present
108 |     {
109 |         generateUUID {
110 |             field:id
111 |         }
112 |     }
113 | 
114 |     # convert the timestamp field to "yyyy-MM-dd'T'HH:mm:ss.SSSZ" format
115 |     {
116 |        #  21/Nov/2014:22:08:27
117 |         convertTimestamp {
118 |             field : time 
119 |             inputFormats : ["[dd/MMM/yyyy:HH:mm:ss", "EEE, d MMM yyyy HH:mm:ss Z", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"]
120 |             inputTimezone : America/Los_Angeles
121 |            outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
122 |             outputTimezone : UTC
123 |         }
124 |     }
125 | 
126 |     # Consume the output record of the previous command and pipe another
127 |     # record downstream.
128 |     #
129 |     # This command sanitizes record fields that are unknown to Solr schema.xml
130 |     # by deleting them. Recall that Solr throws an exception on any attempt to
131 |     # load a document that contains a field that isn't specified in schema.xml
132 |     {
133 |         sanitizeUnknownSolrFields {
134 |             # Location from which to fetch Solr schema
135 |             solrLocator : ${SOLR_LOCATOR}
136 |         }
137 |     }
138 | 
139 |     # load the record into a SolrServer or MapReduce SolrOutputFormat.
140 |     {
141 |         loadSolr {
142 |             solrLocator : ${SOLR_LOCATOR}
143 |         }
144 |     }
145 |     ]
146 | }
147 | ]
148 | 
149 | 
150 | 


--------------------------------------------------------------------------------
/apache-real-time/schema.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8" ?>
  2 | <!--
  3 |  Licensed to the Apache Software Foundation (ASF) under one or more
  4 |  contributor license agreements.  See the NOTICE file distributed with
  5 |  this work for additional information regarding copyright ownership.
  6 |  The ASF licenses this file to You under the Apache License, Version 2.0
  7 |  (the "License"); you may not use this file except in compliance with
  8 |  the License.  You may obtain a copy of the License at
  9 | 
 10 |      http://www.apache.org/licenses/LICENSE-2.0
 11 | 
 12 |  Unless required by applicable law or agreed to in writing, software
 13 |  distributed under the License is distributed on an "AS IS" BASIS,
 14 |  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 |  See the License for the specific language governing permissions and
 16 |  limitations under the License.
 17 | -->
 18 | 
 19 | <!--  
 20 |  This is the Solr schema file. This file should be named "schema.xml" and
 21 |  should be in the conf directory under the solr home
 22 |  (i.e. ./solr/conf/schema.xml by default) 
 23 |  or located where the classloader for the Solr webapp can find it.
 24 | 
 25 |  This example schema is the recommended starting point for users.
 26 |  It should be kept correct and concise, usable out-of-the-box.
 27 | 
 28 |  For more information, on how to customize this file, please see
 29 |  http://wiki.apache.org/solr/SchemaXml
 30 | 
 31 |  PERFORMANCE NOTE: this schema includes many optional features and should not
 32 |  be used for benchmarking.  To improve performance one could
 33 |   - set stored="false" for all fields possible (esp large fields) when you
 34 |     only need to search on the field but don't need to return the original
 35 |     value.
 36 |   - set indexed="false" if you don't need to search on the field, but only
 37 |     return the field as a result of searching on other indexed fields.
 38 |   - remove all unneeded copyField statements
 39 |   - for best index size and searching performance, set "index" to false
 40 |     for all general text fields, use copyField to copy them to the
 41 |     catchall "text" field, and use that for searching.
 42 |   - For maximum indexing performance, use the StreamingUpdateSolrServer
 43 |     java client.
 44 |   - Remember to run the JVM in server mode, and use a higher logging level
 45 |     that avoids logging every request
 46 | -->
 47 | 
 48 | <schema name="example" version="1.5">
 49 |   <!-- attribute "name" is the name of this schema and is only used for display purposes.
 50 |        version="x.y" is Solr's version number for the schema syntax and 
 51 |        semantics.  It should not normally be changed by applications.
 52 | 
 53 |        1.0: multiValued attribute did not exist, all fields are multiValued 
 54 |             by nature
 55 |        1.1: multiValued attribute introduced, false by default 
 56 |        1.2: omitTermFreqAndPositions attribute introduced, true by default 
 57 |             except for text fields.
 58 |        1.3: removed optional field compress feature
 59 |        1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser
 60 |             behavior when a single string produces multiple tokens.  Defaults 
 61 |             to off for version >= 1.4
 62 |        1.5: omitNorms defaults to true for primitive field types 
 63 |             (int, float, boolean, string...)
 64 |      -->
 65 | 
 66 |  <fields>
 67 |    <!-- Valid attributes for fields:
 68 |      name: mandatory - the name for the field
 69 |      type: mandatory - the name of a field type from the 
 70 |        <types> fieldType section
 71 |      indexed: true if this field should be indexed (searchable or sortable)
 72 |      stored: true if this field should be retrievable
 73 |      docValues: true if this field should have doc values. Doc values are
 74 |        useful for faceting, grouping, sorting and function queries. Although not
 75 |        required, doc values will make the index faster to load, more
 76 |        NRT-friendly and more memory-efficient. They however come with some
 77 |        limitations: they are currently only supported by StrField, UUIDField
 78 |        and all Trie*Fields, and depending on the field type, they might
 79 |        require the field to be single-valued, be required or have a default
 80 |        value (check the documentation of the field type you're interested in
 81 |        for more information)
 82 |      multiValued: true if this field may contain multiple values per document
 83 |      omitNorms: (expert) set to true to omit the norms associated with
 84 |        this field (this disables length normalization and index-time
 85 |        boosting for the field, and saves some memory).  Only full-text
 86 |        fields or fields that need an index-time boost need norms.
 87 |        Norms are omitted for primitive (non-analyzed) types by default.
 88 |      termVectors: [false] set to true to store the term vector for a
 89 |        given field.
 90 |        When using MoreLikeThis, fields used for similarity should be
 91 |        stored for best performance.
 92 |      termPositions: Store position information with the term vector.  
 93 |        This will increase storage costs.
 94 |      termOffsets: Store offset information with the term vector. This 
 95 |        will increase storage costs.
 96 |      required: The field is required.  It will throw an error if the
 97 |        value does not exist
 98 |      default: a value that should be used if no value is specified
 99 |        when adding a document.
100 |    -->
101 | 
102 |    <!-- field names should consist of alphanumeric or underscore characters only and
103 |       not start with a digit.  This is not currently strictly enforced,
104 |       but other field names will not have first class support from all components
105 |       and back compatibility is not guaranteed.  Names with both leading and
106 |       trailing underscores (e.g. _version_) are reserved.
107 |    -->
108 | 
109 |    <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" />
110 |    <field name="time" type="tdate" indexed="true" stored="true" />
111 |    <field name="record" type="text_general" indexed="true" stored="false" multiValued="true"/>
112 |    <field name="client_ip" type="string" indexed="true" stored="true" />
113 |    <field name="code" type="string" indexed="true" stored="true" />
114 |    <field name="user_agent" type="string" indexed="true" stored="true" />
115 |    <field name="protocol" type="string" indexed="true" stored="true" />   
116 |    <field name="url" type="string" indexed="true" stored="true" />   
117 |    <field name="request" type="string" indexed="true" stored="true" />
118 |    <field name="referer" type="string" indexed="true" stored="true" />
119 |    <field name="bytes" type="tint" indexed="true" stored="true" />
120 |    <field name="method" type="string" indexed="true" stored="true" />
121 |    
122 |    <field name="extension" type="string" indexed="true" stored="true" />   
123 |    <field name="app" type="string" indexed="true" stored="true" />      
124 |    <field name="subapp" type="string" indexed="true" stored="true" />
125 |       
126 |    <field name="device_family" type="string" indexed="true" stored="true" />
127 |    <field name="user_agent_major" type="string" indexed="true" stored="true" />   
128 |    <field name="user_agent_family" type="string" indexed="true" stored="true" />
129 |    <field name="os_family" type="string" indexed="true" stored="true" />   
130 |    <field name="os_major" type="string" indexed="true" stored="true" />
131 |    
132 |    <field name="region_code" type="string" indexed="true" stored="true" />
133 |    <field name="country_code" type="string" indexed="true" stored="true" />
134 |    <field name="city" type="string" indexed="true" stored="true" />
135 |    <field name="latitude" type="float" indexed="true" stored="true" />
136 |    <field name="longitude" type="float" indexed="true" stored="true" />
137 |    <field name="country_name" type="string" indexed="true" stored="true" />
138 |    <field name="country_code3" type="string" indexed="true" stored="true" />
139 | 
140 |    <field name="_version_" type="long" indexed="true" stored="true"/>
141 |    <dynamicField name="ignored_*" type="ignored"/>
142 |    
143 |  </fields>
144 | 
145 | 
146 |  <!-- Field to use to determine and enforce document uniqueness. 
147 |       Unless this field is marked with required="false", it will be a required field
148 |    -->
149 |  <uniqueKey>id</uniqueKey>
150 | 
151 | 
152 |   <copyField source="time" dest="record"/>
153 |   <copyField source="client_ip" dest="record"/>
154 |   <copyField source="code" dest="record"/>
155 |   <copyField source="user_agent" dest="record"/>
156 |   <copyField source="url" dest="record"/>
157 |   <copyField source="protocol" dest="record"/>
158 |   <copyField source="referer" dest="record"/>
159 |   <copyField source="request" dest="record"/>
160 |   <copyField source="bytes" dest="record"/>
161 |   <copyField source="method" dest="record"/>
162 |   <copyField source="extension" dest="record"/>
163 |   <copyField source="app" dest="record"/>
164 |   <copyField source="subapp" dest="record"/>
165 |   <copyField source="device_family" dest="record"/>
166 |   <copyField source="user_agent_major" dest="record"/>
167 |   <copyField source="user_agent_family" dest="record"/>
168 |   <copyField source="os_family" dest="record"/>
169 |   <copyField source="os_major" dest="record"/>
170 |   <copyField source="region_code" dest="record"/>
171 |   <copyField source="country_code" dest="record"/>
172 |   <copyField source="city" dest="record"/>
173 |   <copyField source="latitude" dest="record"/>
174 |   <copyField source="longitude" dest="record"/>
175 |   <copyField source="country_name" dest="record"/>
176 | 
177 |  <!-- DEPRECATED: The defaultSearchField is consulted by various query parsers when
178 |   parsing a query string that isn't explicit about the field.  Machine (non-user)
179 |   generated queries are best made explicit, or they can use the "df" request parameter
180 |   which takes precedence over this.
181 |   Note: Un-commenting defaultSearchField will be insufficient if your request handler
182 |   in solrconfig.xml defines "df", which takes precedence. That would need to be removed.
183 |  <defaultSearchField>text</defaultSearchField> -->
184 | 
185 |  <!-- DEPRECATED: The defaultOperator (AND|OR) is consulted by various query parsers
186 |   when parsing a query string to determine if a clause of the query should be marked as
187 |   required or optional, assuming the clause isn't already marked by some operator.
188 |   The default is OR, which is generally assumed so it is not a good idea to change it
189 |   globally here.  The "q.op" request parameter takes precedence over this.
190 |  <solrQueryParser defaultOperator="OR"/> -->
191 | 
192 |   <!-- copyField commands copy one field to another at the time a document
193 |         is added to the index.  It's used either to index the same field differently,
194 |         or to add multiple fields to the same field for easier/faster searching.  -->
195 | 
196 |   
197 |   <types>
198 |     <!-- field type definitions. The "name" attribute is
199 |        just a label to be used by field definitions.  The "class"
200 |        attribute and any other attributes determine the real
201 |        behavior of the fieldType.
202 |          Class names starting with "solr" refer to java classes in a
203 |        standard package such as org.apache.solr.analysis
204 |     -->
205 | 
206 |     <!-- The StrField type is not analyzed, but indexed/stored verbatim.
207 |        It supports doc values but in that case the field needs to be
208 |        single-valued and either required or have a default value.
209 |       -->
210 |     <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
211 | 
212 |     <!-- boolean type: "true" or "false" -->
213 |     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
214 | 
215 |     <!-- sortMissingLast and sortMissingFirst attributes are optional attributes are
216 |          currently supported on types that are sorted internally as strings
217 |          and on numeric types.
218 | 	     This includes "string","boolean", and, as of 3.5 (and 4.x),
219 | 	     int, float, long, date, double, including the "Trie" variants.
220 |        - If sortMissingLast="true", then a sort on this field will cause documents
221 |          without the field to come after documents with the field,
222 |          regardless of the requested sort order (asc or desc).
223 |        - If sortMissingFirst="true", then a sort on this field will cause documents
224 |          without the field to come before documents with the field,
225 |          regardless of the requested sort order.
226 |        - If sortMissingLast="false" and sortMissingFirst="false" (the default),
227 |          then default lucene sorting will be used which places docs without the
228 |          field first in an ascending sort and last in a descending sort.
229 |     -->    
230 | 
231 |     <!--
232 |       Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
233 | 
234 |       These fields support doc values, but they require the field to be
235 |       single-valued and either be required or have a default value.
236 |     -->
237 |     <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
238 |     <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
239 |     <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
240 |     <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
241 | 
242 |     <!--
243 |      Numeric field types that index each value at various levels of precision
244 |      to accelerate range queries when the number of values between the range
245 |      endpoints is large. See the javadoc for NumericRangeQuery for internal
246 |      implementation details.
247 | 
248 |      Smaller precisionStep values (specified in bits) will lead to more tokens
249 |      indexed per value, slightly larger index size, and faster range queries.
250 |      A precisionStep of 0 disables indexing at different precision levels.
251 |     -->
252 |     <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
253 |     <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
254 |     <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
255 |     <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
256 | 
257 |     <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
258 |          is a more restricted form of the canonical representation of dateTime
259 |          http://www.w3.org/TR/xmlschema-2/#dateTime    
260 |          The trailing "Z" designates UTC time and is mandatory.
261 |          Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
262 |          All other components are mandatory.
263 | 
264 |          Expressions can also be used to denote calculations that should be
265 |          performed relative to "NOW" to determine the value, ie...
266 | 
267 |                NOW/HOUR
268 |                   ... Round to the start of the current hour
269 |                NOW-1DAY
270 |                   ... Exactly 1 day prior to now
271 |                NOW/DAY+6MONTHS+3DAYS
272 |                   ... 6 months and 3 days in the future from the start of
273 |                       the current day
274 |                       
275 |          Consult the DateField javadocs for more information.
276 | 
277 |          Note: For faster range queries, consider the tdate type
278 |       -->
279 |     <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
280 | 
281 |     <!-- A Trie based date field for faster date range queries and date faceting. -->
282 |     <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
283 | 
284 | 
285 |     <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
286 |     <fieldtype name="binary" class="solr.BinaryField"/>
287 | 
288 |     <!--
289 |       Note:
290 |       These should only be used for compatibility with existing indexes (created with lucene or older Solr versions).
291 |       Use Trie based fields instead. As of Solr 3.5 and 4.x, Trie based fields support sortMissingFirst/Last
292 |       
293 |       Plain numeric field types that store and index the text
294 |       value verbatim (and hence don't correctly support range queries, since the
295 |       lexicographic ordering isn't equal to the numeric ordering)
296 |     -->
297 |     <fieldType name="pint" class="solr.IntField"/>
298 |     <fieldType name="plong" class="solr.LongField"/>
299 |     <fieldType name="pfloat" class="solr.FloatField"/>
300 |     <fieldType name="pdouble" class="solr.DoubleField"/>
301 |     <fieldType name="pdate" class="solr.DateField" sortMissingLast="true"/>
302 | 
303 |     <!-- The "RandomSortField" is not used to store or search any
304 |          data.  You can declare fields of this type it in your schema
305 |          to generate pseudo-random orderings of your docs for sorting 
306 |          or function purposes.  The ordering is generated based on the field
307 |          name and the version of the index. As long as the index version
308 |          remains unchanged, and the same field name is reused,
309 |          the ordering of the docs will be consistent.  
310 |          If you want different psuedo-random orderings of documents,
311 |          for the same version of the index, use a dynamicField and
312 |          change the field name in the request.
313 |      -->
314 |     <fieldType name="random" class="solr.RandomSortField" indexed="true" />
315 | 
316 |     <!-- solr.TextField allows the specification of custom text analyzers
317 |          specified as a tokenizer and a list of token filters. Different
318 |          analyzers may be specified for indexing and querying.
319 | 
320 |          The optional positionIncrementGap puts space between multiple fields of
321 |          this type on the same document, with the purpose of preventing false phrase
322 |          matching across fields.
323 | 
324 |          For more info on customizing your analyzer chain, please see
325 |          http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
326 |      -->
327 | 
328 |     <!-- One can also specify an existing Analyzer class that has a
329 |          default constructor via the class attribute on the analyzer element.
330 |          Example:
331 |     <fieldType name="text_greek" class="solr.TextField">
332 |       <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
333 |     </fieldType>
334 |     -->
335 | 
336 |     <!-- A text field that only splits on whitespace for exact matching of words -->
337 |     <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
338 |       <analyzer>
339 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
340 |       </analyzer>
341 |     </fieldType>
342 | 
343 |     <!-- A general text field that has reasonable, generic
344 |          cross-language defaults: it tokenizes with StandardTokenizer,
345 | 	 removes stop words from case-insensitive "stopwords.txt"
346 | 	 (empty by default), and down cases.  At query time only, it
347 | 	 also applies synonyms. -->
348 |     <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
349 |       <analyzer type="index">
350 |         <tokenizer class="solr.StandardTokenizerFactory"/>
351 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
352 |         <!-- in this example, we will only use synonyms at query time
353 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
354 |         -->
355 |         <filter class="solr.LowerCaseFilterFactory"/>
356 |       </analyzer>
357 |       <analyzer type="query">
358 |         <tokenizer class="solr.StandardTokenizerFactory"/>
359 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
360 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
361 |         <filter class="solr.LowerCaseFilterFactory"/>
362 |       </analyzer>
363 |     </fieldType>
364 | 
365 |     <!-- A text field with defaults appropriate for English: it
366 |          tokenizes with StandardTokenizer, removes English stop words
367 |          (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
368 |          finally applies Porter's stemming.  The query time analyzer
369 |          also applies synonyms from synonyms.txt. -->
370 |     <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
371 |       <analyzer type="index">
372 |         <tokenizer class="solr.StandardTokenizerFactory"/>
373 |         <!-- in this example, we will only use synonyms at query time
374 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
375 |         -->
376 |         <!-- Case insensitive stop word removal.
377 |           add enablePositionIncrements=true in both the index and query
378 |           analyzers to leave a 'gap' for more accurate phrase queries.
379 |         -->
380 |         <filter class="solr.StopFilterFactory"
381 |                 ignoreCase="true"
382 |                 words="lang/stopwords_en.txt"
383 |                 enablePositionIncrements="true"
384 |                 />
385 |         <filter class="solr.LowerCaseFilterFactory"/>
386 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
387 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
388 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
389 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
390 | 	-->
391 |         <filter class="solr.PorterStemFilterFactory"/>
392 |       </analyzer>
393 |       <analyzer type="query">
394 |         <tokenizer class="solr.StandardTokenizerFactory"/>
395 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
396 |         <filter class="solr.StopFilterFactory"
397 |                 ignoreCase="true"
398 |                 words="lang/stopwords_en.txt"
399 |                 enablePositionIncrements="true"
400 |                 />
401 |         <filter class="solr.LowerCaseFilterFactory"/>
402 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
403 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
404 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
405 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
406 | 	-->
407 |         <filter class="solr.PorterStemFilterFactory"/>
408 |       </analyzer>
409 |     </fieldType>
410 | 
411 |     <!-- A text field with defaults appropriate for English, plus
412 | 	 aggressive word-splitting and autophrase features enabled.
413 | 	 This field is just like text_en, except it adds
414 | 	 WordDelimiterFilter to enable splitting and matching of
415 | 	 words on case-change, alpha numeric boundaries, and
416 | 	 non-alphanumeric chars.  This means certain compound word
417 | 	 cases will work, for example query "wi fi" will match
418 | 	 document "WiFi" or "wi-fi".
419 |         -->
420 |     <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
421 |       <analyzer type="index">
422 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
423 |         <!-- in this example, we will only use synonyms at query time
424 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
425 |         -->
426 |         <!-- Case insensitive stop word removal.
427 |           add enablePositionIncrements=true in both the index and query
428 |           analyzers to leave a 'gap' for more accurate phrase queries.
429 |         -->
430 |         <filter class="solr.StopFilterFactory"
431 |                 ignoreCase="true"
432 |                 words="lang/stopwords_en.txt"
433 |                 enablePositionIncrements="true"
434 |                 />
435 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
436 |         <filter class="solr.LowerCaseFilterFactory"/>
437 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
438 |         <filter class="solr.PorterStemFilterFactory"/>
439 |       </analyzer>
440 |       <analyzer type="query">
441 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
442 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
443 |         <filter class="solr.StopFilterFactory"
444 |                 ignoreCase="true"
445 |                 words="lang/stopwords_en.txt"
446 |                 enablePositionIncrements="true"
447 |                 />
448 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
449 |         <filter class="solr.LowerCaseFilterFactory"/>
450 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
451 |         <filter class="solr.PorterStemFilterFactory"/>
452 |       </analyzer>
453 |     </fieldType>
454 | 
455 |     <!-- Less flexible matching, but less false matches.  Probably not ideal for product names,
456 |          but may be good for SKUs.  Can insert dashes in the wrong place and still match. -->
457 |     <fieldType name="text_en_splitting_tight" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
458 |       <analyzer>
459 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
460 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
461 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
462 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
463 |         <filter class="solr.LowerCaseFilterFactory"/>
464 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
465 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
466 |         <!-- this filter can remove any duplicate tokens that appear at the same position - sometimes
467 |              possible with WordDelimiterFilter in conjuncton with stemming. -->
468 |         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
469 |       </analyzer>
470 |     </fieldType>
471 | 
472 |     <!-- Just like text_general except it reverses the characters of
473 | 	 each token, to enable more efficient leading wildcard queries. -->
474 |     <fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
475 |       <analyzer type="index">
476 |         <tokenizer class="solr.StandardTokenizerFactory"/>
477 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
478 |         <filter class="solr.LowerCaseFilterFactory"/>
479 |         <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
480 |            maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
481 |       </analyzer>
482 |       <analyzer type="query">
483 |         <tokenizer class="solr.StandardTokenizerFactory"/>
484 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
485 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
486 |         <filter class="solr.LowerCaseFilterFactory"/>
487 |       </analyzer>
488 |     </fieldType>
489 | 
490 |     <!-- charFilter + WhitespaceTokenizer  -->
491 |     <!--
492 |     <fieldType name="text_char_norm" class="solr.TextField" positionIncrementGap="100" >
493 |       <analyzer>
494 |         <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
495 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
496 |       </analyzer>
497 |     </fieldType>
498 |     -->
499 | 
500 |     <!-- This is an example of using the KeywordTokenizer along
501 |          With various TokenFilterFactories to produce a sortable field
502 |          that does not include some properties of the source text
503 |       -->
504 |     <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
505 |       <analyzer>
506 |         <!-- KeywordTokenizer does no actual tokenizing, so the entire
507 |              input string is preserved as a single token
508 |           -->
509 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
510 |         <!-- The LowerCase TokenFilter does what you expect, which can be
511 |              when you want your sorting to be case insensitive
512 |           -->
513 |         <filter class="solr.LowerCaseFilterFactory" />
514 |         <!-- The TrimFilter removes any leading or trailing whitespace -->
515 |         <filter class="solr.TrimFilterFactory" />
516 |         <!-- The PatternReplaceFilter gives you the flexibility to use
517 |              Java Regular expression to replace any sequence of characters
518 |              matching a pattern with an arbitrary replacement string, 
519 |              which may include back references to portions of the original
520 |              string matched by the pattern.
521 |              
522 |              See the Java Regular Expression documentation for more
523 |              information on pattern and replacement string syntax.
524 |              
525 |              http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
526 |           -->
527 |         <filter class="solr.PatternReplaceFilterFactory"
528 |                 pattern="([^a-z])" replacement="" replace="all"
529 |         />
530 |       </analyzer>
531 |     </fieldType>
532 |     
533 |     <fieldtype name="phonetic" stored="false" indexed="true" class="solr.TextField" >
534 |       <analyzer>
535 |         <tokenizer class="solr.StandardTokenizerFactory"/>
536 |         <filter class="solr.DoubleMetaphoneFilterFactory" inject="false"/>
537 |       </analyzer>
538 |     </fieldtype>
539 | 
540 |     <fieldtype name="payloads" stored="false" indexed="true" class="solr.TextField" >
541 |       <analyzer>
542 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
543 |         <!--
544 |         The DelimitedPayloadTokenFilter can put payloads on tokens... for example,
545 |         a token of "foo|1.4"  would be indexed as "foo" with a payload of 1.4f
546 |         Attributes of the DelimitedPayloadTokenFilterFactory : 
547 |          "delimiter" - a one character delimiter. Default is | (pipe)
548 | 	 "encoder" - how to encode the following value into a playload
549 | 	    float -> org.apache.lucene.analysis.payloads.FloatEncoder,
550 | 	    integer -> o.a.l.a.p.IntegerEncoder
551 | 	    identity -> o.a.l.a.p.IdentityEncoder
552 |             Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor.
553 |          -->
554 |         <filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="float"/>
555 |       </analyzer>
556 |     </fieldtype>
557 | 
558 |     <!-- lowercases the entire field value, keeping it as a single token.  -->
559 |     <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
560 |       <analyzer>
561 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
562 |         <filter class="solr.LowerCaseFilterFactory" />
563 |       </analyzer>
564 |     </fieldType>
565 | 
566 |     <!-- 
567 |       Example of using PathHierarchyTokenizerFactory at index time, so
568 |       queries for paths match documents at that path, or in descendent paths
569 |     -->
570 |     <fieldType name="descendent_path" class="solr.TextField">
571 |       <analyzer type="index">
572 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
573 |       </analyzer>
574 |       <analyzer type="query">
575 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
576 |       </analyzer>
577 |     </fieldType>
578 |     <!-- 
579 |       Example of using PathHierarchyTokenizerFactory at query time, so
580 |       queries for paths match documents at that path, or in ancestor paths
581 |     -->
582 |     <fieldType name="ancestor_path" class="solr.TextField">
583 |       <analyzer type="index">
584 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
585 |       </analyzer>
586 |       <analyzer type="query">
587 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
588 |       </analyzer>
589 |     </fieldType>
590 | 
591 |     <!-- since fields of this type are by default not stored or indexed,
592 |          any data added to them will be ignored outright.  --> 
593 |     <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
594 | 
595 |     <!-- This point type indexes the coordinates as separate fields (subFields)
596 |       If subFieldType is defined, it references a type, and a dynamic field
597 |       definition is created matching *___<typename>.  Alternately, if 
598 |       subFieldSuffix is defined, that is used to create the subFields.
599 |       Example: if subFieldType="double", then the coordinates would be
600 |         indexed in fields myloc_0___double,myloc_1___double.
601 |       Example: if subFieldSuffix="_d" then the coordinates would be indexed
602 |         in fields myloc_0_d,myloc_1_d
603 |       The subFields are an implementation detail of the fieldType, and end
604 |       users normally should not need to know about them.
605 |      -->
606 |     <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
607 | 
608 |     <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
609 |     <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
610 | 
611 |     <!-- An alternative geospatial field type new to Solr 4.  It supports multiValued and polygon shapes.
612 |       For more information about this and other Spatial fields new to Solr 4, see:
613 |       http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
614 |     -->
615 |     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
616 |         geo="true" distErrPct="0.025" maxDistErr="0.000009" units="degrees" />
617 | 
618 |    <!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType
619 |         Parameters:
620 |           defaultCurrency: Specifies the default currency if none specified. Defaults to "USD"
621 |           precisionStep:   Specifies the precisionStep for the TrieLong field used for the amount
622 |           providerClass:   Lets you plug in other exchange provider backend:
623 |                            solr.FileExchangeRateProvider is the default and takes one parameter:
624 |                              currencyConfig: name of an xml file holding exchange rates
625 |                            solr.OpenExchangeRatesOrgProvider uses rates from openexchangerates.org:
626 |                              ratesFileLocation: URL or path to rates JSON file (default latest.json on the web)
627 |                              refreshInterval: Number of minutes between each rates fetch (default: 1440, min: 60)
628 |    -->
629 |     <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
630 |              
631 | 
632 | 
633 |    <!-- some examples for different languages (generally ordered by ISO code) -->
634 | 
635 |     <!-- Arabic -->
636 |     <fieldType name="text_ar" class="solr.TextField" positionIncrementGap="100">
637 |       <analyzer> 
638 |         <tokenizer class="solr.StandardTokenizerFactory"/>
639 |         <!-- for any non-arabic -->
640 |         <filter class="solr.LowerCaseFilterFactory"/>
641 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ar.txt" enablePositionIncrements="true"/>
642 |         <!-- normalizes ﻯ to ﻱ, etc -->
643 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
644 |         <filter class="solr.ArabicStemFilterFactory"/>
645 |       </analyzer>
646 |     </fieldType>
647 | 
648 |     <!-- Bulgarian -->
649 |     <fieldType name="text_bg" class="solr.TextField" positionIncrementGap="100">
650 |       <analyzer> 
651 |         <tokenizer class="solr.StandardTokenizerFactory"/> 
652 |         <filter class="solr.LowerCaseFilterFactory"/>
653 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_bg.txt" enablePositionIncrements="true"/>
654 |         <filter class="solr.BulgarianStemFilterFactory"/>       
655 |       </analyzer>
656 |     </fieldType>
657 |     
658 |     <!-- Catalan -->
659 |     <fieldType name="text_ca" class="solr.TextField" positionIncrementGap="100">
660 |       <analyzer> 
661 |         <tokenizer class="solr.StandardTokenizerFactory"/>
662 |         <!-- removes l', etc -->
663 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ca.txt"/>
664 |         <filter class="solr.LowerCaseFilterFactory"/>
665 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ca.txt" enablePositionIncrements="true"/>
666 |         <filter class="solr.SnowballPorterFilterFactory" language="Catalan"/>       
667 |       </analyzer>
668 |     </fieldType>
669 |     
670 |     <!-- CJK bigram (see text_ja for a Japanese configuration using morphological analysis) -->
671 |     <fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
672 |       <analyzer>
673 |         <tokenizer class="solr.StandardTokenizerFactory"/>
674 |         <!-- normalize width before bigram, as e.g. half-width dakuten combine  -->
675 |         <filter class="solr.CJKWidthFilterFactory"/>
676 |         <!-- for any non-CJK -->
677 |         <filter class="solr.LowerCaseFilterFactory"/>
678 |         <filter class="solr.CJKBigramFilterFactory"/>
679 |       </analyzer>
680 |     </fieldType>
681 | 
682 |     <!-- Czech -->
683 |     <fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
684 |       <analyzer> 
685 |         <tokenizer class="solr.StandardTokenizerFactory"/>
686 |         <filter class="solr.LowerCaseFilterFactory"/>
687 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_cz.txt" enablePositionIncrements="true"/>
688 |         <filter class="solr.CzechStemFilterFactory"/>       
689 |       </analyzer>
690 |     </fieldType>
691 |     
692 |     <!-- Danish -->
693 |     <fieldType name="text_da" class="solr.TextField" positionIncrementGap="100">
694 |       <analyzer> 
695 |         <tokenizer class="solr.StandardTokenizerFactory"/>
696 |         <filter class="solr.LowerCaseFilterFactory"/>
697 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_da.txt" format="snowball" enablePositionIncrements="true"/>
698 |         <filter class="solr.SnowballPorterFilterFactory" language="Danish"/>       
699 |       </analyzer>
700 |     </fieldType>
701 |     
702 |     <!-- German -->
703 |     <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
704 |       <analyzer> 
705 |         <tokenizer class="solr.StandardTokenizerFactory"/>
706 |         <filter class="solr.LowerCaseFilterFactory"/>
707 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_de.txt" format="snowball" enablePositionIncrements="true"/>
708 |         <filter class="solr.GermanNormalizationFilterFactory"/>
709 |         <filter class="solr.GermanLightStemFilterFactory"/>
710 |         <!-- less aggressive: <filter class="solr.GermanMinimalStemFilterFactory"/> -->
711 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="German2"/> -->
712 |       </analyzer>
713 |     </fieldType>
714 |     
715 |     <!-- Greek -->
716 |     <fieldType name="text_el" class="solr.TextField" positionIncrementGap="100">
717 |       <analyzer> 
718 |         <tokenizer class="solr.StandardTokenizerFactory"/>
719 |         <!-- greek specific lowercase for sigma -->
720 |         <filter class="solr.GreekLowerCaseFilterFactory"/>
721 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_el.txt" enablePositionIncrements="true"/>
722 |         <filter class="solr.GreekStemFilterFactory"/>
723 |       </analyzer>
724 |     </fieldType>
725 |     
726 |     <!-- Spanish -->
727 |     <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
728 |       <analyzer> 
729 |         <tokenizer class="solr.StandardTokenizerFactory"/>
730 |         <filter class="solr.LowerCaseFilterFactory"/>
731 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" format="snowball" enablePositionIncrements="true"/>
732 |         <filter class="solr.SpanishLightStemFilterFactory"/>
733 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Spanish"/> -->
734 |       </analyzer>
735 |     </fieldType>
736 |     
737 |     <!-- Basque -->
738 |     <fieldType name="text_eu" class="solr.TextField" positionIncrementGap="100">
739 |       <analyzer> 
740 |         <tokenizer class="solr.StandardTokenizerFactory"/>
741 |         <filter class="solr.LowerCaseFilterFactory"/>
742 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_eu.txt" enablePositionIncrements="true"/>
743 |         <filter class="solr.SnowballPorterFilterFactory" language="Basque"/>
744 |       </analyzer>
745 |     </fieldType>
746 |     
747 |     <!-- Persian -->
748 |     <fieldType name="text_fa" class="solr.TextField" positionIncrementGap="100">
749 |       <analyzer>
750 |         <!-- for ZWNJ -->
751 |         <charFilter class="solr.PersianCharFilterFactory"/>
752 |         <tokenizer class="solr.StandardTokenizerFactory"/>
753 |         <filter class="solr.LowerCaseFilterFactory"/>
754 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
755 |         <filter class="solr.PersianNormalizationFilterFactory"/>
756 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fa.txt" enablePositionIncrements="true"/>
757 |       </analyzer>
758 |     </fieldType>
759 |     
760 |     <!-- Finnish -->
761 |     <fieldType name="text_fi" class="solr.TextField" positionIncrementGap="100">
762 |       <analyzer> 
763 |         <tokenizer class="solr.StandardTokenizerFactory"/>
764 |         <filter class="solr.LowerCaseFilterFactory"/>
765 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fi.txt" format="snowball" enablePositionIncrements="true"/>
766 |         <filter class="solr.SnowballPorterFilterFactory" language="Finnish"/>
767 |         <!-- less aggressive: <filter class="solr.FinnishLightStemFilterFactory"/> -->
768 |       </analyzer>
769 |     </fieldType>
770 |     
771 |     <!-- French -->
772 |     <fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
773 |       <analyzer> 
774 |         <tokenizer class="solr.StandardTokenizerFactory"/>
775 |         <!-- removes l', etc -->
776 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_fr.txt"/>
777 |         <filter class="solr.LowerCaseFilterFactory"/>
778 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" format="snowball" enablePositionIncrements="true"/>
779 |         <filter class="solr.FrenchLightStemFilterFactory"/>
780 |         <!-- less aggressive: <filter class="solr.FrenchMinimalStemFilterFactory"/> -->
781 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="French"/> -->
782 |       </analyzer>
783 |     </fieldType>
784 |     
785 |     <!-- Irish -->
786 |     <fieldType name="text_ga" class="solr.TextField" positionIncrementGap="100">
787 |       <analyzer> 
788 |         <tokenizer class="solr.StandardTokenizerFactory"/>
789 |         <!-- removes d', etc -->
790 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ga.txt"/>
791 |         <!-- removes n-, etc. position increments is intentionally false! -->
792 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/hyphenations_ga.txt" enablePositionIncrements="false"/>
793 |         <filter class="solr.IrishLowerCaseFilterFactory"/>
794 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ga.txt" enablePositionIncrements="true"/>
795 |         <filter class="solr.SnowballPorterFilterFactory" language="Irish"/>
796 |       </analyzer>
797 |     </fieldType>
798 |     
799 |     <!-- Galician -->
800 |     <fieldType name="text_gl" class="solr.TextField" positionIncrementGap="100">
801 |       <analyzer> 
802 |         <tokenizer class="solr.StandardTokenizerFactory"/>
803 |         <filter class="solr.LowerCaseFilterFactory"/>
804 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_gl.txt" enablePositionIncrements="true"/>
805 |         <filter class="solr.GalicianStemFilterFactory"/>
806 |         <!-- less aggressive: <filter class="solr.GalicianMinimalStemFilterFactory"/> -->
807 |       </analyzer>
808 |     </fieldType>
809 |     
810 |     <!-- Hindi -->
811 |     <fieldType name="text_hi" class="solr.TextField" positionIncrementGap="100">
812 |       <analyzer> 
813 |         <tokenizer class="solr.StandardTokenizerFactory"/>
814 |         <filter class="solr.LowerCaseFilterFactory"/>
815 |         <!-- normalizes unicode representation -->
816 |         <filter class="solr.IndicNormalizationFilterFactory"/>
817 |         <!-- normalizes variation in spelling -->
818 |         <filter class="solr.HindiNormalizationFilterFactory"/>
819 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hi.txt" enablePositionIncrements="true"/>
820 |         <filter class="solr.HindiStemFilterFactory"/>
821 |       </analyzer>
822 |     </fieldType>
823 |     
824 |     <!-- Hungarian -->
825 |     <fieldType name="text_hu" class="solr.TextField" positionIncrementGap="100">
826 |       <analyzer> 
827 |         <tokenizer class="solr.StandardTokenizerFactory"/>
828 |         <filter class="solr.LowerCaseFilterFactory"/>
829 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hu.txt" format="snowball" enablePositionIncrements="true"/>
830 |         <filter class="solr.SnowballPorterFilterFactory" language="Hungarian"/>
831 |         <!-- less aggressive: <filter class="solr.HungarianLightStemFilterFactory"/> -->   
832 |       </analyzer>
833 |     </fieldType>
834 |     
835 |     <!-- Armenian -->
836 |     <fieldType name="text_hy" class="solr.TextField" positionIncrementGap="100">
837 |       <analyzer> 
838 |         <tokenizer class="solr.StandardTokenizerFactory"/>
839 |         <filter class="solr.LowerCaseFilterFactory"/>
840 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hy.txt" enablePositionIncrements="true"/>
841 |         <filter class="solr.SnowballPorterFilterFactory" language="Armenian"/>
842 |       </analyzer>
843 |     </fieldType>
844 |     
845 |     <!-- Indonesian -->
846 |     <fieldType name="text_id" class="solr.TextField" positionIncrementGap="100">
847 |       <analyzer> 
848 |         <tokenizer class="solr.StandardTokenizerFactory"/>
849 |         <filter class="solr.LowerCaseFilterFactory"/>
850 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_id.txt" enablePositionIncrements="true"/>
851 |         <!-- for a less aggressive approach (only inflectional suffixes), set stemDerivational to false -->
852 |         <filter class="solr.IndonesianStemFilterFactory" stemDerivational="true"/>
853 |       </analyzer>
854 |     </fieldType>
855 |     
856 |     <!-- Italian -->
857 |     <fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
858 |       <analyzer> 
859 |         <tokenizer class="solr.StandardTokenizerFactory"/>
860 |         <!-- removes l', etc -->
861 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_it.txt"/>
862 |         <filter class="solr.LowerCaseFilterFactory"/>
863 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_it.txt" format="snowball" enablePositionIncrements="true"/>
864 |         <filter class="solr.ItalianLightStemFilterFactory"/>
865 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Italian"/> -->
866 |       </analyzer>
867 |     </fieldType>
868 |     
869 |     <!-- Latvian -->
870 |     <fieldType name="text_lv" class="solr.TextField" positionIncrementGap="100">
871 |       <analyzer> 
872 |         <tokenizer class="solr.StandardTokenizerFactory"/>
873 |         <filter class="solr.LowerCaseFilterFactory"/>
874 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_lv.txt" enablePositionIncrements="true"/>
875 |         <filter class="solr.LatvianStemFilterFactory"/>
876 |       </analyzer>
877 |     </fieldType>
878 |     
879 |     <!-- Dutch -->
880 |     <fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
881 |       <analyzer> 
882 |         <tokenizer class="solr.StandardTokenizerFactory"/>
883 |         <filter class="solr.LowerCaseFilterFactory"/>
884 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_nl.txt" format="snowball" enablePositionIncrements="true"/>
885 |         <filter class="solr.StemmerOverrideFilterFactory" dictionary="lang/stemdict_nl.txt" ignoreCase="false"/>
886 |         <filter class="solr.SnowballPorterFilterFactory" language="Dutch"/>
887 |       </analyzer>
888 |     </fieldType>
889 |     
890 |     <!-- Norwegian -->
891 |     <fieldType name="text_no" class="solr.TextField" positionIncrementGap="100">
892 |       <analyzer> 
893 |         <tokenizer class="solr.StandardTokenizerFactory"/>
894 |         <filter class="solr.LowerCaseFilterFactory"/>
895 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_no.txt" format="snowball" enablePositionIncrements="true"/>
896 |         <filter class="solr.SnowballPorterFilterFactory" language="Norwegian"/>
897 |         <!-- less aggressive: <filter class="solr.NorwegianLightStemFilterFactory"/> -->
898 |         <!-- singular/plural: <filter class="solr.NorwegianMinimalStemFilterFactory"/> -->
899 |       </analyzer>
900 |     </fieldType>
901 |     
902 |     <!-- Portuguese -->
903 |     <fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
904 |       <analyzer> 
905 |         <tokenizer class="solr.StandardTokenizerFactory"/>
906 |         <filter class="solr.LowerCaseFilterFactory"/>
907 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_pt.txt" format="snowball" enablePositionIncrements="true"/>
908 |         <filter class="solr.PortugueseLightStemFilterFactory"/>
909 |         <!-- less aggressive: <filter class="solr.PortugueseMinimalStemFilterFactory"/> -->
910 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Portuguese"/> -->
911 |         <!-- most aggressive: <filter class="solr.PortugueseStemFilterFactory"/> -->
912 |       </analyzer>
913 |     </fieldType>
914 |     
915 |     <!-- Romanian -->
916 |     <fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
917 |       <analyzer> 
918 |         <tokenizer class="solr.StandardTokenizerFactory"/>
919 |         <filter class="solr.LowerCaseFilterFactory"/>
920 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ro.txt" enablePositionIncrements="true"/>
921 |         <filter class="solr.SnowballPorterFilterFactory" language="Romanian"/>
922 |       </analyzer>
923 |     </fieldType>
924 |     
925 |     <!-- Russian -->
926 |     <fieldType name="text_ru" class="solr.TextField" positionIncrementGap="100">
927 |       <analyzer> 
928 |         <tokenizer class="solr.StandardTokenizerFactory"/>
929 |         <filter class="solr.LowerCaseFilterFactory"/>
930 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" format="snowball" enablePositionIncrements="true"/>
931 |         <filter class="solr.SnowballPorterFilterFactory" language="Russian"/>
932 |         <!-- less aggressive: <filter class="solr.RussianLightStemFilterFactory"/> -->
933 |       </analyzer>
934 |     </fieldType>
935 |     
936 |     <!-- Swedish -->
937 |     <fieldType name="text_sv" class="solr.TextField" positionIncrementGap="100">
938 |       <analyzer> 
939 |         <tokenizer class="solr.StandardTokenizerFactory"/>
940 |         <filter class="solr.LowerCaseFilterFactory"/>
941 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_sv.txt" format="snowball" enablePositionIncrements="true"/>
942 |         <filter class="solr.SnowballPorterFilterFactory" language="Swedish"/>
943 |         <!-- less aggressive: <filter class="solr.SwedishLightStemFilterFactory"/> -->
944 |       </analyzer>
945 |     </fieldType>
946 |     
947 |     <!-- Thai -->
948 |     <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
949 |       <analyzer> 
950 |         <tokenizer class="solr.StandardTokenizerFactory"/>
951 |         <filter class="solr.LowerCaseFilterFactory"/>
952 |         <filter class="solr.ThaiWordFilterFactory"/>
953 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_th.txt" enablePositionIncrements="true"/>
954 |       </analyzer>
955 |     </fieldType>
956 |     
957 |     <!-- Turkish -->
958 |     <fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
959 |       <analyzer> 
960 |         <tokenizer class="solr.StandardTokenizerFactory"/>
961 |         <filter class="solr.TurkishLowerCaseFilterFactory"/>
962 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_tr.txt" enablePositionIncrements="true"/>
963 |         <filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
964 |       </analyzer>
965 |     </fieldType>
966 | 
967 |  </types>
968 |   
969 |   <!-- Similarity is the scoring routine for each document vs. a query.
970 |        A custom Similarity or SimilarityFactory may be specified here, but 
971 |        the default is fine for most applications.  
972 |        For more info: http://wiki.apache.org/solr/SchemaXml#Similarity
973 |     -->
974 |   <!--
975 |      <similarity class="com.example.solr.CustomSimilarityFactory">
976 |        <str name="paramkey">param value</str>
977 |      </similarity>
978 |     -->
979 | 
980 | </schema>
981 | 


--------------------------------------------------------------------------------
/email-search/email-schema.xml:
--------------------------------------------------------------------------------
   1 | <!--  
   2 |  This is the Solr schema file. This file should be named "schema.xml" and
   3 |  should be in the conf directory under the solr home
   4 |  (i.e. ./solr/conf/schema.xml by default) 
   5 |  or located where the classloader for the Solr webapp can find it.
   6 | 
   7 |  This example schema is the recommended starting point for users.
   8 |  It should be kept correct and concise, usable out-of-the-box.
   9 | 
  10 |  For more information, on how to customize this file, please see
  11 |  http://wiki.apache.org/solr/SchemaXml
  12 | 
  13 |  PERFORMANCE NOTE: this schema includes many optional features and should not
  14 |  be used for benchmarking.  To improve performance one could
  15 |   - set stored="false" for all fields possible (esp large fields) when you
  16 |     only need to search on the field but don't need to return the original
  17 |     value.
  18 |   - set indexed="false" if you don't need to search on the field, but only
  19 |     return the field as a result of searching on other indexed fields.
  20 |   - remove all unneeded copyField statements
  21 |   - for best index size and searching performance, set "index" to false
  22 |     for all general text fields, use copyField to copy them to the
  23 |     catchall "text" field, and use that for searching.
  24 |   - For maximum indexing performance, use the StreamingUpdateSolrServer
  25 |     java client.
  26 |   - Remember to run the JVM in server mode, and use a higher logging level
  27 |     that avoids logging every request
  28 | -->
  29 | 
  30 | <schema name="example" version="1.5">
  31 |   <!-- attribute "name" is the name of this schema and is only used for display purposes.
  32 |        version="x.y" is Solr's version number for the schema syntax and 
  33 |        semantics.  It should not normally be changed by applications.
  34 | 
  35 |        1.0: multiValued attribute did not exist, all fields are multiValued 
  36 |             by nature
  37 |        1.1: multiValued attribute introduced, false by default 
  38 |        1.2: omitTermFreqAndPositions attribute introduced, true by default 
  39 |             except for text fields.
  40 |        1.3: removed optional field compress feature
  41 |        1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser
  42 |             behavior when a single string produces multiple tokens.  Defaults 
  43 |             to off for version >= 1.4
  44 |        1.5: omitNorms defaults to true for primitive field types 
  45 |             (int, float, boolean, string...)
  46 |      -->
  47 | 
  48 |  <fields>
  49 |    <!-- Valid attributes for fields:
  50 |      name: mandatory - the name for the field
  51 |      type: mandatory - the name of a field type from the 
  52 |        <types> fieldType section
  53 |      indexed: true if this field should be indexed (searchable or sortable)
  54 |      stored: true if this field should be retrievable
  55 |      docValues: true if this field should have doc values. Doc values are
  56 |        useful for faceting, grouping, sorting and function queries. Although not
  57 |        required, doc values will make the index faster to load, more
  58 |        NRT-friendly and more memory-efficient. They however come with some
  59 |        limitations: they are currently only supported by StrField, UUIDField
  60 |        and all Trie*Fields, and depending on the field type, they might
  61 |        require the field to be single-valued, be required or have a default
  62 |        value (check the documentation of the field type you're interested in
  63 |        for more information)
  64 |      multiValued: true if this field may contain multiple values per document
  65 |      omitNorms: (expert) set to true to omit the norms associated with
  66 |        this field (this disables length normalization and index-time
  67 |        boosting for the field, and saves some memory).  Only full-text
  68 |        fields or fields that need an index-time boost need norms.
  69 |        Norms are omitted for primitive (non-analyzed) types by default.
  70 |      termVectors: [false] set to true to store the term vector for a
  71 |        given field.
  72 |        When using MoreLikeThis, fields used for similarity should be
  73 |        stored for best performance.
  74 |      termPositions: Store position information with the term vector.  
  75 |        This will increase storage costs.
  76 |      termOffsets: Store offset information with the term vector. This 
  77 |        will increase storage costs.
  78 |      required: The field is required.  It will throw an error if the
  79 |        value does not exist
  80 |      default: a value that should be used if no value is specified
  81 |        when adding a document.
  82 |    -->
  83 | 
  84 |    <!-- field names should consist of alphanumeric or underscore characters only and
  85 |       not start with a digit.  This is not currently strictly enforced,
  86 |       but other field names will not have first class support from all components
  87 |       and back compatibility is not guaranteed.  Names with both leading and
  88 |       trailing underscores (e.g. _version_) are reserved.
  89 |    -->
  90 |         
  91 |    <field name="message_id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> 
  92 |    <field name="id" type="string" indexed="true" stored="true" required="true"  /> 
  93 |    <field name="date" type="tdate" indexed="true" stored="true"/>
  94 |    <field name="from" type="email_general" indexed="true" stored="true"/>
  95 |    <field name="to" type="email_general" indexed="true" stored="true"/>
  96 |    <field name="subject" type="text_general" indexed="true" stored="true"/>
  97 |    <field name="from_names" type="names_general" indexed="true" stored="true"/>
  98 |    <field name="to_names" type="names_general" indexed="true" stored="true"/>
  99 |    <field name="cc" type="email_general" indexed="true" stored="true"/>
 100 |    <field name="bcc" type="text_general" indexed="true" stored="true"/>
 101 |    <field name="cc_names" type="names_general" indexed="true" stored="true"/>
 102 |    <field name="bcc_names" type="names_general" indexed="true" stored="true" />
 103 |    <field name="x_folder" type="text_general" indexed="true" stored="false" />
 104 |    <field name="x_origin" type="text_general" indexed="true" stored="false" />
 105 |    <field name="x_filename" type="text_general" indexed="true" stored="false"/>
 106 |    <field name="message" type="text_general" indexed="false" stored="false"/>
 107 |    <field name="body" type="text_general" indexed="true" stored="true"/>
 108 |    <field name="text" type="text_general" indexed="true" stored="false" multiValued="true"/>
 109 |    <field name="_version_" type="long" indexed="true" stored="true"/> 
 110 | 
 111 |  </fields>
 112 | 
 113 | 
 114 |  <!-- Field to use to determine and enforce document uniqueness. 
 115 |       Unless this field is marked with required="false", it will be a required field
 116 |    -->
 117 |  <uniqueKey>id</uniqueKey>
 118 | 
 119 | 
 120 |  <!-- DEPRECATED: The defaultOperator (AND|OR) is consulted by various query parsers
 121 |   when parsing a query string to determine if a clause of the query should be marked as
 122 |   required or optional, assuming the clause isn't already marked by some operator.
 123 |   The default is OR, which is generally assumed so it is not a good idea to change it
 124 |   globally here.  The "q.op" request parameter takes precedence over this.
 125 |  <solrQueryParser defaultOperator="OR"/> -->
 126 | 
 127 |   <!-- copyField commands copy one field to another at the time a document
 128 |         is added to the index.  It's used either to index the same field differently,
 129 |         or to add multiple fields to the same field for easier/faster searching.  -->
 130 | 
 131 |    <copyField source="message_id" dest="text"/>
 132 |    <copyField source="id" dest="text"/>
 133 |    <copyField source="date" dest="text"/>
 134 |    <copyField source="from" dest="text"/>
 135 |    <copyField source="to" dest="text"/>
 136 |    <copyField source="subject" dest="text"/>
 137 |    <copyField source="from_names" dest="text"/>
 138 |    <copyField source="cc" dest="text"/>
 139 |    <copyField source="cc_names" dest="text"/>
 140 |    <copyField source="bcc" dest="text"/>
 141 |    <copyField source="bcc_names" dest="text"/>
 142 |    <copyField source="x_folder" dest="text"/>
 143 |    <copyField source="x_origin" dest="text"/>
 144 |    <copyField source="x_filename" dest="text"/>
 145 |    <copyField source="to_names" dest="text"/>
 146 |    <copyField source="message" dest="text"/>
 147 |    <copyField source="body" dest="text"/>
 148 | 
 149 |  
 150 |   <types>
 151 |     <!-- field type definitions. The "name" attribute is
 152 |        just a label to be used by field definitions.  The "class"
 153 |        attribute and any other attributes determine the real
 154 |        behavior of the fieldType.
 155 |          Class names starting with "solr" refer to java classes in a
 156 |        standard package such as org.apache.solr.analysis
 157 |     -->
 158 | 
 159 |     <!-- The StrField type is not analyzed, but indexed/stored verbatim.
 160 |        It supports doc values but in that case the field needs to be
 161 |        single-valued and either required or have a default value.
 162 |       -->
 163 |     <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
 164 | 
 165 |     <!-- boolean type: "true" or "false" -->
 166 |     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
 167 | 
 168 |     <!-- sortMissingLast and sortMissingFirst attributes are optional attributes are
 169 |          currently supported on types that are sorted internally as strings
 170 |          and on numeric types.
 171 | 	     This includes "string","boolean", and, as of 3.5 (and 4.x),
 172 | 	     int, float, long, date, double, including the "Trie" variants.
 173 |        - If sortMissingLast="true", then a sort on this field will cause documents
 174 |          without the field to come after documents with the field,
 175 |          regardless of the requested sort order (asc or desc).
 176 |        - If sortMissingFirst="true", then a sort on this field will cause documents
 177 |          without the field to come before documents with the field,
 178 |          regardless of the requested sort order.
 179 |        - If sortMissingLast="false" and sortMissingFirst="false" (the default),
 180 |          then default lucene sorting will be used which places docs without the
 181 |          field first in an ascending sort and last in a descending sort.
 182 |     -->    
 183 | 
 184 |     <!--
 185 |       Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
 186 | 
 187 |       These fields support doc values, but they require the field to be
 188 |       single-valued and either be required or have a default value.
 189 |     -->
 190 |     <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
 191 |     <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
 192 |     <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
 193 |     <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
 194 | 
 195 |     <!--
 196 |      Numeric field types that index each value at various levels of precision
 197 |      to accelerate range queries when the number of values between the range
 198 |      endpoints is large. See the javadoc for NumericRangeQuery for internal
 199 |      implementation details.
 200 | 
 201 |      Smaller precisionStep values (specified in bits) will lead to more tokens
 202 |      indexed per value, slightly larger index size, and faster range queries.
 203 |      A precisionStep of 0 disables indexing at different precision levels.
 204 |     -->
 205 |     <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
 206 |     <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
 207 |     <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
 208 |     <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
 209 | 
 210 |     <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
 211 |          is a more restricted form of the canonical representation of dateTime
 212 |          http://www.w3.org/TR/xmlschema-2/#dateTime    
 213 |          The trailing "Z" designates UTC time and is mandatory.
 214 |          Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
 215 |          All other components are mandatory.
 216 | 
 217 |          Expressions can also be used to denote calculations that should be
 218 |          performed relative to "NOW" to determine the value, ie...
 219 | 
 220 |                NOW/HOUR
 221 |                   ... Round to the start of the current hour
 222 |                NOW-1DAY
 223 |                   ... Exactly 1 day prior to now
 224 |                NOW/DAY+6MONTHS+3DAYS
 225 |                   ... 6 months and 3 days in the future from the start of
 226 |                       the current day
 227 |                       
 228 |          Consult the DateField javadocs for more information.
 229 | 
 230 |          Note: For faster range queries, consider the tdate type
 231 |       -->
 232 |     <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
 233 | 
 234 |     <!-- A Trie based date field for faster date range queries and date faceting. -->
 235 |     <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
 236 | 
 237 | 
 238 |     <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
 239 |     <fieldtype name="binary" class="solr.BinaryField"/>
 240 | 
 241 |     <!--
 242 |       Note:
 243 |       These should only be used for compatibility with existing indexes (created with lucene or older Solr versions).
 244 |       Use Trie based fields instead. As of Solr 3.5 and 4.x, Trie based fields support sortMissingFirst/Last
 245 |       
 246 |       Plain numeric field types that store and index the text
 247 |       value verbatim (and hence don't correctly support range queries, since the
 248 |       lexicographic ordering isn't equal to the numeric ordering)
 249 |     -->
 250 |     <fieldType name="pint" class="solr.IntField"/>
 251 |     <fieldType name="plong" class="solr.LongField"/>
 252 |     <fieldType name="pfloat" class="solr.FloatField"/>
 253 |     <fieldType name="pdouble" class="solr.DoubleField"/>
 254 |     <fieldType name="pdate" class="solr.DateField" sortMissingLast="true"/>
 255 | 
 256 |     <!-- The "RandomSortField" is not used to store or search any
 257 |          data.  You can declare fields of this type it in your schema
 258 |          to generate pseudo-random orderings of your docs for sorting 
 259 |          or function purposes.  The ordering is generated based on the field
 260 |          name and the version of the index. As long as the index version
 261 |          remains unchanged, and the same field name is reused,
 262 |          the ordering of the docs will be consistent.  
 263 |          If you want different psuedo-random orderings of documents,
 264 |          for the same version of the index, use a dynamicField and
 265 |          change the field name in the request.
 266 |      -->
 267 |     <fieldType name="random" class="solr.RandomSortField" indexed="true" />
 268 | 
 269 |     <!-- solr.TextField allows the specification of custom text analyzers
 270 |          specified as a tokenizer and a list of token filters. Different
 271 |          analyzers may be specified for indexing and querying.
 272 | 
 273 |          The optional positionIncrementGap puts space between multiple fields of
 274 |          this type on the same document, with the purpose of preventing false phrase
 275 |          matching across fields.
 276 | 
 277 |          For more info on customizing your analyzer chain, please see
 278 |          http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
 279 |      -->
 280 | 
 281 |     <!-- One can also specify an existing Analyzer class that has a
 282 |          default constructor via the class attribute on the analyzer element.
 283 |          Example:
 284 |     <fieldType name="text_greek" class="solr.TextField">
 285 |       <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
 286 |     </fieldType>
 287 |     -->
 288 | 
 289 |     <!-- A text field that only splits on whitespace for exact matching of words -->
 290 |     <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
 291 |       <analyzer>
 292 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 293 |       </analyzer>
 294 |     </fieldType>
 295 | 
 296 |     <!-- A general text field that has reasonable, generic
 297 |          cross-language defaults: it tokenizes with StandardTokenizer,
 298 | 	 removes stop words from case-insensitive "stopwords.txt"
 299 | 	 (empty by default), and down cases.  At query time only, it
 300 | 	 also applies synonyms. -->
 301 |     <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
 302 |       <analyzer type="index">
 303 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 304 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 305 |         <!-- in this example, we will only use synonyms at query time
 306 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 307 |         -->
 308 |         <filter class="solr.LowerCaseFilterFactory"/>
 309 |       </analyzer>
 310 |       <analyzer type="query">
 311 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 312 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 313 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 314 |         <filter class="solr.LowerCaseFilterFactory"/>
 315 |       </analyzer>
 316 |     </fieldType>
 317 | 
 318 |     <fieldType name="names_general" class="solr.TextField" >
 319 |       <analyzer type="index">
 320 | 	<charFilter class="solr.HTMLStripCharFilterFactory" />
 321 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 322 |         <filter class="solr.LowerCaseFilterFactory"/>
 323 |       </analyzer>
 324 |       <analyzer type="query">
 325 | 	<charFilter class="solr.HTMLStripCharFilterFactory" />
 326 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 327 |         <filter class="solr.LowerCaseFilterFactory"/>
 328 |       </analyzer>
 329 |     </fieldType>
 330 | 
 331 |     <fieldType name="email_general" class="solr.TextField" >
 332 |       <analyzer type="index">
 333 |         <tokenizer class="solr.UAX29URLEmailTokenizerFactory" />
 334 |         <filter class="solr.LowerCaseFilterFactory"/>
 335 |       </analyzer>
 336 |       <analyzer type="query">
 337 |         <tokenizer class="solr.UAX29URLEmailTokenizerFactory" />
 338 |         <filter class="solr.LowerCaseFilterFactory"/>
 339 |       </analyzer>
 340 |     </fieldType>
 341 | 
 342 |     <!-- A text field with defaults appropriate for English: it
 343 |          tokenizes with StandardTokenizer, removes English stop words
 344 |          (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
 345 |          finally applies Porter's stemming.  The query time analyzer
 346 |          also applies synonyms from synonyms.txt. -->
 347 | 
 348 |     <!-- A text field with defaults appropriate for English: it
 349 |          tokenizes with StandardTokenizer, removes English stop words
 350 |          (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
 351 |          finally applies Porter's stemming.  The query time analyzer
 352 |          also applies synonyms from synonyms.txt. -->
 353 |     <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
 354 |       <analyzer type="index">
 355 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 356 |         <!-- in this example, we will only use synonyms at query time
 357 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 358 |         -->
 359 |         <!-- Case insensitive stop word removal.
 360 |           add enablePositionIncrements=true in both the index and query
 361 |           analyzers to leave a 'gap' for more accurate phrase queries.
 362 |         -->
 363 |         <filter class="solr.StopFilterFactory"
 364 |                 ignoreCase="true"
 365 |                 words="lang/stopwords_en.txt"
 366 |                 enablePositionIncrements="true"
 367 |                 />
 368 |         <filter class="solr.LowerCaseFilterFactory"/>
 369 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
 370 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 371 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
 372 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 373 | 	-->
 374 |         <filter class="solr.PorterStemFilterFactory"/>
 375 |       </analyzer>
 376 |       <analyzer type="query">
 377 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 378 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 379 |         <filter class="solr.StopFilterFactory"
 380 |                 ignoreCase="true"
 381 |                 words="lang/stopwords_en.txt"
 382 |                 enablePositionIncrements="true"
 383 |                 />
 384 |         <filter class="solr.LowerCaseFilterFactory"/>
 385 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
 386 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 387 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
 388 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 389 | 	-->
 390 |         <filter class="solr.PorterStemFilterFactory"/>
 391 |       </analyzer>
 392 |     </fieldType>
 393 | 
 394 |     <!-- A text field with defaults appropriate for English, plus
 395 | 	 aggressive word-splitting and autophrase features enabled.
 396 | 	 This field is just like text_en, except it adds
 397 | 	 WordDelimiterFilter to enable splitting and matching of
 398 | 	 words on case-change, alpha numeric boundaries, and
 399 | 	 non-alphanumeric chars.  This means certain compound word
 400 | 	 cases will work, for example query "wi fi" will match
 401 | 	 document "WiFi" or "wi-fi".
 402 |         -->
 403 |     <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
 404 |       <analyzer type="index">
 405 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 406 |         <!-- in this example, we will only use synonyms at query time
 407 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 408 |         -->
 409 |         <!-- Case insensitive stop word removal.
 410 |           add enablePositionIncrements=true in both the index and query
 411 |           analyzers to leave a 'gap' for more accurate phrase queries.
 412 |         -->
 413 |         <filter class="solr.StopFilterFactory"
 414 |                 ignoreCase="true"
 415 |                 words="lang/stopwords_en.txt"
 416 |                 enablePositionIncrements="true"
 417 |                 />
 418 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
 419 |         <filter class="solr.LowerCaseFilterFactory"/>
 420 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 421 |         <filter class="solr.PorterStemFilterFactory"/>
 422 |       </analyzer>
 423 |       <analyzer type="query">
 424 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 425 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 426 |         <filter class="solr.StopFilterFactory"
 427 |                 ignoreCase="true"
 428 |                 words="lang/stopwords_en.txt"
 429 |                 enablePositionIncrements="true"
 430 |                 />
 431 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
 432 |         <filter class="solr.LowerCaseFilterFactory"/>
 433 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 434 |         <filter class="solr.PorterStemFilterFactory"/>
 435 |       </analyzer>
 436 |     </fieldType>
 437 | 
 438 |     <!-- Less flexible matching, but less false matches.  Probably not ideal for product names,
 439 |          but may be good for SKUs.  Can insert dashes in the wrong place and still match. -->
 440 |     <fieldType name="text_en_splitting_tight" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
 441 |       <analyzer>
 442 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 443 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
 444 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
 445 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
 446 |         <filter class="solr.LowerCaseFilterFactory"/>
 447 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 448 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 449 |         <!-- this filter can remove any duplicate tokens that appear at the same position - sometimes
 450 |              possible with WordDelimiterFilter in conjuncton with stemming. -->
 451 |         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 452 |       </analyzer>
 453 |     </fieldType>
 454 | 
 455 |     <!-- Just like text_general except it reverses the characters of
 456 | 	 each token, to enable more efficient leading wildcard queries. -->
 457 |     <fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
 458 |       <analyzer type="index">
 459 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 460 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 461 |         <filter class="solr.LowerCaseFilterFactory"/>
 462 |         <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
 463 |            maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
 464 |       </analyzer>
 465 |       <analyzer type="query">
 466 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 467 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 468 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 469 |         <filter class="solr.LowerCaseFilterFactory"/>
 470 |       </analyzer>
 471 |     </fieldType>
 472 | 
 473 |     <!-- charFilter + WhitespaceTokenizer  -->
 474 |     <!--
 475 |     <fieldType name="text_char_norm" class="solr.TextField" positionIncrementGap="100" >
 476 |       <analyzer>
 477 |         <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
 478 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 479 |       </analyzer>
 480 |     </fieldType>
 481 |     -->
 482 | 
 483 |     <!-- This is an example of using the KeywordTokenizer along
 484 |          With various TokenFilterFactories to produce a sortable field
 485 |          that does not include some properties of the source text
 486 |       -->
 487 |     <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
 488 |       <analyzer>
 489 |         <!-- KeywordTokenizer does no actual tokenizing, so the entire
 490 |              input string is preserved as a single token
 491 |           -->
 492 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
 493 |         <!-- The LowerCase TokenFilter does what you expect, which can be
 494 |              when you want your sorting to be case insensitive
 495 |           -->
 496 |         <filter class="solr.LowerCaseFilterFactory" />
 497 |         <!-- The TrimFilter removes any leading or trailing whitespace -->
 498 |         <filter class="solr.TrimFilterFactory" />
 499 |         <!-- The PatternReplaceFilter gives you the flexibility to use
 500 |              Java Regular expression to replace any sequence of characters
 501 |              matching a pattern with an arbitrary replacement string, 
 502 |              which may include back references to portions of the original
 503 |              string matched by the pattern.
 504 |              
 505 |              See the Java Regular Expression documentation for more
 506 |              information on pattern and replacement string syntax.
 507 |              
 508 |              http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
 509 |           -->
 510 |         <filter class="solr.PatternReplaceFilterFactory"
 511 |                 pattern="([^a-z])" replacement="" replace="all"
 512 |         />
 513 |       </analyzer>
 514 |     </fieldType>
 515 |     
 516 |     <fieldtype name="phonetic" stored="false" indexed="true" class="solr.TextField" >
 517 |       <analyzer>
 518 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 519 |         <filter class="solr.DoubleMetaphoneFilterFactory" inject="false"/>
 520 |       </analyzer>
 521 |     </fieldtype>
 522 | 
 523 |     <fieldtype name="payloads" stored="false" indexed="true" class="solr.TextField" >
 524 |       <analyzer>
 525 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 526 |         <!--
 527 |         The DelimitedPayloadTokenFilter can put payloads on tokens... for example,
 528 |         a token of "foo|1.4"  would be indexed as "foo" with a payload of 1.4f
 529 |         Attributes of the DelimitedPayloadTokenFilterFactory : 
 530 |          "delimiter" - a one character delimiter. Default is | (pipe)
 531 | 	 "encoder" - how to encode the following value into a playload
 532 | 	    float -> org.apache.lucene.analysis.payloads.FloatEncoder,
 533 | 	    integer -> o.a.l.a.p.IntegerEncoder
 534 | 	    identity -> o.a.l.a.p.IdentityEncoder
 535 |             Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor.
 536 |          -->
 537 |         <filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="float"/>
 538 |       </analyzer>
 539 |     </fieldtype>
 540 | 
 541 |     <!-- lowercases the entire field value, keeping it as a single token.  -->
 542 |     <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
 543 |       <analyzer>
 544 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
 545 |         <filter class="solr.LowerCaseFilterFactory" />
 546 |       </analyzer>
 547 |     </fieldType>
 548 | 
 549 |     <!-- 
 550 |       Example of using PathHierarchyTokenizerFactory at index time, so
 551 |       queries for paths match documents at that path, or in descendent paths
 552 |     -->
 553 |     <fieldType name="descendent_path" class="solr.TextField">
 554 |       <analyzer type="index">
 555 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
 556 |       </analyzer>
 557 |       <analyzer type="query">
 558 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
 559 |       </analyzer>
 560 |     </fieldType>
 561 |     <!-- 
 562 |       Example of using PathHierarchyTokenizerFactory at query time, so
 563 |       queries for paths match documents at that path, or in ancestor paths
 564 |     -->
 565 |     <fieldType name="ancestor_path" class="solr.TextField">
 566 |       <analyzer type="index">
 567 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
 568 |       </analyzer>
 569 |       <analyzer type="query">
 570 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
 571 |       </analyzer>
 572 |     </fieldType>
 573 | 
 574 |     <!-- since fields of this type are by default not stored or indexed,
 575 |          any data added to them will be ignored outright.  --> 
 576 |     <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
 577 | 
 578 |     <!-- This point type indexes the coordinates as separate fields (subFields)
 579 |       If subFieldType is defined, it references a type, and a dynamic field
 580 |       definition is created matching *___<typename>.  Alternately, if 
 581 |       subFieldSuffix is defined, that is used to create the subFields.
 582 |       Example: if subFieldType="double", then the coordinates would be
 583 |         indexed in fields myloc_0___double,myloc_1___double.
 584 |       Example: if subFieldSuffix="_d" then the coordinates would be indexed
 585 |         in fields myloc_0_d,myloc_1_d
 586 |       The subFields are an implementation detail of the fieldType, and end
 587 |       users normally should not need to know about them.
 588 |      -->
 589 |     <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
 590 | 
 591 |     <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
 592 |     <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
 593 | 
 594 |     <!-- An alternative geospatial field type new to Solr 4.  It supports multiValued and polygon shapes.
 595 |       For more information about this and other Spatial fields new to Solr 4, see:
 596 |       http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
 597 |     -->
 598 |     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
 599 |         geo="true" distErrPct="0.025" maxDistErr="0.000009" units="degrees" />
 600 | 
 601 |    <!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType
 602 |         Parameters:
 603 |           defaultCurrency: Specifies the default currency if none specified. Defaults to "USD"
 604 |           precisionStep:   Specifies the precisionStep for the TrieLong field used for the amount
 605 |           providerClass:   Lets you plug in other exchange provider backend:
 606 |                            solr.FileExchangeRateProvider is the default and takes one parameter:
 607 |                              currencyConfig: name of an xml file holding exchange rates
 608 |                            solr.OpenExchangeRatesOrgProvider uses rates from openexchangerates.org:
 609 |                              ratesFileLocation: URL or path to rates JSON file (default latest.json on the web)
 610 |                              refreshInterval: Number of minutes between each rates fetch (default: 1440, min: 60)
 611 |    -->
 612 |    <!--  <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
 613 |    -->
 614 |              
 615 | 
 616 | 
 617 |    <!-- some examples for different languages (generally ordered by ISO code) -->
 618 | 
 619 |     <!-- Arabic -->
 620 |     <fieldType name="text_ar" class="solr.TextField" positionIncrementGap="100">
 621 |       <analyzer> 
 622 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 623 |         <!-- for any non-arabic -->
 624 |         <filter class="solr.LowerCaseFilterFactory"/>
 625 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ar.txt" enablePositionIncrements="true"/>
 626 |         <!-- normalizes ﻯ to ﻱ, etc -->
 627 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
 628 |         <filter class="solr.ArabicStemFilterFactory"/>
 629 |       </analyzer>
 630 |     </fieldType>
 631 | 
 632 |     <!-- Bulgarian -->
 633 |     <fieldType name="text_bg" class="solr.TextField" positionIncrementGap="100">
 634 |       <analyzer> 
 635 |         <tokenizer class="solr.StandardTokenizerFactory"/> 
 636 |         <filter class="solr.LowerCaseFilterFactory"/>
 637 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_bg.txt" enablePositionIncrements="true"/>
 638 |         <filter class="solr.BulgarianStemFilterFactory"/>       
 639 |       </analyzer>
 640 |     </fieldType>
 641 |     
 642 |     <!-- Catalan -->
 643 |     <fieldType name="text_ca" class="solr.TextField" positionIncrementGap="100">
 644 |       <analyzer> 
 645 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 646 |         <!-- removes l', etc -->
 647 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ca.txt"/>
 648 |         <filter class="solr.LowerCaseFilterFactory"/>
 649 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ca.txt" enablePositionIncrements="true"/>
 650 |         <filter class="solr.SnowballPorterFilterFactory" language="Catalan"/>       
 651 |       </analyzer>
 652 |     </fieldType>
 653 |     
 654 |     <!-- CJK bigram (see text_ja for a Japanese configuration using morphological analysis) -->
 655 |     <fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
 656 |       <analyzer>
 657 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 658 |         <!-- normalize width before bigram, as e.g. half-width dakuten combine  -->
 659 |         <filter class="solr.CJKWidthFilterFactory"/>
 660 |         <!-- for any non-CJK -->
 661 |         <filter class="solr.LowerCaseFilterFactory"/>
 662 |         <filter class="solr.CJKBigramFilterFactory"/>
 663 |       </analyzer>
 664 |     </fieldType>
 665 | 
 666 |     <!-- Czech -->
 667 |     <fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
 668 |       <analyzer> 
 669 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 670 |         <filter class="solr.LowerCaseFilterFactory"/>
 671 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_cz.txt" enablePositionIncrements="true"/>
 672 |         <filter class="solr.CzechStemFilterFactory"/>       
 673 |       </analyzer>
 674 |     </fieldType>
 675 |     
 676 |     <!-- Danish -->
 677 |     <fieldType name="text_da" class="solr.TextField" positionIncrementGap="100">
 678 |       <analyzer> 
 679 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 680 |         <filter class="solr.LowerCaseFilterFactory"/>
 681 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_da.txt" format="snowball" enablePositionIncrements="true"/>
 682 |         <filter class="solr.SnowballPorterFilterFactory" language="Danish"/>       
 683 |       </analyzer>
 684 |     </fieldType>
 685 |     
 686 |     <!-- German -->
 687 |     <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
 688 |       <analyzer> 
 689 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 690 |         <filter class="solr.LowerCaseFilterFactory"/>
 691 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_de.txt" format="snowball" enablePositionIncrements="true"/>
 692 |         <filter class="solr.GermanNormalizationFilterFactory"/>
 693 |         <filter class="solr.GermanLightStemFilterFactory"/>
 694 |         <!-- less aggressive: <filter class="solr.GermanMinimalStemFilterFactory"/> -->
 695 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="German2"/> -->
 696 |       </analyzer>
 697 |     </fieldType>
 698 |     
 699 |     <!-- Greek -->
 700 |     <fieldType name="text_el" class="solr.TextField" positionIncrementGap="100">
 701 |       <analyzer> 
 702 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 703 |         <!-- greek specific lowercase for sigma -->
 704 |         <filter class="solr.GreekLowerCaseFilterFactory"/>
 705 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_el.txt" enablePositionIncrements="true"/>
 706 |         <filter class="solr.GreekStemFilterFactory"/>
 707 |       </analyzer>
 708 |     </fieldType>
 709 |     
 710 |     <!-- Spanish -->
 711 |     <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
 712 |       <analyzer> 
 713 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 714 |         <filter class="solr.LowerCaseFilterFactory"/>
 715 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" format="snowball" enablePositionIncrements="true"/>
 716 |         <filter class="solr.SpanishLightStemFilterFactory"/>
 717 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Spanish"/> -->
 718 |       </analyzer>
 719 |     </fieldType>
 720 |     
 721 |     <!-- Basque -->
 722 |     <fieldType name="text_eu" class="solr.TextField" positionIncrementGap="100">
 723 |       <analyzer> 
 724 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 725 |         <filter class="solr.LowerCaseFilterFactory"/>
 726 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_eu.txt" enablePositionIncrements="true"/>
 727 |         <filter class="solr.SnowballPorterFilterFactory" language="Basque"/>
 728 |       </analyzer>
 729 |     </fieldType>
 730 |     
 731 |     <!-- Persian -->
 732 |     <fieldType name="text_fa" class="solr.TextField" positionIncrementGap="100">
 733 |       <analyzer>
 734 |         <!-- for ZWNJ -->
 735 |         <charFilter class="solr.PersianCharFilterFactory"/>
 736 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 737 |         <filter class="solr.LowerCaseFilterFactory"/>
 738 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
 739 |         <filter class="solr.PersianNormalizationFilterFactory"/>
 740 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fa.txt" enablePositionIncrements="true"/>
 741 |       </analyzer>
 742 |     </fieldType>
 743 |     
 744 |     <!-- Finnish -->
 745 |     <fieldType name="text_fi" class="solr.TextField" positionIncrementGap="100">
 746 |       <analyzer> 
 747 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 748 |         <filter class="solr.LowerCaseFilterFactory"/>
 749 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fi.txt" format="snowball" enablePositionIncrements="true"/>
 750 |         <filter class="solr.SnowballPorterFilterFactory" language="Finnish"/>
 751 |         <!-- less aggressive: <filter class="solr.FinnishLightStemFilterFactory"/> -->
 752 |       </analyzer>
 753 |     </fieldType>
 754 |     
 755 |     <!-- French -->
 756 |     <fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
 757 |       <analyzer> 
 758 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 759 |         <!-- removes l', etc -->
 760 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_fr.txt"/>
 761 |         <filter class="solr.LowerCaseFilterFactory"/>
 762 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" format="snowball" enablePositionIncrements="true"/>
 763 |         <filter class="solr.FrenchLightStemFilterFactory"/>
 764 |         <!-- less aggressive: <filter class="solr.FrenchMinimalStemFilterFactory"/> -->
 765 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="French"/> -->
 766 |       </analyzer>
 767 |     </fieldType>
 768 |     
 769 |     <!-- Irish -->
 770 |     <fieldType name="text_ga" class="solr.TextField" positionIncrementGap="100">
 771 |       <analyzer> 
 772 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 773 |         <!-- removes d', etc -->
 774 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ga.txt"/>
 775 |         <!-- removes n-, etc. position increments is intentionally false! -->
 776 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/hyphenations_ga.txt" enablePositionIncrements="false"/>
 777 |         <filter class="solr.IrishLowerCaseFilterFactory"/>
 778 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ga.txt" enablePositionIncrements="true"/>
 779 |         <filter class="solr.SnowballPorterFilterFactory" language="Irish"/>
 780 |       </analyzer>
 781 |     </fieldType>
 782 |     
 783 |     <!-- Galician -->
 784 |     <fieldType name="text_gl" class="solr.TextField" positionIncrementGap="100">
 785 |       <analyzer> 
 786 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 787 |         <filter class="solr.LowerCaseFilterFactory"/>
 788 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_gl.txt" enablePositionIncrements="true"/>
 789 |         <filter class="solr.GalicianStemFilterFactory"/>
 790 |         <!-- less aggressive: <filter class="solr.GalicianMinimalStemFilterFactory"/> -->
 791 |       </analyzer>
 792 |     </fieldType>
 793 |     
 794 |     <!-- Hindi -->
 795 |     <fieldType name="text_hi" class="solr.TextField" positionIncrementGap="100">
 796 |       <analyzer> 
 797 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 798 |         <filter class="solr.LowerCaseFilterFactory"/>
 799 |         <!-- normalizes unicode representation -->
 800 |         <filter class="solr.IndicNormalizationFilterFactory"/>
 801 |         <!-- normalizes variation in spelling -->
 802 |         <filter class="solr.HindiNormalizationFilterFactory"/>
 803 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hi.txt" enablePositionIncrements="true"/>
 804 |         <filter class="solr.HindiStemFilterFactory"/>
 805 |       </analyzer>
 806 |     </fieldType>
 807 |     
 808 |     <!-- Hungarian -->
 809 |     <fieldType name="text_hu" class="solr.TextField" positionIncrementGap="100">
 810 |       <analyzer> 
 811 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 812 |         <filter class="solr.LowerCaseFilterFactory"/>
 813 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hu.txt" format="snowball" enablePositionIncrements="true"/>
 814 |         <filter class="solr.SnowballPorterFilterFactory" language="Hungarian"/>
 815 |         <!-- less aggressive: <filter class="solr.HungarianLightStemFilterFactory"/> -->   
 816 |       </analyzer>
 817 |     </fieldType>
 818 |     
 819 |     <!-- Armenian -->
 820 |     <fieldType name="text_hy" class="solr.TextField" positionIncrementGap="100">
 821 |       <analyzer> 
 822 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 823 |         <filter class="solr.LowerCaseFilterFactory"/>
 824 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hy.txt" enablePositionIncrements="true"/>
 825 |         <filter class="solr.SnowballPorterFilterFactory" language="Armenian"/>
 826 |       </analyzer>
 827 |     </fieldType>
 828 |     
 829 |     <!-- Indonesian -->
 830 |     <fieldType name="text_id" class="solr.TextField" positionIncrementGap="100">
 831 |       <analyzer> 
 832 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 833 |         <filter class="solr.LowerCaseFilterFactory"/>
 834 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_id.txt" enablePositionIncrements="true"/>
 835 |         <!-- for a less aggressive approach (only inflectional suffixes), set stemDerivational to false -->
 836 |         <filter class="solr.IndonesianStemFilterFactory" stemDerivational="true"/>
 837 |       </analyzer>
 838 |     </fieldType>
 839 |     
 840 |     <!-- Italian -->
 841 |     <fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
 842 |       <analyzer> 
 843 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 844 |         <!-- removes l', etc -->
 845 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_it.txt"/>
 846 |         <filter class="solr.LowerCaseFilterFactory"/>
 847 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_it.txt" format="snowball" enablePositionIncrements="true"/>
 848 |         <filter class="solr.ItalianLightStemFilterFactory"/>
 849 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Italian"/> -->
 850 |       </analyzer>
 851 |     </fieldType>
 852 |     
 853 |     <!-- Japanese using morphological analysis (see text_cjk for a configuration using bigramming)
 854 | 
 855 |          NOTE: If you want to optimize search for precision, use default operator AND in your query
 856 |          parser config with <solrQueryParser defaultOperator="AND"/> further down in this file.  Use 
 857 |          OR if you would like to optimize for recall (default).
 858 |     -->
 859 |     <fieldType name="text_ja" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
 860 |       <analyzer>
 861 |       <!-- Kuromoji Japanese morphological analyzer/tokenizer (JapaneseTokenizer)
 862 | 
 863 |            Kuromoji has a search mode (default) that does segmentation useful for search.  A heuristic
 864 |            is used to segment compounds into its parts and the compound itself is kept as synonym.
 865 | 
 866 |            Valid values for attribute mode are:
 867 |               normal: regular segmentation
 868 |               search: segmentation useful for search with synonyms compounds (default)
 869 |             extended: same as search mode, but unigrams unknown words (experimental)
 870 | 
 871 |            For some applications it might be good to use search mode for indexing and normal mode for
 872 |            queries to reduce recall and prevent parts of compounds from being matched and highlighted.
 873 |            Use <analyzer type="index"> and <analyzer type="query"> for this and mode normal in query.
 874 | 
 875 |            Kuromoji also has a convenient user dictionary feature that allows overriding the statistical
 876 |            model with your own entries for segmentation, part-of-speech tags and readings without a need
 877 |            to specify weights.  Notice that user dictionaries have not been subject to extensive testing.
 878 | 
 879 |            User dictionary attributes are:
 880 |                      userDictionary: user dictionary filename
 881 |              userDictionaryEncoding: user dictionary encoding (default is UTF-8)
 882 | 
 883 |            See lang/userdict_ja.txt for a sample user dictionary file.
 884 | 
 885 |            Punctuation characters are discarded by default.  Use discardPunctuation="false" to keep them.
 886 | 
 887 |            See http://wiki.apache.org/solr/JapaneseLanguageSupport for more on Japanese language support.
 888 |         -->
 889 |         <tokenizer class="solr.JapaneseTokenizerFactory" mode="search"/>
 890 |         <!--<tokenizer class="solr.JapaneseTokenizerFactory" mode="search" userDictionary="lang/userdict_ja.txt"/>-->
 891 |         <!-- Reduces inflected verbs and adjectives to their base/dictionary forms (辞書形) -->
 892 |         <filter class="solr.JapaneseBaseFormFilterFactory"/>
 893 |         <!-- Removes tokens with certain part-of-speech tags -->
 894 |         <filter class="solr.JapanesePartOfSpeechStopFilterFactory" tags="lang/stoptags_ja.txt" enablePositionIncrements="true"/>
 895 |         <!-- Normalizes full-width romaji to half-width and half-width kana to full-width (Unicode NFKC subset) -->
 896 |         <filter class="solr.CJKWidthFilterFactory"/>
 897 |         <!-- Removes common tokens typically not useful for search, but have a negative effect on ranking -->
 898 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ja.txt" enablePositionIncrements="true" />
 899 |         <!-- Normalizes common katakana spelling variations by removing any last long sound character (U+30FC) -->
 900 |         <filter class="solr.JapaneseKatakanaStemFilterFactory" minimumLength="4"/>
 901 |         <!-- Lower-cases romaji characters -->
 902 |         <filter class="solr.LowerCaseFilterFactory"/>
 903 |       </analyzer>
 904 |     </fieldType>
 905 |     
 906 |     <!-- Latvian -->
 907 |     <fieldType name="text_lv" class="solr.TextField" positionIncrementGap="100">
 908 |       <analyzer> 
 909 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 910 |         <filter class="solr.LowerCaseFilterFactory"/>
 911 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_lv.txt" enablePositionIncrements="true"/>
 912 |         <filter class="solr.LatvianStemFilterFactory"/>
 913 |       </analyzer>
 914 |     </fieldType>
 915 |     
 916 |     <!-- Dutch -->
 917 |     <fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
 918 |       <analyzer> 
 919 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 920 |         <filter class="solr.LowerCaseFilterFactory"/>
 921 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_nl.txt" format="snowball" enablePositionIncrements="true"/>
 922 |         <filter class="solr.StemmerOverrideFilterFactory" dictionary="lang/stemdict_nl.txt" ignoreCase="false"/>
 923 |         <filter class="solr.SnowballPorterFilterFactory" language="Dutch"/>
 924 |       </analyzer>
 925 |     </fieldType>
 926 |     
 927 |     <!-- Norwegian -->
 928 |     <fieldType name="text_no" class="solr.TextField" positionIncrementGap="100">
 929 |       <analyzer> 
 930 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 931 |         <filter class="solr.LowerCaseFilterFactory"/>
 932 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_no.txt" format="snowball" enablePositionIncrements="true"/>
 933 |         <filter class="solr.SnowballPorterFilterFactory" language="Norwegian"/>
 934 |         <!-- less aggressive: <filter class="solr.NorwegianLightStemFilterFactory"/> -->
 935 |         <!-- singular/plural: <filter class="solr.NorwegianMinimalStemFilterFactory"/> -->
 936 |       </analyzer>
 937 |     </fieldType>
 938 |     
 939 |     <!-- Portuguese -->
 940 |     <fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
 941 |       <analyzer> 
 942 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 943 |         <filter class="solr.LowerCaseFilterFactory"/>
 944 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_pt.txt" format="snowball" enablePositionIncrements="true"/>
 945 |         <filter class="solr.PortugueseLightStemFilterFactory"/>
 946 |         <!-- less aggressive: <filter class="solr.PortugueseMinimalStemFilterFactory"/> -->
 947 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Portuguese"/> -->
 948 |         <!-- most aggressive: <filter class="solr.PortugueseStemFilterFactory"/> -->
 949 |       </analyzer>
 950 |     </fieldType>
 951 |     
 952 |     <!-- Romanian -->
 953 |     <fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
 954 |       <analyzer> 
 955 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 956 |         <filter class="solr.LowerCaseFilterFactory"/>
 957 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ro.txt" enablePositionIncrements="true"/>
 958 |         <filter class="solr.SnowballPorterFilterFactory" language="Romanian"/>
 959 |       </analyzer>
 960 |     </fieldType>
 961 |     
 962 |     <!-- Russian -->
 963 |     <fieldType name="text_ru" class="solr.TextField" positionIncrementGap="100">
 964 |       <analyzer> 
 965 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 966 |         <filter class="solr.LowerCaseFilterFactory"/>
 967 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" format="snowball" enablePositionIncrements="true"/>
 968 |         <filter class="solr.SnowballPorterFilterFactory" language="Russian"/>
 969 |         <!-- less aggressive: <filter class="solr.RussianLightStemFilterFactory"/> -->
 970 |       </analyzer>
 971 |     </fieldType>
 972 |     
 973 |     <!-- Swedish -->
 974 |     <fieldType name="text_sv" class="solr.TextField" positionIncrementGap="100">
 975 |       <analyzer> 
 976 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 977 |         <filter class="solr.LowerCaseFilterFactory"/>
 978 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_sv.txt" format="snowball" enablePositionIncrements="true"/>
 979 |         <filter class="solr.SnowballPorterFilterFactory" language="Swedish"/>
 980 |         <!-- less aggressive: <filter class="solr.SwedishLightStemFilterFactory"/> -->
 981 |       </analyzer>
 982 |     </fieldType>
 983 |     
 984 |     <!-- Thai -->
 985 |     <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
 986 |       <analyzer> 
 987 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 988 |         <filter class="solr.LowerCaseFilterFactory"/>
 989 |         <filter class="solr.ThaiWordFilterFactory"/>
 990 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_th.txt" enablePositionIncrements="true"/>
 991 |       </analyzer>
 992 |     </fieldType>
 993 |     
 994 |     <!-- Turkish -->
 995 |     <fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
 996 |       <analyzer> 
 997 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 998 |         <filter class="solr.TurkishLowerCaseFilterFactory"/>
 999 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_tr.txt" enablePositionIncrements="true"/>
1000 |         <filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
1001 |       </analyzer>
1002 |     </fieldType>
1003 | 
1004 |  </types>
1005 |   
1006 |   <!-- Similarity is the scoring routine for each document vs. a query.
1007 |        A custom Similarity or SimilarityFactory may be specified here, but 
1008 |        the default is fine for most applications.  
1009 |        For more info: http://wiki.apache.org/solr/SchemaXml#Similarity
1010 |     -->
1011 |   <!--
1012 |      <similarity class="com.example.solr.CustomSimilarityFactory">
1013 |        <str name="paramkey">param value</str>
1014 |      </similarity>
1015 |     -->
1016 | 
1017 | </schema>
1018 | 


--------------------------------------------------------------------------------
/email-search/flume.config:
--------------------------------------------------------------------------------
 1 | # Please paste flume.conf here. Example:
 2 | 
 3 | # Sources, channels, and sinks are defined per
 4 | # agent name, in this case 'tier1'.
 5 | tier1.sources=emailsSrc
 6 | tier1.channels=emailsChannel solrChannel
 7 | tier1.sinks=emailsSink solrSink
 8 | 
 9 | 
10 | # For each source, channel, and sink, set
11 | # standard properties.
12 | #tier1.sources.emailsSrc.command=tail -F /tmp/emails.txt
13 | #tier1.sources.emailsSrc.type=exec
14 | #tier1.sources.emailsSrc.command=cat /tmp/emails/emails_sample.txt
15 | tier1.sources.emailsSrc.type=spooldir
16 | tier1.sources.emailsSrc.spoolDir =/tmp/emails/spool
17 | tier1.sources.emailsSrc.deserializer = org.apache.flume.sink.solr.morphline.BlobDeserializer$Builder
18 | tier1.sources.emailsSrc.deletePolicy=immediate
19 | tier1.sources.emailsSrc.channels=emailsChannel solrChannel
20 | tier1.sources.emailsSrc.selector.type=replicating
21 | 
22 | 
23 |  tier1.channels.emailsChannel.type=memory
24 | tier1.channels.solrChannel.type=memory
25 | 
26 | 
27 | # HDFS Sink
28 | tier1.sinks.emailsSink.type=hdfs
29 | tier1.sinks.emailsSink.hdfs.path=/user/flume
30 | tier1.sinks.emailsSink.hdfs.filePrefix=email_log
31 | tier1.sinks.emailsSink.hdfs.rollSize=102400000
32 | tier1.sinks.emailsSink.hdfs.batchSize=10000
33 | tier1.sinks.emailsSink.hdfs.rollCount=0
34 | tier1.sinks.emailsSink.hdfs.minBlockReplicas=1
35 | tier1.sinks.emailsSink.hdfs.txnEventMax=10000
36 | tier1.sinks.emailsSink.hdfs.callTimeout=1000000
37 | tier1.sinks.emailsSink.channel=emailsChannel
38 | 
39 | # SOLR Sink
40 | tier1.sinks.solrSink.type=org.apache.flume.sink.solr.morphline.MorphlineSolrSink
41 | tier1.sinks.solrSink.channel=solrChannel
42 | tier1.sinks.solrSink.morphlineFile=/tmp/morphline.conf
43 | 
44 | 
45 | # Other properties are specific to each type of
46 | # source, channel, or sink. In this case, we
47 | # specify the capacity of the memory channel.
48 | tier1.channels.emailsChannel.capacity=100000000
49 | tier1.channels.emailsChannel.transactionCapacity=10000000
50 | 
51 | tier1.channels.solrChannel.capacity=100000000
52 | tier1.channels.solrChannel.transactionCapacity=10000000
53 | 
54 | 


--------------------------------------------------------------------------------
/email-search/indexer-config.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | <indexer table="inbox" mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper" mapping-type="column" >
 3 | 
 4 |    <!-- The relative or absolute path on the local file system to the morphline configuration file. -->
 5 |    <!-- Use relative path "morphlines.conf" for morphlines managed by Cloudera Manager -->
 6 |    <param name="morphlineFile" value="morphlines.conf"/>
 7 | 
 8 |    <!-- The optional morphlineId identifies a morphline if there are multiple morphlines in morphlines.conf -->
 9 |    <!-- <param name="morphlineId" value="morphline1"/> -->
10 | 
11 | </indexer>
12 | 


--------------------------------------------------------------------------------
/email-search/ks-indexer-email-morphlines.conf:
--------------------------------------------------------------------------------
 1 | SOLR_LOCATOR : {
 2 |   # Name of solr collection
 3 |   collection : collection1
 4 |   
 5 |   # ZooKeeper ensemble
 6 |   zkHost : "$ZK_HOST" 
 7 | }
 8 | 
 9 | 
10 | morphlines : [
11 | {
12 | id : morphline1
13 | importCommands : ["com.cloudera.**", "com.ngdata.**"]
14 | 
15 | commands : [                    
16 |   {
17 |     extractHBaseCells {
18 |       mappings : [
19 |         {
20 |           inputColumn : "messages:*"
21 |           outputField : "message" 
22 |           type : string 
23 |           source : value
24 |         }
25 |       ]
26 |     }
27 |   }
28 |       ## Break up the email text into SOLR fields
29 |     {
30 |         if {
31 |             conditions: [
32 |             {
33 |                 not{
34 |                     grok {
35 |                         expressions : {
36 |                             message: """(?s)(.*?)(Message-ID: <)(?<message_id>(.+?))(>.*?)(Date: )(?<date>(.+?))( \(.+?)(From: )(?<from>(.*?))((To: )(?<to>(.+?)))?(Subject: )(?<subject>(.*?))((Cc: )(?<cc>(.*)))?(Mime.+?)((Bcc: )(?<bcc>(.*)))?(X-From: )(?<from_names>(.*?))(X-To: )(?<to_names>(.*?))(X-cc: )(?<cc_names>(.*?))(X-bcc: )(?<bcc_names>(.*?))(X-Folder: )(?<x_folder>(.*?))(X-Origin: )(?<x_origin>(.*?))(X-FileName: )(?<x_filename>(.*?))(\n)(?<body>(.*))"""
37 |                         }
38 |                         extract: inplace
39 |                         findSubstrings: false
40 |                         addEmptyStrings: false
41 |                         numRequiredMatches: all
42 |                     }
43 |                 }
44 |             }
45 |             ]
46 |             then:[
47 |             { logInfo { format : "found no grok match: {}", args : ["@{}"] } }
48 |             { dropRecord {} }
49 |             ]
50 |         }
51 |     }
52 | 
53 |     # add Unique ID, in case our message_id field from above is not present
54 |     {
55 |         generateUUID {
56 |             field:message_id
57 |         }
58 |     }
59 | 
60 |     # convert the timestamp field to "yyyy-MM-dd'T'HH:mm:ss.SSSZ" format
61 |     {
62 |         convertTimestamp {
63 |             field : date
64 |             inputFormats : ["EEE, d MMM yyyy HH:mm:ss Z", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"]
65 |             inputTimezone : America/Los_Angeles
66 |             outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
67 |             outputTimezone : UTC
68 |         }
69 |     }
70 | 
71 |     # Consume the output record of the previous command and pipe another
72 |     # record downstream.
73 |     #
74 |     # This command sanitizes record fields that are unknown to Solr schema.xml
75 |     # by deleting them. Recall that Solr throws an exception on any attempt to
76 |     # load a document that contains a field that isn't specified in schema.xml
77 |     {
78 |         sanitizeUnknownSolrFields {
79 |             # Location from which to fetch Solr schema
80 |             solrLocator : ${SOLR_LOCATOR}
81 |         }
82 |     }
83 |     ]
84 | }
85 | ]
86 | 


--------------------------------------------------------------------------------
/email-search/morphlines.conf:
--------------------------------------------------------------------------------
  1 | # Specify server locations in a SOLR_LOCATOR variable;
  2 | # used later in variable substitutions
  3 | # Change the zkHost to point to your own Zookeeper quorum
  4 | SOLR_LOCATOR : {
  5 |     # Name of solr collection
  6 |     collection : email_collection
  7 |     # ZooKeeper ensemble
  8 |     zkHost : "clust2:2181/solr"
  9 | }
 10 | 
 11 | # Specify an array of one or more morphlines, each of which defines an ETL
 12 | # transformation chain. A morphline consists of one or more (potentially
 13 | # nested) commands. A morphline is a way to consume records (e.g. Flume events,
 14 | # HDFS files or blocks), turn them into a stream of records, and pipe the stream
 15 | # of records through a set of easily configurable transformations on it's way to
 16 | # Solr (or a MapReduceIndexerTool RecordWriter that feeds via a Reducer into Solr).
 17 | morphlines : [
 18 | {
 19 |     # Name used to identify a morphline. E.g. used if there are multiple morphlines in a
 20 |     # morphline config file
 21 |     id : morphline1
 22 |     # Import all morphline commands in these java packages and their subpackages.
 23 |     # Other commands that may be present on the classpath are not visible to this morphline.
 24 |     importCommands : ["com.cloudera.**", "org.apache.solr.**"]
 25 |     commands : [
 26 |     {
 27 |         ## Read the email stream and break it up into individual messages.
 28 |         ## The beginning of a message is marked by regex clause below
 29 |         ## The reason we use this command is that one event can have multiple
 30 |         ## messages
 31 |         readMultiLine {
 32 |             regex : "Message-ID:.*"
 33 |             what : next
 34 |             charset : UTF-8
 35 |         }
 36 |     }
 37 |     ## Break up the email text into SOLR fields
 38 |     {
 39 |         if {
 40 |             conditions: [
 41 |             {
 42 |                 not{
 43 |                     grok {
 44 |                         expressions : {
 45 |                             message: """(?s)(.*?)(Message-ID: <)(?<message_id>(.+?))(>.*?)(Date: )(?<date>(.+?))( \(.+?)(From: )(?<from>(.*?))((To: )(?<to>(.+?)))?(Subject: )(?<subject>(.*?))((Cc: )(?<cc>(.*)))?(Mime.+?)((Bcc: )(?<bcc>(.*)))?(X-From: )(?<from_names>(.*?))(X-To: )(?<to_names>(.*?))(X-cc: )(?<cc_names>(.*?))(X-bcc: )(?<bcc_names>(.*?))(X-Folder: )(?<x_folder>(.*?))(X-Origin: )(?<x_origin>(.*?))(X-FileName: )(?<x_filename>(.*?))(\n)(?<body>(.*))"""
 46 |                         }
 47 |                         extract: inplace
 48 |                         findSubstrings: false
 49 |                         addEmptyStrings: false
 50 |                         numRequiredMatches: all
 51 |                     }
 52 |                 }
 53 |             }
 54 |             ]
 55 |             then:[
 56 |             { logInfo { format : "found no grok match: {}", args : ["@{}"] } }
 57 |             { dropRecord {} }
 58 |             ]
 59 |         }
 60 |     }
 61 | 
 62 |     # add Unique ID, in case our message_id field from above is not present
 63 |     {
 64 |         generateUUID {
 65 |             field:message_id
 66 |         }
 67 |     }
 68 | 
 69 |     # convert the timestamp field to "yyyy-MM-dd'T'HH:mm:ss.SSSZ" format
 70 |     {
 71 |         convertTimestamp {
 72 |             field : date
 73 |             inputFormats : ["EEE, d MMM yyyy HH:mm:ss Z", "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"]
 74 |             inputTimezone : America/Los_Angeles
 75 |             outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
 76 |             outputTimezone : UTC
 77 |         }
 78 |     }
 79 | 
 80 |     # Consume the output record of the previous command and pipe another
 81 |     # record downstream.
 82 |     #
 83 |     # This command sanitizes record fields that are unknown to Solr schema.xml
 84 |     # by deleting them. Recall that Solr throws an exception on any attempt to
 85 |     # load a document that contains a field that isn't specified in schema.xml
 86 |     {
 87 |         sanitizeUnknownSolrFields {
 88 |             # Location from which to fetch Solr schema
 89 |             solrLocator : ${SOLR_LOCATOR}
 90 |         }
 91 |     }
 92 | 
 93 |     # load the record into a SolrServer or MapReduce SolrOutputFormat.
 94 |     {
 95 |         loadSolr {
 96 |             solrLocator : ${SOLR_LOCATOR}
 97 |         }
 98 |     }
 99 |     ]
100 | }
101 | ]
102 | 


--------------------------------------------------------------------------------
/email-search/schema.xml:
--------------------------------------------------------------------------------
   1 | 
   2 | <!--  
   3 |  This is the Solr schema file. This file should be named "schema.xml" and
   4 |  should be in the conf directory under the solr home
   5 |  (i.e. ./solr/conf/schema.xml by default) 
   6 |  or located where the classloader for the Solr webapp can find it.
   7 | 
   8 |  This example schema is the recommended starting point for users.
   9 |  It should be kept correct and concise, usable out-of-the-box.
  10 | 
  11 |  For more information, on how to customize this file, please see
  12 |  http://wiki.apache.org/solr/SchemaXml
  13 | 
  14 |  PERFORMANCE NOTE: this schema includes many optional features and should not
  15 |  be used for benchmarking.  To improve performance one could
  16 |   - set stored="false" for all fields possible (esp large fields) when you
  17 |     only need to search on the field but don't need to return the original
  18 |     value.
  19 |   - set indexed="false" if you don't need to search on the field, but only
  20 |     return the field as a result of searching on other indexed fields.
  21 |   - remove all unneeded copyField statements
  22 |   - for best index size and searching performance, set "index" to false
  23 |     for all general text fields, use copyField to copy them to the
  24 |     catchall "text" field, and use that for searching.
  25 |   - For maximum indexing performance, use the StreamingUpdateSolrServer
  26 |     java client.
  27 |   - Remember to run the JVM in server mode, and use a higher logging level
  28 |     that avoids logging every request
  29 | -->
  30 | 
  31 | <schema name="example" version="1.5">
  32 |   <!-- attribute "name" is the name of this schema and is only used for display purposes.
  33 |        version="x.y" is Solr's version number for the schema syntax and 
  34 |        semantics.  It should not normally be changed by applications.
  35 | 
  36 |        1.0: multiValued attribute did not exist, all fields are multiValued 
  37 |             by nature
  38 |        1.1: multiValued attribute introduced, false by default 
  39 |        1.2: omitTermFreqAndPositions attribute introduced, true by default 
  40 |             except for text fields.
  41 |        1.3: removed optional field compress feature
  42 |        1.4: autoGeneratePhraseQueries attribute introduced to drive QueryParser
  43 |             behavior when a single string produces multiple tokens.  Defaults 
  44 |             to off for version >= 1.4
  45 |        1.5: omitNorms defaults to true for primitive field types 
  46 |             (int, float, boolean, string...)
  47 |      -->
  48 | 
  49 |  <fields>
  50 |    <!-- Valid attributes for fields:
  51 |      name: mandatory - the name for the field
  52 |      type: mandatory - the name of a field type from the 
  53 |        <types> fieldType section
  54 |      indexed: true if this field should be indexed (searchable or sortable)
  55 |      stored: true if this field should be retrievable
  56 |      docValues: true if this field should have doc values. Doc values are
  57 |        useful for faceting, grouping, sorting and function queries. Although not
  58 |        required, doc values will make the index faster to load, more
  59 |        NRT-friendly and more memory-efficient. They however come with some
  60 |        limitations: they are currently only supported by StrField, UUIDField
  61 |        and all Trie*Fields, and depending on the field type, they might
  62 |        require the field to be single-valued, be required or have a default
  63 |        value (check the documentation of the field type you're interested in
  64 |        for more information)
  65 |      multiValued: true if this field may contain multiple values per document
  66 |      omitNorms: (expert) set to true to omit the norms associated with
  67 |        this field (this disables length normalization and index-time
  68 |        boosting for the field, and saves some memory).  Only full-text
  69 |        fields or fields that need an index-time boost need norms.
  70 |        Norms are omitted for primitive (non-analyzed) types by default.
  71 |      termVectors: [false] set to true to store the term vector for a
  72 |        given field.
  73 |        When using MoreLikeThis, fields used for similarity should be
  74 |        stored for best performance.
  75 |      termPositions: Store position information with the term vector.  
  76 |        This will increase storage costs.
  77 |      termOffsets: Store offset information with the term vector. This 
  78 |        will increase storage costs.
  79 |      required: The field is required.  It will throw an error if the
  80 |        value does not exist
  81 |      default: a value that should be used if no value is specified
  82 |        when adding a document.
  83 |    -->
  84 | 
  85 |    <!-- field names should consist of alphanumeric or underscore characters only and
  86 |       not start with a digit.  This is not currently strictly enforced,
  87 |       but other field names will not have first class support from all components
  88 |       and back compatibility is not guaranteed.  Names with both leading and
  89 |       trailing underscores (e.g. _version_) are reserved.
  90 |    -->
  91 |         
  92 |    <field name="message_id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> 
  93 |    <field name="date" type="tdate" indexed="true" stored="true"/>
  94 |    <field name="from" type="email_general" indexed="true" stored="true"/>
  95 |    <field name="to" type="email_general" indexed="true" stored="true"/>
  96 |    <field name="subject" type="text_general" indexed="true" stored="true"/>
  97 |    <field name="from_names" type="names_general" indexed="true" stored="true"/>
  98 |    <field name="to_names" type="names_general" indexed="true" stored="true"/>
  99 |    <field name="cc" type="email_general" indexed="true" stored="true"/>
 100 |    <field name="bcc" type="text_general" indexed="true" stored="true"/>
 101 |    <field name="cc_names" type="names_general" indexed="true" stored="true"/>
 102 |    <field name="bcc_names" type="names_general" indexed="true" stored="true" />
 103 |    <field name="x_folder" type="text_general" indexed="true" stored="false" />
 104 |    <field name="x_origin" type="text_general" indexed="true" stored="false" />
 105 |    <field name="x_filename" type="text_general" indexed="true" stored="false"/>
 106 |    <field name="message" type="text_general" indexed="false" stored="false"/>
 107 |    <field name="body" type="text_general" indexed="true" stored="true"/>
 108 |    <field name="text" type="text_general" indexed="true" stored="false" multiValued="true"/>
 109 |    <field name="_version_" type="long" indexed="true" stored="true"/> 
 110 | 
 111 |  </fields>
 112 | 
 113 | 
 114 |  <!-- Field to use to determine and enforce document uniqueness. 
 115 |       Unless this field is marked with required="false", it will be a required field
 116 |    -->
 117 |  <uniqueKey>message_id</uniqueKey>
 118 | 
 119 | 
 120 |  <!-- DEPRECATED: The defaultOperator (AND|OR) is consulted by various query parsers
 121 |   when parsing a query string to determine if a clause of the query should be marked as
 122 |   required or optional, assuming the clause isn't already marked by some operator.
 123 |   The default is OR, which is generally assumed so it is not a good idea to change it
 124 |   globally here.  The "q.op" request parameter takes precedence over this.
 125 |  <solrQueryParser defaultOperator="OR"/> -->
 126 | 
 127 |   <!-- copyField commands copy one field to another at the time a document
 128 |         is added to the index.  It's used either to index the same field differently,
 129 |         or to add multiple fields to the same field for easier/faster searching.  -->
 130 | 
 131 |    <copyField source="message_id" dest="text"/>
 132 |    <copyField source="date" dest="text"/>
 133 |    <copyField source="from" dest="text"/>
 134 |    <copyField source="to" dest="text"/>
 135 |    <copyField source="subject" dest="text"/>
 136 |    <copyField source="from_names" dest="text"/>
 137 |    <copyField source="cc" dest="text"/>
 138 |    <copyField source="cc_names" dest="text"/>
 139 |    <copyField source="bcc" dest="text"/>
 140 |    <copyField source="bcc_names" dest="text"/>
 141 |    <copyField source="x_folder" dest="text"/>
 142 |    <copyField source="x_origin" dest="text"/>
 143 |    <copyField source="x_filename" dest="text"/>
 144 |    <copyField source="to_names" dest="text"/>
 145 |    <copyField source="message" dest="text"/>
 146 |    <copyField source="body" dest="text"/>
 147 | 
 148 |  
 149 |   <types>
 150 |     <!-- field type definitions. The "name" attribute is
 151 |        just a label to be used by field definitions.  The "class"
 152 |        attribute and any other attributes determine the real
 153 |        behavior of the fieldType.
 154 |          Class names starting with "solr" refer to java classes in a
 155 |        standard package such as org.apache.solr.analysis
 156 |     -->
 157 | 
 158 |     <!-- The StrField type is not analyzed, but indexed/stored verbatim.
 159 |        It supports doc values but in that case the field needs to be
 160 |        single-valued and either required or have a default value.
 161 |       -->
 162 |     <fieldType name="string" class="solr.StrField" sortMissingLast="true" />
 163 | 
 164 |     <!-- boolean type: "true" or "false" -->
 165 |     <fieldType name="boolean" class="solr.BoolField" sortMissingLast="true"/>
 166 | 
 167 |     <!-- sortMissingLast and sortMissingFirst attributes are optional attributes are
 168 |          currently supported on types that are sorted internally as strings
 169 |          and on numeric types.
 170 | 	     This includes "string","boolean", and, as of 3.5 (and 4.x),
 171 | 	     int, float, long, date, double, including the "Trie" variants.
 172 |        - If sortMissingLast="true", then a sort on this field will cause documents
 173 |          without the field to come after documents with the field,
 174 |          regardless of the requested sort order (asc or desc).
 175 |        - If sortMissingFirst="true", then a sort on this field will cause documents
 176 |          without the field to come before documents with the field,
 177 |          regardless of the requested sort order.
 178 |        - If sortMissingLast="false" and sortMissingFirst="false" (the default),
 179 |          then default lucene sorting will be used which places docs without the
 180 |          field first in an ascending sort and last in a descending sort.
 181 |     -->    
 182 | 
 183 |     <!--
 184 |       Default numeric field types. For faster range queries, consider the tint/tfloat/tlong/tdouble types.
 185 | 
 186 |       These fields support doc values, but they require the field to be
 187 |       single-valued and either be required or have a default value.
 188 |     -->
 189 |     <fieldType name="int" class="solr.TrieIntField" precisionStep="0" positionIncrementGap="0"/>
 190 |     <fieldType name="float" class="solr.TrieFloatField" precisionStep="0" positionIncrementGap="0"/>
 191 |     <fieldType name="long" class="solr.TrieLongField" precisionStep="0" positionIncrementGap="0"/>
 192 |     <fieldType name="double" class="solr.TrieDoubleField" precisionStep="0" positionIncrementGap="0"/>
 193 | 
 194 |     <!--
 195 |      Numeric field types that index each value at various levels of precision
 196 |      to accelerate range queries when the number of values between the range
 197 |      endpoints is large. See the javadoc for NumericRangeQuery for internal
 198 |      implementation details.
 199 | 
 200 |      Smaller precisionStep values (specified in bits) will lead to more tokens
 201 |      indexed per value, slightly larger index size, and faster range queries.
 202 |      A precisionStep of 0 disables indexing at different precision levels.
 203 |     -->
 204 |     <fieldType name="tint" class="solr.TrieIntField" precisionStep="8" positionIncrementGap="0"/>
 205 |     <fieldType name="tfloat" class="solr.TrieFloatField" precisionStep="8" positionIncrementGap="0"/>
 206 |     <fieldType name="tlong" class="solr.TrieLongField" precisionStep="8" positionIncrementGap="0"/>
 207 |     <fieldType name="tdouble" class="solr.TrieDoubleField" precisionStep="8" positionIncrementGap="0"/>
 208 | 
 209 |     <!-- The format for this date field is of the form 1995-12-31T23:59:59Z, and
 210 |          is a more restricted form of the canonical representation of dateTime
 211 |          http://www.w3.org/TR/xmlschema-2/#dateTime    
 212 |          The trailing "Z" designates UTC time and is mandatory.
 213 |          Optional fractional seconds are allowed: 1995-12-31T23:59:59.999Z
 214 |          All other components are mandatory.
 215 | 
 216 |          Expressions can also be used to denote calculations that should be
 217 |          performed relative to "NOW" to determine the value, ie...
 218 | 
 219 |                NOW/HOUR
 220 |                   ... Round to the start of the current hour
 221 |                NOW-1DAY
 222 |                   ... Exactly 1 day prior to now
 223 |                NOW/DAY+6MONTHS+3DAYS
 224 |                   ... 6 months and 3 days in the future from the start of
 225 |                       the current day
 226 |                       
 227 |          Consult the DateField javadocs for more information.
 228 | 
 229 |          Note: For faster range queries, consider the tdate type
 230 |       -->
 231 |     <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/>
 232 | 
 233 |     <!-- A Trie based date field for faster date range queries and date faceting. -->
 234 |     <fieldType name="tdate" class="solr.TrieDateField" precisionStep="6" positionIncrementGap="0"/>
 235 | 
 236 | 
 237 |     <!--Binary data type. The data should be sent/retrieved in as Base64 encoded Strings -->
 238 |     <fieldtype name="binary" class="solr.BinaryField"/>
 239 | 
 240 |     <!--
 241 |       Note:
 242 |       These should only be used for compatibility with existing indexes (created with lucene or older Solr versions).
 243 |       Use Trie based fields instead. As of Solr 3.5 and 4.x, Trie based fields support sortMissingFirst/Last
 244 |       
 245 |       Plain numeric field types that store and index the text
 246 |       value verbatim (and hence don't correctly support range queries, since the
 247 |       lexicographic ordering isn't equal to the numeric ordering)
 248 |     -->
 249 |     <fieldType name="pint" class="solr.IntField"/>
 250 |     <fieldType name="plong" class="solr.LongField"/>
 251 |     <fieldType name="pfloat" class="solr.FloatField"/>
 252 |     <fieldType name="pdouble" class="solr.DoubleField"/>
 253 |     <fieldType name="pdate" class="solr.DateField" sortMissingLast="true"/>
 254 | 
 255 |     <!-- The "RandomSortField" is not used to store or search any
 256 |          data.  You can declare fields of this type it in your schema
 257 |          to generate pseudo-random orderings of your docs for sorting 
 258 |          or function purposes.  The ordering is generated based on the field
 259 |          name and the version of the index. As long as the index version
 260 |          remains unchanged, and the same field name is reused,
 261 |          the ordering of the docs will be consistent.  
 262 |          If you want different psuedo-random orderings of documents,
 263 |          for the same version of the index, use a dynamicField and
 264 |          change the field name in the request.
 265 |      -->
 266 |     <fieldType name="random" class="solr.RandomSortField" indexed="true" />
 267 | 
 268 |     <!-- solr.TextField allows the specification of custom text analyzers
 269 |          specified as a tokenizer and a list of token filters. Different
 270 |          analyzers may be specified for indexing and querying.
 271 | 
 272 |          The optional positionIncrementGap puts space between multiple fields of
 273 |          this type on the same document, with the purpose of preventing false phrase
 274 |          matching across fields.
 275 | 
 276 |          For more info on customizing your analyzer chain, please see
 277 |          http://wiki.apache.org/solr/AnalyzersTokenizersTokenFilters
 278 |      -->
 279 | 
 280 |     <!-- One can also specify an existing Analyzer class that has a
 281 |          default constructor via the class attribute on the analyzer element.
 282 |          Example:
 283 |     <fieldType name="text_greek" class="solr.TextField">
 284 |       <analyzer class="org.apache.lucene.analysis.el.GreekAnalyzer"/>
 285 |     </fieldType>
 286 |     -->
 287 | 
 288 |     <!-- A text field that only splits on whitespace for exact matching of words -->
 289 |     <fieldType name="text_ws" class="solr.TextField" positionIncrementGap="100">
 290 |       <analyzer>
 291 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 292 |       </analyzer>
 293 |     </fieldType>
 294 | 
 295 |     <!-- A general text field that has reasonable, generic
 296 |          cross-language defaults: it tokenizes with StandardTokenizer,
 297 | 	 removes stop words from case-insensitive "stopwords.txt"
 298 | 	 (empty by default), and down cases.  At query time only, it
 299 | 	 also applies synonyms. -->
 300 |     <fieldType name="text_general" class="solr.TextField" positionIncrementGap="100">
 301 |       <analyzer type="index">
 302 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 303 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 304 |         <!-- in this example, we will only use synonyms at query time
 305 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 306 |         -->
 307 |         <filter class="solr.LowerCaseFilterFactory"/>
 308 |       </analyzer>
 309 |       <analyzer type="query">
 310 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 311 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 312 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 313 |         <filter class="solr.LowerCaseFilterFactory"/>
 314 |       </analyzer>
 315 |     </fieldType>
 316 | 
 317 |     <fieldType name="names_general" class="solr.TextField" >
 318 |       <analyzer type="index">
 319 | 	<charFilter class="solr.HTMLStripCharFilterFactory" />
 320 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 321 |         <filter class="solr.LowerCaseFilterFactory"/>
 322 |       </analyzer>
 323 |       <analyzer type="query">
 324 | 	<charFilter class="solr.HTMLStripCharFilterFactory" />
 325 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 326 |         <filter class="solr.LowerCaseFilterFactory"/>
 327 |       </analyzer>
 328 |     </fieldType>
 329 | 
 330 |     <fieldType name="email_general" class="solr.TextField" >
 331 |       <analyzer type="index">
 332 |         <tokenizer class="solr.UAX29URLEmailTokenizerFactory" />
 333 |         <filter class="solr.LowerCaseFilterFactory"/>
 334 |       </analyzer>
 335 |       <analyzer type="query">
 336 |         <tokenizer class="solr.UAX29URLEmailTokenizerFactory" />
 337 |         <filter class="solr.LowerCaseFilterFactory"/>
 338 |       </analyzer>
 339 |     </fieldType>
 340 | 
 341 |     <!-- A text field with defaults appropriate for English: it
 342 |          tokenizes with StandardTokenizer, removes English stop words
 343 |          (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
 344 |          finally applies Porter's stemming.  The query time analyzer
 345 |          also applies synonyms from synonyms.txt. -->
 346 | 
 347 |     <!-- A text field with defaults appropriate for English: it
 348 |          tokenizes with StandardTokenizer, removes English stop words
 349 |          (lang/stopwords_en.txt), down cases, protects words from protwords.txt, and
 350 |          finally applies Porter's stemming.  The query time analyzer
 351 |          also applies synonyms from synonyms.txt. -->
 352 |     <fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
 353 |       <analyzer type="index">
 354 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 355 |         <!-- in this example, we will only use synonyms at query time
 356 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 357 |         -->
 358 |         <!-- Case insensitive stop word removal.
 359 |           add enablePositionIncrements=true in both the index and query
 360 |           analyzers to leave a 'gap' for more accurate phrase queries.
 361 |         -->
 362 |         <filter class="solr.StopFilterFactory"
 363 |                 ignoreCase="true"
 364 |                 words="lang/stopwords_en.txt"
 365 |                 enablePositionIncrements="true"
 366 |                 />
 367 |         <filter class="solr.LowerCaseFilterFactory"/>
 368 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
 369 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 370 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
 371 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 372 | 	-->
 373 |         <filter class="solr.PorterStemFilterFactory"/>
 374 |       </analyzer>
 375 |       <analyzer type="query">
 376 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 377 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 378 |         <filter class="solr.StopFilterFactory"
 379 |                 ignoreCase="true"
 380 |                 words="lang/stopwords_en.txt"
 381 |                 enablePositionIncrements="true"
 382 |                 />
 383 |         <filter class="solr.LowerCaseFilterFactory"/>
 384 | 	<filter class="solr.EnglishPossessiveFilterFactory"/>
 385 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 386 | 	<!-- Optionally you may want to use this less aggressive stemmer instead of PorterStemFilterFactory:
 387 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 388 | 	-->
 389 |         <filter class="solr.PorterStemFilterFactory"/>
 390 |       </analyzer>
 391 |     </fieldType>
 392 | 
 393 |     <!-- A text field with defaults appropriate for English, plus
 394 | 	 aggressive word-splitting and autophrase features enabled.
 395 | 	 This field is just like text_en, except it adds
 396 | 	 WordDelimiterFilter to enable splitting and matching of
 397 | 	 words on case-change, alpha numeric boundaries, and
 398 | 	 non-alphanumeric chars.  This means certain compound word
 399 | 	 cases will work, for example query "wi fi" will match
 400 | 	 document "WiFi" or "wi-fi".
 401 |         -->
 402 |     <fieldType name="text_en_splitting" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
 403 |       <analyzer type="index">
 404 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 405 |         <!-- in this example, we will only use synonyms at query time
 406 |         <filter class="solr.SynonymFilterFactory" synonyms="index_synonyms.txt" ignoreCase="true" expand="false"/>
 407 |         -->
 408 |         <!-- Case insensitive stop word removal.
 409 |           add enablePositionIncrements=true in both the index and query
 410 |           analyzers to leave a 'gap' for more accurate phrase queries.
 411 |         -->
 412 |         <filter class="solr.StopFilterFactory"
 413 |                 ignoreCase="true"
 414 |                 words="lang/stopwords_en.txt"
 415 |                 enablePositionIncrements="true"
 416 |                 />
 417 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="1" catenateNumbers="1" catenateAll="0" splitOnCaseChange="1"/>
 418 |         <filter class="solr.LowerCaseFilterFactory"/>
 419 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 420 |         <filter class="solr.PorterStemFilterFactory"/>
 421 |       </analyzer>
 422 |       <analyzer type="query">
 423 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 424 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 425 |         <filter class="solr.StopFilterFactory"
 426 |                 ignoreCase="true"
 427 |                 words="lang/stopwords_en.txt"
 428 |                 enablePositionIncrements="true"
 429 |                 />
 430 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="1" generateNumberParts="1" catenateWords="0" catenateNumbers="0" catenateAll="0" splitOnCaseChange="1"/>
 431 |         <filter class="solr.LowerCaseFilterFactory"/>
 432 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 433 |         <filter class="solr.PorterStemFilterFactory"/>
 434 |       </analyzer>
 435 |     </fieldType>
 436 | 
 437 |     <!-- Less flexible matching, but less false matches.  Probably not ideal for product names,
 438 |          but may be good for SKUs.  Can insert dashes in the wrong place and still match. -->
 439 |     <fieldType name="text_en_splitting_tight" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="true">
 440 |       <analyzer>
 441 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 442 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="false"/>
 443 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_en.txt"/>
 444 |         <filter class="solr.WordDelimiterFilterFactory" generateWordParts="0" generateNumberParts="0" catenateWords="1" catenateNumbers="1" catenateAll="0"/>
 445 |         <filter class="solr.LowerCaseFilterFactory"/>
 446 |         <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
 447 |         <filter class="solr.EnglishMinimalStemFilterFactory"/>
 448 |         <!-- this filter can remove any duplicate tokens that appear at the same position - sometimes
 449 |              possible with WordDelimiterFilter in conjuncton with stemming. -->
 450 |         <filter class="solr.RemoveDuplicatesTokenFilterFactory"/>
 451 |       </analyzer>
 452 |     </fieldType>
 453 | 
 454 |     <!-- Just like text_general except it reverses the characters of
 455 | 	 each token, to enable more efficient leading wildcard queries. -->
 456 |     <fieldType name="text_general_rev" class="solr.TextField" positionIncrementGap="100">
 457 |       <analyzer type="index">
 458 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 459 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 460 |         <filter class="solr.LowerCaseFilterFactory"/>
 461 |         <filter class="solr.ReversedWildcardFilterFactory" withOriginal="true"
 462 |            maxPosAsterisk="3" maxPosQuestion="2" maxFractionAsterisk="0.33"/>
 463 |       </analyzer>
 464 |       <analyzer type="query">
 465 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 466 |         <filter class="solr.SynonymFilterFactory" synonyms="synonyms.txt" ignoreCase="true" expand="true"/>
 467 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="stopwords.txt" enablePositionIncrements="true" />
 468 |         <filter class="solr.LowerCaseFilterFactory"/>
 469 |       </analyzer>
 470 |     </fieldType>
 471 | 
 472 |     <!-- charFilter + WhitespaceTokenizer  -->
 473 |     <!--
 474 |     <fieldType name="text_char_norm" class="solr.TextField" positionIncrementGap="100" >
 475 |       <analyzer>
 476 |         <charFilter class="solr.MappingCharFilterFactory" mapping="mapping-ISOLatin1Accent.txt"/>
 477 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 478 |       </analyzer>
 479 |     </fieldType>
 480 |     -->
 481 | 
 482 |     <!-- This is an example of using the KeywordTokenizer along
 483 |          With various TokenFilterFactories to produce a sortable field
 484 |          that does not include some properties of the source text
 485 |       -->
 486 |     <fieldType name="alphaOnlySort" class="solr.TextField" sortMissingLast="true" omitNorms="true">
 487 |       <analyzer>
 488 |         <!-- KeywordTokenizer does no actual tokenizing, so the entire
 489 |              input string is preserved as a single token
 490 |           -->
 491 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
 492 |         <!-- The LowerCase TokenFilter does what you expect, which can be
 493 |              when you want your sorting to be case insensitive
 494 |           -->
 495 |         <filter class="solr.LowerCaseFilterFactory" />
 496 |         <!-- The TrimFilter removes any leading or trailing whitespace -->
 497 |         <filter class="solr.TrimFilterFactory" />
 498 |         <!-- The PatternReplaceFilter gives you the flexibility to use
 499 |              Java Regular expression to replace any sequence of characters
 500 |              matching a pattern with an arbitrary replacement string, 
 501 |              which may include back references to portions of the original
 502 |              string matched by the pattern.
 503 |              
 504 |              See the Java Regular Expression documentation for more
 505 |              information on pattern and replacement string syntax.
 506 |              
 507 |              http://java.sun.com/j2se/1.6.0/docs/api/java/util/regex/package-summary.html
 508 |           -->
 509 |         <filter class="solr.PatternReplaceFilterFactory"
 510 |                 pattern="([^a-z])" replacement="" replace="all"
 511 |         />
 512 |       </analyzer>
 513 |     </fieldType>
 514 |     
 515 |     <fieldtype name="phonetic" stored="false" indexed="true" class="solr.TextField" >
 516 |       <analyzer>
 517 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 518 |         <filter class="solr.DoubleMetaphoneFilterFactory" inject="false"/>
 519 |       </analyzer>
 520 |     </fieldtype>
 521 | 
 522 |     <fieldtype name="payloads" stored="false" indexed="true" class="solr.TextField" >
 523 |       <analyzer>
 524 |         <tokenizer class="solr.WhitespaceTokenizerFactory"/>
 525 |         <!--
 526 |         The DelimitedPayloadTokenFilter can put payloads on tokens... for example,
 527 |         a token of "foo|1.4"  would be indexed as "foo" with a payload of 1.4f
 528 |         Attributes of the DelimitedPayloadTokenFilterFactory : 
 529 |          "delimiter" - a one character delimiter. Default is | (pipe)
 530 | 	 "encoder" - how to encode the following value into a playload
 531 | 	    float -> org.apache.lucene.analysis.payloads.FloatEncoder,
 532 | 	    integer -> o.a.l.a.p.IntegerEncoder
 533 | 	    identity -> o.a.l.a.p.IdentityEncoder
 534 |             Fully Qualified class name implementing PayloadEncoder, Encoder must have a no arg constructor.
 535 |          -->
 536 |         <filter class="solr.DelimitedPayloadTokenFilterFactory" encoder="float"/>
 537 |       </analyzer>
 538 |     </fieldtype>
 539 | 
 540 |     <!-- lowercases the entire field value, keeping it as a single token.  -->
 541 |     <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100">
 542 |       <analyzer>
 543 |         <tokenizer class="solr.KeywordTokenizerFactory"/>
 544 |         <filter class="solr.LowerCaseFilterFactory" />
 545 |       </analyzer>
 546 |     </fieldType>
 547 | 
 548 |     <!-- 
 549 |       Example of using PathHierarchyTokenizerFactory at index time, so
 550 |       queries for paths match documents at that path, or in descendent paths
 551 |     -->
 552 |     <fieldType name="descendent_path" class="solr.TextField">
 553 |       <analyzer type="index">
 554 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
 555 |       </analyzer>
 556 |       <analyzer type="query">
 557 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
 558 |       </analyzer>
 559 |     </fieldType>
 560 |     <!-- 
 561 |       Example of using PathHierarchyTokenizerFactory at query time, so
 562 |       queries for paths match documents at that path, or in ancestor paths
 563 |     -->
 564 |     <fieldType name="ancestor_path" class="solr.TextField">
 565 |       <analyzer type="index">
 566 | 	<tokenizer class="solr.KeywordTokenizerFactory" />
 567 |       </analyzer>
 568 |       <analyzer type="query">
 569 | 	<tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
 570 |       </analyzer>
 571 |     </fieldType>
 572 | 
 573 |     <!-- since fields of this type are by default not stored or indexed,
 574 |          any data added to them will be ignored outright.  --> 
 575 |     <fieldtype name="ignored" stored="false" indexed="false" multiValued="true" class="solr.StrField" />
 576 | 
 577 |     <!-- This point type indexes the coordinates as separate fields (subFields)
 578 |       If subFieldType is defined, it references a type, and a dynamic field
 579 |       definition is created matching *___<typename>.  Alternately, if 
 580 |       subFieldSuffix is defined, that is used to create the subFields.
 581 |       Example: if subFieldType="double", then the coordinates would be
 582 |         indexed in fields myloc_0___double,myloc_1___double.
 583 |       Example: if subFieldSuffix="_d" then the coordinates would be indexed
 584 |         in fields myloc_0_d,myloc_1_d
 585 |       The subFields are an implementation detail of the fieldType, and end
 586 |       users normally should not need to know about them.
 587 |      -->
 588 |     <fieldType name="point" class="solr.PointType" dimension="2" subFieldSuffix="_d"/>
 589 | 
 590 |     <!-- A specialized field for geospatial search. If indexed, this fieldType must not be multivalued. -->
 591 |     <fieldType name="location" class="solr.LatLonType" subFieldSuffix="_coordinate"/>
 592 | 
 593 |     <!-- An alternative geospatial field type new to Solr 4.  It supports multiValued and polygon shapes.
 594 |       For more information about this and other Spatial fields new to Solr 4, see:
 595 |       http://wiki.apache.org/solr/SolrAdaptersForLuceneSpatial4
 596 |     -->
 597 |     <fieldType name="location_rpt" class="solr.SpatialRecursivePrefixTreeFieldType"
 598 |         geo="true" distErrPct="0.025" maxDistErr="0.000009" units="degrees" />
 599 | 
 600 |    <!-- Money/currency field type. See http://wiki.apache.org/solr/MoneyFieldType
 601 |         Parameters:
 602 |           defaultCurrency: Specifies the default currency if none specified. Defaults to "USD"
 603 |           precisionStep:   Specifies the precisionStep for the TrieLong field used for the amount
 604 |           providerClass:   Lets you plug in other exchange provider backend:
 605 |                            solr.FileExchangeRateProvider is the default and takes one parameter:
 606 |                              currencyConfig: name of an xml file holding exchange rates
 607 |                            solr.OpenExchangeRatesOrgProvider uses rates from openexchangerates.org:
 608 |                              ratesFileLocation: URL or path to rates JSON file (default latest.json on the web)
 609 |                              refreshInterval: Number of minutes between each rates fetch (default: 1440, min: 60)
 610 |    -->
 611 |    <!--  <fieldType name="currency" class="solr.CurrencyField" precisionStep="8" defaultCurrency="USD" currencyConfig="currency.xml" />
 612 |    -->
 613 |              
 614 | 
 615 | 
 616 |    <!-- some examples for different languages (generally ordered by ISO code) -->
 617 | 
 618 |     <!-- Arabic -->
 619 |     <fieldType name="text_ar" class="solr.TextField" positionIncrementGap="100">
 620 |       <analyzer> 
 621 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 622 |         <!-- for any non-arabic -->
 623 |         <filter class="solr.LowerCaseFilterFactory"/>
 624 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ar.txt" enablePositionIncrements="true"/>
 625 |         <!-- normalizes ﻯ to ﻱ, etc -->
 626 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
 627 |         <filter class="solr.ArabicStemFilterFactory"/>
 628 |       </analyzer>
 629 |     </fieldType>
 630 | 
 631 |     <!-- Bulgarian -->
 632 |     <fieldType name="text_bg" class="solr.TextField" positionIncrementGap="100">
 633 |       <analyzer> 
 634 |         <tokenizer class="solr.StandardTokenizerFactory"/> 
 635 |         <filter class="solr.LowerCaseFilterFactory"/>
 636 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_bg.txt" enablePositionIncrements="true"/>
 637 |         <filter class="solr.BulgarianStemFilterFactory"/>       
 638 |       </analyzer>
 639 |     </fieldType>
 640 |     
 641 |     <!-- Catalan -->
 642 |     <fieldType name="text_ca" class="solr.TextField" positionIncrementGap="100">
 643 |       <analyzer> 
 644 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 645 |         <!-- removes l', etc -->
 646 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ca.txt"/>
 647 |         <filter class="solr.LowerCaseFilterFactory"/>
 648 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ca.txt" enablePositionIncrements="true"/>
 649 |         <filter class="solr.SnowballPorterFilterFactory" language="Catalan"/>       
 650 |       </analyzer>
 651 |     </fieldType>
 652 |     
 653 |     <!-- CJK bigram (see text_ja for a Japanese configuration using morphological analysis) -->
 654 |     <fieldType name="text_cjk" class="solr.TextField" positionIncrementGap="100">
 655 |       <analyzer>
 656 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 657 |         <!-- normalize width before bigram, as e.g. half-width dakuten combine  -->
 658 |         <filter class="solr.CJKWidthFilterFactory"/>
 659 |         <!-- for any non-CJK -->
 660 |         <filter class="solr.LowerCaseFilterFactory"/>
 661 |         <filter class="solr.CJKBigramFilterFactory"/>
 662 |       </analyzer>
 663 |     </fieldType>
 664 | 
 665 |     <!-- Czech -->
 666 |     <fieldType name="text_cz" class="solr.TextField" positionIncrementGap="100">
 667 |       <analyzer> 
 668 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 669 |         <filter class="solr.LowerCaseFilterFactory"/>
 670 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_cz.txt" enablePositionIncrements="true"/>
 671 |         <filter class="solr.CzechStemFilterFactory"/>       
 672 |       </analyzer>
 673 |     </fieldType>
 674 |     
 675 |     <!-- Danish -->
 676 |     <fieldType name="text_da" class="solr.TextField" positionIncrementGap="100">
 677 |       <analyzer> 
 678 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 679 |         <filter class="solr.LowerCaseFilterFactory"/>
 680 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_da.txt" format="snowball" enablePositionIncrements="true"/>
 681 |         <filter class="solr.SnowballPorterFilterFactory" language="Danish"/>       
 682 |       </analyzer>
 683 |     </fieldType>
 684 |     
 685 |     <!-- German -->
 686 |     <fieldType name="text_de" class="solr.TextField" positionIncrementGap="100">
 687 |       <analyzer> 
 688 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 689 |         <filter class="solr.LowerCaseFilterFactory"/>
 690 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_de.txt" format="snowball" enablePositionIncrements="true"/>
 691 |         <filter class="solr.GermanNormalizationFilterFactory"/>
 692 |         <filter class="solr.GermanLightStemFilterFactory"/>
 693 |         <!-- less aggressive: <filter class="solr.GermanMinimalStemFilterFactory"/> -->
 694 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="German2"/> -->
 695 |       </analyzer>
 696 |     </fieldType>
 697 |     
 698 |     <!-- Greek -->
 699 |     <fieldType name="text_el" class="solr.TextField" positionIncrementGap="100">
 700 |       <analyzer> 
 701 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 702 |         <!-- greek specific lowercase for sigma -->
 703 |         <filter class="solr.GreekLowerCaseFilterFactory"/>
 704 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_el.txt" enablePositionIncrements="true"/>
 705 |         <filter class="solr.GreekStemFilterFactory"/>
 706 |       </analyzer>
 707 |     </fieldType>
 708 |     
 709 |     <!-- Spanish -->
 710 |     <fieldType name="text_es" class="solr.TextField" positionIncrementGap="100">
 711 |       <analyzer> 
 712 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 713 |         <filter class="solr.LowerCaseFilterFactory"/>
 714 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_es.txt" format="snowball" enablePositionIncrements="true"/>
 715 |         <filter class="solr.SpanishLightStemFilterFactory"/>
 716 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Spanish"/> -->
 717 |       </analyzer>
 718 |     </fieldType>
 719 |     
 720 |     <!-- Basque -->
 721 |     <fieldType name="text_eu" class="solr.TextField" positionIncrementGap="100">
 722 |       <analyzer> 
 723 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 724 |         <filter class="solr.LowerCaseFilterFactory"/>
 725 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_eu.txt" enablePositionIncrements="true"/>
 726 |         <filter class="solr.SnowballPorterFilterFactory" language="Basque"/>
 727 |       </analyzer>
 728 |     </fieldType>
 729 |     
 730 |     <!-- Persian -->
 731 |     <fieldType name="text_fa" class="solr.TextField" positionIncrementGap="100">
 732 |       <analyzer>
 733 |         <!-- for ZWNJ -->
 734 |         <charFilter class="solr.PersianCharFilterFactory"/>
 735 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 736 |         <filter class="solr.LowerCaseFilterFactory"/>
 737 |         <filter class="solr.ArabicNormalizationFilterFactory"/>
 738 |         <filter class="solr.PersianNormalizationFilterFactory"/>
 739 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fa.txt" enablePositionIncrements="true"/>
 740 |       </analyzer>
 741 |     </fieldType>
 742 |     
 743 |     <!-- Finnish -->
 744 |     <fieldType name="text_fi" class="solr.TextField" positionIncrementGap="100">
 745 |       <analyzer> 
 746 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 747 |         <filter class="solr.LowerCaseFilterFactory"/>
 748 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fi.txt" format="snowball" enablePositionIncrements="true"/>
 749 |         <filter class="solr.SnowballPorterFilterFactory" language="Finnish"/>
 750 |         <!-- less aggressive: <filter class="solr.FinnishLightStemFilterFactory"/> -->
 751 |       </analyzer>
 752 |     </fieldType>
 753 |     
 754 |     <!-- French -->
 755 |     <fieldType name="text_fr" class="solr.TextField" positionIncrementGap="100">
 756 |       <analyzer> 
 757 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 758 |         <!-- removes l', etc -->
 759 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_fr.txt"/>
 760 |         <filter class="solr.LowerCaseFilterFactory"/>
 761 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_fr.txt" format="snowball" enablePositionIncrements="true"/>
 762 |         <filter class="solr.FrenchLightStemFilterFactory"/>
 763 |         <!-- less aggressive: <filter class="solr.FrenchMinimalStemFilterFactory"/> -->
 764 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="French"/> -->
 765 |       </analyzer>
 766 |     </fieldType>
 767 |     
 768 |     <!-- Irish -->
 769 |     <fieldType name="text_ga" class="solr.TextField" positionIncrementGap="100">
 770 |       <analyzer> 
 771 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 772 |         <!-- removes d', etc -->
 773 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_ga.txt"/>
 774 |         <!-- removes n-, etc. position increments is intentionally false! -->
 775 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/hyphenations_ga.txt" enablePositionIncrements="false"/>
 776 |         <filter class="solr.IrishLowerCaseFilterFactory"/>
 777 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ga.txt" enablePositionIncrements="true"/>
 778 |         <filter class="solr.SnowballPorterFilterFactory" language="Irish"/>
 779 |       </analyzer>
 780 |     </fieldType>
 781 |     
 782 |     <!-- Galician -->
 783 |     <fieldType name="text_gl" class="solr.TextField" positionIncrementGap="100">
 784 |       <analyzer> 
 785 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 786 |         <filter class="solr.LowerCaseFilterFactory"/>
 787 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_gl.txt" enablePositionIncrements="true"/>
 788 |         <filter class="solr.GalicianStemFilterFactory"/>
 789 |         <!-- less aggressive: <filter class="solr.GalicianMinimalStemFilterFactory"/> -->
 790 |       </analyzer>
 791 |     </fieldType>
 792 |     
 793 |     <!-- Hindi -->
 794 |     <fieldType name="text_hi" class="solr.TextField" positionIncrementGap="100">
 795 |       <analyzer> 
 796 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 797 |         <filter class="solr.LowerCaseFilterFactory"/>
 798 |         <!-- normalizes unicode representation -->
 799 |         <filter class="solr.IndicNormalizationFilterFactory"/>
 800 |         <!-- normalizes variation in spelling -->
 801 |         <filter class="solr.HindiNormalizationFilterFactory"/>
 802 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hi.txt" enablePositionIncrements="true"/>
 803 |         <filter class="solr.HindiStemFilterFactory"/>
 804 |       </analyzer>
 805 |     </fieldType>
 806 |     
 807 |     <!-- Hungarian -->
 808 |     <fieldType name="text_hu" class="solr.TextField" positionIncrementGap="100">
 809 |       <analyzer> 
 810 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 811 |         <filter class="solr.LowerCaseFilterFactory"/>
 812 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hu.txt" format="snowball" enablePositionIncrements="true"/>
 813 |         <filter class="solr.SnowballPorterFilterFactory" language="Hungarian"/>
 814 |         <!-- less aggressive: <filter class="solr.HungarianLightStemFilterFactory"/> -->   
 815 |       </analyzer>
 816 |     </fieldType>
 817 |     
 818 |     <!-- Armenian -->
 819 |     <fieldType name="text_hy" class="solr.TextField" positionIncrementGap="100">
 820 |       <analyzer> 
 821 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 822 |         <filter class="solr.LowerCaseFilterFactory"/>
 823 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_hy.txt" enablePositionIncrements="true"/>
 824 |         <filter class="solr.SnowballPorterFilterFactory" language="Armenian"/>
 825 |       </analyzer>
 826 |     </fieldType>
 827 |     
 828 |     <!-- Indonesian -->
 829 |     <fieldType name="text_id" class="solr.TextField" positionIncrementGap="100">
 830 |       <analyzer> 
 831 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 832 |         <filter class="solr.LowerCaseFilterFactory"/>
 833 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_id.txt" enablePositionIncrements="true"/>
 834 |         <!-- for a less aggressive approach (only inflectional suffixes), set stemDerivational to false -->
 835 |         <filter class="solr.IndonesianStemFilterFactory" stemDerivational="true"/>
 836 |       </analyzer>
 837 |     </fieldType>
 838 |     
 839 |     <!-- Italian -->
 840 |     <fieldType name="text_it" class="solr.TextField" positionIncrementGap="100">
 841 |       <analyzer> 
 842 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 843 |         <!-- removes l', etc -->
 844 |         <filter class="solr.ElisionFilterFactory" ignoreCase="true" articles="lang/contractions_it.txt"/>
 845 |         <filter class="solr.LowerCaseFilterFactory"/>
 846 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_it.txt" format="snowball" enablePositionIncrements="true"/>
 847 |         <filter class="solr.ItalianLightStemFilterFactory"/>
 848 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Italian"/> -->
 849 |       </analyzer>
 850 |     </fieldType>
 851 |     
 852 |     <!-- Japanese using morphological analysis (see text_cjk for a configuration using bigramming)
 853 | 
 854 |          NOTE: If you want to optimize search for precision, use default operator AND in your query
 855 |          parser config with <solrQueryParser defaultOperator="AND"/> further down in this file.  Use 
 856 |          OR if you would like to optimize for recall (default).
 857 |     -->
 858 |     <fieldType name="text_ja" class="solr.TextField" positionIncrementGap="100" autoGeneratePhraseQueries="false">
 859 |       <analyzer>
 860 |       <!-- Kuromoji Japanese morphological analyzer/tokenizer (JapaneseTokenizer)
 861 | 
 862 |            Kuromoji has a search mode (default) that does segmentation useful for search.  A heuristic
 863 |            is used to segment compounds into its parts and the compound itself is kept as synonym.
 864 | 
 865 |            Valid values for attribute mode are:
 866 |               normal: regular segmentation
 867 |               search: segmentation useful for search with synonyms compounds (default)
 868 |             extended: same as search mode, but unigrams unknown words (experimental)
 869 | 
 870 |            For some applications it might be good to use search mode for indexing and normal mode for
 871 |            queries to reduce recall and prevent parts of compounds from being matched and highlighted.
 872 |            Use <analyzer type="index"> and <analyzer type="query"> for this and mode normal in query.
 873 | 
 874 |            Kuromoji also has a convenient user dictionary feature that allows overriding the statistical
 875 |            model with your own entries for segmentation, part-of-speech tags and readings without a need
 876 |            to specify weights.  Notice that user dictionaries have not been subject to extensive testing.
 877 | 
 878 |            User dictionary attributes are:
 879 |                      userDictionary: user dictionary filename
 880 |              userDictionaryEncoding: user dictionary encoding (default is UTF-8)
 881 | 
 882 |            See lang/userdict_ja.txt for a sample user dictionary file.
 883 | 
 884 |            Punctuation characters are discarded by default.  Use discardPunctuation="false" to keep them.
 885 | 
 886 |            See http://wiki.apache.org/solr/JapaneseLanguageSupport for more on Japanese language support.
 887 |         -->
 888 |         <tokenizer class="solr.JapaneseTokenizerFactory" mode="search"/>
 889 |         <!--<tokenizer class="solr.JapaneseTokenizerFactory" mode="search" userDictionary="lang/userdict_ja.txt"/>-->
 890 |         <!-- Reduces inflected verbs and adjectives to their base/dictionary forms (辞書形) -->
 891 |         <filter class="solr.JapaneseBaseFormFilterFactory"/>
 892 |         <!-- Removes tokens with certain part-of-speech tags -->
 893 |         <filter class="solr.JapanesePartOfSpeechStopFilterFactory" tags="lang/stoptags_ja.txt" enablePositionIncrements="true"/>
 894 |         <!-- Normalizes full-width romaji to half-width and half-width kana to full-width (Unicode NFKC subset) -->
 895 |         <filter class="solr.CJKWidthFilterFactory"/>
 896 |         <!-- Removes common tokens typically not useful for search, but have a negative effect on ranking -->
 897 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ja.txt" enablePositionIncrements="true" />
 898 |         <!-- Normalizes common katakana spelling variations by removing any last long sound character (U+30FC) -->
 899 |         <filter class="solr.JapaneseKatakanaStemFilterFactory" minimumLength="4"/>
 900 |         <!-- Lower-cases romaji characters -->
 901 |         <filter class="solr.LowerCaseFilterFactory"/>
 902 |       </analyzer>
 903 |     </fieldType>
 904 |     
 905 |     <!-- Latvian -->
 906 |     <fieldType name="text_lv" class="solr.TextField" positionIncrementGap="100">
 907 |       <analyzer> 
 908 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 909 |         <filter class="solr.LowerCaseFilterFactory"/>
 910 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_lv.txt" enablePositionIncrements="true"/>
 911 |         <filter class="solr.LatvianStemFilterFactory"/>
 912 |       </analyzer>
 913 |     </fieldType>
 914 |     
 915 |     <!-- Dutch -->
 916 |     <fieldType name="text_nl" class="solr.TextField" positionIncrementGap="100">
 917 |       <analyzer> 
 918 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 919 |         <filter class="solr.LowerCaseFilterFactory"/>
 920 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_nl.txt" format="snowball" enablePositionIncrements="true"/>
 921 |         <filter class="solr.StemmerOverrideFilterFactory" dictionary="lang/stemdict_nl.txt" ignoreCase="false"/>
 922 |         <filter class="solr.SnowballPorterFilterFactory" language="Dutch"/>
 923 |       </analyzer>
 924 |     </fieldType>
 925 |     
 926 |     <!-- Norwegian -->
 927 |     <fieldType name="text_no" class="solr.TextField" positionIncrementGap="100">
 928 |       <analyzer> 
 929 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 930 |         <filter class="solr.LowerCaseFilterFactory"/>
 931 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_no.txt" format="snowball" enablePositionIncrements="true"/>
 932 |         <filter class="solr.SnowballPorterFilterFactory" language="Norwegian"/>
 933 |         <!-- less aggressive: <filter class="solr.NorwegianLightStemFilterFactory"/> -->
 934 |         <!-- singular/plural: <filter class="solr.NorwegianMinimalStemFilterFactory"/> -->
 935 |       </analyzer>
 936 |     </fieldType>
 937 |     
 938 |     <!-- Portuguese -->
 939 |     <fieldType name="text_pt" class="solr.TextField" positionIncrementGap="100">
 940 |       <analyzer> 
 941 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 942 |         <filter class="solr.LowerCaseFilterFactory"/>
 943 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_pt.txt" format="snowball" enablePositionIncrements="true"/>
 944 |         <filter class="solr.PortugueseLightStemFilterFactory"/>
 945 |         <!-- less aggressive: <filter class="solr.PortugueseMinimalStemFilterFactory"/> -->
 946 |         <!-- more aggressive: <filter class="solr.SnowballPorterFilterFactory" language="Portuguese"/> -->
 947 |         <!-- most aggressive: <filter class="solr.PortugueseStemFilterFactory"/> -->
 948 |       </analyzer>
 949 |     </fieldType>
 950 |     
 951 |     <!-- Romanian -->
 952 |     <fieldType name="text_ro" class="solr.TextField" positionIncrementGap="100">
 953 |       <analyzer> 
 954 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 955 |         <filter class="solr.LowerCaseFilterFactory"/>
 956 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ro.txt" enablePositionIncrements="true"/>
 957 |         <filter class="solr.SnowballPorterFilterFactory" language="Romanian"/>
 958 |       </analyzer>
 959 |     </fieldType>
 960 |     
 961 |     <!-- Russian -->
 962 |     <fieldType name="text_ru" class="solr.TextField" positionIncrementGap="100">
 963 |       <analyzer> 
 964 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 965 |         <filter class="solr.LowerCaseFilterFactory"/>
 966 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_ru.txt" format="snowball" enablePositionIncrements="true"/>
 967 |         <filter class="solr.SnowballPorterFilterFactory" language="Russian"/>
 968 |         <!-- less aggressive: <filter class="solr.RussianLightStemFilterFactory"/> -->
 969 |       </analyzer>
 970 |     </fieldType>
 971 |     
 972 |     <!-- Swedish -->
 973 |     <fieldType name="text_sv" class="solr.TextField" positionIncrementGap="100">
 974 |       <analyzer> 
 975 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 976 |         <filter class="solr.LowerCaseFilterFactory"/>
 977 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_sv.txt" format="snowball" enablePositionIncrements="true"/>
 978 |         <filter class="solr.SnowballPorterFilterFactory" language="Swedish"/>
 979 |         <!-- less aggressive: <filter class="solr.SwedishLightStemFilterFactory"/> -->
 980 |       </analyzer>
 981 |     </fieldType>
 982 |     
 983 |     <!-- Thai -->
 984 |     <fieldType name="text_th" class="solr.TextField" positionIncrementGap="100">
 985 |       <analyzer> 
 986 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 987 |         <filter class="solr.LowerCaseFilterFactory"/>
 988 |         <filter class="solr.ThaiWordFilterFactory"/>
 989 |         <filter class="solr.StopFilterFactory" ignoreCase="true" words="lang/stopwords_th.txt" enablePositionIncrements="true"/>
 990 |       </analyzer>
 991 |     </fieldType>
 992 |     
 993 |     <!-- Turkish -->
 994 |     <fieldType name="text_tr" class="solr.TextField" positionIncrementGap="100">
 995 |       <analyzer> 
 996 |         <tokenizer class="solr.StandardTokenizerFactory"/>
 997 |         <filter class="solr.TurkishLowerCaseFilterFactory"/>
 998 |         <filter class="solr.StopFilterFactory" ignoreCase="false" words="lang/stopwords_tr.txt" enablePositionIncrements="true"/>
 999 |         <filter class="solr.SnowballPorterFilterFactory" language="Turkish"/>
1000 |       </analyzer>
1001 |     </fieldType>
1002 | 
1003 |  </types>
1004 |   
1005 |   <!-- Similarity is the scoring routine for each document vs. a query.
1006 |        A custom Similarity or SimilarityFactory may be specified here, but 
1007 |        the default is fine for most applications.  
1008 |        For more info: http://wiki.apache.org/solr/SchemaXml#Similarity
1009 |     -->
1010 |   <!--
1011 |      <similarity class="com.example.solr.CustomSimilarityFactory">
1012 |        <str name="paramkey">param value</str>
1013 |      </similarity>
1014 |     -->
1015 | 
1016 | </schema>
1017 | 


--------------------------------------------------------------------------------
/ocr/IdmpExtraction.scala:
--------------------------------------------------------------------------------
  1 | / ** 
  2 |  * @author Jeff Shmain
  3 |  * @author Vartika Singh
  4 |  * /
  5 | 
  6 | package com.cloudera.sa.OCR
  7 | 
  8 | import org.bytedeco.javacpp.lept._;
  9 | import org.bytedeco.javacpp.tesseract._;
 10 | import java.io.ByteArrayOutputStream;
 11 | import java.nio.ByteBuffer;
 12 | import java.nio.ByteBuffer
 13 | import java.awt.Image
 14 | import java.awt.image.RenderedImage;
 15 | import java.io.File;
 16 | import scala.collection.mutable.StringBuilder
 17 | import collection.JavaConversions._
 18 | import java.io.IOException;
 19 | import java.util.List;
 20 | import javax.imageio.ImageIO;
 21 | import org.ghost4j.analyzer.AnalysisItem;
 22 | import org.ghost4j.analyzer.FontAnalyzer;
 23 | import org.ghost4j.document.PDFDocument;
 24 | import org.ghost4j.renderer.SimpleRenderer;
 25 | import org.bytedeco.javacpp._;
 26 | import java.io._
 27 | import org.apache.spark.{Logging, SerializableWritable, SparkConf, SparkContext}
 28 | import org.apache.hadoop.hbase.HBaseConfiguration
 29 | import org.apache.hadoop.hbase.client.Connection
 30 | import org.apache.hadoop.hbase.client.ConnectionFactory
 31 | import org.apache.hadoop.hbase.TableName
 32 | import org.apache.hadoop.hbase.client.Put
 33 | import org.apache.hadoop.hbase.client.Get
 34 | import org.apache.hadoop.hbase.util.Bytes
 35 | 
 36 | object IdmpExtraction {
 37 |   def main(args: Array[String]) {
 38 | 
 39 | 
 40 |     val conf = new SparkConf().setAppName("IDMP Processor")
 41 |     val sc = new SparkContext(conf)
 42 | 
 43 |     /** Read in PDFs into the RDD */
 44 |     val files = sc.binaryFiles ("hdfs://nameservice1/data/raw")
 45 |     files.map(convertFunc(_)).count
 46 |   }
 47 | 
 48 | 
 49 |   /** Populate the HBase table
 50 |     *	@param fileName This corresponds to the rowID in HBase
 51 |     * 	@param lines The parsed output.
 52 |     *	dataformat: binaryPDF:PortableDataStream
 53 |     */
 54 |   def populateHbase (
 55 |                       fileName:String,
 56 |                       lines: String,
 57 |                       pdf:org.apache.spark.input.PortableDataStream) : Unit =
 58 |   {
 59 |     /** Configure and open a HBase connection */
 60 |     val conf = HBaseConfiguration.create()
 61 |     val conn=  ConnectionFactory.createConnection( conf );
 62 |     val mddsTbl = conn.getTable( TableName.valueOf( "mdds" ));
 63 |     val cf = "info"
 64 |     val put = new Put( Bytes.toBytes( fileName ))
 65 | 
 66 |     /**
 67 |      *	Extract Fields here using Regexes
 68 |      *	Create Put objects and send to hbase
 69 |      */
 70 |     val aAndCP = """(?s)(?m).*\d\d\d\d\d-\d\d\d\d(.*)\nRe: (\w\d\d\d\d\d\d).*""".r
 71 |     val approvedP = """(?s)(?m).*(You may, therefore, market the device, subject to the general controls provisions of the Act).*""".r
 72 | 
 73 |     lines match {
 74 |       case
 75 |         aAndCP( addr, casenum ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "submitter_info" ), Bytes.toBytes( addr ) ).add( Bytes.toBytes( cf ), Bytes.toBytes( "case_num" ), Bytes.toBytes( casenum ))
 76 |       case _ => println( "did not match a regex" )
 77 |     }
 78 | 
 79 |     lines match {
 80 |       case
 81 |         approvedP( approved ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "approved" ), Bytes.toBytes( "yes" ))
 82 |       case _ => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "approved" ), Bytes.toBytes( "no" ))
 83 |     }
 84 | 
 85 |     lines.split("\n").foreach {
 86 | 
 87 |       val regNumRegex = """Regulation Number:\s+(.+)""".r
 88 |       val regNameRegex = """Regulation Name:\s+(.+)""".r
 89 |       val regClassRegex = """Regulatory Class:\s+(.+)""".r
 90 |       val productCodeRegex = """Product Code:\s+(.+)""".r
 91 |       val datedRegex = """Dated:\s+(\w{3,10}\s+\d{1,2},\s+\d{4}).*""".r
 92 |       val receivedRegex = """Received:\s+(\w{3,10}\s+\d{1,2},\s+\d{4}).*""".r
 93 |       val deviceNameRegex = """Trade/Device Name:\s+(.+)""".r
 94 | 
 95 |       _ match {
 96 |         case regNumRegex( regNum ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "reg_num" ), Bytes.toBytes( regNum ))
 97 |         case regNameRegex(regName) => put.add(Bytes.toBytes( cf ), Bytes.toBytes( "reg_name" ), Bytes.toBytes( regName ))
 98 |         case regClassRegex( regClass ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "reg_class" ), Bytes.toBytes( regClass ))
 99 |         case productCodeRegex( productCode ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "product_code" ), Bytes.toBytes( productCode ))
100 |         case datedRegex( dated ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "dated" ), Bytes.toBytes( dated ))
101 |         case receivedRegex( received ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "received" ), Bytes.toBytes( received ))
102 |         case deviceNameRegex( deviceName ) => put.add( Bytes.toBytes( cf ), Bytes.toBytes( "device_name" ), Bytes.toBytes( deviceName ))
103 | 
104 |         case _ => print( "" )
105 |       }
106 |     }
107 |     put.add( Bytes.toBytes( cf ), Bytes.toBytes( "text" ), Bytes.toBytes( lines ))
108 |     val pdfBytes = pdf.toArray.clone
109 |     put.add(Bytes.toBytes( "obj" ), Bytes.toBytes( "pdf" ), pdfBytes )
110 | 
111 |     mddsTbl.put( put )
112 |     mddsTbl.close
113 |     conn.close
114 |   }
115 | 
116 |   /** Method to convert a PDF document to images and hence OCR
117 |     * @param PDF File(s) to process
118 |     */
119 |   def convertFunc (
120 |                     file: (String, org.apache.spark.input.PortableDataStream)
121 |                     ) : Unit  =
122 |   {
123 |     /** Render the PDF into a list of images with 300 dpi resolution
124 |       * One image per PDF page, a PDF document may have multiple pages
125 |       */
126 |     val document: PDFDocument = new PDFDocument( );
127 |     document.load( file._2.open )
128 |     file._2.close
129 |     val renderer :SimpleRenderer = new SimpleRenderer( )
130 |     renderer.setResolution( 300 )
131 |     val images:List[Image] = renderer.render( document )
132 | 
133 |     /**  Iterate through the image list and extract OCR
134 |       * using Tesseract API.
135 |       */
136 |     var r:StringBuilder = new StringBuilder
137 |     images.toList.foreach{ x=>
138 |       val imageByteStream = new ByteArrayOutputStream( )
139 |       ImageIO.write(
140 |         x.asInstanceOf[RenderedImage], "png", imageByteStream )
141 |       val pix: PIX = pixReadMem(
142 |         ByteBuffer.wrap( imageByteStream.toByteArray( ) ).array( ),
143 |         ByteBuffer.wrap( imageByteStream.toByteArray( ) ).capacity( )
144 |       )
145 |       val api: TessBaseAPI = new TessBaseAPI( )
146 |       /** We assume the documents are in English here, hence \”eng\” */
147 |       api.Init( null, "eng" )
148 |       api.SetImage(pix)
149 |       r.append(api.GetUTF8Text().getString())
150 |       imageByteStream.close
151 |       pixDestroy(pix)
152 |       api.End
153 |     }
154 | 
155 |     /** Write the generated data into HBase */
156 |     populateHbase( file._1, r.toString( ), file._2 )
157 |   }
158 | }
159 | 


--------------------------------------------------------------------------------
/ocr/indexer-config.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | <indexer table="mdds"  mapper="com.ngdata.hbaseindexer.morphline.MorphlineResultToSolrMapper" mapping-type="row" >
 3 |  
 4 |   <!-- The relative or absolute path on the local file system to the morphline configuration file. -->
 5 |   <!-- Use relative path "morphlines.conf" for morphlines managed by Cloudera Manager -->
 6 |   <param name="morphlineFile" value="morphlines.conf"/>
 7 |  
 8 |   <!-- The optional morphlineId identifies a morphline if there are multiple morphlines in morphlines.conf -->
 9 |   <!-- <param name="morphlineId" value="morphline1"/> -->
10 |  
11 | </indexer>
12 | 


--------------------------------------------------------------------------------
/ocr/morphlines.conf:
--------------------------------------------------------------------------------
  1 | SOLR_LOCATOR : {
  2 |  # Name of solr collection
  3 |  collection : mdds_collection
  4 |  
  5 |  # ZooKeeper ensemble
  6 |  zkHost : "$ZK_HOST"
  7 | }
  8 |  
  9 | morphlines : [
 10 | {
 11 | id : morphline
 12 | importCommands : ["org.kitesdk.**", "com.ngdata.**"]
 13 |  
 14 | commands : [                    
 15 |  {
 16 |    extractHBaseCells {
 17 |      mappings : [
 18 |        {
 19 |          inputColumn : "info:submitter_info"
 20 |          outputField : "submitter_info"
 21 |          type : string
 22 |          source : value
 23 |        }
 24 |        {
 25 |          inputColumn : "info:case_num"
 26 |          outputField : "case_num"
 27 |          type : string
 28 |          source : value
 29 |        }
 30 |        {
 31 |          inputColumn : "info:device_name"
 32 |          outputField : "device_name"
 33 |          type : string
 34 |          source : value
 35 |        }
 36 |        {
 37 |          inputColumn : "info:reg_num"
 38 |          outputField : "reg_num"
 39 |          type : string
 40 |          source : value
 41 |        }
 42 |        {
 43 |          inputColumn : "info:reg_name"
 44 |          outputField : "reg_name"
 45 |          type : string
 46 |          source : value
 47 |        }
 48 |        {
 49 |          inputColumn : "info:reg_class"
 50 |          outputField : "reg_class"
 51 |          type : string
 52 |          source : value
 53 |        }
 54 |        {
 55 |          inputColumn : "info:product_code"
 56 |          outputField : "product_code"
 57 |          type : string
 58 |          source : value
 59 |        }
 60 |        {
 61 |          inputColumn : "info:dated"
 62 |          outputField : "dated"
 63 |          type : string
 64 |          source : value
 65 |        }
 66 |        {
 67 |          inputColumn : "info:received"
 68 |          outputField : "received"
 69 |          type : string
 70 |          source : value
 71 |        }
 72 |        {
 73 |          inputColumn : "info:approved"
 74 |          outputField : "approved"
 75 |          type : string
 76 |          source : value
 77 |        }
 78 |        {
 79 |          inputColumn : "info:text"
 80 |          outputField : "text"
 81 |          type : string
 82 |          source : value
 83 |        }
 84 |      ]
 85 |    }
 86 |  }
 87 |  
 88 |    {
 89 |        convertTimestamp {
 90 |            field : dated
 91 |            inputFormats : ["MMMM d, yyyy", "yyyy-MM-dd","MMMM d,yyyy"]
 92 |            inputTimezone : America/Los_Angeles
 93 |            outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
 94 |            outputTimezone : UTC
 95 |        }
 96 |    }
 97 |    {
 98 |        convertTimestamp {
 99 |            field : received
100 |            inputFormats : ["MMMM d, yyyy", "yyyy-MM-dd","MMMM d,yyyy"]
101 |            inputTimezone : America/Los_Angeles
102 |            outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
103 |            outputTimezone : UTC
104 |        }
105 |    }
106 |    # Consume the output record of the previous command and pipe another
107 |    # record downstream.
108 |    #
109 |    # This command sanitizes record fields that are unknown to Solr schema.xml
110 |    # by deleting them. Recall that Solr throws an exception on any attempt to
111 |    # load a document that contains a field that isn't specified in schema.xml
112 |    {
113 |        sanitizeUnknownSolrFields {
114 |            # Location from which to fetch Solr schema
115 |            solrLocator : ${SOLR_LOCATOR}
116 |        }
117 |    }
118 | # { logDebug { format : "output record: {}", args : ["@{}"] } }
119 | ]
120 | }
121 | ]
122 | 


--------------------------------------------------------------------------------