45 | @@ -50,7 +39,6 @@
46 |
51 | `
52 | // If you want to see the full request going to Elasticsearch, you can include
53 | // the following snippet in the renderer above:
54 | //
55 |
56 | // The parameter "q" should be included in this pagination
57 | prototype(Flowpack.Listable:PaginationParameters) {
58 | q = ${request.arguments.q}
59 | nodeTypes = ${request.arguments.nodeTypesFilter}
60 | }
61 |
62 | // We configure the cache mode "dynamic" here.
63 | @cache {
64 | mode = 'dynamic'
65 | entryIdentifier {
66 | node = ${node}
67 | type = 'searchForm'
68 | }
69 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
70 | context {
71 | 1 = 'node'
72 | 2 = 'documentNode'
73 | 3 = 'site'
74 | }
75 | entryTags {
76 | 1 = ${Neos.Caching.nodeTag(node)}
77 | }
78 | }
79 | }
80 |
81 | // The result display is done here.
82 | // In the context, you'll find an object `searchResultDocument` which is of type
83 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
84 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
85 | faqEntries {
86 | condition = ${searchResultDocument.property('index_discriminator') == 'faq'}
87 | renderer = afx`
88 | {searchResultDocument.properties.faqEntryTitle}
89 | `
90 | }
91 | neosNodes {
92 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
93 | // This is in preparation for displaying other kinds of data.
94 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
95 | renderer.@context.node = ${searchResultDocument.loadNode()}
96 | renderer = afx`
97 |
98 | `
99 | // If you want to see the full Search Response hit, you can include the following
100 | // snippet in the renderer above:
101 | //
102 | }
103 | }
104 |
--------------------------------------------------------------------------------
/Documentation/02_FacetedSearchTemplate.fusion:
--------------------------------------------------------------------------------
1 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
2 | // this is the base query from the user which should *always* be applied.
3 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
4 |
5 | // register a Terms aggregation with the URL parameter "nodeTypesFilter"
6 | _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
7 |
8 | // This is the main elasticsearch query which determines the search results:
9 | // - this._elasticsearchBaseQuery is applied
10 | // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
11 | // <-- if you add additional aggregations, you need to add them here to this list.
12 | @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
13 |
14 | // The Request is for displaying the Node Types aggregation (faceted search).
15 | //
16 | // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
17 | // This means, for the `.aggregation()` part, we take the aggregation itself.
18 | // For the `.filter()` part, we add:
19 | // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
20 | // <-- if you add additional aggregations, you need to add them here to the list.
21 | @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
22 |
23 | // Search Result Display is controlled through Flowpack.Listable
24 | searchResults = Flowpack.Listable:PaginatedCollection {
25 | collection = ${mainSearchRequest}
26 | itemsPerPage = 12
27 |
28 | // we use cache mode "dynamic" for the full Search component; so we do not need an additional cache entry
29 | // for the PaginatedCollection.
30 | @cache.mode = "embed"
31 | }
32 |
33 | nodeTypesFacet = Neos.Fusion:Component {
34 | // the nodeTypesFacet is a "Terms" aggregation...
35 | // ...so we can access nodeTypesFacet.buckets.
36 | // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
37 | // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
38 | // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
39 | renderer = afx`
40 |
41 |
42 | - {bucket.key} {bucket.doc_count} (selected)
43 |
44 |
45 | CLEAR FACET
46 | `
47 | }
48 |
49 | renderer = afx`
50 |
63 | `
64 | // If you want to see the full request going to Elasticsearch, you can include
65 | // the following snippet in the renderer above:
66 | //
67 |
68 | // The parameter "q" should be included in this pagination
69 | prototype(Flowpack.Listable:PaginationParameters) {
70 | q = ${request.arguments.q}
71 | // <-- if you add additional aggregations, you need to add the parameter names here
72 | nodeTypesFilter = ${request.arguments.nodeTypesFilter}
73 | }
74 |
75 | // We configure the cache mode "dynamic" here.
76 | @cache {
77 | mode = 'dynamic'
78 | entryIdentifier {
79 | node = ${node}
80 | type = 'searchForm'
81 | }
82 | // <-- if you add additional aggregations, you need to add the parameter names to the entryDiscriminator
83 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
84 | context {
85 | 1 = 'node'
86 | 2 = 'documentNode'
87 | 3 = 'site'
88 | }
89 | entryTags {
90 | 1 = ${Neos.Caching.nodeTag(node)}
91 | }
92 | }
93 | }
94 |
95 | // The result display is done here.
96 | // In the context, you'll find an object `searchResultDocument` which is of type
97 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
98 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
99 | neosNodes {
100 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
101 | // This is in preparation for displaying other kinds of data.
102 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
103 | renderer.@context.node = ${searchResultDocument.loadNode()}
104 | renderer = afx`
105 |
106 | `
107 | // If you want to see the full Search Response hit, you can include the following
108 | // snippet in the renderer above:
109 | //
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/Documentation/02a_FacetedHighlightedSearchTemplate.fusion:
--------------------------------------------------------------------------------
1 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
2 | // this is the base query from the user which should *always* be applied.
3 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
4 |
5 | // register a Terms aggregation with the URL parameter "nodeTypesFilter"
6 | _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
7 |
8 | // This is the main elasticsearch query which determines the search results:
9 | // - this._elasticsearchBaseQuery is applied
10 | // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
11 | // <-- if you add additional aggregations, you need to add them here to this list.
12 | @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation)).highlight(Elasticsearch.createNeosFulltextHighlight())}
13 |
14 | // The Request is for displaying the Node Types aggregation (faceted search).
15 | //
16 | // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
17 | // This means, for the `.aggregation()` part, we take the aggregation itself.
18 | // For the `.filter()` part, we add:
19 | // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
20 | // <-- if you add additional aggregations, you need to add them here to the list.
21 | @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
22 |
23 | // Search Result Display is controlled through Flowpack.Listable
24 | searchResults = Flowpack.Listable:PaginatedCollection {
25 | collection = ${mainSearchRequest}
26 | itemsPerPage = 12
27 |
28 | // we use cache mode "dynamic" for the full Search component; so we do not need an additional cache entry
29 | // for the PaginatedCollection.
30 | @cache.mode = "embed"
31 | }
32 |
33 | nodeTypesFacet = Neos.Fusion:Component {
34 | // the nodeTypesFacet is a "Terms" aggregation...
35 | // ...so we can access nodeTypesFacet.buckets.
36 | // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
37 | // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
38 | // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
39 | renderer = afx`
40 |
41 |
42 | - {bucket.key} {bucket.doc_count} (selected)
43 |
44 |
45 | CLEAR FACET
46 | `
47 | }
48 |
49 | renderer = afx`
50 |
63 | `
64 | // If you want to see the full request going to Elasticsearch, you can include
65 | // the following snippet in the renderer above:
66 | //
67 |
68 | // The parameter "q" should be included in this pagination
69 | prototype(Flowpack.Listable:PaginationParameters) {
70 | q = ${request.arguments.q}
71 | // <-- if you add additional aggregations, you need to add the parameter names here
72 | nodeTypesFilter = ${request.arguments.nodeTypesFilter}
73 | }
74 |
75 | // We configure the cache mode "dynamic" here.
76 | @cache {
77 | mode = 'dynamic'
78 | entryIdentifier {
79 | node = ${node}
80 | type = 'searchForm'
81 | }
82 | // <-- if you add additional aggregations, you need to add the parameter names to the entryDiscriminator
83 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
84 | context {
85 | 1 = 'node'
86 | 2 = 'documentNode'
87 | 3 = 'site'
88 | }
89 | entryTags {
90 | 1 = ${Neos.Caching.nodeTag(node)}
91 | }
92 | }
93 | }
94 |
95 | // The result display is done here.
96 | // In the context, you'll find an object `searchResultDocument` which is of type
97 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
98 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
99 | neosNodes {
100 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
101 | // This is in preparation for displaying other kinds of data.
102 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
103 | renderer.@context.node = ${searchResultDocument.loadNode()}
104 | renderer = afx`
105 |
106 | {Array.join(searchResultDocument.processedHighlights, '…')}
107 |
108 | `
109 | // If you want to see the full Search Response hit, you can include the following
110 | // snippet in the renderer above:
111 | //
112 | }
113 | }
114 |
--------------------------------------------------------------------------------
/Classes/CustomIndexing/CustomIndexer.php:
--------------------------------------------------------------------------------
1 | createIndexWithMapping([
19 | * 'properties' => [
20 | * 'faqEntry' => ['type' => 'text']
21 | * ]
22 | * ]);
23 | * $indexer->index([
24 | * 'faqEntryTitle' => 'FAQ Dresden'
25 | * ]);
26 | * $indexer->finalizeAndSwitchAlias();
27 | *
28 | *
29 | *
30 | * USAGE - CLEANING OLD INDICES
31 | *
32 | * $indexer = CustomIndexer::create('faq');
33 | * $indexer->removeObsoleteIndices();
34 | */
35 | class CustomIndexer
36 | {
37 | /**
38 | * @Flow\Inject
39 | * @var ClientFactory
40 | */
41 | protected $clientFactory;
42 |
43 | /**
44 | * @Flow\Inject
45 | * @var IndexAliasManager
46 | */
47 | protected $indexAliasManager;
48 |
49 | /**
50 | * @Flow\Inject
51 | * @var IndexDriverInterface
52 | */
53 | protected $indexDriver;
54 |
55 |
56 | private \Flowpack\ElasticSearch\Domain\Model\Client $elasticsearchClient;
57 |
58 | protected string $aliasName;
59 | protected string $discriminatorValue;
60 | protected string $indexName;
61 | protected ?Index $index;
62 |
63 | protected int $bulkSize = 100;
64 | protected array $currentBulkRequest = [];
65 |
66 |
67 | /**
68 | * Create a custom indexer with the given $aliasName as index alias (i.e. what you specify in the 2nd argument
69 | * of `Elasticsearch.createRequest(site, ['myAlias'])` in Eel).
70 | *
71 | * The given $discriminatorValue is used as value of the `index_discriminator` key in every indexed document;
72 | * and can be used to distinguish different document types inside a query.
73 | *
74 | * If no $discriminatorValue is specified, the $aliasName is used by default.
75 | *
76 | * @param string $aliasName name of the Elasticsearch alias to create/update when indexing is completed
77 | * @param string|null $discriminatorValue value of the index_discriminator field of all documents indexed by this indexer. If null, $aliasName is used.
78 | * @return CustomIndexer
79 | */
80 | public static function create(string $aliasName, string $discriminatorValue = null): CustomIndexer
81 | {
82 | if ($discriminatorValue === null) {
83 | $discriminatorValue = $aliasName;
84 | }
85 |
86 | return new CustomIndexer($aliasName, $discriminatorValue);
87 | }
88 |
89 | protected function __construct(string $aliasName, string $discriminatorValue)
90 | {
91 | $this->aliasName = $aliasName;
92 | $this->discriminatorValue = $discriminatorValue;
93 | $this->indexName = $aliasName . '-' . time();
94 | }
95 |
96 | public function initializeObject()
97 | {
98 | $this->elasticsearchClient = $this->clientFactory->create();
99 | $this->index = new Index($this->indexName, $this->elasticsearchClient);
100 | }
101 |
102 | /**
103 | * Create a new index with the given Elasticsearch mapping.
104 | *
105 | * @param array $fullMapping
106 | * @throws \Flowpack\ElasticSearch\Exception
107 | * @throws \Neos\Flow\Http\Exception
108 | */
109 | public function createIndexWithMapping(array $fullMapping): void
110 | {
111 | $this->index->create();
112 | $mapping = new Mapping(new GenericType($this->index, $this->discriminatorValue));
113 |
114 | // enforce correct type of index_discriminator
115 | $fullMapping['properties']['index_discriminator'] = [
116 | 'type' => 'keyword'
117 | ];
118 |
119 | $mapping->setFullMapping($fullMapping);
120 | $mapping->apply();
121 | }
122 |
123 | /**
124 | * Determines after how many calls to index() the request is sent to Elasticsearch
125 | * @param int $bulkSize
126 | */
127 | public function setBulkSize(int $bulkSize): void
128 | {
129 | $this->bulkSize = $bulkSize;
130 | }
131 |
132 | /**
133 | * Index a document, optionally also specifying the document ID.
134 | *
135 | * @param array $documentProperties
136 | * @param string|null $documentId
137 | */
138 | public function index(array $documentProperties, ?string $documentId = null): void
139 | {
140 | if ($documentId !== null) {
141 | $this->currentBulkRequest[] = [
142 | 'index' => [
143 | '_id' => $documentId,
144 | ]
145 | ];
146 | } else {
147 | $this->currentBulkRequest[] = [
148 | 'index' => new \stdClass()
149 | ];
150 | }
151 |
152 | $documentProperties['index_discriminator'] = $this->discriminatorValue;
153 | $this->currentBulkRequest[] = $documentProperties;
154 |
155 | // for every request, we have two rows in $this->currentBulkRequest
156 | if (count($this->currentBulkRequest) / 2 >= $this->bulkSize) {
157 | $this->sendCurrentBulkRequest();
158 | }
159 | }
160 |
161 | protected function sendCurrentBulkRequest(): void
162 | {
163 | if (count($this->currentBulkRequest) > 0) {
164 | // Bulk request MUST end with line return
165 | $request = implode("\n", array_map(fn($requestPart) => json_encode($requestPart), $this->currentBulkRequest)) . "\n";
166 |
167 | $this->index->request('POST', '/_bulk', [], $request);
168 | }
169 |
170 | $this->currentBulkRequest = [];
171 | }
172 |
173 | /**
174 | * Send the last bulk request to ensure indexing is completed; and then switch the index alias, so that
175 | * documents can be found.
176 | *
177 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception
178 | * @throws \Flowpack\ElasticSearch\Exception
179 | * @throws \Flowpack\ElasticSearch\Transfer\Exception\ApiException
180 | */
181 | public function finalizeAndSwitchAlias(): void
182 | {
183 | $this->sendCurrentBulkRequest();
184 | $this->indexAliasManager->updateIndexAlias($this->aliasName, $this->indexName);
185 | }
186 |
187 | /**
188 | * Remove old indices which are not active anymore (remember, each bulk index creates a new index from scratch,
189 | * making the "old" index a stale one).
190 | *
191 | * @return array a list of index names which were removed
192 | * @throws \Flowpack\ElasticSearch\Transfer\Exception
193 | * @throws \Flowpack\ElasticSearch\Transfer\Exception\ApiException
194 | * @throws \Neos\Flow\Http\Exception
195 | */
196 | public function removeObsoleteIndices(): array
197 | {
198 | $currentlyLiveIndices = $this->indexDriver->getIndexNamesByAlias($this->aliasName);
199 |
200 | $indexStatus = $this->elasticsearchClient->request('GET', '/_stats')->getTreatedContent();
201 | $allIndices = array_keys($indexStatus['indices']);
202 |
203 | $indicesToBeRemoved = [];
204 |
205 | foreach ($allIndices as $indexName) {
206 | if (strpos($indexName, $this->aliasName . '-') !== 0) {
207 | // filter out all indices not starting with the alias-name, as they are unrelated to our application
208 | continue;
209 | }
210 |
211 | if (in_array($indexName, $currentlyLiveIndices, true)) {
212 | // skip the currently live index names from deletion
213 | continue;
214 | }
215 |
216 | $indicesToBeRemoved[] = $indexName;
217 | }
218 |
219 | array_map(function ($index) {
220 | $this->indexDriver->deleteIndex($index);
221 | }, $indicesToBeRemoved);
222 |
223 | return $indicesToBeRemoved;
224 | }
225 | }
226 |
--------------------------------------------------------------------------------
/Documentation/README_template.md:
--------------------------------------------------------------------------------
1 | # Sandstorm.LightweightElasticsearch
2 |
3 | ...this is an attempt for a more lightweight elasticsearch integration for Neos CMS. This is built because I wanted
4 | to try out some different design decisions in parts of the Neos <-> Elasticsearch integration.
5 |
6 | This is a wrapper around Flowpack.Elasticsearch.ContentRepositoryAdaptor, which replaces parts of its API.
7 | A huge thanks goes to everybody maintaining Flowpack.Elasticsearch.ContentRepositoryAdaptor and Flowpack.Elasticsearch,
8 | as we build upon this work and profit from it greatly.
9 |
10 |
11 |
12 | - [Sandstorm.LightweightElasticsearch](#sandstormlightweightelasticsearch)
13 | - [Goals and Limitations](#goals-and-limitations)
14 | - [Starting Elasticsearch for development](#starting-elasticsearch-for-development)
15 | - [Indexing](#indexing)
16 | - [Configurations per property index field](#configurations-per-property-index-field)
17 | - [Exclude NodeTypes from indexing](#exclude-nodetypes-from-indexing)
18 | - [Indexing configuration per data type](#indexing-configuration-per-data-type)
19 | - [Indexing configuration per property](#indexing-configuration-per-property)
20 | - [Skip indexing and mapping of a property](#skip-indexing-and-mapping-of-a-property)
21 | - [Fulltext Indexing](#fulltext-indexing)
22 | - [Working with Dates](#working-with-dates)
23 | - [Working with Assets / Attachments](#working-with-assets--attachments)
24 | - [Search Component](#search-component)
25 | - [Query API](#query-api)
26 | - [Aggregations and Faceting](#aggregations-and-faceting)
27 | - [Result Highlighting](#result-highlighting)
28 | - [Indexing other data](#indexing-other-data)
29 | - [Querying other data](#querying-other-data)
30 | - [Debugging Elasticsearch queries](#debugging-elasticsearch-queries)
31 | - [Developing](#developing)
32 | - [Changing this readme](#changing-this-readme)
33 | - [License](#license)
34 |
35 |
36 |
37 | ## Goals and Limitations
38 |
39 | The project has the following goals and limitations:
40 |
41 | - **Only for fulltext search**
42 |
43 | This means only document nodes or anything which can potentially appear in fulltext search results is put into
44 | the Elasticsearch index (everything marked in the NodeTypes as `search.fulltext.isRoot = TRUE`).
45 | That means (by default) no content nodes or ContentCollections are stored inside the index.
46 |
47 | - **Easier Fulltext indexing implementation**
48 |
49 | Fulltext collection is done in PHP instead of inside Elasticsearch with Painless.
50 |
51 | - **Query Results not specific to Neos**
52 |
53 | You can easily write queries which target anything stored in Elasticsearch; and not just Neos Nodes.
54 | We provide examples and utilities how other data sources can be indexed in Elasticsearch.
55 |
56 | - **More flexible and simple Query API**
57 |
58 | The Query API is aligned to the Elasticsearch API; and it is possible to write arbitrary Elasticsearch Search
59 | queries. We do not support the `Neos\Flow\Persistence\QueryResultInterface`, and thus no ``
60 | to keep things simple.
61 |
62 | - **Only supports Batch Indexing**
63 |
64 | We currently only support batch indexing, as this removes many errors in the Neos UI if there are problems
65 | with the Elasticsearch indexing.
66 |
67 | This is an "artificial limitation" which could be removed; but we do not provide support for this removal
68 | right now.
69 |
70 | - **Only support for a single Elasticsearch version**
71 |
72 | We only support Elasticsearch 7 right now.
73 |
74 | - **Only index live the workspace**
75 |
76 | We only index the live workspace, as this is the 99% case to be supported.
77 |
78 | - **Faceting using multiple Elasticsearch requests / One Aggregation per Request**
79 |
80 | Building a huge Elasticsearch request for all facets and queries at the same time is possible, but
81 | hard to debug and understand.
82 |
83 | That's why we keep it simple here; and if you use aggregations, there will be one query per aggregation
84 | which is done.
85 |
86 | ## Starting Elasticsearch for development
87 |
88 | ```bash
89 | docker run --rm --name neos7-es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.10.2
90 | ```
91 |
92 | ## Indexing
93 |
94 | > **Tip: Indexing behaves to the user in the same way as defined in [Flowpack.ElasticSearch.ContentRepositoryAdaptor](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor).
95 | > The only difference is the internal implementation:** Instead of indexing every node (content and document) individually and letting Elasticsearch
96 | > do the merging, we merge the content to the parent document in PHP, as this is easier to handle.
97 | >
98 | > **The full configuration for indexing is exactly the same as in [Flowpack.ElasticSearch.ContentRepositoryAdaptor](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor)**.
99 | > It is included below for your convenience.
100 |
101 | The following commands are needed for indexing:
102 |
103 | ```bash
104 | ./flow nodeindex:build
105 | ./flow nodeindex:cleanup
106 | ```
107 |
108 | **NOTE:** Only nodes which are marked as `search.fulltext.isRoot` in the corresponding `NodeTypes.yaml`
109 | will become part of the search index, and all their children Content nodes' texts will be indexed as part of this.
110 |
111 | **Under the Covers**
112 |
113 | - The different indexing strategy is implemented using a custom `DocumentNodeIndexer`, which then calls a custom
114 | `DocumentIndexerDriver`.
115 |
116 | As an example, you can then query the Elasticsearch index using:
117 |
118 | ```bash
119 | curl -X GET "localhost:9200/neoscr/_search?pretty" -H 'Content-Type: application/json' -d'
120 | {
121 | "query": {
122 | "match_all": {}
123 | }
124 | }
125 | '
126 | ```
127 |
128 | ### Configurations per property (index field)
129 |
130 | You can change the analyzers on a per-field level; or e.g. reconfigure the _all field with the following snippet
131 | in the NodeTypes.yaml. Generally this works by defining the global mapping at `[nodeType].search.elasticSearchMapping`:
132 |
133 | ```yaml
134 | 'Neos.Neos:Node':
135 | search:
136 | elasticSearchMapping:
137 | myProperty:
138 | analyzer: custom_french_analyzer
139 | ```
140 |
141 | ### Exclude NodeTypes from indexing
142 |
143 | By default the indexing processes all NodeTypes, but you can change this in your *Settings.yaml*:
144 |
145 | ```yaml
146 | Neos:
147 | ContentRepository:
148 | Search:
149 | defaultConfigurationPerNodeType:
150 | '*':
151 | indexed: true
152 | 'Neos.Neos:FallbackNode':
153 | indexed: false
154 | 'Neos.Neos:Shortcut':
155 | indexed: false
156 | 'Neos.Neos:ContentCollection':
157 | indexed: false
158 | ```
159 |
160 | You need to explicitly configure the individual NodeTypes (this feature does not check the Super Type configuration).
161 | But you can use a special notation to configure a full namespace, `Acme.AcmeCom:*` will be applied for all node
162 | types in the `Acme.AcmeCom` namespace. The most specific configuration is used in this order:
163 |
164 | - NodeType name (`Neos.Neos:Shortcut`)
165 | - Full namespace notation (`Neos.Neos:*`)
166 | - Catch all (`*`)
167 |
168 | ### Indexing configuration per data type
169 |
170 | **The default configuration supports most use cases and often may not need to be touched, as this package comes
171 | with sane defaults for all Neos data types.**
172 |
173 | Indexing of properties is configured at two places. The defaults per-data-type are configured
174 | inside `Neos.ContentRepository.Search.defaultConfigurationPerType` of `Settings.yaml`.
175 | Furthermore, this can be overridden using the `properties.[....].search` path inside
176 | `NodeTypes.yaml`.
177 |
178 | This configuration contains two parts:
179 |
180 | * Underneath `elasticSearchMapping`, the Elasticsearch property mapping can be defined.
181 | * Underneath `indexing`, an Eel expression which processes the value before indexing has to be
182 | specified. It has access to the current `value` and the current `node`.
183 |
184 | Example (from the default configuration):
185 |
186 | ```yaml
187 | # Settings.yaml
188 | Neos:
189 | ContentRepository:
190 | Search:
191 | defaultConfigurationPerType:
192 |
193 | # strings should just be indexed with their simple value.
194 | string:
195 | elasticSearchMapping:
196 | type: string
197 | indexing: '${value}'
198 | ```
199 |
200 | ### Indexing configuration per property
201 |
202 | ```yaml
203 | # NodeTypes.yaml
204 | 'Neos.Neos:Timable':
205 | properties:
206 | '_hiddenBeforeDateTime':
207 | search:
208 |
209 | # A date should be mapped differently, and in this case we want to use a date format which
210 | # Elasticsearch understands
211 | elasticSearchMapping:
212 | type: DateTime
213 | format: 'date_time_no_millis'
214 | indexing: '${(node.hiddenBeforeDateTime ? Date.format(node.hiddenBeforeDateTime, "Y-m-d\TH:i:sP") : null)}'
215 | ```
216 |
217 | If your nodetypes schema defines custom properties of type DateTime, you have got to provide similar configuration for
218 | them as well in your `NodeTypes.yaml`, or else they will not be indexed correctly.
219 |
220 | There are a few indexing helpers inside the `Indexing` namespace which are usable inside the
221 | `indexing` expression. In most cases, you don't need to touch this, but they were needed to build up
222 | the standard indexing configuration:
223 |
224 | * `Indexing.buildAllPathPrefixes`: for a path such as `foo/bar/baz`, builds up a list of path
225 | prefixes, e.g. `['foo', 'foo/bar', 'foo/bar/baz']`.
226 | * `Indexing.extractNodeTypeNamesAndSupertypes(NodeType)`: extracts a list of node type names for
227 | the passed node type and all of its supertypes
228 | * `Indexing.convertArrayOfNodesToArrayOfNodeIdentifiers(array $nodes)`: convert the given nodes to
229 | their node identifiers.
230 |
231 | ### Skip indexing and mapping of a property
232 |
233 | If you don't want a property to be indexed, set `indexing: false`. In this case no mapping is configured for this field.
234 | This can be used to also solve a type conflict of two node properties with same name and different type.
235 |
236 | ### Fulltext Indexing
237 |
238 | In order to enable fulltext indexing, every `Document` node must be configured as *fulltext root*. Thus,
239 | the following is configured in the default configuration:
240 |
241 | ```yaml
242 | 'Neos.Neos:Document':
243 | search:
244 | fulltext:
245 | isRoot: true
246 | ```
247 |
248 | A *fulltext root* contains all the *content* of its non-document children, such that when one searches
249 | inside these texts, the document itself is returned as result.
250 |
251 | In order to specify how the fulltext of a property in a node should be extracted, this is configured
252 | in `NodeTypes.yaml` at `properties.[propertyName].search.fulltextExtractor`.
253 |
254 | An example:
255 |
256 | ```yaml
257 | 'Neos.Neos.NodeTypes:Text':
258 | properties:
259 | 'text':
260 | search:
261 | fulltextExtractor: '${Indexing.extractHtmlTags(value)}'
262 |
263 | 'My.Blog:Post':
264 | properties:
265 | title:
266 | search:
267 | fulltextExtractor: '${Indexing.extractInto("h1", value)}'
268 | ```
269 |
270 |
271 | ### Working with Dates
272 |
273 | As a default, Elasticsearch indexes dates in the UTC Timezone. In order to have it index using the timezone
274 | currently configured in PHP, the configuration for any property in a node which represents a date should look like this:
275 |
276 | ```yaml
277 | 'My.Blog:Post':
278 | properties:
279 | date:
280 | search:
281 | elasticSearchMapping:
282 | type: 'date'
283 | format: 'date_time_no_millis'
284 | indexing: '${(value ? Date.format(value, "Y-m-d\TH:i:sP") : null)}'
285 | ```
286 |
287 | This is important so that Date- and Time-based searches work as expected, both when using formatted DateTime strings and
288 | when using relative DateTime calculations (eg.: `now`, `now+1d`).
289 |
290 | If you want to filter items by date, e.g. to show items with date later than today, you can create a query like this:
291 |
292 | ```
293 | ${...greaterThan('date', Date.format(Date.Now(), "Y-m-d\TH:i:sP"))...}
294 | ```
295 |
296 | For more information on Elasticsearch's Date Formats,
297 | [click here](http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html).
298 |
299 |
300 | ### Working with Assets / Attachments
301 |
302 | If you want to index attachments, you need to install the [Elasticsearch Ingest-Attachment Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-attachment.html).
303 | Then, you can add the following to your `Settings.yaml`:
304 |
305 | ```yaml
306 | Neos:
307 | ContentRepository:
308 | Search:
309 | defaultConfigurationPerType:
310 | 'Neos\Media\Domain\Model\Asset':
311 | elasticSearchMapping:
312 | type: text
313 | indexing: ${Indexing.Indexing.extractAssetContent(value)}
314 | ```
315 |
316 | or add the attachments content to a fulletxt field in your NodeType configuration:
317 |
318 | ```yaml
319 | properties:
320 | file:
321 | type: 'Neos\Media\Domain\Model\Asset'
322 | ui:
323 | search:
324 | fulltextExtractor: ${Indexing.extractInto('text', Indexing.extractAssetContent(value))}
325 | ```
326 |
327 | By default `Indexing.extractAssetContent(value)` returns the asset content. You can use the second parameter to return asset meta data. The field parameter can be set to one of the following: `content, title, name, author, keywords, date, content_type, content_length, language`.
328 |
329 | With that, you can for example add the keywords of a file to a higher boosted field:
330 |
331 | ```yaml
332 | properties:
333 | file:
334 | type: 'Neos\Media\Domain\Model\Asset'
335 | ui:
336 | search:
337 | fulltextExtractor: ${Indexing.extractInto('h2', Indexing.extractAssetContent(value, 'keywords'))}
338 | ```
339 |
340 |
341 | ## Search Component
342 |
343 | As the search component usually needs to be heavily adjusted, we only include a snippet which can be copy/pasted
344 | and adjusted into your project:
345 |
346 | ```neosfusion
347 | ###01_BasicSearchTemplate.fusion###
348 | ```
349 |
350 | ## Query API
351 |
352 | Simple Example as Eel expression:
353 |
354 | ```
355 | Elasticsearch.createRequest(site)
356 | .query(
357 | Elasticsearch.createNeosFulltextQuery(site)
358 | .fulltext(request.arguments.q)
359 | )
360 | ```
361 |
362 | - If you want to search Neos nodes, we need to pass in a *context node* as first argument to `Elasticsearch.createRequest()`
363 | - This way, the correct index (in the current language) is automatically searched
364 | - You are able to call `searchResultDocument.loadNode()` on the individual document
365 | - `Elasticsearch.createNeosFulltextQuery` *also needs a context node*, which specifies the part of the node tree which
366 | we want to search.
367 |
368 | There exists a query API for more complex cases, i.e. you can do the following:
369 |
370 | ```
371 | Elasticsearch.createRequest(site)
372 | .query(
373 | Elasticsearch.createNeosFulltextQuery(site)
374 | .fulltext(request.arguments.q)
375 | // only the results to documents where myKey = myValue
376 | .filter(Elasticsearch.createTermQuery("myKey", "myValue"))
377 | )
378 | ```
379 |
380 | More complex queries for searching through multiple indices can look like this:
381 |
382 | ```
383 | Elasticsearch.createRequest(site, ['index2', 'index3')
384 | .query(
385 | Elasticsearch.createBooleanQuery()
386 | .should(
387 | Elasticsearch.createNeosFulltextQuery(site)
388 | .fulltext(request.arguments.q)
389 | .filter(Elasticsearch.createTermQuery("index_discriminator", "neos_nodes"))
390 | )
391 | .should(
392 | // add query for index2 here
393 | )
394 | )
395 | ```
396 |
397 | **We recommend to build more complex queries through Custom Eel helpers; directly calling the Query Builders of this package**
398 |
399 | ## Aggregations and Faceting
400 |
401 | Implementing Faceted Search is more difficult than it looks at first sight - so let's first build a mental model
402 | of how the queries need to work.
403 |
404 | Faceting usually looks like:
405 |
406 | ```
407 | [ Global Search Input Field ] <-- global info
408 |
409 | Categories
410 | - News
411 | - FAQ Entries
412 | - ...
413 |
414 | Products
415 | - Product 1
416 | - Product 2 (chosen)
417 |
418 | [Result Listing]
419 | ```
420 |
421 | The global search input field is the easiest - as it influences both the facets (Categories and Products) above,
422 | and the Result Listing.
423 |
424 | For a facet, things are a bit more difficult. To *calculate* the facet values (i.e. what is shown underneath the "Categories"
425 | headline), an *aggregation query* needs to be executed. In this query, we need to take into account the Global Search field,
426 | and the choices of all other facets (but not our own one).
427 |
428 | For the result listing, we need to take into account the global search, and the choices of all facets.
429 |
430 | To model that in Elasticsearch, we recommend to use multiple queries: One for each facet; and one for rendering
431 | the result listing.
432 |
433 | Here follows the list of modifications done to the template above:
434 |
435 | ```diff
436 | ###02_FacetedSearchTemplate.fusion.diff###
437 | ```
438 |
439 | You can also copy/paste the full file:
440 |
441 |
442 | See the faceted search example
443 |
444 | ```
445 | ###02_FacetedSearchTemplate.fusion###
446 | ```
447 |
448 |
449 |
450 | ## Result Highlighting
451 |
452 | Result highlighting is implemented using the [highlight API](https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html)
453 | of Elasticsearch.
454 |
455 | To enable it, you need to change thef following parts:
456 |
457 | - To use a default highlighting, add the `.highlight(Elasticsearch.createNeosFulltextHighlight())`
458 | part to your main Elasticsearch query.
459 | - Additionally, you can call the getter `searchResultDocument.processedHighlights` for each
460 | result, which contains the highlighted extracts, which you can simply join together like this:
461 |
462 | `Array.join(searchResultDocument.processedHighlights, '…')`
463 |
464 | A full example can be found below:
465 |
466 | ```diff
467 | ###02a_FacetedHighlightedSearchTemplate.fusion.diff###
468 | ```
469 |
470 | You can also copy/paste the full file:
471 |
472 |
473 | See the faceted + highlighted search example
474 |
475 | ```
476 | ###02a_FacetedHighlightedSearchTemplate.fusion###
477 | ```
478 |
479 |
480 |
481 |
482 | ## Indexing other data
483 |
484 | We suggest to set `index_discriminator` to different values for different data sources, to be able to
485 | identify different sources properly.
486 |
487 | You can use the `CustomIndexer` as a basis for indexing as follows:
488 |
489 | ```php
490 | $indexer = CustomIndexer::create('faq');
491 | $indexer->createIndexWithMapping(['properties' => [
492 | 'faqEntryTitle' => [
493 | 'type' => 'text'
494 | ]
495 | ]]);
496 | $indexer->index([
497 | 'faqEntryTitle' => 'FAQ Dresden'
498 | ]);
499 | // index other documents here
500 | $indexer->finalizeAndSwitchAlias();
501 |
502 | // Optionally for cleanup
503 | $indexer->removeObsoleteIndices();
504 | ```
505 |
506 | For your convenience, a full CommandController can be copied/pasted below:
507 |
508 |
509 | Command Controller for custom indexing
510 |
511 | ```
512 | ###03_CommandController.php###
513 | ```
514 |
515 |
516 |
517 | See the next section for querying other data sources
518 |
519 | ## Querying other data
520 |
521 | Three parts need to be adjusted for querying other data sources:
522 |
523 | - adjust `Elasticsearch.createRequest()` and `Elasticsearch.createAggregationRequest()` calls
524 | - build up and use the fulltext query for your custom data
525 | - customize result rendering.
526 |
527 | We'll now go through these steps one by one.
528 |
529 | **Adjust `Elasticsearch.createRequest()` and `Elasticsearch.createAggregationRequest()`**
530 |
531 | Here, you need to include the other index as second parameter; so for example `Elasticsearch.createRequest(site, ['faq'])`
532 | is a valid invocation.
533 |
534 |
535 | **Build up the fulltext query**
536 |
537 | We suggest that you build custom Eel helpers for doing the fulltext search in your custom data,
538 | e.g. by filtering on `index_discriminator` and using a `simple_query_string` query as follows:
539 |
540 | ```php
541 | return BooleanQueryBuilder::create()
542 | ->filter(TermQueryBuilder::create('index_discriminator', 'faq'))
543 | ->must(
544 | SimpleQueryStringBuilder::create($query ?? '')->fields([
545 | 'faqEntryTitle^5',
546 | ])
547 | );
548 | ```
549 |
550 | As an example, you can also check out the full Eel helper:
551 |
552 |
553 | Eel helper for fulltext querying custom data
554 |
555 | ```
556 | ###03_FulltextEelHelper.php###
557 | ```
558 |
559 |
560 |
561 | **Remember to register the Eel helper in `Settings.yaml` as usual:
562 |
563 | ```yaml
564 | Neos:
565 | Fusion:
566 | defaultContext:
567 | MyQueries: My\Package\Eel\MyQueries
568 | ```
569 |
570 |
571 | **Use the fulltext query**
572 |
573 | To use both the Neos and your custom fulltext query, these two queries should be combined using a `should` clause
574 | in the `Terms` query; so this is like an "or" query combination:
575 |
576 | ```
577 | Elasticsearch.createBooleanQuery()
578 | .should(Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q))
579 | .should(MyQueries.faqQuery(request.arguments.q))}
580 | ```
581 |
582 |
583 | **Adjust Result Rendering**
584 |
585 | By adding a conditional branch to `prototype(Sandstorm.LightweightElasticsearch:SearchResultCase)`, you can have a custom
586 | result rendering:
587 |
588 | ```neosfusion
589 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
590 | faqEntries {
591 | condition = ${searchResultDocument.property('index_discriminator') == 'faq'}
592 | renderer = afx`
593 | {searchResultDocument.properties.faqEntryTitle}
594 | `
595 | }
596 | }
597 | ```
598 |
599 |
600 | **Putting it all together**
601 |
602 | See the following diff, or the full source code below:
603 |
604 |
605 | ```diff
606 | ###03_ExternalDataTemplate.fusion.diff###
607 | ```
608 |
609 | You can also copy/paste the full file:
610 |
611 |
612 | See the faceted search example
613 |
614 | ```
615 | ###03_ExternalDataTemplate.fusion###
616 | ```
617 |
618 |
619 |
620 |
621 | ## Debugging Elasticsearch queries
622 |
623 | 1. add `.log("!!!FOO")` to your Eel ElasticSearch query
624 | 2. Check the System_Development log file for the full query and save it to a file (req.json)
625 | 3. We suggest to use https://httpie.io/ (which can be installed using `brew install httpie`) to
626 | do a request:
627 |
628 | ```
629 | http 127.0.0.1:9200/_cat/aliases
630 | cat req.json | http 127.0.0.1:9200/neoscr,foo/_search
631 | ```
632 |
633 | ## Developing
634 |
635 | We gladly accept pull requests or contributors :-)
636 |
637 | ### Changing this readme
638 |
639 | First change Documentation/README_template.php; then run `php Documentation/build_readme.php`.
640 |
641 | ## License
642 |
643 | MIT
644 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Sandstorm.LightweightElasticsearch
2 |
3 | ...this is an attempt for a more lightweight elasticsearch integration for Neos CMS. This is built because I wanted
4 | to try out some different design decisions in parts of the Neos <-> Elasticsearch integration.
5 |
6 | This is a wrapper around Flowpack.Elasticsearch.ContentRepositoryAdaptor, which replaces parts of its API.
7 | A huge thanks goes to everybody maintaining Flowpack.Elasticsearch.ContentRepositoryAdaptor and Flowpack.Elasticsearch,
8 | as we build upon this work and profit from it greatly.
9 |
10 |
11 |
12 | - [Sandstorm.LightweightElasticsearch](#sandstormlightweightelasticsearch)
13 | - [Goals and Limitations](#goals-and-limitations)
14 | - [Starting Elasticsearch for development](#starting-elasticsearch-for-development)
15 | - [Indexing](#indexing)
16 | - [Configurations per property index field](#configurations-per-property-index-field)
17 | - [Exclude NodeTypes from indexing](#exclude-nodetypes-from-indexing)
18 | - [Indexing configuration per data type](#indexing-configuration-per-data-type)
19 | - [Indexing configuration per property](#indexing-configuration-per-property)
20 | - [Skip indexing and mapping of a property](#skip-indexing-and-mapping-of-a-property)
21 | - [Fulltext Indexing](#fulltext-indexing)
22 | - [Working with Dates](#working-with-dates)
23 | - [Working with Assets / Attachments](#working-with-assets--attachments)
24 | - [Search Component](#search-component)
25 | - [Query API](#query-api)
26 | - [Aggregations and Faceting](#aggregations-and-faceting)
27 | - [Result Highlighting](#result-highlighting)
28 | - [Indexing other data](#indexing-other-data)
29 | - [Querying other data](#querying-other-data)
30 | - [Debugging Elasticsearch queries](#debugging-elasticsearch-queries)
31 | - [Developing](#developing)
32 | - [Changing this readme](#changing-this-readme)
33 | - [License](#license)
34 |
35 |
36 |
37 | ## Goals and Limitations
38 |
39 | The project has the following goals and limitations:
40 |
41 | - **Only for fulltext search**
42 |
43 | This means only document nodes or anything which can potentially appear in fulltext search results is put into
44 | the Elasticsearch index (everything marked in the NodeTypes as `search.fulltext.isRoot = TRUE`).
45 | That means (by default) no content nodes or ContentCollections are stored inside the index.
46 |
47 | - **Easier Fulltext indexing implementation**
48 |
49 | Fulltext collection is done in PHP instead of inside Elasticsearch with Painless.
50 |
51 | - **Query Results not specific to Neos**
52 |
53 | You can easily write queries which target anything stored in Elasticsearch; and not just Neos Nodes.
54 | We provide examples and utilities how other data sources can be indexed in Elasticsearch.
55 |
56 | - **More flexible and simple Query API**
57 |
58 | The Query API is aligned to the Elasticsearch API; and it is possible to write arbitrary Elasticsearch Search
59 | queries. We do not support the `Neos\Flow\Persistence\QueryResultInterface`, and thus no ``
60 | to keep things simple.
61 |
62 | - **Only supports Batch Indexing**
63 |
64 | We currently only support batch indexing, as this removes many errors in the Neos UI if there are problems
65 | with the Elasticsearch indexing.
66 |
67 | This is an "artificial limitation" which could be removed; but we do not provide support for this removal
68 | right now.
69 |
70 | - **Only support for a single Elasticsearch version**
71 |
72 | We only support Elasticsearch 7 right now.
73 |
74 | - **Only index live the workspace**
75 |
76 | We only index the live workspace, as this is the 99% case to be supported.
77 |
78 | - **Faceting using multiple Elasticsearch requests / One Aggregation per Request**
79 |
80 | Building a huge Elasticsearch request for all facets and queries at the same time is possible, but
81 | hard to debug and understand.
82 |
83 | That's why we keep it simple here; and if you use aggregations, there will be one query per aggregation
84 | which is done.
85 |
86 | ## Starting Elasticsearch for development
87 |
88 | ```bash
89 | docker run --rm --name neos7-es -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.10.2
90 | ```
91 |
92 | ## Indexing
93 |
94 | > **Tip: Indexing behaves to the user in the same way as defined in [Flowpack.ElasticSearch.ContentRepositoryAdaptor](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor).
95 | > The only difference is the internal implementation:** Instead of indexing every node (content and document) individually and letting Elasticsearch
96 | > do the merging, we merge the content to the parent document in PHP, as this is easier to handle.
97 | >
98 | > **The full configuration for indexing is exactly the same as in [Flowpack.ElasticSearch.ContentRepositoryAdaptor](https://github.com/Flowpack/Flowpack.ElasticSearch.ContentRepositoryAdaptor)**.
99 | > It is included below for your convenience.
100 |
101 | The following commands are needed for indexing:
102 |
103 | ```bash
104 | ./flow nodeindex:build
105 | ./flow nodeindex:cleanup
106 | ```
107 |
108 | **NOTE:** Only nodes which are marked as `search.fulltext.isRoot` in the corresponding `NodeTypes.yaml`
109 | will become part of the search index, and all their children Content nodes' texts will be indexed as part of this.
110 |
111 | **Under the Covers**
112 |
113 | - The different indexing strategy is implemented using a custom `DocumentNodeIndexer`, which then calls a custom
114 | `DocumentIndexerDriver`.
115 |
116 | As an example, you can then query the Elasticsearch index using:
117 |
118 | ```bash
119 | curl -X GET "localhost:9200/neoscr/_search?pretty" -H 'Content-Type: application/json' -d'
120 | {
121 | "query": {
122 | "match_all": {}
123 | }
124 | }
125 | '
126 | ```
127 |
128 | ### Configurations per property (index field)
129 |
130 | You can change the analyzers on a per-field level; or e.g. reconfigure the _all field with the following snippet
131 | in the NodeTypes.yaml. Generally this works by defining the global mapping at `[nodeType].search.elasticSearchMapping`:
132 |
133 | ```yaml
134 | 'Neos.Neos:Node':
135 | search:
136 | elasticSearchMapping:
137 | myProperty:
138 | analyzer: custom_french_analyzer
139 | ```
140 |
141 | ### Exclude NodeTypes from indexing
142 |
143 | By default the indexing processes all NodeTypes, but you can change this in your *Settings.yaml*:
144 |
145 | ```yaml
146 | Neos:
147 | ContentRepository:
148 | Search:
149 | defaultConfigurationPerNodeType:
150 | '*':
151 | indexed: true
152 | 'Neos.Neos:FallbackNode':
153 | indexed: false
154 | 'Neos.Neos:Shortcut':
155 | indexed: false
156 | 'Neos.Neos:ContentCollection':
157 | indexed: false
158 | ```
159 |
160 | You need to explicitly configure the individual NodeTypes (this feature does not check the Super Type configuration).
161 | But you can use a special notation to configure a full namespace, `Acme.AcmeCom:*` will be applied for all node
162 | types in the `Acme.AcmeCom` namespace. The most specific configuration is used in this order:
163 |
164 | - NodeType name (`Neos.Neos:Shortcut`)
165 | - Full namespace notation (`Neos.Neos:*`)
166 | - Catch all (`*`)
167 |
168 | ### Indexing configuration per data type
169 |
170 | **The default configuration supports most use cases and often may not need to be touched, as this package comes
171 | with sane defaults for all Neos data types.**
172 |
173 | Indexing of properties is configured at two places. The defaults per-data-type are configured
174 | inside `Neos.ContentRepository.Search.defaultConfigurationPerType` of `Settings.yaml`.
175 | Furthermore, this can be overridden using the `properties.[....].search` path inside
176 | `NodeTypes.yaml`.
177 |
178 | This configuration contains two parts:
179 |
180 | * Underneath `elasticSearchMapping`, the Elasticsearch property mapping can be defined.
181 | * Underneath `indexing`, an Eel expression which processes the value before indexing has to be
182 | specified. It has access to the current `value` and the current `node`.
183 |
184 | Example (from the default configuration):
185 |
186 | ```yaml
187 | # Settings.yaml
188 | Neos:
189 | ContentRepository:
190 | Search:
191 | defaultConfigurationPerType:
192 |
193 | # strings should just be indexed with their simple value.
194 | string:
195 | elasticSearchMapping:
196 | type: string
197 | indexing: '${value}'
198 | ```
199 |
200 | ### Indexing configuration per property
201 |
202 | ```yaml
203 | # NodeTypes.yaml
204 | 'Neos.Neos:Timable':
205 | properties:
206 | '_hiddenBeforeDateTime':
207 | search:
208 |
209 | # A date should be mapped differently, and in this case we want to use a date format which
210 | # Elasticsearch understands
211 | elasticSearchMapping:
212 | type: DateTime
213 | format: 'date_time_no_millis'
214 | indexing: '${(node.hiddenBeforeDateTime ? Date.format(node.hiddenBeforeDateTime, "Y-m-d\TH:i:sP") : null)}'
215 | ```
216 |
217 | If your nodetypes schema defines custom properties of type DateTime, you have got to provide similar configuration for
218 | them as well in your `NodeTypes.yaml`, or else they will not be indexed correctly.
219 |
220 | There are a few indexing helpers inside the `Indexing` namespace which are usable inside the
221 | `indexing` expression. In most cases, you don't need to touch this, but they were needed to build up
222 | the standard indexing configuration:
223 |
224 | * `Indexing.buildAllPathPrefixes`: for a path such as `foo/bar/baz`, builds up a list of path
225 | prefixes, e.g. `['foo', 'foo/bar', 'foo/bar/baz']`.
226 | * `Indexing.extractNodeTypeNamesAndSupertypes(NodeType)`: extracts a list of node type names for
227 | the passed node type and all of its supertypes
228 | * `Indexing.convertArrayOfNodesToArrayOfNodeIdentifiers(array $nodes)`: convert the given nodes to
229 | their node identifiers.
230 |
231 | ### Skip indexing and mapping of a property
232 |
233 | If you don't want a property to be indexed, set `indexing: false`. In this case no mapping is configured for this field.
234 | This can be used to also solve a type conflict of two node properties with same name and different type.
235 |
236 | ### Fulltext Indexing
237 |
238 | In order to enable fulltext indexing, every `Document` node must be configured as *fulltext root*. Thus,
239 | the following is configured in the default configuration:
240 |
241 | ```yaml
242 | 'Neos.Neos:Document':
243 | search:
244 | fulltext:
245 | isRoot: true
246 | ```
247 |
248 | A *fulltext root* contains all the *content* of its non-document children, such that when one searches
249 | inside these texts, the document itself is returned as result.
250 |
251 | In order to specify how the fulltext of a property in a node should be extracted, this is configured
252 | in `NodeTypes.yaml` at `properties.[propertyName].search.fulltextExtractor`.
253 |
254 | An example:
255 |
256 | ```yaml
257 | 'Neos.Neos.NodeTypes:Text':
258 | properties:
259 | 'text':
260 | search:
261 | fulltextExtractor: '${Indexing.extractHtmlTags(value)}'
262 |
263 | 'My.Blog:Post':
264 | properties:
265 | title:
266 | search:
267 | fulltextExtractor: '${Indexing.extractInto("h1", value)}'
268 | ```
269 |
270 |
271 | ### Working with Dates
272 |
273 | As a default, Elasticsearch indexes dates in the UTC Timezone. In order to have it index using the timezone
274 | currently configured in PHP, the configuration for any property in a node which represents a date should look like this:
275 |
276 | ```yaml
277 | 'My.Blog:Post':
278 | properties:
279 | date:
280 | search:
281 | elasticSearchMapping:
282 | type: 'date'
283 | format: 'date_time_no_millis'
284 | indexing: '${(value ? Date.format(value, "Y-m-d\TH:i:sP") : null)}'
285 | ```
286 |
287 | This is important so that Date- and Time-based searches work as expected, both when using formatted DateTime strings and
288 | when using relative DateTime calculations (eg.: `now`, `now+1d`).
289 |
290 | If you want to filter items by date, e.g. to show items with date later than today, you can create a query like this:
291 |
292 | ```
293 | ${...greaterThan('date', Date.format(Date.Now(), "Y-m-d\TH:i:sP"))...}
294 | ```
295 |
296 | For more information on Elasticsearch's Date Formats,
297 | [click here](http://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-date-format.html).
298 |
299 |
300 | ### Working with Assets / Attachments
301 |
302 | If you want to index attachments, you need to install the [Elasticsearch Ingest-Attachment Plugin](https://www.elastic.co/guide/en/elasticsearch/plugins/master/ingest-attachment.html).
303 | Then, you can add the following to your `Settings.yaml`:
304 |
305 | ```yaml
306 | Neos:
307 | ContentRepository:
308 | Search:
309 | defaultConfigurationPerType:
310 | 'Neos\Media\Domain\Model\Asset':
311 | elasticSearchMapping:
312 | type: text
313 | indexing: ${Indexing.Indexing.extractAssetContent(value)}
314 | ```
315 |
316 | or add the attachments content to a fulletxt field in your NodeType configuration:
317 |
318 | ```yaml
319 | properties:
320 | file:
321 | type: 'Neos\Media\Domain\Model\Asset'
322 | ui:
323 | search:
324 | fulltextExtractor: ${Indexing.extractInto('text', Indexing.extractAssetContent(value))}
325 | ```
326 |
327 | By default `Indexing.extractAssetContent(value)` returns the asset content. You can use the second parameter to return asset meta data. The field parameter can be set to one of the following: `content, title, name, author, keywords, date, content_type, content_length, language`.
328 |
329 | With that, you can for example add the keywords of a file to a higher boosted field:
330 |
331 | ```yaml
332 | properties:
333 | file:
334 | type: 'Neos\Media\Domain\Model\Asset'
335 | ui:
336 | search:
337 | fulltextExtractor: ${Indexing.extractInto('h2', Indexing.extractAssetContent(value, 'keywords'))}
338 | ```
339 |
340 |
341 | ## Search Component
342 |
343 | As the search component usually needs to be heavily adjusted, we only include a snippet which can be copy/pasted
344 | and adjusted into your project:
345 |
346 | ```neosfusion
347 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
348 | // for possibilities on how to build the query, see the next section in the documentation
349 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
350 | @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(this._elasticsearchBaseQuery))}
351 |
352 | // Search Result Display is controlled through Flowpack.Listable
353 | searchResults = Flowpack.Listable:PaginatedCollection {
354 | collection = ${mainSearchRequest}
355 | itemsPerPage = 12
356 |
357 | // we use cache mode "dynamic" for the full Search component; so we do not need an additional cache entry
358 | // for the PaginatedCollection.
359 | @cache.mode = "embed"
360 | }
361 | renderer = afx`
362 |
373 | `
374 | // If you want to see the full request going to Elasticsearch, you can include
375 | // the following snippet in the renderer above:
376 | //
377 |
378 | // The parameter "q" should be included in this pagination
379 | prototype(Flowpack.Listable:PaginationParameters) {
380 | q = ${request.arguments.q}
381 | }
382 |
383 | // We configure the cache mode "dynamic" here.
384 | @cache {
385 | mode = 'dynamic'
386 | entryIdentifier {
387 | node = ${node}
388 | type = 'searchForm'
389 | }
390 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage}
391 | context {
392 | 1 = 'node'
393 | 2 = 'documentNode'
394 | 3 = 'site'
395 | }
396 | entryTags {
397 | 1 = ${Neos.Caching.nodeTag(node)}
398 | }
399 | }
400 | }
401 |
402 | // The result display is done here.
403 | // In the context, you'll find an object `searchResultDocument` which is of type
404 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
405 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
406 | neosNodes {
407 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
408 | // This is in preparation for displaying other kinds of data.
409 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
410 | renderer.@context.node = ${searchResultDocument.loadNode()}
411 | renderer = afx`
412 |
413 | `
414 | // If you want to see the full Search Response hit, you can include the following
415 | // snippet in the renderer above:
416 | //
417 | }
418 | }
419 |
420 | ```
421 |
422 | ## Query API
423 |
424 | Simple Example as Eel expression:
425 |
426 | ```
427 | Elasticsearch.createRequest(site)
428 | .query(
429 | Elasticsearch.createNeosFulltextQuery(site)
430 | .fulltext(request.arguments.q)
431 | )
432 | ```
433 |
434 | - If you want to search Neos nodes, we need to pass in a *context node* as first argument to `Elasticsearch.createRequest()`
435 | - This way, the correct index (in the current language) is automatically searched
436 | - You are able to call `searchResultDocument.loadNode()` on the individual document
437 | - `Elasticsearch.createNeosFulltextQuery` *also needs a context node*, which specifies the part of the node tree which
438 | we want to search.
439 |
440 | There exists a query API for more complex cases, i.e. you can do the following:
441 |
442 | ```
443 | Elasticsearch.createRequest(site)
444 | .query(
445 | Elasticsearch.createNeosFulltextQuery(site)
446 | .fulltext(request.arguments.q)
447 | // only the results to documents where myKey = myValue
448 | .filter(Elasticsearch.createTermQuery("myKey", "myValue"))
449 | )
450 | ```
451 |
452 | More complex queries for searching through multiple indices can look like this:
453 |
454 | ```
455 | Elasticsearch.createRequest(site, ['index2', 'index3')
456 | .query(
457 | Elasticsearch.createBooleanQuery()
458 | .should(
459 | Elasticsearch.createNeosFulltextQuery(site)
460 | .fulltext(request.arguments.q)
461 | .filter(Elasticsearch.createTermQuery("index_discriminator", "neos_nodes"))
462 | )
463 | .should(
464 | // add query for index2 here
465 | )
466 | )
467 | ```
468 |
469 | **We recommend to build more complex queries through Custom Eel helpers; directly calling the Query Builders of this package**
470 |
471 | ## Aggregations and Faceting
472 |
473 | Implementing Faceted Search is more difficult than it looks at first sight - so let's first build a mental model
474 | of how the queries need to work.
475 |
476 | Faceting usually looks like:
477 |
478 | ```
479 | [ Global Search Input Field ] <-- global info
480 |
481 | Categories
482 | - News
483 | - FAQ Entries
484 | - ...
485 |
486 | Products
487 | - Product 1
488 | - Product 2 (chosen)
489 |
490 | [Result Listing]
491 | ```
492 |
493 | The global search input field is the easiest - as it influences both the facets (Categories and Products) above,
494 | and the Result Listing.
495 |
496 | For a facet, things are a bit more difficult. To *calculate* the facet values (i.e. what is shown underneath the "Categories"
497 | headline), an *aggregation query* needs to be executed. In this query, we need to take into account the Global Search field,
498 | and the choices of all other facets (but not our own one).
499 |
500 | For the result listing, we need to take into account the global search, and the choices of all facets.
501 |
502 | To model that in Elasticsearch, we recommend to use multiple queries: One for each facet; and one for rendering
503 | the result listing.
504 |
505 | Here follows the list of modifications done to the template above:
506 |
507 | ```diff
508 | @@ -1,7 +1,24 @@
509 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
510 | - // for possibilities on how to build the query, see the next section in the documentation
511 | + // this is the base query from the user which should *always* be applied.
512 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
513 | - @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(this._elasticsearchBaseQuery))}
514 | +
515 | + // register a Terms aggregation with the URL parameter "nodeTypesFilter"
516 | + _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
517 | +
518 | + // This is the main elasticsearch query which determines the search results:
519 | + // - this._elasticsearchBaseQuery is applied
520 | + // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
521 | + // <-- if you add additional aggregations, you need to add them here to this list.
522 | + @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
523 | +
524 | + // The Request is for displaying the Node Types aggregation (faceted search).
525 | + //
526 | + // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
527 | + // This means, for the `.aggregation()` part, we take the aggregation itself.
528 | + // For the `.filter()` part, we add:
529 | + // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
530 | + // <-- if you add additional aggregations, you need to add them here to the list.
531 | + @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
532 |
533 | // Search Result Display is controlled through Flowpack.Listable
534 | searchResults = Flowpack.Listable:PaginatedCollection {
535 | @@ -12,6 +29,23 @@
536 | // for the PaginatedCollection.
537 | @cache.mode = "embed"
538 | }
539 | +
540 | + nodeTypesFacet = Neos.Fusion:Component {
541 | + // the nodeTypesFacet is a "Terms" aggregation...
542 | + // ...so we can access nodeTypesFacet.buckets.
543 | + // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
544 | + // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
545 | + // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
546 | + renderer = afx`
547 | +
548 | +
549 | + - {bucket.key} {bucket.doc_count} (selected)
550 | +
551 | +
552 | + CLEAR FACET
553 | + `
554 | + }
555 | +
556 | renderer = afx`
557 |
567 | `
568 | @@ -32,6 +68,8 @@
569 | // The parameter "q" should be included in this pagination
570 | prototype(Flowpack.Listable:PaginationParameters) {
571 | q = ${request.arguments.q}
572 | + // <-- if you add additional aggregations, you need to add the parameter names here
573 | + nodeTypesFilter = ${request.arguments.nodeTypesFilter}
574 | }
575 |
576 | // We configure the cache mode "dynamic" here.
577 | @@ -41,7 +79,8 @@
578 | node = ${node}
579 | type = 'searchForm'
580 | }
581 | - entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage}
582 | + // <-- if you add additional aggregations, you need to add the parameter names to the entryDiscriminator
583 | + entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
584 | context {
585 | 1 = 'node'
586 | 2 = 'documentNode'
587 |
588 | ```
589 |
590 | You can also copy/paste the full file:
591 |
592 |
593 | See the faceted search example
594 |
595 | ```
596 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
597 | // this is the base query from the user which should *always* be applied.
598 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
599 |
600 | // register a Terms aggregation with the URL parameter "nodeTypesFilter"
601 | _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
602 |
603 | // This is the main elasticsearch query which determines the search results:
604 | // - this._elasticsearchBaseQuery is applied
605 | // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
606 | // <-- if you add additional aggregations, you need to add them here to this list.
607 | @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
608 |
609 | // The Request is for displaying the Node Types aggregation (faceted search).
610 | //
611 | // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
612 | // This means, for the `.aggregation()` part, we take the aggregation itself.
613 | // For the `.filter()` part, we add:
614 | // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
615 | // <-- if you add additional aggregations, you need to add them here to the list.
616 | @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
617 |
618 | // Search Result Display is controlled through Flowpack.Listable
619 | searchResults = Flowpack.Listable:PaginatedCollection {
620 | collection = ${mainSearchRequest}
621 | itemsPerPage = 12
622 |
623 | // we use cache mode "dynamic" for the full Search component; so we do not need an additional cache entry
624 | // for the PaginatedCollection.
625 | @cache.mode = "embed"
626 | }
627 |
628 | nodeTypesFacet = Neos.Fusion:Component {
629 | // the nodeTypesFacet is a "Terms" aggregation...
630 | // ...so we can access nodeTypesFacet.buckets.
631 | // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
632 | // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
633 | // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
634 | renderer = afx`
635 |
636 |
637 | - {bucket.key} {bucket.doc_count} (selected)
638 |
639 |
640 | CLEAR FACET
641 | `
642 | }
643 |
644 | renderer = afx`
645 |
658 | `
659 | // If you want to see the full request going to Elasticsearch, you can include
660 | // the following snippet in the renderer above:
661 | //
662 |
663 | // The parameter "q" should be included in this pagination
664 | prototype(Flowpack.Listable:PaginationParameters) {
665 | q = ${request.arguments.q}
666 | // <-- if you add additional aggregations, you need to add the parameter names here
667 | nodeTypesFilter = ${request.arguments.nodeTypesFilter}
668 | }
669 |
670 | // We configure the cache mode "dynamic" here.
671 | @cache {
672 | mode = 'dynamic'
673 | entryIdentifier {
674 | node = ${node}
675 | type = 'searchForm'
676 | }
677 | // <-- if you add additional aggregations, you need to add the parameter names to the entryDiscriminator
678 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
679 | context {
680 | 1 = 'node'
681 | 2 = 'documentNode'
682 | 3 = 'site'
683 | }
684 | entryTags {
685 | 1 = ${Neos.Caching.nodeTag(node)}
686 | }
687 | }
688 | }
689 |
690 | // The result display is done here.
691 | // In the context, you'll find an object `searchResultDocument` which is of type
692 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
693 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
694 | neosNodes {
695 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
696 | // This is in preparation for displaying other kinds of data.
697 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
698 | renderer.@context.node = ${searchResultDocument.loadNode()}
699 | renderer = afx`
700 |
701 | `
702 | // If you want to see the full Search Response hit, you can include the following
703 | // snippet in the renderer above:
704 | //
705 | }
706 | }
707 |
708 | ```
709 |
710 |
711 |
712 | ## Result Highlighting
713 |
714 | Result highlighting is implemented using the [highlight API](https://www.elastic.co/guide/en/elasticsearch/reference/current/highlighting.html)
715 | of Elasticsearch.
716 |
717 | To enable it, you need to change thef following parts:
718 |
719 | - To use a default highlighting, add the `.highlight(Elasticsearch.createNeosFulltextHighlight())`
720 | part to your main Elasticsearch query.
721 | - Additionally, you can call the getter `searchResultDocument.processedHighlights` for each
722 | result, which contains the highlighted extracts, which you can simply join together like this:
723 |
724 | `Array.join(searchResultDocument.processedHighlights, '…')`
725 |
726 | A full example can be found below:
727 |
728 | ```diff
729 | @@ -9,7 +9,7 @@
730 | // - this._elasticsearchBaseQuery is applied
731 | // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
732 | // <-- if you add additional aggregations, you need to add them here to this list.
733 | - @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
734 | + @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation)).highlight(Elasticsearch.createNeosFulltextHighlight())}
735 |
736 | // The Request is for displaying the Node Types aggregation (faceted search).
737 | //
738 | @@ -102,7 +102,9 @@
739 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
740 | renderer.@context.node = ${searchResultDocument.loadNode()}
741 | renderer = afx`
742 | -
743 | +
744 | + {Array.join(searchResultDocument.processedHighlights, '…')}
745 | +
746 | `
747 | // If you want to see the full Search Response hit, you can include the following
748 | // snippet in the renderer above:
749 |
750 | ```
751 |
752 | You can also copy/paste the full file:
753 |
754 |
755 | See the faceted + highlighted search example
756 |
757 | ```
758 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
759 | // this is the base query from the user which should *always* be applied.
760 | _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
761 |
762 | // register a Terms aggregation with the URL parameter "nodeTypesFilter"
763 | _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
764 |
765 | // This is the main elasticsearch query which determines the search results:
766 | // - this._elasticsearchBaseQuery is applied
767 | // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
768 | // <-- if you add additional aggregations, you need to add them here to this list.
769 | @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation)).highlight(Elasticsearch.createNeosFulltextHighlight())}
770 |
771 | // The Request is for displaying the Node Types aggregation (faceted search).
772 | //
773 | // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
774 | // This means, for the `.aggregation()` part, we take the aggregation itself.
775 | // For the `.filter()` part, we add:
776 | // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
777 | // <-- if you add additional aggregations, you need to add them here to the list.
778 | @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
779 |
780 | // Search Result Display is controlled through Flowpack.Listable
781 | searchResults = Flowpack.Listable:PaginatedCollection {
782 | collection = ${mainSearchRequest}
783 | itemsPerPage = 12
784 |
785 | // we use cache mode "dynamic" for the full Search component; so we do not need an additional cache entry
786 | // for the PaginatedCollection.
787 | @cache.mode = "embed"
788 | }
789 |
790 | nodeTypesFacet = Neos.Fusion:Component {
791 | // the nodeTypesFacet is a "Terms" aggregation...
792 | // ...so we can access nodeTypesFacet.buckets.
793 | // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
794 | // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
795 | // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
796 | renderer = afx`
797 |
798 |
799 | - {bucket.key} {bucket.doc_count} (selected)
800 |
801 |
802 | CLEAR FACET
803 | `
804 | }
805 |
806 | renderer = afx`
807 |
820 | `
821 | // If you want to see the full request going to Elasticsearch, you can include
822 | // the following snippet in the renderer above:
823 | //
824 |
825 | // The parameter "q" should be included in this pagination
826 | prototype(Flowpack.Listable:PaginationParameters) {
827 | q = ${request.arguments.q}
828 | // <-- if you add additional aggregations, you need to add the parameter names here
829 | nodeTypesFilter = ${request.arguments.nodeTypesFilter}
830 | }
831 |
832 | // We configure the cache mode "dynamic" here.
833 | @cache {
834 | mode = 'dynamic'
835 | entryIdentifier {
836 | node = ${node}
837 | type = 'searchForm'
838 | }
839 | // <-- if you add additional aggregations, you need to add the parameter names to the entryDiscriminator
840 | entryDiscriminator = ${request.arguments.q + '-' + request.arguments.currentPage + '-' + request.arguments.nodeTypesFilter}
841 | context {
842 | 1 = 'node'
843 | 2 = 'documentNode'
844 | 3 = 'site'
845 | }
846 | entryTags {
847 | 1 = ${Neos.Caching.nodeTag(node)}
848 | }
849 | }
850 | }
851 |
852 | // The result display is done here.
853 | // In the context, you'll find an object `searchResultDocument` which is of type
854 | // Sandstorm\LightweightElasticsearch\Query\Result\SearchResultDocument.
855 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
856 | neosNodes {
857 | // all Documents in the index which are Nodes have a property "index_discriminator" set to "neos_nodes";
858 | // This is in preparation for displaying other kinds of data.
859 | condition = ${searchResultDocument.property('index_discriminator') == 'neos_nodes'}
860 | renderer.@context.node = ${searchResultDocument.loadNode()}
861 | renderer = afx`
862 |
863 | {Array.join(searchResultDocument.processedHighlights, '…')}
864 |
865 | `
866 | // If you want to see the full Search Response hit, you can include the following
867 | // snippet in the renderer above:
868 | //
869 | }
870 | }
871 |
872 | ```
873 |
874 |
875 |
876 |
877 | ## Indexing other data
878 |
879 | We suggest to set `index_discriminator` to different values for different data sources, to be able to
880 | identify different sources properly.
881 |
882 | You can use the `CustomIndexer` as a basis for indexing as follows:
883 |
884 | ```php
885 | $indexer = CustomIndexer::create('faq');
886 | $indexer->createIndexWithMapping(['properties' => [
887 | 'faqEntryTitle' => [
888 | 'type' => 'text'
889 | ]
890 | ]]);
891 | $indexer->index([
892 | 'faqEntryTitle' => 'FAQ Dresden'
893 | ]);
894 | // index other documents here
895 | $indexer->finalizeAndSwitchAlias();
896 |
897 | // Optionally for cleanup
898 | $indexer->removeObsoleteIndices();
899 | ```
900 |
901 | For your convenience, a full CommandController can be copied/pasted below:
902 |
903 |
904 | Command Controller for custom indexing
905 |
906 | ```
907 | createIndexWithMapping(['properties' => [
921 | 'faqEntryTitle' => [
922 | 'type' => 'text'
923 | ]
924 | ]]);
925 | $indexer->index([
926 | 'faqEntryTitle' => 'FAQ Dresden'
927 | ]);
928 | $indexer->index([
929 | 'faqEntryTitle' => 'FAQ Berlin'
930 | ]);
931 | $indexer->finalizeAndSwitchAlias();
932 | }
933 |
934 | public function cleanupCommand()
935 | {
936 | $indexer = CustomIndexer::create('faq');
937 | $removedIndices = $indexer->removeObsoleteIndices();
938 | foreach ($removedIndices as $index) {
939 | $this->outputLine('Removed ' . $index);
940 | }
941 | }
942 | }
943 |
944 | ```
945 |
946 |
947 |
948 | See the next section for querying other data sources
949 |
950 | ## Querying other data
951 |
952 | Three parts need to be adjusted for querying other data sources:
953 |
954 | - adjust `Elasticsearch.createRequest()` and `Elasticsearch.createAggregationRequest()` calls
955 | - build up and use the fulltext query for your custom data
956 | - customize result rendering.
957 |
958 | We'll now go through these steps one by one.
959 |
960 | **Adjust `Elasticsearch.createRequest()` and `Elasticsearch.createAggregationRequest()`**
961 |
962 | Here, you need to include the other index as second parameter; so for example `Elasticsearch.createRequest(site, ['faq'])`
963 | is a valid invocation.
964 |
965 |
966 | **Build up the fulltext query**
967 |
968 | We suggest that you build custom Eel helpers for doing the fulltext search in your custom data,
969 | e.g. by filtering on `index_discriminator` and using a `simple_query_string` query as follows:
970 |
971 | ```php
972 | return BooleanQueryBuilder::create()
973 | ->filter(TermQueryBuilder::create('index_discriminator', 'faq'))
974 | ->must(
975 | SimpleQueryStringBuilder::create($query ?? '')->fields([
976 | 'faqEntryTitle^5',
977 | ])
978 | );
979 | ```
980 |
981 | As an example, you can also check out the full Eel helper:
982 |
983 |
984 | Eel helper for fulltext querying custom data
985 |
986 | ```
987 | filter(TermQueryBuilder::create('index_discriminator', 'faq'))
1005 | ->must(
1006 | SimpleQueryStringBuilder::create($query ?? '')->fields([
1007 | 'faqEntryTitle^5',
1008 | ])
1009 | );
1010 | }
1011 |
1012 | public function allowsCallOfMethod($methodName)
1013 | {
1014 | return true;
1015 | }
1016 | }
1017 |
1018 | ```
1019 |
1020 |
1021 |
1022 | **Remember to register the Eel helper in `Settings.yaml` as usual:
1023 |
1024 | ```yaml
1025 | Neos:
1026 | Fusion:
1027 | defaultContext:
1028 | MyQueries: My\Package\Eel\MyQueries
1029 | ```
1030 |
1031 |
1032 | **Use the fulltext query**
1033 |
1034 | To use both the Neos and your custom fulltext query, these two queries should be combined using a `should` clause
1035 | in the `Terms` query; so this is like an "or" query combination:
1036 |
1037 | ```
1038 | Elasticsearch.createBooleanQuery()
1039 | .should(Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q))
1040 | .should(MyQueries.faqQuery(request.arguments.q))}
1041 | ```
1042 |
1043 |
1044 | **Adjust Result Rendering**
1045 |
1046 | By adding a conditional branch to `prototype(Sandstorm.LightweightElasticsearch:SearchResultCase)`, you can have a custom
1047 | result rendering:
1048 |
1049 | ```neosfusion
1050 | prototype(Sandstorm.LightweightElasticsearch:SearchResultCase) {
1051 | faqEntries {
1052 | condition = ${searchResultDocument.property('index_discriminator') == 'faq'}
1053 | renderer = afx`
1054 | {searchResultDocument.properties.faqEntryTitle}
1055 | `
1056 | }
1057 | }
1058 | ```
1059 |
1060 |
1061 | **Putting it all together**
1062 |
1063 | See the following diff, or the full source code below:
1064 |
1065 |
1066 | ```diff
1067 | @@ -1,26 +1,15 @@
1068 | prototype(My.Package:Search) < prototype(Neos.Fusion:Component) {
1069 | - // this is the base query from the user which should *always* be applied.
1070 | - _elasticsearchBaseQuery = ${Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)}
1071 | + // for possibilities on how to build the query, see the next section in the documentation
1072 | + _elasticsearchBaseQuery = ${Elasticsearch.createBooleanQuery().should(Elasticsearch.createNeosFulltextQuery(site).fulltext(request.arguments.q)).should(MyQueries.faqQuery(request.arguments.q))}
1073 |
1074 | - // register a Terms aggregation with the URL parameter "nodeTypesFilter"
1075 | + // register a Terms aggregation with the URL parameter "nodeTypesFilter".
1076 | + // we also need to pass in the request, so that the aggregation can extract the currently selected value.
1077 | _nodeTypesAggregation = ${Elasticsearch.createTermsAggregation("neos_type", request.arguments.nodeTypesFilter)}
1078 |
1079 | - // This is the main elasticsearch query which determines the search results:
1080 | - // - this._elasticsearchBaseQuery is applied
1081 | - // - this._nodeTypesAggregation is applied as well (if the user chose a facet value)
1082 | - // <-- if you add additional aggregations, you need to add them here to this list.
1083 | - @context.mainSearchRequest = ${Elasticsearch.createRequest(site).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
1084 | -
1085 | - // The Request is for displaying the Node Types aggregation (faceted search).
1086 | - //
1087 | - // For faceted search to work properly, we need to add all OTHER query parts as filter; so NOT ourselves.
1088 | - // This means, for the `.aggregation()` part, we take the aggregation itself.
1089 | - // For the `.filter()` part, we add:
1090 | - // - this._elasticsearchBaseQuery to ensure the entered query string by the user is taken into account
1091 | - // <-- if you add additional aggregations, you need to add them here to the list.
1092 | - @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
1093 | -
1094 | - // Search Result Display is controlled through Flowpack.Listable
1095 | + // this is the main elasticsearch query which determines the search results - here, we also apply any restrictions imposed
1096 | + // by the _nodeTypesAggregation
1097 | + @context.mainSearchRequest = ${Elasticsearch.createRequest(site, ['faq']).query(Elasticsearch.createBooleanQuery().must(this._elasticsearchBaseQuery).filter(this._nodeTypesAggregation))}
1098 | + @context.nodeTypesFacet = ${Elasticsearch.createAggregationRequest(site, ['faq']).aggregation(this._nodeTypesAggregation).filter(this._elasticsearchBaseQuery).execute()}
1099 | searchResults = Flowpack.Listable:PaginatedCollection {
1100 | collection = ${mainSearchRequest}
1101 | itemsPerPage = 12
1102 | @@ -35,7 +24,7 @@
1103 | // ...so we can access nodeTypesFacet.buckets.
1104 | // To build a link to the facet, we use Neos.Neos:NodeLink with two additions:
1105 | // - addQueryString must be set to TRUE, to keep the search query and potentially other facets.
1106 | - // - to build the arguments, we need to set `nodeTypesFilter` to the current bucket key (or to null in case we want to clear the facet)
1107 | + // - to build the arguments, each aggregation result type (e.g. TermsAggregationResult) has a specific method with the required arguments.
1108 | renderer = afx`
1109 |
1110 |
1111 | @@ -50,7 +39,6 @@
1112 |