├── .styleci.yml ├── Configuration ├── Testing │ ├── ElasticVersion6 │ │ └── Settings.yaml │ ├── Settings.yaml │ └── NodeTypes.yaml ├── Settings.Neos.Fusion.yaml ├── NodeTypes.Override.Hidable.yaml ├── NodeTypes.Override.TextMixin.yaml ├── NodeTypes.Override.TitleMixin.yaml ├── Development │ └── Settings.yaml ├── NodeTypes.Override.Document.yaml ├── Settings.Neos.ContentRepository.Search.yaml ├── NodeTypes.Override.Node.yaml ├── Objects.yaml └── Settings.yaml ├── Documentation ├── ElasticConfiguration-5.x.md ├── Upgrade-8-to-9.md ├── ElasticMapping-5.x.md └── Upgrade-6-to-7.md ├── Classes ├── Exception.php ├── Exception │ ├── RuntimeException.php │ ├── ConfigurationException.php │ └── QueryBuildingException.php ├── Service │ ├── IndexNameStrategyInterface.php │ ├── DocumentIdentifier │ │ ├── DocumentIdentifierGeneratorInterface.php │ │ ├── NodeIdentifierBasedDocumentIdentifierGenerator.php │ │ └── NodeAddressBasedDocumentIdentifierGenerator.php │ ├── IndexNameService.php │ ├── IndexNameStrategy.php │ ├── NodeTypeIndexingConfiguration.php │ └── DimensionsService.php ├── Driver │ ├── SystemDriverInterface.php │ ├── DocumentDriverInterface.php │ ├── RequestDriverInterface.php │ ├── AbstractDriver.php │ ├── Version6 │ │ ├── SystemDriver.php │ │ ├── DocumentDriver.php │ │ ├── RequestDriver.php │ │ ├── Query │ │ │ ├── FilteredQuery.php │ │ │ └── FunctionScoreQuery.php │ │ ├── IndexDriver.php │ │ ├── Mapping │ │ │ └── NodeTypeMappingBuilder.php │ │ └── IndexerDriver.php │ ├── NodeTypeMappingBuilderInterface.php │ ├── IndexDriverInterface.php │ ├── IndexerDriverInterface.php │ ├── AbstractNodeTypeMappingBuilder.php │ ├── AbstractIndexerDriver.php │ ├── QueryInterface.php │ └── AbstractQuery.php ├── ErrorHandling │ ├── ErrorStorageInterface.php │ ├── ErrorHandlingService.php │ └── FileStorage.php ├── Factory │ ├── QueryFactory.php │ ├── NodeTypeMappingBuilderFactory.php │ ├── AbstractDriverSpecificObjectFactory.php │ └── DriverFactory.php ├── Dto │ └── SearchResult.php ├── Client │ └── ClientFactory.php ├── Indexer │ ├── BulkRequestPart.php │ └── WorkspaceIndexer.php ├── ViewHelpers │ └── GetHitArrayForNodeViewHelper.php ├── Eel │ ├── SearchResultHelper.php │ ├── ElasticSearchQueryResult.php │ └── ElasticSearchQuery.php ├── Command │ ├── NodeTypeCommandController.php │ ├── NodeIndexMappingCommandController.php │ └── SearchCommandController.php ├── ElasticSearchClient.php └── AssetExtraction │ └── IngestAttachmentAssetExtractor.php ├── Tests ├── Unit │ ├── Eel │ │ └── ElasticSearchQueryBuilderResultTest.php │ ├── Service │ │ ├── DimensionServiceTest.php │ │ └── NodeIndexerTest.php │ └── ViewHelpers │ │ └── GetHitArrayForNodeViewHelperTest.php └── Functional │ ├── Traits │ ├── Assertions.php │ ├── ContentRepositorySetupTrait.php │ ├── ContentRepositoryNodeCreationTrait.php │ └── ContentRepositoryMultiDimensionNodeCreationTrait.php │ ├── Service │ └── NodeTypeIndexingConfigurationTest.php │ ├── Eel │ └── ElasticSearchMultiDimensionQueryTest.php │ ├── BaseElasticsearchContentRepositoryAdapterTest.php │ └── Indexer │ └── NodeIndexerTest.php ├── .travis.yml ├── composer.json └── LICENSE /.styleci.yml: -------------------------------------------------------------------------------- 1 | preset: psr2 2 | 3 | finder: 4 | path: 5 | - "Classes" 6 | - "Tests" -------------------------------------------------------------------------------- /Configuration/Testing/ElasticVersion6/Settings.yaml: -------------------------------------------------------------------------------- 1 | Flowpack: 2 | ElasticSearch: 3 | ContentRepositoryAdaptor: 4 | driver: 5 | version: '6.x' 6 | -------------------------------------------------------------------------------- /Configuration/Settings.Neos.Fusion.yaml: -------------------------------------------------------------------------------- 1 | Neos: 2 | Fusion: 3 | defaultContext: 4 | SearchResult: Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\SearchResultHelper 5 | -------------------------------------------------------------------------------- /Configuration/NodeTypes.Override.Hidable.yaml: -------------------------------------------------------------------------------- 1 | 'Neos.Neos:Hidable': 2 | properties: 3 | 'neos_hidden': 4 | type: boolean 5 | search: 6 | indexing: '${Neos.Node.isDisabled(node)}' 7 | -------------------------------------------------------------------------------- /Configuration/NodeTypes.Override.TextMixin.yaml: -------------------------------------------------------------------------------- 1 | 'Neos.NodeTypes.BaseMixins:TextMixin': 2 | properties: 3 | 'text': 4 | search: 5 | fulltextExtractor: '${Indexing.extractHtmlTags(value)}' 6 | -------------------------------------------------------------------------------- /Configuration/NodeTypes.Override.TitleMixin.yaml: -------------------------------------------------------------------------------- 1 | 'Neos.NodeTypes.BaseMixins:TitleMixin': 2 | properties: 3 | 'title': 4 | search: 5 | fulltextExtractor: '${Indexing.extractHtmlTags(value)}' 6 | -------------------------------------------------------------------------------- /Configuration/Development/Settings.yaml: -------------------------------------------------------------------------------- 1 | Neos: 2 | ContentRepository: 3 | Search: 4 | elasticSearch: 5 | log: 6 | backendOptions: 7 | fileBackend: 8 | logFileURL: '%FLOW_PATH_DATA%Logs/ElasticSearch_Development.log' 9 | severityThreshold: '%LOG_DEBUG%' 10 | -------------------------------------------------------------------------------- /Documentation/ElasticConfiguration-5.x.md: -------------------------------------------------------------------------------- 1 | # Needed Configuration in configuration.yml for Elasticsearch 5.x 2 | 3 | No special configuration is needed for Elasticsearch 5.x. 4 | 5 | **Note:** Compared to what you may know from earlier versions of Elasticsearch, 6 | changes to the types may need to be done in your mapping for version 5. More 7 | information on the [mapping in ElasticSearch 5.x](ElasticMapping-5.x.md). 8 | -------------------------------------------------------------------------------- /Classes/Exception.php: -------------------------------------------------------------------------------- 1 | searchClient->request('GET', '/_stats')->getTreatedContent(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Classes/Factory/QueryFactory.php: -------------------------------------------------------------------------------- 1 | resolve('query'); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Classes/Dto/SearchResult.php: -------------------------------------------------------------------------------- 1 | hits = $hits; 31 | $this->total = $total; 32 | } 33 | 34 | /** 35 | * @return int 36 | */ 37 | public function getTotal(): int 38 | { 39 | return $this->total; 40 | } 41 | 42 | /** 43 | * @return array 44 | */ 45 | public function getHits(): array 46 | { 47 | return $this->hits; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Classes/Service/IndexNameService.php: -------------------------------------------------------------------------------- 1 | clientFactory->create(null, ElasticSearchClient::class); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Configuration/Testing/NodeTypes.yaml: -------------------------------------------------------------------------------- 1 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:BaseType': 2 | superTypes: { } 3 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type1': 4 | superTypes: 5 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:BaseType': true 6 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type2': 7 | superTypes: { } 8 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type3': 9 | superTypes: 10 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type1': true 11 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type2': true 12 | 13 | 'Neos.Neos:Document': 14 | properties: 15 | title_analyzed: 16 | type: string 17 | search: 18 | elasticSearchMapping: 19 | type: text 20 | 21 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document': 22 | superTypes: 23 | 'Neos.Neos:Document': true 24 | childNodes: 25 | main: 26 | type: 'Neos.Neos:ContentCollection' 27 | 28 | 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Content': 29 | superTypes: 30 | 'Neos.Neos:Content': true 31 | properties: 32 | text: 33 | type: string 34 | defaultValue: '' 35 | search: 36 | fulltextExtractor: '${Indexing.extractHtmlTags(value)}' 37 | -------------------------------------------------------------------------------- /Classes/Factory/NodeTypeMappingBuilderFactory.php: -------------------------------------------------------------------------------- 1 | resolve('nodeTypeMappingBuilder'); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Configuration/NodeTypes.Override.Document.yaml: -------------------------------------------------------------------------------- 1 | 'Neos.Neos:Document': 2 | search: 3 | fulltext: 4 | isRoot: true 5 | properties: 6 | 'uriPathSegment': 7 | search: 8 | elasticSearchMapping: 9 | type: keyword 10 | title: 11 | search: 12 | fulltextExtractor: ${Indexing.extractInto('h1', value)} 13 | 'neos_fulltext_parts': 14 | search: 15 | elasticSearchMapping: 16 | type: object 17 | enabled: false 18 | indexing: '' 19 | 'neos_fulltext': 20 | search: 21 | indexing: '' 22 | elasticSearchMapping: 23 | type: object 24 | properties: 25 | 'h1': 26 | type: text 27 | 'h2': 28 | type: text 29 | 'h3': 30 | type: text 31 | 'h4': 32 | type: text 33 | 'h5': 34 | type: text 35 | 'h6': 36 | type: text 37 | 'text': 38 | type: text 39 | 'neos_hidden_in_menu': 40 | type: boolean 41 | search: 42 | indexing: '${node.hiddenInMenu}' 43 | 44 | # deliberately don't map or index this 45 | 'hiddenInMenu': 46 | search: 47 | indexing: false 48 | -------------------------------------------------------------------------------- /Classes/Service/DocumentIdentifier/NodeIdentifierBasedDocumentIdentifierGenerator.php: -------------------------------------------------------------------------------- 1 | workspaceName; 29 | 30 | return sha1($node->aggregateId->value . $workspaceName->value); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/DocumentDriver.php: -------------------------------------------------------------------------------- 1 | [ 36 | '_id' => $identifier 37 | ] 38 | ] 39 | ]; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Classes/Driver/NodeTypeMappingBuilderInterface.php: -------------------------------------------------------------------------------- 1 | 32 | */ 33 | public function buildMappingInformation(ContentRepositoryId $contentRepositoryId, Index $index): MappingCollection; 34 | 35 | /** 36 | * @return Result 37 | */ 38 | public function getLastMappingErrors(): Result; 39 | } 40 | -------------------------------------------------------------------------------- /Classes/Service/IndexNameStrategy.php: -------------------------------------------------------------------------------- 1 | indexName; 39 | if ($name === '') { 40 | throw new Exception\ConfigurationException('Index name can not be null, check Settings at path: Neos.ContentRepository.Search.elasticSearch.indexName', 1574327388); 41 | } 42 | 43 | return $name; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Classes/Driver/IndexDriverInterface.php: -------------------------------------------------------------------------------- 1 | request('POST', '/_bulk', [], $request)->getTreatedContent(); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Classes/ErrorHandling/ErrorHandlingService.php: -------------------------------------------------------------------------------- 1 | errorCount++; 44 | $this->logger->error($message, $context); 45 | } 46 | 47 | /** 48 | * @return int 49 | */ 50 | public function getErrorCount(): int 51 | { 52 | return $this->errorCount; 53 | } 54 | 55 | /** 56 | * @return bool 57 | */ 58 | public function hasError(): bool 59 | { 60 | return $this->errorCount > 0; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Classes/Service/DocumentIdentifier/NodeAddressBasedDocumentIdentifierGenerator.php: -------------------------------------------------------------------------------- 1 | contentRepositoryId, 31 | $targetWorkspaceName ?: $node->workspaceName, 32 | $node->dimensionSpacePoint, 33 | $node->aggregateId 34 | ); 35 | 36 | return sha1($nodeAddress->toJson()); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Classes/Driver/IndexerDriverInterface.php: -------------------------------------------------------------------------------- 1 | ['some', 'nodes'] 34 | ]; 35 | 36 | $queryBuilder = $this->getMockBuilder(ElasticSearchQueryBuilder::class)->setMethods(['fetch'])->getMock(); 37 | $queryBuilder->method('fetch')->willReturn($resultArrayWithoutAggregations); 38 | 39 | $esQuery = new ElasticSearchQuery($queryBuilder); 40 | 41 | $queryResult = new ElasticSearchQueryResult($esQuery); 42 | 43 | $actual = $queryResult->getAggregations(); 44 | 45 | static::assertIsArray($actual); 46 | static::assertEmpty($actual); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Classes/Indexer/BulkRequestPart.php: -------------------------------------------------------------------------------- 1 | targetDimensionsHash = $targetDimensionsHash; 40 | $this->requests = array_map(function (array $request) { 41 | $data = json_encode($request); 42 | if ($data === false) { 43 | return null; 44 | } 45 | $this->size += strlen($data); 46 | return $data; 47 | }, $requests); 48 | } 49 | 50 | public function getTargetDimensionsHash(): string 51 | { 52 | return $this->targetDimensionsHash; 53 | } 54 | 55 | public function getRequest(): Generator 56 | { 57 | foreach ($this->requests as $request) { 58 | yield $request; 59 | } 60 | } 61 | 62 | public function getSize(): int 63 | { 64 | return $this->size; 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tests/Unit/Service/DimensionServiceTest.php: -------------------------------------------------------------------------------- 1 | [ 25 | 'indexNames' => ['neoscr-4f534b1eb0c1a785da31e681fb5e91ff-1582128256', 'neoscr-4f534b1eb0c1a785da31e681fb5e91ff-1582128111'], 26 | 'postfix' => '1582128111', 27 | 'expected' => ['neoscr-4f534b1eb0c1a785da31e681fb5e91ff-1582128111'], 28 | ], 29 | 'prefixUsesDash' => [ 30 | 'indexNames' => ['neos-cr-4f534b1eb0c1a785da31e681fb5e91ff-1582128256', 'neos-cr-4f534b1eb0c1a785da31e681fb5e91ff-1582128111'], 31 | 'postfix' => '1582128111', 32 | 'expected' => ['neos-cr-4f534b1eb0c1a785da31e681fb5e91ff-1582128111'], 33 | ] 34 | ]; 35 | } 36 | 37 | /** 38 | * @test 39 | * @dataProvider indexNameDataProvider 40 | * 41 | * @param array $indexNames 42 | * @param string $postfix 43 | * @param array $expected 44 | */ 45 | public function filterIndexNamesByPostfix(array $indexNames, string $postfix, array $expected): void 46 | { 47 | self::assertEquals($expected, IndexNameService::filterIndexNamesByPostfix($indexNames, $postfix)); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Classes/ViewHelpers/GetHitArrayForNodeViewHelper.php: -------------------------------------------------------------------------------- 1 | 29 | * {esCrAdapter:geHitArrayForNode(queryResultObject: result, node: node)} 30 | * 31 | * 32 | * array 33 | * 34 | * 35 | * You can also return specific data 36 | * 37 | * {esCrAdapter:geHitArrayForNode(queryResultObject: result, node: node, path: 'sort')} 38 | * 39 | * 40 | * array or single value 41 | * 42 | */ 43 | class GetHitArrayForNodeViewHelper extends AbstractViewHelper 44 | { 45 | /** 46 | * @param ElasticSearchQueryResult $queryResultObject 47 | * @param Node $node 48 | * @param array|string|null $path 49 | * @return mixed 50 | */ 51 | public function render(ElasticSearchQueryResult $queryResultObject, Node $node, $path = null) 52 | { 53 | $hitArray = $queryResultObject->searchHitForNode($node); 54 | 55 | if (!empty($path)) { 56 | return Arrays::getValueByPath($hitArray, $path); 57 | } 58 | 59 | return $hitArray; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /Configuration/Settings.Neos.ContentRepository.Search.yaml: -------------------------------------------------------------------------------- 1 | Neos: 2 | ContentRepository: 3 | Search: 4 | 5 | # API. If set to FALSE, only index the "live" workspace and not user workspaces. 6 | # If you only index the live workspace, Search will not work for your editors in the user workspaces. 7 | # Furthermore, if you heavily rely on Search for collecting content, this might be strange for editors to 8 | # work with -- as unpublished changes are not indexed right away. 9 | indexAllWorkspaces: true 10 | 11 | elasticSearch: 12 | 13 | # API. name of the Elasticsearch index to use. Will create many indices prefixed by this indexName. 14 | indexName: neoscr 15 | 16 | defaultConfigurationPerType: 17 | 18 | string: 19 | elasticSearchMapping: 20 | type: keyword 21 | ignore_above: 8191 22 | 23 | boolean: 24 | elasticSearchMapping: 25 | type: boolean 26 | 27 | array: 28 | elasticSearchMapping: 29 | type: keyword 30 | ignore_above: 8191 31 | 32 | integer: 33 | elasticSearchMapping: 34 | type: integer 35 | 36 | DateTime: 37 | elasticSearchMapping: 38 | type: date 39 | format: 'date_time_no_millis' 40 | indexing: '${(value ? Date.format(value, "Y-m-d\TH:i:sP") : null)}' 41 | 42 | 'Neos\Media\Domain\Model\Asset': 43 | elasticSearchMapping: '' # deliberately don't map or index this 44 | 45 | 'array': 46 | elasticSearchMapping: '' # deliberately don't map or index this 47 | 48 | 'Neos\Media\Domain\Model\ImageInterface': 49 | elasticSearchMapping: '' # deliberately don't map or index this 50 | 51 | 'references': 52 | elasticSearchMapping: 53 | type: keyword # an array of keywords, to be precise 54 | 55 | 'reference': 56 | elasticSearchMapping: 57 | type: keyword 58 | -------------------------------------------------------------------------------- /Tests/Functional/Traits/Assertions.php: -------------------------------------------------------------------------------- 1 | searchClient->request('HEAD', '/' . $indexName); 32 | self::assertEquals(200, $response->getStatusCode()); 33 | } 34 | 35 | protected function assertAliasesEquals(string $aliasPrefix, array $expectdAliases): void 36 | { 37 | $content = $this->searchClient->request('GET', '/_alias/' . $aliasPrefix . '*')->getTreatedContent(); 38 | static::assertEquals($expectdAliases, array_keys($content)); 39 | } 40 | 41 | private static function extractNodeNames(ElasticSearchQueryResult $result): array 42 | { 43 | return array_map(static function (NodeInterface $node) { 44 | return $node->getName(); 45 | }, $result->toArray()); 46 | } 47 | 48 | private static function assertNodeNames(array $expectedNames, ElasticSearchQueryResult $actualResult): void 49 | { 50 | sort($expectedNames); 51 | 52 | $actualNames = self::extractNodeNames($actualResult); 53 | sort($actualNames); 54 | 55 | self::assertEquals($expectedNames, $actualNames); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /Classes/Factory/AbstractDriverSpecificObjectFactory.php: -------------------------------------------------------------------------------- 1 | driverVersion); 54 | if (!isset($this->mapping[$version][$type]['className']) || trim($this->driverVersion) === '') { 55 | throw new ConfigurationException(sprintf('Missing or wrongly configured driver type "%s" with the given version: %s', $type, $version ?: '[missing]'), 1485933538); 56 | } 57 | 58 | $className = trim($this->mapping[$version][$type]['className']); 59 | 60 | if (!isset($this->mapping[$version][$type]['arguments'])) { 61 | return new $className(); 62 | } 63 | 64 | return new $className(...array_values($this->mapping[$version][$type]['arguments'])); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Tests/Functional/Traits/ContentRepositorySetupTrait.php: -------------------------------------------------------------------------------- 1 | workspaceRepository = $this->objectManager->get(WorkspaceRepository::class); 59 | $liveWorkspace = new Workspace('live'); 60 | $this->workspaceRepository->add($liveWorkspace); 61 | 62 | $this->nodeTypeManager = $this->objectManager->get(NodeTypeManager::class); 63 | $this->contextFactory = $this->objectManager->get(ContextFactoryInterface::class); 64 | $this->nodeDataRepository = $this->objectManager->get(NodeDataRepository::class); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Configuration/NodeTypes.Override.Node.yaml: -------------------------------------------------------------------------------- 1 | 'Neos.Neos:Node': &node 2 | search: 3 | fulltext: 4 | enable: true 5 | properties: 6 | 'neos_node_identifier': 7 | search: 8 | elasticSearchMapping: 9 | type: keyword 10 | indexing: '${node.aggregateId}' 11 | 12 | 'neos_workspace': 13 | search: 14 | elasticSearchMapping: 15 | type: keyword 16 | indexing: '${node.workspaceName}' 17 | 18 | 'neos_path': 19 | search: 20 | elasticSearchMapping: 21 | type: keyword 22 | indexing: '${Indexing.aggregateIdPath(node)}' 23 | 24 | 'neos_parent_path': 25 | search: 26 | elasticSearchMapping: 27 | type: keyword 28 | # we index *all* parent paths as separate tokens to allow for efficient searching without a prefix query 29 | indexing: '${Array.pop(Indexing.buildAllPathPrefixes(Indexing.aggregateIdPath(node)))}' 30 | 31 | # we index the node type INCLUDING ALL SUPERTYPES 32 | 'neos_type_and_supertypes': 33 | search: 34 | elasticSearchMapping: 35 | type: keyword 36 | indexing: '${Indexing.extractNodeTypeNamesAndSupertypes(node)}' 37 | 38 | 'neos_last_modification_date_time': 39 | search: 40 | elasticSearchMapping: 41 | type: date 42 | format: 'date_time_no_millis' 43 | indexing: '${(node.timestamps.lastModified ? Date.format(node.timestamps.lastModified, "Y-m-d\TH:i:sP") : null)}' 44 | 45 | 'neos_last_publication_date_time': 46 | search: 47 | elasticSearchMapping: 48 | type: date 49 | format: 'date_time_no_millis' 50 | indexing: '${(node.timestamps.originalLastModified ? Date.format(node.timestamps.originalLastModified, "Y-m-d\TH:i:sP") : null)}' 51 | 52 | 'neos_creation_date_time': 53 | search: 54 | elasticSearchMapping: 55 | type: date 56 | format: 'date_time_no_millis' 57 | indexing: '${(node.timestamps.created ? Date.format(node.timestamps.created, "Y-m-d\TH:i:sP") : null)}' 58 | # deliberately don't map or index this 59 | '_nodeType': 60 | search: 61 | indexing: false 62 | '_hidden': 63 | search: 64 | indexing: false 65 | 'unstructured': *node 66 | -------------------------------------------------------------------------------- /Classes/ErrorHandling/FileStorage.php: -------------------------------------------------------------------------------- 1 | renderErrorResult($errorResult)); 46 | } else { 47 | throw new RuntimeException('Elasticsearch error response could not be written to ' . $filename, 1588835331); 48 | } 49 | 50 | return $message; 51 | } 52 | 53 | /** 54 | * @param array $errorResult 55 | * @return string 56 | */ 57 | protected function renderErrorResult(array $errorResult): string 58 | { 59 | $error = json_encode($errorResult, JSON_PRETTY_PRINT); 60 | return sprintf("Error:\n=======\n\n%s\n\n", $error); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Classes/Driver/AbstractNodeTypeMappingBuilder.php: -------------------------------------------------------------------------------- 1 | configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, 'Neos.ContentRepository.Search'); 58 | $this->defaultConfigurationPerType = $settings['defaultConfigurationPerType']; 59 | } 60 | } 61 | 62 | /** 63 | * @return Result 64 | */ 65 | public function getLastMappingErrors(): Result 66 | { 67 | return $this->lastMappingErrors; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | sudo: false 3 | git: 4 | depth: 5 5 | submodules: false 6 | addons: 7 | apt: 8 | packages: 9 | - openjdk-8-jre-headless 10 | matrix: 11 | include: 12 | - php: 7.3 13 | env: ES=6 14 | - php: 7.3 15 | env: ES=7 16 | - php: 7.4 17 | env: ES=6 18 | - php: 7.4 19 | env: ES=7 20 | 21 | cache: 22 | directories: 23 | - $HOME/.composer/cache 24 | 25 | before_install: 26 | - export NEOS_TARGET_VERSION=7.0 27 | - cd .. 28 | - if [ "$ES" = 6 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-6.8.6.tar.gz && tar xvfz elasticsearch-6.8.6.tar.gz && mv elasticsearch-6.8.6 elasticsearch; fi 29 | - if [ "$ES" = 7 ]; then wget --no-check-certificate https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.0-linux-x86_64.tar.gz && tar xvfz elasticsearch-7.9.0-linux-x86_64.tar.gz && mv elasticsearch-7.9.0 elasticsearch; fi 30 | - cd elasticsearch 31 | - bin/elasticsearch -d 32 | - cd .. 33 | - git clone https://github.com/neos/neos-base-distribution.git -b ${NEOS_TARGET_VERSION} 34 | - cd neos-base-distribution 35 | - composer require --no-update --no-interaction neos/content-repository-search:dev-master 36 | - composer require --no-update --no-interaction flowpack/elasticsearch:dev-master 37 | - composer require --no-update --no-interaction flowpack/elasticsearch-contentrepositoryadaptor:dev-master 38 | install: 39 | - composer install --no-interaction 40 | - cd .. 41 | - rm -rf neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor 42 | - mv Flowpack.ElasticSearch.ContentRepositoryAdaptor neos-base-distribution/Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor 43 | - cd neos-base-distribution 44 | script: 45 | - bin/phpunit --colors -c Build/BuildEssentials/PhpUnit/UnitTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Unit 46 | - if [ "$ES" = 6 ]; then FLOW_CONTEXT="Testing/ElasticVersion6" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi 47 | - if [ "$ES" = 7 ]; then FLOW_CONTEXT="Testing/ElasticVersion6" bin/phpunit --colors --stop-on-failure -c Build/BuildEssentials/PhpUnit/FunctionalTests.xml Packages/Application/Flowpack.ElasticSearch.ContentRepositoryAdaptor/Tests/Functional; fi 48 | -------------------------------------------------------------------------------- /Classes/Factory/DriverFactory.php: -------------------------------------------------------------------------------- 1 | resolve('document'); 37 | } 38 | 39 | /** 40 | * @return IndexerDriverInterface 41 | * @throws ConfigurationException 42 | */ 43 | public function createIndexerDriver(): IndexerDriverInterface 44 | { 45 | return $this->resolve('indexer'); 46 | } 47 | 48 | /** 49 | * @return IndexDriverInterface 50 | * @throws ConfigurationException 51 | */ 52 | public function createIndexManagementDriver(): IndexDriverInterface 53 | { 54 | return $this->resolve('indexManagement'); 55 | } 56 | 57 | /** 58 | * @return RequestDriverInterface 59 | * @throws ConfigurationException 60 | */ 61 | public function createRequestDriver(): RequestDriverInterface 62 | { 63 | return $this->resolve('request'); 64 | } 65 | 66 | /** 67 | * @return SystemDriverInterface 68 | * @throws ConfigurationException 69 | */ 70 | public function createSystemDriver(): SystemDriverInterface 71 | { 72 | return $this->resolve('system'); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Classes/Eel/SearchResultHelper.php: -------------------------------------------------------------------------------- 1 | getSuggestions()[$suggestionName] as $suggestion) { 47 | if (array_key_exists('options', $suggestion) && !empty($suggestion['options'])) { 48 | $bestSuggestion = current($suggestion['options']); 49 | $maxScore = $bestSuggestion['score'] > $maxScore ? $bestSuggestion['score'] : $maxScore; 50 | $suggestionParts[] = $bestSuggestion['text']; 51 | } else { 52 | $suggestionParts[] = $suggestion['text']; 53 | } 54 | } 55 | if ($maxScore >= $scoreThreshold) { 56 | return implode(' ', $suggestionParts); 57 | } 58 | 59 | return ''; 60 | } 61 | 62 | /** 63 | * @param string $methodName 64 | * @return boolean 65 | */ 66 | public function allowsCallOfMethod($methodName) 67 | { 68 | return true; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Configuration/Objects.yaml: -------------------------------------------------------------------------------- 1 | Neos\ContentRepository\Search\Search\QueryBuilderInterface: 2 | className: Flowpack\ElasticSearch\ContentRepositoryAdaptor\Eel\ElasticSearchQueryBuilder 3 | 4 | Neos\ContentRepository\Search\Indexer\NodeIndexerInterface: 5 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Indexer\NodeIndexer' 6 | 7 | Neos\ContentRepository\Search\AssetExtraction\AssetExtractorInterface: 8 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\AssetExtraction\IngestAttachmentAssetExtractor' 9 | 10 | 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\DocumentIdentifier\DocumentIdentifierGeneratorInterface': 11 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Service\DocumentIdentifier\NodeAddressBasedDocumentIdentifierGenerator' 12 | 13 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\QueryInterface: 14 | scope: prototype 15 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\QueryFactory' 16 | factoryMethodName: createQuery 17 | 18 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\DocumentDriverInterface: 19 | scope: singleton 20 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' 21 | factoryMethodName: createDocumentDriver 22 | 23 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexerDriverInterface: 24 | scope: singleton 25 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' 26 | factoryMethodName: createIndexerDriver 27 | 28 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\IndexDriverInterface: 29 | scope: singleton 30 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' 31 | factoryMethodName: createIndexManagementDriver 32 | 33 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\RequestDriverInterface: 34 | scope: singleton 35 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' 36 | factoryMethodName: createRequestDriver 37 | 38 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\SystemDriverInterface: 39 | scope: singleton 40 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\DriverFactory' 41 | factoryMethodName: createSystemDriver 42 | 43 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\NodeTypeMappingBuilderInterface: 44 | scope: singleton 45 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Factory\NodeTypeMappingBuilderFactory' 46 | factoryMethodName: createNodeTypeMappingBuilder 47 | 48 | Flowpack\ElasticSearch\ContentRepositoryAdaptor\ElasticSearchClient: 49 | scope: singleton 50 | factoryObjectName: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Client\ClientFactory' 51 | factoryMethodName: create 52 | -------------------------------------------------------------------------------- /Classes/Service/NodeTypeIndexingConfiguration.php: -------------------------------------------------------------------------------- 1 | settings === null || !is_array($this->settings)) { 44 | return true; 45 | } 46 | 47 | if (isset($this->settings[$nodeType->name->value]['indexed'])) { 48 | return (bool)$this->settings[$nodeType->name->value]['indexed']; 49 | } 50 | 51 | $nodeTypeParts = explode(':', $nodeType->name->value); 52 | $namespace = reset($nodeTypeParts) . ':*'; 53 | if (isset($this->settings[$namespace]['indexed'])) { 54 | return (bool)$this->settings[$namespace]['indexed']; 55 | } 56 | if (isset($this->settings['*']['indexed'])) { 57 | return (bool)$this->settings['*']['indexed']; 58 | } 59 | 60 | return false; 61 | } 62 | 63 | /** 64 | * @return array 65 | * @throws Exception 66 | */ 67 | public function getIndexableConfiguration(ContentRepositoryId $contentRepositoryId): array 68 | { 69 | $nodeConfigurations = []; 70 | $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); 71 | /** @var NodeType $nodeType */ 72 | foreach ($contentRepository->getNodeTypeManager()->getNodeTypes(false) as $nodeType) { 73 | $nodeConfigurations[$nodeType->name->value] = $this->isIndexable($nodeType); 74 | } 75 | 76 | return $nodeConfigurations; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Tests/Functional/Service/NodeTypeIndexingConfigurationTest.php: -------------------------------------------------------------------------------- 1 | nodeTypeManager = $this->objectManager->get(NodeTypeManager::class); 37 | $this->nodeTypeIndexingConfiguration = $this->objectManager->get(NodeTypeIndexingConfiguration::class); 38 | } 39 | 40 | public function nodeTypeDataProvider(): array 41 | { 42 | return [ 43 | 'notIndexable' => [ 44 | 'nodeTypeName' => 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type1', 45 | 'expected' => false, 46 | ], 47 | 'indexable' => [ 48 | 'nodeTypeName' => 'Flowpack.ElasticSearch.ContentRepositoryAdaptor:Type2', 49 | 'expected' => true, 50 | ], 51 | ]; 52 | } 53 | 54 | /** 55 | * @test 56 | * @dataProvider nodeTypeDataProvider 57 | * 58 | * @param string $nodeTypeName 59 | * @param bool $expected 60 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 61 | * @throws \Neos\ContentRepository\Exception\NodeTypeNotFoundException 62 | */ 63 | public function isIndexable(string $nodeTypeName, bool $expected): void 64 | { 65 | self::assertEquals($expected, $this->nodeTypeIndexingConfiguration->isIndexable($this->nodeTypeManager->getNodeType($nodeTypeName))); 66 | } 67 | 68 | /** 69 | * @test 70 | * @dataProvider nodeTypeDataProvider 71 | * 72 | * @param string $nodeTypeName 73 | * @param bool $expected 74 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 75 | */ 76 | public function getIndexableConfiguration(string $nodeTypeName, bool $expected): void 77 | { 78 | $indexableConfiguration = $this->nodeTypeIndexingConfiguration->getIndexableConfiguration(); 79 | self::assertEquals($indexableConfiguration[$nodeTypeName], $expected); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/Query/FilteredQuery.php: -------------------------------------------------------------------------------- 1 | request; 31 | foreach ($this->unsupportedFieldsInCountRequest as $field) { 32 | if (isset($request[$field])) { 33 | unset($request[$field]); 34 | } 35 | } 36 | 37 | return json_encode($request); 38 | } 39 | 40 | /** 41 | * {@inheritdoc} 42 | */ 43 | public function size(int $size): void 44 | { 45 | $this->request['size'] = $size; 46 | } 47 | 48 | /** 49 | * {@inheritdoc} 50 | */ 51 | public function from(int $size): void 52 | { 53 | $this->request['from'] = $size; 54 | } 55 | 56 | /** 57 | * {@inheritdoc} 58 | */ 59 | public function fulltext(string $searchWord, array $options = []): void 60 | { 61 | $this->appendAtPath('query.bool.must', [ 62 | 'query_string' => array_merge( 63 | $this->queryStringParameters, 64 | $options, 65 | [ 'query' => $searchWord ] 66 | ) 67 | ]); 68 | } 69 | 70 | /** 71 | * {@inheritdoc} 72 | */ 73 | public function simpleQueryStringFulltext(string $searchWord, array $options = []): void 74 | { 75 | $this->appendAtPath('query.bool.must', [ 76 | 'simple_query_string' => array_merge( 77 | $this->queryStringParameters, 78 | $options, 79 | [ 'query' => $searchWord ] 80 | ) 81 | ]); 82 | } 83 | 84 | /** 85 | * {@inheritdoc} 86 | */ 87 | public function queryFilter(string $filterType, $filterOptions, string $clauseType = 'must'): void 88 | { 89 | if (!in_array($clauseType, ['must', 'should', 'must_not', 'filter'])) { 90 | throw new QueryBuildingException('The given clause type "' . $clauseType . '" is not supported. Must be one of "must", "should", "must_not".', 1383716082); 91 | } 92 | 93 | $this->appendAtPath('query.bool.filter.bool.' . $clauseType, [$filterType => $filterOptions]); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Classes/Service/DimensionsService.php: -------------------------------------------------------------------------------- 1 | coordinates === []) { 49 | $this->dimensionsRegistry[self::HASH_DEFAULT] = []; 50 | return self::HASH_DEFAULT; 51 | } 52 | 53 | $this->dimensionsRegistry[$dimensionSpacePoint->hash] = $dimensionSpacePoint; 54 | 55 | return $dimensionSpacePoint->hash; 56 | } 57 | 58 | /** 59 | * @param Node $node 60 | * @return string|null 61 | */ 62 | public function hashByNode(Node $node): ?string 63 | { 64 | return $this->hash($node->dimensionSpacePoint); 65 | } 66 | 67 | /** 68 | * @return array 69 | */ 70 | public function getDimensionsRegistry(): array 71 | { 72 | return $this->dimensionsRegistry; 73 | } 74 | 75 | public function reset(): void 76 | { 77 | $this->dimensionsRegistry = []; 78 | } 79 | 80 | /** 81 | * Only return the dimensions of the current node and all dimensions 82 | * that fall back to the current nodes dimensions. 83 | * 84 | * @param Node $node 85 | * @return array 86 | */ 87 | public function getDimensionCombinationsForIndexing(Node $node): DimensionSpacePointSet 88 | { 89 | $dimensionsHash = $this->hash($node->dimensionSpacePoint); 90 | 91 | if (!isset($this->dimensionCombinationsForIndexing[$dimensionsHash])) { 92 | 93 | $contentRepository = $this->contentRepositoryRegistry->get($node->contentRepositoryId); 94 | $this->dimensionCombinationsForIndexing[$dimensionsHash] = $contentRepository->getVariationGraph()->getSpecializationSet($node->dimensionSpacePoint); 95 | 96 | } 97 | 98 | return $this->dimensionCombinationsForIndexing[$dimensionsHash]; 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Configuration/Settings.yaml: -------------------------------------------------------------------------------- 1 | Flowpack: 2 | ElasticSearch: 3 | ContentRepositoryAdaptor: 4 | command: 5 | useSubProcesses: true 6 | indexing: 7 | retryOnConflict: 3 8 | batchSize: 9 | elements: 500 10 | octets: 40000000 11 | assetExtraction: 12 | # The maximum size of files to be ingested in bytes (100 Mb) 13 | maximumFileSize: 104857600 14 | configuration: 15 | nodeTypes: 16 | '*': 17 | indexed: true 18 | driver: 19 | version: '6.x' 20 | mapping: 21 | 6.x: &v6x 22 | query: 23 | className: Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Query\FilteredQuery 24 | arguments: 25 | request: 26 | query: 27 | bool: 28 | must: 29 | - match_all: 30 | boost: 1.0 # force match_all to be an object 31 | filter: 32 | bool: 33 | must: [] 34 | should: [] 35 | must_not: 36 | - term: 37 | neos_hidden: true 38 | _source: 39 | - 'neos_path' 40 | - 'neos_node_identifier' 41 | 42 | unsupportedFieldsInCountRequest: 43 | - '_source' 44 | - 'sort' 45 | - 'from' 46 | - 'size' 47 | - 'highlight' 48 | - 'aggs' 49 | - 'aggregations' 50 | - 'suggest' 51 | 52 | # Parameters for the query string query used by the fullText() and simpleQueryStringFulltext() operation 53 | # See https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-query-string-query.html#query-string-multi-field 54 | # for all available parameters 55 | queryStringParameters: 56 | default_operator: or 57 | fields: 58 | - neos_fulltext.h1^20 59 | - neos_fulltext.h2^12 60 | - neos_fulltext.h3^10 61 | - neos_fulltext.h4^5 62 | - neos_fulltext.h5^3 63 | - neos_fulltext.h6^2 64 | - neos_fulltext.text^1 65 | 66 | document: 67 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\DocumentDriver' 68 | indexer: 69 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\IndexerDriver' 70 | indexManagement: 71 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\IndexDriver' 72 | request: 73 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\RequestDriver' 74 | system: 75 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\SystemDriver' 76 | nodeTypeMappingBuilder: 77 | className: 'Flowpack\ElasticSearch\ContentRepositoryAdaptor\Driver\Version6\Mapping\NodeTypeMappingBuilder' 78 | 7.x: *v6x 79 | 8.x: *v6x 80 | -------------------------------------------------------------------------------- /Tests/Unit/Service/NodeIndexerTest.php: -------------------------------------------------------------------------------- 1 | buildAccessibleProxy(DimensionsService::class); 20 | $this->dimensionService = new $proxyClass; 21 | } 22 | 23 | public function dimensionCombinationProvider(): array 24 | { 25 | return [ 26 | 'multiDimension' => [ 27 | 'dimensionCombinations' => [ 28 | [ 29 | 'country' => ['uk', 'us'], 30 | 'language' => ['en_UK', 'en_US'], 31 | ], 32 | [ 33 | 'country' => ['us'], 34 | 'language' => ['en_US'], 35 | ], 36 | [ 37 | 'country' => ['de'], 38 | 'language' => ['de'], 39 | ] 40 | ], 41 | 'nodeDimensions' => [ 42 | 'country' => ['us'], 43 | 'language' => ['en_US'], 44 | ], 45 | 'expected' => [ 46 | [ 47 | 'country' => ['uk', 'us'], 48 | 'language' => ['en_UK', 'en_US'], 49 | ], 50 | [ 51 | 'country' => ['us'], 52 | 'language' => ['en_US'], 53 | ] 54 | ] 55 | ], 56 | 'singleDimension' => [ 57 | 'dimensionCombinations' => [ 58 | [ 59 | 'language' => ['en_UK', 'en_US'], 60 | ], 61 | [ 62 | 'language' => ['en_US'], 63 | ], 64 | [ 65 | 'language' => ['de'], 66 | ] 67 | ], 68 | 'nodeDimensions' => [ 69 | 'language' => ['en_US'], 70 | ], 71 | 'expected' => [ 72 | [ 73 | 'language' => ['en_UK', 'en_US'], 74 | ], 75 | [ 76 | 'language' => ['en_US'], 77 | ] 78 | ] 79 | ] 80 | ]; 81 | } 82 | 83 | /** 84 | * @test 85 | * 86 | * @dataProvider dimensionCombinationProvider 87 | * @param array $dimensionCombinations 88 | * @param array $nodeDimensions 89 | * @param array $expected 90 | */ 91 | public function reduceDimensionCombinationstoSelfAndFallback(array $dimensionCombinations, array $nodeDimensions, array $expected): void 92 | { 93 | self::assertEquals($expected, $this->dimensionService->_call('reduceDimensionCombinationstoSelfAndFallback', $dimensionCombinations, $nodeDimensions)); 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Classes/Command/NodeTypeCommandController.php: -------------------------------------------------------------------------------- 1 | contentRepositoryRegistry->get($contentRepositoryId); 52 | $nodeTypeManager = $contentRepository->getNodeTypeManager(); 53 | 54 | if ($nodeType !== null) { 55 | $nodeType = $nodeTypeManager->getNodeType(NodeTypeName::fromString($nodeType)); 56 | $configuration = $nodeType->getFullConfiguration(); 57 | } else { 58 | $nodeTypes = $nodeTypeManager->getNodeTypes(); 59 | $configuration = []; 60 | foreach ($nodeTypes as $nodeTypeName => $nodeType) { 61 | $configuration[$nodeTypeName] = $nodeType->getFullConfiguration(); 62 | } 63 | } 64 | $this->output(Yaml::dump($configuration, 5, 2)); 65 | } 66 | 67 | /** 68 | * Shows a list of NodeTypes and if they are configured to be indexable or not 69 | * 70 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 71 | */ 72 | public function showIndexableConfigurationCommand(string $contentRepository = 'default'): void 73 | { 74 | $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); 75 | 76 | $indexableConfiguration = $this->nodeTypeIndexingConfiguration->getIndexableConfiguration($contentRepositoryId); 77 | $indexTable = []; 78 | foreach ($indexableConfiguration as $nodeTypeName => $value) { 79 | $indexTable[] = [$nodeTypeName, $value ? 'true' : 'false']; 80 | } 81 | 82 | $this->output->outputTable($indexTable, ['NodeType', 'indexable']); 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /Classes/Driver/AbstractIndexerDriver.php: -------------------------------------------------------------------------------- 1 | contentRepositoryRegistry->get($node->contentRepositoryId)->getNodeTypeManager()->getNodeType($node->nodeTypeName); 53 | 54 | if ($nodeType->hasConfiguration('search')) { 55 | $elasticSearchSettingsForNode = $nodeType->getConfiguration('search'); 56 | if (isset($elasticSearchSettingsForNode['fulltext']['isRoot']) && $elasticSearchSettingsForNode['fulltext']['isRoot'] === true) { 57 | return true; 58 | } 59 | } 60 | 61 | return false; 62 | } 63 | 64 | /** 65 | * @param Node $node 66 | * @return Node|null 67 | */ 68 | protected function findClosestFulltextRoot(Node $node): ?Node 69 | { 70 | $subgraph = $this->contentRepositoryRegistry->get($node->contentRepositoryId)->getContentGraph($node->workspaceName)->getSubgraph($node->dimensionSpacePoint, VisibilityConstraints::withoutRestrictions()); 71 | 72 | $closestFulltextNode = $node; 73 | while (!$this->isFulltextRoot($closestFulltextNode)) { 74 | $closestFulltextNode = $subgraph->findParentNode($closestFulltextNode->aggregateId); 75 | if ($closestFulltextNode === null) { 76 | // root of hierarchy, no fulltext root found anymore, abort silently... 77 | if (!$node->nodeTypeName->equals(NodeTypeNameFactory::forRoot()) && 78 | !$node->nodeTypeName->equals(NodeTypeNameFactory::forSites())) { 79 | $this->logger->warning(sprintf('NodeIndexer: No fulltext root found for node %s', (string)$node->aggregateId), LogEnvironment::fromMethodName(__METHOD__)); 80 | } 81 | 82 | return null; 83 | } 84 | } 85 | 86 | return $closestFulltextNode; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Tests/Functional/Traits/ContentRepositoryNodeCreationTrait.php: -------------------------------------------------------------------------------- 1 | context = $this->contextFactory->create([ 25 | 'workspaceName' => 'live', 26 | 'dimensions' => ['language' => ['en_US']], 27 | 'targetDimensions' => ['language' => 'en_US'] 28 | ]); 29 | $rootNode = $this->context->getRootNode(); 30 | 31 | $this->siteNode = $rootNode->createNode('welcome', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 32 | $this->siteNode->setProperty('title', 'welcome'); 33 | 34 | $newDocumentNode1 = $this->siteNode->createNode('test-node-1', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 35 | $newDocumentNode1->setProperty('title', 'chicken'); 36 | $newDocumentNode1->setProperty('title_analyzed', 'chicken'); 37 | 38 | $newContentNode1 = $newDocumentNode1->getNode('main')->createNode('document-1-text-1', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Content')); 39 | $newContentNode1->setProperty('text', 'A Scout smiles and whistles under all circumstances.'); 40 | 41 | $newDocumentNode2 = $this->siteNode->createNode('test-node-2', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 42 | $newDocumentNode2->setProperty('title', 'chicken'); 43 | $newDocumentNode2->setProperty('title_analyzed', 'chicken'); 44 | 45 | // Nodes for cacheLifetime test 46 | $newContentNode2 = $newDocumentNode2->getNode('main')->createNode('document-2-text-1', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Content')); 47 | $newContentNode2->setProperty('text', 'Hidden after 2025-01-01'); 48 | $newContentNode2->setHiddenAfterDateTime(new \DateTime('@1735686000')); 49 | $newContentNode3 = $newDocumentNode2->getNode('main')->createNode('document-2-text-2', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Content')); 50 | $newContentNode3->setProperty('text', 'Hidden before 2018-07-18'); 51 | $newContentNode3->setHiddenBeforeDateTime(new \DateTime('@1531864800')); 52 | 53 | $newDocumentNode3 = $this->siteNode->createNode('test-node-3', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 54 | $newDocumentNode3->setProperty('title', 'egg'); 55 | $newDocumentNode3->setProperty('title_analyzed', 'egg'); 56 | 57 | $dimensionContext = $this->contextFactory->create([ 58 | 'workspaceName' => 'live', 59 | 'dimensions' => ['language' => ['de']] 60 | ]); 61 | $translatedNode3 = $dimensionContext->adoptNode($newDocumentNode3, true); 62 | $translatedNode3->setProperty('title', 'De'); 63 | 64 | $this->persistenceManager->persistAll(); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/Query/FunctionScoreQuery.php: -------------------------------------------------------------------------------- 1 | [] 26 | ]; 27 | 28 | /** 29 | * @param array $functions 30 | * @return void 31 | */ 32 | public function functions(array $functions): void 33 | { 34 | if (isset($functions['functions'])) { 35 | $this->functionScoreRequest = $functions; 36 | } else { 37 | $this->functionScoreRequest['functions'] = $functions; 38 | } 39 | } 40 | 41 | /** 42 | * @param string $scoreMode 43 | * @return void 44 | * @throws Exception\QueryBuildingException 45 | */ 46 | public function scoreMode(string $scoreMode): void 47 | { 48 | if (!in_array($scoreMode, ['multiply', 'first', 'sum', 'avg', 'max', 'min'])) { 49 | throw new Exception\QueryBuildingException('Invalid score mode', 1454016230); 50 | } 51 | $this->functionScoreRequest['score_mode'] = $scoreMode; 52 | } 53 | 54 | /** 55 | * @param string $boostMode 56 | * @return void 57 | * @throws Exception\QueryBuildingException 58 | */ 59 | public function boostMode(string $boostMode): void 60 | { 61 | if (!in_array($boostMode, ['multiply', 'replace', 'sum', 'avg', 'max', 'min'])) { 62 | throw new Exception\QueryBuildingException('Invalid boost mode', 1454016229); 63 | } 64 | $this->functionScoreRequest['boost_mode'] = $boostMode; 65 | } 66 | 67 | /** 68 | * @param integer|float $boost 69 | * @return void 70 | * @throws Exception\QueryBuildingException 71 | */ 72 | public function maxBoost($boost): void 73 | { 74 | if (!is_numeric($boost)) { 75 | throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); 76 | } 77 | $this->functionScoreRequest['max_boost'] = $boost; 78 | } 79 | 80 | /** 81 | * @param integer|float $score 82 | * @return void 83 | * @throws Exception\QueryBuildingException 84 | */ 85 | public function minScore($score): void 86 | { 87 | if (!is_numeric($score)) { 88 | throw new Exception\QueryBuildingException('Invalid max boost', 1454016230); 89 | } 90 | $this->functionScoreRequest['min_score'] = $score; 91 | } 92 | 93 | /** 94 | * {@inheritdoc} 95 | */ 96 | protected function prepareRequest(): array 97 | { 98 | if ($this->functionScoreRequest['functions'] === []) { 99 | return parent::prepareRequest(); 100 | } 101 | $currentQuery = $this->request['query']; 102 | 103 | $baseQuery = $this->request; 104 | unset($baseQuery['query']); 105 | 106 | $functionScore = $this->functionScoreRequest; 107 | $functionScore['query'] = $currentQuery; 108 | $query = Arrays::arrayMergeRecursiveOverrule($baseQuery, [ 109 | 'query' => [ 110 | 'function_score' => $functionScore 111 | ] 112 | ]); 113 | 114 | return $query; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /Tests/Functional/Eel/ElasticSearchMultiDimensionQueryTest.php: -------------------------------------------------------------------------------- 1 | setupContentRepository(); 46 | $this->createNodesForNodeSearchTest(); 47 | $this->indexNodes(); 48 | } 49 | 50 | /** 51 | * @test 52 | */ 53 | public function countDefaultDimensionNodesTest(): void 54 | { 55 | $resultDefault = $this->getQueryBuilder() 56 | ->query($this->siteNodeDefault) 57 | ->log($this->getLogMessagePrefix(__METHOD__)) 58 | ->nodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document') 59 | ->sortDesc('title') 60 | ->execute(); 61 | 62 | static::assertCount(3, $resultDefault->toArray()); 63 | static::assertNodeNames(['root', 'document1', 'document-untranslated'], $resultDefault); 64 | } 65 | 66 | /** 67 | * @test 68 | */ 69 | public function countDeDimensionNodesTest(): void 70 | { 71 | $resultDe = $this->getQueryBuilder() 72 | ->query($this->siteNodeDe) 73 | ->log($this->getLogMessagePrefix(__METHOD__)) 74 | ->nodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document') 75 | ->sortDesc('title') 76 | ->execute(); 77 | 78 | // expecting: root, document1, document2, document3, document4, untranslated (fallback from en_us) = 6 79 | static::assertCount(6, $resultDe->toArray(), 'Found nodes: ' . implode(',', self::extractNodeNames($resultDe))); 80 | static::assertNodeNames(['root', 'document1', 'document2-de', 'document3-de', 'document4-de', 'document-untranslated'], $resultDe); 81 | } 82 | 83 | /** 84 | * @test 85 | */ 86 | public function countDkDimensionNodesTest(): void 87 | { 88 | $resultDk = $this->getQueryBuilder() 89 | ->query($this->siteNodeDk) 90 | ->log($this->getLogMessagePrefix(__METHOD__)) 91 | ->nodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document') 92 | ->sortDesc('title') 93 | ->execute(); 94 | 95 | // expecting: root, document1, document2, untranslated (fallback from en_us) = 4 96 | static::assertCount(4, $resultDk->toArray()); 97 | static::assertNodeNames(['root', 'document1', 'document2-dk', 'document-untranslated'], $resultDk); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/IndexDriver.php: -------------------------------------------------------------------------------- 1 | searchClient->request('POST', '/_aliases', [], \json_encode(['actions' => $actions])); 37 | } 38 | 39 | /** 40 | * @param string $index 41 | * @throws Exception 42 | * @throws TransferException 43 | * @throws ApiException 44 | * @throws \Neos\Flow\Http\Exception 45 | */ 46 | public function deleteIndex(string $index): void 47 | { 48 | $response = $this->searchClient->request('HEAD', '/' . $index); 49 | if ($response->getStatusCode() === 200) { 50 | $response = $this->searchClient->request('DELETE', '/' . $index); 51 | if ($response->getStatusCode() !== 200) { 52 | throw new Exception('The index "' . $index . '" could not be deleted. (return code: ' . $response->getStatusCode() . ')', 1395419177); 53 | } 54 | } 55 | } 56 | 57 | /** 58 | * @param string $alias 59 | * @return array 60 | * @throws Exception 61 | * @throws TransferException 62 | * @throws ApiException 63 | * @throws \Neos\Flow\Http\Exception 64 | */ 65 | public function getIndexNamesByAlias(string $alias): array 66 | { 67 | $response = $this->searchClient->request('GET', '/_alias/' . $alias); 68 | $statusCode = $response->getStatusCode(); 69 | if ($statusCode !== 200 && $statusCode !== 404) { 70 | throw new Exception('The alias "' . $alias . '" was not found with some unexpected error... (return code: ' . $statusCode . ')', 1383650137); 71 | } 72 | 73 | // return empty array if content from response cannot be read as an array 74 | $treatedContent = $response->getTreatedContent(); 75 | 76 | return is_array($treatedContent) ? array_keys($treatedContent) : []; 77 | } 78 | 79 | /** 80 | * @param string $prefix 81 | * @return array 82 | * @throws TransferException 83 | * @throws ApiException 84 | * @throws \Neos\Flow\Http\Exception 85 | */ 86 | public function getIndexNamesByPrefix(string $prefix): array 87 | { 88 | $treatedContent = $this->searchClient->request('GET', '/_alias/')->getTreatedContent(); 89 | 90 | // return empty array if content from response cannot be read as an array 91 | if (!\is_array($treatedContent)) { 92 | return []; 93 | } 94 | 95 | return \array_filter(\array_keys($treatedContent), static function ($indexName) use ($prefix) { 96 | $prefix .= IndexNameService::INDEX_PART_SEPARATOR; 97 | return strpos($indexName, $prefix) === 0; 98 | }); 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Tests/Unit/ViewHelpers/GetHitArrayForNodeViewHelperTest.php: -------------------------------------------------------------------------------- 1 | viewHelper = new GetHitArrayForNodeViewHelper(); 45 | $this->mockNode = $this->createMock(NodeInterface::class); 46 | $this->mockQueryResult = $this->getMockBuilder(ElasticSearchQueryResult::class)->setMethods(['searchHitForNode'])->disableOriginalConstructor()->getMock(); 47 | } 48 | 49 | /** 50 | * @test 51 | */ 52 | public function ifNoPathIsSetTheFullHitArrayWillBeReturned(): void 53 | { 54 | $hitArray = [ 55 | 'sort' => [ 56 | 0 => '14' 57 | ] 58 | ]; 59 | 60 | $this->mockQueryResult->expects($this->once())->method('searchHitForNode')->willReturn($hitArray); 61 | 62 | $result = $this->viewHelper->render($this->mockQueryResult, $this->mockNode); 63 | $this->assertEquals($hitArray, $result, 'The full hit array will be returned'); 64 | } 65 | 66 | /** 67 | * @test 68 | */ 69 | public function viewHelperWillReturnAPathFromHitArray(): void 70 | { 71 | $path = 'sort'; 72 | $hitArray = [ 73 | 'foo' => 'bar', 74 | $path => [ 75 | 0 => '14', 76 | 1 => '18' 77 | ] 78 | ]; 79 | 80 | $this->mockQueryResult->expects($this->once())->method('searchHitForNode')->willReturn($hitArray); 81 | 82 | $result = $this->viewHelper->render($this->mockQueryResult, $this->mockNode, $path); 83 | $this->assertEquals($hitArray[$path], $result, 'Just a path from the full hit array will be returned'); 84 | } 85 | 86 | /** 87 | * @test 88 | */ 89 | public function aSingleValueWillBeReturnedForADottedPath(): void 90 | { 91 | $singleValue = 'bar'; 92 | $hitArray = [ 93 | 'foo' => [ 94 | 0 => $singleValue 95 | ], 96 | 'sort' => [ 97 | 0 => '14', 98 | 1 => '18' 99 | ] 100 | ]; 101 | 102 | $this->mockQueryResult->expects($this->exactly(2))->method('searchHitForNode')->willReturn($hitArray); 103 | 104 | $singleResult = $this->viewHelper->render($this->mockQueryResult, $this->mockNode, 'foo.0'); 105 | $this->assertEquals($singleValue, $singleResult, 'Only a single value will be returned if path is dotted'); 106 | 107 | $fullResult = $this->viewHelper->render($this->mockQueryResult, $this->mockNode, 'sort'); 108 | $this->assertEquals($hitArray['sort'], $fullResult, 'Full array will be returned if there are multiple values'); 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Tests/Functional/BaseElasticsearchContentRepositoryAdapterTest.php: -------------------------------------------------------------------------------- 1 | nodeIndexCommandController = $this->objectManager->get(NodeIndexCommandController::class); 48 | $this->searchClient = $this->objectManager->get(ElasticSearchClient::class); 49 | } 50 | 51 | public function tearDown(): void 52 | { 53 | parent::tearDown(); 54 | 55 | if (isset($this->contextFactory) && $this->contextFactory instanceof ContextFactoryInterface) { 56 | $this->inject($this->contextFactory, 'contextInstances', []); 57 | } 58 | 59 | if (!$this->isIndexInitialized()) { 60 | // clean up any existing indices 61 | $aliases = $this->searchClient->request('GET', '_aliases')->getTreatedContent(); 62 | 63 | foreach ($aliases as $alias => $aliasConfiguration) { 64 | if(str_starts_with($alias, self::TESTING_INDEX_PREFIX)) { 65 | $this->searchClient->request('DELETE', '/' . $alias); 66 | } 67 | } 68 | } 69 | } 70 | 71 | /** 72 | * @param string $method 73 | * @return string 74 | */ 75 | protected function getLogMessagePrefix(string $method): string 76 | { 77 | return substr(strrchr($method, '\\'), 1); 78 | } 79 | 80 | protected function indexNodes(): void 81 | { 82 | if ($this->isIndexInitialized()) { 83 | return; 84 | } 85 | 86 | $this->nodeIndexCommandController->buildCommand(null, false, null, 'functionaltest'); 87 | $this->setIndexInitialized(); 88 | } 89 | 90 | /** 91 | * @return ElasticSearchQueryBuilder 92 | */ 93 | protected function getQueryBuilder(): ElasticSearchQueryBuilder 94 | { 95 | try { 96 | /** @var ElasticSearchQueryBuilder $elasticSearchQueryBuilder */ 97 | $elasticSearchQueryBuilder = $this->objectManager->get(ElasticSearchQueryBuilder::class); 98 | $this->inject($elasticSearchQueryBuilder, 'now', new \DateTimeImmutable('@1735685400')); // Dec. 31, 2024 23:50:00 99 | 100 | return $elasticSearchQueryBuilder; 101 | } catch (\Exception $exception) { 102 | static::fail('Setting up the QueryBuilder failed: ' . $exception->getMessage()); 103 | } 104 | } 105 | 106 | protected function isIndexInitialized(): bool 107 | { 108 | return self::$instantiatedIndexes[get_class($this)] ?? false; 109 | } 110 | 111 | protected function setIndexInitialized(): void 112 | { 113 | self::$instantiatedIndexes[get_class($this)] = true; 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /Classes/Indexer/WorkspaceIndexer.php: -------------------------------------------------------------------------------- 1 | contentRepositoryRegistry->get($contentRepositoryId); 52 | $dimensionSpacePoints = $contentRepository->getVariationGraph()->getDimensionSpacePoints(); 53 | 54 | if ($dimensionSpacePoints->isEmpty()) { 55 | $count += $this->indexWithDimensions($contentRepositoryId, $workspaceName, DimensionSpacePoint::createWithoutDimensions(), $limit, $callback); 56 | } else { 57 | foreach ($dimensionSpacePoints as $dimensionSpacePoint) { 58 | $count += $this->indexWithDimensions($contentRepositoryId, $workspaceName, $dimensionSpacePoint, $limit, $callback); 59 | } 60 | } 61 | 62 | return $count; 63 | } 64 | 65 | /** 66 | * @param string $workspaceName 67 | * @param array $dimensions 68 | * @param int|null $limit 69 | * @param callable $callback 70 | * @return int 71 | */ 72 | public function indexWithDimensions(ContentRepositoryId $contentRepositoryId, WorkspaceName $workspaceName, DimensionSpacePoint $dimensionSpacePoint, ?int $limit = null, ?callable $callback = null): int 73 | { 74 | $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); 75 | $contentGraph = $contentRepository->getContentGraph($workspaceName); 76 | 77 | $rootNodeAggregate = $contentGraph->findRootNodeAggregateByType(NodeTypeNameFactory::forSites()); 78 | $subgraph = $contentGraph->getSubgraph($dimensionSpacePoint, NeosVisibilityConstraints::excludeRemoved()); 79 | 80 | $rootNode = $subgraph->findNodeById($rootNodeAggregate->nodeAggregateId); 81 | $indexedNodes = 0; 82 | 83 | $this->nodeIndexingManager->indexNode($rootNode); 84 | $indexedNodes++; 85 | 86 | foreach ($subgraph->findDescendantNodes($rootNode->aggregateId, FindDescendantNodesFilter::create()) as $descendantNode) { 87 | if ($limit !== null && $indexedNodes > $limit) { 88 | break; 89 | } 90 | 91 | $this->nodeIndexingManager->indexNode($descendantNode); 92 | $indexedNodes++; 93 | 94 | }; 95 | 96 | $this->nodeIndexingManager->flushQueues(); 97 | 98 | if ($callback !== null) { 99 | $callback($workspaceName, $indexedNodes, $dimensionSpacePoint); 100 | } 101 | 102 | return $indexedNodes; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flowpack/elasticsearch-contentrepositoryadaptor", 3 | "type": "neos-package", 4 | "description": "This package provides functionality for using Elasticsearch on top of Neos.ContentRepository.Search", 5 | "homepage": "http://flowpack.org/", 6 | "license": ["LGPL-3.0-only"], 7 | "require": { 8 | "php": "^8.2", 9 | "ext-json": "*", 10 | "flowpack/elasticsearch": "^5.4 || dev-main", 11 | "neos/contentrepository-core": "^9.0 || dev-master", 12 | "neos/contentrepositoryregistry": "^9.0 || dev-master", 13 | "neos/content-repository-search": "^5.0 || dev-main", 14 | "neos/eel": "^9.0 || dev-master", 15 | "neos/flow": "^9.0 || dev-master", 16 | "neos/fluid-adaptor": "^9.0 || dev-master", 17 | "neos/utility-arrays": "^9.0 || dev-master" 18 | }, 19 | "autoload": { 20 | "psr-4": { 21 | "Flowpack\\ElasticSearch\\ContentRepositoryAdaptor\\": "Classes" 22 | } 23 | }, 24 | "extra": { 25 | "applied-flow-migrations": [ 26 | "TYPO3.FLOW3-201201261636", 27 | "TYPO3.Fluid-201205031303", 28 | "TYPO3.FLOW3-201205292145", 29 | "TYPO3.FLOW3-201206271128", 30 | "TYPO3.FLOW3-201209201112", 31 | "TYPO3.Flow-201209251426", 32 | "TYPO3.Flow-201211151101", 33 | "TYPO3.Flow-201212051340", 34 | "TYPO3.TypoScript-130516234520", 35 | "TYPO3.TypoScript-130516235550", 36 | "TYPO3.TYPO3CR-130523180140", 37 | "TYPO3.Neos.NodeTypes-201309111655", 38 | "TYPO3.Flow-201310031523", 39 | "TYPO3.Flow-201405111147", 40 | "TYPO3.Neos-201407061038", 41 | "TYPO3.Neos-201409071922", 42 | "TYPO3.TYPO3CR-140911160326", 43 | "TYPO3.Neos-201410010000", 44 | "TYPO3.TYPO3CR-141101082142", 45 | "TYPO3.Neos-20141113115300", 46 | "TYPO3.Fluid-20141113120800", 47 | "TYPO3.Flow-20141113121400", 48 | "TYPO3.Fluid-20141121091700", 49 | "TYPO3.Neos-20141218134700", 50 | "TYPO3.Fluid-20150214130800", 51 | "TYPO3.Neos-20150303231600", 52 | "TYPO3.TYPO3CR-20150510103823", 53 | "TYPO3.Flow-20151113161300", 54 | "TYPO3.Form-20160601101500", 55 | "TYPO3.Flow-20161115140400", 56 | "TYPO3.Flow-20161115140430", 57 | "Neos.Flow-20161124204700", 58 | "Neos.Flow-20161124204701", 59 | "Neos.Twitter.Bootstrap-20161124204912", 60 | "Neos.Form-20161124205254", 61 | "Neos.Flow-20161124224015", 62 | "Neos.Party-20161124225257", 63 | "Neos.Eel-20161124230101", 64 | "Neos.Kickstart-20161124230102", 65 | "Neos.Setup-20161124230842", 66 | "Neos.Imagine-20161124231742", 67 | "Neos.Media-20161124233100", 68 | "Neos.NodeTypes-20161125002300", 69 | "Neos.SiteKickstarter-20161125002311", 70 | "Neos.Neos-20161125002322", 71 | "Neos.ContentRepository-20161125012000", 72 | "Neos.Fusion-20161125013710", 73 | "Neos.Setup-20161125014759", 74 | "Neos.SiteKickstarter-20161125095901", 75 | "Neos.Fusion-20161125104701", 76 | "Neos.NodeTypes-20161125104800", 77 | "Neos.Neos-20161125104802", 78 | "Neos.Kickstarter-20161125110814", 79 | "Neos.Neos-20161125122412", 80 | "Neos.Flow-20161125124112", 81 | "TYPO3.FluidAdaptor-20161130112935", 82 | "Neos.Fusion-20161201202543", 83 | "Neos.Neos-20161201222211", 84 | "Neos.Fusion-20161202215034", 85 | "Neos.ContentRepository.Search-20161210231100", 86 | "Neos.Fusion-20161219092345", 87 | "Neos.ContentRepository-20161219093512", 88 | "Neos.Media-20161219094126", 89 | "Neos.Neos-20161219094403", 90 | "Neos.Neos-20161219122512", 91 | "Neos.Fusion-20161219130100", 92 | "Neos.Neos-20161220163741", 93 | "Neos.Neos-20170115114620", 94 | "Neos.Fusion-20170120013047", 95 | "Neos.Flow-20170125103800", 96 | "Neos.Seo-20170127154600", 97 | "Neos.Flow-20170127183102" 98 | ] 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Classes/Command/NodeIndexMappingCommandController.php: -------------------------------------------------------------------------------- 1 | nodeIndexer->getIndexName(); 56 | 57 | $headers = ['Dimension Preset', 'Index Name']; 58 | $rows = []; 59 | 60 | $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); 61 | $variationGraph = $this->contentRepositoryRegistry->get($contentRepositoryId)->getVariationGraph(); 62 | 63 | foreach ($variationGraph->getDimensionSpacePoints() as $dimensionSpacePoint) { 64 | $rows[] = [ 65 | $dimensionSpacePoint->toJson(), 66 | sprintf('%s-%s', $indexName, $dimensionSpacePoint->hash) 67 | ]; 68 | } 69 | 70 | $this->output->outputTable($rows, $headers); 71 | } 72 | 73 | /** 74 | * Show the mapping which would be sent to the ElasticSearch server 75 | * 76 | * @return void 77 | * @throws \Flowpack\ElasticSearch\Exception 78 | */ 79 | public function mappingCommand(string $contentRepository = 'default'): void 80 | { 81 | $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); 82 | try { 83 | $nodeTypeMappingCollection = $this->nodeTypeMappingBuilder->buildMappingInformation($contentRepositoryId, $this->nodeIndexer->getIndex()); 84 | } catch (Exception $e) { 85 | $this->outputLine('Unable to get the current index'); 86 | $this->sendAndExit(1); 87 | } 88 | 89 | foreach ($nodeTypeMappingCollection as $mapping) { 90 | /** @var Mapping $mapping */ 91 | $this->output(Yaml::dump($mapping->asArray(), 5, 2)); 92 | $this->outputLine(); 93 | } 94 | $this->outputLine('------------'); 95 | 96 | $mappingErrors = $this->nodeTypeMappingBuilder->getLastMappingErrors(); 97 | if ($mappingErrors->hasErrors()) { 98 | $this->outputLine('Mapping Errors'); 99 | foreach ($mappingErrors->getFlattenedErrors() as $errors) { 100 | foreach ($errors as $error) { 101 | $this->outputLine('%s', [$error]); 102 | } 103 | } 104 | } 105 | 106 | if ($mappingErrors->hasWarnings()) { 107 | $this->outputLine('Mapping Warnings'); 108 | foreach ($mappingErrors->getFlattenedWarnings() as $warnings) { 109 | foreach ($warnings as $warning) { 110 | $this->outputLine('%s', [$warning]); 111 | } 112 | } 113 | } 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /Classes/ElasticSearchClient.php: -------------------------------------------------------------------------------- 1 | contextNode; 58 | } 59 | 60 | /** 61 | * @param Node $contextNode 62 | */ 63 | public function setContextNode(Node $contextNode): void 64 | { 65 | $this->setDimensions($contextNode->dimensionSpacePoint); 66 | $this->contextNode = $contextNode; 67 | } 68 | 69 | /** 70 | * @param array $dimensionSpacePoint 71 | */ 72 | public function setDimensions(DimensionSpacePoint $dimensionSpacePoint): void 73 | { 74 | $this->dimensionsHash = $dimensionSpacePoint->hash; 75 | } 76 | 77 | /** 78 | * @return string 79 | */ 80 | public function getDimensionsHash(): string 81 | { 82 | return $this->dimensionsHash; 83 | } 84 | 85 | /** 86 | * @param \Closure $closure 87 | * @param array $dimensionValues 88 | * @throws \Exception 89 | */ 90 | public function withDimensions(\Closure $closure, DimensionSpacePoint $dimensionSpacePoint): void 91 | { 92 | $previousDimensionHash = $this->dimensionsHash; 93 | try { 94 | $this->setDimensions($dimensionSpacePoint); 95 | $closure(); 96 | } finally { 97 | $this->dimensionsHash = $previousDimensionHash; 98 | } 99 | } 100 | 101 | /** 102 | * Get the index name to be used 103 | * 104 | * @return string 105 | * @throws Exception 106 | * @throws ConfigurationException 107 | * @todo Add a contraints, if the system use content dimensions, the dimensionsHash MUST be set 108 | */ 109 | public function getIndexName(): string 110 | { 111 | $name = $this->getIndexNamePrefix(); 112 | if ($this->dimensionsHash !== null) { 113 | $name .= '-' . $this->dimensionsHash; 114 | } 115 | return $name; 116 | } 117 | 118 | /** 119 | * @return string 120 | * @throws Exception 121 | * @throws ConfigurationException 122 | */ 123 | public function getIndexNamePrefix(): string 124 | { 125 | $name = trim($this->indexNameStrategy->get()); 126 | if ($name === '') { 127 | throw new ConfigurationException('IndexNameStrategy ' . get_class($this->indexNameStrategy) . ' returned an empty index name', 1582538800); 128 | } 129 | 130 | return $name; 131 | } 132 | 133 | /** 134 | * Retrieve the index to be used for querying or on-the-fly indexing. 135 | * In Elasticsearch, this index is an *alias* to the currently used index. 136 | * 137 | * @return \Flowpack\ElasticSearch\Domain\Model\Index 138 | * @throws Exception 139 | * @throws \Flowpack\ElasticSearch\Exception 140 | * @throws ConfigurationException 141 | */ 142 | public function getIndex(): Index 143 | { 144 | return $this->findIndex($this->getIndexName()); 145 | } 146 | } 147 | -------------------------------------------------------------------------------- /Tests/Functional/Traits/ContentRepositoryMultiDimensionNodeCreationTrait.php: -------------------------------------------------------------------------------- 1 | contextFactory->create([ 24 | 'workspaceName' => 'live', 25 | 'dimensions' => ['language' => ['en_US']], 26 | 'targetDimensions' => ['language' => 'en_US'] 27 | ]); 28 | $deLanguageDimensionContext = $this->contextFactory->create([ 29 | 'workspaceName' => 'live', 30 | 'dimensions' => ['language' => ['de', 'en_US']], 31 | 'targetDimensions' => ['language' => 'de'] 32 | ]); 33 | $dkLanguageDimensionContext = $this->contextFactory->create([ 34 | 'workspaceName' => 'live', 35 | 'dimensions' => ['language' => ['dk', 'en_US']], 36 | 'targetDimensions' => ['language' => 'dk'] 37 | ]); 38 | 39 | $rootNode = $defaultLanguageDimensionContext->getRootNode(); 40 | $this->siteNodeDefault = $rootNode->createNode('root', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 41 | $this->siteNodeDefault->setProperty('title', 'root-default'); 42 | $this->siteNodeDe = $deLanguageDimensionContext->adoptNode($this->siteNodeDefault, true); 43 | $this->siteNodeDe->setProperty('title', 'root-de'); 44 | $this->siteNodeDk = $dkLanguageDimensionContext->adoptNode($this->siteNodeDefault, true); 45 | $this->siteNodeDk->setProperty('title', 'root-dk'); 46 | 47 | // add a document node that is translated in two languages 48 | $newDocumentNode1 = $this->siteNodeDefault->createNode('document1', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 49 | $newDocumentNode1->setProperty('title', 'document1-default'); 50 | 51 | $translatedDocumentNode1De = $deLanguageDimensionContext->adoptNode($newDocumentNode1, true); 52 | $translatedDocumentNode1De->setProperty('title', 'document1-de'); 53 | $translatedDocumentNode1Dk = $dkLanguageDimensionContext->adoptNode($newDocumentNode1, true); 54 | $translatedDocumentNode1Dk->setProperty('title', 'document1-dk'); 55 | 56 | // add a document node that is not translated 57 | $newDocumentNode_untranslated = $this->siteNodeDefault->createNode('document-untranslated', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 58 | $newDocumentNode_untranslated->setProperty('title', 'document-untranslated'); 59 | 60 | // add additional, but separate nodes here 61 | $standaloneDocumentNode2De = $this->siteNodeDe->createNode('document2-de', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 62 | $standaloneDocumentNode2De->setProperty('title', 'document2-de'); 63 | 64 | $standaloneDocumentNode2Dk = $this->siteNodeDk->createNode('document2-dk', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 65 | $standaloneDocumentNode2Dk->setProperty('title', 'document2-dk'); 66 | 67 | // add an additional german node 68 | $documentNodeDe3 = $standaloneDocumentNode2De->createNode('document3-de', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 69 | $documentNodeDe3->setProperty('title', 'document3-de'); 70 | 71 | // add another german node, but translate it to danish 72 | $documentNodeDe4 = $standaloneDocumentNode2De->createNode('document4-de', $this->nodeTypeManager->getNodeType('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document')); 73 | $documentNodeDe4->setProperty('title', 'document4-de'); 74 | 75 | // This one has no connetion to the DK root and DK does not fallback to DE, so this node should not be indexed! 76 | $translatedDocumentNode4Dk = $dkLanguageDimensionContext->adoptNode($documentNodeDe4, true); 77 | $translatedDocumentNode4Dk->setProperty('title', 'document4-dk'); 78 | 79 | $this->persistenceManager->persistAll(); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/Mapping/NodeTypeMappingBuilder.php: -------------------------------------------------------------------------------- 1 | 47 | * @throws Exception 48 | */ 49 | public function buildMappingInformation(ContentRepositoryId $contentRepositoryId, Index $index): MappingCollection 50 | { 51 | $this->lastMappingErrors = new Result(); 52 | 53 | $mappings = new MappingCollection(MappingCollection::TYPE_ENTITY); 54 | 55 | $contentRepository = $this->contentRepositoryRegistry->get($contentRepositoryId); 56 | 57 | /** @var \Neos\ContentRepository\Core\NodeType\NodeType $nodeType */ 58 | foreach ($contentRepository->getNodeTypeManager()->getNodeTypes() as $nodeTypeName => $nodeType) { 59 | if ($nodeTypeName === 'unstructured' || $nodeType->isAbstract()) { 60 | continue; 61 | } 62 | 63 | if ($this->nodeTypeIndexingConfiguration->isIndexable($nodeType) === false) { 64 | continue; 65 | } 66 | 67 | $mapping = new Mapping($index->findType($nodeTypeName)); 68 | $fullConfiguration = $nodeType->getFullConfiguration(); 69 | if (isset($fullConfiguration['search']['elasticSearchMapping'])) { 70 | $fullMapping = $fullConfiguration['search']['elasticSearchMapping']; 71 | $mapping->setFullMapping($fullMapping); 72 | } 73 | 74 | $propertiesAndReferences = array_merge( 75 | $nodeType->getProperties(), 76 | array_map(function ($reference) { 77 | $reference['type'] = 'references'; 78 | return $reference; 79 | }, $nodeType->getReferences()) 80 | ); 81 | 82 | foreach ($propertiesAndReferences as $propertyName => $propertyConfiguration) { 83 | // This property is configured to not be indexed, so do not add a mapping for it 84 | if (isset($propertyConfiguration['search']) && array_key_exists('indexing', $propertyConfiguration['search']) && $propertyConfiguration['search']['indexing'] === false) { 85 | continue; 86 | } 87 | 88 | if (isset($propertyConfiguration['search']['elasticSearchMapping'])) { 89 | if (is_array($propertyConfiguration['search']['elasticSearchMapping'])) { 90 | $propertyMapping = array_filter($propertyConfiguration['search']['elasticSearchMapping'], static function ($value) { 91 | return $value !== null; 92 | }); 93 | $mapping->setPropertyByPath($propertyName, $propertyMapping); 94 | } 95 | } elseif (isset($propertyConfiguration['type'], $this->defaultConfigurationPerType[$propertyConfiguration['type']]['elasticSearchMapping'])) { 96 | if (is_array($this->defaultConfigurationPerType[$propertyConfiguration['type']]['elasticSearchMapping'])) { 97 | $mapping->setPropertyByPath($propertyName, $this->defaultConfigurationPerType[$propertyConfiguration['type']]['elasticSearchMapping']); 98 | } 99 | } else { 100 | $this->lastMappingErrors->addWarning(new Warning('Node Type "' . $nodeTypeName . '" - property "' . $propertyName . '": No ElasticSearch Mapping found.')); 101 | } 102 | } 103 | 104 | $mappings->add($mapping); 105 | } 106 | 107 | return $mappings; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /Classes/AssetExtraction/IngestAttachmentAssetExtractor.php: -------------------------------------------------------------------------------- 1 | getResource()->getFileSize() > $this->maximumFileSize) { 67 | $this->logger->info(sprintf('The asset %s with size of %s bytes exceeds the maximum size of %s bytes. The file content was not ingested.', $asset->getResource()->getFilename(), $asset->getResource()->getFileSize(), $this->maximumFileSize), LogEnvironment::fromMethodName(__METHOD__)); 68 | return $this->buildAssetContentObject([]); 69 | } 70 | 71 | $extractedAsset = null; 72 | 73 | $request = [ 74 | 'pipeline' => [ 75 | 'description' => 'Attachment Extraction', 76 | 'processors' => [ 77 | [ 78 | 'attachment' => [ 79 | 'field' => 'neos_asset', 80 | 'indexed_chars' => 100000, 81 | 'ignore_missing' => true, 82 | ] 83 | ] 84 | ] 85 | ], 86 | 'docs' => [ 87 | [ 88 | '_source' => [ 89 | 'neos_asset' => $this->getAssetContent($asset) 90 | ] 91 | ] 92 | ] 93 | ]; 94 | 95 | $result = $this->elasticsearchClient->request('POST', '_ingest/pipeline/_simulate', [], json_encode($request))->getTreatedContent(); 96 | 97 | if (is_array($result)) { 98 | $extractedAsset = Arrays::getValueByPath($result, 'docs.0.doc._source.attachment'); 99 | } 100 | 101 | if (!is_array($extractedAsset)) { 102 | $this->logger->error(sprintf('Error while extracting fulltext data from file "%s". See Elasticsearch error log line for details.', $asset->getResource()->getFilename()), LogEnvironment::fromMethodName(__METHOD__)); 103 | } else { 104 | $this->logger->debug(sprintf('Extracted asset %s of type %s. Extracted %s characters of content', $asset->getResource()->getFilename(), $extractedAsset['content_type'] ?? '-no-content-type-', $extractedAsset['content_length'] ?? '0'), LogEnvironment::fromMethodName(__METHOD__)); 105 | } 106 | 107 | return $this->buildAssetContentObject($extractedAsset); 108 | } 109 | 110 | /** 111 | * @param AssetInterface $asset 112 | * @return string 113 | */ 114 | protected function getAssetContent(AssetInterface $asset): string 115 | { 116 | try { 117 | $stream = $asset->getResource()->getStream(); 118 | } catch (\Exception $e) { 119 | $message = $this->throwableStorage->logThrowable($e); 120 | $this->logger->error(sprintf('An exception occured while fetching resource with sha1 %s of asset %s. %s', $asset->getResource()->getSha1(), $asset->getResource()->getFilename(), $message), LogEnvironment::fromMethodName(__METHOD__)); 121 | return ''; 122 | } 123 | 124 | if ($stream === false) { 125 | $this->logger->error(sprintf('Could not get the file stream of resource with sha1 %s of asset %s.', $asset->getResource()->getSha1(), $asset->getResource()->getFilename()), LogEnvironment::fromMethodName(__METHOD__)); 126 | return ''; 127 | } 128 | 129 | stream_filter_append($stream, 'convert.base64-encode'); 130 | $result = stream_get_contents($stream); 131 | return $result !== false ? $result : ''; 132 | } 133 | 134 | /** 135 | * @param $extractedAsset 136 | * @return AssetContent 137 | */ 138 | protected function buildAssetContentObject(?array $extractedAsset): AssetContent 139 | { 140 | return new AssetContent( 141 | $extractedAsset['content'] ?? '', 142 | $extractedAsset['title'] ?? '', 143 | $extractedAsset['name'] ?? '', 144 | $extractedAsset['author'] ?? '', 145 | $extractedAsset['keywords'] ?? '', 146 | $extractedAsset['date'] ?? '', 147 | $extractedAsset['content_type'] ?? '', 148 | $extractedAsset['content_length'] ?? 0, 149 | $extractedAsset['language'] ?? '' 150 | ); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /Classes/Driver/QueryInterface.php: -------------------------------------------------------------------------------- 1 | request. 162 | * 163 | * Low-level method to manipulate the Elasticsearch Query 164 | * 165 | * @param string $path 166 | * @param array $data 167 | * @return void 168 | * @throws Exception\QueryBuildingException 169 | */ 170 | public function appendAtPath(string $path, array $data): void; 171 | 172 | /** 173 | * Modify a part of the Elasticsearch Request denoted by $path, merging together 174 | * the existing values and the passed-in values. 175 | * 176 | * @param string $path 177 | * @param mixed $requestPart 178 | * @return $this 179 | */ 180 | public function setByPath(string $path, $requestPart): QueryInterface; 181 | 182 | /** 183 | * @param array $request 184 | * @return void 185 | */ 186 | public function replaceRequest(array $request): void; 187 | } 188 | -------------------------------------------------------------------------------- /Classes/Driver/Version6/IndexerDriver.php: -------------------------------------------------------------------------------- 1 | isFulltextRoot($node)) { 47 | // for fulltext root documents, we need to preserve the "neos_fulltext" field. That's why we use the 48 | // "update" API instead of the "index" API, with a custom script internally; as we 49 | // shall not delete the "neos_fulltext" part of the document if it has any. 50 | return [ 51 | [ 52 | 'update' => [ 53 | '_id' => $document->getId(), 54 | '_index' => $indexName, 55 | 'retry_on_conflict' => $this->retryOnConflict ?: 3 56 | ] 57 | ], 58 | // https://www.elastic.co/guide/en/elasticsearch/reference/5.0/docs-update.html 59 | [ 60 | 'script' => [ 61 | 'lang' => 'painless', 62 | 'source' => ' 63 | HashMap fulltext = (ctx._source.containsKey("neos_fulltext") && ctx._source.neos_fulltext instanceof Map ? ctx._source.neos_fulltext : new HashMap()); 64 | HashMap fulltextParts = (ctx._source.containsKey("neos_fulltext_parts") && ctx._source.neos_fulltext_parts instanceof Map ? ctx._source.neos_fulltext_parts : new HashMap()); 65 | ctx._source = params.newData; 66 | ctx._source.neos_fulltext = fulltext; 67 | ctx._source.neos_fulltext_parts = fulltextParts', 68 | 'params' => [ 69 | 'newData' => $documentData 70 | ] 71 | ], 72 | 'upsert' => $documentData 73 | ] 74 | ]; 75 | } 76 | 77 | // non-fulltext-root documents can be indexed as-they-are 78 | return [ 79 | [ 80 | 'index' => [ 81 | '_id' => $document->getId(), 82 | '_index' => $indexName, 83 | 'retry_on_conflict' => $this->retryOnConflict ?: 3 84 | ] 85 | ], 86 | $documentData 87 | ]; 88 | } 89 | 90 | /** 91 | * {@inheritdoc} 92 | * @param Node $node 93 | * @param array $fulltextIndexOfNode 94 | * @param string|null $targetWorkspaceName 95 | * @return array 96 | */ 97 | public function fulltext(Node $node, array $fulltextIndexOfNode, ?WorkspaceName $targetWorkspaceName = null): array 98 | { 99 | $closestFulltextNode = $this->findClosestFulltextRoot($node); 100 | if ($closestFulltextNode === null) { 101 | return []; 102 | } 103 | 104 | $closestFulltextNodeDocumentIdentifier = $this->documentIdentifierGenerator->generate($closestFulltextNode, $targetWorkspaceName); 105 | 106 | $upsertFulltextParts = []; 107 | if (!empty($fulltextIndexOfNode)) { 108 | $upsertFulltextParts[$node->aggregateId->value] = $fulltextIndexOfNode; 109 | } 110 | 111 | return [ 112 | [ 113 | 'update' => [ 114 | '_id' => $closestFulltextNodeDocumentIdentifier, 115 | 'retry_on_conflict' => $this->retryOnConflict ?: 3 116 | ] 117 | ], 118 | // https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-update.html 119 | [ 120 | // first, update the neos_fulltext_parts, then re-generate the neos_fulltext from all neos_fulltext_parts 121 | 'script' => [ 122 | 'lang' => 'painless', 123 | 'source' => ' 124 | ctx._source.neos_fulltext = new HashMap(); 125 | if (!ctx._source.containsKey("neos_fulltext_parts") || !(ctx._source.neos_fulltext_parts instanceof Map)) { 126 | ctx._source.neos_fulltext_parts = new HashMap(); 127 | } 128 | 129 | if (params.nodeIsHidden || params.fulltext.size() == 0) { 130 | if (ctx._source.neos_fulltext_parts.containsKey(params.identifier)) { 131 | ctx._source.neos_fulltext_parts.remove(params.identifier); 132 | } 133 | } else { 134 | ctx._source.neos_fulltext_parts.put(params.identifier, params.fulltext); 135 | } 136 | 137 | for (fulltextPart in ctx._source.neos_fulltext_parts.entrySet()) { 138 | for (entry in fulltextPart.getValue().entrySet()) { 139 | def value = ""; 140 | if (ctx._source.neos_fulltext.containsKey(entry.getKey())) { 141 | value = ctx._source.neos_fulltext[entry.getKey()] + " " + entry.getValue().trim(); 142 | } else { 143 | value = entry.getValue().trim(); 144 | } 145 | ctx._source.neos_fulltext[entry.getKey()] = value; 146 | } 147 | }', 148 | 'params' => [ 149 | 'identifier' => $node->aggregateId->value, 150 | 'nodeIsHidden' => $node->tags->contain(SubtreeTag::disabled()), 151 | 'fulltext' => $fulltextIndexOfNode 152 | ], 153 | ], 154 | 'upsert' => [ 155 | 'neos_fulltext' => $fulltextIndexOfNode, 156 | 'neos_fulltext_parts' => $upsertFulltextParts 157 | ] 158 | ] 159 | ]; 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /Classes/Eel/ElasticSearchQueryResult.php: -------------------------------------------------------------------------------- 1 | elasticSearchQuery = $elasticSearchQuery; 46 | } 47 | 48 | /** 49 | * Initialize the results by really executing the query 50 | * 51 | * @return void 52 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 53 | * @throws \Flowpack\ElasticSearch\Exception 54 | * @throws \Neos\Flow\Http\Exception 55 | */ 56 | protected function initialize(): void 57 | { 58 | if ($this->result === null) { 59 | $queryBuilder = $this->elasticSearchQuery->getQueryBuilder(); 60 | $this->result = $queryBuilder->fetch(); 61 | $this->nodes = $this->result['nodes']; 62 | $this->count = $queryBuilder->getTotalItems(); 63 | } 64 | } 65 | 66 | /** 67 | * @return ElasticSearchQuery 68 | */ 69 | public function getQuery(): QueryInterface 70 | { 71 | return clone $this->elasticSearchQuery; 72 | } 73 | 74 | /** 75 | * {@inheritdoc} 76 | */ 77 | public function current() 78 | { 79 | $this->initialize(); 80 | 81 | return current($this->nodes); 82 | } 83 | 84 | /** 85 | * {@inheritdoc} 86 | */ 87 | public function next() 88 | { 89 | $this->initialize(); 90 | 91 | return next($this->nodes); 92 | } 93 | 94 | /** 95 | * {@inheritdoc} 96 | */ 97 | public function key() 98 | { 99 | $this->initialize(); 100 | 101 | return key($this->nodes); 102 | } 103 | 104 | /** 105 | * {@inheritdoc} 106 | */ 107 | public function valid() 108 | { 109 | $this->initialize(); 110 | 111 | return current($this->nodes) !== false; 112 | } 113 | 114 | /** 115 | * {@inheritdoc} 116 | */ 117 | public function rewind() 118 | { 119 | $this->initialize(); 120 | reset($this->nodes); 121 | } 122 | 123 | /** 124 | * {@inheritdoc} 125 | */ 126 | public function offsetExists($offset) 127 | { 128 | $this->initialize(); 129 | 130 | return isset($this->nodes[$offset]); 131 | } 132 | 133 | /** 134 | * {@inheritdoc} 135 | */ 136 | public function offsetGet($offset) 137 | { 138 | $this->initialize(); 139 | 140 | return $this->nodes[$offset]; 141 | } 142 | 143 | /** 144 | * {@inheritdoc} 145 | */ 146 | public function offsetSet($offset, $value) 147 | { 148 | $this->initialize(); 149 | $this->nodes[$offset] = $value; 150 | } 151 | 152 | /** 153 | * {@inheritdoc} 154 | */ 155 | public function offsetUnset($offset) 156 | { 157 | $this->initialize(); 158 | unset($this->nodes[$offset]); 159 | } 160 | 161 | /** 162 | * {@inheritdoc} 163 | */ 164 | public function getFirst() 165 | { 166 | $this->initialize(); 167 | if (count($this->nodes) > 0) { 168 | return array_values($this->nodes)[0]; 169 | } 170 | } 171 | 172 | /** 173 | * {@inheritdoc} 174 | */ 175 | public function toArray(): array 176 | { 177 | $this->initialize(); 178 | 179 | return $this->nodes; 180 | } 181 | 182 | /** 183 | * {@inheritdoc} 184 | * @throws \Flowpack\ElasticSearch\Exception 185 | */ 186 | public function count() 187 | { 188 | if ($this->count === null) { 189 | $this->count = $this->elasticSearchQuery->getQueryBuilder()->count(); 190 | } 191 | 192 | return $this->count; 193 | } 194 | 195 | /** 196 | * @return int the current number of results which can be iterated upon 197 | * @api 198 | */ 199 | public function getAccessibleCount(): int 200 | { 201 | $this->initialize(); 202 | 203 | return count($this->nodes); 204 | } 205 | 206 | /** 207 | * @return array 208 | */ 209 | public function getAggregations(): array 210 | { 211 | $this->initialize(); 212 | if (array_key_exists('aggregations', $this->result)) { 213 | return $this->result['aggregations']; 214 | } 215 | 216 | return []; 217 | } 218 | 219 | /** 220 | * Returns an array of type 221 | * [ 222 | * => [ 223 | * 'text' => 224 | * 'options' => [ 225 | * [ 226 | * 'text' => 227 | * 'score' => 228 | * ], 229 | * [ 230 | * ... 231 | * ] 232 | * ] 233 | * ] 234 | * ] 235 | * 236 | * @return array 237 | */ 238 | public function getSuggestions(): array 239 | { 240 | $this->initialize(); 241 | if (array_key_exists('suggest', $this->result)) { 242 | return $this->result['suggest']; 243 | } 244 | 245 | return []; 246 | } 247 | 248 | /** 249 | * Returns the Elasticsearch "hit" (e.g. the raw content being transferred back from Elasticsearch) 250 | * for the given node. 251 | * 252 | * Can be used for example to access highlighting information. 253 | * 254 | * @param Node $node 255 | * @return array the Elasticsearch hit, or NULL if it does not exist. 256 | * @api 257 | */ 258 | public function searchHitForNode(Node $node): ?array 259 | { 260 | return $this->elasticSearchQuery->getQueryBuilder()->getFullElasticSearchHitForNode($node); 261 | } 262 | 263 | /** 264 | * Returns the array with all sort values for a given node. The values are fetched from the raw content 265 | * Elasticsearch returns within the hit data 266 | * 267 | * @param Node $node 268 | * @return array 269 | */ 270 | public function getSortValuesForNode(Node $node): array 271 | { 272 | $hit = $this->searchHitForNode($node); 273 | if (is_array($hit) && array_key_exists('sort', $hit)) { 274 | return $hit['sort']; 275 | } 276 | 277 | return []; 278 | } 279 | 280 | /** 281 | * @param string $methodName 282 | * @return boolean 283 | */ 284 | public function allowsCallOfMethod($methodName) 285 | { 286 | return true; 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU LESSER GENERAL PUBLIC LICENSE 2 | Version 3, 29 June 2007 3 | 4 | Copyright (C) 2007 Free Software Foundation, Inc. 5 | Everyone is permitted to copy and distribute verbatim copies 6 | of this license document, but changing it is not allowed. 7 | 8 | 9 | This version of the GNU Lesser General Public License incorporates 10 | the terms and conditions of version 3 of the GNU General Public 11 | License, supplemented by the additional permissions listed below. 12 | 13 | 0. Additional Definitions. 14 | 15 | As used herein, "this License" refers to version 3 of the GNU Lesser 16 | General Public License, and the "GNU GPL" refers to version 3 of the GNU 17 | General Public License. 18 | 19 | "The Library" refers to a covered work governed by this License, 20 | other than an Application or a Combined Work as defined below. 21 | 22 | An "Application" is any work that makes use of an interface provided 23 | by the Library, but which is not otherwise based on the Library. 24 | Defining a subclass of a class defined by the Library is deemed a mode 25 | of using an interface provided by the Library. 26 | 27 | A "Combined Work" is a work produced by combining or linking an 28 | Application with the Library. The particular version of the Library 29 | with which the Combined Work was made is also called the "Linked 30 | Version". 31 | 32 | The "Minimal Corresponding Source" for a Combined Work means the 33 | Corresponding Source for the Combined Work, excluding any source code 34 | for portions of the Combined Work that, considered in isolation, are 35 | based on the Application, and not on the Linked Version. 36 | 37 | The "Corresponding Application Code" for a Combined Work means the 38 | object code and/or source code for the Application, including any data 39 | and utility programs needed for reproducing the Combined Work from the 40 | Application, but excluding the System Libraries of the Combined Work. 41 | 42 | 1. Exception to Section 3 of the GNU GPL. 43 | 44 | You may convey a covered work under sections 3 and 4 of this License 45 | without being bound by section 3 of the GNU GPL. 46 | 47 | 2. Conveying Modified Versions. 48 | 49 | If you modify a copy of the Library, and, in your modifications, a 50 | facility refers to a function or data to be supplied by an Application 51 | that uses the facility (other than as an argument passed when the 52 | facility is invoked), then you may convey a copy of the modified 53 | version: 54 | 55 | a) under this License, provided that you make a good faith effort to 56 | ensure that, in the event an Application does not supply the 57 | function or data, the facility still operates, and performs 58 | whatever part of its purpose remains meaningful, or 59 | 60 | b) under the GNU GPL, with none of the additional permissions of 61 | this License applicable to that copy. 62 | 63 | 3. Object Code Incorporating Material from Library Header Files. 64 | 65 | The object code form of an Application may incorporate material from 66 | a header file that is part of the Library. You may convey such object 67 | code under terms of your choice, provided that, if the incorporated 68 | material is not limited to numerical parameters, data structure 69 | layouts and accessors, or small macros, inline functions and templates 70 | (ten or fewer lines in length), you do both of the following: 71 | 72 | a) Give prominent notice with each copy of the object code that the 73 | Library is used in it and that the Library and its use are 74 | covered by this License. 75 | 76 | b) Accompany the object code with a copy of the GNU GPL and this license 77 | document. 78 | 79 | 4. Combined Works. 80 | 81 | You may convey a Combined Work under terms of your choice that, 82 | taken together, effectively do not restrict modification of the 83 | portions of the Library contained in the Combined Work and reverse 84 | engineering for debugging such modifications, if you also do each of 85 | the following: 86 | 87 | a) Give prominent notice with each copy of the Combined Work that 88 | the Library is used in it and that the Library and its use are 89 | covered by this License. 90 | 91 | b) Accompany the Combined Work with a copy of the GNU GPL and this license 92 | document. 93 | 94 | c) For a Combined Work that displays copyright notices during 95 | execution, include the copyright notice for the Library among 96 | these notices, as well as a reference directing the user to the 97 | copies of the GNU GPL and this license document. 98 | 99 | d) Do one of the following: 100 | 101 | 0) Convey the Minimal Corresponding Source under the terms of this 102 | License, and the Corresponding Application Code in a form 103 | suitable for, and under terms that permit, the user to 104 | recombine or relink the Application with a modified version of 105 | the Linked Version to produce a modified Combined Work, in the 106 | manner specified by section 6 of the GNU GPL for conveying 107 | Corresponding Source. 108 | 109 | 1) Use a suitable shared library mechanism for linking with the 110 | Library. A suitable mechanism is one that (a) uses at run time 111 | a copy of the Library already present on the user's computer 112 | system, and (b) will operate properly with a modified version 113 | of the Library that is interface-compatible with the Linked 114 | Version. 115 | 116 | e) Provide Installation Information, but only if you would otherwise 117 | be required to provide such information under section 6 of the 118 | GNU GPL, and only to the extent that such information is 119 | necessary to install and execute a modified version of the 120 | Combined Work produced by recombining or relinking the 121 | Application with a modified version of the Linked Version. (If 122 | you use option 4d0, the Installation Information must accompany 123 | the Minimal Corresponding Source and Corresponding Application 124 | Code. If you use option 4d1, you must provide the Installation 125 | Information in the manner specified by section 6 of the GNU GPL 126 | for conveying Corresponding Source.) 127 | 128 | 5. Combined Libraries. 129 | 130 | You may place library facilities that are a work based on the 131 | Library side by side in a single library together with other library 132 | facilities that are not Applications and are not covered by this 133 | License, and convey such a combined library under terms of your 134 | choice, if you do both of the following: 135 | 136 | a) Accompany the combined library with a copy of the same work based 137 | on the Library, uncombined with any other library facilities, 138 | conveyed under the terms of this License. 139 | 140 | b) Give prominent notice with the combined library that part of it 141 | is a work based on the Library, and explaining where to find the 142 | accompanying uncombined form of the same work. 143 | 144 | 6. Revised Versions of the GNU Lesser General Public License. 145 | 146 | The Free Software Foundation may publish revised and/or new versions 147 | of the GNU Lesser General Public License from time to time. Such new 148 | versions will be similar in spirit to the present version, but may 149 | differ in detail to address new problems or concerns. 150 | 151 | Each version is given a distinguishing version number. If the 152 | Library as you received it specifies that a certain numbered version 153 | of the GNU Lesser General Public License "or any later version" 154 | applies to it, you have the option of following the terms and 155 | conditions either of that published version or of any later version 156 | published by the Free Software Foundation. If the Library as you 157 | received it does not specify a version number of the GNU Lesser 158 | General Public License, you may choose any version of the GNU Lesser 159 | General Public License ever published by the Free Software Foundation. 160 | 161 | If the Library as you received it specifies that a proxy can decide 162 | whether future versions of the GNU Lesser General Public License shall 163 | apply, that proxy's public statement of acceptance of any version is 164 | permanent authorization for you to choose that version for the 165 | Library. 166 | -------------------------------------------------------------------------------- /Classes/Eel/ElasticSearchQuery.php: -------------------------------------------------------------------------------- 1 | getQuery(), so that pagination 27 | * widgets etc work in the same manner for Elasticsearch results. 28 | */ 29 | class ElasticSearchQuery implements QueryInterface 30 | { 31 | /** 32 | * @var ElasticSearchQueryBuilder 33 | */ 34 | protected $queryBuilder; 35 | 36 | /** 37 | * @var array 38 | */ 39 | protected static $runtimeQueryResultCache; 40 | 41 | /** 42 | * ElasticSearchQuery constructor. 43 | * 44 | * @param ElasticSearchQueryBuilder $elasticSearchQueryBuilder 45 | */ 46 | public function __construct(ElasticSearchQueryBuilder $elasticSearchQueryBuilder) 47 | { 48 | $this->queryBuilder = $elasticSearchQueryBuilder; 49 | } 50 | 51 | /** 52 | * Executes the query and returns the result. 53 | * 54 | * @param bool $cacheResult If the result cache should be used 55 | * @return ElasticSearchQueryResult The query result 56 | * @throws Exception 57 | * @throws Exception\ConfigurationException 58 | * @throws JsonException 59 | * @api 60 | */ 61 | public function execute($cacheResult = false): QueryResultInterface 62 | { 63 | $queryHash = md5($this->queryBuilder->getIndexName() . json_encode($this->queryBuilder->getRequest(), JSON_THROW_ON_ERROR)); 64 | if ($cacheResult === true && isset(self::$runtimeQueryResultCache[$queryHash])) { 65 | return self::$runtimeQueryResultCache[$queryHash]; 66 | } 67 | $queryResult = new ElasticSearchQueryResult($this); 68 | self::$runtimeQueryResultCache[$queryHash] = $queryResult; 69 | 70 | return $queryResult; 71 | } 72 | 73 | /** 74 | * {@inheritdoc} 75 | */ 76 | public function count(): int 77 | { 78 | return $this->queryBuilder->getTotalItems(); 79 | } 80 | 81 | /** 82 | * @param int|null $limit 83 | * @return QueryInterface 84 | * @throws IllegalObjectTypeException 85 | */ 86 | public function setLimit(?int $limit): QueryInterface 87 | { 88 | if ($limit < 1 || !is_int($limit)) { 89 | throw new InvalidArgumentException('Expecting integer greater than zero for limit'); 90 | } 91 | 92 | $this->queryBuilder->limit($limit); 93 | return $this; 94 | } 95 | 96 | /** 97 | * {@inheritdoc} 98 | */ 99 | public function getLimit(): int 100 | { 101 | return $this->queryBuilder->getLimit(); 102 | } 103 | 104 | /** 105 | * {@inheritdoc} 106 | */ 107 | public function setOffset(?int $offset): QueryInterface 108 | { 109 | if ($offset < 1 || !is_int($offset)) { 110 | throw new InvalidArgumentException('Expecting integer greater than zero for offset', 1605474906); 111 | } 112 | 113 | $this->queryBuilder->from($offset); 114 | return $this; 115 | } 116 | 117 | /** 118 | * {@inheritdoc} 119 | */ 120 | public function getOffset(): int 121 | { 122 | return $this->queryBuilder->getFrom(); 123 | } 124 | 125 | /** 126 | * {@inheritdoc} 127 | */ 128 | public function getType(): string 129 | { 130 | return Node::class; 131 | } 132 | 133 | /** 134 | * {@inheritdoc} 135 | * @throws Exception 136 | */ 137 | public function setOrderings(array $orderings): QueryInterface 138 | { 139 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749035); 140 | } 141 | 142 | /** 143 | * {@inheritdoc} 144 | * @throws Exception 145 | */ 146 | public function getOrderings(): array 147 | { 148 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749036); 149 | } 150 | 151 | /** 152 | * {@inheritdoc} 153 | * @throws Exception 154 | */ 155 | public function matching($constraint): QueryInterface 156 | { 157 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749037); 158 | } 159 | 160 | /** 161 | * {@inheritdoc} 162 | * @throws Exception 163 | */ 164 | public function getConstraint() 165 | { 166 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749038); 167 | } 168 | 169 | /** 170 | * {@inheritdoc} 171 | * @throws Exception 172 | */ 173 | public function logicalAnd(mixed $constraint1, mixed ...$constraints) 174 | { 175 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749039); 176 | } 177 | 178 | /** 179 | * {@inheritdoc} 180 | * @throws Exception 181 | */ 182 | public function logicalOr(mixed $constraint1, mixed ...$constraints) 183 | { 184 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749040); 185 | } 186 | 187 | /** 188 | * {@inheritdoc} 189 | * @throws Exception 190 | */ 191 | public function logicalNot($constraint) 192 | { 193 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749041); 194 | } 195 | 196 | /** 197 | * {@inheritdoc} 198 | * @throws Exception 199 | */ 200 | public function equals($propertyName, $operand, $caseSensitive = true) 201 | { 202 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749042); 203 | } 204 | 205 | /** 206 | * {@inheritdoc} 207 | * @throws Exception 208 | */ 209 | public function like($propertyName, $operand, $caseSensitive = true) 210 | { 211 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749043); 212 | } 213 | 214 | /** 215 | * {@inheritdoc} 216 | * @throws Exception 217 | */ 218 | public function contains($propertyName, $operand) 219 | { 220 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749044); 221 | } 222 | 223 | /** 224 | * {@inheritdoc} 225 | * @throws Exception 226 | */ 227 | public function isEmpty($propertyName) 228 | { 229 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749045); 230 | } 231 | 232 | /** 233 | * {@inheritdoc} 234 | * @throws Exception 235 | */ 236 | public function in($propertyName, $operand) 237 | { 238 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749046); 239 | } 240 | 241 | /** 242 | * {@inheritdoc} 243 | * @throws Exception 244 | */ 245 | public function lessThan($propertyName, $operand) 246 | { 247 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749047); 248 | } 249 | 250 | /** 251 | * {@inheritdoc} 252 | * @throws Exception 253 | */ 254 | public function lessThanOrEqual($propertyName, $operand) 255 | { 256 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749048); 257 | } 258 | 259 | /** 260 | * {@inheritdoc} 261 | * @throws Exception 262 | */ 263 | public function greaterThan($propertyName, $operand) 264 | { 265 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749049); 266 | } 267 | 268 | /** 269 | * {@inheritdoc} 270 | * @throws Exception 271 | */ 272 | public function greaterThanOrEqual($propertyName, $operand) 273 | { 274 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749050); 275 | } 276 | 277 | /** 278 | * {@inheritdoc} 279 | * @throws Exception 280 | */ 281 | public function setDistinct(bool $distinct = true): QueryInterface 282 | { 283 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749051); 284 | } 285 | 286 | /** 287 | * {@inheritdoc} 288 | * @throws Exception 289 | */ 290 | public function isDistinct(): bool 291 | { 292 | throw new Exception(__FUNCTION__ . ' not implemented', 1421749052); 293 | } 294 | 295 | /** 296 | * @return ElasticSearchQueryBuilder 297 | */ 298 | public function getQueryBuilder(): ElasticSearchQueryBuilder 299 | { 300 | return $this->queryBuilder; 301 | } 302 | } 303 | -------------------------------------------------------------------------------- /Tests/Functional/Indexer/NodeIndexerTest.php: -------------------------------------------------------------------------------- 1 | nodeIndexer = $this->objectManager->get(NodeIndexer::class); 58 | $this->dimensionService = $this->objectManager->get(DimensionsService::class); 59 | $this->nodeTypeMappingBuilder = $this->objectManager->get(NodeTypeMappingBuilderInterface::class); 60 | } 61 | 62 | /** 63 | * @test 64 | */ 65 | public function getIndexWithoutDimensionConfigured(): void 66 | { 67 | $this->nodeIndexer->setIndexNamePostfix(''); 68 | $this->nodeIndexer->setDimensions([]); 69 | $index = $this->nodeIndexer->getIndex(); 70 | static::assertEquals(self::TESTING_INDEX_PREFIX . '-default', $index->getName()); 71 | } 72 | 73 | /** 74 | * @test 75 | * 76 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 77 | * @throws ConfigurationException 78 | * @throws Exception 79 | */ 80 | public function getIndexForDimensionConfiguration(): void 81 | { 82 | $dimensionValues = ['language' => ['de']]; 83 | $this->nodeIndexer->setDimensions($dimensionValues); 84 | $index = $this->nodeIndexer->getIndex(); 85 | $dimesionHash = $this->dimensionService->hash($dimensionValues); 86 | 87 | static::assertEquals(self::TESTING_INDEX_PREFIX . '-' . $dimesionHash, $index->getName()); 88 | } 89 | 90 | /** 91 | * @test 92 | */ 93 | public function updateIndexAlias(): void 94 | { 95 | $dimensionValues = ['language' => ['de']]; 96 | $this->nodeIndexer->setDimensions($dimensionValues); 97 | $this->nodeIndexer->setIndexNamePostfix((string)time()); 98 | $this->nodeIndexer->getIndex()->create(); 99 | 100 | $this->assertIndexExists($this->nodeIndexer->getIndexName()); 101 | $this->nodeIndexer->updateIndexAlias(); 102 | 103 | $this->assertAliasesEquals(self::TESTING_INDEX_PREFIX, [$this->nodeIndexer->getIndexName()]); 104 | } 105 | 106 | /** 107 | * @test 108 | */ 109 | public function indexAndDeleteNode(): void 110 | { 111 | $testNode = $this->setupCrAndIndexTestNode(); 112 | self::assertTrue($this->nodeExistsInIndex($testNode), 'Node was not successfully indexed.'); 113 | 114 | $this->nodeIndexer->removeNode($testNode); 115 | $this->nodeIndexer->flush(); 116 | sleep(1); 117 | self::assertFalse($this->nodeExistsInIndex($testNode), 'Node still exists after delete'); 118 | } 119 | 120 | /** 121 | * @test 122 | */ 123 | public function nodeMoveIsHandledCorrectly(): void 124 | { 125 | $testNode = $this->setupCrAndIndexTestNode(); 126 | self::assertTrue($this->nodeExistsInIndex($testNode), 'Node was not successfully indexed.'); 127 | 128 | $testNode2 = $this->siteNode->getNode('test-node-2'); 129 | 130 | // move this node (test-node-1) into test-node-2 131 | $testNode->setProperty('title', 'changed'); 132 | $testNode->moveInto($testNode2); 133 | 134 | // re-index 135 | $this->nodeIndexer->indexNode($testNode); 136 | $this->nodeIndexer->flush(); 137 | sleep(1); 138 | 139 | // check if we do have more than one single occurrence (nodeExistsInIndex will check that indirectly) 140 | self::assertTrue($this->nodeExistsInIndex($testNode), 'Node was not successfully indexed.'); 141 | 142 | // check the node path in es after indexing 143 | $pathInEs = $this->getNeosPathOfNodeInIndex($testNode); 144 | self::assertNotNull($pathInEs, 'Node does not exist after indexing'); 145 | self::assertEquals($pathInEs, $testNode->getPath(), 'Wrong node path in elasticsearch after indexing'); 146 | } 147 | 148 | /** 149 | * Fetch the node path (stored in elasticsearch) of the given node 150 | */ 151 | private function getNeosPathOfNodeInIndex(NodeInterface $node): ?string 152 | { 153 | $this->searchClient->setContextNode($this->siteNode); 154 | /** @var FilteredQuery $query */ 155 | $query = $this->objectManager->get(QueryInterface::class); 156 | $query->queryFilter('term', ['neos_node_identifier' => $node->getIdentifier()]); 157 | 158 | $result = $this->nodeIndexer->getIndex()->request('GET', '/_search', [], $query->toArray())->getTreatedContent(); 159 | 160 | $firstHit = current(Arrays::getValueByPath($result, 'hits.hits')); 161 | 162 | if ($firstHit === false) { 163 | return null; 164 | } 165 | 166 | return Arrays::getValueByPath($firstHit, '_source.neos_path'); 167 | } 168 | 169 | /** 170 | * @param NodeInterface $testNode 171 | * @return bool 172 | * @throws ConfigurationException 173 | * @throws Exception 174 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 175 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception\QueryBuildingException 176 | * @throws \Neos\Flow\Http\Exception 177 | */ 178 | private function nodeExistsInIndex(NodeInterface $testNode): bool 179 | { 180 | $this->searchClient->setContextNode($this->siteNode); 181 | /** @var FilteredQuery $query */ 182 | $query = $this->objectManager->get(QueryInterface::class); 183 | $query->queryFilter('term', ['neos_node_identifier' => $testNode->getIdentifier()]); 184 | 185 | $result = $this->nodeIndexer->getIndex()->request('GET', '/_search', [], $query->toArray())->getTreatedContent(); 186 | return count(Arrays::getValueByPath($result, 'hits.hits')) === 1; 187 | } 188 | 189 | /** 190 | * @return NodeInterface 191 | * @throws ConfigurationException 192 | * @throws Exception 193 | * @throws \Flowpack\ElasticSearch\ContentRepositoryAdaptor\Exception 194 | * @throws \Neos\Flow\Http\Exception 195 | */ 196 | protected function setupCrAndIndexTestNode(): NodeInterface 197 | { 198 | $this->setupContentRepository(); 199 | $this->createNodesForNodeSearchTest(); 200 | /** @var NodeInterface $testNode */ 201 | $testNode = current($this->siteNode->getChildNodes('Flowpack.ElasticSearch.ContentRepositoryAdaptor:Document', 1)); 202 | 203 | $this->nodeIndexer->setDimensions($testNode->getDimensions()); 204 | $this->nodeIndexer->getIndex()->create(); 205 | 206 | $nodeTypeMappingCollection = $this->nodeTypeMappingBuilder->buildMappingInformation($this->nodeIndexer->getIndex()); 207 | foreach ($nodeTypeMappingCollection as $mapping) { 208 | /** @var Mapping $mapping */ 209 | $mapping->apply(); 210 | } 211 | 212 | $this->nodeIndexer->indexNode($testNode); 213 | $this->nodeIndexer->flush(); 214 | sleep(1); 215 | return $testNode; 216 | } 217 | } 218 | -------------------------------------------------------------------------------- /Classes/Driver/AbstractQuery.php: -------------------------------------------------------------------------------- 1 | request = $request; 54 | $this->unsupportedFieldsInCountRequest = $unsupportedFieldsInCountRequest; 55 | $this->queryStringParameters = $queryStringParameters; 56 | } 57 | 58 | /** 59 | * Modify a part of the Elasticsearch Request denoted by $path, merging together 60 | * the existing values and the passed-in values. 61 | * 62 | * @param string $path 63 | * @param mixed $requestPart 64 | * @return AbstractQuery 65 | */ 66 | public function setByPath(string $path, $requestPart): QueryInterface 67 | { 68 | $valueAtPath = Arrays::getValueByPath($this->request, $path); 69 | if (is_array($valueAtPath)) { 70 | $result = Arrays::arrayMergeRecursiveOverrule($valueAtPath, $requestPart); 71 | } else { 72 | $result = $requestPart; 73 | } 74 | 75 | $this->request = Arrays::setValueByPath($this->request, $path, $result); 76 | 77 | return $this; 78 | } 79 | 80 | /** 81 | * {@inheritdoc} 82 | */ 83 | public function toArray(): array 84 | { 85 | return $this->prepareRequest(); 86 | } 87 | 88 | /** 89 | * {@inheritdoc} 90 | * @throws \JsonException 91 | */ 92 | public function getRequestAsJson(): string 93 | { 94 | return json_encode($this); 95 | } 96 | 97 | /** 98 | * {@inheritdoc} 99 | */ 100 | public function addSortFilter(array $configuration): void 101 | { 102 | if (!isset($this->request['sort'])) { 103 | $this->request['sort'] = []; 104 | } 105 | $this->request['sort'][] = $configuration; 106 | } 107 | 108 | /** 109 | * {@inheritdoc} 110 | */ 111 | public function aggregation(string $name, array $aggregationDefinition, string $parentPath = ''): void 112 | { 113 | if (!array_key_exists('aggregations', $this->request)) { 114 | $this->request['aggregations'] = []; 115 | } 116 | 117 | if ($parentPath !== '') { 118 | $this->addSubAggregation($parentPath, $name, $aggregationDefinition); 119 | } else { 120 | $this->request['aggregations'][$name] = $aggregationDefinition; 121 | } 122 | } 123 | 124 | /** 125 | * {@inheritdoc} 126 | * @throws Exception\QueryBuildingException 127 | */ 128 | public function moreLikeThis(array $like, array $fields = [], array $options = []): void 129 | { 130 | $moreLikeThis = $options; 131 | $moreLikeThis['like'] = $like; 132 | 133 | if (!empty($fields)) { 134 | $moreLikeThis['fields'] = $fields; 135 | } 136 | 137 | $this->appendAtPath('query.bool.filter.bool.must', ['more_like_this' => $moreLikeThis]); 138 | } 139 | 140 | /** 141 | * This is an low level method for internal usage. 142 | * 143 | * You can add a custom $aggregationConfiguration under a given $parentPath. The $parentPath foo.bar would 144 | * insert your $aggregationConfiguration under 145 | * $this->request['aggregations']['foo']['aggregations']['bar']['aggregations'][$name] 146 | * 147 | * @param string $parentPath The parent path to add the sub aggregation to 148 | * @param string $name The name to identify the resulting aggregation 149 | * @param array $aggregationConfiguration 150 | * @return void 151 | * @throws Exception\QueryBuildingException 152 | */ 153 | protected function addSubAggregation(string $parentPath, string $name, array $aggregationConfiguration): void 154 | { 155 | // Find the parentPath 156 | $path =& $this->request['aggregations']; 157 | 158 | foreach (explode('.', $parentPath) as $subPart) { 159 | if ($path === null || !array_key_exists($subPart, $path)) { 160 | throw new Exception\QueryBuildingException(sprintf('The parent path segment "%s" could not be found when adding a sub aggregation to parent path "%s"', $subPart, $parentPath)); 161 | } 162 | $path =& $path[$subPart]['aggregations']; 163 | } 164 | 165 | $path[$name] = $aggregationConfiguration; 166 | } 167 | 168 | /** 169 | * {@inheritdoc} 170 | */ 171 | public function suggestions(string $name, array $suggestionDefinition): void 172 | { 173 | if (!array_key_exists('suggest', $this->request)) { 174 | $this->request['suggest'] = []; 175 | } 176 | 177 | $this->request['suggest'][$name] = $suggestionDefinition; 178 | } 179 | 180 | /** 181 | * {@inheritdoc} 182 | */ 183 | public function highlight($fragmentSize, ?int $fragmentCount = null, int $noMatchSize = 150, string $field = 'neos_fulltext.*'): void 184 | { 185 | if ($fragmentSize === false) { 186 | // Highlighting is disabled. 187 | unset($this->request['highlight']); 188 | return; 189 | } 190 | 191 | $this->request['highlight']['fields'][$field] = [ 192 | 'fragment_size' => $fragmentSize, 193 | 'no_match_size' => $noMatchSize, 194 | 'number_of_fragments' => $fragmentCount 195 | ]; 196 | } 197 | 198 | /** 199 | * {@inheritdoc} 200 | */ 201 | public function setValueByPath(string $path, $value): void 202 | { 203 | $this->request = Arrays::setValueByPath($this->request, $path, $value); 204 | } 205 | 206 | /** 207 | * {@inheritdoc} 208 | */ 209 | public function appendAtPath(string $path, array $data): void 210 | { 211 | $currentElement =& $this->request; 212 | foreach (explode('.', $path) as $pathPart) { 213 | if (!isset($currentElement[$pathPart])) { 214 | throw new Exception\QueryBuildingException('The element at path "' . $path . '" was not an array (failed at "' . $pathPart . '").', 1383716367); 215 | } 216 | $currentElement =& $currentElement[$pathPart]; 217 | } 218 | $currentElement[] = $data; 219 | } 220 | 221 | /** 222 | * {@inheritdoc} 223 | */ 224 | public function jsonSerialize() 225 | { 226 | return $this->prepareRequest(); 227 | } 228 | 229 | /** 230 | * {@inheritdoc} 231 | */ 232 | public function offsetSet($offset, $value) 233 | { 234 | if ($offset === null) { 235 | $this->request[] = $value; 236 | } else { 237 | $this->request[$offset] = $value; 238 | } 239 | } 240 | 241 | /** 242 | * {@inheritdoc} 243 | */ 244 | public function offsetExists($offset) 245 | { 246 | return isset($this->request[$offset]); 247 | } 248 | 249 | /** 250 | * {@inheritdoc} 251 | */ 252 | public function offsetUnset($offset) 253 | { 254 | unset($this->request[$offset]); 255 | } 256 | 257 | /** 258 | * {@inheritdoc} 259 | */ 260 | public function offsetGet($offset) 261 | { 262 | return $this->request[$offset] ?? null; 263 | } 264 | 265 | /** 266 | * Prepare the final request array 267 | * 268 | * This method is useful if you extend the current query implementation. 269 | * 270 | * @return array 271 | */ 272 | protected function prepareRequest(): array 273 | { 274 | return $this->request; 275 | } 276 | 277 | /** 278 | * All methods are considered safe 279 | * 280 | * @param string $methodName 281 | * @return boolean 282 | */ 283 | public function allowsCallOfMethod($methodName) 284 | { 285 | return true; 286 | } 287 | 288 | /** 289 | * @param array $request 290 | */ 291 | public function replaceRequest(array $request): void 292 | { 293 | $this->request = $request; 294 | } 295 | } 296 | -------------------------------------------------------------------------------- /Classes/Command/SearchCommandController.php: -------------------------------------------------------------------------------- 1 | outputLine('Error: The Dimensions must be given as a JSON array like \'{"language":["de"]}\''); 71 | $this->sendAndExit(1); 72 | } 73 | 74 | $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); 75 | $dimensionSpacePoint = $dimensions ? DimensionSpacePoint::fromJsonString($dimensions) : DimensionSpacePoint::createWithoutDimensions(); 76 | 77 | $contentGraph = $this->contentRepositoryRegistry->get($contentRepositoryId)->getContentGraph(WorkspaceName::forLive()); 78 | $subgraph = $contentGraph->getSubgraph($dimensionSpacePoint, VisibilityConstraints::withoutRestrictions()); 79 | 80 | if ($nodeAggregateId !== null) { 81 | $contextNode = $subgraph->findNodeById(NodeAggregateId::fromString($nodeAggregateId)); 82 | } else { 83 | $contextNode = $subgraph->findRootNodeByType(NodeTypeNameFactory::forSites()); 84 | } 85 | 86 | if ($contextNode === null) { 87 | $this->outputLine('Context node not found'); 88 | $this->sendAndExit(1); 89 | } 90 | 91 | $queryBuilder = new ElasticSearchQueryBuilder(); 92 | $queryBuilder = $queryBuilder->query($contextNode) 93 | ->fulltext($searchWord) 94 | ->limit(10) 95 | ->termSuggestions($searchWord) 96 | ->log(__CLASS__); 97 | 98 | /** @var ElasticSearchQueryResult $results */ 99 | $results = $queryBuilder->execute(); 100 | 101 | $didYouMean = (new SearchResultHelper())->didYouMean($results); 102 | if (trim($didYouMean) !== '') { 103 | $this->outputLine('Did you mean %s', [$didYouMean]); 104 | } 105 | 106 | $this->outputLine(); 107 | $this->outputLine('Results'); 108 | $this->outputLine('Number of result(s): %d', [$queryBuilder->count()]); 109 | $this->outputLine('Index name: %s', [$this->elasticSearchClient->getIndexName()]); 110 | $this->outputResults($results); 111 | $this->outputLine(); 112 | } 113 | 114 | /** 115 | * Prints the index content of the given node identifier. 116 | * 117 | * @param string $identifier The node identifier 118 | * @param string $contentRepository 119 | * @param string|null $dimensions Dimensions, specified in JSON format, like '{"language":"en"}' 120 | * @param string $field Name or path to a source field to display. Eg. "__fulltext.h1" 121 | * @throws Exception 122 | * @throws IllegalObjectTypeException 123 | * @throws QueryBuildingException 124 | * @throws \Flowpack\ElasticSearch\Exception 125 | * @throws \Neos\Flow\Http\Exception 126 | */ 127 | public function viewNodeCommand(string $identifier, string $contentRepository = 'default', ?string $dimensions = null, string $field = ''): void 128 | { 129 | if ($dimensions !== null && is_array(json_decode($dimensions, true)) === false) { 130 | $this->outputLine('Error: The Dimensions must be given as a JSON array like \'{"language":["de"]}\''); 131 | $this->sendAndExit(1); 132 | } 133 | 134 | $contentRepositoryId = ContentRepositoryId::fromString($contentRepository); 135 | $dimensionSpacePoint = $dimensions ? DimensionSpacePoint::fromJsonString($dimensions) : DimensionSpacePoint::createWithoutDimensions(); 136 | 137 | $contentGraph = $this->contentRepositoryRegistry->get($contentRepositoryId)->getContentGraph(WorkspaceName::forLive()); 138 | $subgraph = $contentGraph->getSubgraph($dimensionSpacePoint, VisibilityConstraints::withoutRestrictions()); 139 | 140 | $rootNode = $subgraph->findRootNodeByType(NodeTypeNameFactory::forSites()); 141 | 142 | if ($rootNode === null) { 143 | $this->outputLine('Error: No root node found for the given dimensions'); 144 | return; 145 | } 146 | 147 | $queryBuilder = new ElasticSearchQueryBuilder(); 148 | $queryBuilder->query($rootNode); 149 | $queryBuilder->exactMatch('neos_node_identifier', $identifier); 150 | 151 | $queryBuilder->getRequest()->setValueByPath('_source', []); 152 | 153 | if ($queryBuilder->count() > 0) { 154 | $this->outputLine(); 155 | $this->outputLine('Results'); 156 | 157 | foreach ($queryBuilder->execute() as $node) { 158 | $this->outputLine('%s', [(string)$node->aggregateId->value]); 159 | $data = $queryBuilder->getFullElasticSearchHitForNode($node); 160 | 161 | if ($field !== '') { 162 | $data = Arrays::getValueByPath($data, '_source.' . $field); 163 | } 164 | 165 | $this->outputLine(print_r($data, true)); 166 | $this->outputLine(); 167 | } 168 | } else { 169 | $this->outputLine(); 170 | $this->outputLine('No document matching the given node identifier'); 171 | } 172 | } 173 | 174 | /** 175 | * @param ElasticSearchQueryResult $result 176 | */ 177 | private function outputResults(ElasticSearchQueryResult $result): void 178 | { 179 | $results = array_map(function (Node $node) { 180 | $properties = []; 181 | 182 | foreach ($node->properties as $propertyName => $propertyValue) { 183 | if ($propertyValue instanceof ResourceBasedInterface) { 184 | $properties[$propertyName] = '' . $propertyName . ': ' . (string)$propertyValue->getResource()->getFilename(); 185 | } elseif ($propertyValue instanceof \DateTimeInterface) { 186 | $properties[$propertyName] = '' . $propertyName . ': ' . $propertyValue->format('Y-m-d H:i'); 187 | } elseif (is_array($propertyValue)) { 188 | $properties[$propertyName] = '' . $propertyName . ': ' . 'array'; 189 | } elseif ($propertyValue instanceof Node) { 190 | $properties[$propertyName] = '' . $propertyName . ': ' . $propertyValue->aggregateId->value; 191 | } else { 192 | $properties[$propertyName] = '' . $propertyName . ': ' . (string)$propertyValue; 193 | } 194 | } 195 | 196 | return [ 197 | 'identifier' => $node->aggregateId->value, 198 | 'label' => $this->nodeLabelGenerator->getLabel($node), 199 | 'nodeType' => $node->nodeTypeName->value, 200 | 'properties' => implode(PHP_EOL, $properties), 201 | ]; 202 | }, $result->toArray()); 203 | 204 | $this->output->outputTable($results, ['Identifier', 'Label', 'Node Type', 'Properties']); 205 | } 206 | } 207 | --------------------------------------------------------------------------------