├── Admin ├── AdminElasticsearchHelper.php ├── AdminIndexingBehavior.php ├── AdminSearchController.php ├── AdminSearchIndexingMessage.php ├── AdminSearchRegistry.php ├── AdminSearcher.php ├── ElasticsearchAdminException.php ├── Indexer │ ├── AbstractAdminIndexer.php │ ├── CategoryAdminSearchIndexer.php │ ├── CmsPageAdminSearchIndexer.php │ ├── CustomerAdminSearchIndexer.php │ ├── CustomerGroupAdminSearchIndexer.php │ ├── LandingPageAdminSearchIndexer.php │ ├── ManufacturerAdminSearchIndexer.php │ ├── MediaAdminSearchIndexer.php │ ├── NewsletterRecipientAdminSearchIndexer.php │ ├── OrderAdminSearchIndexer.php │ ├── PaymentMethodAdminSearchIndexer.php │ ├── ProductAdminSearchIndexer.php │ ├── ProductStreamAdminSearchIndexer.php │ ├── PromotionAdminSearchIndexer.php │ ├── PropertyGroupAdminSearchIndexer.php │ ├── SalesChannelAdminSearchIndexer.php │ └── ShippingMethodAdminSearchIndexer.php └── Subscriber │ └── RefreshIndexSubscriber.php ├── DependencyInjection ├── Configuration.php ├── ElasticsearchExtension.php └── ElasticsearchMigrationCompilerPass.php ├── Elasticsearch.php ├── ElasticsearchException.php ├── Event └── ElasticsearchCustomFieldsMappingEvent.php ├── Framework ├── AbstractElasticsearchDefinition.php ├── AsyncAwsSigner.php ├── ClientFactory.php ├── Command │ ├── ElasticsearchAdminIndexingCommand.php │ ├── ElasticsearchAdminResetCommand.php │ ├── ElasticsearchAdminTestCommand.php │ ├── ElasticsearchAdminUpdateMappingCommand.php │ ├── ElasticsearchCleanIndicesCommand.php │ ├── ElasticsearchCreateAliasCommand.php │ ├── ElasticsearchIndexingCommand.php │ ├── ElasticsearchResetCommand.php │ ├── ElasticsearchStatusCommand.php │ ├── ElasticsearchTestAnalyzerCommand.php │ └── ElasticsearchUpdateMappingCommand.php ├── DataAbstractionLayer │ ├── AbstractElasticsearchAggregationHydrator.php │ ├── AbstractElasticsearchSearchHydrator.php │ ├── CriteriaParser.php │ ├── ElasticsearchEntityAggregator.php │ ├── ElasticsearchEntityAggregatorHydrator.php │ ├── ElasticsearchEntitySearchHydrator.php │ ├── ElasticsearchEntitySearcher.php │ └── Event │ │ ├── ElasticsearchEntityAggregatorSearchEvent.php │ │ ├── ElasticsearchEntityAggregatorSearchedEvent.php │ │ ├── ElasticsearchEntitySearcherSearchEvent.php │ │ └── ElasticsearchEntitySearcherSearchedEvent.php ├── ElasticsearchDateHistogramAggregation.php ├── ElasticsearchFieldBuilder.php ├── ElasticsearchFieldMapper.php ├── ElasticsearchHelper.php ├── ElasticsearchIndexingUtils.php ├── ElasticsearchLanguageProvider.php ├── ElasticsearchOutdatedIndexDetector.php ├── ElasticsearchRangeAggregation.php ├── ElasticsearchRegistry.php ├── ElasticsearchStagingHandler.php ├── Indexing │ ├── CreateAliasTask.php │ ├── CreateAliasTaskHandler.php │ ├── ElasticsearchIndexer.php │ ├── ElasticsearchIndexingMessage.php │ ├── Event │ │ ├── ElasticsearchIndexAliasSwitchedEvent.php │ │ ├── ElasticsearchIndexConfigEvent.php │ │ ├── ElasticsearchIndexCreatedEvent.php │ │ ├── ElasticsearchIndexIteratorEvent.php │ │ └── ElasticsearchIndexerLanguageCriteriaEvent.php │ ├── IndexCreator.php │ ├── IndexMappingProvider.php │ ├── IndexMappingUpdater.php │ ├── IndexerOffset.php │ ├── IndexingDto.php │ └── Scripts │ │ ├── cheapest_price.groovy │ │ ├── cheapest_price_filter.groovy │ │ ├── cheapest_price_percentage.groovy │ │ ├── cheapest_price_percentage_filter.groovy │ │ ├── numeric_translated_field_sorting.groovy │ │ └── translated_field_sorting.groovy └── SystemUpdateListener.php ├── LICENSE ├── Migration ├── Traits │ └── ElasticsearchTriggerTrait.php └── V6_5 │ ├── Migration1689083660ElasticsearchIndexTask.php │ └── Migration1689084023AdminElasticsearchIndexTask.php ├── Product ├── AbstractProductSearchQueryBuilder.php ├── CustomFieldSetGateway.php ├── CustomFieldUpdater.php ├── ElasticsearchProductDefinition.php ├── ElasticsearchProductException.php ├── LanguageSubscriber.php ├── ProductSearchBuilder.php ├── ProductSearchQueryBuilder.php ├── ProductUpdater.php ├── SearchConfigLoader.php ├── SearchFieldConfig.php ├── SearchKeywordReplacement.php └── StopwordTokenFilter.php ├── Profiler ├── ClientProfiler.php ├── DataCollector.php └── ElasticsearchProfileCompilerPass.php ├── README.md ├── Resources ├── config │ ├── packages │ │ ├── elasticsearch.yaml │ │ ├── framework.yaml │ │ ├── monolog.yaml │ │ └── test │ │ │ └── elasticsearch.yaml │ ├── routes.xml │ └── services.xml └── views │ └── Collector │ ├── elasticsearch.html.twig │ └── icon.svg ├── Sort └── CountSort.php ├── Test ├── AdminElasticsearchTestBehaviour.php └── ElasticsearchTestTestBehaviour.php ├── TokenQueryBuilder.php └── composer.json /Admin/AdminElasticsearchHelper.php: -------------------------------------------------------------------------------- 1 | adminEsEnabled; 25 | } 26 | 27 | /** 28 | * Only used for unit tests because the container parameter bag is frozen and can not be changed at runtime. 29 | * Therefore this function can be used to test different behaviours 30 | * 31 | * @internal 32 | */ 33 | public function setEnabled(bool $enabled): self 34 | { 35 | $this->adminEsEnabled = $enabled; 36 | 37 | return $this; 38 | } 39 | 40 | public function getRefreshIndices(): bool 41 | { 42 | return $this->refreshIndices; 43 | } 44 | 45 | public function getPrefix(): string 46 | { 47 | return $this->adminIndexPrefix; 48 | } 49 | 50 | public function getIndex(string $name): string 51 | { 52 | return $this->adminIndexPrefix . '-' . \strtolower(\str_replace(['_', ' '], '-', $name)); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Admin/AdminIndexingBehavior.php: -------------------------------------------------------------------------------- 1 | $skipEntities 12 | * @param array $onlyEntities 13 | */ 14 | public function __construct( 15 | protected bool $noQueue = false, 16 | protected array $skipEntities = [], 17 | private readonly array $onlyEntities = [] 18 | ) { 19 | } 20 | 21 | public function getNoQueue(): bool 22 | { 23 | return $this->noQueue; 24 | } 25 | 26 | /** 27 | * @return array 28 | */ 29 | public function getSkipEntities(): array 30 | { 31 | return $this->skipEntities; 32 | } 33 | 34 | /** 35 | * @return array 36 | */ 37 | public function getOnlyEntities(): array 38 | { 39 | return $this->onlyEntities; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Admin/AdminSearchController.php: -------------------------------------------------------------------------------- 1 | ['administration']])] 32 | public function elastic(Request $request, Context $context): Response 33 | { 34 | if ($this->adminEsHelper->getEnabled() === false) { 35 | throw ElasticsearchAdminException::esNotEnabled(); 36 | } 37 | 38 | $term = trim($request->request->getString('term')); 39 | $entities = $request->request->all('entities'); 40 | 41 | if ($term === '') { 42 | throw ElasticsearchAdminException::missingTermParameter(); 43 | } 44 | 45 | $limit = $request->get('limit', 10); 46 | 47 | $results = $this->searcher->search($term, $entities, $context, $limit); 48 | 49 | foreach ($results as $entityName => $result) { 50 | $definition = $this->definitionRegistry->getByEntityName($entityName); 51 | 52 | /** @var EntityCollection $entityCollection */ 53 | $entityCollection = $result['data']; 54 | $entities = []; 55 | 56 | foreach ($entityCollection->getElements() as $key => $entity) { 57 | $entities[$key] = $this->entityEncoder->encode(new Criteria(), $definition, $entity, '/api'); 58 | } 59 | 60 | $results[$entityName]['data'] = $entities; 61 | } 62 | 63 | return new JsonResponse(['data' => $results]); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Admin/AdminSearchIndexingMessage.php: -------------------------------------------------------------------------------- 1 | $indices 16 | * @param array $ids 17 | */ 18 | public function __construct( 19 | private readonly string $entity, 20 | private readonly string $indexer, 21 | private readonly array $indices, 22 | private readonly array $ids 23 | ) { 24 | } 25 | 26 | public function getEntity(): string 27 | { 28 | return $this->entity; 29 | } 30 | 31 | public function getIndexer(): string 32 | { 33 | return $this->indexer; 34 | } 35 | 36 | /** 37 | * @return array 38 | */ 39 | public function getIndices(): array 40 | { 41 | return $this->indices; 42 | } 43 | 44 | /** 45 | * @return array 46 | */ 47 | public function getIds(): array 48 | { 49 | return $this->ids; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Admin/ElasticsearchAdminException.php: -------------------------------------------------------------------------------- 1 | >> $mapping 23 | * 24 | * @return array>> 25 | */ 26 | public function mapping(array $mapping): array 27 | { 28 | return $mapping; 29 | } 30 | 31 | abstract public function getIterator(): IterableQuery; 32 | 33 | /** 34 | * @param array $ids 35 | * 36 | * @return array 37 | */ 38 | abstract public function fetch(array $ids): array; 39 | 40 | /** 41 | * @param array $result 42 | * 43 | * @return array{total:int, data: EntityCollection} 44 | * 45 | * Returns EntityCollection and their total by ids in the result parameter 46 | */ 47 | abstract public function globalData(array $result, Context $context): array; 48 | 49 | public function globalCriteria(string $term, Search $criteria): Search 50 | { 51 | return $criteria; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Admin/Indexer/CategoryAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return CategoryDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'category-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(category.id)) as id, 69 | GROUP_CONCAT(DISTINCT category_translation.name SEPARATOR " ") as name, 70 | GROUP_CONCAT(DISTINCT tag.name SEPARATOR " ") as tags 71 | FROM category 72 | INNER JOIN category_translation 73 | ON category_translation.category_id = category.id 74 | LEFT JOIN category_tag 75 | ON category_tag.category_id = category.id 76 | LEFT JOIN tag 77 | ON category_tag.tag_id = tag.id 78 | WHERE category.id IN (:ids) 79 | GROUP BY category.id 80 | ', 81 | [ 82 | 'ids' => Uuid::fromHexToBytesList($ids), 83 | ], 84 | [ 85 | 'ids' => ArrayParameterType::BINARY, 86 | ] 87 | ); 88 | 89 | $mapped = []; 90 | foreach ($data as $row) { 91 | $id = (string) $row['id']; 92 | $text = \implode(' ', array_filter(array_unique(array_values($row)))); 93 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 94 | } 95 | 96 | return $mapped; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Admin/Indexer/CmsPageAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return CmsPageDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'cms-page-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator(CmsPageDefinition::ENTITY_NAME, null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(cms_page.id)) as id, 69 | GROUP_CONCAT(DISTINCT cms_page_translation.name SEPARATOR " ") as name 70 | FROM cms_page 71 | INNER JOIN cms_page_translation 72 | ON cms_page_translation.cms_page_id = cms_page.id 73 | WHERE cms_page.id IN (:ids) 74 | GROUP BY cms_page.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/CustomerGroupAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return CustomerGroupDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'customer-group-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(customer_group.id)) as id, 69 | GROUP_CONCAT(DISTINCT customer_group_translation.name SEPARATOR " ") as name 70 | FROM customer_group 71 | INNER JOIN customer_group_translation 72 | ON customer_group.id = customer_group_translation.customer_group_id 73 | WHERE customer_group.id IN (:ids) 74 | GROUP BY customer_group.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/LandingPageAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return LandingPageDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'landing-page-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(landing_page.id)) as id, 69 | GROUP_CONCAT(DISTINCT landing_page_translation.name SEPARATOR " ") as name, 70 | GROUP_CONCAT(DISTINCT tag.name SEPARATOR " ") as tags 71 | FROM landing_page 72 | INNER JOIN landing_page_translation 73 | ON landing_page.id = landing_page_translation.landing_page_id 74 | LEFT JOIN landing_page_tag 75 | ON landing_page.id = landing_page_tag.landing_page_id 76 | LEFT JOIN tag 77 | ON landing_page_tag.tag_id = tag.id 78 | WHERE landing_page.id IN (:ids) 79 | GROUP BY landing_page.id 80 | ', 81 | [ 82 | 'ids' => Uuid::fromHexToBytesList($ids), 83 | ], 84 | [ 85 | 'ids' => ArrayParameterType::BINARY, 86 | ] 87 | ); 88 | 89 | $mapped = []; 90 | foreach ($data as $row) { 91 | $id = (string) $row['id']; 92 | $text = \implode(' ', array_filter(array_unique(array_values($row)))); 93 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 94 | } 95 | 96 | return $mapped; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Admin/Indexer/ManufacturerAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return ProductManufacturerDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'manufacturer-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(product_manufacturer.id)) as id, 69 | GROUP_CONCAT(DISTINCT product_manufacturer_translation.name SEPARATOR " ") as name 70 | FROM product_manufacturer 71 | INNER JOIN product_manufacturer_translation 72 | ON product_manufacturer.id = product_manufacturer_translation.product_manufacturer_id 73 | WHERE product_manufacturer.id IN (:ids) 74 | GROUP BY product_manufacturer.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/MediaAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return MediaDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'media-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(media.id)) as id, 69 | media.file_name, 70 | media.path, 71 | GROUP_CONCAT(DISTINCT media_translation.alt SEPARATOR " ") as alt, 72 | GROUP_CONCAT(DISTINCT media_translation.title SEPARATOR " ") as title, 73 | media_folder.name, 74 | GROUP_CONCAT(tag.name SEPARATOR " ") as tags 75 | FROM media 76 | INNER JOIN media_translation 77 | ON media.id = media_translation.media_id 78 | LEFT JOIN media_folder 79 | ON media.media_folder_id = media_folder.id 80 | LEFT JOIN media_tag 81 | ON media.id = media_tag.media_id 82 | LEFT JOIN tag 83 | ON media_tag.tag_id = tag.id 84 | WHERE media.id IN (:ids) 85 | GROUP BY media.id 86 | ', 87 | [ 88 | 'ids' => Uuid::fromHexToBytesList($ids), 89 | ], 90 | [ 91 | 'ids' => ArrayParameterType::BINARY, 92 | ] 93 | ); 94 | 95 | $mapped = []; 96 | foreach ($data as $row) { 97 | $id = (string) $row['id']; 98 | $text = \implode(' ', array_filter(array_unique(array_values($row)))); 99 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 100 | } 101 | 102 | return $mapped; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /Admin/Indexer/NewsletterRecipientAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return NewsletterRecipientDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'newsletter-recipient-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(newsletter_recipient.id)) as id, 69 | newsletter_recipient.email 70 | FROM newsletter_recipient 71 | WHERE newsletter_recipient.id IN (:ids) 72 | GROUP BY newsletter_recipient.id 73 | ', 74 | [ 75 | 'ids' => Uuid::fromHexToBytesList($ids), 76 | ], 77 | [ 78 | 'ids' => ArrayParameterType::BINARY, 79 | ] 80 | ); 81 | 82 | $mapped = []; 83 | foreach ($data as $row) { 84 | $id = (string) $row['id']; 85 | $text = \implode(' ', array_filter(array_unique(array_values($row)))); 86 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 87 | } 88 | 89 | return $mapped; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Admin/Indexer/PaymentMethodAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return PaymentMethodDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'payment-method-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(payment_method.id)) as id, 69 | GROUP_CONCAT(DISTINCT payment_method_translation.name SEPARATOR " ") as name 70 | FROM payment_method 71 | INNER JOIN payment_method_translation 72 | ON payment_method.id = payment_method_translation.payment_method_id 73 | WHERE payment_method.id IN (:ids) 74 | GROUP BY payment_method.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/ProductStreamAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return ProductStreamDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'product-stream-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(product_stream.id)) as id, 69 | GROUP_CONCAT(DISTINCT product_stream_translation.name SEPARATOR " ") as name 70 | FROM product_stream 71 | INNER JOIN product_stream_translation 72 | ON product_stream.id = product_stream_translation.product_stream_id 73 | WHERE product_stream.id IN (:ids) 74 | GROUP BY product_stream.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter(array_unique(array_values($row)))); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/PromotionAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 28 | */ 29 | public function __construct( 30 | private readonly Connection $connection, 31 | private readonly IteratorFactory $factory, 32 | private readonly EntityRepository $repository, 33 | private readonly int $indexingBatchSize 34 | ) { 35 | } 36 | 37 | public function getDecorated(): AbstractAdminIndexer 38 | { 39 | throw new DecorationPatternException(self::class); 40 | } 41 | 42 | public function getEntity(): string 43 | { 44 | return PromotionDefinition::ENTITY_NAME; 45 | } 46 | 47 | public function getName(): string 48 | { 49 | return 'promotion-listing'; 50 | } 51 | 52 | public function getIterator(): IterableQuery 53 | { 54 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 55 | } 56 | 57 | public function globalData(array $result, Context $context): array 58 | { 59 | $ids = array_column($result['hits'], 'id'); 60 | 61 | return [ 62 | 'total' => (int) $result['total'], 63 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 64 | ]; 65 | } 66 | 67 | public function fetch(array $ids): array 68 | { 69 | $data = $this->connection->fetchAllAssociative( 70 | ' 71 | SELECT LOWER(HEX(promotion.id)) as id, 72 | GROUP_CONCAT(DISTINCT promotion_translation.name SEPARATOR " ") as name 73 | FROM promotion 74 | INNER JOIN promotion_translation 75 | ON promotion.id = promotion_translation.promotion_id 76 | WHERE promotion.id IN (:ids) 77 | GROUP BY promotion.id 78 | ', 79 | [ 80 | 'ids' => Uuid::fromHexToBytesList($ids), 81 | ], 82 | [ 83 | 'ids' => ArrayParameterType::BINARY, 84 | ] 85 | ); 86 | 87 | $mapped = []; 88 | foreach ($data as $row) { 89 | $id = (string) $row['id']; 90 | $text = \implode(' ', array_filter($row)); 91 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 92 | } 93 | 94 | return $mapped; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Admin/Indexer/PropertyGroupAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return PropertyGroupDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'property-group-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(property_group.id)) as id, 69 | GROUP_CONCAT(DISTINCT property_group_translation.name SEPARATOR " ") as name 70 | FROM property_group 71 | INNER JOIN property_group_translation 72 | ON property_group.id = property_group_translation.property_group_id 73 | WHERE property_group.id IN (:ids) 74 | GROUP BY property_group.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/SalesChannelAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return SalesChannelDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'sales-channel-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(sales_channel.id)) as id, 69 | GROUP_CONCAT(DISTINCT sales_channel_translation.name SEPARATOR " ") as name 70 | FROM sales_channel 71 | INNER JOIN sales_channel_translation 72 | ON sales_channel.id = sales_channel_translation.sales_channel_id 73 | WHERE sales_channel.id IN (:ids) 74 | GROUP BY sales_channel.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Indexer/ShippingMethodAdminSearchIndexer.php: -------------------------------------------------------------------------------- 1 | $repository 25 | */ 26 | public function __construct( 27 | private readonly Connection $connection, 28 | private readonly IteratorFactory $factory, 29 | private readonly EntityRepository $repository, 30 | private readonly int $indexingBatchSize 31 | ) { 32 | } 33 | 34 | public function getDecorated(): AbstractAdminIndexer 35 | { 36 | throw new DecorationPatternException(self::class); 37 | } 38 | 39 | public function getEntity(): string 40 | { 41 | return ShippingMethodDefinition::ENTITY_NAME; 42 | } 43 | 44 | public function getName(): string 45 | { 46 | return 'shipping-method-listing'; 47 | } 48 | 49 | public function getIterator(): IterableQuery 50 | { 51 | return $this->factory->createIterator($this->getEntity(), null, $this->indexingBatchSize); 52 | } 53 | 54 | public function globalData(array $result, Context $context): array 55 | { 56 | $ids = array_column($result['hits'], 'id'); 57 | 58 | return [ 59 | 'total' => (int) $result['total'], 60 | 'data' => $this->repository->search(new Criteria($ids), $context)->getEntities(), 61 | ]; 62 | } 63 | 64 | public function fetch(array $ids): array 65 | { 66 | $data = $this->connection->fetchAllAssociative( 67 | ' 68 | SELECT LOWER(HEX(shipping_method.id)) as id, 69 | GROUP_CONCAT(DISTINCT shipping_method_translation.name SEPARATOR " ") as name 70 | FROM shipping_method 71 | INNER JOIN shipping_method_translation 72 | ON shipping_method.id = shipping_method_translation.shipping_method_id 73 | WHERE shipping_method.id IN (:ids) 74 | GROUP BY shipping_method.id 75 | ', 76 | [ 77 | 'ids' => Uuid::fromHexToBytesList($ids), 78 | ], 79 | [ 80 | 'ids' => ArrayParameterType::BINARY, 81 | ] 82 | ); 83 | 84 | $mapped = []; 85 | foreach ($data as $row) { 86 | $id = (string) $row['id']; 87 | $text = \implode(' ', array_filter($row)); 88 | $mapped[$id] = ['id' => $id, 'text' => \strtolower($text)]; 89 | } 90 | 91 | return $mapped; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Admin/Subscriber/RefreshIndexSubscriber.php: -------------------------------------------------------------------------------- 1 | > 23 | */ 24 | public static function getSubscribedEvents(): array 25 | { 26 | return [ 27 | RefreshIndexEvent::class => 'handled', 28 | ]; 29 | } 30 | 31 | public function handled(RefreshIndexEvent $event): void 32 | { 33 | $this->registry->iterate( 34 | new AdminIndexingBehavior( 35 | $event->getNoQueue(), 36 | $event->getSkipEntities(), 37 | $event->getOnlyEntities() 38 | ) 39 | ); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /DependencyInjection/Configuration.php: -------------------------------------------------------------------------------- 1 | getRootNode(); 17 | $rootNode 18 | ->children() 19 | ->booleanNode('enabled')->end() 20 | ->booleanNode('indexing_enabled')->end() 21 | ->integerNode('indexing_batch_size')->defaultValue(100)->end() 22 | ->scalarNode('hosts')->end() 23 | ->scalarNode('index_prefix')->end() 24 | ->scalarNode('throw_exception')->end() 25 | ->arrayNode('ssl') 26 | ->children() 27 | ->scalarNode('cert_path')->end() 28 | ->scalarNode('cert_password')->end() 29 | ->scalarNode('cert_key_path')->end() 30 | ->scalarNode('cert_key_password')->end() 31 | ->booleanNode('verify_server_cert')->defaultValue(true)->end() 32 | ->arrayNode('sigV4') 33 | ->children() 34 | ->scalarNode('enabled')->defaultValue(false)->end() 35 | ->scalarNode('region')->end() 36 | ->scalarNode('service')->end() 37 | ->arrayNode('credentials_provider') 38 | ->children() 39 | ->scalarNode('key_id')->end() 40 | ->scalarNode('secret_key')->end() 41 | ->end() 42 | ->end() 43 | ->end() 44 | ->end() 45 | ->end() 46 | ->end() 47 | ->arrayNode('index_settings')->variablePrototype()->end()->end() 48 | ->arrayNode('analysis')->performNoDeepMerging()->variablePrototype()->end()->end() 49 | ->arrayNode('language_analyzer_mapping')->defaultValue([])->scalarPrototype()->end()->end() 50 | ->arrayNode('dynamic_templates')->performNoDeepMerging()->variablePrototype()->end()->end() 51 | ->arrayNode('product') 52 | ->children() 53 | ->arrayNode('custom_fields_mapping') 54 | ->variablePrototype()->end() 55 | ->end() 56 | ->booleanNode('exclude_source')->end() 57 | ->end() 58 | ->end() 59 | ->arrayNode('search') 60 | ->children() 61 | ->scalarNode('timeout')->end() 62 | ->integerNode('term_max_length')->end() 63 | ->scalarNode('search_type')->end() 64 | ->end() 65 | ->end() 66 | ->arrayNode('administration') 67 | ->children() 68 | ->scalarNode('hosts')->end() 69 | ->booleanNode('enabled')->end() 70 | ->booleanNode('refresh_indices')->end() 71 | ->scalarNode('index_prefix')->end() 72 | ->arrayNode('index_settings')->variablePrototype()->end()->end() 73 | ->arrayNode('analysis')->performNoDeepMerging()->variablePrototype()->end()->end() 74 | ->arrayNode('dynamic_templates')->performNoDeepMerging()->variablePrototype()->end()->end() 75 | ->arrayNode('search') 76 | ->children() 77 | ->scalarNode('timeout')->end() 78 | ->integerNode('term_max_length')->end() 79 | ->scalarNode('search_type')->end() 80 | ->end() 81 | ->end() 82 | ->end() 83 | ->end() 84 | ->end(); 85 | 86 | return $treeBuilder; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /DependencyInjection/ElasticsearchExtension.php: -------------------------------------------------------------------------------- 1 | processConfiguration($this->getConfiguration($configs, $container), $configs); 18 | $this->addConfig($container, $this->getAlias(), $config); 19 | 20 | $loader = new XmlFileLoader($container, new FileLocator(__DIR__ . '/../Resources/config')); 21 | 22 | $loader->load('services.xml'); 23 | } 24 | 25 | public function getConfiguration(array $config, ContainerBuilder $container): ConfigurationInterface 26 | { 27 | return new Configuration(); 28 | } 29 | 30 | private function addConfig(ContainerBuilder $container, string $alias, array $options): void 31 | { 32 | foreach ($options as $key => $option) { 33 | $container->setParameter($alias . '.' . $key, $option); 34 | 35 | if (\is_array($option)) { 36 | $this->addConfig($container, $alias . '.' . $key, $option); 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /DependencyInjection/ElasticsearchMigrationCompilerPass.php: -------------------------------------------------------------------------------- 1 | buildDefaultConfig($container); 29 | 30 | $container->addCompilerPass(new ElasticsearchMigrationCompilerPass()); 31 | 32 | // Needs to run before the ProfilerPass 33 | $container->addCompilerPass(new ElasticsearchProfileCompilerPass(), PassConfig::TYPE_BEFORE_OPTIMIZATION, 5000); 34 | } 35 | 36 | protected function createContainerExtension(): ?ExtensionInterface 37 | { 38 | return new ElasticsearchExtension(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Event/ElasticsearchCustomFieldsMappingEvent.php: -------------------------------------------------------------------------------- 1 | $mapping 18 | */ 19 | public function __construct( 20 | private readonly string $entity, 21 | private array $mapping, 22 | private readonly Context $context 23 | ) { 24 | } 25 | 26 | public function getContext(): Context 27 | { 28 | return $this->context; 29 | } 30 | 31 | /** 32 | * @param CustomFieldTypes::* $type 33 | */ 34 | public function setMapping(string $field, string $type): void 35 | { 36 | $this->mapping[$field] = $type; 37 | } 38 | 39 | /** 40 | * @return CustomFieldTypes::*|null 41 | * @return string|null 42 | */ 43 | public function getMapping(string $field) 44 | { 45 | return $this->mapping[$field] ?? null; 46 | } 47 | 48 | public function removeMapping(string $field): void 49 | { 50 | if (isset($this->mapping[$field])) { 51 | unset($this->mapping[$field]); 52 | } 53 | } 54 | 55 | /** 56 | * @return array 57 | */ 58 | public function getMappings(): array 59 | { 60 | return $this->mapping; 61 | } 62 | 63 | public function getEntity(): string 64 | { 65 | return $this->entity; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Framework/AbstractElasticsearchDefinition.php: -------------------------------------------------------------------------------- 1 | 'keyword', 17 | 'ignore_above' => 10000, 18 | 'normalizer' => 'sw_lowercase_normalizer', 19 | ]; 20 | 21 | final public const BOOLEAN_FIELD = ['type' => 'boolean']; 22 | 23 | final public const FLOAT_FIELD = ['type' => 'double']; 24 | 25 | final public const INT_FIELD = ['type' => 'long']; 26 | 27 | final public const SEARCH_FIELD = [ 28 | 'fields' => [ 29 | 'search' => ['type' => 'text', 'analyzer' => 'sw_whitespace_analyzer'], 30 | 'ngram' => ['type' => 'text', 'analyzer' => 'sw_ngram_analyzer'], 31 | ], 32 | ]; 33 | 34 | abstract public function getEntityDefinition(): EntityDefinition; 35 | 36 | /** 37 | * @return array{_source?: array{includes: string[]}, properties: array} 38 | */ 39 | abstract public function getMapping(Context $context): array; 40 | 41 | /** 42 | * Can be used to define custom queries to define the data to be indexed. 43 | */ 44 | public function getIterator(): ?IterableQuery 45 | { 46 | return null; 47 | } 48 | 49 | /** 50 | * @param array $ids 51 | * 52 | * @return array> 53 | */ 54 | public function fetch(array $ids, Context $context): array 55 | { 56 | return []; 57 | } 58 | 59 | abstract public function buildTermQuery(Context $context, Criteria $criteria): BuilderInterface; 60 | 61 | /** 62 | * @return array 63 | */ 64 | protected static function getTextFieldConfig(): array 65 | { 66 | return self::KEYWORD_FIELD + self::SEARCH_FIELD; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Framework/AsyncAwsSigner.php: -------------------------------------------------------------------------------- 1 | $request 34 | */ 35 | public function __invoke(array $request): CompletedFutureArray 36 | { 37 | try { 38 | $transformed = $this->transformRequest($request); 39 | 40 | $credentials = $this->credentialProvider->getCredentials($this->configuration); 41 | if ($credentials === null) { 42 | throw ElasticsearchException::awsCredentialsNotFound(); 43 | } 44 | 45 | $signer = new SignerV4($this->service, $this->region); 46 | 47 | $signer->sign($transformed, $credentials, new RequestContext()); 48 | 49 | $request['headers'] = []; 50 | foreach ($transformed->getHeaders() as $key => $value) { 51 | $request['headers'][$key] = [$value]; 52 | } 53 | 54 | return \call_user_func(ClientBuilder::defaultHandler(), $request); 55 | } catch (\Throwable $e) { 56 | $this->logger->error('Error signing request: ' . $e->getMessage()); 57 | 58 | throw $e; 59 | } 60 | } 61 | 62 | /** 63 | * @param array $request 64 | */ 65 | private function transformRequest(array $request): Request 66 | { 67 | // fix for uppercase 'Host' array key in elasticsearch-php 5.3.1 and backward compatible 68 | // https://github.com/aws/aws-sdk-php/issues/1225 69 | $hostKey = isset($request['headers']['Host']) ? 'Host' : 'host'; 70 | 71 | // Amazon ES/OS listens on standard ports (443 for HTTPS, 80 for HTTP). 72 | // Consequently, the port should be stripped from the host header. 73 | $parsedUrl = parse_url($request['headers'][$hostKey][0]); 74 | 75 | if (isset($parsedUrl['host'])) { 76 | $request['headers'][$hostKey][0] = $parsedUrl['host']; 77 | } 78 | 79 | parse_str($request['query_string'] ?? '', $query); 80 | $query = array_filter($query, 'is_string'); 81 | $query = array_combine(array_map('strval', array_keys($query)), $query); 82 | 83 | $headers = []; 84 | foreach ($request['headers'] as $key => $value) { 85 | $headers[$key] = $value[0]; 86 | } 87 | 88 | $url = $request['scheme'] . '://' . $request['headers'][$hostKey][0] . $request['uri']; 89 | 90 | $request = new Request( 91 | $request['http_method'], 92 | $url, 93 | $query, 94 | $headers, 95 | StringStream::create($request['body'] ?? '') 96 | ); 97 | $request->setEndpoint($url); 98 | 99 | return $request; 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /Framework/ClientFactory.php: -------------------------------------------------------------------------------- 1 | setHosts($hosts); 24 | 25 | if ($debug) { 26 | $clientBuilder->setTracer($logger); 27 | } 28 | 29 | $clientBuilder->setLogger($logger); 30 | 31 | if ($sslConfig['verify_server_cert'] === false) { 32 | $clientBuilder->setSSLVerification(false); 33 | } 34 | 35 | if (isset($sslConfig['cert_path'])) { 36 | $clientBuilder->setSSLCert($sslConfig['cert_path'], $sslConfig['cert_password'] ?? null); 37 | } 38 | 39 | if (isset($sslConfig['cert_key_path'])) { 40 | $clientBuilder->setSSLKey($sslConfig['cert_key_path'], $sslConfig['cert_key_password'] ?? null); 41 | } 42 | 43 | // Apply SigV4 signing if configured 44 | if ($sslConfig['sigV4']['enabled'] ?? false) { 45 | $region = $sslConfig['sigV4']['region'] ?? ''; 46 | $service = $sslConfig['sigV4']['service'] ?? 'es'; 47 | $credentials = $sslConfig['sigV4']['credentials_provider'] ?? []; 48 | 49 | $configuration = Configuration::create([ 50 | 'region' => $region, 51 | 'accessKeyId' => $credentials['key_id'] ?? null, 52 | 'accessKeySecret' => $credentials['secret_key'] ?? null, 53 | ]); 54 | 55 | $credentialProvider = ChainProvider::createDefaultChain(null, $logger); 56 | 57 | $signer = new AsyncAwsSigner($configuration, $logger, $service, $region, $credentialProvider); 58 | $clientBuilder->setHandler($signer); 59 | } 60 | 61 | return $clientBuilder->build(); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchAdminIndexingCommand.php: -------------------------------------------------------------------------------- 1 | addOption('no-progress', null, null, 'Do not output progress bar'); 43 | $this->addOption('no-queue', null, null, 'Do not use the queue for indexing'); 44 | $this->addOption('skip', null, InputArgument::OPTIONAL, 'Comma separated list of entity names to be skipped'); 45 | $this->addOption('only', null, InputArgument::OPTIONAL, 'Comma separated list of entity names to be generated'); 46 | } 47 | 48 | protected function execute(InputInterface $input, OutputInterface $output): int 49 | { 50 | $this->io = $input->getOption('no-progress') ? null : new ShopwareStyle($input, $output); 51 | 52 | $skip = \is_string($input->getOption('skip')) ? explode(',', $input->getOption('skip')) : []; 53 | $only = \is_string($input->getOption('only')) ? explode(',', $input->getOption('only')) : []; 54 | 55 | $this->registry->iterate( 56 | new AdminIndexingBehavior( 57 | (bool) $input->getOption('no-queue'), 58 | $skip, 59 | $only 60 | ) 61 | ); 62 | 63 | return self::SUCCESS; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchAdminResetCommand.php: -------------------------------------------------------------------------------- 1 | adminEsHelper->getEnabled() !== true) { 45 | $io->error('Admin elasticsearch is not enabled'); 46 | 47 | return self::FAILURE; 48 | } 49 | 50 | $confirm = $io->confirm('Are you sure you want to reset the Admin Elasticsearch indexing?'); 51 | 52 | if (!$confirm) { 53 | $io->caution('Canceled clearing indexing process'); 54 | 55 | return self::SUCCESS; 56 | } 57 | 58 | $allIndices = $this->client->indices()->get(['index' => $this->adminEsHelper->getPrefix() . '*']); 59 | 60 | foreach ($allIndices as $index) { 61 | $this->client->indices()->delete(['index' => $index['settings']['index']['provided_name']]); 62 | } 63 | 64 | $this->connection->executeStatement('TRUNCATE admin_elasticsearch_index_task'); 65 | 66 | try { 67 | $gateway = $this->gatewayRegistry->get(IncrementGatewayRegistry::MESSAGE_QUEUE_POOL); 68 | $gateway->reset('message_queue_stats', AdminSearchIndexingMessage::class); 69 | } catch (IncrementGatewayNotFoundException) { 70 | // In case message_queue pool is disabled 71 | } 72 | 73 | $io->success('Admin Elasticsearch indices deleted and queue cleared'); 74 | 75 | return self::SUCCESS; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchAdminTestCommand.php: -------------------------------------------------------------------------------- 1 | addArgument('term', InputArgument::REQUIRED); 55 | } 56 | 57 | protected function execute(InputInterface $input, OutputInterface $output): int 58 | { 59 | $this->io = new ShopwareStyle($input, $output); 60 | 61 | $term = $input->getArgument('term'); 62 | $entities = [ 63 | CmsPageDefinition::ENTITY_NAME, 64 | CustomerDefinition::ENTITY_NAME, 65 | CustomerGroupDefinition::ENTITY_NAME, 66 | LandingPageDefinition::ENTITY_NAME, 67 | ProductManufacturerDefinition::ENTITY_NAME, 68 | MediaDefinition::ENTITY_NAME, 69 | OrderDefinition::ENTITY_NAME, 70 | PaymentMethodDefinition::ENTITY_NAME, 71 | ProductDefinition::ENTITY_NAME, 72 | PromotionDefinition::ENTITY_NAME, 73 | PropertyGroupDefinition::ENTITY_NAME, 74 | SalesChannelDefinition::ENTITY_NAME, 75 | ShippingMethodDefinition::ENTITY_NAME, 76 | ]; 77 | 78 | $result = $this->searcher->search($term, $entities, Context::createCLIContext()); 79 | 80 | $rows = []; 81 | foreach ($result as $data) { 82 | $rows[] = [$data['index'], $data['indexer'], $data['total']]; 83 | } 84 | 85 | $this->io->table(['Index', 'Indexer', 'total'], $rows); 86 | 87 | return self::SUCCESS; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchAdminUpdateMappingCommand.php: -------------------------------------------------------------------------------- 1 | registry->updateMappings(); 33 | 34 | $io->success('Updated mapping for admin indices'); 35 | 36 | return self::SUCCESS; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchCleanIndicesCommand.php: -------------------------------------------------------------------------------- 1 | addOption('force', 'f', InputOption::VALUE_NONE, 'Do not ask for confirmation'); 41 | } 42 | } 43 | 44 | protected function execute(InputInterface $input, OutputInterface $output): int 45 | { 46 | $io = new SymfonyStyle($input, $output); 47 | $indices = $this->outdatedIndexDetector->get(); 48 | 49 | if (empty($indices)) { 50 | $io->writeln('No indices to be deleted.'); 51 | 52 | return self::SUCCESS; 53 | } 54 | 55 | $io->table(['Indices to be deleted:'], array_map(static fn (string $name) => [$name], $indices)); 56 | 57 | if (Feature::isActive('v6.8.0.0') || !$input->getOption('force')) { 58 | $confirm = $io->confirm(\sprintf('Delete these %d indices?', \count($indices))); 59 | 60 | if (!$confirm) { 61 | $io->caution('Deletion aborted.'); 62 | 63 | return self::SUCCESS; 64 | } 65 | } 66 | 67 | foreach ($indices as $index) { 68 | $this->client->indices()->delete(['index' => $index]); 69 | } 70 | 71 | $io->writeln('Indices deleted.'); 72 | 73 | return self::SUCCESS; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchCreateAliasCommand.php: -------------------------------------------------------------------------------- 1 | handler->run(); 37 | 38 | return self::SUCCESS; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchIndexingCommand.php: -------------------------------------------------------------------------------- 1 | addOption('no-progress', null, null, 'Do not output progress bar'); 47 | $this->addOption('no-queue', null, null, 'Do not use the queue for indexing'); 48 | $this->addOption('only', null, InputOption::VALUE_REQUIRED, 'Add entities separated by comma to indexing'); 49 | } 50 | 51 | protected function execute(InputInterface $input, OutputInterface $output): int 52 | { 53 | $stopwatch = new Stopwatch(); 54 | $stopwatch->start('es-indexing'); 55 | $this->io = new ShopwareStyle($input, $output); 56 | 57 | if (!$this->enabled) { 58 | $this->io->error('Elasticsearch indexing is disabled'); 59 | 60 | return self::FAILURE; 61 | } 62 | 63 | $progressBar = new ProgressBar($input->getOption('no-progress') ? new NullOutput() : $output); 64 | $progressBar->start(); 65 | 66 | $entities = $input->getOption('only') ? explode(',', $input->getOption('only')) : []; 67 | $offset = null; 68 | while ($message = $this->indexer->iterate($offset, $entities)) { 69 | $offset = $message->getOffset(); 70 | 71 | $step = \count($message->getData()->getIds()); 72 | 73 | if ($input->getOption('no-queue')) { 74 | $this->indexer->__invoke($message); 75 | 76 | $progressBar->advance($step); 77 | 78 | continue; 79 | } 80 | 81 | $this->messageBus->dispatch($message); 82 | 83 | $progressBar->advance($step); 84 | } 85 | 86 | $progressBar->finish(); 87 | 88 | if ($input->getOption('no-queue')) { 89 | $this->aliasHandler->run(); 90 | } 91 | 92 | $event = (string) $stopwatch->stop('es-indexing'); 93 | 94 | $this->io->info($event); 95 | 96 | return self::SUCCESS; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchResetCommand.php: -------------------------------------------------------------------------------- 1 | confirm('Are you sure you want to reset the Elasticsearch indexing?'); 48 | 49 | if (!$confirm) { 50 | $io->caution('Canceled clearing indexing process'); 51 | 52 | return self::SUCCESS; 53 | } 54 | 55 | $indices = $this->detector->getAllUsedIndices(); 56 | 57 | foreach ($indices as $index) { 58 | $this->client->indices()->delete(['index' => $index]); 59 | } 60 | 61 | $this->connection->executeStatement('TRUNCATE elasticsearch_index_task'); 62 | 63 | try { 64 | $gateway = $this->gatewayRegistry->get(IncrementGatewayRegistry::MESSAGE_QUEUE_POOL); 65 | $gateway->reset('message_queue_stats', ElasticsearchIndexingMessage::class); 66 | } catch (IncrementGatewayNotFoundException) { 67 | // In case message_queue pool is disabled 68 | } 69 | 70 | $this->connection->executeStatement('DELETE FROM `messenger_messages` WHERE `headers` LIKE "%ElasticsearchIndexingMessage%"'); 71 | 72 | $io->success('Elasticsearch indices deleted and queue cleared'); 73 | 74 | return self::SUCCESS; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchStatusCommand.php: -------------------------------------------------------------------------------- 1 | client->ping()) { 49 | throw ElasticsearchException::serverNotAvailable(); 50 | } 51 | 52 | $table = new Table($output); 53 | $table->setHeaders(['Name', 'Status']); 54 | $health = $this->client->cluster()->health(); 55 | 56 | $table->addRow(['Cluster Status', $health['status']]); 57 | $table->addRow(['Available Nodes', $health['number_of_nodes']]); 58 | 59 | $indexTask = $this->connection->fetchAssociative('SELECT * FROM elasticsearch_index_task WHERE entity = "product" LIMIT 1'); 60 | $totalProducts = (int) $this->connection->fetchOne('SELECT COUNT(*) FROM product WHERE version_id = :liveVersionId AND child_count = 0 OR parent_id IS NOT NULL', ['liveVersionId' => Uuid::fromHexToBytes(Defaults::LIVE_VERSION)]); 61 | 62 | // No entry in key 63 | if ($indexTask === false) { 64 | $table->addRow(['Indexing', 'completed']); 65 | $table->render(); 66 | $output->writeln(''); 67 | 68 | return self::SUCCESS; 69 | } 70 | 71 | if ((int) $indexTask['doc_count'] > 0) { 72 | $table->addRow(['Indexing', 'in progress']); 73 | 74 | $table->render(); 75 | $output->writeln(''); 76 | 77 | $progressBar = new ProgressBar($output, $totalProducts); 78 | $progressBar->advance($totalProducts - $indexTask['doc_count']); 79 | $output->writeln(''); 80 | } else { 81 | $table->addRow(['Indexing', 'completed']); 82 | $table->render(); 83 | $output->writeln(''); 84 | } 85 | 86 | /** @var list $usedIndices */ 87 | $usedIndices = array_keys($this->client->indices()->getAlias(['name' => $indexTask['alias']])); 88 | 89 | $indexName = $indexTask['index']; 90 | \assert(\is_string($indexName)); 91 | if (!\in_array($indexName, $usedIndices, true)) { 92 | $io = new SymfonyStyle($input, $output); 93 | $io->warning(\sprintf('Alias will swap at the end of the indexing process from %s to %s', $usedIndices[0], $indexName)); 94 | } 95 | 96 | return self::SUCCESS; 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchTestAnalyzerCommand.php: -------------------------------------------------------------------------------- 1 | addArgument('term', InputArgument::REQUIRED); 38 | } 39 | 40 | protected function execute(InputInterface $input, OutputInterface $output): int 41 | { 42 | $this->io = new ShopwareStyle($input, $output); 43 | 44 | $term = $input->getArgument('term'); 45 | 46 | $iteration = $this->getAnalyzers(); 47 | 48 | $rows = []; 49 | foreach ($iteration as $headline => $analyzers) { 50 | $rows[] = [$headline]; 51 | $rows[] = ['###############']; 52 | foreach ($analyzers as $analyzer) { 53 | /** @var array{'tokens': array{token: string}[]} $analyzed */ 54 | $analyzed = $this->client->indices()->analyze([ 55 | 'body' => [ 56 | 'analyzer' => $analyzer, 57 | 'text' => $term, 58 | ], 59 | ]); 60 | 61 | $rows[] = [ 62 | 'Analyzer' => $analyzer, 63 | 'Tokens' => implode(' ', array_column($analyzed['tokens'], 'token')), 64 | ]; 65 | } 66 | 67 | $rows[] = [' ']; 68 | $rows[] = [' ']; 69 | } 70 | 71 | $this->io->table(['Analyzer', 'Tokens'], $rows); 72 | 73 | return self::SUCCESS; 74 | } 75 | 76 | /** 77 | * @return array> 78 | */ 79 | protected function getAnalyzers(): array 80 | { 81 | return [ 82 | 'Default analyzers' => [ 83 | 'standard', 84 | 'simple', 85 | 'whitespace', 86 | 'stop', 87 | 'keyword', 88 | 'pattern', 89 | 'fingerprint', 90 | ], 91 | 'Custom analyzers' => [], 92 | 'Default language analyzers' => [ 93 | 'arabic', 94 | 'armenian', 95 | 'basque', 96 | 'bengali', 97 | 'brazilian', 98 | 'bulgarian', 99 | 'catalan', 100 | 'cjk', 101 | 'czech', 102 | 'danish', 103 | 'dutch', 104 | 'english', 105 | 'finnish', 106 | 'french', 107 | 'galician', 108 | 'german', 109 | 'greek', 110 | 'hindi', 111 | 'hungarian', 112 | 'indonesian', 113 | 'irish', 114 | 'italian', 115 | 'latvian', 116 | 'lithuanian', 117 | 'norwegian', 118 | 'persian', 119 | 'portuguese', 120 | 'romanian', 121 | 'russian', 122 | 'sorani', 123 | 'spanish', 124 | 'swedish', 125 | 'turkish', 126 | 'thai', 127 | ], 128 | ]; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /Framework/Command/ElasticsearchUpdateMappingCommand.php: -------------------------------------------------------------------------------- 1 | indexMappingUpdater->update(Context::createCLIContext()); 32 | 33 | return self::SUCCESS; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/AbstractElasticsearchAggregationHydrator.php: -------------------------------------------------------------------------------- 1 | helper->allowSearch($definition, $context, $criteria)) { 42 | return $this->decorated->aggregate($definition, $criteria, $context); 43 | } 44 | 45 | if (\count($criteria->getAggregations()) === 0) { 46 | return new AggregationResultCollection(); 47 | } 48 | 49 | $search = $this->createSearch($definition, $criteria, $context); 50 | 51 | $this->eventDispatcher->dispatch( 52 | new ElasticsearchEntityAggregatorSearchEvent($search, $definition, $criteria, $context) 53 | ); 54 | 55 | $searchArray = $search->toArray(); 56 | $searchArray['timeout'] = $this->timeout; 57 | 58 | $result = $this->client->search([ 59 | 'index' => $this->helper->getIndexName($definition), 60 | 'track_total_hits' => false, 61 | 'body' => $searchArray, 62 | 'search_type' => $this->searchType, 63 | ]); 64 | 65 | $result = $this->hydrator->hydrate($definition, $criteria, $context, $result); 66 | 67 | $this->eventDispatcher->dispatch(new ElasticsearchEntityAggregatorSearchedEvent($result, $search, $definition, $criteria, $context)); 68 | 69 | $result->addState(self::RESULT_STATE); 70 | 71 | return $result; 72 | } catch (\Throwable $e) { 73 | if ($e instanceof ElasticsearchException && $e->getErrorCode() === ElasticsearchException::EMPTY_QUERY) { 74 | return new AggregationResultCollection(); 75 | } 76 | 77 | $this->helper->logAndThrowException($e); 78 | 79 | return $this->decorated->aggregate($definition, $criteria, $context); 80 | } 81 | } 82 | 83 | private function createSearch(EntityDefinition $definition, Criteria $criteria, Context $context): Search 84 | { 85 | $search = new Search(); 86 | $this->helper->addFilters($definition, $criteria, $search, $context); 87 | $this->helper->addQueries($definition, $criteria, $search, $context); 88 | $this->helper->addAggregations($definition, $criteria, $search, $context); 89 | $this->helper->addTerm($criteria, $search, $context, $definition); 90 | $this->helper->handleIds($definition, $criteria, $search, $context); 91 | $search->setSize(0); 92 | 93 | return $search; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/ElasticsearchEntitySearchHydrator.php: -------------------------------------------------------------------------------- 1 | , inner_hits?: array{ inner?: array}}>}, aggregations?: array>} $result 22 | */ 23 | public function hydrate(EntityDefinition $definition, Criteria $criteria, Context $context, array $result): IdSearchResult 24 | { 25 | if (!isset($result['hits'])) { 26 | return new IdSearchResult(0, [], $criteria, $context); 27 | } 28 | 29 | $hits = $this->extractHits($result); 30 | 31 | $data = []; 32 | foreach ($hits as $hit) { 33 | $id = $hit['_id']; 34 | 35 | $data[$id] = [ 36 | 'primaryKey' => $id, 37 | 'data' => array_merge( 38 | $hit['_source'] ?? [], 39 | ['id' => $id, '_score' => $hit['_score']] 40 | ), 41 | ]; 42 | } 43 | 44 | $total = $this->getTotalValue($criteria, $result); 45 | if ($criteria->useIdSorting()) { 46 | $data = $this->sortByIdArray($criteria->getIds(), $data); 47 | } 48 | 49 | return new IdSearchResult($total, $data, $criteria, $context); 50 | } 51 | 52 | /** 53 | * @param array{ hits: array{ hits: array}}>}} $result 54 | * 55 | * @return array 56 | */ 57 | private function extractHits(array $result): array 58 | { 59 | $records = []; 60 | $hits = $result['hits']['hits']; 61 | 62 | foreach ($hits as $hit) { 63 | if (!isset($hit['inner_hits']['inner'])) { 64 | $records[] = $hit; 65 | 66 | continue; 67 | } 68 | 69 | /** @var array{ hits: array{ hits: array>}} $inner */ 70 | $inner = $hit['inner_hits']['inner']; 71 | 72 | $nested = $this->extractHits($inner); 73 | 74 | foreach ($nested as $inner) { 75 | $records[] = $inner; 76 | } 77 | } 78 | 79 | return $records; 80 | } 81 | 82 | /** 83 | * @param array{ hits: array{ hits: array, total?: array{ value: int } }, aggregations?: array>} $result 84 | */ 85 | private function getTotalValue(Criteria $criteria, array $result): int 86 | { 87 | if ($criteria->getTotalCountMode() !== Criteria::TOTAL_COUNT_MODE_EXACT) { 88 | return empty($result['hits']['hits']) ? 0 : \count($result['hits']['hits']); 89 | } 90 | 91 | if (!$criteria->getGroupFields()) { 92 | return (int) ($result['hits']['total']['value'] ?? 0); 93 | } 94 | 95 | if (!$criteria->getPostFilters()) { 96 | return empty($result['aggregations']['total-count']['value']) ? 0 : (int) $result['aggregations']['total-count']['value']; 97 | } 98 | 99 | return empty($result['aggregations']['total-filtered-count']['total-count']['value']) ? 0 : (int) $result['aggregations']['total-filtered-count']['total-count']['value']; 100 | } 101 | 102 | /** 103 | * @param array> $ids 104 | * @param array> $data 105 | * 106 | * @return array> 107 | */ 108 | private function sortByIdArray(array $ids, array $data): array 109 | { 110 | $sorted = []; 111 | 112 | foreach ($ids as $id) { 113 | if (\is_array($id)) { 114 | $id = implode('-', $id); 115 | } 116 | 117 | if (\array_key_exists($id, $data)) { 118 | $sorted[$id] = $data[$id]; 119 | } 120 | } 121 | 122 | return $sorted; 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/Event/ElasticsearchEntityAggregatorSearchEvent.php: -------------------------------------------------------------------------------- 1 | search; 27 | } 28 | 29 | public function getContext(): Context 30 | { 31 | return $this->context; 32 | } 33 | 34 | public function getDefinition(): EntityDefinition 35 | { 36 | return $this->definition; 37 | } 38 | 39 | public function getCriteria(): Criteria 40 | { 41 | return $this->criteria; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/Event/ElasticsearchEntityAggregatorSearchedEvent.php: -------------------------------------------------------------------------------- 1 | context; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/Event/ElasticsearchEntitySearcherSearchEvent.php: -------------------------------------------------------------------------------- 1 | search; 27 | } 28 | 29 | public function getContext(): Context 30 | { 31 | return $this->context; 32 | } 33 | 34 | public function getDefinition(): EntityDefinition 35 | { 36 | return $this->definition; 37 | } 38 | 39 | public function getCriteria(): Criteria 40 | { 41 | return $this->criteria; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /Framework/DataAbstractionLayer/Event/ElasticsearchEntitySearcherSearchedEvent.php: -------------------------------------------------------------------------------- 1 | $response 22 | */ 23 | public function __construct( 24 | public readonly IdSearchResult $result, 25 | public readonly Search $search, 26 | public readonly EntityDefinition $definition, 27 | public readonly Criteria $criteria, 28 | private readonly Context $context, 29 | public readonly array $response = [], 30 | ) { 31 | } 32 | 33 | public function getContext(): Context 34 | { 35 | return $this->context; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Framework/ElasticsearchDateHistogramAggregation.php: -------------------------------------------------------------------------------- 1 | setField($field); 30 | $this->setInterval($interval); 31 | $this->setFormat($format); 32 | } 33 | 34 | public function getInterval(): string 35 | { 36 | return $this->interval; 37 | } 38 | 39 | public function setInterval(string $interval): self 40 | { 41 | $this->interval = $interval; 42 | 43 | return $this; 44 | } 45 | 46 | public function setFormat(?string $format): self 47 | { 48 | $this->format = $format; 49 | 50 | return $this; 51 | } 52 | 53 | public function getType(): string 54 | { 55 | return 'date_histogram'; 56 | } 57 | 58 | /** 59 | * {@inheritdoc} 60 | */ 61 | protected function getArray(): array 62 | { 63 | $out = [ 64 | 'field' => $this->getField(), 65 | 'calendar_interval' => $this->getInterval(), 66 | ]; 67 | 68 | if (!empty($this->format)) { 69 | $out['format'] = $this->format; 70 | } 71 | 72 | return $out; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Framework/ElasticsearchFieldBuilder.php: -------------------------------------------------------------------------------- 1 | $languageAnalyzerMapping 17 | */ 18 | public function __construct( 19 | private readonly LanguageLoaderInterface $languageLoader, 20 | private readonly ElasticsearchIndexingUtils $indexingUtils, 21 | private readonly array $languageAnalyzerMapping 22 | ) { 23 | } 24 | 25 | /** 26 | * @param array $fieldConfig 27 | * 28 | * @description This method is used to build the mapping for translated fields 29 | * 30 | * @return array{properties: array} 31 | */ 32 | public function translated(array $fieldConfig): array 33 | { 34 | $languages = $this->languageLoader->loadLanguages(); 35 | 36 | $languageFields = []; 37 | 38 | foreach ($languages as $languageId => $language) { 39 | $code = $language['code'] ?? $language['parentCode']; 40 | $parts = explode('-', $code); 41 | $locale = $parts[0]; 42 | 43 | $languageFields[$languageId] = $fieldConfig; 44 | 45 | if (\array_key_exists($locale, $this->languageAnalyzerMapping)) { 46 | $languageFields[$languageId]['fields']['search']['analyzer'] = $this->languageAnalyzerMapping[$locale]; 47 | } 48 | } 49 | 50 | return ['properties' => $languageFields]; 51 | } 52 | 53 | /** 54 | * @description This method is used to build the mapping for translated custom fields 55 | * 56 | * @return array{ properties: array> } 57 | */ 58 | public function customFields(string $entity, Context $context): array 59 | { 60 | $languages = $this->languageLoader->loadLanguages(); 61 | 62 | $customFields = []; 63 | 64 | foreach (array_keys($languages) as $languageId) { 65 | $customFields[$languageId] = $this->getCustomFieldsMapping($entity, $context); 66 | } 67 | 68 | return ['properties' => $customFields]; 69 | } 70 | 71 | /** 72 | * @description This method is used to build the mapping for datetime fields 73 | * 74 | * @param array $override 75 | * 76 | * @return array 77 | */ 78 | public static function datetime(array $override = []): array 79 | { 80 | return array_merge([ 81 | 'type' => 'date', 82 | 'format' => 'yyyy-MM-dd HH:mm:ss.000||strict_date_optional_time||epoch_millis', 83 | 'ignore_malformed' => true, 84 | ], $override); 85 | } 86 | 87 | /** 88 | * @description This method is used to build the mapping for nested fields 89 | * 90 | * @param array $properties 91 | * 92 | * @return array{type: 'nested', properties: array} 93 | */ 94 | public static function nested(array $properties = []): array 95 | { 96 | return [ 97 | 'type' => 'nested', 98 | 'properties' => array_filter(array_merge([ 99 | 'id' => AbstractElasticsearchDefinition::KEYWORD_FIELD, 100 | '_count' => AbstractElasticsearchDefinition::INT_FIELD, 101 | ], $properties)), 102 | ]; 103 | } 104 | 105 | /** 106 | * @return array 107 | */ 108 | private function getCustomFieldsMapping(string $entity, Context $context): array 109 | { 110 | $fieldMapping = $this->indexingUtils->getCustomFieldTypes($entity, $context); 111 | 112 | $mapping = [ 113 | 'type' => 'object', 114 | 'dynamic' => true, 115 | 'properties' => [], 116 | ]; 117 | 118 | foreach ($fieldMapping as $name => $type) { 119 | /** @var array $esType */ 120 | $esType = CustomFieldUpdater::getTypeFromCustomFieldType($type); 121 | 122 | $mapping['properties'][$name] = $esType; 123 | } 124 | 125 | return $mapping; 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /Framework/ElasticsearchIndexingUtils.php: -------------------------------------------------------------------------------- 1 | > 23 | */ 24 | private array $customFieldsTypes = []; 25 | 26 | /** 27 | * @internal 28 | */ 29 | public function __construct( 30 | private readonly Connection $connection, 31 | private readonly EventDispatcherInterface $eventDispatcher, 32 | private readonly ParameterBagInterface $parameterBag, 33 | ) { 34 | } 35 | 36 | /** 37 | * @throws Exception 38 | * 39 | * @return array 40 | */ 41 | public function getCustomFieldTypes(string $entity, Context $context): array 42 | { 43 | if (\array_key_exists($entity, $this->customFieldsTypes)) { 44 | return $this->customFieldsTypes[$entity]; 45 | } 46 | 47 | $mappingKey = \sprintf('elasticsearch.%s.custom_fields_mapping', $entity); 48 | $customFieldsMapping = $this->parameterBag->has($mappingKey) ? $this->parameterBag->get($mappingKey) : []; 49 | 50 | /** @var array $mappings */ 51 | $mappings = $this->connection->fetchAllKeyValue(' 52 | SELECT 53 | custom_field.`name`, 54 | custom_field.type 55 | FROM custom_field_set_relation 56 | INNER JOIN custom_field ON(custom_field.set_id = custom_field_set_relation.set_id) 57 | WHERE custom_field_set_relation.entity_name = :entity 58 | ', ['entity' => $entity]) + $customFieldsMapping; 59 | 60 | $event = new ElasticsearchCustomFieldsMappingEvent($entity, $mappings, $context); 61 | 62 | $this->eventDispatcher->dispatch($event); 63 | 64 | $this->customFieldsTypes[$entity] = $event->getMappings(); 65 | 66 | return $this->customFieldsTypes[$entity]; 67 | } 68 | 69 | /** 70 | * @description strip html tags from text and truncate to 32766 characters 71 | */ 72 | public static function stripText(string $text): string 73 | { 74 | // Remove all html elements to save up space 75 | $text = strip_tags($text); 76 | 77 | if (mb_strlen($text) >= self::TEXT_MAX_LENGTH) { 78 | return mb_substr($text, 0, self::TEXT_MAX_LENGTH); 79 | } 80 | 81 | return $text; 82 | } 83 | 84 | /** 85 | * @param array $record 86 | * 87 | * @throws \JsonException 88 | * 89 | * @return array 90 | */ 91 | public static function parseJson(array $record, string $field): array 92 | { 93 | if (!\array_key_exists($field, $record)) { 94 | return []; 95 | } 96 | 97 | return json_decode($record[$field] ?? '[]', true, 512, \JSON_THROW_ON_ERROR); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /Framework/ElasticsearchLanguageProvider.php: -------------------------------------------------------------------------------- 1 | $languageRepository 23 | */ 24 | public function __construct( 25 | private readonly EntityRepository $languageRepository, 26 | private readonly EventDispatcherInterface $eventDispatcher 27 | ) { 28 | } 29 | 30 | public function getLanguages(Context $context): LanguageCollection 31 | { 32 | $criteria = new Criteria(); 33 | $criteria->addFilter(new NandFilter([new EqualsFilter('salesChannels.id', null)])); 34 | $criteria->addSorting(new FieldSorting('id')); 35 | 36 | $this->eventDispatcher->dispatch(new ElasticsearchIndexerLanguageCriteriaEvent($criteria, $context)); 37 | 38 | return $this->languageRepository->search($criteria, $context)->getEntities(); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /Framework/ElasticsearchOutdatedIndexDetector.php: -------------------------------------------------------------------------------- 1 | 23 | */ 24 | public function get(): ?array 25 | { 26 | $allIndices = $this->getAllIndices(); 27 | 28 | if (empty($allIndices)) { 29 | return []; 30 | } 31 | 32 | $indicesToBeDeleted = []; 33 | foreach ($allIndices as $index) { 34 | if (\count($index['aliases']) > 0) { 35 | continue; 36 | } 37 | 38 | $indicesToBeDeleted[] = $index['settings']['index']['provided_name']; 39 | } 40 | 41 | return $indicesToBeDeleted; 42 | } 43 | 44 | /** 45 | * @return array 46 | */ 47 | public function getAllUsedIndices(): array 48 | { 49 | $allIndices = $this->getAllIndices(); 50 | 51 | return array_map(fn (array $index) => $index['settings']['index']['provided_name'], $allIndices); 52 | } 53 | 54 | /** 55 | * @return array 56 | */ 57 | private function getPrefixes(): array 58 | { 59 | $definitions = $this->registry->getDefinitions(); 60 | 61 | $prefixes = []; 62 | 63 | foreach ($definitions as $definition) { 64 | $prefixes[] = \sprintf('%s_*', $this->helper->getIndexName($definition->getEntityDefinition())); 65 | } 66 | 67 | return $prefixes; 68 | } 69 | 70 | /** 71 | * @return array{aliases: array, settings: array}[] 72 | */ 73 | private function getAllIndices(): array 74 | { 75 | $prefixes = array_chunk($this->getPrefixes(), 5); 76 | 77 | $allIndices = []; 78 | 79 | foreach ($prefixes as $prefix) { 80 | $indices = $this->client->indices()->get( 81 | ['index' => implode(',', $prefix)] 82 | ); 83 | 84 | $allIndices = array_merge($allIndices, $indices); 85 | } 86 | 87 | return $allIndices; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Framework/ElasticsearchRangeAggregation.php: -------------------------------------------------------------------------------- 1 | > 19 | */ 20 | private array $ranges = []; 21 | 22 | /** 23 | * @param array> $ranges 24 | */ 25 | public function __construct( 26 | string $name, 27 | string $field, 28 | array $ranges 29 | ) { 30 | parent::__construct($name); 31 | 32 | $this->setField($field); 33 | $this->setRanges($ranges); 34 | } 35 | 36 | /** 37 | * @param array> $ranges 38 | */ 39 | public function setRanges(array $ranges): void 40 | { 41 | $this->ranges = $ranges; 42 | } 43 | 44 | /** 45 | * @return array> 46 | */ 47 | public function getRanges(): array 48 | { 49 | return $this->ranges; 50 | } 51 | 52 | public function getType(): string 53 | { 54 | return 'ranges'; 55 | } 56 | 57 | /** 58 | * {@inheritdoc} 59 | * 60 | * @return array 61 | */ 62 | protected function getArray(): array 63 | { 64 | return [ 65 | 'field' => $this->getField(), 66 | 'ranges' => $this->getRanges(), 67 | ]; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Framework/ElasticsearchRegistry.php: -------------------------------------------------------------------------------- 1 | definitions; 25 | } 26 | 27 | /** 28 | * @return iterable 29 | */ 30 | public function getDefinitionNames(): iterable 31 | { 32 | $names = []; 33 | 34 | foreach ($this->getDefinitions() as $definition) { 35 | $names[] = $definition->getEntityDefinition()->getEntityName(); 36 | } 37 | 38 | return $names; 39 | } 40 | 41 | public function get(string $entityName): ?AbstractElasticsearchDefinition 42 | { 43 | foreach ($this->getDefinitions() as $definition) { 44 | if ($definition->getEntityDefinition()->getEntityName() === $entityName) { 45 | return $definition; 46 | } 47 | } 48 | 49 | return null; 50 | } 51 | 52 | public function has(string $entityName): bool 53 | { 54 | foreach ($this->getDefinitions() as $definition) { 55 | if ($definition->getEntityDefinition()->getEntityName() === $entityName) { 56 | return true; 57 | } 58 | } 59 | 60 | return false; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Framework/ElasticsearchStagingHandler.php: -------------------------------------------------------------------------------- 1 | checkElasticsearch || !$this->helper->allowIndexing()) { 24 | return; 25 | } 26 | 27 | if (!empty($this->detector->getAllUsedIndices())) { 28 | $event->io->error('Found existing Elasticsearch indices, please delete them before setting up a staging environment or consider setting a index prefix'); 29 | $event->canceled = true; 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Framework/Indexing/CreateAliasTask.php: -------------------------------------------------------------------------------- 1 | get('elasticsearch.enabled'); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Framework/Indexing/CreateAliasTaskHandler.php: -------------------------------------------------------------------------------- 1 | $scheduledTaskRepository 30 | * @param array $config 31 | */ 32 | public function __construct( 33 | EntityRepository $scheduledTaskRepository, 34 | LoggerInterface $logger, 35 | private readonly Client $client, 36 | private readonly Connection $connection, 37 | private readonly ElasticsearchHelper $elasticsearchHelper, 38 | private readonly array $config, 39 | private readonly EventDispatcherInterface $eventDispatcher, 40 | ) { 41 | parent::__construct($scheduledTaskRepository, $logger); 42 | } 43 | 44 | public function run(): void 45 | { 46 | try { 47 | $this->handleQueue(); 48 | } catch (\Throwable $e) { 49 | // catch exception - otherwise the task will never be called again 50 | $this->elasticsearchHelper->logAndThrowException($e); 51 | } 52 | } 53 | 54 | private function createAlias(string $index, string $alias): void 55 | { 56 | $exist = $this->client->indices()->existsAlias(['name' => $alias]); 57 | 58 | if (!$exist) { 59 | $this->client->indices()->refresh([ 60 | 'index' => $index, 61 | ]); 62 | $this->client->indices()->putAlias(['index' => $index, 'name' => $alias]); 63 | 64 | return; 65 | } 66 | 67 | $current = $this->client->indices()->getAlias(['name' => $alias]); 68 | $current = array_keys($current); 69 | 70 | $actions = []; 71 | foreach ($current as $value) { 72 | if ($value === $index) { 73 | continue; 74 | } 75 | $actions[] = ['remove' => ['index' => $value, 'alias' => $alias]]; 76 | } 77 | $actions[] = ['add' => ['index' => $index, 'alias' => $alias]]; 78 | 79 | $this->client->indices()->updateAliases(['body' => ['actions' => $actions]]); 80 | } 81 | 82 | private function handleQueue(): void 83 | { 84 | $indices = $this->connection->fetchAllAssociative('SELECT * FROM elasticsearch_index_task'); 85 | if (empty($indices)) { 86 | return; 87 | } 88 | 89 | $changes = []; 90 | 91 | foreach ($indices as $row) { 92 | $index = $row['index']; 93 | $count = (int) $row['doc_count']; 94 | 95 | $this->client->indices()->refresh(['index' => $index]); 96 | 97 | if ($count > 0) { 98 | continue; 99 | } 100 | 101 | $alias = $row['alias']; 102 | 103 | $this->createAlias($index, $alias); 104 | 105 | $this->client->indices()->putSettings([ 106 | 'index' => $index, 107 | 'body' => [ 108 | 'number_of_replicas' => $this->config['settings']['index']['number_of_replicas'], 109 | 'refresh_interval' => null, 110 | ], 111 | ]); 112 | 113 | $this->connection->executeStatement( 114 | 'DELETE FROM elasticsearch_index_task WHERE id = :id', 115 | ['id' => $row['id']] 116 | ); 117 | 118 | $changes[(string) $index] = $alias; 119 | } 120 | 121 | $this->eventDispatcher->dispatch(new ElasticsearchIndexAliasSwitchedEvent($changes)); 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Framework/Indexing/ElasticsearchIndexingMessage.php: -------------------------------------------------------------------------------- 1 | data; 25 | } 26 | 27 | public function getOffset(): ?IndexerOffset 28 | { 29 | return $this->offset; 30 | } 31 | 32 | public function getContext(): Context 33 | { 34 | return $this->context; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Framework/Indexing/Event/ElasticsearchIndexAliasSwitchedEvent.php: -------------------------------------------------------------------------------- 1 | $changes 12 | */ 13 | public function __construct(private readonly array $changes) 14 | { 15 | } 16 | 17 | /** 18 | * Returns the index as key and the alias as value. 19 | * 20 | * @return array 21 | */ 22 | public function getChanges(): array 23 | { 24 | return $this->changes; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Framework/Indexing/Event/ElasticsearchIndexConfigEvent.php: -------------------------------------------------------------------------------- 1 | $config 15 | */ 16 | public function __construct( 17 | private readonly string $indexName, 18 | private array $config, 19 | private readonly AbstractElasticsearchDefinition $definition, 20 | private readonly Context $context 21 | ) { 22 | } 23 | 24 | public function getIndexName(): string 25 | { 26 | return $this->indexName; 27 | } 28 | 29 | /** 30 | * @return array 31 | */ 32 | public function getConfig(): array 33 | { 34 | return $this->config; 35 | } 36 | 37 | public function getDefinition(): AbstractElasticsearchDefinition 38 | { 39 | return $this->definition; 40 | } 41 | 42 | /** 43 | * @param array $config 44 | */ 45 | public function setConfig(array $config): void 46 | { 47 | $this->config = $config; 48 | } 49 | 50 | public function getContext(): Context 51 | { 52 | return $this->context; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /Framework/Indexing/Event/ElasticsearchIndexCreatedEvent.php: -------------------------------------------------------------------------------- 1 | indexName; 20 | } 21 | 22 | public function getDefinition(): AbstractElasticsearchDefinition 23 | { 24 | return $this->definition; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /Framework/Indexing/Event/ElasticsearchIndexIteratorEvent.php: -------------------------------------------------------------------------------- 1 | context; 23 | } 24 | 25 | public function getCriteria(): Criteria 26 | { 27 | return $this->criteria; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Framework/Indexing/IndexCreator.php: -------------------------------------------------------------------------------- 1 | 18 | */ 19 | private readonly array $config; 20 | 21 | /** 22 | * @internal 23 | * 24 | * @param array $config 25 | */ 26 | public function __construct( 27 | private readonly Client $client, 28 | array $config, 29 | private readonly IndexMappingProvider $mappingProvider, 30 | private readonly EventDispatcherInterface $eventDispatcher 31 | ) { 32 | if (isset($config['settings']['index'])) { 33 | if (\array_key_exists('number_of_shards', $config['settings']['index']) && $config['settings']['index']['number_of_shards'] === null) { 34 | unset($config['settings']['index']['number_of_shards']); 35 | } 36 | 37 | if (\array_key_exists('number_of_replicas', $config['settings']['index']) && $config['settings']['index']['number_of_replicas'] === null) { 38 | unset($config['settings']['index']['number_of_replicas']); 39 | } 40 | } 41 | 42 | $this->config = $config; 43 | } 44 | 45 | public function createIndex(AbstractElasticsearchDefinition $definition, string $index, string $alias, Context $context): void 46 | { 47 | // @codeCoverageIgnoreStart - does not execute if there's no index yet 48 | if ($this->indexExists($index)) { 49 | $this->client->indices()->delete(['index' => $index]); 50 | } 51 | // @codeCoverageIgnoreEnd 52 | 53 | $mapping = $this->mappingProvider->build($definition, $context); 54 | 55 | $body = array_merge( 56 | $this->config, 57 | ['mappings' => $mapping] 58 | ); 59 | 60 | $event = new ElasticsearchIndexConfigEvent($index, $body, $definition, $context); 61 | $this->eventDispatcher->dispatch($event); 62 | 63 | $this->client->indices()->create([ 64 | 'index' => $index, 65 | 'body' => $event->getConfig(), 66 | ]); 67 | 68 | $this->createAliasIfNotExisting($index, $alias); 69 | 70 | $this->eventDispatcher->dispatch(new ElasticsearchIndexCreatedEvent($index, $definition)); 71 | } 72 | 73 | public function aliasExists(string $alias): bool 74 | { 75 | return $this->client->indices()->existsAlias(['name' => $alias]); 76 | } 77 | 78 | private function indexExists(string $index): bool 79 | { 80 | return $this->client->indices()->exists(['index' => $index]); 81 | } 82 | 83 | private function createAliasIfNotExisting(string $index, string $alias): void 84 | { 85 | $exist = $this->client->indices()->existsAlias(['name' => $alias]); 86 | 87 | if ($exist) { 88 | return; 89 | } 90 | 91 | $this->client->indices()->refresh([ 92 | 'index' => $index, 93 | ]); 94 | 95 | $this->client->indices()->putAlias(['index' => $index, 'name' => $alias]); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Framework/Indexing/IndexMappingProvider.php: -------------------------------------------------------------------------------- 1 | $mapping 16 | */ 17 | public function __construct( 18 | private readonly array $mapping, 19 | ) { 20 | } 21 | 22 | /** 23 | * @return array 24 | */ 25 | public function build(AbstractElasticsearchDefinition $definition, Context $context): array 26 | { 27 | $mapping = $definition->getMapping($context); 28 | 29 | return array_merge_recursive($mapping, $this->mapping); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Framework/Indexing/IndexMappingUpdater.php: -------------------------------------------------------------------------------- 1 | elasticsearchHelper->allowIndexing()) { 34 | return; 35 | } 36 | 37 | $entitiesToReindex = $this->storage->get(SystemUpdateListener::CONFIG_KEY, []) ?? []; 38 | 39 | if (\is_string($entitiesToReindex)) { 40 | $entitiesToReindex = \json_decode($entitiesToReindex, true); 41 | } 42 | 43 | if (!\is_array($entitiesToReindex)) { 44 | $entitiesToReindex = []; 45 | } 46 | 47 | foreach ($this->registry->getDefinitions() as $definition) { 48 | $indexName = $this->elasticsearchHelper->getIndexName($definition->getEntityDefinition()); 49 | 50 | try { 51 | $this->client->indices()->putMapping([ 52 | 'index' => $indexName, 53 | 'body' => $this->indexMappingProvider->build($definition, $context), 54 | ]); 55 | } catch (BadRequest400Exception $exception) { 56 | if (str_contains($exception->getMessage(), 'cannot be changed from type') || str_contains($exception->getMessage(), 'can\'t merge a non object mapping')) { 57 | $entitiesToReindex[] = $definition->getEntityDefinition()->getEntityName(); 58 | 59 | $exception = ElasticsearchProductException::cannotChangeFieldType($exception); 60 | } 61 | 62 | $this->elasticsearchHelper->logAndThrowException($exception); 63 | } catch (Missing404Exception $exception) { 64 | $this->elasticsearchHelper->logAndThrowException($exception); 65 | } 66 | } 67 | 68 | if (!empty($entitiesToReindex)) { 69 | $this->storage->set(SystemUpdateListener::CONFIG_KEY, \array_values(\array_unique($entitiesToReindex))); 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Framework/Indexing/IndexerOffset.php: -------------------------------------------------------------------------------- 1 | 12 | */ 13 | protected array $definitions; 14 | 15 | /** 16 | * @var list 17 | */ 18 | protected array $allDefinitions; 19 | 20 | protected ?string $definition = null; 21 | 22 | /** 23 | * @param iterable $mappingDefinitions 24 | * @param array{offset: int|null}|null $lastId 25 | */ 26 | public function __construct( 27 | iterable $mappingDefinitions, 28 | protected ?int $timestamp, 29 | protected ?array $lastId = null 30 | ) { 31 | $mapping = []; 32 | /** @var string $mappingDefinition */ 33 | foreach ($mappingDefinitions as $mappingDefinition) { 34 | $mapping[] = $mappingDefinition; 35 | } 36 | 37 | $this->allDefinitions = $mapping; 38 | $this->definitions = $mapping; 39 | 40 | $this->selectNextDefinition(); 41 | } 42 | 43 | public function selectNextDefinition(): void 44 | { 45 | $this->definition = array_shift($this->definitions); 46 | } 47 | 48 | public function resetDefinitions(): void 49 | { 50 | $this->definitions = $this->allDefinitions; 51 | $this->definition = array_shift($this->definitions); 52 | } 53 | 54 | public function hasNextDefinition(): bool 55 | { 56 | return !empty($this->definitions); 57 | } 58 | 59 | /** 60 | * @return list 61 | */ 62 | public function getDefinitions(): array 63 | { 64 | return $this->definitions; 65 | } 66 | 67 | public function getTimestamp(): ?int 68 | { 69 | return $this->timestamp; 70 | } 71 | 72 | /** 73 | * @return array{offset: int|null}|null 74 | */ 75 | public function getLastId(): ?array 76 | { 77 | return $this->lastId; 78 | } 79 | 80 | public function getDefinition(): ?string 81 | { 82 | return $this->definition; 83 | } 84 | 85 | /** 86 | * @param array{offset: int|null}|null $lastId 87 | */ 88 | public function setLastId(?array $lastId): void 89 | { 90 | $this->lastId = $lastId; 91 | } 92 | 93 | /** 94 | * @internal This method is internal and will be used by Symfony serializer 95 | * 96 | * @return array 97 | */ 98 | public function getAllDefinitions(): array 99 | { 100 | return $this->allDefinitions; 101 | } 102 | 103 | /** 104 | * @param list $allDefinitions 105 | * 106 | * @internal This method is internal and will be used by Symfony serializer 107 | */ 108 | public function setAllDefinitions(array $allDefinitions): void 109 | { 110 | $this->allDefinitions = $allDefinitions; 111 | } 112 | 113 | /** 114 | * @param list $definitions 115 | * 116 | * @internal This method is internal and will be used by Symfony serializer 117 | */ 118 | public function setDefinitions(array $definitions): void 119 | { 120 | $this->definitions = $definitions; 121 | } 122 | 123 | /** 124 | * @internal This method is internal and will be used by Symfony serializer 125 | */ 126 | public function setDefinition(?string $definition): void 127 | { 128 | $this->definition = $definition; 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /Framework/Indexing/IndexingDto.php: -------------------------------------------------------------------------------- 1 | ids = array_values($ids); 18 | } 19 | 20 | public function getIds(): array 21 | { 22 | return $this->ids; 23 | } 24 | 25 | public function getIndex(): string 26 | { 27 | return $this->index; 28 | } 29 | 30 | public function getEntity(): string 31 | { 32 | return $this->entity; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/cheapest_price.groovy: -------------------------------------------------------------------------------- 1 | double getPrice(def accessors, def doc, def decimals, def round, def multiplier) { 2 | for (accessor in accessors) { 3 | def key = accessor['key']; 4 | 5 | if (!doc.containsKey(key) || doc[key].empty) { 6 | continue; 7 | } 8 | 9 | def factor = accessor['factor']; 10 | def value = doc[key].value * factor; 11 | 12 | value = Math.round(value * decimals); 13 | value = (double) value / decimals; 14 | 15 | if (!round) { 16 | return (double) value; 17 | } 18 | 19 | value = Math.round(value * multiplier); 20 | 21 | value = (double) value / multiplier; 22 | 23 | return (double) value; 24 | } 25 | 26 | return 0; 27 | } 28 | 29 | return getPrice(params['accessors'], doc, params['decimals'], params['round'], params['multiplier']); 30 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/cheapest_price_filter.groovy: -------------------------------------------------------------------------------- 1 | double getPrice(def accessors, def doc, def decimals, def round, def multiplier) { 2 | for (accessor in accessors) { 3 | def key = accessor['key']; 4 | if (!doc.containsKey(key) || doc[key].empty) { 5 | continue; 6 | } 7 | 8 | def factor = accessor['factor']; 9 | def value = doc[key].value * factor; 10 | 11 | value = Math.round(value * decimals); 12 | value = (double) value / decimals; 13 | 14 | if (!round) { 15 | return (double) value; 16 | } 17 | 18 | value = Math.round(value * multiplier); 19 | 20 | value = (double) value / multiplier; 21 | 22 | return (double) value; 23 | } 24 | 25 | return 0; 26 | } 27 | 28 | def price = getPrice(params['accessors'], doc, params['decimals'], params['round'], params['multiplier']); 29 | 30 | def match = true; 31 | if (params.containsKey('eq')) { 32 | match = match && price == params['eq']; 33 | } 34 | if (params.containsKey('gte')) { 35 | match = match && price >= params['gte']; 36 | } 37 | if (params.containsKey('gt')) { 38 | match = match && price > params['gt']; 39 | } 40 | if (params.containsKey('lte')) { 41 | match = match && price <= params['lte']; 42 | } 43 | if (params.containsKey('lt')) { 44 | match = match && price < params['lt']; 45 | } 46 | 47 | return match; 48 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/cheapest_price_percentage.groovy: -------------------------------------------------------------------------------- 1 | double getPercentage(def accessors, def doc) { 2 | for (accessor in accessors) { 3 | def key = accessor['key']; 4 | if (!doc.containsKey(key) || doc[key].empty) { 5 | continue; 6 | } 7 | 8 | return (double) doc[key].value; 9 | } 10 | 11 | return 0; 12 | } 13 | 14 | return getPercentage(params['accessors'], doc); 15 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/cheapest_price_percentage_filter.groovy: -------------------------------------------------------------------------------- 1 | String getPercentageKey(def accessors, def doc) { 2 | for (accessor in accessors) { 3 | def key = accessor['key']; 4 | if (!doc.containsKey(key) || doc[key].empty) { 5 | continue; 6 | } 7 | 8 | return key; 9 | } 10 | 11 | return ''; 12 | } 13 | 14 | def percentageKey = getPercentageKey(params['accessors'], doc); 15 | 16 | if (percentageKey == '') { 17 | if (params.containsKey('eq') && params['eq'] === null) { 18 | return true; 19 | } 20 | 21 | return false; 22 | } 23 | 24 | def percentage = (double) doc[percentageKey].value; 25 | 26 | def match = true; 27 | if (params.containsKey('eq')) { 28 | match = match && percentage == params['eq']; 29 | } 30 | if (params.containsKey('gte')) { 31 | match = match && percentage >= params['gte']; 32 | } 33 | if (params.containsKey('gt')) { 34 | match = match && percentage > params['gt']; 35 | } 36 | if (params.containsKey('lte')) { 37 | match = match && percentage <= params['lte']; 38 | } 39 | if (params.containsKey('lt')) { 40 | match = match && percentage < params['lt']; 41 | } 42 | 43 | return match; 44 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/numeric_translated_field_sorting.groovy: -------------------------------------------------------------------------------- 1 | def languages = params['languages']; 2 | def suffix = params.containsKey('suffix') ? '.' + params['suffix'] : ''; 3 | 4 | for (int i = 0; i < languages.length; i++) { 5 | def field_name = params['field'] + '.' + languages[i] + suffix; 6 | 7 | if (doc[field_name].size() > 0 && doc[field_name].value != null && doc[field_name].value.toString().length() > 0) { 8 | def fieldValue = doc[field_name].value; 9 | 10 | return fieldValue; 11 | } 12 | } 13 | 14 | if (params['order'] == 'asc') { 15 | return Double.MAX_VALUE; 16 | } 17 | 18 | return Double.MIN_VALUE; 19 | -------------------------------------------------------------------------------- /Framework/Indexing/Scripts/translated_field_sorting.groovy: -------------------------------------------------------------------------------- 1 | def languages = params['languages']; 2 | def suffix = params.containsKey('suffix') ? '.' + params['suffix'] : ''; 3 | 4 | for (int i = 0; i < languages.length; i++) { 5 | def field_name = params['field'] + '.' + languages[i] + suffix; 6 | 7 | if (doc[field_name].size() > 0 && doc[field_name].value != null && doc[field_name].value.toString().length() > 0) { 8 | def fieldValue = doc[field_name].value; 9 | 10 | return fieldValue.toString(); 11 | } 12 | } 13 | 14 | return ''; 15 | -------------------------------------------------------------------------------- /Framework/SystemUpdateListener.php: -------------------------------------------------------------------------------- 1 | mappingUpdater->update($event->getContext()); 36 | 37 | $entitiesToReindex = $this->storage->get(self::CONFIG_KEY, []); 38 | 39 | if (empty($entitiesToReindex)) { 40 | return; 41 | } 42 | 43 | $offset = null; 44 | while ($message = $this->indexer->iterate($offset)) { 45 | $offset = $message->getOffset(); 46 | 47 | $this->messageBus->dispatch($message); 48 | } 49 | 50 | $this->storage->remove(self::CONFIG_KEY); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2019 shopware AG 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 4 | documentation files (the "Software"), to deal in the Software without restriction, including without limitation the 5 | rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit 6 | persons to whom the Software is furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in all 9 | copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO 12 | THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 13 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 14 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Migration/Traits/ElasticsearchTriggerTrait.php: -------------------------------------------------------------------------------- 1 | executeStatement( 16 | ' 17 | REPLACE INTO app_config (`key`, `value`) VALUES 18 | (?, ?) 19 | ', 20 | [SystemUpdateListener::CONFIG_KEY, json_encode(['*'], \JSON_THROW_ON_ERROR)] 21 | ); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /Migration/V6_5/Migration1689083660ElasticsearchIndexTask.php: -------------------------------------------------------------------------------- 1 | executeStatement(' 28 | CREATE TABLE `elasticsearch_index_task` ( 29 | `id` binary(16) NOT NULL, 30 | `index` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 31 | `alias` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 32 | `entity` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 33 | `doc_count` int(11) NOT NULL, 34 | PRIMARY KEY (`id`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci 36 | '); 37 | } 38 | 39 | public function updateDestructive(Connection $connection): void 40 | { 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Migration/V6_5/Migration1689084023AdminElasticsearchIndexTask.php: -------------------------------------------------------------------------------- 1 | executeStatement(' 28 | CREATE TABLE IF NOT EXISTS `admin_elasticsearch_index_task` ( 29 | `id` binary(16) NOT NULL, 30 | `index` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 31 | `alias` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 32 | `entity` varchar(500) COLLATE utf8mb4_unicode_ci NOT NULL, 33 | `doc_count` int(11) NOT NULL, 34 | PRIMARY KEY (`id`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci 36 | '); 37 | } 38 | 39 | public function updateDestructive(Connection $connection): void 40 | { 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Product/AbstractProductSearchQueryBuilder.php: -------------------------------------------------------------------------------- 1 | $setIds 26 | * 27 | * @return array> 28 | */ 29 | public function fetchCustomFieldsForSets(array $setIds): array 30 | { 31 | /** @var list $result */ 32 | $result = $this->connection->fetchAllAssociative( 33 | <<<'SQL' 34 | SELECT LOWER(HEX(set_id)) as set_id, LOWER(HEX(id)) AS id, name, type 35 | FROM custom_field 36 | WHERE set_id IN (:setIds) 37 | SQL, 38 | ['setIds' => Uuid::fromHexToBytesList($setIds)], 39 | ['setIds' => ArrayParameterType::STRING] 40 | ); 41 | 42 | /** @var array> $customFields */ 43 | $customFields = FetchModeHelper::group($result); 44 | 45 | return $customFields; 46 | } 47 | 48 | /** 49 | * @param array $customFieldIds 50 | * 51 | * @return array 52 | */ 53 | public function fetchFieldSetIds(array $customFieldIds): array 54 | { 55 | /** @var array $result */ 56 | $result = $this->connection->fetchAllKeyValue( 57 | 'SELECT LOWER(HEX(id)), LOWER(HEX(set_id)) FROM custom_field WHERE id IN (:ids)', 58 | ['ids' => Uuid::fromHexToBytesList($customFieldIds)], 59 | ['ids' => ArrayParameterType::STRING] 60 | ); 61 | 62 | return $result; 63 | } 64 | 65 | /** 66 | * @param array $fieldSetIds 67 | * 68 | * @return array> 69 | */ 70 | public function fetchFieldSetEntityMappings(array $fieldSetIds): array 71 | { 72 | /** @var list $fieldSets */ 73 | $fieldSets = $this->connection->fetchAllAssociative( 74 | <<<'SQL' 75 | SELECT LOWER(HEX(custom_field_set.id)) AS set_id, entity_name 76 | FROM custom_field_set 77 | LEFT JOIN custom_field_set_relation ON custom_field_set.id = custom_field_set_relation.set_id 78 | WHERE custom_field_set.id IN (:ids) 79 | SQL, 80 | ['ids' => Uuid::fromHexToBytesList($fieldSetIds)], 81 | ['ids' => ArrayParameterType::STRING] 82 | ); 83 | 84 | return FetchModeHelper::group($fieldSets, static fn (array $row): string => (string) $row['entity_name']); 85 | } 86 | 87 | /** 88 | * @return array 89 | */ 90 | public function fetchLanguageIds(): array 91 | { 92 | /** @var array $languageIds */ 93 | $languageIds = $this->connection->fetchFirstColumn('SELECT LOWER(HEX(`id`)) FROM language'); 94 | 95 | return $languageIds; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Product/ElasticsearchProductException.php: -------------------------------------------------------------------------------- 1 | 'onLanguageWritten', 31 | ]; 32 | } 33 | 34 | public function onLanguageWritten(EntityWrittenEvent $event): void 35 | { 36 | if (!$this->elasticsearchHelper->allowIndexing()) { 37 | return; 38 | } 39 | 40 | $context = $event->getContext(); 41 | 42 | foreach ($event->getWriteResults() as $writeResult) { 43 | if ($writeResult->getOperation() !== EntityWriteResult::OPERATION_INSERT) { 44 | continue; 45 | } 46 | 47 | foreach ($this->registry->getDefinitions() as $definition) { 48 | $indexName = $this->elasticsearchHelper->getIndexName($definition->getEntityDefinition()); 49 | 50 | // index doesn't exist, don't need to do anything 51 | if (!$this->client->indices()->exists(['index' => $indexName])) { 52 | continue; 53 | } 54 | 55 | $this->client->indices()->putMapping([ 56 | 'index' => $indexName, 57 | 'body' => [ 58 | 'properties' => $definition->getMapping($context)['properties'], 59 | ], 60 | ]); 61 | } 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Product/ProductSearchBuilder.php: -------------------------------------------------------------------------------- 1 | helper->allowSearch($this->productDefinition, $context->getContext(), $criteria)) { 31 | $this->decorated->build($request, $criteria, $context); 32 | 33 | return; 34 | } 35 | 36 | $search = $request->get('search'); 37 | 38 | if (\is_array($search)) { 39 | $term = implode(' ', $search); 40 | } else { 41 | $term = (string) $search; 42 | } 43 | 44 | $term = mb_substr(trim($term), 0, $this->searchTermMaxLength); 45 | 46 | if (empty($term)) { 47 | throw RoutingException::missingRequestParameter('search'); 48 | } 49 | 50 | // reset queries and set term to criteria. 51 | $criteria->resetQueries(); 52 | 53 | // elasticsearch will interpret this on demand 54 | $criteria->setTerm($term); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Product/ProductSearchQueryBuilder.php: -------------------------------------------------------------------------------- 1 | getTerm()); 44 | 45 | $tokens = $this->tokenizer->tokenize($originalTerm); 46 | $tokens = $this->tokenFilter->filter($tokens, $context); 47 | 48 | if (empty(array_filter($tokens))) { 49 | throw ElasticsearchException::emptyQuery(); 50 | } 51 | 52 | $searchConfig = $this->configLoader->load($context); 53 | 54 | $configs = array_map(function (array $item): SearchFieldConfig { 55 | return new SearchFieldConfig( 56 | (string) $item['field'], 57 | (float) $item['ranking'], 58 | (bool) $item['tokenize'], 59 | (bool) $item['and_logic'], 60 | ); 61 | }, $searchConfig); 62 | 63 | if (!$configs[0]->isAndLogic()) { 64 | $tokens = [$originalTerm]; 65 | } 66 | 67 | $queries = []; 68 | 69 | foreach ($tokens as $token) { 70 | $query = $this->tokenQueryBuilder->build( 71 | $this->productDefinition->getEntityName(), 72 | $token, 73 | $configs, 74 | $context, 75 | ); 76 | 77 | if ($query) { 78 | $queries[] = $query; 79 | } 80 | } 81 | 82 | if (empty($queries)) { 83 | throw ElasticsearchException::emptyQuery(); 84 | } 85 | 86 | if (\count($queries) === 1 && $queries[0] instanceof BoolQuery) { 87 | return $queries[0]; 88 | } 89 | 90 | $andSearch = $configs[0]->isAndLogic() ? BoolQuery::MUST : BoolQuery::SHOULD; 91 | 92 | $tokensQuery = new BoolQuery([$andSearch => $queries]); 93 | 94 | if (\in_array($originalTerm, $tokens, true)) { 95 | return $tokensQuery; 96 | } 97 | 98 | $originalTermQuery = $this->tokenQueryBuilder->build( 99 | $this->productDefinition->getEntityName(), 100 | $originalTerm, 101 | $configs, 102 | $context 103 | ); 104 | 105 | if (!$originalTermQuery) { 106 | return $tokensQuery; 107 | } 108 | 109 | $dismax = new DisMaxQuery(); 110 | 111 | $dismax->addQuery($tokensQuery); 112 | $dismax->addQuery($originalTermQuery); 113 | 114 | return $dismax; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /Product/ProductUpdater.php: -------------------------------------------------------------------------------- 1 | 'update', 31 | ProductStockAlteredEvent::class => 'update', 32 | ]; 33 | } 34 | 35 | public function update(ProductIndexerEvent|ProductStockAlteredEvent $event): void 36 | { 37 | $this->indexer->updateIds($this->definition, $event->getIds()); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Product/SearchConfigLoader.php: -------------------------------------------------------------------------------- 1 | 33 | */ 34 | public function load(Context $context): array 35 | { 36 | foreach ($context->getLanguageIdChain() as $languageId) { 37 | /** @var array $config */ 38 | $config = $this->connection->fetchAllAssociative( 39 | 'SELECT 40 | product_search_config.and_logic, 41 | product_search_config_field.field, 42 | product_search_config_field.tokenize, 43 | product_search_config_field.ranking 44 | 45 | FROM product_search_config 46 | INNER JOIN product_search_config_field ON(product_search_config_field.product_search_config_id = product_search_config.id) 47 | WHERE product_search_config.language_id = :languageId AND product_search_config_field.searchable = 1 AND product_search_config_field.field NOT IN(:excludedFields)', 48 | [ 49 | 'languageId' => Uuid::fromHexToBytes($languageId), 50 | 'excludedFields' => self::NOT_SUPPORTED_FIELDS, 51 | ], 52 | [ 53 | 'excludedFields' => ArrayParameterType::STRING, 54 | ] 55 | ); 56 | 57 | if (!empty($config)) { 58 | return $config; 59 | } 60 | } 61 | 62 | throw ElasticsearchProductException::configNotFound(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Product/SearchFieldConfig.php: -------------------------------------------------------------------------------- 1 | tokenize; 22 | } 23 | 24 | public function getRanking(): float 25 | { 26 | return $this->ranking; 27 | } 28 | 29 | public function getField(): string 30 | { 31 | return $this->field; 32 | } 33 | 34 | public function isCustomField(): bool 35 | { 36 | return str_contains($this->field, 'customFields'); 37 | } 38 | 39 | public function isAndLogic(): bool 40 | { 41 | return $this->andLogic; 42 | } 43 | 44 | public function setRanking(float $ranking): void 45 | { 46 | $this->ranking = $ranking; 47 | } 48 | 49 | public function usePrefixMatch(): bool 50 | { 51 | return $this->prefixMatch; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Product/SearchKeywordReplacement.php: -------------------------------------------------------------------------------- 1 | $ids 24 | */ 25 | public function update(array $ids, Context $context): void 26 | { 27 | if ($this->helper->allowIndexing()) { 28 | return; 29 | } 30 | 31 | $this->decorated->update($ids, $context); 32 | } 33 | 34 | public function reset(): void 35 | { 36 | $this->decorated->reset(); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Product/StopwordTokenFilter.php: -------------------------------------------------------------------------------- 1 | 17 | */ 18 | private array $config = []; 19 | 20 | /** 21 | * @internal 22 | */ 23 | public function __construct( 24 | private readonly Connection $connection 25 | ) { 26 | } 27 | 28 | public function getDecorated(): AbstractTokenFilter 29 | { 30 | throw new DecorationPatternException(self::class); 31 | } 32 | 33 | /** 34 | * {@inheritdoc} 35 | */ 36 | public function filter(array $tokens, Context $context): array 37 | { 38 | if (empty($tokens)) { 39 | return $tokens; 40 | } 41 | 42 | $minSearchLength = $this->getMinSearchLength($context->getLanguageId()); 43 | 44 | if ($minSearchLength === null) { 45 | return $tokens; 46 | } 47 | 48 | return $this->searchTermLengthFilter($tokens, $minSearchLength); 49 | } 50 | 51 | public function reset(): void 52 | { 53 | $this->config = []; 54 | } 55 | 56 | /** 57 | * @param list $tokens 58 | * 59 | * @return list 60 | */ 61 | private function searchTermLengthFilter(array $tokens, int $minSearchTermLength): array 62 | { 63 | $filtered = []; 64 | foreach ($tokens as $tag) { 65 | $tag = trim($tag); 66 | 67 | if (empty($tag) || mb_strlen($tag) < $minSearchTermLength) { 68 | continue; 69 | } 70 | 71 | $filtered[] = $tag; 72 | } 73 | 74 | return $filtered; 75 | } 76 | 77 | private function getMinSearchLength(string $languageId): ?int 78 | { 79 | if (isset($this->config[$languageId])) { 80 | return $this->config[$languageId]; 81 | } 82 | 83 | $config = $this->connection->fetchAssociative(' 84 | SELECT `min_search_length` 85 | FROM product_search_config 86 | WHERE language_id = :languageId 87 | LIMIT 1 88 | ', ['languageId' => Uuid::fromHexToBytes($languageId)]); 89 | 90 | if (empty($config)) { 91 | return null; 92 | } 93 | 94 | return $this->config[$languageId] = (int) ($config['min_search_length'] ?? self::DEFAULT_MIN_SEARCH_TERM_LENGTH); 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /Profiler/DataCollector.php: -------------------------------------------------------------------------------- 1 | attributes->get(PlatformRequest::ATTRIBUTE_CONTEXT_OBJECT); 33 | $enabled = $this->enabled; 34 | $client = $this->client; 35 | 36 | if (empty($client->getCalledRequests()) && $context instanceof Context && $context->getSource() instanceof AdminApiSource) { 37 | $enabled = $this->adminEnabled; 38 | $client = $this->adminClient; 39 | } 40 | 41 | $clientRequests = array_merge($this->client->getCalledRequests(), $this->adminClient->getCalledRequests()); 42 | $this->data = [ 43 | 'enabled' => $enabled, 44 | 'requests' => $clientRequests, 45 | 'time' => 0, 46 | ]; 47 | 48 | if (!$enabled) { 49 | return; 50 | } 51 | 52 | foreach ($clientRequests as $calledRequest) { 53 | $this->data['time'] += $calledRequest['time']; 54 | } 55 | 56 | $this->data['clusterInfo'] = $client->cluster()->health(); 57 | $this->data['indices'] = $client->cat()->indices(); 58 | } 59 | 60 | public function getName(): string 61 | { 62 | return 'elasticsearch'; 63 | } 64 | 65 | public function reset(): void 66 | { 67 | $this->data = []; 68 | $this->client->resetRequests(); 69 | $this->adminClient->resetRequests(); 70 | } 71 | 72 | public function getTime(): float 73 | { 74 | $time = 0; 75 | 76 | foreach ($this->data['requests'] ?? [] as $calledRequest) { 77 | $time += $calledRequest['time']; 78 | } 79 | 80 | return (int) ($time * 1000); 81 | } 82 | 83 | public function getRequestAmount(): int 84 | { 85 | return is_countable($this->data['requests']) ? \count($this->data['requests']) : 0; 86 | } 87 | 88 | /** 89 | * @return RequestInfo[] 90 | */ 91 | public function getRequests(): array 92 | { 93 | return $this->data['requests'] ?? []; 94 | } 95 | 96 | /** 97 | * @return array{cluster_name: string, status: string, number_of_nodes: int} 98 | */ 99 | public function getClusterInfo(): array 100 | { 101 | return $this->data['clusterInfo']; 102 | } 103 | 104 | /** 105 | * @return array{index: string, status: string, pri: int, rep: int, 'docs.count': int}[] 106 | */ 107 | public function getIndices(): array 108 | { 109 | return $this->data['indices']; 110 | } 111 | 112 | public function isEnabled(): bool 113 | { 114 | return (bool) $this->data['enabled']; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /Profiler/ElasticsearchProfileCompilerPass.php: -------------------------------------------------------------------------------- 1 | getParameterBag()->resolveValue($container->getParameter('kernel.debug')); 18 | 19 | if (!$isDebugEnabled) { 20 | $container->removeDefinition(DataCollector::class); 21 | 22 | return; 23 | } 24 | 25 | $clientDecorator = new Definition(ClientProfiler::class); 26 | $clientDecorator->setArguments([ 27 | new Reference('shopware.es.profiled.client.inner'), 28 | ]); 29 | $clientDecorator->setDecoratedService(Client::class); 30 | 31 | $container->setDefinition('shopware.es.profiled.client', $clientDecorator); 32 | 33 | $adminClientDecorator = new Definition(ClientProfiler::class); 34 | $adminClientDecorator->setArguments([ 35 | new Reference('shopware.es.profiled.adminClient.inner'), 36 | ]); 37 | $adminClientDecorator->setDecoratedService('admin.openSearch.client'); 38 | 39 | $container->setDefinition('shopware.es.profiled.adminClient', $adminClientDecorator); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Elasticsearch Component 2 | ============== 3 | 4 | The Elasticsearch component is the elasticsearch adapter for the shopware/core. 5 | It contains the indexing of entities and an adapter for the entity search. 6 | 7 | This repository is considered **read-only**. Please send pull requests 8 | to our [main Shopware repository](https://github.com/shopware/shopware). 9 | 10 | Resources 11 | --------- 12 | 13 | * [Documentation](https://developer.shopware.com) 14 | * [Contributing](https://developer.shopware.com/docs/resources/guidelines/code/contribution.html) 15 | * [Report issues](https://github.com/shopware/shopware/issues) and 16 | [send Pull Requests](https://github.com/shopware/shopware/pulls) 17 | in the [main Shopware\Core repository](https://github.com/shopware/shopware) 18 | -------------------------------------------------------------------------------- /Resources/config/packages/elasticsearch.yaml: -------------------------------------------------------------------------------- 1 | elasticsearch: 2 | enabled: "%env(bool:SHOPWARE_ES_ENABLED)%" 3 | indexing_enabled: "%env(bool:SHOPWARE_ES_INDEXING_ENABLED)%" 4 | indexing_batch_size: "%env(int:SHOPWARE_ES_INDEXING_BATCH_SIZE)%" 5 | hosts: "%env(string:OPENSEARCH_URL)%" 6 | index_prefix: "%env(string:SHOPWARE_ES_INDEX_PREFIX)%" 7 | throw_exception: "%env(string:SHOPWARE_ES_THROW_EXCEPTION)%" 8 | search: 9 | timeout: 5s 10 | term_max_length: 300 11 | search_type: "query_then_fetch" 12 | administration: 13 | hosts: "%env(string:ADMIN_OPENSEARCH_URL)%" 14 | enabled: "%env(bool:SHOPWARE_ADMIN_ES_ENABLED)%" 15 | refresh_indices: "%env(bool:SHOPWARE_ADMIN_ES_REFRESH_INDICES)%" 16 | index_prefix: "%env(string:SHOPWARE_ADMIN_ES_INDEX_PREFIX)%" 17 | search: 18 | timeout: 5s 19 | term_max_length: 300 20 | search_type: "query_then_fetch" 21 | index_settings: 22 | number_of_shards: 3 23 | number_of_replicas: 3 24 | 'mapping.total_fields.limit': 50000 25 | 'mapping.nested_fields.limit': 500 26 | 'mapping.nested_objects.limit': 1000000 27 | max_result_window: 10000 28 | analysis: 29 | normalizer: 30 | sw_lowercase_normalizer: 31 | type: custom 32 | filter: [ 'lowercase' ] 33 | dynamic_templates: 34 | - keywords: 35 | match_mapping_type: string 36 | mapping: 37 | type: keyword 38 | normalizer: sw_lowercase_normalizer 39 | fields: 40 | text: 41 | type: text 42 | product: 43 | custom_fields_mapping: 44 | exclude_source: "%env(bool:SHOPWARE_ES_EXCLUDE_SOURCE)%" 45 | ssl: 46 | verify_server_cert: true 47 | index_settings: 48 | number_of_shards: null 49 | number_of_replicas: null 50 | 'mapping.total_fields.limit': 50000 51 | 'mapping.nested_fields.limit': 500 52 | 'mapping.nested_objects.limit': 1000000 53 | max_result_window: 10000 54 | analysis: 55 | normalizer: 56 | sw_lowercase_normalizer: 57 | type: custom 58 | filter: ['lowercase'] 59 | analyzer: 60 | sw_whitespace_analyzer: 61 | type: custom 62 | tokenizer: whitespace 63 | filter: ['lowercase'] 64 | sw_ngram_analyzer: 65 | type: custom 66 | tokenizer: whitespace 67 | filter: ['lowercase', 'sw_ngram_filter'] 68 | sw_english_analyzer: 69 | type: custom 70 | tokenizer: whitespace 71 | filter: ['lowercase', 'sw_english_stop_filter'] 72 | sw_german_analyzer: 73 | type: custom 74 | tokenizer: whitespace 75 | filter: ['lowercase', 'sw_german_stop_filter'] 76 | filter: 77 | sw_ngram_filter: 78 | type: ngram 79 | min_gram: 4 80 | max_gram: 5 81 | sw_english_stop_filter: 82 | type: 'stop' 83 | stopwords: '_english_' 84 | sw_german_stop_filter: 85 | type: 'stop' 86 | stopwords: '_german_' 87 | language_analyzer_mapping: 88 | en: sw_english_analyzer 89 | de: sw_german_analyzer 90 | gsw: sw_german_analyzer 91 | nds: sw_german_analyzer 92 | dynamic_templates: 93 | - keywords: 94 | match_mapping_type: string 95 | mapping: 96 | type: keyword 97 | normalizer: sw_lowercase_normalizer 98 | fields: 99 | text: 100 | type: text 101 | 102 | parameters: 103 | default_elasticsearch_prefix: "sw" 104 | default_whitespace: " " 105 | env(SHOPWARE_ES_ENABLED): "" 106 | env(SHOPWARE_ES_INDEXING_ENABLED): "" 107 | env(OPENSEARCH_URL): "" 108 | env(SHOPWARE_ES_INDEX_PREFIX): "sw" 109 | env(SHOPWARE_ES_THROW_EXCEPTION): "1" 110 | env(SHOPWARE_ADMIN_ES_ENABLED): "" 111 | env(ADMIN_OPENSEARCH_URL): "" 112 | env(SHOPWARE_ADMIN_ES_INDEX_PREFIX): "sw-admin" 113 | env(SHOPWARE_ADMIN_ES_REFRESH_INDICES): "" 114 | env(SHOPWARE_ES_INDEXING_BATCH_SIZE): "100" 115 | env(SHOPWARE_ES_EXCLUDE_SOURCE): "0" 116 | -------------------------------------------------------------------------------- /Resources/config/packages/framework.yaml: -------------------------------------------------------------------------------- 1 | framework: 2 | messenger: 3 | serializer: 4 | symfony_serializer: 5 | context: 6 | default_constructor_arguments: 7 | Shopware\Elasticsearch\Framework\Indexing\IndexerOffset: 8 | mappingDefinitions: [ ] 9 | -------------------------------------------------------------------------------- /Resources/config/packages/monolog.yaml: -------------------------------------------------------------------------------- 1 | monolog: 2 | channels: ['elasticsearch'] 3 | handlers: 4 | elasticsearch: 5 | type: rotating_file 6 | path: "%kernel.logs_dir%/elasticsearch_%kernel.environment%.log" 7 | max_files: 10 8 | level: "error" 9 | channels: [ "elasticsearch"] 10 | -------------------------------------------------------------------------------- /Resources/config/packages/test/elasticsearch.yaml: -------------------------------------------------------------------------------- 1 | elasticsearch: 2 | index_settings: 3 | number_of_shards: 1 4 | number_of_replicas: 0 5 | 6 | -------------------------------------------------------------------------------- /Resources/config/routes.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /Resources/views/Collector/icon.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /Sort/CountSort.php: -------------------------------------------------------------------------------- 1 | $params 14 | */ 15 | public function __construct( 16 | string $field, 17 | ?string $order = null, 18 | ?BuilderInterface $nestedFilter = null, 19 | array $params = [] 20 | ) { 21 | $path = explode('.', $field); 22 | array_pop($path); 23 | 24 | $params = array_merge( 25 | $params, 26 | [ 27 | 'mode' => 'sum', 28 | 'nested' => ['path' => implode('.', $path)], 29 | 'missing' => 0, 30 | ] 31 | ); 32 | 33 | $path[] = '_count'; 34 | 35 | parent::__construct(implode('.', $path), $order, $nestedFilter, $params); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Test/AdminElasticsearchTestBehaviour.php: -------------------------------------------------------------------------------- 1 | getDiContainer() 25 | ->get(ElasticsearchHelper::class) 26 | ->setEnabled(true); 27 | } 28 | 29 | #[After] 30 | public function disableElasticsearch(): void 31 | { 32 | $this->getDiContainer() 33 | ->get(ElasticsearchHelper::class) 34 | ->setEnabled(false); 35 | } 36 | 37 | #[Before] 38 | public function enableAdminElasticsearch(): void 39 | { 40 | $this->getDiContainer() 41 | ->get(AdminElasticsearchHelper::class) 42 | ->setEnabled(true); 43 | } 44 | 45 | #[After] 46 | public function disableAdminElasticsearch(): void 47 | { 48 | $this->getDiContainer() 49 | ->get(AdminElasticsearchHelper::class) 50 | ->setEnabled(false); 51 | } 52 | 53 | /** 54 | * @param array $input 55 | */ 56 | public function indexElasticSearch(array $input = []): void 57 | { 58 | $this->getDiContainer() 59 | ->get(ElasticsearchAdminIndexingCommand::class) 60 | ->run(new ArrayInput([...$input, '--no-queue' => true]), new NullOutput()); 61 | 62 | $this->runWorker(); 63 | 64 | $this->refreshIndex(); 65 | } 66 | 67 | public function refreshIndex(): void 68 | { 69 | $this->getDiContainer()->get(Client::class) 70 | ->indices() 71 | ->refresh(['index' => '_all']); 72 | } 73 | 74 | abstract protected function getDiContainer(): ContainerInterface; 75 | 76 | abstract protected function runWorker(): void; 77 | 78 | protected function clearElasticsearch(): void 79 | { 80 | $c = $this->getDiContainer(); 81 | 82 | $client = $c->get(Client::class); 83 | 84 | $indices = $client->indices()->get(['index' => EnvironmentHelper::getVariable('SHOPWARE_ADMIN_ES_INDEX_PREFIX') . '*']); 85 | 86 | foreach ($indices as $index) { 87 | $client->indices()->delete(['index' => $index['settings']['index']['provided_name']]); 88 | } 89 | 90 | $connection = $c->get(Connection::class); 91 | $connection->executeStatement('TRUNCATE admin_elasticsearch_index_task'); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /Test/ElasticsearchTestTestBehaviour.php: -------------------------------------------------------------------------------- 1 | getDiContainer() 31 | ->get(ElasticsearchHelper::class) 32 | ->setEnabled(true); 33 | } 34 | 35 | #[After] 36 | public function disableElasticsearch(): void 37 | { 38 | $this->getDiContainer() 39 | ->get(ElasticsearchHelper::class) 40 | ->setEnabled(false); 41 | } 42 | 43 | public function indexElasticSearch(): void 44 | { 45 | $this->getDiContainer() 46 | ->get(ElasticsearchIndexingCommand::class) 47 | ->run(new ArrayInput([]), new NullOutput()); 48 | 49 | $this->runWorker(); 50 | 51 | $this->refreshIndex(); 52 | } 53 | 54 | public function refreshIndex(): void 55 | { 56 | $this->getDiContainer()->get(Client::class) 57 | ->indices() 58 | ->refresh(['index' => '_all']); 59 | } 60 | 61 | protected function createEntityAggregator(): ElasticsearchEntityAggregator 62 | { 63 | $decorated = $this->createMock(EntityAggregator::class); 64 | 65 | $decorated 66 | ->expects(static::never()) 67 | ->method('aggregate'); 68 | 69 | return new ElasticsearchEntityAggregator( 70 | $this->getDiContainer()->get(ElasticsearchHelper::class), 71 | $this->getDiContainer()->get(Client::class), 72 | $decorated, 73 | $this->getDiContainer()->get(AbstractElasticsearchAggregationHydrator::class), 74 | $this->getDiContainer()->get('event_dispatcher'), 75 | '5s', 76 | 'dfs_query_then_fetch' 77 | ); 78 | } 79 | 80 | protected function createEntitySearcher(): ElasticsearchEntitySearcher 81 | { 82 | $decorated = $this->createMock(EntitySearcher::class); 83 | 84 | $decorated 85 | ->expects(static::never()) 86 | ->method('search'); 87 | 88 | return new ElasticsearchEntitySearcher( 89 | $this->getDiContainer()->get(Client::class), 90 | $decorated, 91 | $this->getDiContainer()->get(ElasticsearchHelper::class), 92 | $this->getDiContainer()->get(CriteriaParser::class), 93 | $this->getDiContainer()->get(AbstractElasticsearchSearchHydrator::class), 94 | $this->getDiContainer()->get('event_dispatcher'), 95 | '5s', 96 | 'dfs_query_then_fetch' 97 | ); 98 | } 99 | 100 | abstract protected function getDiContainer(): ContainerInterface; 101 | 102 | abstract protected function runWorker(): void; 103 | 104 | protected function clearElasticsearch(): void 105 | { 106 | $c = $this->getDiContainer(); 107 | 108 | $client = $c->get(Client::class); 109 | 110 | $indices = $client->indices()->get(['index' => EnvironmentHelper::getVariable('SHOPWARE_ES_INDEX_PREFIX') . '*']); 111 | 112 | foreach ($indices as $name => $index) { 113 | $client->indices()->delete(['index' => $name]); 114 | } 115 | 116 | $connection = $c->get(Connection::class); 117 | $connection->executeStatement('DELETE FROM elasticsearch_index_task'); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "shopware/elasticsearch", 3 | "description": "Elasticsearch for Shopware", 4 | "type": "library", 5 | "license": "MIT", 6 | "autoload": { 7 | "psr-4": { 8 | "Shopware\\Elasticsearch\\": "" 9 | }, 10 | "exclude-from-classmap": [ 11 | "/Test/" 12 | ] 13 | }, 14 | "config": { 15 | "sort-packages": true 16 | }, 17 | "support": { 18 | "issues": "https://github.com/shopware/shopware/issues/", 19 | "forum": "https://forum.shopware.com", 20 | "wiki": "https://developer.shopware.com", 21 | "docs": "https://developer.shopware.com", 22 | "chat": "https://slack.shopware.com" 23 | }, 24 | "extra": { 25 | "branch-alias": { 26 | "dev-master": "6.7.x-dev", 27 | "dev-trunk": "6.7.x-dev" 28 | } 29 | }, 30 | "minimum-stability": "stable", 31 | "require": { 32 | "php": "~8.2.0 || ~8.3.0 || ~8.4.0", 33 | "ext-curl": "*", 34 | "async-aws/core": "^1.22", 35 | "doctrine/dbal": "^4.2", 36 | "monolog/monolog": "^3.3.1", 37 | "opensearch-project/opensearch-php": "^2.3.1", 38 | "shopware/core": "dev-trunk", 39 | "shyim/opensearch-php-dsl": "^1.0.5", 40 | "symfony/http-foundation": "~7.2.0", 41 | "symfony/messenger": "~7.2.0" 42 | }, 43 | "require-dev": { 44 | "phpunit/phpunit": "^11.5.17" 45 | } 46 | } 47 | --------------------------------------------------------------------------------