├── .styleci.yml ├── Configuration ├── Objects.yaml ├── Settings.yaml └── Testing │ └── Settings.yaml ├── Classes ├── Exception.php ├── Indexer │ ├── Object │ │ ├── Signal │ │ │ ├── EmitterAdapterInterface.php │ │ │ ├── SignalEmitter.php │ │ │ └── Doctrine │ │ │ │ └── EmitterAdapter.php │ │ ├── Transform │ │ │ ├── TransformerInterface.php │ │ │ ├── TextCastTransformer.php │ │ │ ├── StringCastTransformer.php │ │ │ ├── DateTransformer.php │ │ │ ├── CollectionStringCastTransformer.php │ │ │ ├── ObjectIdentifierTransformer.php │ │ │ └── TransformerFactory.php │ │ ├── IndexInformer.php │ │ └── ObjectIndexer.php │ └── Aspect │ │ └── IndexerAspect.php ├── Domain │ ├── Model │ │ ├── GenericType.php │ │ ├── TypeGroup.php │ │ ├── Client.php │ │ ├── Client │ │ │ └── ClientConfiguration.php │ │ ├── AbstractType.php │ │ ├── Mapping.php │ │ ├── Document.php │ │ └── Index.php │ ├── Exception │ │ └── DocumentPropertiesMismatchException.php │ └── Factory │ │ ├── DocumentFactory.php │ │ └── ClientFactory.php ├── Annotations │ ├── Transform.php │ ├── Indexable.php │ └── Mapping.php ├── Transfer │ ├── Exception │ │ └── ApiException.php │ ├── Exception.php │ ├── Response.php │ └── RequestService.php ├── Service │ ├── IndexSettingProcessorInterface.php │ └── DynamicIndexSettingService.php ├── Package.php ├── Mapping │ ├── MappingCollection.php │ ├── BackendMappingBuilder.php │ └── EntityMappingBuilder.php └── Command │ ├── MappingCommandController.php │ └── IndexCommandController.php ├── Tests └── Functional │ ├── Fixtures │ ├── TwitterType.php │ ├── TweetRepository.php │ ├── JustFewPropertiesToIndex.php │ └── Tweet.php │ ├── Mapping │ └── MappingBuilderTest.php │ ├── Indexer │ └── Object │ │ ├── IndexInformerTest.php │ │ └── ObjectIndexerTest.php │ └── Domain │ ├── AbstractTest.php │ ├── DocumentTest.php │ └── IndexTest.php ├── README.md ├── LICENSE ├── composer.json └── Documentation ├── Transformer.rst ├── Indexer.rst └── Index.rst /.styleci.yml: -------------------------------------------------------------------------------- 1 | preset: psr2 2 | 3 | finder: 4 | path: 5 | - "Classes" 6 | - "Tests" 7 | -------------------------------------------------------------------------------- /Configuration/Objects.yaml: -------------------------------------------------------------------------------- 1 | Flowpack\ElasticSearch\Indexer\Object\ObjectIndexer: 2 | properties: 3 | client: 4 | object: 5 | factoryObjectName: Flowpack\ElasticSearch\Domain\Factory\ClientFactory 6 | arguments: 7 | 1: 8 | setting: Flowpack.ElasticSearch.realtimeIndexing.client 9 | -------------------------------------------------------------------------------- /Classes/Exception.php: -------------------------------------------------------------------------------- 1 | errorResult = $result; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Transform/TransformerInterface.php: -------------------------------------------------------------------------------- 1 | 23 | */ 24 | protected $types = []; 25 | 26 | /** 27 | * @param Index $index 28 | * @param array $types 29 | */ 30 | public function __construct(Index $index, array $types) 31 | { 32 | parent::__construct($index); 33 | $this->types = $types; 34 | 35 | $names = []; 36 | foreach ($this->types as $type) { 37 | $names[] = $type->getName(); 38 | } 39 | $this->name = implode(',', $names); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Signal/SignalEmitter.php: -------------------------------------------------------------------------------- 1 | mappingBuilder = $this->objectManager->get(EntityMappingBuilder::class); 31 | } 32 | 33 | /** 34 | * @test 35 | */ 36 | public function basicTest() 37 | { 38 | $information = $this->mappingBuilder->buildMappingInformation(); 39 | static::assertGreaterThanOrEqual(2, count($information)); 40 | static::assertInstanceOf(Mapping::class, $information[0]); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Transform/DateTransformer.php: -------------------------------------------------------------------------------- 1 | format($annotation->options['format'] ?: 'Y-m-d'); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Transform/CollectionStringCastTransformer.php: -------------------------------------------------------------------------------- 1 | persistenceManager->getIdentifierByObject($source); 48 | } 49 | 50 | return ''; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Transform/TransformerFactory.php: -------------------------------------------------------------------------------- 1 | objectManager->get($annotatedTransformer); 40 | if (!$transformer instanceof TransformerInterface) { 41 | throw new ElasticSearchException(sprintf('The transformer instance "%s" does not implement the TransformerInterface.', $annotatedTransformer), 1339598316); 42 | } 43 | 44 | return $transformer; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/Signal/Doctrine/EmitterAdapter.php: -------------------------------------------------------------------------------- 1 | signalEmitter->emitObjectUpdated($eventArguments->getEntity()); 37 | } 38 | 39 | /** 40 | * @param LifecycleEventArgs $eventArguments 41 | * @return void 42 | */ 43 | public function postPersist(LifecycleEventArgs $eventArguments) 44 | { 45 | $this->signalEmitter->emitObjectPersisted($eventArguments->getEntity()); 46 | } 47 | 48 | /** 49 | * @param LifecycleEventArgs $eventArguments 50 | * @return void 51 | */ 52 | public function postRemove(LifecycleEventArgs $eventArguments) 53 | { 54 | $this->signalEmitter->emitObjectRemoved($eventArguments->getEntity()); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Classes/Indexer/Aspect/IndexerAspect.php: -------------------------------------------------------------------------------- 1 | (add|update)())") 35 | * @param JoinPointInterface $joinPoint 36 | * @return string 37 | */ 38 | public function updateObjectToIndex(JoinPointInterface $joinPoint) 39 | { 40 | $arguments = $joinPoint->getMethodArguments(); 41 | $object = reset($arguments); 42 | $this->objectIndexer->indexObject($object); 43 | } 44 | 45 | /** 46 | * @Flow\AfterReturning("setting(Flowpack.ElasticSearch.realtimeIndexing.enabled) && within(Neos\Flow\Persistence\PersistenceManagerInterface) && method(public .+->(remove)())") 47 | * @param JoinPointInterface $joinPoint 48 | * @return string 49 | */ 50 | public function removeObjectFromIndex(JoinPointInterface $joinPoint) 51 | { 52 | $arguments = $joinPoint->getMethodArguments(); 53 | $object = reset($arguments); 54 | $this->objectIndexer->removeObject($object); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /Tests/Functional/Indexer/Object/IndexInformerTest.php: -------------------------------------------------------------------------------- 1 | informer = $this->objectManager->get(IndexInformer::class); 32 | } 33 | 34 | /** 35 | * @test 36 | */ 37 | public function classAnnotationTest() 38 | { 39 | $actual = $this->informer->getClassAnnotation(Fixtures\JustFewPropertiesToIndex::class); 40 | static::assertInstanceOf(IndexableAnnotation::class, $actual); 41 | static::assertSame('dummyindex', $actual->indexName); 42 | static::assertSame('sampletype', $actual->typeName); 43 | } 44 | 45 | /** 46 | * @test 47 | */ 48 | public function classWithOnlyOnePropertyAnnotatedHasOnlyThisPropertyToBeIndexed() 49 | { 50 | $actual = $this->informer->getClassProperties(Fixtures\JustFewPropertiesToIndex::class); 51 | static::assertCount(1, $actual); 52 | } 53 | 54 | /** 55 | * @test 56 | */ 57 | public function classWithNoPropertyAnnotatedHasAllPropertiesToBeIndexed() 58 | { 59 | $actual = $this->informer->getClassProperties(Fixtures\Tweet::class); 60 | static::assertGreaterThan(1, $actual); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Tests/Functional/Fixtures/Tweet.php: -------------------------------------------------------------------------------- 1 | date = $date; 49 | } 50 | 51 | /** 52 | * @param string $message 53 | */ 54 | public function setMessage($message) 55 | { 56 | $this->message = $message; 57 | } 58 | 59 | /** 60 | * @param string $username 61 | */ 62 | public function setUsername($username) 63 | { 64 | $this->username = $username; 65 | } 66 | 67 | /** 68 | * @return \DateTime 69 | */ 70 | public function getDate() 71 | { 72 | return $this->date; 73 | } 74 | 75 | /** 76 | * @return string 77 | */ 78 | public function getMessage() 79 | { 80 | return $this->message; 81 | } 82 | 83 | /** 84 | * @return string 85 | */ 86 | public function getUsername() 87 | { 88 | return $this->username; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /Tests/Functional/Domain/AbstractTest.php: -------------------------------------------------------------------------------- 1 | clientFactory = $this->objectManager->get(ClientFactory::class); 43 | $client = $this->clientFactory->create("FunctionalTests"); 44 | $this->testingIndex = $client->findIndex('flow_elasticsearch_functionaltests'); 45 | 46 | if ($this->testingIndex->exists()) { 47 | throw new \Exception('The index "flow_elasticsearch_functionaltests" already existed, aborting.', 1338967487); 48 | } else { 49 | $this->testingIndex->create(); 50 | $this->removeIndexOnTearDown = true; 51 | } 52 | 53 | $this->additionalSetUp(); 54 | } 55 | 56 | /** 57 | * may be implemented by inheritors because setUp() is final. 58 | */ 59 | protected function additionalSetUp() 60 | { 61 | } 62 | 63 | /** 64 | * set to final because this is an important step which may not be overridden. 65 | */ 66 | final public function tearDown(): void 67 | { 68 | parent::tearDown(); 69 | 70 | if ($this->removeIndexOnTearDown === true) { 71 | $this->testingIndex->delete(); 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /Classes/Transfer/Exception.php: -------------------------------------------------------------------------------- 1 | response = $response; 45 | $this->request = $request; 46 | 47 | $this->request->getBody()->rewind(); 48 | $this->response->getBody()->rewind(); 49 | 50 | if ($request !== null) { 51 | $message = sprintf( 52 | "Elasticsearch request failed: 53 | \n[%s %s]: %s 54 | \n\n 55 | Request data: 56 | \n%s 57 | \n\n 58 | Response body: 59 | \n%s", 60 | $request->getMethod(), 61 | $request->getUri(), 62 | $message, 63 | $request->getBody()->getContents(), 64 | $response->getBody()->getContents() 65 | ); 66 | } 67 | 68 | parent::__construct($message, $code, $previous); 69 | } 70 | 71 | /** 72 | * @return RequestInterface 73 | */ 74 | public function getRequest(): RequestInterface 75 | { 76 | return $this->request; 77 | } 78 | 79 | /** 80 | * @return ResponseInterface 81 | */ 82 | public function getResponse(): ResponseInterface 83 | { 84 | return $this->response; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Classes/Transfer/Response.php: -------------------------------------------------------------------------------- 1 | originalResponse = $response; 43 | 44 | $content = $response->getBody()->getContents(); 45 | $treatedContent = json_decode($content, true); 46 | 47 | if ($content !== '') { 48 | if ($treatedContent === null) { 49 | throw new Exception('The request returned an invalid JSON string which was "' . $content . '".', 1338976439, $response, $request); 50 | } 51 | 52 | if (array_key_exists('error', $treatedContent)) { 53 | $exceptionMessage = print_r($treatedContent['error'], true); 54 | throw new Exception\ApiException($exceptionMessage, 1338977435, $response, $request); 55 | } 56 | } 57 | 58 | $this->treatedContent = $treatedContent; 59 | } 60 | 61 | /** 62 | * Shortcut to response's getStatusCode 63 | * 64 | * @return int 65 | */ 66 | public function getStatusCode(): int 67 | { 68 | return $this->originalResponse->getStatusCode(); 69 | } 70 | 71 | /** 72 | * @return mixed 73 | */ 74 | public function getTreatedContent() 75 | { 76 | return $this->treatedContent; 77 | } 78 | 79 | /** 80 | * @return ResponseInterface 81 | */ 82 | public function getOriginalResponse(): ResponseInterface 83 | { 84 | return $this->originalResponse; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Classes/Package.php: -------------------------------------------------------------------------------- 1 | getSignalSlotDispatcher(); 41 | $package = $this; 42 | $dispatcher->connect(Sequence::class, 'afterInvokeStep', function (Step $step) use ($package, $bootstrap) { 43 | if ($step->getIdentifier() === 'neos.flow:objectmanagement:runtime') { 44 | $package->prepareRealtimeIndexing($bootstrap); 45 | } 46 | }); 47 | } 48 | 49 | /** 50 | * @param Bootstrap $bootstrap 51 | * @return void 52 | */ 53 | public function prepareRealtimeIndexing(Bootstrap $bootstrap) 54 | { 55 | $this->configurationManager = $bootstrap->getObjectManager()->get(ConfigurationManager::class); 56 | $settings = $this->configurationManager->getConfiguration(ConfigurationManager::CONFIGURATION_TYPE_SETTINGS, $this->getPackageKey()); 57 | 58 | if (isset($settings['realtimeIndexing']['enabled']) && $settings['realtimeIndexing']['enabled'] === true) { 59 | $bootstrap->getSignalSlotDispatcher()->connect(SignalEmitter::class, 'objectUpdated', ObjectIndexer::class, 'indexObject'); 60 | $bootstrap->getSignalSlotDispatcher()->connect(SignalEmitter::class, 'objectPersisted', ObjectIndexer::class, 'indexObject'); 61 | $bootstrap->getSignalSlotDispatcher()->connect(SignalEmitter::class, 'objectRemoved', ObjectIndexer::class, 'removeObject'); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Tests/Functional/Domain/DocumentTest.php: -------------------------------------------------------------------------------- 1 | 'kimchy', 29 | 'post_date' => '2009-11-15T14:12:12', 30 | 'message' => 'trying out Elastic Search' 31 | ] 32 | ] 33 | ]; 34 | } 35 | 36 | /** 37 | * @dataProvider simpleDocumentDataProvider 38 | * @test 39 | */ 40 | public function idOfFreshNewDocumentIsPopulatedAfterStoring(?array $data = null) 41 | { 42 | $document = new Document(new TwitterType($this->testingIndex), $data); 43 | static::assertNull($document->getId()); 44 | $document->store(); 45 | static::assertMatchesRegularExpression('/\w+/', $document->getId()); 46 | } 47 | 48 | /** 49 | * @dataProvider simpleDocumentDataProvider 50 | * @test 51 | */ 52 | public function versionOfFreshNewDocumentIsCreatedAfterStoringAndIncreasedAfterSubsequentStoring(?array $data = null) 53 | { 54 | $document = new Document(new TwitterType($this->testingIndex), $data); 55 | static::assertNull($document->getVersion()); 56 | $document->store(); 57 | $idAfterFirstStoring = $document->getId(); 58 | static::assertSame(1, $document->getVersion()); 59 | $document->store(); 60 | static::assertSame(2, $document->getVersion()); 61 | static::assertSame($idAfterFirstStoring, $document->getId()); 62 | } 63 | 64 | /** 65 | * @dataProvider simpleDocumentDataProvider 66 | * @test 67 | */ 68 | public function existingIdOfDocumentIsNotModifiedAfterStoring(array $data) 69 | { 70 | $id = '42-1010-42'; 71 | $document = new Document(new TwitterType($this->testingIndex), $data, $id); 72 | $document->store(); 73 | static::assertSame($id, $document->getId()); 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Classes/Service/DynamicIndexSettingService.php: -------------------------------------------------------------------------------- 1 | objectManager) as $configuration) { 44 | /** @var IndexSettingProcessorInterface $processor */ 45 | $processor = $this->objectManager->get($configuration['className']); 46 | if ($processor->canProcess($settings, $path)) { 47 | $settings = $processor->process($settings, $path, $indexName); 48 | } 49 | } 50 | 51 | return $settings; 52 | } 53 | 54 | /** 55 | * Returns all class names implementing the IndexSettingProcessorInterface. 56 | * 57 | * @Flow\CompileStatic 58 | * @param ObjectManagerInterface $objectManager 59 | * @return array 60 | */ 61 | public static function getAllProcessors($objectManager) 62 | { 63 | /** @var ReflectionService $reflectionService */ 64 | $reflectionService = $objectManager->get(ReflectionService::class); 65 | $processorClassNames = $reflectionService->getAllImplementationClassNamesForInterface(IndexSettingProcessorInterface::class); 66 | 67 | $processors = []; 68 | foreach ($processorClassNames as $processorClassName) { 69 | $processors[$processorClassName] = [ 70 | 'priority' => $processorClassName::getPriority(), 71 | 'className' => $processorClassName 72 | ]; 73 | } 74 | 75 | return array_reverse( 76 | (new PositionalArraySorter($processors, 'priority'))->toArray() 77 | ); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Classes/Domain/Factory/DocumentFactory.php: -------------------------------------------------------------------------------- 1 | getTreatedContent(); 43 | 44 | $verificationResults = new ErrorResult(); 45 | if (isset($content['_index']) && $type->getIndex()->getName() !== $content['_index']) { 46 | $error = new Error('The received index name "%s" does not match the expected one "%s".', 1340264838, [$content['_index'], $type->getIndex()->getName()]); 47 | $verificationResults->addError($error); 48 | } 49 | if ($type->getName() !== Arrays::getValueByPath($content, '_source.neos_type')) { 50 | $error = new Error('The received type name "%s" does not match the expected one "%s".', 1340265103, [$content['_type'], $type->getName()]); 51 | $verificationResults->addError($error); 52 | } 53 | 54 | if (isset($content['_id']) && $id !== null && $id !== $content['_id']) { 55 | $error = new Error('The received id "%s" does not match the expected one "%s".', 1340269758, [$content['_id'], $id]); 56 | $verificationResults->addError($error); 57 | } 58 | 59 | if ($verificationResults->hasErrors()) { 60 | $exception = new DocumentPropertiesMismatchException('The document\'s properties do not match the expected ones.', 1340265248); 61 | $exception->setErrorResult($verificationResults); 62 | throw $exception; 63 | } 64 | 65 | $version = $content['_version']; 66 | $data = $content['_source']; 67 | 68 | return new Model\Document($type, $data, $id, $version); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Client.php: -------------------------------------------------------------------------------- 1 | bundle; 55 | } 56 | 57 | /** 58 | * @param string $bundle 59 | * @return void 60 | */ 61 | public function setBundle(string $bundle): void 62 | { 63 | $this->bundle = $bundle; 64 | } 65 | 66 | /** 67 | * @return array 68 | */ 69 | public function getClientConfigurations(): array 70 | { 71 | return $this->clientConfigurations; 72 | } 73 | 74 | /** 75 | * @param array $clientConfigurations 76 | * @return void 77 | */ 78 | public function setClientConfigurations(array $clientConfigurations): void 79 | { 80 | $this->clientConfigurations = $clientConfigurations; 81 | } 82 | 83 | /** 84 | * @param string $indexName 85 | * @return Index 86 | * @throws \Flowpack\ElasticSearch\Exception 87 | */ 88 | public function findIndex(string $indexName): Index 89 | { 90 | if (!array_key_exists($indexName, $this->indexCollection)) { 91 | $this->indexCollection[$indexName] = new Index($indexName, $this); 92 | } 93 | 94 | return $this->indexCollection[$indexName]; 95 | } 96 | 97 | /** 98 | * Passes a request through to the request service 99 | * 100 | * @param string $method 101 | * @param string $path 102 | * @param array $arguments 103 | * @param string|array $content 104 | * @return Response 105 | * @throws \Flowpack\ElasticSearch\Transfer\Exception 106 | * @throws \Flowpack\ElasticSearch\Transfer\Exception\ApiException 107 | * @throws \Neos\Flow\Http\Exception 108 | */ 109 | public function request(string $method, ?string $path = null, array $arguments = [], $content = null): Response 110 | { 111 | return $this->requestService->request($method, $this, $path, $arguments, $content); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /Classes/Domain/Factory/ClientFactory.php: -------------------------------------------------------------------------------- 1 | settings = $settings; 39 | } 40 | 41 | /** 42 | * @param string $bundle 43 | * @param string $clientClassName 44 | * @return Client 45 | * @throws ElasticSearchException 46 | */ 47 | public function create($bundle = null, $clientClassName = Client::class) 48 | { 49 | if ($bundle === null) { 50 | $bundle = 'default'; 51 | } 52 | 53 | if (!isset($this->settings['clients'][$bundle]) || !is_array($this->settings['clients'][$bundle])) { 54 | throw new ElasticSearchException('The inquired client settings bundle "' . $bundle . '" is not present in setting "Flowpack.ElasticSearch.clients".', 1338890487); 55 | } 56 | $clientsSettings = $this->settings['clients'][$bundle]; 57 | 58 | $clientConfigurations = $this->buildClientConfigurations($clientsSettings); 59 | 60 | $client = new $clientClassName(); 61 | $client->setClientConfigurations($clientConfigurations); 62 | $client->setBundle($bundle); 63 | 64 | return $client; 65 | } 66 | 67 | /** 68 | * @param array $clientsSettings 69 | * @return array 70 | * @throws ElasticSearchException 71 | */ 72 | protected function buildClientConfigurations(array $clientsSettings) 73 | { 74 | $clientConfigurations = []; 75 | $clientConfiguration = new ClientConfiguration(); 76 | foreach ($clientsSettings as $clientSettings) { 77 | $configuration = clone $clientConfiguration; 78 | foreach ($clientSettings as $settingKey => $settingValue) { 79 | $setterMethodName = 'set' . ucfirst($settingKey); 80 | try { 81 | $configuration->$setterMethodName($settingValue); 82 | } catch (FlowErrorException $exception) { 83 | $exceptionMessage = 'Setting key "' . $settingKey . '" as client configuration value is not allowed. Refer to the Settings.yaml.example for the supported keys.'; 84 | throw new ElasticSearchException($exceptionMessage, 1338886877, $exception); 85 | } 86 | } 87 | $clientConfigurations[] = $configuration; 88 | } 89 | 90 | return $clientConfigurations; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "flowpack/elasticsearch", 3 | "type": "neos-package", 4 | "license": "MIT", 5 | "description": "This package provides wrapper functionality for the Elasticsearch engine.", 6 | "require": { 7 | "ext-curl": "*", 8 | "doctrine/annotations": "^1.0", 9 | "doctrine/collections": "^1.0", 10 | "doctrine/orm": "^2.0", 11 | "neos/error-messages": "^7.3 || ^8.0 || ^9.0 || dev-master", 12 | "neos/flow": "^7.3 || ^8.0 || ^9.0 || dev-master", 13 | "neos/utility-arrays": "^7.3 || ^8.0 || ^9.0 || dev-master", 14 | "neos/utility-objecthandling": "^7.3 || ^8.0 || ^9.0 || dev-master" 15 | }, 16 | "autoload": { 17 | "psr-4": { 18 | "Flowpack\\ElasticSearch\\": "Classes" 19 | } 20 | }, 21 | "extra": { 22 | "applied-flow-migrations": [ 23 | "TYPO3.Flow-201209251426", 24 | "TYPO3.FLOW3-201209201112", 25 | "TYPO3.FLOW3-201201261636", 26 | "TYPO3.Fluid-201205031303", 27 | "TYPO3.FLOW3-201205292145", 28 | "TYPO3.FLOW3-201206271128", 29 | "TYPO3.Flow-201211151101", 30 | "TYPO3.Flow-201212051340", 31 | "TYPO3.TypoScript-130516234520", 32 | "TYPO3.TypoScript-130516235550", 33 | "TYPO3.TYPO3CR-130523180140", 34 | "TYPO3.Neos.NodeTypes-201309111655", 35 | "TYPO3.Flow-201310031523", 36 | "TYPO3.Flow-201405111147", 37 | "TYPO3.Neos-201407061038", 38 | "TYPO3.Neos-201409071922", 39 | "TYPO3.TYPO3CR-140911160326", 40 | "TYPO3.Neos-201410010000", 41 | "TYPO3.TYPO3CR-141101082142", 42 | "TYPO3.Neos-20141113115300", 43 | "TYPO3.Fluid-20141113120800", 44 | "TYPO3.Flow-20141113121400", 45 | "TYPO3.Fluid-20141121091700", 46 | "TYPO3.Neos-20141218134700", 47 | "TYPO3.Fluid-20150214130800", 48 | "TYPO3.Neos-20150303231600", 49 | "TYPO3.TYPO3CR-20150510103823", 50 | "TYPO3.Flow-20151113161300", 51 | "TYPO3.Form-20160601101500", 52 | "TYPO3.Flow-20161115140400", 53 | "TYPO3.Flow-20161115140430", 54 | "Neos.Flow-20161124204700", 55 | "Neos.Flow-20161124204701", 56 | "Neos.Twitter.Bootstrap-20161124204912", 57 | "Neos.Form-20161124205254", 58 | "Neos.Flow-20161124224015", 59 | "Neos.Party-20161124225257", 60 | "Neos.Eel-20161124230101", 61 | "Neos.Kickstart-20161124230102", 62 | "Neos.Setup-20161124230842", 63 | "Neos.Imagine-20161124231742", 64 | "Neos.Media-20161124233100", 65 | "Neos.NodeTypes-20161125002300", 66 | "Neos.SiteKickstarter-20161125002311", 67 | "Neos.Neos-20161125002322", 68 | "Neos.ContentRepository-20161125012000", 69 | "Neos.Fusion-20161125013710", 70 | "Neos.Setup-20161125014759", 71 | "Neos.SiteKickstarter-20161125095901", 72 | "Neos.Fusion-20161125104701", 73 | "Neos.NodeTypes-20161125104800", 74 | "Neos.Neos-20161125104802", 75 | "Neos.Kickstarter-20161125110814", 76 | "Neos.Neos-20161125122412", 77 | "Neos.Flow-20161125124112" 78 | ] 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Client/ClientConfiguration.php: -------------------------------------------------------------------------------- 1 | host; 62 | } 63 | 64 | /** 65 | * @param string $host 66 | * @return void 67 | */ 68 | public function setHost(string $host): void 69 | { 70 | $this->host = $host; 71 | } 72 | 73 | /** 74 | * @return int 75 | */ 76 | public function getPort(): int 77 | { 78 | return $this->port; 79 | } 80 | 81 | /** 82 | * @param mixed $port Cast to int internally 83 | * @return void 84 | */ 85 | public function setPort($port): void 86 | { 87 | $this->port = (int)$port; 88 | } 89 | 90 | /** 91 | * @return string 92 | */ 93 | public function getScheme(): string 94 | { 95 | return $this->scheme; 96 | } 97 | 98 | /** 99 | * @param string $scheme 100 | * @return void 101 | */ 102 | public function setScheme(string $scheme): void 103 | { 104 | $this->scheme = $scheme; 105 | } 106 | 107 | /** 108 | * Returns username 109 | * 110 | * @return string 111 | */ 112 | public function getUsername(): string 113 | { 114 | return $this->username; 115 | } 116 | 117 | /** 118 | * Sets username 119 | * 120 | * @param string $username 121 | * @return void 122 | */ 123 | public function setUsername(?string $username): void 124 | { 125 | $this->username = $username; 126 | } 127 | 128 | /** 129 | * Returns password 130 | * 131 | * @return string 132 | */ 133 | public function getPassword(): string 134 | { 135 | return $this->password; 136 | } 137 | 138 | /** 139 | * Sets password 140 | * 141 | * @param string|null $password 142 | * @return void 143 | */ 144 | public function setPassword(?string $password): void 145 | { 146 | $this->password = $password; 147 | } 148 | 149 | /** 150 | * @return UriInterface 151 | */ 152 | public function getUri(): UriInterface 153 | { 154 | $uriWithoutAuthorization = $this->uriFactory->createUri() 155 | ->withScheme($this->scheme) 156 | ->withHost($this->host) 157 | ->withPort($this->port); 158 | 159 | if ($this->username === null && $this->password === null) { 160 | return $uriWithoutAuthorization; 161 | } 162 | 163 | return $uriWithoutAuthorization->withUserInfo($this->username, $this->password); 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /Classes/Mapping/MappingCollection.php: -------------------------------------------------------------------------------- 1 | type = $type; 44 | } 45 | 46 | /** 47 | * Returns a new collection of mappings of this collection that are not member of the $complementCollection. 48 | * 49 | * @param MappingCollection $complementCollection 50 | * @return MappingCollection 51 | */ 52 | public function diffAgainstCollection(MappingCollection $complementCollection) 53 | { 54 | $returnMappings = new MappingCollection(); 55 | foreach ($this as $entityMapping) { 56 | /** @var $entityMapping Mapping */ 57 | $mapping = new Mapping(clone $entityMapping->getType()); 58 | $saveMapping = false; 59 | foreach ($entityMapping->getProperties() as $propertyName => $propertySettings) { 60 | foreach ($propertySettings as $entitySettingKey => $entitySettingValue) { 61 | $backendSettingValue = $complementCollection->getMappingSetting($entityMapping, $propertyName, $entitySettingKey); 62 | if ($entitySettingValue !== $backendSettingValue) { 63 | $mapping->setPropertyByPath([$propertyName, $entitySettingKey], $entitySettingValue); 64 | $saveMapping = true; 65 | } 66 | } 67 | } 68 | if ($saveMapping) { 69 | $returnMappings->add($mapping); 70 | } 71 | } 72 | 73 | return $returnMappings; 74 | } 75 | 76 | /** 77 | * Tells whether a member of this collection has a specific index/type/property settings value 78 | * 79 | * @param Mapping $inquirerMapping 80 | * @param string $propertyName 81 | * @param string $settingKey 82 | * @return mixed 83 | */ 84 | public function getMappingSetting(Mapping $inquirerMapping, $propertyName, $settingKey) 85 | { 86 | foreach ($this as $memberMapping) { 87 | /** @var $memberMapping Mapping */ 88 | if ($inquirerMapping->getType()->getName() === $memberMapping->getType()->getName() 89 | && $inquirerMapping->getType()->getIndex()->getName() === $memberMapping->getType()->getIndex()->getName() 90 | ) { 91 | return $memberMapping->getPropertyByPath([$propertyName, $settingKey]); 92 | } 93 | } 94 | 95 | return null; 96 | } 97 | 98 | /** 99 | * @return ElasticSearchClient 100 | */ 101 | public function getClient() 102 | { 103 | return $this->client; 104 | } 105 | 106 | /** 107 | * @param ElasticSearchClient $client 108 | * @return void 109 | */ 110 | public function setClient(ElasticSearchClient $client) 111 | { 112 | $this->client = $client; 113 | } 114 | 115 | /** 116 | * @return string 117 | */ 118 | public function getType() 119 | { 120 | return $this->type; 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /Documentation/Transformer.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Transformer 3 | =========== 4 | 5 | The Transformer classes allows you to transform your data before it's stored in elasticsearch. 6 | 7 | Transformer annotations 8 | ======================= 9 | This package ships several Transformer by default: 10 | 11 | * CollectionStringCastTransformer: 12 | 13 | * Iterates over an Collection/Array and casts all items to a string and returns an array with strings 14 | 15 | * DateTransformer 16 | 17 | * Converts a DateTime-Object to formatted string. Defaults to Y-m-d 18 | 19 | * ObectIdentifierTransformer 20 | 21 | * Converts an object to it's persistence identifier 22 | 23 | * StringCastTransformer 24 | 25 | * Converts a value to a string 26 | 27 | * TextCastTransformer 28 | 29 | * Converts a value to a string 30 | 31 | 32 | Usage 33 | ===== 34 | 35 | To transform an objects property at index time you need to annotate your value like this: 36 | 37 | *Example: Use a Transformer* :: 38 | 39 | /** 40 | * @var \DateTime 41 | * @ElasticSearch\Transform(options={"format" = "Y-m-d H:m:s"}, type="date") 42 | */ 43 | protected $date; 44 | 45 | 46 | The `type` option is usded to determine the corresponding transformer class. Date would resolve to `Flowpack\ElasticSearch\Indexer\Object\Transform\DateTransformer`. 47 | and call it's implementation of `transformByAnnotation($source, TransformAnnotation $annotation)`. 48 | All default transformer can be used just like this. 49 | 50 | 51 | Implement a custom Transformer 52 | ============================== 53 | If you need a custom transformer you need to implement the TrnasformerInterface. 54 | It declares two methods: 55 | 56 | * getTargetMappingType: return value is used as mapping type in elasticsearch (one of https://www.elastic.co/guide/en/elasticsearch/reference/current/mapping-types.html) 57 | 58 | * transformByAnnotation: actual implementation of your transform process 59 | 60 | 61 | *Example: Custom Transformer that implements TransformerInterface and crops the source length* :: 62 | 63 | options['length']) { 98 | return substr((string) $source, 0, $annotation->options['length']); 99 | } 100 | 101 | return (string) $source 102 | } 103 | } 104 | 105 | 106 | 107 | *Example: Annotation usage:* :: 108 | 109 | /** 110 | * @Flow\Entity 111 | * @ElasticSearch\Indexable("twitter", typeName="tweet") 112 | */ 113 | class Tweet { 114 | 115 | /** 116 | * @var string 117 | */ 118 | protected $username; 119 | 120 | /** 121 | * @var string 122 | * @ElasticSearch\Transform(options={"length" = 20}, type="Some\Vendor\Indexer\Object\Transform\CropTransformer") 123 | */ 124 | protected $message; 125 | 126 | /** 127 | * @var \DateTime 128 | */ 129 | protected $date; 130 | } 131 | 132 | 133 | With this configuration the message will always be cropped to 20 chars when it's indexed 134 | 135 | -------------------------------------------------------------------------------- /Classes/Mapping/BackendMappingBuilder.php: -------------------------------------------------------------------------------- 1 | 51 | * @throws ElasticSearchException 52 | * @throws \Neos\Flow\Http\Exception 53 | */ 54 | public function buildMappingInformation(): MappingCollection 55 | { 56 | if (!$this->client instanceof Model\Client) { 57 | throw new ElasticSearchException('No client was given for mapping retrieval. Set a client BackendMappingBuilder->setClient().', 1339678111); 58 | } 59 | 60 | $this->indicesWithoutTypeInformation = []; 61 | 62 | $response = $this->client->request('GET', '/_mapping'); 63 | $mappingInformation = new MappingCollection(MappingCollection::TYPE_BACKEND); 64 | $mappingInformation->setClient($this->client); 65 | $indexNames = $this->indexInformer->getAllIndexNames(); 66 | 67 | foreach ($response->getTreatedContent() as $indexName => $indexSettings) { 68 | if (!in_array($indexName, $indexNames)) { 69 | continue; 70 | } 71 | $index = new Model\Index($indexName); 72 | if (empty($indexSettings)) { 73 | $this->indicesWithoutTypeInformation[] = $indexName; 74 | } 75 | foreach ($indexSettings as $typeName => $typeSettings) { 76 | $type = new Model\GenericType($index, $typeName); 77 | $mapping = new Model\Mapping($type); 78 | if (isset($typeSettings['properties'])) { 79 | foreach ($typeSettings['properties'] as $propertyName => $propertySettings) { 80 | foreach ($propertySettings as $key => $value) { 81 | $mapping->setPropertyByPath([$propertyName, $key], $value); 82 | } 83 | } 84 | } 85 | $mappingInformation->add($mapping); 86 | } 87 | } 88 | 89 | return $mappingInformation; 90 | } 91 | 92 | /** 93 | * @param Model\Client $client 94 | * @return void 95 | */ 96 | public function setClient(Model\Client $client): void 97 | { 98 | $this->client = $client; 99 | } 100 | 101 | /** 102 | * @return array 103 | * @throws ElasticSearchException 104 | */ 105 | public function getIndicesWithoutTypeInformation(): array 106 | { 107 | if ($this->indicesWithoutTypeInformation === null) { 108 | throw new ElasticSearchException('For getting the indices having no mapping information attached, BackendMappingBuilder->buildMappingInformation() has to be run first.', 1339751812); 109 | } 110 | 111 | return $this->indicesWithoutTypeInformation; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /Tests/Functional/Domain/IndexTest.php: -------------------------------------------------------------------------------- 1 | createMock(Client::class); 27 | $clientMock->method('getBundle') 28 | ->willReturn('FunctionalTests'); 29 | 30 | $clientMock->expects($this->exactly(2))->method('request') 31 | ->withConsecutive( 32 | [ 33 | 'PUT', 34 | '/index_without_prefix/', 35 | [], 36 | json_encode([ 37 | 'settings' => [ 38 | 'index' => [ 39 | 'number_of_replicas' => 2, 40 | 'soft_deletes' => [ 41 | 'enabled' => true 42 | ] 43 | ] 44 | ] 45 | ], JSON_THROW_ON_ERROR) 46 | ], 47 | // updateSettings should correctly filter soft_deletes as it's not in the allow list 48 | [ 49 | 'PUT', 50 | '/index_without_prefix/_settings', 51 | [], 52 | json_encode([ 53 | 'index' => [ 54 | 'number_of_replicas' => 2 55 | ] 56 | ], JSON_THROW_ON_ERROR) 57 | ] 58 | ) 59 | ->willReturn($this->createStub(Response::class)); 60 | 61 | $testObject = new Index('index_without_prefix', $clientMock); 62 | $testObject->create(); 63 | $testObject->updateSettings(); 64 | 65 | static::assertSame('index_without_prefix', $testObject->getOriginalName()); 66 | static::assertSame('index_without_prefix', $testObject->getName()); 67 | } 68 | 69 | /** 70 | * @test 71 | */ 72 | public function indexWithPrefix() 73 | { 74 | $clientMock = $this->createMock(Client::class); 75 | $clientMock->method('getBundle') 76 | ->willReturn('FunctionalTests'); 77 | 78 | $clientMock->expects($this->exactly(2))->method('request') 79 | ->withConsecutive( 80 | [ 81 | 'PUT', 82 | '/prefix-index_with_prefix/', 83 | [], 84 | json_encode([ 85 | 'settings' => [ 86 | 'index' => [ 87 | 'number_of_replicas' => 1, 88 | 'soft_deletes' => [ 89 | 'enabled' => true 90 | ] 91 | ] 92 | ] 93 | ], JSON_THROW_ON_ERROR) 94 | ], 95 | // updateSettings should correctly filter soft_deletes as it's not in the allow list 96 | [ 97 | 'PUT', 98 | '/prefix-index_with_prefix/_settings', 99 | [], 100 | json_encode([ 101 | 'index' => [ 102 | 'number_of_replicas' => 1 103 | ] 104 | ], JSON_THROW_ON_ERROR) 105 | ] 106 | ) 107 | ->willReturn($this->createStub(Response::class)); 108 | 109 | $testObject = new Index('index_with_prefix', $clientMock); 110 | $testObject->create(); 111 | $testObject->updateSettings(); 112 | 113 | static::assertSame('index_with_prefix', $testObject->getOriginalName()); 114 | static::assertSame('prefix-index_with_prefix', $testObject->getName()); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /Classes/Domain/Model/AbstractType.php: -------------------------------------------------------------------------------- 1 | index = $index; 50 | 51 | if ($name === null) { 52 | $this->name = str_replace('\\', '_', get_class($this)); 53 | } else { 54 | $this->name = $name; 55 | } 56 | } 57 | 58 | /** 59 | * Gets this type's name. 60 | * 61 | * @return string 62 | */ 63 | public function getName(): string 64 | { 65 | return $this->name; 66 | } 67 | 68 | /** 69 | * @return Index 70 | */ 71 | public function getIndex(): Index 72 | { 73 | return $this->index; 74 | } 75 | 76 | /** 77 | * Returns a document 78 | * 79 | * @param string $id 80 | * @return Document 81 | * @throws DocumentPropertiesMismatchException 82 | * @throws \Flowpack\ElasticSearch\Exception 83 | * @throws \Neos\Flow\Http\Exception 84 | */ 85 | public function findDocumentById(string $id): ?Document 86 | { 87 | $response = $this->request('GET', '/_doc/' . $id); 88 | if ($response->getStatusCode() !== 200) { 89 | return null; 90 | } 91 | 92 | return $this->documentFactory->createFromResponse($this, $id, $response); 93 | } 94 | 95 | /** 96 | * @param string $method 97 | * @param string $path 98 | * @param array $arguments 99 | * @param string $content 100 | * @return Response 101 | * @throws \Flowpack\ElasticSearch\Exception 102 | * @throws \Neos\Flow\Http\Exception 103 | */ 104 | public function request(string $method, ?string $path = null, array $arguments = [], ?string $content = null): Response 105 | { 106 | return $this->index->request($method, $path, $arguments, $content); 107 | } 108 | 109 | /** 110 | * @param string $id 111 | * @return boolean ...whether the deletion is considered successful 112 | * @throws \Flowpack\ElasticSearch\Exception 113 | * @throws \Neos\Flow\Http\Exception 114 | */ 115 | public function deleteDocumentById(string $id): bool 116 | { 117 | $response = $this->request('DELETE', '/_doc/' . $id); 118 | $treatedContent = $response->getTreatedContent(); 119 | 120 | return $response->getStatusCode() === 200 && $treatedContent['result'] === 'deleted'; 121 | } 122 | 123 | /** 124 | * @return int 125 | * @throws \Flowpack\ElasticSearch\Exception 126 | * @throws \Neos\Flow\Http\Exception 127 | */ 128 | public function count(): ?int 129 | { 130 | $response = $this->request('GET', '/_count'); 131 | if ($response->getStatusCode() !== 200) { 132 | return null; 133 | } 134 | $treatedContent = $response->getTreatedContent(); 135 | 136 | return (integer)$treatedContent['count']; 137 | } 138 | 139 | /** 140 | * @param array $searchQuery The search query TODO: make it an object 141 | * @return Response 142 | * @throws \Flowpack\ElasticSearch\Exception 143 | * @throws \Neos\Flow\Http\Exception 144 | */ 145 | public function search(array $searchQuery): Response 146 | { 147 | return $this->request('GET', '/_search', [], json_encode($searchQuery)); 148 | } 149 | } 150 | -------------------------------------------------------------------------------- /Documentation/Indexer.rst: -------------------------------------------------------------------------------- 1 | ======= 2 | Indexer 3 | ======= 4 | 5 | The indexer allows collecting data that has to be indexed. 6 | 7 | Indexing annotations 8 | ==================== 9 | 10 | The package introduces a new annotation called "Indexable". Use this annotation to define what objects/entities you want 11 | to have indexed. If you annotate a class, all supported properties [#suppProperties]_ will be mapped. If you annotated 12 | single properties, only these will be indexed. However, you have to annotate the class in every case. 13 | 14 | *Example: Class where every property will be indexed* :: 15 | 16 | /** 17 | * @Flow\Entity 18 | * @ElasticSearch\Indexable("twitter", typeName="tweet") 19 | */ 20 | class Tweet { 21 | 22 | /** 23 | * @var string 24 | */ 25 | protected $username; 26 | 27 | /** 28 | * @var string 29 | */ 30 | protected $message; 31 | 32 | /** 33 | * @var \DateTime 34 | */ 35 | protected $date; 36 | } 37 | 38 | *Example: Class where only the ``message`` property will be indexed* :: 39 | 40 | /** 41 | * @Flow\Entity 42 | * @ElasticSearch\Indexable("twitter", typeName="tweet") 43 | */ 44 | class Tweet { 45 | 46 | /** 47 | * @var string 48 | */ 49 | protected $username; 50 | 51 | /** 52 | * @var string 53 | * @ElasticSearch\Indexable 54 | */ 55 | protected $message; 56 | 57 | /** 58 | * @var \DateTime 59 | */ 60 | protected $date; 61 | } 62 | 63 | Mapping annotations 64 | =================== 65 | 66 | ElasticSearch allows the mapping configuration done via annotations. See the example how to define mapping annotations: 67 | 68 | *Example: Annotations to set up mapping directives* :: 69 | 70 | /** 71 | * @var string 72 | * @ElasticSearch\Mapping(boost=2.0, term_vector="with_offsets") 73 | */ 74 | protected $username; 75 | 76 | /** 77 | * @var string 78 | */ 79 | protected $message; 80 | 81 | /** 82 | * @var \DateTime 83 | * @ElasticSearch\Mapping(format="YYYY-MM-dd") 84 | */ 85 | protected $date; 86 | 87 | Note that for mapping creation, the type will automatically be determined from the PHP type the property is of. 88 | 89 | Value transformations 90 | ===================== 91 | 92 | For some properties it'll be necessary to conduct specific conversions in order to meet the requirements of 93 | ElasticSearch. Declare custom type converters via their appropriate annotation:: 94 | 95 | /** 96 | * @var \DateTime 97 | * @ElasticSearch\Mapping(format="YYYY-MM-dd") 98 | * @ElasticSearch\Transform("Date") 99 | */ 100 | protected $date; 101 | 102 | This will call the (supplied with the package) Date transformer and hand the converted value over to the ElasticSearch 103 | engine. 104 | 105 | Setting up the indexes 106 | ====================== 107 | 108 | As soon as you have proper configuration for your entities, you can create your index, with the CLI utility:: 109 | 110 | flow index:create --index-name twitter 111 | 112 | If you need advanced settings you can define them in your ``Settings.yaml``:: 113 | 114 | Flowpack: 115 | ElasticSearch: 116 | indexes: 117 | default: 118 | 'twitter': 119 | analysis: 120 | filter: 121 | elision: 122 | type: 'elision' 123 | articles: [ 'l', 'm', 't', 'qu', 'n', 's', 'j', 'd' ] 124 | analyzer: 125 | custom_french_analyzer: 126 | tokenizer: 'letter' 127 | filter: [ 'asciifolding', 'lowercase', 'french_stem', 'elision', 'stop' ] 128 | tag_analyzer: 129 | tokenizer: 'keyword' 130 | filter: [ 'asciifolding', 'lowercase' ] 131 | 132 | If you use multiple client configurations, please change the ``default`` key just below the ``indexes``. 133 | 134 | You can update the index configuration with the following CLI:: 135 | 136 | flow index:updateSettings --index-name twitter 137 | 138 | Please check the ElasticSearch configuration to know witch settings are updatable. For any other settings changes, you 139 | need to delete your indexes:: 140 | 141 | flow index:delete --index-name twitter 142 | 143 | .. [#suppProperties] *supported properties* are all scalar types, unless value transformation is applied. 144 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Mapping.php: -------------------------------------------------------------------------------- 1 | type = $type; 58 | $this->properties[static::NEOS_TYPE_FIELD] = ['type' => 'keyword']; 59 | } 60 | 61 | /** 62 | * Gets a property setting by its path 63 | * 64 | * @param array|string $path 65 | * @return mixed 66 | */ 67 | public function getPropertyByPath($path) 68 | { 69 | return Arrays::getValueByPath($this->properties, $path); 70 | } 71 | 72 | /** 73 | * Gets a property setting by its path 74 | * 75 | * @param array|string $path 76 | * @param string $value 77 | * @return void 78 | */ 79 | public function setPropertyByPath($path, $value) 80 | { 81 | $this->properties = Arrays::setValueByPath($this->properties, $path, $value); 82 | } 83 | 84 | /** 85 | * @return AbstractType 86 | */ 87 | public function getType(): AbstractType 88 | { 89 | return $this->type; 90 | } 91 | 92 | /** 93 | * Sets this mapping to the server 94 | * 95 | * @return Response 96 | * @throws \Flowpack\ElasticSearch\Exception 97 | * @throws \Neos\Flow\Http\Exception 98 | */ 99 | public function apply(): Response 100 | { 101 | $content = json_encode($this->asArray()); 102 | 103 | return $this->type->request('PUT', '/_mapping', [], $content); 104 | } 105 | 106 | /** 107 | * Return the mapping which would be sent to the server as array 108 | * 109 | * @return array 110 | */ 111 | public function asArray(): array 112 | { 113 | return Arrays::arrayMergeRecursiveOverrule([ 114 | 'dynamic_templates' => $this->getDynamicTemplates(), 115 | 'properties' => $this->getProperties(), 116 | ], $this->fullMapping); 117 | } 118 | 119 | /** 120 | * @return array 121 | */ 122 | public function getDynamicTemplates(): array 123 | { 124 | return $this->dynamicTemplates; 125 | } 126 | 127 | /** 128 | * @return array 129 | */ 130 | public function getProperties(): array 131 | { 132 | return $this->properties; 133 | } 134 | 135 | /** 136 | * Dynamic templates allow to define mapping templates 137 | * 138 | * @param string $dynamicTemplateName 139 | * @param array $mappingConfiguration 140 | * @return void 141 | */ 142 | public function addDynamicTemplate(string $dynamicTemplateName, array $mappingConfiguration): void 143 | { 144 | $this->dynamicTemplates[] = [ 145 | $dynamicTemplateName => $mappingConfiguration, 146 | ]; 147 | } 148 | 149 | /** 150 | * See {@link setFullMapping} for documentation 151 | * 152 | * @return array 153 | */ 154 | public function getFullMapping(): array 155 | { 156 | return $this->fullMapping; 157 | } 158 | 159 | /** 160 | * This is the full / raw ElasticSearch mapping which is merged with the properties and dynamicTemplates. 161 | * 162 | * It can be used to specify arbitrary ElasticSearch mapping options, like f.e. configuring the _all field. 163 | * 164 | * @param array $fullMapping 165 | * @return void 166 | */ 167 | public function setFullMapping(array $fullMapping): void 168 | { 169 | $this->fullMapping = $fullMapping; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /Classes/Transfer/RequestService.php: -------------------------------------------------------------------------------- 1 | settings = $settings; 61 | } 62 | 63 | /** 64 | * @return void 65 | */ 66 | public function initializeObject() 67 | { 68 | $requestEngine = new CurlEngine(); 69 | $requestEngine->setOption(CURLOPT_TIMEOUT, $this->settings['transfer']['connectionTimeout']); 70 | $requestEngine->setOption(CURLOPT_SSL_VERIFYPEER, $this->settings['transfer']['sslVerifyPeer'] ?? true ? 2 : 0); 71 | $requestEngine->setOption(CURLOPT_SSL_VERIFYHOST, $this->settings['transfer']['sslVerifyHost'] ?? true ? 2 : 0); 72 | 73 | if (!empty($this->settings['transfer']['sslCaInfo'])) { 74 | $requestEngine->setOption(CURLOPT_CAINFO, $this->settings['transfer']['sslCaInfo']); 75 | } 76 | 77 | if (!empty($this->settings['transfer']['sslKey'])) { 78 | $requestEngine->setOption(CURLOPT_SSLKEY, $this->settings['transfer']['sslKey']); 79 | } 80 | 81 | if (!empty($this->settings['transfer']['sslCert'])) { 82 | $requestEngine->setOption(CURLOPT_SSLCERT, $this->settings['transfer']['sslCert']); 83 | } 84 | 85 | if (!empty($this->settings['transfer']['sslKeyPasswd'])) { 86 | $requestEngine->setOption(CURLOPT_SSLKEYPASSWD, $this->settings['transfer']['sslKeyPasswd']); 87 | } 88 | 89 | $this->browser->setRequestEngine($requestEngine); 90 | } 91 | 92 | /** 93 | * @param string $method 94 | * @param ElasticSearchClient $client 95 | * @param string $path 96 | * @param array $arguments 97 | * @param string|array $content 98 | * @return Response 99 | * @throws Exception 100 | * @throws Exception\ApiException 101 | * @throws \Neos\Flow\Http\Exception 102 | */ 103 | public function request($method, ElasticSearchClient $client, ?string $path = null, array $arguments = [], $content = null): Response 104 | { 105 | $clientConfigurations = $client->getClientConfigurations(); 106 | $clientConfiguration = $clientConfigurations[0]; 107 | /** @var ClientConfiguration $clientConfiguration */ 108 | 109 | $uri = clone $clientConfiguration->getUri(); 110 | 111 | if ($path !== null) { 112 | if (strpos($path, '?') !== false) { 113 | list($path, $query) = explode('?', $path); 114 | $uri = $uri->withQuery($query); 115 | } 116 | 117 | $uri = $uri->withPath($uri->getPath() . $path); 118 | } 119 | 120 | if (!empty($arguments)) { 121 | $uri = $uri->withQuery($uri->getQuery() . '&' . http_build_query($arguments)); 122 | } 123 | 124 | $request = $this->requestFactory->createServerRequest($method, $uri); 125 | 126 | // In some cases, $content will contain "null" as a string. Better be safe and handle this weird case: 127 | if ($content !== 'null' && $content !== null) { 128 | $request = $request->withBody($this->contentStreamFactory->createStream(is_array($content) ? json_encode($content) : (string)$content)); 129 | } 130 | 131 | $request = $request->withHeader('Content-Type', 'application/json'); 132 | $response = $this->browser->sendRequest($request); 133 | 134 | return new Response($response, $this->browser->getLastRequest()); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Classes/Annotations/Mapping.php: -------------------------------------------------------------------------------- 1 | fields; 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /Tests/Functional/Indexer/Object/ObjectIndexerTest.php: -------------------------------------------------------------------------------- 1 | testEntityRepository = new TweetRepository(); 45 | $this->testClient = $this->objectManager->get(ObjectIndexer::class)->getClient(); 46 | } 47 | 48 | /** 49 | * @test 50 | */ 51 | public function persistingNewObjectTriggersIndexing() 52 | { 53 | $testEntity = $this->createAndPersistTestEntity(); 54 | $documentId = $this->persistenceManager->getIdentifierByObject($testEntity); 55 | 56 | $resultDocument = $this->testClient 57 | ->findIndex('flow_elasticsearch_functionaltests_twitter') 58 | ->findType('tweet') 59 | ->findDocumentById($documentId); 60 | $resultData = $resultDocument->getData(); 61 | 62 | static::assertEquals($testEntity->getMessage(), $resultData['message']); 63 | static::assertEquals($testEntity->getUsername(), $resultData['username']); 64 | } 65 | 66 | /** 67 | * @test 68 | */ 69 | public function updatingExistingObjectTriggersReindexing() 70 | { 71 | $testEntity = $this->createAndPersistTestEntity(); 72 | $identifier = $this->persistenceManager->getIdentifierByObject($testEntity); 73 | 74 | $initialVersion = $this->testClient 75 | ->findIndex('flow_elasticsearch_functionaltests_twitter') 76 | ->findType('tweet') 77 | ->findDocumentById($identifier) 78 | ->getVersion(); 79 | static::assertIsInt($initialVersion); 80 | 81 | $persistedTestEntity = $this->testEntityRepository->findByIdentifier($identifier); 82 | $persistedTestEntity->setMessage('changed message.'); 83 | $this->testEntityRepository->update($persistedTestEntity); 84 | $this->persistenceManager->persistAll(); 85 | $this->persistenceManager->clearState(); 86 | 87 | $changedDocument = $this->testClient 88 | ->findIndex('flow_elasticsearch_functionaltests_twitter') 89 | ->findType('tweet') 90 | ->findDocumentById($identifier); 91 | 92 | // the version increments by two, since we index via AOP and via Doctrine lifecycle events 93 | // see https://github.com/Flowpack/Flowpack.ElasticSearch/pull/36 94 | static::assertSame($initialVersion + 2, $changedDocument->getVersion()); 95 | static::assertSame($changedDocument->getField('message'), 'changed message.'); 96 | } 97 | 98 | /** 99 | * @test 100 | */ 101 | public function removingObjectTriggersIndexRemoval() 102 | { 103 | $testEntity = $this->createAndPersistTestEntity(); 104 | $identifier = $this->persistenceManager->getIdentifierByObject($testEntity); 105 | 106 | $initialDocument = $this->testClient 107 | ->findIndex('flow_elasticsearch_functionaltests_twitter') 108 | ->findType('tweet') 109 | ->findDocumentById($identifier); 110 | static::assertInstanceOf(Document::class, $initialDocument); 111 | 112 | $persistedTestEntity = $this->testEntityRepository->findByIdentifier($identifier); 113 | $this->testEntityRepository->remove($persistedTestEntity); 114 | $this->persistenceManager->persistAll(); 115 | $this->persistenceManager->clearState(); 116 | 117 | $foundDocument = $this->testClient 118 | ->findIndex('flow_elasticsearch_functionaltests_twitter') 119 | ->findType('tweet') 120 | ->findDocumentById($identifier); 121 | static::assertNull($foundDocument); 122 | } 123 | 124 | protected function createAndPersistTestEntity() 125 | { 126 | $testEntity = new Tweet(); 127 | $testEntity->setDate(new \DateTime()); 128 | $testEntity->setMessage('This is a test message ' . Algorithms::generateRandomString(8)); 129 | $testEntity->setUsername('Zak McKracken' . Algorithms::generateRandomString(8)); 130 | 131 | $this->testEntityRepository->add($testEntity); 132 | $this->persistenceManager->persistAll(); 133 | $this->persistenceManager->clearState(); 134 | return $testEntity; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/IndexInformer.php: -------------------------------------------------------------------------------- 1 | indexAnnotations = self::buildIndexClassesAndProperties($this->objectManager); 51 | } 52 | 53 | /** 54 | * Creates the source array of what classes and properties have to be annotated. 55 | * The returned array consists of class names, with a sub-key having both 'annotation' and 'properties' set. 56 | * The annotation contains the class's annotation, while properties contains each property that has to be indexed. 57 | * Each property might either have TRUE as value, or also an annotation instance, if given. 58 | * 59 | * @param ObjectManagerInterface $objectManager 60 | * @return array 61 | * @throws ElasticSearchException 62 | */ 63 | public static function buildIndexClassesAndProperties($objectManager) 64 | { 65 | /** @var ReflectionService $reflectionService */ 66 | $reflectionService = $objectManager->get(ReflectionService::class); 67 | 68 | $indexAnnotations = []; 69 | 70 | $annotationClassName = Indexable::class; 71 | foreach ($reflectionService->getClassNamesByAnnotation($annotationClassName) as $className) { 72 | if ($reflectionService->isClassAbstract($className)) { 73 | throw new ElasticSearchException(sprintf('The class with name "%s" is annotated with %s, but is abstract. Indexable classes must not be abstract.', $className, $annotationClassName), 1339595182); 74 | } 75 | $indexAnnotations[$className]['annotation'] = $reflectionService->getClassAnnotation($className, $annotationClassName); 76 | 77 | // if no single properties are set to be indexed, consider all properties to be indexed. 78 | $annotatedProperties = $reflectionService->getPropertyNamesByAnnotation($className, $annotationClassName); 79 | if (!empty($annotatedProperties)) { 80 | $indexAnnotations[$className]['properties'] = $annotatedProperties; 81 | } else { 82 | foreach ($reflectionService->getClassPropertyNames($className) as $propertyName) { 83 | $indexAnnotations[$className]['properties'][] = $propertyName; 84 | } 85 | } 86 | } 87 | 88 | return $indexAnnotations; 89 | } 90 | 91 | /** 92 | * Returns all indexes name deplared in class annotations 93 | * 94 | * @return array 95 | */ 96 | public function getAllIndexNames() 97 | { 98 | $indexes = []; 99 | foreach ($this->getClassesAndAnnotations() as $configuration) { 100 | /** @var Indexable $configuration */ 101 | $indexes[$configuration->indexName] = $configuration->indexName; 102 | } 103 | 104 | return array_keys($indexes); 105 | } 106 | 107 | /** 108 | * Returns the to-index classes and their annotation 109 | * 110 | * @return array 111 | */ 112 | public function getClassesAndAnnotations() 113 | { 114 | static $classesAndAnnotations; 115 | if ($classesAndAnnotations === null) { 116 | $classesAndAnnotations = []; 117 | foreach (array_keys($this->indexAnnotations) as $className) { 118 | $classesAndAnnotations[$className] = $this->indexAnnotations[$className]['annotation']; 119 | } 120 | } 121 | 122 | return $classesAndAnnotations; 123 | } 124 | 125 | /** 126 | * @param string $className 127 | * @return Indexable The annotation for this class 128 | */ 129 | public function getClassAnnotation($className) 130 | { 131 | if (!isset($this->indexAnnotations[$className])) { 132 | return null; 133 | } 134 | 135 | return $this->indexAnnotations[$className]['annotation']; 136 | } 137 | 138 | /** 139 | * @param string $className 140 | * @return array 141 | */ 142 | public function getClassProperties($className) 143 | { 144 | if (!isset($this->indexAnnotations[$className])) { 145 | return null; 146 | } 147 | 148 | return $this->indexAnnotations[$className]['properties']; 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Document.php: -------------------------------------------------------------------------------- 1 | type = $type; 66 | $this->data = $data; 67 | $this->id = $id; 68 | $this->version = $version; 69 | } 70 | 71 | /** 72 | * When cloning (locally), the cloned object doesn't represent a stored one anymore, 73 | * so reset id, version and the dirty state. 74 | * 75 | * @return void 76 | */ 77 | public function __clone() 78 | { 79 | $this->id = null; 80 | $this->version = null; 81 | $this->setDirty(); 82 | } 83 | 84 | /** 85 | * Stores this document. If ID is given, PUT will be used; else POST 86 | * 87 | * @return void 88 | * @throws \Neos\Flow\Http\Exception 89 | * @throws ElasticSearchException 90 | */ 91 | public function store(): void 92 | { 93 | if ($this->id !== null) { 94 | $method = 'PUT'; 95 | $path = '/_doc/' . $this->id; 96 | } else { 97 | $method = 'POST'; 98 | $path = '/_doc/'; 99 | } 100 | 101 | $response = $this->request($method, $path, [], json_encode($this->getData())); 102 | $treatedContent = $response->getTreatedContent(); 103 | 104 | $this->id = $treatedContent['_id']; 105 | $this->version = $treatedContent['_version']; 106 | $this->dirty = false; 107 | } 108 | 109 | /** 110 | * @return boolean 111 | */ 112 | public function isDirty(): bool 113 | { 114 | return $this->dirty; 115 | } 116 | 117 | /** 118 | * @return int|nulll 119 | */ 120 | public function getVersion(): ?int 121 | { 122 | return $this->version; 123 | } 124 | 125 | /** 126 | * The contents of this document 127 | * 128 | * @return array 129 | */ 130 | public function getData(): array 131 | { 132 | $this->data[Mapping::NEOS_TYPE_FIELD] = $this->type->getName(); 133 | return $this->data; 134 | } 135 | 136 | /** 137 | * @param array $data 138 | * @return void 139 | */ 140 | public function setData(array $data): void 141 | { 142 | $this->data = $data; 143 | $this->setDirty(); 144 | } 145 | 146 | /** 147 | * @return string|null 148 | */ 149 | public function getId(): ?string 150 | { 151 | return $this->id; 152 | } 153 | 154 | /** 155 | * Gets a specific field's value from this' data 156 | * 157 | * @param string $fieldName 158 | * @param boolean $silent 159 | * @return mixed 160 | * @throws ElasticSearchException 161 | */ 162 | public function getField(string $fieldName, bool $silent = false) 163 | { 164 | if (!array_key_exists($fieldName, $this->data) && $silent === false) { 165 | throw new ElasticSearchException(sprintf('The field %s was not present in data of document in %s/%s.', $fieldName, $this->type->getIndex()->getName(), $this->type->getName()), 1340274696); 166 | } 167 | 168 | return $this->data[$fieldName]; 169 | } 170 | 171 | /** 172 | * @return AbstractType the type of this Document 173 | */ 174 | public function getType(): AbstractType 175 | { 176 | return $this->type; 177 | } 178 | 179 | /** 180 | * @param string $method 181 | * @param string|null $path 182 | * @param array $arguments 183 | * @param string|null $content 184 | * @return Response 185 | * @throws ElasticSearchException 186 | * @throws \Neos\Flow\Http\Exception 187 | */ 188 | protected function request(string $method, ?string $path = null, array $arguments = [], ?string $content = null): Response 189 | { 190 | return $this->type->request($method, $path, $arguments, $content); 191 | } 192 | 193 | /** 194 | * @param boolean $dirty 195 | * @return void 196 | */ 197 | protected function setDirty(bool $dirty = true): void 198 | { 199 | $this->dirty = $dirty; 200 | } 201 | } 202 | -------------------------------------------------------------------------------- /Documentation/Index.rst: -------------------------------------------------------------------------------- 1 | ==================================== 2 | Flowpack.ElasticSearch documentation 3 | ==================================== 4 | 5 | This package provides an API for using Elasticsearch with Flow. The intention is to provide a simple, 6 | `fluent-interface`_-driven framework respecting the paradigm of Elasticsearch, made for working best 7 | with the features Flow already provides. 8 | 9 | Setting up the clients 10 | ====================== 11 | 12 | Usually you will only need one *client*. A *client* is one target the search communicates against, hence will consist 13 | of many *nodes*. Perform the setting in the appropriate ``Settings.yaml``:: 14 | 15 | Flowpack: 16 | ElasticSearch: 17 | clients: 18 | # default bundle that will be used if no more specific bundle name was supplied. 19 | default: 20 | - host: localhost 21 | port: 9200 22 | - host: localhost 23 | port: 9201 24 | 25 | # reserved bundle name that's used when running the package's functional tests. 26 | FunctionalTests: 27 | - host: localhost 28 | port: 9200 29 | 30 | In this example setup, there's one *client* with two *nodes*, both at ``localhost``, one at port ``9200``, the other 31 | at port ``9201``. The ``FunctionalTests`` client is a reserved one that acts for functional testing. If you like to 32 | have a dedicated server (recommended) for running the Functional Tests, set it up here. 33 | 34 | During runtime, you have to provide the client's name you want to connect to, if it is not `default`. 35 | 36 | When access to the Elasticsearch instance is protected through HTTP Basic Auth, you can provide the necessary username 37 | and password in your client settings:: 38 | 39 | Flowpack: 40 | ElasticSearch: 41 | clients: 42 | # default bundle that will be used if no more specific bundle name was supplied. 43 | default: 44 | - host: my.elasticsearch-service.com 45 | port: 443 46 | scheme: https 47 | username: john 48 | password: mysecretpassword 49 | 50 | The following options are available to configure TLS connections. These correspond to the options provided by cURL:: 51 | 52 | sslVerifyHost: true 53 | sslVerifyPeer: true 54 | 55 | # CA certificate to verify the peer with 56 | sslCaInfo: './root-ca.pem' 57 | 58 | # file containing the private SSL key 59 | sslKey: './client-key.pem' 60 | 61 | # file containing the PEM formatted certificate 62 | sslCert: './client.pem' 63 | 64 | # password needed for the private SSL key 65 | sslKeyPasswd: 'some-password' 66 | 67 | Running the Functional Tests 68 | ============================ 69 | 70 | For running the Functional Tests, the API will connect to the server and host you've set up in the ``Settings.yaml`` 71 | directive ``Flowpack.ElasticSearch.clients.FunctionalTests`` (see above). The test scenario will try to create a temporary 72 | test *index* named ``flow_elasticsearch_functionaltests`` where it will work the test data on. If this index already 73 | exists, the test will stop with a notification in order not to destroy some real-life-data in the unlikely case it has 74 | that name. 75 | 76 | After the tests run through, this index will be dropped rigorously. 77 | 78 | Fetching a Client instance to work on 79 | ===================================== 80 | 81 | For whatever you want to achieve, you need a *client* representation to work on. This is done via the ``ClientFactory``, 82 | inject it wherever you need:: 83 | 84 | class SampleClass { 85 | 86 | /** 87 | * @Flow\Inject 88 | * @var \Flowpack\ElasticSearch\Client\ClientFactory 89 | */ 90 | protected $clientFactory; 91 | 92 | public function sampleMethod() { 93 | $client = $this->clientFactory->create(); 94 | } 95 | 96 | } 97 | 98 | This will create a fresh client instance connecting to the client set (i.e. *nodes* you've configured in the 99 | ``Settings.yaml``). In this case, the ``create()`` method carries no argument, meaning the ``default`` client will be 100 | taken. 101 | 102 | Handling documents 103 | ================== 104 | 105 | While handling *documents* (the actual data that is to be indexed), besides the *client*, there's additionally an 106 | *index* and a *type* involved. And *index*, if not yet present, will be created automatically. A *type* specifies, 107 | as its name allows to guess, the type of a document, for example "Tweet" that represents a Twitter tweet, or "Actor" 108 | that represents a movie actor. 109 | 110 | While a document itself is very generic (it consists of data, its mother *index* and the *type*), the type is specific 111 | and reflects some real existing Model. Therefore the API provides an AbstractType where you as the developer inherit 112 | your specific, intended types from, for example:: 113 | 114 | class TwitterType extends \Flowpack\ElasticSearch\Domain\Model\AbstractType { 115 | } 116 | 117 | This class might even be empty like in this case, it just has to be there. Per default, the name of the type is 118 | determined from the full namespace. If you want to change that, just override the ``getName()`` method which is provided 119 | by the ``AbstractType`` class. 120 | 121 | So for storing a Twitter document, follow this example:: 122 | 123 | class SampleClass { 124 | 125 | /** 126 | * @Flow\Inject 127 | * @var \Flowpack\ElasticSearch\Client\ClientFactory 128 | */ 129 | protected $clientFactory; 130 | 131 | public function sampleMethod() { 132 | $client = $this->clientFactory->create(); 133 | $tweetsIndex = $client->findIndex('tweets'); 134 | $twitterType = new TwitterType($tweetsIndex); 135 | $document = new \Flowpack\ElasticSearch\Document($twitterType, array( 136 | 'user' => 'John', 137 | 'date' => '2012-06-12', 138 | 'text' => 'This is an example document data' 139 | )); 140 | $document->store(); 141 | } 142 | 143 | } 144 | 145 | This will make the document being stored by transforming the object chain to its corresponding REST service call. 146 | 147 | 148 | .. _fluent-interface: http://martinfowler.com/bliki/FluentInterface.html -------------------------------------------------------------------------------- /Classes/Mapping/EntityMappingBuilder.php: -------------------------------------------------------------------------------- 1 | 65 | * @throws ElasticSearchException 66 | */ 67 | public function buildMappingInformation() 68 | { 69 | $mappings = new MappingCollection(MappingCollection::TYPE_ENTITY); 70 | foreach ($this->indexInformer->getClassesAndAnnotations() as $className => $annotation) { 71 | $mappings->add($this->buildMappingFromClassAndAnnotation($className, $annotation)); 72 | } 73 | 74 | return $mappings; 75 | } 76 | 77 | /** 78 | * @param string $className 79 | * @param IndexableAnnotation $annotation 80 | * @return Mapping 81 | * @throws ElasticSearchException 82 | */ 83 | protected function buildMappingFromClassAndAnnotation($className, IndexableAnnotation $annotation) 84 | { 85 | $index = new ElasticSearchIndex($annotation->indexName); 86 | $type = new GenericType($index, $annotation->typeName); 87 | $mapping = new Mapping($type); 88 | foreach ($this->indexInformer->getClassProperties($className) as $propertyName) { 89 | $this->augmentMappingByProperty($mapping, $className, $propertyName); 90 | } 91 | 92 | return $mapping; 93 | } 94 | 95 | /** 96 | * @param Mapping $mapping 97 | * @param string $className 98 | * @param string $propertyName 99 | * @return void 100 | * @throws ElasticSearchException 101 | */ 102 | protected function augmentMappingByProperty(Mapping $mapping, string $className, string $propertyName): void 103 | { 104 | list($propertyType) = $this->reflectionService->getPropertyTagValues($className, $propertyName, 'var'); 105 | if (($transformAnnotation = $this->reflectionService->getPropertyAnnotation($className, $propertyName, Transform::class)) !== null) { 106 | $mappingType = $this->transformerFactory->create($transformAnnotation->type)->getTargetMappingType(); 107 | } elseif ($propertyType === 'string') { 108 | // string must be mapped to text as elasticsearch does not support the 'string' type for version >=5.0 109 | $mappingType = 'text'; 110 | } elseif (TypeHandling::isSimpleType($propertyType)) { 111 | $mappingType = $propertyType; 112 | } elseif ($propertyType === '\DateTime') { 113 | $mappingType = 'date'; 114 | } else { 115 | throw new ElasticSearchException('Mapping is only supported for simple types and DateTime objects; "' . $propertyType . '" given but without a Transform directive.'); 116 | } 117 | 118 | $mapping->setPropertyByPath($propertyName, ['type' => $mappingType]); 119 | 120 | $annotation = $this->reflectionService->getPropertyAnnotation($className, $propertyName, MappingAnnotation::class); 121 | 122 | if ($annotation instanceof MappingAnnotation) { 123 | $mapping->setPropertyByPath($propertyName, $this->processMappingAnnotation($annotation, $mapping->getPropertyByPath($propertyName))); 124 | if ($annotation->getFields()) { 125 | $multiFields = []; 126 | foreach ($annotation->getFields() as $multiFieldAnnotation) { 127 | $multiFieldIndexName = trim($multiFieldAnnotation->index_name); 128 | if ($multiFieldIndexName === '') { 129 | throw new ElasticSearchException('Multi field require an unique index name "' . $className . '::' . $propertyName . '".'); 130 | } 131 | if (isset($multiFields[$multiFieldIndexName])) { 132 | throw new ElasticSearchException('Duplicate index name in the same multi field is not allowed "' . $className . '::' . $propertyName . '".'); 133 | } 134 | if (!$multiFieldAnnotation->type) { 135 | // Fallback to the parent's type if not specified on multi-field 136 | $multiFieldAnnotation->type = $mappingType; 137 | } 138 | $multiFields[$multiFieldIndexName] = $this->processMappingAnnotation($multiFieldAnnotation); 139 | } 140 | $mapping->setPropertyByPath([$propertyName, 'fields'], $multiFields); 141 | } 142 | } 143 | } 144 | 145 | /** 146 | * @param MappingAnnotation $annotation 147 | * @param array $propertyMapping 148 | * @return array 149 | */ 150 | protected function processMappingAnnotation(MappingAnnotation $annotation, array $propertyMapping = []) 151 | { 152 | foreach ($annotation->getPropertiesArray() as $mappingDirective => $directiveValue) { 153 | if ($directiveValue === null) { 154 | continue; 155 | } 156 | $propertyMapping = Arrays::setValueByPath($propertyMapping, $mappingDirective, $directiveValue); 157 | } 158 | 159 | return $propertyMapping; 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /Classes/Indexer/Object/ObjectIndexer.php: -------------------------------------------------------------------------------- 1 | getIndexTypeForObject($object, $client); 84 | if ($type === null) { 85 | return null; 86 | } 87 | $data = $this->getIndexablePropertiesAndValuesFromObject($object); 88 | 89 | $id = $this->persistenceManager->getIdentifierByObject($object); 90 | $document = new Document($type, $data, $id); 91 | $document->store(); 92 | } 93 | 94 | /** 95 | * Returns the ElasticSearch type for a specific object, by its annotation 96 | * 97 | * @param object $object 98 | * @param Client $client 99 | * @return GenericType 100 | */ 101 | protected function getIndexTypeForObject($object, ?Client $client = null) 102 | { 103 | if ($client === null) { 104 | $client = $this->client; 105 | } 106 | $className = TypeHandling::getTypeForValue($object); 107 | $indexAnnotation = $this->indexInformer->getClassAnnotation($className); 108 | if ($indexAnnotation === null) { 109 | return null; 110 | } 111 | $index = $client->findIndex($indexAnnotation->indexName); 112 | 113 | return new GenericType($index, $indexAnnotation->typeName); 114 | } 115 | 116 | /** 117 | * Returns a multidimensional array with the indexable, probably transformed values of an object 118 | * 119 | * @param object $object 120 | * @return array 121 | */ 122 | protected function getIndexablePropertiesAndValuesFromObject($object) 123 | { 124 | $className = TypeHandling::getTypeForValue($object); 125 | foreach ($this->indexInformer->getClassProperties($className) as $propertyName) { 126 | if (ObjectAccess::isPropertyGettable($object, $propertyName) === false) { 127 | continue; 128 | } 129 | 130 | $value = ObjectAccess::getProperty($object, $propertyName); 131 | if (($transformAnnotation = $this->reflectionService->getPropertyAnnotation($className, $propertyName, TransformAnnotation::class)) !== null) { 132 | $value = $this->transformerFactory->create($transformAnnotation->type)->transformByAnnotation($value, $transformAnnotation); 133 | } 134 | 135 | $data[$propertyName] = $value; 136 | } 137 | 138 | return $data; 139 | } 140 | 141 | /** 142 | * @param object $object 143 | * @param string $signalInformation Signal information, if called from a signal 144 | * @param Client $client 145 | * @return void 146 | */ 147 | public function removeObject($object, $signalInformation = null, ?Client $client = null) 148 | { 149 | $type = $this->getIndexTypeForObject($object, $client); 150 | if ($type === null) { 151 | return; 152 | } 153 | $id = $this->persistenceManager->getIdentifierByObject($object); 154 | $type->deleteDocumentById($id); 155 | } 156 | 157 | /** 158 | * Returns if, and what, treatment an object requires regarding the index state, 159 | * i.e. it checks the given object against the index and tells whether deletion, update or creation is required. 160 | * 161 | * @param object $object 162 | * @param Client $client 163 | * @return string one of this' ACTION_TYPE_* constants or NULL if no action is required 164 | */ 165 | public function objectIndexActionRequired($object, ?Client $client = null) 166 | { 167 | $type = $this->getIndexTypeForObject($object, $client); 168 | if ($type === null) { 169 | return null; 170 | } 171 | $id = $this->persistenceManager->getIdentifierByObject($object); 172 | $document = $type->findDocumentById($id); 173 | if ($document !== null) { 174 | $objectData = $this->getIndexablePropertiesAndValuesFromObject($object); 175 | $objectData[Mapping::NEOS_TYPE_FIELD] = $type->getName(); 176 | if (strcmp(json_encode($objectData), json_encode($document->getData())) === 0) { 177 | $actionType = null; 178 | } else { 179 | $actionType = self::ACTION_TYPE_UPDATE; 180 | } 181 | } else { 182 | $actionType = self::ACTION_TYPE_CREATE; 183 | } 184 | 185 | return $actionType; 186 | } 187 | 188 | /** 189 | * Returns the currently used client, used for functional testing 190 | * 191 | * @return Client 192 | */ 193 | public function getClient() 194 | { 195 | return $this->client; 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /Classes/Domain/Model/Index.php: -------------------------------------------------------------------------------- 1 | name = $name; 119 | $this->settingsKey = $name; 120 | $this->client = $client; 121 | } 122 | 123 | /** 124 | * Inject the framework settings 125 | * 126 | * @param array $settings 127 | * @return void 128 | */ 129 | public function injectSettings(array $settings): void 130 | { 131 | $this->settings = $settings; 132 | } 133 | 134 | /** 135 | * @param string $typeName 136 | * @return AbstractType 137 | */ 138 | public function findType(string $typeName): AbstractType 139 | { 140 | return new GenericType($this, $typeName); 141 | } 142 | 143 | /** 144 | * @param array $types 145 | * @return TypeGroup 146 | */ 147 | public function findTypeGroup(array $types): TypeGroup 148 | { 149 | return new TypeGroup($this, $types); 150 | } 151 | 152 | /** 153 | * @return bool 154 | * @throws \Exception 155 | */ 156 | public function exists(): bool 157 | { 158 | $response = $this->request('HEAD'); 159 | 160 | return $response->getStatusCode() === 200; 161 | } 162 | 163 | /** 164 | * @param string $method 165 | * @param string $path 166 | * @param array $arguments 167 | * @param string|array $content 168 | * @param bool $prefixIndex 169 | * @return Response 170 | * @throws ElasticSearchException 171 | * @throws \Neos\Flow\Http\Exception 172 | */ 173 | public function request(string $method, ?string $path = null, array $arguments = [], $content = null, bool $prefixIndex = true): Response 174 | { 175 | if ($this->client === null) { 176 | throw new ElasticSearchException('The client of the index "' . $this->prefixName() . '" is not set, hence no requests can be done.', 1566313883); 177 | } 178 | $path = ltrim($path ? trim($path) : '', '/'); 179 | if ($prefixIndex === true) { 180 | $path = '/' . $this->prefixName() . '/' . $path; 181 | } else { 182 | $path = '/' . ltrim($path, '/'); 183 | } 184 | 185 | return $this->client->request($method, $path, $arguments, $content); 186 | } 187 | 188 | /** 189 | * @throws ElasticSearchException 190 | * @throws \Neos\Flow\Http\Exception 191 | */ 192 | public function create(): void 193 | { 194 | $indexConfiguration = $this->getConfiguration() ?? []; 195 | $indexCreateObject = array_filter($indexConfiguration, static fn($key) => in_array($key, self::$allowedIndexCreateKeys, true), ARRAY_FILTER_USE_KEY); 196 | $this->request('PUT', null, [], $this->encodeRequestBody($indexCreateObject)); 197 | } 198 | 199 | /** 200 | * @return array|null 201 | */ 202 | protected function getConfiguration(): ?array 203 | { 204 | if ($this->client instanceof Client) { 205 | $path = 'indexes.' . $this->client->getBundle() . '.' . $this->settingsKey; 206 | } else { 207 | $path = 'indexes.default' . '.' . $this->settingsKey; 208 | } 209 | 210 | $cconfiguration = Arrays::getValueByPath($this->settings, $path); 211 | return $cconfiguration !== null ? $this->dynamicIndexSettingService->process($cconfiguration, $path, $this->name) : $cconfiguration; 212 | } 213 | 214 | /** 215 | * @throws ElasticSearchException 216 | * @throws \Neos\Flow\Http\Exception 217 | */ 218 | public function updateSettings(): void 219 | { 220 | // we only ever need the settings path from all the settings. 221 | $settings = $this->getConfiguration()['settings'] ?? []; 222 | $updatableSettings = []; 223 | foreach (static::$updatableSettings as $settingPath) { 224 | $setting = Arrays::getValueByPath($settings, $settingPath); 225 | if ($setting !== null) { 226 | $updatableSettings = Arrays::setValueByPath($updatableSettings, $settingPath, $setting); 227 | } 228 | } 229 | if ($updatableSettings !== []) { 230 | $this->request('PUT', '/_settings', [], $this->encodeRequestBody($updatableSettings)); 231 | } 232 | } 233 | 234 | /** 235 | * @return Response 236 | * @throws ElasticSearchException 237 | * @throws \Neos\Flow\Http\Exception 238 | */ 239 | public function delete(): Response 240 | { 241 | return $this->request('DELETE'); 242 | } 243 | 244 | /** 245 | * Refresh the index 246 | * 247 | * @return Response 248 | * @throws ElasticSearchException 249 | * @throws \Neos\Flow\Http\Exception 250 | */ 251 | public function refresh(): Response 252 | { 253 | return $this->request('POST', '/_refresh'); 254 | } 255 | 256 | /** 257 | * @return string 258 | */ 259 | public function getName(): string 260 | { 261 | return $this->prefixName(); 262 | } 263 | 264 | public function getOriginalName(): string 265 | { 266 | return $this->name; 267 | } 268 | 269 | /** 270 | * @param Client $client 271 | * @return void 272 | */ 273 | public function setClient(Client $client): void 274 | { 275 | $this->client = $client; 276 | } 277 | 278 | /** 279 | * @param string $settingsKey 280 | * @return void 281 | */ 282 | public function setSettingsKey(string $settingsKey): void 283 | { 284 | $this->settingsKey = $settingsKey; 285 | } 286 | 287 | /** 288 | * Prepends configured preset to the base index name 289 | * 290 | * @return string 291 | */ 292 | private function prefixName(): string 293 | { 294 | $indexConfiguration = $this->getConfiguration(); 295 | if (!isset($indexConfiguration['prefix']) || empty($indexConfiguration['prefix'])) { 296 | return $this->name; 297 | } 298 | 299 | return $indexConfiguration['prefix'] . '-' . $this->name; 300 | } 301 | 302 | private function encodeRequestBody(array $content): string 303 | { 304 | if ($content === []) { 305 | return ''; 306 | } 307 | 308 | return json_encode($content); 309 | } 310 | } 311 | -------------------------------------------------------------------------------- /Classes/Command/MappingCommandController.php: -------------------------------------------------------------------------------- 1 | entityMappingBuilder->buildMappingInformation(); 58 | $entityMappingCollection = $this->buildArrayFromMappingCollection($entityMappingCollection); 59 | 60 | $client = $this->clientFactory->create($clientName); 61 | $this->backendMappingBuilder->setClient($client); 62 | $backendMappingCollection = $this->backendMappingBuilder->buildMappingInformation(); 63 | $backendMappingCollection = $this->buildArrayFromMappingCollection($backendMappingCollection); 64 | 65 | $this->printLegend(); 66 | $this->outputFormatted('Mapping status:'); 67 | $this->outputFormatted('---------------'); 68 | 69 | $mergedMappingCollection = array_merge_recursive($entityMappingCollection, $backendMappingCollection); 70 | foreach ($mergedMappingCollection as $indexName => $typeSet) { 71 | $this->outputFormatted('index %s:', [$this->markupDiffValue(isset($entityMappingCollection[$indexName]) ? $indexName : null, isset($backendMappingCollection[$indexName]) ? $indexName : null)]); 72 | foreach ($typeSet as $typeName => $mappingSet) { 73 | $propertiesSet = $mappingSet['properties']; 74 | $this->outputFormatted('type %s:', [$this->markupDiffValue(isset($entityMappingCollection[$indexName][$typeName]) ? $typeName : null, isset($backendMappingCollection[$indexName][$typeName]) ? $typeName : null)], 4); 75 | foreach ($propertiesSet as $propertyName => $properties) { 76 | $entityProperties = Arrays::getValueByPath($entityMappingCollection, [ 77 | $indexName, 78 | $typeName, 79 | 'properties', 80 | $propertyName, 81 | ]); 82 | $backendProperties = Arrays::getValueByPath($backendMappingCollection, [ 83 | $indexName, 84 | $typeName, 85 | 'properties', 86 | $propertyName, 87 | ]); 88 | 89 | $this->outputFormatted('property %s:', [$this->markupDiffValue($entityProperties ? $propertyName : null, $backendProperties ? $propertyName : null)], 8); 90 | foreach ($properties as $key => $value) { 91 | $keyMarkup = $this->markupDiffValue(isset($entityProperties[$key]) ? $key : null, isset($backendProperties[$key]) ? $key : null); 92 | $valueMarkup = $this->markupDiffValue(isset($entityProperties[$key]) ? $entityProperties[$key] : null, isset($backendProperties[$key]) ? $backendProperties[$key] : null); 93 | $this->outputFormatted("%s : %s", [$keyMarkup, $valueMarkup], 12); 94 | } 95 | } 96 | $this->outputLine(); 97 | } 98 | $this->outputLine(); 99 | } 100 | 101 | if (count($indicesWithoutTypeInformation = $this->backendMappingBuilder->getIndicesWithoutTypeInformation())) { 102 | $this->outputFormatted("\x1b[43mNotice:\x1b[0m The following indices are present in the backend's mapping but having no type configuration, can hence be regarded as garbage:"); 103 | foreach ($indicesWithoutTypeInformation as $indexName) { 104 | $this->outputFormatted('* %s', [$indexName], 4); 105 | } 106 | } 107 | } 108 | 109 | /** 110 | * Traverses through mappingInformation array and aggregates by index and type names 111 | * 112 | * @param MappingCollection $mappingCollection 113 | * @throws ElasticSearchException 114 | * @return array with index names as keys, second level type names as keys 115 | */ 116 | protected function buildArrayFromMappingCollection(MappingCollection $mappingCollection) 117 | { 118 | $return = []; 119 | 120 | /** @var $mappingInformation Mapping */ 121 | foreach ($mappingCollection as $mappingInformation) { 122 | $indexName = $mappingInformation->getType()->getIndex()->getName(); 123 | $typeName = $mappingInformation->getType()->getName(); 124 | if (isset($return[$indexName][$typeName])) { 125 | throw new ElasticSearchException('There was more than one mapping present in index %s with type %s, which must not happen.', 1339758480); 126 | } 127 | 128 | $return[$indexName][$typeName]['mappingInstance'] = $mappingInformation; 129 | $return[$indexName][$typeName]['properties'] = $mappingInformation->getProperties(); 130 | } 131 | 132 | return $return; 133 | } 134 | 135 | /** 136 | * @return void 137 | */ 138 | protected function printLegend() 139 | { 140 | $legendText = " 141 | " . $this->markupDiffValue(null, 'something') . " defined in backend, but not in entities 142 | " . $this->markupDiffValue('something', null) . " defined in entities, but not in backend 143 | " . $this->markupDiffValue('something', 'something') . " defined both in entities and backend, all OK 144 | " . $this->markupDiffValue('something', 'different') . " different in entities and backend 145 | "; 146 | $this->outputFormatted('Legend:'); 147 | $this->outputFormatted($legendText, [], 4); 148 | } 149 | 150 | /** 151 | * @param mixed $entityValue 152 | * @param mixed $backendValue 153 | * @return string 154 | */ 155 | protected function markupDiffValue($entityValue, $backendValue) 156 | { 157 | $markup = ''; 158 | if ($entityValue === null || $backendValue === null || $entityValue === $backendValue) { 159 | $markup .= "\x1b[" . ($entityValue ? '31' : '30') . ';' . ($backendValue ? '42' : '0') . 'm'; 160 | if (is_array($entityValue)) { 161 | $entityValue = var_export($entityValue, true); 162 | } 163 | if (is_array($backendValue)) { 164 | $backendValue = var_export($backendValue, true); 165 | } 166 | $markup .= $entityValue ?: $backendValue; 167 | $markup .= "\x1b[0m"; 168 | } else { 169 | if (is_array($entityValue)) { 170 | $entityValue = var_export($entityValue, true); 171 | } 172 | if (is_array($backendValue)) { 173 | $backendValue = var_export($backendValue, true); 174 | } 175 | $markup .= "\x1b[31m" . $entityValue . "\x1b[0m"; 176 | $markup .= "\x1b[30;42m" . $backendValue . "\x1b[0m"; 177 | } 178 | 179 | return $markup; 180 | } 181 | 182 | /** 183 | * This command will adjust the backend's mapping to the mapping the entity status prescribes. 184 | * 185 | * @param string $clientName The client name for the configuration. Defaults to the default client configured. 186 | * @return void 187 | */ 188 | public function convergeCommand($clientName = null) 189 | { 190 | $client = $this->clientFactory->create($clientName); 191 | 192 | $entityMappingCollection = $this->entityMappingBuilder->buildMappingInformation(); 193 | $this->backendMappingBuilder->setClient($client); 194 | $backendMappingCollection = $this->backendMappingBuilder->buildMappingInformation(); 195 | 196 | $additiveMappings = $entityMappingCollection->diffAgainstCollection($backendMappingCollection); 197 | /** @var $mapping Mapping */ 198 | foreach ($additiveMappings as $mapping) { 199 | $index = $mapping->getType()->getIndex(); 200 | $index->setClient($client); 201 | if (!$index->exists()) { 202 | $this->outputFormatted('Index %s does not exist', [$index->getName()]); 203 | continue; 204 | } 205 | $this->outputLine('Attempt to apply properties to %s/%s: %s... ', [ 206 | $index->getName(), 207 | $mapping->getType()->getName(), 208 | print_r($mapping->getProperties(), true), 209 | ]); 210 | $response = $mapping->apply(); 211 | if ($response->getStatusCode() === 200) { 212 | $this->outputFormatted('OK'); 213 | } else { 214 | $this->outputFormatted('NOT OK, response code was %d, response body was: %s', [ 215 | $response->getStatusCode(), 216 | $response->getOriginalResponse()->getContent(), 217 | ], 4); 218 | } 219 | } 220 | if ($additiveMappings->count() === 0) { 221 | $this->outputLine('No mappings were to be applied.'); 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /Classes/Command/IndexCommandController.php: -------------------------------------------------------------------------------- 1 | indexInformer->getAllIndexNames())) { 70 | $this->outputFormatted("The index %s is not configured in the current application", [$indexName]); 71 | $this->quit(1); 72 | } 73 | 74 | $client = $this->clientFactory->create($clientName); 75 | try { 76 | $index = new Index($indexName, $client); 77 | if ($index->exists()) { 78 | $this->outputFormatted("The index %s exists", [$indexName]); 79 | $this->quit(1); 80 | } 81 | $index->create(); 82 | $this->outputFormatted("Index %s created with success", [$indexName]); 83 | } catch (Exception $exception) { 84 | $this->outputFormatted("Unable to create an index named: %s", [$indexName]); 85 | $this->quit(1); 86 | } 87 | } 88 | 89 | /** 90 | * Update index settings 91 | * 92 | * @param string $indexName The name of the new index 93 | * @param string $clientName The client name to use 94 | * @return void 95 | * @throws \Flowpack\ElasticSearch\Exception 96 | * @throws \Neos\Flow\Cli\Exception\StopCommandException 97 | */ 98 | public function updateSettingsCommand($indexName, $clientName = null) 99 | { 100 | if (!in_array($indexName, $this->indexInformer->getAllIndexNames())) { 101 | $this->outputFormatted("The index %s is not configured in the current application", [$indexName]); 102 | $this->quit(1); 103 | } 104 | 105 | $client = $this->clientFactory->create($clientName); 106 | try { 107 | $index = new Index($indexName, $client); 108 | if (!$index->exists()) { 109 | $this->outputFormatted("The index %s does not exists", [$indexName]); 110 | $this->quit(1); 111 | } 112 | $index->updateSettings(); 113 | $this->outputFormatted("Index settings %s updated with success", [$indexName]); 114 | } catch (Exception $exception) { 115 | $this->outputFormatted("Unable to update settings for %s index", [$indexName]); 116 | $this->quit(1); 117 | } 118 | } 119 | 120 | /** 121 | * Delete an index in ElasticSearch 122 | * 123 | * @param string $indexName The name of the index to be removed 124 | * @param string $clientName The client name to use 125 | * @return void 126 | */ 127 | public function deleteCommand($indexName, $clientName = null) 128 | { 129 | if (!in_array($indexName, $this->indexInformer->getAllIndexNames())) { 130 | $this->outputFormatted("The index %s is not configured in the current application", [$indexName]); 131 | $this->quit(1); 132 | } 133 | 134 | $client = $this->clientFactory->create($clientName); 135 | try { 136 | $index = new Index($indexName, $client); 137 | if (!$index->exists()) { 138 | $this->outputFormatted("The index %s does not exists", [$indexName]); 139 | $this->quit(1); 140 | } 141 | $index->delete(); 142 | $this->outputFormatted("Index %s deleted with success", [$indexName]); 143 | } catch (Exception $exception) { 144 | $this->outputFormatted("Unable to delete an index named: %s", [$indexName]); 145 | $this->quit(1); 146 | } 147 | } 148 | 149 | /** 150 | * Refresh an index in ElasticSearch 151 | * 152 | * @param string $indexName The name of the index to be removed 153 | * @param string $clientName The client name to use 154 | * @return void 155 | */ 156 | public function refreshCommand($indexName, $clientName = null) 157 | { 158 | if (!in_array($indexName, $this->indexInformer->getAllIndexNames())) { 159 | $this->outputFormatted("The index %s is not configured in the current application", [$indexName]); 160 | } 161 | 162 | $client = $this->clientFactory->create($clientName); 163 | try { 164 | $index = new Index($indexName, $client); 165 | if (!$index->exists()) { 166 | $this->outputFormatted("The index %s does not exists", [$indexName]); 167 | $this->quit(1); 168 | } 169 | $index->refresh(); 170 | $this->outputFormatted("Index %s refreshed with success", [$indexName]); 171 | } catch (Exception $exception) { 172 | $this->outputFormatted("Unable to refresh an index named: %s", [$indexName]); 173 | $this->quit(1); 174 | } 175 | } 176 | 177 | /** 178 | * List available document type 179 | * 180 | * @return void 181 | */ 182 | public function showConfiguredTypesCommand() 183 | { 184 | $classesAndAnnotations = $this->indexInformer->getClassesAndAnnotations(); 185 | $this->outputFormatted("Available document type"); 186 | /** @var $annotation Indexable */ 187 | foreach ($classesAndAnnotations as $className => $annotation) { 188 | $this->outputFormatted("%s", [$className], 4); 189 | } 190 | } 191 | 192 | /** 193 | * Shows the status of the current mapping 194 | * 195 | * @param string $object Class name of a domain object. If given, will only work on this single object 196 | * @param boolean $conductUpdate Set to TRUE to conduct the required corrections 197 | * @param string $clientName The client name to use 198 | * @return void 199 | */ 200 | public function statusCommand($object = null, $conductUpdate = false, $clientName = null) 201 | { 202 | $result = new ErrorResult(); 203 | 204 | $client = $this->clientFactory->create($clientName); 205 | 206 | $classesAndAnnotations = $this->indexInformer->getClassesAndAnnotations(); 207 | if ($object !== null) { 208 | if (!isset($classesAndAnnotations[$object])) { 209 | $this->outputFormatted("Error: Object '%s' is not configured correctly, check the Indexable annotation.", [$object]); 210 | $this->quit(1); 211 | } 212 | $classesAndAnnotations = [$object => $classesAndAnnotations[$object]]; 213 | } 214 | array_walk($classesAndAnnotations, function (Indexable $annotation, $className) use ($result, $client, $conductUpdate) { 215 | $this->outputFormatted("Object \x1b[33m%s\x1b[0m", [$className], 4); 216 | $this->outputFormatted("Index %s Type %s", [ 217 | $annotation->indexName, 218 | $annotation->typeName, 219 | ], 8); 220 | $count = $client->findIndex($annotation->indexName)->findType($annotation->typeName)->count(); 221 | if ($count === null) { 222 | $result->forProperty($className)->addError(new Error('ElasticSearch was unable to retrieve a count for the type "%s" at index "%s". Probably these don\' exist.', 1340289921, [ 223 | $annotation->typeName, 224 | $annotation->indexName, 225 | ])); 226 | } 227 | $this->outputFormatted("Documents in Search: %s", [$count !== null ? $count : "\x1b[41mError\x1b[0m"], 8); 228 | 229 | try { 230 | $count = $this->persistenceManager->createQueryForType($className)->count(); 231 | } catch (\Exception $exception) { 232 | $count = null; 233 | $result->forProperty($className)->addError(new Error('The persistence backend was unable to retrieve a count for the type "%s". The exception message was "%s".', 1340290088, [ 234 | $className, 235 | $exception->getMessage(), 236 | ])); 237 | } 238 | $this->outputFormatted("Documents in Persistence: %s", [$count !== null ? $count : "\x1b[41mError\x1b[0m"], 8); 239 | if (!$result->forProperty($className)->hasErrors()) { 240 | $states = $this->getModificationsNeededStatesAndIdentifiers($client, $className); 241 | if ($conductUpdate) { 242 | $inserted = 0; 243 | $updated = 0; 244 | foreach ($states[ObjectIndexer::ACTION_TYPE_CREATE] as $identifier) { 245 | try { 246 | $this->objectIndexer->indexObject($this->persistenceManager->getObjectByIdentifier($identifier, $className)); 247 | $inserted++; 248 | } catch (\Exception $exception) { 249 | $result->forProperty($className)->addError(new Error('An error occurred while trying to add an object to the ElasticSearch backend. The exception message was "%s".', 1340356330, [$exception->getMessage()])); 250 | } 251 | } 252 | foreach ($states[ObjectIndexer::ACTION_TYPE_UPDATE] as $identifier) { 253 | try { 254 | $this->objectIndexer->indexObject($this->persistenceManager->getObjectByIdentifier($identifier, $className)); 255 | $updated++; 256 | } catch (\Exception $exception) { 257 | $result->forProperty($className)->addError(new Error('An error occurred while trying to update an object to the ElasticSearch backend. The exception message was "%s".', 1340358590, [$exception->getMessage()])); 258 | } 259 | } 260 | $this->outputFormatted("Objects inserted: %s", [$inserted], 8); 261 | $this->outputFormatted("Objects updated: %s", [$updated], 8); 262 | } else { 263 | $this->outputFormatted("Modifications needed: create %d, update %d", [ 264 | count($states[ObjectIndexer::ACTION_TYPE_CREATE]), 265 | count($states[ObjectIndexer::ACTION_TYPE_UPDATE]), 266 | ], 8); 267 | } 268 | } 269 | }); 270 | 271 | if ($result->hasErrors()) { 272 | $this->outputLine(); 273 | $this->outputLine('The following errors occurred:'); 274 | /** @var $error Error */ 275 | foreach ($result->getFlattenedErrors() as $className => $errors) { 276 | foreach ($errors as $error) { 277 | $this->outputLine(); 278 | $this->outputFormatted("\x1b[41mError\x1b[0m for \x1b[33m%s\x1b[0m:", [$className], 8); 279 | $this->outputFormatted((string)$error, [], 4); 280 | } 281 | } 282 | } 283 | } 284 | 285 | /** 286 | * @param Client $client 287 | * @param string $className 288 | * @return array 289 | */ 290 | protected function getModificationsNeededStatesAndIdentifiers(Client $client, $className) 291 | { 292 | $query = $this->persistenceManager->createQueryForType($className); 293 | $states = [ 294 | ObjectIndexer::ACTION_TYPE_CREATE => [], 295 | ObjectIndexer::ACTION_TYPE_UPDATE => [], 296 | ObjectIndexer::ACTION_TYPE_DELETE => [], 297 | ]; 298 | foreach ($query->execute() as $object) { 299 | $state = $this->objectIndexer->objectIndexActionRequired($object, $client); 300 | $states[$state][] = $this->persistenceManager->getIdentifierByObject($object); 301 | } 302 | 303 | return $states; 304 | } 305 | } 306 | --------------------------------------------------------------------------------