├── .gitignore ├── DependencyInjection ├── Configuration.php └── EnqueueElasticaExtension.php ├── Doctrine ├── Queue │ ├── Commands.php │ └── SyncIndexWithObjectChangeProcessor.php └── SyncIndexWithObjectChangeListener.php ├── EnqueueElasticaBundle.php ├── LICENSE ├── Persister ├── Listener │ └── PurgePopulateQueueListener.php └── QueuePagerPersister.php ├── Queue ├── Commands.php └── PopulateProcessor.php ├── README.md └── composer.json /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /composer.lock 3 | /composer.phar 4 | /phpunit.xml 5 | /vendor/ 6 | /.idea/ 7 | -------------------------------------------------------------------------------- /DependencyInjection/Configuration.php: -------------------------------------------------------------------------------- 1 | getRootNode(); 19 | } else { 20 | // BC layer for symfony/config 4.1 and older 21 | $rootNode = $tb->root('enqueue_elastica'); 22 | } 23 | $rootNode 24 | ->children() 25 | ->booleanNode('enabled')->defaultValue(true)->end() 26 | ->scalarNode('transport')->defaultValue('%enqueue.default_transport%')->cannotBeEmpty()->isRequired()->end() 27 | ->arrayNode('doctrine') 28 | ->children() 29 | ->scalarNode('driver')->defaultValue('orm')->cannotBeEmpty() 30 | ->validate()->ifNotInArray(['orm', 'mongodb'])->thenInvalid('Invalid driver') 31 | ->end()->end() 32 | ->arrayNode('queue_listeners') 33 | ->prototype('array') 34 | ->addDefaultsIfNotSet() 35 | ->children() 36 | ->booleanNode('insert')->defaultTrue()->end() 37 | ->booleanNode('update')->defaultTrue()->end() 38 | ->booleanNode('remove')->defaultTrue()->end() 39 | ->scalarNode('connection')->defaultValue('default')->cannotBeEmpty()->end() 40 | ->scalarNode('index_name')->isRequired()->cannotBeEmpty()->end() 41 | ->scalarNode('model_class')->isRequired()->cannotBeEmpty()->end() 42 | ->scalarNode('model_id')->defaultValue('id')->cannotBeEmpty()->end() 43 | ->scalarNode('repository_method')->defaultValue('find')->cannotBeEmpty()->end() 44 | ->end() 45 | ; 46 | 47 | return $tb; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /DependencyInjection/EnqueueElasticaExtension.php: -------------------------------------------------------------------------------- 1 | processConfiguration(new Configuration(), $configs); 24 | 25 | if (!$config['enabled']) { 26 | return; 27 | } 28 | 29 | $transport = $container->getParameterBag()->resolveValue($config['transport']); 30 | 31 | $diUtils = new DiUtils(TransportFactory::MODULE, $transport); 32 | $container->setAlias('enqueue_elastica.context', $diUtils->format('context')); 33 | 34 | $container->register('enqueue_elastica.populate_processor', PopulateProcessor::class) 35 | ->addArgument(new Reference('fos_elastica.pager_provider_registry')) 36 | ->addArgument(new Reference('fos_elastica.pager_persister_registry')) 37 | ->addArgument(new Reference('fos_elastica.index_manager')) 38 | 39 | ->addTag('enqueue.command_subscriber', ['client' => $transport]) 40 | ->addTag('enqueue.transport.processor', ['transport' => $transport]) 41 | ; 42 | 43 | $container->register('enqueue_elastica.purge_populate_queue_listener', PurgePopulateQueueListener::class) 44 | ->addArgument(new Reference('enqueue_elastica.context')) 45 | 46 | ->addTag('kernel.event_subscriber') 47 | ; 48 | 49 | $container->register('enqueue_elastica.queue_pager_persister', QueuePagerPersister::class) 50 | ->addArgument(new Reference('enqueue_elastica.context')) 51 | ->addArgument(new Reference('fos_elastica.persister_registry')) 52 | ->addArgument(new Reference('event_dispatcher')) 53 | ->addArgument(new Reference('fos_elastica.index_manager')) 54 | 55 | ->addTag('fos_elastica.pager_persister', ['persisterName' => 'queue']) 56 | ->setPublic(true) 57 | ; 58 | 59 | if (false == empty($config['doctrine']['queue_listeners'])) { 60 | $doctrineDriver = $config['doctrine']['driver']; 61 | 62 | $container->register('enqueue_elastica.doctrine.sync_index_with_object_change_processor', SyncIndexWithObjectChangeProcessor::class) 63 | ->addArgument(new Reference($this->getManagerRegistry($doctrineDriver))) 64 | ->addArgument(new Reference('fos_elastica.persister_registry')) 65 | ->addArgument(new Reference('fos_elastica.indexable')) 66 | ->addTag('enqueue.command_subscriber', ['client' => $transport]) 67 | ->addTag('enqueue.transport.processor', ['transport' => $transport]) 68 | ; 69 | 70 | foreach ($config['doctrine']['queue_listeners'] as $listenerConfig) { 71 | $listenerId = sprintf( 72 | 'enqueue_elastica.doctrine_queue_listener.%s', 73 | $listenerConfig['index_name'] 74 | ); 75 | 76 | $container->register($listenerId, SyncIndexWithObjectChangeListener::class) 77 | ->setPublic(true) 78 | ->addArgument(new Reference('enqueue_elastica.context')) 79 | ->addArgument($listenerConfig['model_class']) 80 | ->addArgument($listenerConfig) 81 | ->addTag($this->getEventSubscriber($doctrineDriver), ['connection' => $listenerConfig['connection']]) 82 | ; 83 | } 84 | } 85 | } 86 | 87 | private function getManagerRegistry(string $driver): string 88 | { 89 | switch ($driver) { 90 | case 'mongodb': 91 | return 'doctrine_mongodb'; 92 | break; 93 | case 'orm': 94 | default: 95 | return 'doctrine'; 96 | } 97 | } 98 | 99 | private function getEventSubscriber(string $driver): string 100 | { 101 | switch ($driver) { 102 | case 'mongodb': 103 | return 'doctrine_mongodb.odm.event_subscriber'; 104 | break; 105 | case 'orm': 106 | default: 107 | return 'doctrine.event_subscriber'; 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /Doctrine/Queue/Commands.php: -------------------------------------------------------------------------------- 1 | persisterRegistry = $persisterRegistry; 32 | $this->indexable = $indexable; 33 | $this->doctrine = $doctrine; 34 | } 35 | 36 | public function process(Message $message, Context $context): Result 37 | { 38 | $data = JSON::decode($message->getBody()); 39 | 40 | if (false == isset($data['action'])) { 41 | return Result::reject('The message data misses action'); 42 | } 43 | if (false == isset($data['model_class'])) { 44 | return Result::reject('The message data misses model_class'); 45 | } 46 | if (false == isset($data['id'])) { 47 | return Result::reject('The message data misses id'); 48 | } 49 | if (false == isset($data['index_name'])) { 50 | return Result::reject('The message data misses index_name'); 51 | } 52 | if (false == isset($data['repository_method'])) { 53 | return Result::reject('The message data misses repository_method'); 54 | } 55 | 56 | $action = $data['action']; 57 | $modelClass = $data['model_class']; 58 | $id = $data['id']; 59 | $index = $data['index_name']; 60 | $repositoryMethod = $data['repository_method']; 61 | 62 | $repository = $this->doctrine->getManagerForClass($modelClass)->getRepository($modelClass); 63 | $persister = $this->persisterRegistry->getPersister($index); 64 | 65 | switch ($action) { 66 | case self::UPDATE_ACTION: 67 | if (false == $object = $repository->{$repositoryMethod}($id)) { 68 | $persister->deleteById($id); 69 | 70 | return Result::ack(sprintf('The object "%s" with id "%s" could not be found.', $modelClass, $id)); 71 | } 72 | 73 | if ($persister->handlesObject($object)) { 74 | if ($this->indexable->isObjectIndexable($index, $object)) { 75 | $persister->replaceOne($object); 76 | } else { 77 | $persister->deleteOne($object); 78 | } 79 | } 80 | 81 | return Result::ack(); 82 | case self::INSERT_ACTION: 83 | if (false == $object = $repository->{$repositoryMethod}($id)) { 84 | $persister->deleteById($id); 85 | 86 | return Result::ack(sprintf('The object "%s" with id "%s" could not be found.', $modelClass, $id)); 87 | } 88 | 89 | if ($persister->handlesObject($object) && $this->indexable->isObjectIndexable($index, $object)) { 90 | $persister->insertOne($object); 91 | } 92 | 93 | return Result::ack(); 94 | case self::REMOVE_ACTION: 95 | $persister->deleteById($id); 96 | 97 | return Result::ack(); 98 | default: 99 | return Result::reject(sprintf('The action "%s" is not supported', $action)); 100 | } 101 | } 102 | 103 | public static function getSubscribedCommand(): array 104 | { 105 | return [ 106 | 'command' => Commands::SYNC_INDEX_WITH_OBJECT_CHANGE, 107 | 'queue' => Commands::SYNC_INDEX_WITH_OBJECT_CHANGE, 108 | 'prefix_queue' => false, 109 | 'exclusive' => true, 110 | ]; 111 | } 112 | 113 | public static function getSubscribedQueues(): array 114 | { 115 | return [Commands::SYNC_INDEX_WITH_OBJECT_CHANGE]; 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /Doctrine/SyncIndexWithObjectChangeListener.php: -------------------------------------------------------------------------------- 1 | context = $context; 34 | $this->modelClass = $modelClass; 35 | $this->config = $config; 36 | } 37 | 38 | public function postUpdate(LifecycleEventArgs $args) 39 | { 40 | if ($args->getObject() instanceof $this->modelClass) { 41 | $this->scheduledForUpdateIndex[] = [ 42 | 'action' => SyncProcessor::UPDATE_ACTION, 43 | 'id' => $this->extractId($args->getObject()) 44 | ]; 45 | } 46 | } 47 | 48 | public function postPersist(LifecycleEventArgs $args) 49 | { 50 | if ($args->getObject() instanceof $this->modelClass) { 51 | $this->scheduledForUpdateIndex[] = [ 52 | 'action' => SyncProcessor::INSERT_ACTION, 53 | 'id' => $this->extractId($args->getObject()) 54 | ]; 55 | } 56 | } 57 | 58 | public function preRemove(LifecycleEventArgs $args) 59 | { 60 | if ($args->getObject() instanceof $this->modelClass) { 61 | $this->scheduledForUpdateIndex[] = [ 62 | 'action' => SyncProcessor::REMOVE_ACTION, 63 | 'id' => $this->extractId($args->getObject()) 64 | ]; 65 | } 66 | } 67 | 68 | public function postFlush(PostFlushEventArgs $event) 69 | { 70 | if (count($this->scheduledForUpdateIndex)) { 71 | foreach ($this->scheduledForUpdateIndex as $updateIndex) { 72 | $this->sendUpdateIndexMessage($updateIndex['action'], $updateIndex['id']); 73 | } 74 | 75 | $this->scheduledForUpdateIndex = []; 76 | } 77 | } 78 | 79 | public function getSubscribedEvents() 80 | { 81 | return [ 82 | 'postPersist', 83 | 'postUpdate', 84 | 'preRemove', 85 | 'postFlush' 86 | ]; 87 | } 88 | 89 | /** 90 | * @param string $action 91 | * @param $id 92 | */ 93 | private function sendUpdateIndexMessage($action, $id) 94 | { 95 | $queue = $this->context->createQueue(Commands::SYNC_INDEX_WITH_OBJECT_CHANGE); 96 | 97 | $message = $this->context->createMessage(JSON::encode([ 98 | 'action' => $action, 99 | 'model_class' => $this->modelClass, 100 | 'model_id' => $this->config['model_id'], 101 | 'id' => $id, 102 | 'index_name' => $this->config['index_name'], 103 | 'repository_method' => $this->config['repository_method'], 104 | ])); 105 | 106 | $this->context->createProducer()->send($queue, $message); 107 | } 108 | 109 | /** 110 | * @param $object 111 | * @return mixed 112 | * @throws \ReflectionException 113 | */ 114 | private function extractId($object) 115 | { 116 | $rp = (new \ReflectionClass($this->modelClass))->getProperty($this->config['model_id']); 117 | $rp->setAccessible(true); 118 | $id = $rp->getValue($object); 119 | $rp->setAccessible(false); 120 | 121 | return $id; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /EnqueueElasticaBundle.php: -------------------------------------------------------------------------------- 1 | context = $context; 15 | } 16 | 17 | public function purgePopulateQueue(PrePersistEvent $event) 18 | { 19 | $options = $event->getOptions(); 20 | if (empty($options['purge_populate_queue'])) { 21 | return; 22 | } 23 | if (empty($options['populate_queue'])) { 24 | return; 25 | } 26 | 27 | if (method_exists($this->context, 'purge')) { 28 | $queue = $this->context->createQueue($options['populate_queue']); 29 | 30 | $this->context->purge($queue); 31 | } 32 | 33 | if (method_exists($this->context, 'purgeQueue')) { 34 | $queue = $this->context->createQueue($options['populate_queue']); 35 | 36 | $this->context->purgeQueue($queue); 37 | } 38 | } 39 | 40 | /** 41 | * {@inheritdoc} 42 | */ 43 | public static function getSubscribedEvents() 44 | { 45 | return [ 46 | PrePersistEvent::class => 'purgePopulateQueue', 47 | ]; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Persister/QueuePagerPersister.php: -------------------------------------------------------------------------------- 1 | context = $context; 44 | $this->dispatcher = $dispatcher; 45 | $this->registry = $registry; 46 | $this->indexManager = $indexManager; 47 | } 48 | 49 | /** 50 | * {@inheritdoc} 51 | */ 52 | public function insert(PagerInterface $pager, array $options = array()) 53 | { 54 | $pager->setMaxPerPage(empty($options['max_per_page']) ? 100 : $options['max_per_page']); 55 | 56 | $defaultOptions = [ 57 | 'max_per_page' => $pager->getMaxPerPage(), 58 | 'first_page' => $pager->getCurrentPage(), 59 | 'last_page' => $pager->getNbPages(), 60 | 'populate_queue' => Commands::POPULATE, 61 | 'populate_reply_queue' => null, 62 | 'reply_receive_timeout' => 5000, // ms 63 | 'limit_overall_reply_time' => 180 // sec 64 | ]; 65 | $index = $this->indexManager->getIndex($options['indexName']); 66 | if ($index->getName() !== $index->getOriginalName()) { 67 | $defaultOptions['realIndexName'] = $index->getName(); 68 | } 69 | 70 | $options = array_replace($defaultOptions, $options); 71 | 72 | $pager->setCurrentPage($options['first_page']); 73 | 74 | $objectPersister = $this->registry->getPersister($options['indexName']); 75 | 76 | $event = new PrePersistEvent($pager, $objectPersister, $options); 77 | $this->dispatcher->dispatch($event); 78 | $pager = $event->getPager(); 79 | $options = $event->getOptions(); 80 | 81 | $queue = $this->context->createQueue($options['populate_queue']); 82 | $replyQueue = $options['populate_reply_queue'] ? 83 | $this->context->createQueue($options['populate_reply_queue']) : 84 | $this->context->createTemporaryQueue() 85 | ; 86 | $options['populate_reply_queue'] = $replyQueue->getQueueName(); 87 | 88 | $producer = $this->context->createProducer(); 89 | 90 | $lastPage = min($options['last_page'], $pager->getNbPages()); 91 | $page = $pager->getCurrentPage(); 92 | $sentCount = 0; 93 | do { 94 | $pager->setCurrentPage($page); 95 | 96 | $filteredOptions = $options; 97 | unset( 98 | $filteredOptions['first_page'], 99 | $filteredOptions['last_page'], 100 | $filteredOptions['populate_queue'], 101 | $filteredOptions['populate_reply_queue'], 102 | $filteredOptions['reply_receive_timeout'], 103 | $filteredOptions['limit_overall_reply_time'] 104 | ); 105 | 106 | $message = $this->context->createMessage(JSON::encode([ 107 | 'options' => $filteredOptions, 108 | 'page' => $page, 109 | ])); 110 | $message->setReplyTo($replyQueue->getQueueName()); 111 | 112 | // Because of https://github.com/php-enqueue/enqueue-dev/issues/907 113 | \usleep(10); 114 | 115 | $producer->send($queue, $message); 116 | 117 | $page++; 118 | $sentCount++; 119 | } while ($page <= $lastPage); 120 | 121 | $consumer = $this->context->createConsumer($replyQueue); 122 | $limitTime = microtime(true) + $options['limit_overall_reply_time']; 123 | while ($sentCount) { 124 | if ($message = $consumer->receive($options['reply_receive_timeout'])) { 125 | $sentCount--; 126 | 127 | $data = JSON::decode($message->getBody()); 128 | 129 | $errorMessage = $message->getProperty('fos-populate-error', false); 130 | $objectsCount = (int) $message->getProperty('fos-populate-objects-count', false); 131 | 132 | $pager->setCurrentPage($data['page']); 133 | 134 | $event = new PostAsyncInsertObjectsEvent( 135 | $pager, 136 | $objectPersister, 137 | $objectsCount, 138 | $errorMessage, 139 | $data['options'] 140 | ); 141 | $this->dispatcher->dispatch($event); 142 | } 143 | 144 | if (microtime(true) > $limitTime) { 145 | throw new \LogicException(sprintf('Overall reply time (%s seconds) has been exceeded.', $options['limit_overall_reply_time'])); 146 | } 147 | } 148 | 149 | $event = new PostPersistEvent($pager, $objectPersister, $options); 150 | $this->dispatcher->dispatch($event); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /Queue/Commands.php: -------------------------------------------------------------------------------- 1 | pagerPersisterRegistry = $pagerPersisterRegistry; 30 | $this->pagerProviderRegistry = $pagerProviderRegistry; 31 | $this->indexManager = $indexManager; 32 | } 33 | 34 | public function process(Message $message, Context $context): Result 35 | { 36 | if ($message->isRedelivered()) { 37 | $replyMessage = $this->createReplyMessage($context, $message, 0,'The message was redelivered. Chances are that something has gone wrong.'); 38 | 39 | return Result::reply($replyMessage, Result::REJECT); 40 | } 41 | 42 | $objectsCount = 0; 43 | 44 | try { 45 | $data = JSON::decode($message->getBody()); 46 | 47 | if (!isset($data['options'])) { 48 | return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing options.')); 49 | } 50 | if (!isset($data['page'])) { 51 | return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing page.')); 52 | } 53 | if (!isset($data['options']['indexName'])) { 54 | return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing indexName option.')); 55 | } 56 | 57 | $options = $data['options']; 58 | $options['first_page'] = $data['page']; 59 | $options['last_page'] = $data['page']; 60 | 61 | if (isset($options['realIndexName'])) { 62 | $this->indexManager->getIndex($options['indexName'])->overrideName($options['realIndexName']); 63 | } 64 | 65 | $provider = $this->pagerProviderRegistry->getProvider($options['indexName']); 66 | $pager = $provider->provide($options); 67 | $pager->setMaxPerPage($options['max_per_page']); 68 | $pager->setCurrentPage($options['first_page']); 69 | 70 | $objectsCount = count($pager->getCurrentPageResults()); 71 | 72 | $pagerPersister = $this->pagerPersisterRegistry->getPagerPersister(InPlacePagerPersister::NAME); 73 | $pagerPersister->insert($pager, $options); 74 | 75 | return Result::reply($this->createReplyMessage($context, $message, $objectsCount)); 76 | } catch (\Throwable $e) { 77 | return Result::reply($this->createExceptionReplyMessage($context, $message, $objectsCount, $e), Result::REJECT); 78 | } 79 | } 80 | 81 | private function createExceptionReplyMessage(Context $context, Message $message, int $objectsCount, \Throwable $e): Message 82 | { 83 | $errorMessage = sprintf( 84 | 'The queue processor has failed to process the message with exception: %s: %s in file %s at line %s.', 85 | get_class($e), 86 | $e->getMessage(), 87 | $e->getFile(), 88 | $e->getLine() 89 | ); 90 | 91 | return $this->createReplyMessage($context, $message, $objectsCount, $errorMessage); 92 | } 93 | 94 | private function createReplyMessage(Context $context, Message $message, int $objectsCount, string $error = null): Message 95 | { 96 | $replyMessage = $context->createMessage($message->getBody(), $message->getProperties(), $message->getHeaders()); 97 | $replyMessage->setProperty('fos-populate-objects-count', $objectsCount); 98 | 99 | if ($error) { 100 | $replyMessage->setProperty('fos-populate-error', $error); 101 | } 102 | 103 | return $replyMessage; 104 | } 105 | 106 | public static function getSubscribedCommand(): array 107 | { 108 | return [ 109 | 'command' => Commands::POPULATE, 110 | 'queue' => Commands::POPULATE, 111 | 'prefix_queue' => false, 112 | 'exclusive' => true, 113 | ]; 114 | } 115 | 116 | public static function getSubscribedQueues(): array 117 | { 118 | return [Commands::POPULATE]; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Enqueue Elastica Bundle 2 | 3 | Improves performance of `fos:elastica:populate` command from [FOSElasticaBundle](https://github.com/FriendsOfSymfony/FOSElasticaBundle) by distributing the work among consumers. 4 | The performance gain depends on how much consumers you run. 5 | For example 10 consumers may give you 5 to 7 times better performance. 6 | 7 | [Read the documentation](https://github.com/php-enqueue/enqueue-dev/blob/master/docs/elastica-bundle/overview.md) 8 | 9 | ## Developed by Forma-Pro 10 | 11 | Forma-Pro is a full stack development company which interests also spread to open source development. Being a team of strong professionals we have an aim an ability to help community by developing cutting edge solutions in the areas of e-commerce, docker & microservice oriented architecture where we have accumulated a huge many-years experience. Our main specialization is Symfony framework based solution, but we are always looking to the technologies that allow us to do our job the best way. We are committed to creating solutions that revolutionize the way how things are developed in aspects of architecture & scalability. 12 | If you have any questions and inquires about our open source development, this product particularly or any other matter feel free to contact at opensource@forma-pro.com 13 | 14 | ## License 15 | 16 | It is released under the [MIT License](LICENSE). 17 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "enqueue/elastica-bundle", 3 | "type": "symfony-bundle", 4 | "description": "The bundle adds extra features to FOSElasticaBundle bundle", 5 | "keywords": ["elasticsearch", "elastica", "fos", "performance"], 6 | "license": "MIT", 7 | "require": { 8 | "php": "^7.1|^8.0", 9 | "symfony/framework-bundle": "^6.0|^7.0", 10 | "friendsofsymfony/elastica-bundle": "^6.0", 11 | "enqueue/enqueue-bundle": "^0.10" 12 | }, 13 | "require-dev": { 14 | "doctrine/orm": "^2.0" 15 | }, 16 | "autoload": { 17 | "psr-4": { "Enqueue\\ElasticaBundle\\": "" } 18 | }, 19 | "minimum-stability": "dev", 20 | "extra": { 21 | "branch-alias": { 22 | "dev-master": "0.10.x-dev" 23 | } 24 | } 25 | } 26 | --------------------------------------------------------------------------------