├── .gitignore
├── EnqueueElasticaBundle.php
├── Queue
    ├── Commands.php
    └── PopulateProcessor.php
├── Doctrine
    ├── Queue
    │   ├── Commands.php
    │   └── SyncIndexWithObjectChangeProcessor.php
    └── SyncIndexWithObjectChangeListener.php
├── composer.json
├── LICENSE
├── README.md
├── Persister
    ├── Listener
    │   └── PurgePopulateQueueListener.php
    └── QueuePagerPersister.php
└── DependencyInjection
    ├── Configuration.php
    └── EnqueueElasticaExtension.php


/.gitignore:
--------------------------------------------------------------------------------
1 | *~
2 | /composer.lock
3 | /composer.phar
4 | /phpunit.xml
5 | /vendor/
6 | /.idea/
7 | 


--------------------------------------------------------------------------------
/EnqueueElasticaBundle.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace Enqueue\ElasticaBundle;
 4 | 
 5 | use Symfony\Component\HttpKernel\Bundle\Bundle;
 6 | 
 7 | class EnqueueElasticaBundle extends Bundle
 8 | {
 9 | }
10 | 


--------------------------------------------------------------------------------
/Queue/Commands.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace Enqueue\ElasticaBundle\Queue;
 3 | 
 4 | final class Commands
 5 | {
 6 |     const POPULATE = 'fos_elastica_populate';
 7 | 
 8 |     private function __construct()
 9 |     {
10 |     }
11 | }


--------------------------------------------------------------------------------
/Doctrine/Queue/Commands.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace Enqueue\ElasticaBundle\Doctrine\Queue;
 3 | 
 4 | final class Commands
 5 | {
 6 |     const SYNC_INDEX_WITH_OBJECT_CHANGE = 'fos_elastica_doctrine_orm_sync_index_with_object_change';
 7 | 
 8 |     private function __construct()
 9 |     {
10 |     }
11 | }


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "enqueue/elastica-bundle",
 3 |     "type": "symfony-bundle",
 4 |     "description": "The bundle adds extra features to FOSElasticaBundle bundle",
 5 |     "keywords": ["elasticsearch", "elastica", "fos", "performance"],
 6 |     "license": "MIT",
 7 |     "require": {
 8 |         "php": "^7.1|^8.0",
 9 |         "symfony/framework-bundle": "^6.0|^7.0",
10 |         "friendsofsymfony/elastica-bundle": "^6.0",
11 |         "enqueue/enqueue-bundle": "^0.10"
12 |     },
13 |     "require-dev": {
14 |         "doctrine/orm": "^2.0"
15 |     },
16 |     "autoload": {
17 |         "psr-4": { "Enqueue\\ElasticaBundle\\": "" }
18 |     },
19 |     "minimum-stability": "dev",
20 |     "extra": {
21 |         "branch-alias": {
22 |             "dev-master": "0.10.x-dev"
23 |         }
24 |     }
25 | }
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2017 Kotliar Maksym
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is furnished
 8 | to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 | THE SOFTWARE.


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Enqueue Elastica Bundle
 2 | 
 3 | Improves performance of `fos:elastica:populate` command from [FOSElasticaBundle](https://github.com/FriendsOfSymfony/FOSElasticaBundle) by distributing the work among consumers.
 4 | The performance gain depends on how much consumers you run.
 5 | For example 10 consumers may give you 5 to 7 times better performance.  
 6 | 
 7 | [Read the documentation](https://github.com/php-enqueue/enqueue-dev/blob/master/docs/elastica-bundle/overview.md)
 8 | 
 9 | ## Developed by Forma-Pro
10 | 
11 | Forma-Pro is a full stack development company which interests also spread to open source development. Being a team of strong professionals we have an aim an ability to help community by developing cutting edge solutions in the areas of e-commerce, docker & microservice oriented architecture where we have accumulated a huge many-years experience. Our main specialization is Symfony framework based solution, but we are always looking to the technologies that allow us to do our job the best way. We are committed to creating solutions that revolutionize the way how things are developed in aspects of architecture & scalability.
12 | If you have any questions and inquires about our open source development, this product particularly or any other matter feel free to contact at opensource@forma-pro.com
13 | 
14 | ## License
15 | 
16 | It is released under the [MIT License](LICENSE).
17 | 


--------------------------------------------------------------------------------
/Persister/Listener/PurgePopulateQueueListener.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace Enqueue\ElasticaBundle\Persister\Listener;
 3 | 
 4 | use FOS\ElasticaBundle\Persister\Event\PrePersistEvent;
 5 | use Interop\Queue\Context;
 6 | use Symfony\Component\EventDispatcher\EventSubscriberInterface;
 7 | 
 8 | class PurgePopulateQueueListener implements EventSubscriberInterface
 9 | {
10 |     private $context;
11 | 
12 |     public function __construct(Context $context)
13 |     {
14 |         $this->context = $context;
15 |     }
16 | 
17 |     public function purgePopulateQueue(PrePersistEvent $event)
18 |     {
19 |         $options = $event->getOptions();
20 |         if (empty($options['purge_populate_queue'])) {
21 |             return;
22 |         }
23 |         if (empty($options['populate_queue'])) {
24 |             return;
25 |         }
26 | 
27 |         if (method_exists($this->context, 'purge')) {
28 |             $queue = $this->context->createQueue($options['populate_queue']);
29 | 
30 |             $this->context->purge($queue);
31 |         }
32 | 
33 |         if (method_exists($this->context, 'purgeQueue')) {
34 |             $queue = $this->context->createQueue($options['populate_queue']);
35 | 
36 |             $this->context->purgeQueue($queue);
37 |         }
38 |     }
39 | 
40 |     /**
41 |      * {@inheritdoc}
42 |      */
43 |     public static function getSubscribedEvents()
44 |     {
45 |         return [
46 |             PrePersistEvent::class => 'purgePopulateQueue',
47 |         ];
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/DependencyInjection/Configuration.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace Enqueue\ElasticaBundle\DependencyInjection;
 4 | 
 5 | use Symfony\Component\Config\Definition\Builder\TreeBuilder;
 6 | use Symfony\Component\Config\Definition\ConfigurationInterface;
 7 | 
 8 | class Configuration implements ConfigurationInterface
 9 | {
10 |     /**
11 |      * {@inheritdoc}
12 |      */
13 |     public function getConfigTreeBuilder(): TreeBuilder
14 |     {
15 |         
16 |         $tb = new TreeBuilder('enqueue_elastica');
17 |         if (method_exists($tb, 'getRootNode')) {
18 |             $rootNode = $tb->getRootNode();
19 |         } else {
20 |             // BC layer for symfony/config 4.1 and older
21 |             $rootNode = $tb->root('enqueue_elastica');
22 |         }
23 |         $rootNode
24 |             ->children()
25 |                 ->booleanNode('enabled')->defaultValue(true)->end()
26 |                 ->scalarNode('transport')->defaultValue('%enqueue.default_transport%')->cannotBeEmpty()->isRequired()->end()
27 |                 ->arrayNode('doctrine')
28 |                     ->children()
29 |                         ->scalarNode('driver')->defaultValue('orm')->cannotBeEmpty()
30 |                             ->validate()->ifNotInArray(['orm', 'mongodb'])->thenInvalid('Invalid driver')
31 |                         ->end()->end()
32 |                         ->arrayNode('queue_listeners')
33 |                             ->prototype('array')
34 |                                 ->addDefaultsIfNotSet()
35 |                                 ->children()
36 |                                     ->booleanNode('insert')->defaultTrue()->end()
37 |                                     ->booleanNode('update')->defaultTrue()->end()
38 |                                     ->booleanNode('remove')->defaultTrue()->end()
39 |                                     ->scalarNode('connection')->defaultValue('default')->cannotBeEmpty()->end()
40 |                                     ->scalarNode('index_name')->isRequired()->cannotBeEmpty()->end()
41 |                                     ->scalarNode('model_class')->isRequired()->cannotBeEmpty()->end()
42 |                                     ->scalarNode('model_id')->defaultValue('id')->cannotBeEmpty()->end()
43 |                                     ->scalarNode('repository_method')->defaultValue('find')->cannotBeEmpty()->end()
44 |                     ->end()
45 |         ;
46 | 
47 |         return $tb;
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/Doctrine/SyncIndexWithObjectChangeListener.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace Enqueue\ElasticaBundle\Doctrine;
  3 | 
  4 | use Doctrine\Persistence\Event\LifecycleEventArgs;
  5 | use Doctrine\ORM\Event\PostFlushEventArgs;
  6 | use Enqueue\ElasticaBundle\Doctrine\Queue\Commands;
  7 | use Enqueue\ElasticaBundle\Doctrine\Queue\SyncIndexWithObjectChangeProcessor as SyncProcessor;
  8 | use Enqueue\Util\JSON;
  9 | use Interop\Queue\Context;
 10 | use Doctrine\Common\EventSubscriber;
 11 | 
 12 | final class SyncIndexWithObjectChangeListener implements EventSubscriber
 13 | {
 14 |     private $context;
 15 | 
 16 |     /**
 17 |      * @var string
 18 |      */
 19 |     private $modelClass;
 20 | 
 21 |     /**
 22 |      * @var array
 23 |      */
 24 |     private $scheduledForUpdateIndex = [];
 25 | 
 26 |     /**
 27 |      * @var array
 28 |      */
 29 |     private $config;
 30 | 
 31 |     public function __construct(Context $context, $modelClass, array $config)
 32 |     {
 33 |         $this->context = $context;
 34 |         $this->modelClass = $modelClass;
 35 |         $this->config = $config;
 36 |     }
 37 | 
 38 |     public function postUpdate(LifecycleEventArgs $args)
 39 |     {
 40 |         if ($args->getObject() instanceof $this->modelClass) {
 41 |             $this->scheduledForUpdateIndex[] = [
 42 |                 'action' => SyncProcessor::UPDATE_ACTION,
 43 |                 'id'     => $this->extractId($args->getObject())
 44 |             ];
 45 |         }
 46 |     }
 47 | 
 48 |     public function postPersist(LifecycleEventArgs $args)
 49 |     {
 50 |         if ($args->getObject() instanceof $this->modelClass) {
 51 |             $this->scheduledForUpdateIndex[] = [
 52 |                 'action' => SyncProcessor::INSERT_ACTION,
 53 |                 'id'     => $this->extractId($args->getObject())
 54 |             ];
 55 |         }
 56 |     }
 57 | 
 58 |     public function preRemove(LifecycleEventArgs $args)
 59 |     {
 60 |         if ($args->getObject() instanceof $this->modelClass) {
 61 |             $this->scheduledForUpdateIndex[] = [
 62 |                 'action' => SyncProcessor::REMOVE_ACTION,
 63 |                 'id'     => $this->extractId($args->getObject())
 64 |             ];
 65 |         }
 66 |     }
 67 | 
 68 |     public function postFlush(PostFlushEventArgs $event)
 69 |     {
 70 |         if (count($this->scheduledForUpdateIndex)) {
 71 |             foreach ($this->scheduledForUpdateIndex as $updateIndex) {
 72 |                 $this->sendUpdateIndexMessage($updateIndex['action'], $updateIndex['id']);
 73 |             }
 74 | 
 75 |             $this->scheduledForUpdateIndex = [];
 76 |         }
 77 |     }
 78 | 
 79 |     public function getSubscribedEvents()
 80 |     {
 81 |         return [
 82 |             'postPersist',
 83 |             'postUpdate',
 84 |             'preRemove',
 85 |             'postFlush'
 86 |         ];
 87 |     }
 88 | 
 89 |     /**
 90 |      * @param string $action
 91 |      * @param $id
 92 |      */
 93 |     private function sendUpdateIndexMessage($action, $id)
 94 |     {
 95 |         $queue = $this->context->createQueue(Commands::SYNC_INDEX_WITH_OBJECT_CHANGE);
 96 | 
 97 |         $message = $this->context->createMessage(JSON::encode([
 98 |             'action' => $action,
 99 |             'model_class' => $this->modelClass,
100 |             'model_id' => $this->config['model_id'],
101 |             'id' => $id,
102 |             'index_name' => $this->config['index_name'],
103 |             'repository_method' => $this->config['repository_method'],
104 |         ]));
105 | 
106 |         $this->context->createProducer()->send($queue, $message);
107 |     }
108 | 
109 |     /**
110 |      * @param $object
111 |      * @return mixed
112 |      * @throws \ReflectionException
113 |      */
114 |     private function extractId($object)
115 |     {
116 |         $rp = (new \ReflectionClass($this->modelClass))->getProperty($this->config['model_id']);
117 |         $rp->setAccessible(true);
118 |         $id = $rp->getValue($object);
119 |         $rp->setAccessible(false);
120 | 
121 |         return $id;
122 |     }
123 | }
124 | 


--------------------------------------------------------------------------------
/Doctrine/Queue/SyncIndexWithObjectChangeProcessor.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace Enqueue\ElasticaBundle\Doctrine\Queue;
  3 | 
  4 | use Enqueue\Client\CommandSubscriberInterface;
  5 | use Enqueue\Consumption\QueueSubscriberInterface;
  6 | use Enqueue\Consumption\Result;
  7 | use Enqueue\Util\JSON;
  8 | use FOS\ElasticaBundle\Persister\PersisterRegistry;
  9 | use FOS\ElasticaBundle\Provider\IndexableInterface;
 10 | use Interop\Queue\Context;
 11 | use Interop\Queue\Message;
 12 | use Interop\Queue\Processor;
 13 | use Doctrine\Persistence\ManagerRegistry;
 14 | 
 15 | final class SyncIndexWithObjectChangeProcessor implements Processor, CommandSubscriberInterface, QueueSubscriberInterface
 16 | {
 17 |     const INSERT_ACTION = 'insert';
 18 | 
 19 |     const UPDATE_ACTION = 'update';
 20 | 
 21 |     const REMOVE_ACTION = 'remove';
 22 | 
 23 |     private $persisterRegistry;
 24 | 
 25 |     private $indexable;
 26 | 
 27 |     private $doctrine;
 28 | 
 29 |     public function __construct(ManagerRegistry $doctrine, PersisterRegistry $persisterRegistry, IndexableInterface $indexable)
 30 |     {
 31 |         $this->persisterRegistry = $persisterRegistry;
 32 |         $this->indexable = $indexable;
 33 |         $this->doctrine = $doctrine;
 34 |     }
 35 | 
 36 |     public function process(Message $message, Context $context): Result
 37 |     {
 38 |         $data = JSON::decode($message->getBody());
 39 | 
 40 |         if (false == isset($data['action'])) {
 41 |             return Result::reject('The message data misses action');
 42 |         }
 43 |         if (false == isset($data['model_class'])) {
 44 |             return Result::reject('The message data misses model_class');
 45 |         }
 46 |         if (false == isset($data['id'])) {
 47 |             return Result::reject('The message data misses id');
 48 |         }
 49 |         if (false == isset($data['index_name'])) {
 50 |             return Result::reject('The message data misses index_name');
 51 |         }
 52 |         if (false == isset($data['repository_method'])) {
 53 |             return Result::reject('The message data misses repository_method');
 54 |         }
 55 | 
 56 |         $action = $data['action'];
 57 |         $modelClass = $data['model_class'];
 58 |         $id = $data['id'];
 59 |         $index = $data['index_name'];
 60 |         $repositoryMethod = $data['repository_method'];
 61 | 
 62 |         $repository = $this->doctrine->getManagerForClass($modelClass)->getRepository($modelClass);
 63 |         $persister = $this->persisterRegistry->getPersister($index);
 64 | 
 65 |         switch ($action) {
 66 |             case self::UPDATE_ACTION:
 67 |                 if (false == $object = $repository->{$repositoryMethod}($id)) {
 68 |                     $persister->deleteById($id);
 69 | 
 70 |                     return Result::ack(sprintf('The object "%s" with id "%s" could not be found.', $modelClass, $id));
 71 |                 }
 72 | 
 73 |                 if ($persister->handlesObject($object)) {
 74 |                     if ($this->indexable->isObjectIndexable($index, $object)) {
 75 |                         $persister->replaceOne($object);
 76 |                     } else {
 77 |                         $persister->deleteOne($object);
 78 |                     }
 79 |                 }
 80 | 
 81 |                 return Result::ack();
 82 |             case self::INSERT_ACTION:
 83 |                 if (false == $object = $repository->{$repositoryMethod}($id)) {
 84 |                     $persister->deleteById($id);
 85 | 
 86 |                     return Result::ack(sprintf('The object "%s" with id "%s" could not be found.', $modelClass, $id));
 87 |                 }
 88 | 
 89 |                 if ($persister->handlesObject($object) && $this->indexable->isObjectIndexable($index, $object)) {
 90 |                     $persister->insertOne($object);
 91 |                 }
 92 | 
 93 |                 return Result::ack();
 94 |             case self::REMOVE_ACTION:
 95 |                 $persister->deleteById($id);
 96 | 
 97 |                 return Result::ack();
 98 |             default:
 99 |                 return Result::reject(sprintf('The action "%s" is not supported', $action));
100 |         }
101 |     }
102 | 
103 |     public static function getSubscribedCommand(): array
104 |     {
105 |         return [
106 |             'command' => Commands::SYNC_INDEX_WITH_OBJECT_CHANGE,
107 |             'queue' => Commands::SYNC_INDEX_WITH_OBJECT_CHANGE,
108 |             'prefix_queue' => false,
109 |             'exclusive' => true,
110 |         ];
111 |     }
112 | 
113 |     public static function getSubscribedQueues(): array
114 |     {
115 |         return [Commands::SYNC_INDEX_WITH_OBJECT_CHANGE];
116 |     }
117 | }
118 | 


--------------------------------------------------------------------------------
/DependencyInjection/EnqueueElasticaExtension.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace Enqueue\ElasticaBundle\DependencyInjection;
  4 | 
  5 | use Enqueue\ElasticaBundle\Doctrine\Queue\SyncIndexWithObjectChangeProcessor;
  6 | use Enqueue\ElasticaBundle\Doctrine\SyncIndexWithObjectChangeListener;
  7 | use Enqueue\ElasticaBundle\Persister\Listener\PurgePopulateQueueListener;
  8 | use Enqueue\ElasticaBundle\Persister\QueuePagerPersister;
  9 | use Enqueue\ElasticaBundle\Queue\PopulateProcessor;
 10 | use Enqueue\Symfony\DependencyInjection\TransportFactory;
 11 | use Enqueue\Symfony\DiUtils;
 12 | use Symfony\Component\DependencyInjection\ContainerBuilder;
 13 | use Symfony\Component\DependencyInjection\Reference;
 14 | use Symfony\Component\HttpKernel\DependencyInjection\Extension;
 15 | 
 16 | class EnqueueElasticaExtension extends Extension
 17 | {
 18 |     /**
 19 |      * {@inheritdoc}
 20 |      */
 21 |     public function load(array $configs, ContainerBuilder $container)
 22 |     {
 23 |         $config = $this->processConfiguration(new Configuration(), $configs);
 24 | 
 25 |         if (!$config['enabled']) {
 26 |             return;
 27 |         }
 28 | 
 29 |         $transport = $container->getParameterBag()->resolveValue($config['transport']);
 30 | 
 31 |         $diUtils = new DiUtils(TransportFactory::MODULE, $transport);
 32 |         $container->setAlias('enqueue_elastica.context', $diUtils->format('context'));
 33 | 
 34 |         $container->register('enqueue_elastica.populate_processor', PopulateProcessor::class)
 35 |             ->addArgument(new Reference('fos_elastica.pager_provider_registry'))
 36 |             ->addArgument(new Reference('fos_elastica.pager_persister_registry'))
 37 |             ->addArgument(new Reference('fos_elastica.index_manager'))
 38 | 
 39 |             ->addTag('enqueue.command_subscriber', ['client' => $transport])
 40 |             ->addTag('enqueue.transport.processor', ['transport' => $transport])
 41 |         ;
 42 | 
 43 |         $container->register('enqueue_elastica.purge_populate_queue_listener', PurgePopulateQueueListener::class)
 44 |             ->addArgument(new Reference('enqueue_elastica.context'))
 45 | 
 46 |             ->addTag('kernel.event_subscriber')
 47 |         ;
 48 | 
 49 |         $container->register('enqueue_elastica.queue_pager_persister', QueuePagerPersister::class)
 50 |             ->addArgument(new Reference('enqueue_elastica.context'))
 51 |             ->addArgument(new Reference('fos_elastica.persister_registry'))
 52 |             ->addArgument(new Reference('event_dispatcher'))
 53 |             ->addArgument(new Reference('fos_elastica.index_manager'))
 54 | 
 55 |             ->addTag('fos_elastica.pager_persister', ['persisterName' => 'queue'])
 56 |             ->setPublic(true)
 57 |         ;
 58 | 
 59 |         if (false == empty($config['doctrine']['queue_listeners'])) {
 60 |             $doctrineDriver = $config['doctrine']['driver'];
 61 | 
 62 |             $container->register('enqueue_elastica.doctrine.sync_index_with_object_change_processor', SyncIndexWithObjectChangeProcessor::class)
 63 |                 ->addArgument(new Reference($this->getManagerRegistry($doctrineDriver)))
 64 |                 ->addArgument(new Reference('fos_elastica.persister_registry'))
 65 |                 ->addArgument(new Reference('fos_elastica.indexable'))
 66 |                 ->addTag('enqueue.command_subscriber', ['client' => $transport])
 67 |                 ->addTag('enqueue.transport.processor', ['transport' => $transport])
 68 |             ;
 69 | 
 70 |             foreach ($config['doctrine']['queue_listeners'] as $listenerConfig) {
 71 |                 $listenerId = sprintf(
 72 |                     'enqueue_elastica.doctrine_queue_listener.%s',
 73 |                     $listenerConfig['index_name']
 74 |                 );
 75 | 
 76 |                 $container->register($listenerId, SyncIndexWithObjectChangeListener::class)
 77 |                     ->setPublic(true)
 78 |                     ->addArgument(new Reference('enqueue_elastica.context'))
 79 |                     ->addArgument($listenerConfig['model_class'])
 80 |                     ->addArgument($listenerConfig)
 81 |                     ->addTag($this->getEventSubscriber($doctrineDriver), ['connection' => $listenerConfig['connection']])
 82 |                 ;
 83 |             }
 84 |         }
 85 |     }
 86 | 
 87 |     private function getManagerRegistry(string $driver): string
 88 |     {
 89 |         switch ($driver) {
 90 |             case 'mongodb':
 91 |                 return 'doctrine_mongodb';
 92 |                 break;
 93 |             case 'orm':
 94 |             default:
 95 |                 return 'doctrine';
 96 |         }
 97 |     }
 98 | 
 99 |     private function getEventSubscriber(string $driver): string
100 |     {
101 |         switch ($driver) {
102 |             case 'mongodb':
103 |                 return 'doctrine_mongodb.odm.event_subscriber';
104 |                 break;
105 |             case 'orm':
106 |             default:
107 |                 return 'doctrine.event_subscriber';
108 |         }
109 |     }
110 | }
111 | 


--------------------------------------------------------------------------------
/Queue/PopulateProcessor.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace Enqueue\ElasticaBundle\Queue;
  3 | 
  4 | use Enqueue\Client\CommandSubscriberInterface;
  5 | use Enqueue\Consumption\QueueSubscriberInterface;
  6 | use Enqueue\Consumption\Result;
  7 | use FOS\ElasticaBundle\Index\IndexManager;
  8 | use FOS\ElasticaBundle\Persister\InPlacePagerPersister;
  9 | use FOS\ElasticaBundle\Persister\PagerPersisterRegistry;
 10 | use FOS\ElasticaBundle\Provider\PagerProviderRegistry;
 11 | use Interop\Queue\Context;
 12 | use Interop\Queue\Message;
 13 | use Interop\Queue\Processor;
 14 | use Enqueue\Util\JSON;
 15 | 
 16 | final class PopulateProcessor implements Processor, CommandSubscriberInterface, QueueSubscriberInterface
 17 | {
 18 |     private $pagerProviderRegistry;
 19 | 
 20 |     private $pagerPersisterRegistry;
 21 | 
 22 |     private $indexManager;
 23 | 
 24 |     public function __construct(
 25 |         PagerProviderRegistry $pagerProviderRegistry,
 26 |         PagerPersisterRegistry $pagerPersisterRegistry,
 27 |         IndexManager $indexManager
 28 |     ) {
 29 |         $this->pagerPersisterRegistry = $pagerPersisterRegistry;
 30 |         $this->pagerProviderRegistry = $pagerProviderRegistry;
 31 |         $this->indexManager = $indexManager;
 32 |     }
 33 | 
 34 |     public function process(Message $message, Context $context): Result
 35 |     {
 36 |         if ($message->isRedelivered()) {
 37 |             $replyMessage = $this->createReplyMessage($context, $message, 0,'The message was redelivered. Chances are that something has gone wrong.');
 38 | 
 39 |             return Result::reply($replyMessage, Result::REJECT);
 40 |         }
 41 | 
 42 |         $objectsCount = 0;
 43 | 
 44 |         try {
 45 |             $data = JSON::decode($message->getBody());
 46 | 
 47 |             if (!isset($data['options'])) {
 48 |                 return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing options.'));
 49 |             }
 50 |             if (!isset($data['page'])) {
 51 |                 return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing page.'));
 52 |             }
 53 |             if (!isset($data['options']['indexName'])) {
 54 |                 return Result::reply($this->createReplyMessage($context, $message, 0,'The message is invalid. Missing indexName option.'));
 55 |             }
 56 | 
 57 |             $options = $data['options'];
 58 |             $options['first_page'] = $data['page'];
 59 |             $options['last_page'] = $data['page'];
 60 | 
 61 |             if (isset($options['realIndexName'])) {
 62 |                 $this->indexManager->getIndex($options['indexName'])->overrideName($options['realIndexName']);
 63 |             }
 64 | 
 65 |             $provider = $this->pagerProviderRegistry->getProvider($options['indexName']);
 66 |             $pager = $provider->provide($options);
 67 |             $pager->setMaxPerPage($options['max_per_page']);
 68 |             $pager->setCurrentPage($options['first_page']);
 69 | 
 70 |             $objectsCount = count($pager->getCurrentPageResults());
 71 | 
 72 |             $pagerPersister = $this->pagerPersisterRegistry->getPagerPersister(InPlacePagerPersister::NAME);
 73 |             $pagerPersister->insert($pager, $options);
 74 | 
 75 |             return Result::reply($this->createReplyMessage($context, $message, $objectsCount));
 76 |         } catch (\Throwable $e) {
 77 |             return Result::reply($this->createExceptionReplyMessage($context, $message, $objectsCount, $e), Result::REJECT);
 78 |         }
 79 |     }
 80 | 
 81 |     private function createExceptionReplyMessage(Context $context, Message $message, int $objectsCount, \Throwable $e): Message
 82 |     {
 83 |         $errorMessage = sprintf(
 84 |             '<error>The queue processor has failed to process the message with exception: </error><comment>%s: %s in file %s at line %s.</comment>',
 85 |             get_class($e),
 86 |             $e->getMessage(),
 87 |             $e->getFile(),
 88 |             $e->getLine()
 89 |         );
 90 | 
 91 |         return $this->createReplyMessage($context, $message, $objectsCount, $errorMessage);
 92 |     }
 93 | 
 94 |     private function createReplyMessage(Context $context, Message $message, int $objectsCount, string $error = null): Message
 95 |     {
 96 |         $replyMessage = $context->createMessage($message->getBody(), $message->getProperties(), $message->getHeaders());
 97 |         $replyMessage->setProperty('fos-populate-objects-count', $objectsCount);
 98 | 
 99 |         if ($error) {
100 |             $replyMessage->setProperty('fos-populate-error', $error);
101 |         }
102 | 
103 |         return $replyMessage;
104 |     }
105 | 
106 |     public static function getSubscribedCommand(): array
107 |     {
108 |         return [
109 |             'command' => Commands::POPULATE,
110 |             'queue' => Commands::POPULATE,
111 |             'prefix_queue' => false,
112 |             'exclusive' => true,
113 |         ];
114 |     }
115 | 
116 |     public static function getSubscribedQueues(): array
117 |     {
118 |         return [Commands::POPULATE];
119 |     }
120 | }
121 | 


--------------------------------------------------------------------------------
/Persister/QueuePagerPersister.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace Enqueue\ElasticaBundle\Persister;
  3 | 
  4 | use Enqueue\ElasticaBundle\Queue\Commands;
  5 | use Enqueue\Util\JSON;
  6 | use FOS\ElasticaBundle\Index\IndexManager;
  7 | use FOS\ElasticaBundle\Persister\Event\PostAsyncInsertObjectsEvent;
  8 | use FOS\ElasticaBundle\Persister\Event\PostPersistEvent;
  9 | use FOS\ElasticaBundle\Persister\Event\PrePersistEvent;
 10 | use FOS\ElasticaBundle\Persister\PagerPersisterInterface;
 11 | use FOS\ElasticaBundle\Persister\PersisterRegistry;
 12 | use FOS\ElasticaBundle\Provider\PagerInterface;
 13 | use Interop\Queue\Context;
 14 | use Symfony\Component\EventDispatcher\EventDispatcherInterface;
 15 | 
 16 | final class QueuePagerPersister implements PagerPersisterInterface
 17 | {
 18 |     const NAME = 'queue';
 19 | 
 20 |     private $context;
 21 | 
 22 |     /**
 23 |      * @var PersisterRegistry
 24 |      */
 25 |     private $registry;
 26 | 
 27 |     /**
 28 |      * @var EventDispatcherInterface
 29 |      */
 30 |     private $dispatcher;
 31 | 
 32 |     /**
 33 |      * @var IndexManager
 34 |      */
 35 |     private $indexManager;
 36 | 
 37 |     public function __construct(
 38 |         Context $context,
 39 |         PersisterRegistry $registry,
 40 |         EventDispatcherInterface $dispatcher,
 41 |         IndexManager $indexManager
 42 |     ) {
 43 |         $this->context = $context;
 44 |         $this->dispatcher = $dispatcher;
 45 |         $this->registry = $registry;
 46 |         $this->indexManager = $indexManager;
 47 |     }
 48 | 
 49 |     /**
 50 |      * {@inheritdoc}
 51 |      */
 52 |     public function insert(PagerInterface $pager, array $options = array())
 53 |     {
 54 |         $pager->setMaxPerPage(empty($options['max_per_page']) ? 100 : $options['max_per_page']);
 55 | 
 56 |         $defaultOptions = [
 57 |             'max_per_page' => $pager->getMaxPerPage(),
 58 |             'first_page' => $pager->getCurrentPage(),
 59 |             'last_page' => $pager->getNbPages(),
 60 |             'populate_queue' => Commands::POPULATE,
 61 |             'populate_reply_queue' => null,
 62 |             'reply_receive_timeout' => 5000, // ms
 63 |             'limit_overall_reply_time' => 180 // sec
 64 |         ];
 65 |         $index = $this->indexManager->getIndex($options['indexName']);
 66 |         if ($index->getName() !== $index->getOriginalName()) {
 67 |             $defaultOptions['realIndexName'] = $index->getName();
 68 |         }
 69 | 
 70 |         $options = array_replace($defaultOptions, $options);
 71 | 
 72 |         $pager->setCurrentPage($options['first_page']);
 73 | 
 74 |         $objectPersister = $this->registry->getPersister($options['indexName']);
 75 | 
 76 |         $event = new PrePersistEvent($pager, $objectPersister, $options);
 77 |         $this->dispatcher->dispatch($event);
 78 |         $pager = $event->getPager();
 79 |         $options = $event->getOptions();
 80 | 
 81 |         $queue = $this->context->createQueue($options['populate_queue']);
 82 |         $replyQueue = $options['populate_reply_queue'] ?
 83 |             $this->context->createQueue($options['populate_reply_queue']) :
 84 |             $this->context->createTemporaryQueue()
 85 |         ;
 86 |         $options['populate_reply_queue'] = $replyQueue->getQueueName();
 87 | 
 88 |         $producer = $this->context->createProducer();
 89 | 
 90 |         $lastPage = min($options['last_page'], $pager->getNbPages());
 91 |         $page = $pager->getCurrentPage();
 92 |         $sentCount = 0;
 93 |         do {
 94 |             $pager->setCurrentPage($page);
 95 | 
 96 |             $filteredOptions = $options;
 97 |             unset(
 98 |                 $filteredOptions['first_page'],
 99 |                 $filteredOptions['last_page'],
100 |                 $filteredOptions['populate_queue'],
101 |                 $filteredOptions['populate_reply_queue'],
102 |                 $filteredOptions['reply_receive_timeout'],
103 |                 $filteredOptions['limit_overall_reply_time']
104 |             );
105 | 
106 |             $message = $this->context->createMessage(JSON::encode([
107 |                 'options' => $filteredOptions,
108 |                 'page' => $page,
109 |             ]));
110 |             $message->setReplyTo($replyQueue->getQueueName());
111 | 
112 |             // Because of https://github.com/php-enqueue/enqueue-dev/issues/907
113 |             \usleep(10);
114 | 
115 |             $producer->send($queue, $message);
116 | 
117 |             $page++;
118 |             $sentCount++;
119 |         } while ($page <= $lastPage);
120 | 
121 |         $consumer = $this->context->createConsumer($replyQueue);
122 |         $limitTime = microtime(true) + $options['limit_overall_reply_time'];
123 |         while ($sentCount) {
124 |             if ($message = $consumer->receive($options['reply_receive_timeout'])) {
125 |                 $sentCount--;
126 | 
127 |                 $data = JSON::decode($message->getBody());
128 | 
129 |                 $errorMessage = $message->getProperty('fos-populate-error', false);
130 |                 $objectsCount = (int) $message->getProperty('fos-populate-objects-count', false);
131 | 
132 |                 $pager->setCurrentPage($data['page']);
133 | 
134 |                 $event = new PostAsyncInsertObjectsEvent(
135 |                     $pager,
136 |                     $objectPersister,
137 |                     $objectsCount,
138 |                     $errorMessage,
139 |                     $data['options']
140 |                 );
141 |                 $this->dispatcher->dispatch($event);
142 |             }
143 | 
144 |             if (microtime(true) > $limitTime) {
145 |                 throw new \LogicException(sprintf('Overall reply time (%s seconds) has been exceeded.', $options['limit_overall_reply_time']));
146 |             }
147 |         }
148 | 
149 |         $event = new PostPersistEvent($pager, $objectPersister, $options);
150 |         $this->dispatcher->dispatch($event);
151 |     }
152 | }
153 | 


--------------------------------------------------------------------------------