├── .atoum.php ├── .github ├── CODEOWNERS └── workflows │ └── ci.yml ├── .gitignore ├── .php-cs-fixer.dist.php ├── LICENSE ├── README.md ├── composer.json └── src └── M6Web └── Bundle └── ElasticsearchBundle ├── DataCollector └── ElasticsearchDataCollector.php ├── DependencyInjection ├── Configuration.php └── M6WebElasticsearchExtension.php ├── Elasticsearch └── ConnectionPool │ ├── Selector │ └── RandomStickySelector.php │ └── StaticAliveNoPingConnectionPool.php ├── EventDispatcher └── ElasticsearchEvent.php ├── Handler ├── EventHandler.php └── HeadersHandler.php ├── M6WebElasticsearchBundle.php ├── Resources ├── config │ └── services.yml └── views │ └── Collector │ └── elasticsearch.html.twig └── Tests └── Units ├── DependencyInjection └── M6WebElasticsearchExtension.php ├── Elasticsearch ├── ConnectionMocker.php └── ConnectionPool │ ├── Selector │ └── RandomStickySelector.php │ └── StaticAliveNoPingConnectionPool.php └── Handler ├── EventHandler.php └── HeadersHandler.php /.atoum.php: -------------------------------------------------------------------------------- 1 | addTestsFromDirectory(__DIR__.'/src/M6Web/Bundle/ElasticsearchBundle/Tests'); 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @BedrockStreaming/cache-db 2 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | on: [push, pull_request] 3 | 4 | jobs: 5 | tests: 6 | name: Tests 7 | runs-on: ubuntu-20.04 8 | strategy: 9 | matrix: 10 | php-version: [ '7.4', '8.0', '8.1' ] 11 | symfony-version: ['^4.4', '^5.0', '^5.3', '^6.0'] 12 | exclude: 13 | - php-version: '7.4' 14 | symfony-version: '^6.0' 15 | fail-fast: false 16 | steps: 17 | - uses: actions/checkout@master 18 | - uses: shivammathur/setup-php@v2 19 | with: 20 | php-version: ${{ matrix.php-version }} 21 | coverage: xdebug2 22 | extensions: apcu 23 | ini-values: apc.enable_cli=1 24 | - name: Install symfony version from matrix 25 | env: 26 | SYMFONY_VERSION: ${{ matrix.symfony-version }} 27 | run: composer require symfony/symfony:$SYMFONY_VERSION --no-update 28 | - name: Install dependencies 29 | run: composer update --prefer-dist --no-interaction 30 | - name: Unit tests 31 | run: |- 32 | vendor/bin/php-cs-fixer fix --dry-run --using-cache=no --verbose 33 | vendor/bin/atoum 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | composer.lock 3 | .php-cs-fixer.cache 4 | -------------------------------------------------------------------------------- /.php-cs-fixer.dist.php: -------------------------------------------------------------------------------- 1 | in( 5 | [ 6 | __DIR__.'/src', 7 | ] 8 | ); 9 | 10 | $config = new M6Web\CS\Config\BedrockStreaming(); 11 | $config->setFinder($finder); 12 | 13 | return $config; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 M6Web 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ElasticsearchBundle [](https://github.com/BedrockStreaming/ElasticsearchBundle/actions/workflows/ci.yml) [](https://packagist.org/packages/m6web/elasticsearch-bundle) [](https://packagist.org/packages/m6web/elasticsearch-bundle) [](https://packagist.org/packages/m6web/elasticsearch-bundle) 2 | 3 | 4 | Integration of the [Elasticsearch official PHP client](http://github.com/elasticsearch/elasticsearch-php) within a Symfony Project. 5 | 6 | ## Features 7 | 8 | This bundle creates one or more Elasticsearch client services from settings defined in the application configuration. 9 | 10 | ## Usage 11 | 12 | ### Installation 13 | 14 | You must first add the bundle to your `composer.json`: 15 | 16 | ```json 17 | "require": { 18 | "m6web/elasticsearch-bundle": "dev-master" 19 | } 20 | ``` 21 | 22 | Then register the bundle in your `AppKernel` class: 23 | 24 | ```php 25 | =7.4", 14 | "ext-json": "*", 15 | "elasticsearch/elasticsearch": "^5.1.0||^6.0.0||^7.0", 16 | "symfony/event-dispatcher": "^4.4||^5.0||^6.0", 17 | "symfony/yaml": "^4.4||^5.0||^6.0", 18 | "symfony/http-foundation": "^4.4||^5.0||^6.0", 19 | "symfony/config": "^4.4||^5.0||^6.0", 20 | "symfony/http-kernel": "^4.4||^5.0||^6.0", 21 | "symfony/dependency-injection": "^4.4||^5.0||^6.0" 22 | }, 23 | "require-dev": { 24 | "atoum/atoum": "^4.0", 25 | "m6web/php-cs-fixer-config": "^2.0" 26 | }, 27 | "autoload": { 28 | "psr-0": { "": "src/" } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/DataCollector/ElasticsearchDataCollector.php: -------------------------------------------------------------------------------- 1 | reset(); 23 | } 24 | 25 | public function handleEvent(ElasticsearchEvent $event) 26 | { 27 | $query = [ 28 | 'method' => $event->getMethod(), 29 | 'uri' => $event->getUri(), 30 | 'headers' => $this->stringifyVariable($event->getHeaders()), 31 | 'status_code' => $event->getStatusCode(), 32 | 'duration' => $event->getDuration(), 33 | 'took' => $event->getTook(), 34 | 'body' => json_decode($event->getBody()), 35 | 'error' => $event->getError(), 36 | ]; 37 | $this->data['queries'][] = $query; 38 | $this->data['total_execution_time'] += $query['duration']; 39 | } 40 | 41 | /** 42 | * {@inheritdoc} 43 | */ 44 | public function collect(Request $request, Response $response, \Throwable $exception = null) 45 | { 46 | } 47 | 48 | /** 49 | * {@inheritdoc} 50 | */ 51 | public function getName(): string 52 | { 53 | return 'elasticsearch'; 54 | } 55 | 56 | public function getQueries(): array 57 | { 58 | return $this->data['queries']; 59 | } 60 | 61 | public function getTotalExecutionTime(): float 62 | { 63 | return $this->data['total_execution_time']; 64 | } 65 | 66 | /** 67 | * Resets this data collector to its initial state. 68 | */ 69 | public function reset() 70 | { 71 | $this->data = [ 72 | 'queries' => [], 73 | 'total_execution_time' => 0, 74 | ]; 75 | } 76 | 77 | /** 78 | * Converts a PHP variable to a string or 79 | * Converts the variable into a serializable Data instance. 80 | * 81 | * The convert action depend on method available in the sf DataCollector class. 82 | * In sf >= 4, the DataCollector::varToString() doesn't exists anymore 83 | * 84 | * @param mixed $var 85 | * 86 | * @return mixed 87 | */ 88 | protected function stringifyVariable($var) 89 | { 90 | if (method_exists($this, 'varToString')) { 91 | return $this->varToString($var); 92 | } else { 93 | return $this->cloneVar($var); 94 | } 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/DependencyInjection/Configuration.php: -------------------------------------------------------------------------------- 1 | getRootNode(); 24 | 25 | $rootNode 26 | ->children() 27 | ->scalarNode('default_client')->end() 28 | ->arrayNode('clients') 29 | ->useAttributeAsKey('id') 30 | ->prototype('array') 31 | ->children() 32 | ->arrayNode('hosts') 33 | ->isRequired() 34 | ->requiresAtLeastOneElement() 35 | ->performNoDeepMerging() 36 | ->prototype('scalar')->end() 37 | ->end() 38 | // @deprecated 39 | ->scalarNode('client_class')->defaultValue('Elasticsearch\Client')->end() 40 | ->scalarNode('clientBuilderClass')->defaultValue('Elasticsearch\ClientBuilder')->end() 41 | ->scalarNode('connectionPoolClass')->end() 42 | ->scalarNode('selectorClass')->end() 43 | ->variableNode('connectionParams') 44 | ->validate()->ifString()->thenInvalid('connectionParams cannot be a string')->end() 45 | ->defaultValue([]) 46 | ->end() 47 | ->integerNode('retries')->end() 48 | ->scalarNode('logger')->end() 49 | ->variableNode('headers')->end() 50 | ->end() 51 | ->end() 52 | ->end(); 53 | 54 | return $treeBuilder; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/DependencyInjection/M6WebElasticsearchExtension.php: -------------------------------------------------------------------------------- 1 | processConfiguration($configuration, $configs); 28 | 29 | $loader = new Loader\YamlFileLoader($container, new FileLocator(__DIR__.'/../Resources/config')); 30 | $loader->load('services.yml'); 31 | 32 | if (isset($config['clients'])) { 33 | foreach ($config['clients'] as $clientName => $clientConfig) { 34 | $this->createElasticsearchClient($container, $clientName, $clientConfig); 35 | } 36 | 37 | if (isset($config['default_client'])) { 38 | $container->setAlias( 39 | 'm6web_elasticsearch.client.default', 40 | sprintf('m6web_elasticsearch.client.%s', $config['default_client']) 41 | ); 42 | } 43 | } 44 | } 45 | 46 | public function getAlias(): string 47 | { 48 | return 'm6web_elasticsearch'; 49 | } 50 | 51 | /** 52 | * Add a new Elasticsearch client definition in the container 53 | * 54 | * @param string $name 55 | * @param array $config 56 | */ 57 | protected function createElasticsearchClient(ContainerBuilder $container, $name, $config) 58 | { 59 | $definitionId = 'm6web_elasticsearch.client.'.$name; 60 | 61 | $handlerId = $this->createHandler($container, $config, $definitionId); 62 | 63 | $clientConfig = [ 64 | 'hosts' => $config['hosts'], 65 | 'handler' => new Reference($handlerId), 66 | ]; 67 | 68 | if (isset($config['retries'])) { 69 | $clientConfig['retries'] = $config['retries']; 70 | } 71 | 72 | if (isset($config['logger'])) { 73 | $clientConfig['logger'] = new Reference($config['logger']); 74 | } 75 | 76 | if (!empty($config['connectionPoolClass'])) { 77 | $clientConfig['connectionPool'] = $config['connectionPoolClass']; 78 | } 79 | 80 | if (!empty($config['selectorClass'])) { 81 | $clientConfig['selector'] = $config['selectorClass']; 82 | } 83 | 84 | if (!empty($config['connectionParams'])) { 85 | $clientConfig['connectionParams'] = $config['connectionParams']; 86 | } 87 | 88 | $definition = (new Definition($config['client_class'])) 89 | ->setArguments([$clientConfig]); 90 | $this->setFactoryToDefinition($config['clientBuilderClass'], 'fromConfig', $definition); 91 | 92 | $container->setDefinition($definitionId, $definition); 93 | 94 | if ($container->getParameter('kernel.debug')) { 95 | $this->createDataCollector($container); 96 | } 97 | } 98 | 99 | /** 100 | * Create request handler 101 | */ 102 | protected function createHandler(ContainerBuilder $container, array $config, string $definitionId): string 103 | { 104 | // cURL handler 105 | $singleHandler = (new Definition('GuzzleHttp\Ring\Client\CurlHandler')) 106 | ->setPublic(false); 107 | 108 | $this->setFactoryToDefinition('Elasticsearch\ClientBuilder', 'defaultHandler', $singleHandler); 109 | 110 | $singleHandlerId = $definitionId.'.single_handler'; 111 | $container->setDefinition($singleHandlerId, $singleHandler); 112 | 113 | // Headers handler 114 | $headersHandler = (new Definition('M6Web\Bundle\ElasticsearchBundle\Handler\HeadersHandler')) 115 | ->setPublic(false) 116 | ->setArguments([new Reference($singleHandlerId)]); 117 | if (isset($config['headers'])) { 118 | foreach ($config['headers'] as $key => $value) { 119 | $headersHandler->addMethodCall('setHeader', [$key, $value]); 120 | } 121 | } 122 | $headersHandlerId = $definitionId.'.headers_handler'; 123 | $container->setDefinition($headersHandlerId, $headersHandler); 124 | 125 | // Event handler 126 | $eventHandler = (new Definition('M6Web\Bundle\ElasticsearchBundle\Handler\EventHandler')) 127 | ->setPublic(false) 128 | ->setArguments([new Reference('event_dispatcher'), new Reference($headersHandlerId)]); 129 | $eventHandlerId = $definitionId.'.event_handler'; 130 | $container->setDefinition($eventHandlerId, $eventHandler); 131 | 132 | return $eventHandlerId; 133 | } 134 | 135 | private function setFactoryToDefinition(string $className, string $method, Definition $definition) 136 | { 137 | $definition 138 | ->setFactory([$className, $method]); 139 | } 140 | 141 | protected function createDataCollector(ContainerBuilder $container) 142 | { 143 | $collectorDefinition = new Definition( 144 | 'M6Web\Bundle\ElasticsearchBundle\DataCollector\ElasticsearchDataCollector' 145 | ); 146 | $collectorDefinition->addTag( 147 | 'data_collector', 148 | [ 149 | 'template' => '@M6WebElasticsearch/Collector/elasticsearch.html.twig', 150 | 'id' => 'elasticsearch', 151 | ] 152 | ); 153 | 154 | $collectorDefinition->addTag( 155 | 'kernel.event_listener', 156 | [ 157 | 'event' => 'm6web.elasticsearch', 158 | 'method' => 'handleEvent', 159 | ] 160 | ); 161 | 162 | $container->setDefinition('m6web_elasticsearch.data_collector', $collectorDefinition); 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/Elasticsearch/ConnectionPool/Selector/RandomStickySelector.php: -------------------------------------------------------------------------------- 1 | current === null) || !isset($connections[$this->current])) { 32 | $this->current = array_rand($connections); 33 | } 34 | 35 | return $connections[$this->current]; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/Elasticsearch/ConnectionPool/StaticAliveNoPingConnectionPool.php: -------------------------------------------------------------------------------- 1 | Extend StaticNoPingConnectionPool and add the ability to remove any failed connection 16 | * from the list of eligible connections to be chosen from by the selector. 17 | */ 18 | class StaticAliveNoPingConnectionPool extends StaticNoPingConnectionPool 19 | { 20 | private int $pingTimeout = 60; 21 | private int $maxPingTimeout = 3600; 22 | 23 | /** 24 | * > Allow to customize the ping time out. 25 | */ 26 | public function setPingTimeout(int $pingTimeout) 27 | { 28 | $this->pingTimeout = $pingTimeout; 29 | } 30 | 31 | /** 32 | * > Allow to customize the ping max time out. 33 | */ 34 | public function setMaxPingTimeout(int $maxPingTimeout) 35 | { 36 | $this->maxPingTimeout = $maxPingTimeout; 37 | } 38 | 39 | /** 40 | * @throws NoNodesAvailableException 41 | */ 42 | public function nextConnection(bool $force = false): ConnectionInterface 43 | { 44 | // > Replace $this->connections by $connections in order to modify the list later. 45 | $connections = $this->connections; 46 | $total = count($connections); 47 | while ($total--) { 48 | /** @var Connection $connection */ 49 | $connection = $this->selector->select($connections); 50 | if ($connection->isAlive() === true) { 51 | return $connection; 52 | } 53 | 54 | if ($this->readyToRevive($connection) === true) { 55 | return $connection; 56 | } 57 | 58 | // > Remove the failed connection from the list of eligible connections. 59 | $connections = array_filter( 60 | $connections, 61 | function ($baseConnection) use ($connection) { 62 | return $baseConnection !== $connection; 63 | } 64 | ); 65 | } 66 | 67 | throw new NoNodesAvailableException('No alive nodes found in your cluster'); 68 | } 69 | 70 | /** 71 | * > Same as parent private method. 72 | */ 73 | protected function readyToRevive(Connection $connection): bool 74 | { 75 | $timeout = min( 76 | $this->pingTimeout * pow(2, $connection->getPingFailures()), 77 | $this->maxPingTimeout 78 | ); 79 | 80 | if ($connection->getLastPing() + $timeout < time()) { 81 | return true; 82 | } else { 83 | return false; 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/EventDispatcher/ElasticsearchEvent.php: -------------------------------------------------------------------------------- 1 | duration; 34 | } 35 | 36 | public function setDuration(float $duration): self 37 | { 38 | $this->duration = $duration; 39 | 40 | return $this; 41 | } 42 | 43 | public function getMethod(): ?string 44 | { 45 | return $this->method; 46 | } 47 | 48 | public function setMethod(?string $method): self 49 | { 50 | $this->method = $method; 51 | 52 | return $this; 53 | } 54 | 55 | public function getUri(): ?string 56 | { 57 | return $this->uri; 58 | } 59 | 60 | public function setUri(?string $uri): self 61 | { 62 | $this->uri = $uri; 63 | 64 | return $this; 65 | } 66 | 67 | public function getStatusCode(): ?int 68 | { 69 | return $this->statusCode; 70 | } 71 | 72 | public function setStatusCode(?int $statusCode): self 73 | { 74 | $this->statusCode = $statusCode; 75 | 76 | return $this; 77 | } 78 | 79 | public function getTook(): ?int 80 | { 81 | return $this->took; 82 | } 83 | 84 | public function setTook(?int $took): self 85 | { 86 | $this->took = $took; 87 | 88 | return $this; 89 | } 90 | 91 | public function getHeaders(): array 92 | { 93 | return $this->headers; 94 | } 95 | 96 | public function setHeaders(array $headers): self 97 | { 98 | $this->headers = $headers; 99 | 100 | return $this; 101 | } 102 | 103 | public function getBody(): string 104 | { 105 | return $this->body; 106 | } 107 | 108 | public function setBody(string $body): self 109 | { 110 | $this->body = $body; 111 | 112 | return $this; 113 | } 114 | 115 | public function getError(): string 116 | { 117 | return $this->error; 118 | } 119 | 120 | public function setError(string $error): self 121 | { 122 | $this->error = $error; 123 | 124 | return $this; 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/Handler/EventHandler.php: -------------------------------------------------------------------------------- 1 | eventDispatcher = $eventDispatcher; 31 | $this->handler = $handler; 32 | } 33 | 34 | public function __invoke(array $request): \GuzzleHttp\Ring\Future\FutureArray 35 | { 36 | $handler = $this->handler; 37 | 38 | $wrapper = $this; 39 | $getTook = function (array $response) use ($wrapper) { 40 | return $wrapper->extractTookFromResponse($response); 41 | }; 42 | 43 | $dispatchEvent = function ($response) use ($request, $getTook) { 44 | $event = (new ElasticsearchEvent()) 45 | ->setUri($request['uri']) 46 | ->setMethod($request['http_method']) 47 | ->setStatusCode($response['status']) 48 | ->setDuration($response['transfer_stats']['total_time'] * 1000) 49 | ->setTook($getTook($response)); 50 | 51 | if (isset($request['body'])) { 52 | $event->setBody($request['body']); 53 | } 54 | if (isset($request['headers'])) { 55 | $event->setHeaders($request['headers']); 56 | } 57 | if (isset($response['error'])) { 58 | $event->setError($response['error']->getMessage()); 59 | } 60 | 61 | $this->eventDispatcher->dispatch($event, 'm6web.elasticsearch'); 62 | 63 | return $response; 64 | }; 65 | 66 | return Core::proxy($handler($request), $dispatchEvent); 67 | } 68 | 69 | protected function extractTookFromResponse(array $response): ?int 70 | { 71 | if (is_null($response['body'])) { 72 | return null; 73 | } 74 | $content = stream_get_contents($response['body']); 75 | rewind($response['body']); 76 | 77 | if (preg_match('/\"took\":(\d+)/', $content, $matches)) { 78 | return (int) $matches[1]; 79 | } 80 | 81 | return null; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/Handler/HeadersHandler.php: -------------------------------------------------------------------------------- 1 | handler = $handler; 27 | } 28 | 29 | public function setHeader(string $key, $value): self 30 | { 31 | $this->headers[$key] = $value; 32 | 33 | return $this; 34 | } 35 | 36 | public function __invoke(array $request) 37 | { 38 | // By default, host is stored in headers and final url is forged later. 39 | // We need to build url now to handle the case when we want to add a custom Host header. 40 | $request['url'] = Core::url($request); 41 | 42 | $handler = $this->handler; 43 | foreach ($this->headers as $key => $value) { 44 | if ($key == 'Accept-Encoding' && in_array('gzip', $value)) { 45 | $request['client']['decode_content'] = true; 46 | } 47 | $request['headers'][$key] = $value; 48 | } 49 | 50 | return $handler($request); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/M6Web/Bundle/ElasticsearchBundle/M6WebElasticsearchBundle.php: -------------------------------------------------------------------------------- 1 | 6 |
7 | {% if collector.queries|length > 0 %} 8 | 9 | {% endif %} 10 | {% endset %} 11 | {% set text %} 12 | 16 | 20 | {% endset %} 21 | {% include '@WebProfiler/Profiler/toolbar_item.html.twig' with {'link': profiler_url} %} 22 | {% endblock %} 23 | 24 | {% block menu %} 25 | 26 | 27 |