├── .gitignore
├── code-of-conduct.md
├── templates
└── Includes
│ └── AlgoliaSearchResults.ss
├── tests
├── TestAlgoliaServiceResponse.php
├── TestAlgoliaServiceClient.php
├── TestAlgoliaService.php
├── TestAlgoliaServiceIndex.php
├── AlgoliaCustomTestObject.php
├── AlgoliaTestObject.php
├── AlgoliaQuerierTest.php
├── AlgoliaObjectExtensionTest.php
├── AlgoliaIndexerTest.php
└── AlgoliaServiceTest.php
├── phpstan.neon
├── .github
└── workflows
│ └── ci.yml
├── phpcs.xml.dist
├── phpunit.xml.dist
├── .editorconfig
├── _config
└── config.yml
├── src
├── Extensions
│ ├── VirtualPageExtension.php
│ ├── SubsitesVirtualPageExtension.php
│ └── AlgoliaObjectExtension.php
├── Jobs
│ ├── AlgoliaDeleteItemJob.php
│ ├── AlgoliaIndexItemJob.php
│ └── AlgoliaReindexAllJob.php
├── Tasks
│ ├── AlgoliaConfigure.php
│ ├── AlgoliaInspect.php
│ ├── AlgoliaReindexItem.php
│ └── AlgoliaReindex.php
└── Service
│ ├── AlgoliaQuerier.php
│ ├── AlgoliaPageCrawler.php
│ ├── AlgoliaService.php
│ └── AlgoliaIndexer.php
├── LICENSE
├── composer.json
└── README.md
/.gitignore:
--------------------------------------------------------------------------------
1 | /vendor
2 | /assets
3 | /resources
4 | composer.lock
5 | .DS_Store
6 | /public
7 | /.env
8 | /.phpunit.result.cache
9 |
--------------------------------------------------------------------------------
/code-of-conduct.md:
--------------------------------------------------------------------------------
1 | When having discussions about this module in issues or pull request please adhere to the [SilverStripe Community Code of Conduct](https://docs.silverstripe.org/en/contributing/code_of_conduct).
2 |
--------------------------------------------------------------------------------
/templates/Includes/AlgoliaSearchResults.ss:
--------------------------------------------------------------------------------
1 |
2 |
3 |
7 |
--------------------------------------------------------------------------------
/tests/TestAlgoliaServiceResponse.php:
--------------------------------------------------------------------------------
1 |
2 |
3 | CodeSniffer ruleset for SilverStripe coding conventions.
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/tests/TestAlgoliaServiceClient.php:
--------------------------------------------------------------------------------
1 | api, $this->config);
13 | }
14 | }
15 |
--------------------------------------------------------------------------------
/phpunit.xml.dist:
--------------------------------------------------------------------------------
1 |
2 |
3 | tests
4 |
5 |
9 |
10 | src
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | # For more information about the properties used in
2 | # this file, please see the EditorConfig documentation:
3 | # http://editorconfig.org/
4 |
5 | [*]
6 | charset = utf-8
7 | end_of_line = lf
8 | indent_size = 4
9 | indent_style = space
10 | insert_final_newline = true
11 | trim_trailing_whitespace = true
12 |
13 | [*.md]
14 | trim_trailing_whitespace = false
15 |
16 | [*.{yml,js,json,css,scss,eslintrc,feature}]
17 | indent_size = 2
18 | indent_style = space
19 |
20 | [composer.json]
21 | indent_size = 4
22 |
--------------------------------------------------------------------------------
/_config/config.yml:
--------------------------------------------------------------------------------
1 | ---
2 | Name: silverstripe-algolia
3 | ---
4 | SilverStripe\CMS\Model\SiteTree:
5 | extensions:
6 | - Wilr\SilverStripe\Algolia\Extensions\AlgoliaObjectExtension
7 | SilverStripe\Subsites\Pages\SubsitesVirtualPage:
8 | extensions:
9 | - Wilr\SilverStripe\Algolia\Extensions\SubsitesVirtualPageExtension
10 | SilverStripe\CMS\Model\VirtualPage:
11 | extensions:
12 | - Wilr\SilverStripe\Algolia\Extensions\VirtualPageExtension
13 | non_virtual_fields:
14 | - AlgoliaUUID
15 | - AlgoliaIndexed
16 | - AlgoliaError
17 |
--------------------------------------------------------------------------------
/tests/TestAlgoliaService.php:
--------------------------------------------------------------------------------
1 | [
12 | 'includeClasses' => [
13 | AlgoliaTestObject::class
14 | ]
15 | ]
16 | ];
17 |
18 | public function getClient()
19 | {
20 | return TestAlgoliaServiceClient::create('ABC', '123');
21 | }
22 | }
23 |
--------------------------------------------------------------------------------
/tests/TestAlgoliaServiceIndex.php:
--------------------------------------------------------------------------------
1 | [],
21 | 'page' => 1,
22 | 'nbHits' => 1,
23 | 'hitsPerPage' => 10
24 | ];
25 | }
26 |
27 | public function deleteObject($objectId, $requestOptions = array())
28 | {
29 | if (isset($this->objects[$objectId])) {
30 | unset($this->objects[$objectId]);
31 | }
32 | }
33 |
34 | public function saveObject($object, $requestOptions = array())
35 | {
36 | $this->objects[$object['objectID']] = $object;
37 |
38 | return new TestAlgoliaServiceResponse();
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/tests/AlgoliaCustomTestObject.php:
--------------------------------------------------------------------------------
1 | 'Varchar'
16 | ];
17 |
18 | private static $extensions = [
19 | AlgoliaObjectExtension::class
20 | ];
21 |
22 | private static $table_name = 'AlgoliaCustomTestObject';
23 |
24 |
25 | public function AbsoluteLink()
26 | {
27 | return Director::absoluteBaseURL();
28 | }
29 |
30 |
31 | public function exportObjectToAlgolia($data)
32 | {
33 | $data = array_merge($data, [
34 | 'MyCustomField' => 'MyCustomFieldValue'
35 | ]);
36 |
37 | $map = new Map(ArrayList::create());
38 |
39 | foreach ($data as $k => $v) {
40 | $map->push($k, $v);
41 | }
42 |
43 | return $map;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/tests/AlgoliaTestObject.php:
--------------------------------------------------------------------------------
1 | 'Varchar',
15 | 'OtherField' => 'Varchar',
16 | 'NonIndexedField' => 'Varchar',
17 | 'Active' => 'Boolean'
18 | ];
19 |
20 | private static $has_one = [
21 | 'Author' => Member::class
22 | ];
23 |
24 | private static $many_many = [
25 | 'RelatedTestObjects' => AlgoliaTestObject::class
26 | ];
27 |
28 | private static $algolia_index_fields = [
29 | 'OtherField',
30 | 'Active'
31 | ];
32 |
33 | private static $extensions = [
34 | AlgoliaObjectExtension::class
35 | ];
36 |
37 | private static $table_name = 'AlgoliaTestObject';
38 |
39 |
40 | public function AbsoluteLink()
41 | {
42 | return Director::absoluteBaseURL();
43 | }
44 |
45 |
46 | public function canIndexInAlgolia(): bool
47 | {
48 | return ($this->Active) ? true : false;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/tests/AlgoliaQuerierTest.php:
--------------------------------------------------------------------------------
1 | [
23 | AlgoliaObjectExtension::class
24 | ]
25 | ];
26 |
27 | public static function setUpBeforeClass(): void
28 | {
29 | parent::setUpBeforeClass();
30 |
31 | // mock AlgoliaService
32 | Injector::inst()->get(DataObjectSchema::class)->reset();
33 | Injector::inst()->registerService(new TestAlgoliaService(), AlgoliaService::class);
34 | }
35 |
36 | public function testFetchResults()
37 | {
38 | $results = Injector::inst()->get(AlgoliaQuerier::class)->fetchResults('indexName', 'search keywords');
39 |
40 | $this->assertInstanceOf(PaginatedList::class, $results);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/Extensions/VirtualPageExtension.php:
--------------------------------------------------------------------------------
1 | $v) {
18 | if ($k === 'objectClassName') {
19 | continue;
20 | }
21 |
22 | $attributes->push($k, $v);
23 | }
24 |
25 | /** @var AlgoliaIndexer */
26 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
27 | $owner = $this->owner;
28 |
29 | // get original object
30 | $originalObject = $owner->CopyContentFrom();
31 |
32 | if (!$originalObject) {
33 | return $attributes;
34 | }
35 |
36 | $attributes->push('objectClassName', $originalObject->ClassName);
37 | $specs = $originalObject->config()->get('algolia_index_fields');
38 | $attributes = $indexer->addSpecsToAttributes($originalObject, $attributes, $specs);
39 |
40 | $originalObject->invokeWithExtensions('updateAlgoliaAttributes', $attributes);
41 |
42 | return $attributes;
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | BSD 3-Clause License
2 |
3 | Copyright (c) 2020, Fullscreen Interactive All rights reserved.
4 |
5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
6 |
7 | Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8 |
9 | Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
10 |
11 | Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
12 |
13 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
14 |
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "wilr/silverstripe-algolia",
3 | "description": "Algolia Indexer and Search Functionality",
4 | "type": "silverstripe-vendormodule",
5 | "license": "BSD-3-Clause",
6 | "authors": [
7 | {
8 | "name": "Will Rossiter",
9 | "email": "will@fullscreen.io"
10 | }
11 | ],
12 | "require": {
13 | "php": "^8",
14 | "silverstripe/framework": "^6",
15 | "symbiote/silverstripe-queuedjobs": "^6",
16 | "algolia/algoliasearch-client-php": "^3",
17 | "ramsey/uuid": "^4",
18 | "masterminds/html5": "^2.7"
19 | },
20 | "require-dev": {
21 | "phpunit/phpunit": "^11",
22 | "squizlabs/php_codesniffer": "^3",
23 | "cambis/silverstan": "^1.1"
24 | },
25 | "scripts": {
26 | "lint": "phpcs --extensions=php src/ tests/",
27 | "lint:fix": "phpcbf src/ tests/",
28 | "phpstan": "phpstan analyse -c phpstan.neon",
29 | "syntax-check": "find src/ tests/ -type f -name '*.php' -exec php -l {} \\;",
30 | "test": "phpunit tests"
31 | },
32 | "extra": {
33 | "branch-alias": {
34 | "dev-main": "2.x-dev"
35 | }
36 | },
37 | "autoload": {
38 | "psr-4": {
39 | "Wilr\\SilverStripe\\Algolia\\": "src/",
40 | "Wilr\\SilverStripe\\Algolia\\Tests\\": "tests/"
41 | }
42 | },
43 | "prefer-stable": true,
44 | "minimum-stability": "dev",
45 | "config": {
46 | "allow-plugins": {
47 | "composer/installers": true,
48 | "silverstripe/vendor-plugin": true
49 | }
50 | }
51 | }
52 |
--------------------------------------------------------------------------------
/src/Jobs/AlgoliaDeleteItemJob.php:
--------------------------------------------------------------------------------
1 | itemClass = $itemClass;
25 | $this->itemUUID = $itemUUID;
26 | }
27 |
28 |
29 | /**
30 | * Defines the title of the job.
31 | *
32 | * @return string
33 | */
34 | public function getTitle()
35 | {
36 | return sprintf(
37 | 'Algolia remove object %s',
38 | $this->itemUUID
39 | );
40 | }
41 |
42 | /**
43 | * @return int
44 | */
45 | public function getJobType()
46 | {
47 | $this->totalSteps = 1;
48 |
49 | return QueuedJob::IMMEDIATE;
50 | }
51 |
52 | public function process()
53 | {
54 | try {
55 | $indexer = Injector::inst()->create(AlgoliaIndexer::class);
56 | $indexer->deleteItem($this->itemClass, $this->itemUUID);
57 | } catch (Throwable $e) {
58 | Injector::inst()->get(LoggerInterface::class)->error($e);
59 | }
60 |
61 | $this->isComplete = true;
62 |
63 | return;
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/Tasks/AlgoliaConfigure.php:
--------------------------------------------------------------------------------
1 | get(AlgoliaService::class);
29 |
30 | if (!$this->isEnabled()) {
31 | $output->writeln('This task is disabled.');
32 | return Command::FAILURE;
33 | }
34 |
35 | try {
36 | if ($service->syncSettings()) {
37 | $output->writeln('Algolia settings synced successfully.' . PHP_EOL);
38 |
39 | return Command::SUCCESS;
40 | }
41 |
42 | $output->writeln('An error occurred while syncing the settings. Please check your error logs.');
43 | } catch (\Exception $e) {
44 | $output->writeln('An error occurred while syncing the settings. Please check your error logs.');
45 | $output->writeln('Error: ' . $e->getMessage());
46 | }
47 |
48 | return Command::FAILURE;
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/src/Extensions/SubsitesVirtualPageExtension.php:
--------------------------------------------------------------------------------
1 | $v) {
22 | if ($k === 'objectClassName') {
23 | continue;
24 | }
25 |
26 | $attributes->push($k, $v);
27 | }
28 |
29 | /** @var AlgoliaIndexer */
30 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
31 | $owner = $this->owner;
32 |
33 | // get original object
34 | $result = Subsite::withDisabledSubsiteFilter(function () use ($owner, $attributes, $indexer) {
35 | $originalObject = $owner->CopyContentFrom();
36 |
37 | if (!$originalObject) {
38 | return $attributes;
39 | }
40 |
41 | $attributes->push('objectClassName', $originalObject->ClassName);
42 | $attributes->push('objectSubsiteID', $this->owner->SubsiteID);
43 |
44 | $specs = $originalObject->config()->get('algolia_index_fields');
45 | $attributes = $indexer->addSpecsToAttributes($originalObject, $attributes, $specs);
46 |
47 | $originalObject->invokeWithExtensions('updateAlgoliaAttributes', $attributes);
48 |
49 | return $attributes;
50 | });
51 |
52 | $attributes->push('SubsiteID', $this->owner->SubsiteID);
53 |
54 | return $result;
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/src/Tasks/AlgoliaInspect.php:
--------------------------------------------------------------------------------
1 | getOption('class');
23 | $itemId = $input->getOption('id');
24 |
25 | if (!$itemClass || !$itemId) {
26 | $output->writeln('Missing class or id parameters');
27 |
28 | return Command::FAILURE;
29 | }
30 |
31 | $item = $itemClass::get()->byId($itemId);
32 |
33 | if (!$item || !$item->canView()) {
34 | $output->writeln('Missing or unviewable object ' . $itemClass . ' #' . $itemId);
35 | return Command::FAILURE;
36 | }
37 |
38 | $indexer = Injector::inst()->create(AlgoliaIndexer::class);
39 | $indexer->getService()->syncSettings();
40 |
41 | $output->writeln('### LOCAL FIELDS ###');
42 | $output->writeln('');
43 | $output->writeln(print_r($indexer->exportAttributesFromObject($item), true));
44 |
45 | $output->writeln('### REMOTE FIELDS ###');
46 | $output->writeln(print_r($indexer->getObject($item), true));
47 |
48 | $output->writeln('### INDEX SETTINGS ###');
49 | foreach ($item->getAlgoliaIndexes() as $index) {
50 | $output->writeln(print_r($index->getSettings(), true));
51 | }
52 |
53 | $output->writeln('### ALGOLIA STATUS ###');
54 | $output->writeln('Error: ' . $item->AlgoliaError);
55 | $output->writeln('LastIndexed: ' . $item->AlgoliaIndexed);
56 | $output->writeln('Algolia UUID: ' . $item->AlgoliaUUID);
57 |
58 | return Command::SUCCESS;
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/tests/AlgoliaObjectExtensionTest.php:
--------------------------------------------------------------------------------
1 | [
22 | AlgoliaObjectExtension::class
23 | ]
24 | ];
25 |
26 | public static function setUpBeforeClass(): void
27 | {
28 | parent::setUpBeforeClass();
29 |
30 | // mock AlgoliaService
31 | Injector::inst()->get(DataObjectSchema::class)->reset();
32 | Injector::inst()->registerService(new TestAlgoliaService(), AlgoliaService::class);
33 | }
34 |
35 | public function testIndexInAlgolia()
36 | {
37 | $object = AlgoliaTestObject::create();
38 | $object->Active = false;
39 | $object->write();
40 |
41 | $this->assertFalse(
42 | min($object->invokeWithExtensions('canIndexInAlgolia')),
43 | 'Objects with canIndexInAlgolia() false should not index'
44 | );
45 |
46 | $object->Active = true;
47 | $object->write();
48 |
49 | $this->assertTrue(
50 | min($object->invokeWithExtensions('canIndexInAlgolia')),
51 | 'Objects with canIndexInAlgolia() set to true should index'
52 | );
53 |
54 | $index = $object->indexInAlgolia();
55 | $this->assertTrue($index, 'Indexed in Algolia');
56 | }
57 |
58 | public function testTouchAlgoliaIndexedDate()
59 | {
60 | $object = AlgoliaTestObject::create();
61 | $object->write();
62 |
63 | $object->touchAlgoliaIndexedDate();
64 |
65 | $this->assertNotNull(
66 | DB::query(
67 | sprintf(
68 | 'SELECT AlgoliaIndexed FROM AlgoliaTestObject WHERE ID = %s',
69 | $object->ID
70 | )
71 | )->value()
72 | );
73 |
74 | $object->touchAlgoliaIndexedDate(true);
75 |
76 | $this->assertNull(
77 | DB::query(
78 | sprintf(
79 | 'SELECT AlgoliaIndexed FROM AlgoliaTestObject WHERE ID = %s',
80 | $object->ID
81 | )
82 | )->value()
83 | );
84 | }
85 | }
86 |
--------------------------------------------------------------------------------
/tests/AlgoliaIndexerTest.php:
--------------------------------------------------------------------------------
1 | [
23 | AlgoliaObjectExtension::class
24 | ]
25 | ];
26 |
27 | public static function setUpBeforeClass(): void
28 | {
29 | parent::setUpBeforeClass();
30 |
31 | // mock AlgoliaService
32 | Injector::inst()->get(DataObjectSchema::class)->reset();
33 | Injector::inst()->registerService(new TestAlgoliaService(), AlgoliaService::class);
34 | }
35 |
36 | public function testExportAttributesForObject()
37 | {
38 | $object = AlgoliaTestObject::create();
39 | $object->Title = 'Foobar';
40 | $object->write();
41 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
42 | $map = $indexer->exportAttributesFromObject($object)->toArray();
43 |
44 | $this->assertArrayHasKey('objectID', $map);
45 | $this->assertEquals($map['objectTitle'], 'Foobar');
46 |
47 | $object = AlgoliaCustomTestObject::create();
48 | $object->Title = 'Qux';
49 | $object->write();
50 |
51 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
52 | $map = $indexer->exportAttributesFromObject($object)->toArray();
53 |
54 | $this->assertArrayHasKey('objectID', $map);
55 | $this->assertEquals($map['objectTitle'], 'Qux');
56 | $this->assertEquals($map['MyCustomField'], 'MyCustomFieldValue');
57 | }
58 |
59 |
60 | public function testDeleteExistingItem()
61 | {
62 | $object = AlgoliaTestObject::create();
63 | $object->Title = 'Delete This';
64 | $object->write();
65 |
66 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
67 | $deleted = $indexer->deleteItem($object->getClassName(), $object->AlgoliaUUID);
68 |
69 | return $this->assertTrue($deleted);
70 | }
71 |
72 | public function testDeleteNonExistentItem()
73 | {
74 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
75 | $deleted = $indexer->deleteItem(AlgoliaTestObject::class, 9999999);
76 |
77 | return $this->assertTrue($deleted);
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/src/Tasks/AlgoliaReindexItem.php:
--------------------------------------------------------------------------------
1 | getOption('class')) {
31 | $targetClass = $input->getOption('class');
32 | } else {
33 | $output->writeln('Missing class argument');
34 | return Command::FAILURE;
35 | }
36 |
37 | if ($input->getOption('id')) {
38 | $id = $input->getOption('id');
39 | } else {
40 | $output->writeln('Missing id argument');
41 | return Command::FAILURE;
42 | }
43 |
44 | $obj = DataObject::get($targetClass)->byID($id);
45 |
46 | if (!$obj) {
47 | $output->writeln('Object not found');
48 | return Command::FAILURE;
49 | }
50 |
51 | // Set AlgoliaUUID, in case it wasn't previously set
52 | if (!$obj->AlgoliaUUID) {
53 | $output->writeln('No AlgoliaUUID set on object, generating one...');
54 | $obj->assignAlgoliaUUID(true);
55 | }
56 |
57 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
58 | $service = $indexer->getService();
59 |
60 | $output->write('Indexing to Algolia indexes (');
61 | $output->write(implode(', ', array_map(function ($indexName) use ($service) {
62 | return $service->environmentizeIndex($indexName);
63 | }, array_keys($service->initIndexes($obj)))));
64 | $output->writeln(')');
65 |
66 | $result = $obj->doImmediateIndexInAlgolia();
67 |
68 | $output->writeln(sprintf(
69 | 'Indexed: %s%sUUID: %s%s%s',
70 | $result ? 'true ' . '(timestamp ' . $obj->AlgoliaIndexed . ')' : 'false',
71 | PHP_EOL,
72 | $obj->AlgoliaUUID ? $obj->AlgoliaUUID : 'No ID set',
73 | PHP_EOL,
74 | $obj->AlgoliaError ? 'Error from Algolia: ' . $obj->AlgoliaError : ''
75 | ));
76 |
77 | return $result ? Command::SUCCESS : Command::FAILURE;
78 | }
79 |
80 | public function getOptions(): array
81 | {
82 | return [
83 | new InputOption('class', null, InputOption::VALUE_REQUIRED, 'The class name of the object to reindex'),
84 | new InputOption('id', null, InputOption::VALUE_REQUIRED, 'The ID of the object to reindex'),
85 | ];
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/tests/AlgoliaServiceTest.php:
--------------------------------------------------------------------------------
1 | [
24 | AlgoliaObjectExtension::class
25 | ]
26 | ];
27 |
28 | public static function setUpBeforeClass(): void
29 | {
30 | parent::setUpBeforeClass();
31 |
32 | // mock AlgoliaService
33 | Injector::inst()->get(DataObjectSchema::class)->reset();
34 | Injector::inst()->registerService(new TestAlgoliaService(), AlgoliaService::class);
35 | }
36 |
37 |
38 | public function testInitIndexes()
39 | {
40 | $service = Injector::inst()->create(AlgoliaService::class);
41 |
42 | $service->indexes = [
43 | 'testIndexTestObjects' => [
44 | 'includeClasses' => [
45 | AlgoliaTestObject::class
46 | ],
47 | ],
48 | 'testIndexCustomTestObjects' => [
49 | 'includeClasses' => [
50 | AlgoliaCustomTestObject::class
51 | ],
52 | ],
53 | ];
54 |
55 | $testObj = new AlgoliaTestObject();
56 | $testObj->Title = 'Test';
57 | $testObj->Active = 1;
58 | $testObj->write();
59 |
60 | $testObj2 = new AlgoliaCustomTestObject();
61 | $testObj2->Title = 'Test';
62 | $testObj2->Active = 1;
63 |
64 | $this->assertEquals(['testIndexTestObjects'], array_keys($service->initIndexes($testObj)));
65 | $this->assertEquals(['testIndexCustomTestObjects'], array_keys($service->initIndexes($testObj2)));
66 | }
67 |
68 |
69 | public function testInitIndexesWithFilter()
70 | {
71 | $service = Injector::inst()->create(AlgoliaService::class);
72 |
73 | $service->indexes = [
74 | 'testIndexTestObjects' => [
75 | 'includeClasses' => [
76 | AlgoliaTestObject::class
77 | ],
78 | 'includeFilter' => [
79 | AlgoliaTestObject::class => "Title != 'Ted'"
80 | ]
81 | ],
82 | 'testIndexTestObjectsNamedTed' => [
83 | 'includeClasses' => [
84 | AlgoliaTestObject::class
85 | ],
86 | 'includeFilter' => [
87 | AlgoliaTestObject::class => "Title = 'Ted'"
88 | ]
89 | ],
90 | ];
91 |
92 | $testObj = new AlgoliaTestObject();
93 | $testObj->Title = 'Test';
94 | $testObj->Active = 1;
95 | $testObj->write();
96 |
97 |
98 | $testObj2 = new AlgoliaTestObject();
99 | $testObj2->Title = 'Ted';
100 | $testObj2->Active = 1;
101 | $testObj2->write();
102 |
103 | $this->assertEquals(['testIndexTestObjects'], array_keys($service->initIndexes($testObj)));
104 | $this->assertEquals(['testIndexTestObjectsNamedTed'], array_keys($service->initIndexes($testObj2)));
105 | }
106 | }
107 |
--------------------------------------------------------------------------------
/src/Service/AlgoliaQuerier.php:
--------------------------------------------------------------------------------
1 | get(AlgoliaService::class);
32 | $results = false;
33 |
34 | if (!$selectedIndex) {
35 | if (!function_exists('array_key_first')) {
36 | function array_key_first(array $arr)
37 | {
38 | foreach ($arr as $key => $unused) {
39 | return $key;
40 | }
41 | return null;
42 | }
43 | }
44 |
45 | $selectedIndex = array_key_first($service->indexes);
46 | }
47 |
48 | try {
49 | $selectedIndex = $service->environmentizeIndex($selectedIndex);
50 | $index = $service->getSearchClient()->initIndex($selectedIndex);
51 | $results = $index->search($query, $searchParameters);
52 | } catch (Throwable $e) {
53 | Injector::inst()->get(LoggerInterface::class)->error($e);
54 | }
55 |
56 | $records = ArrayList::create();
57 | $totalItems = 0;
58 |
59 | if ($results && isset($results['hits'])) {
60 | $totalItems = isset($results['nbHits']) ? $results['nbHits'] : 0;
61 |
62 | foreach ($results['hits'] as $hit) {
63 | $className = isset($hit['objectClassName']) ? $hit['objectClassName'] : null;
64 | $id = isset($hit['objectSilverstripeID']) ? $hit['objectSilverstripeID'] : null;
65 |
66 | if (!$id || !$className) {
67 | $totalItems--;
68 | continue;
69 | }
70 |
71 | try {
72 | $record = $className::get()->byId($id);
73 |
74 | if ($record && $record->canView()) {
75 | $records->push($record);
76 | } else {
77 | $totalItems--;
78 | }
79 | } catch (Throwable $e) {
80 | Injector::inst()->get(LoggerInterface::class)->notice($e);
81 | }
82 | }
83 | }
84 |
85 | $this->lastResult = $results;
86 |
87 | if (!empty($ORMFilters)) {
88 | $records = $records->filter($ORMFilters);
89 | }
90 |
91 | $output = PaginatedList::create($records);
92 |
93 | if ($results) {
94 | $output = $output->setCurrentPage($results['page'] + 1)
95 | ->setTotalItems($totalItems)
96 | ->setLimitItems(false)
97 | ->setPageStart($results['page'] * $results['hitsPerPage'])
98 | ->setPageLength($results['hitsPerPage']);
99 | }
100 |
101 | return $output;
102 | }
103 |
104 | /**
105 | * @return array|null
106 | */
107 | public function getLastResult()
108 | {
109 | return $this->lastResult;
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/src/Jobs/AlgoliaIndexItemJob.php:
--------------------------------------------------------------------------------
1 | itemClass = $itemClass;
25 | }
26 |
27 | if ($itemIds) {
28 | if (!is_array($itemIds)) {
29 | $this->itemIds = explode(',', $itemIds);
30 | } else {
31 | $this->itemIds = $itemIds;
32 | }
33 | }
34 | }
35 |
36 | /**
37 | * Defines the title of the job.
38 | *
39 | * @return string
40 | */
41 | public function getTitle()
42 | {
43 | return sprintf(
44 | 'Algolia reindex %s (%s)',
45 | $this->itemClass,
46 | implode(', ', $this->itemIds)
47 | );
48 | }
49 |
50 | /**
51 | * @return int
52 | */
53 | public function getJobType()
54 | {
55 | $this->totalSteps = count($this->itemIds);
56 |
57 | return QueuedJob::IMMEDIATE;
58 | }
59 |
60 | /**
61 | * This is called immediately before a job begins - it gives you a chance
62 | * to initialise job data and make sure everything's good to go
63 | *
64 | * What we're doing in our case is to queue up the list of items we know we need to
65 | * process still (it's not everything - just the ones we know at the moment)
66 | *
67 | * When we go through, we'll constantly add and remove from this queue, meaning
68 | * we never overload it with content
69 | */
70 | public function setup()
71 | {
72 | parent::setup();
73 |
74 | $this->remainingIds = $this->itemIds;
75 | $this->currentStep = 0;
76 | $this->totalSteps = count($this->remainingIds);
77 | }
78 |
79 | /**
80 | * Lets process a single node
81 | */
82 | public function process()
83 | {
84 | $remainingChildren = $this->remainingIds;
85 |
86 | if (!$remainingChildren || !count($remainingChildren)) {
87 | $this->isComplete = true;
88 |
89 | return;
90 | }
91 |
92 | $this->currentStep++;
93 |
94 | $id = array_shift($remainingChildren);
95 |
96 | $obj = DataObject::get_by_id($this->itemClass, $id);
97 |
98 | if (!$obj) {
99 | $this->addMessage('Record #'. $id . ' not found');
100 | } elseif (min($obj->invokeWithExtensions('canIndexInAlgolia')) === false) {
101 | $this->addMessage('Record #'. $id .' not indexed, canIndexInAlgolia returned false');
102 | } else {
103 | if (!$obj->AlgoliaUUID) {
104 | $obj->assignAlgoliaUUID();
105 | }
106 |
107 | if ($obj->doImmediateIndexInAlgolia()) {
108 | $this->addMessage('Record #'. $id .' successfully indexed as objectID '. $obj->AlgoliaUUID);
109 | } else {
110 | $this->addMessage('Record #'. $id .' failed to be indexed: '. $obj->AlgoliaError);
111 | }
112 |
113 | unset($obj);
114 | }
115 |
116 | $this->remainingIds = $remainingChildren;
117 |
118 | if (!count($remainingChildren)) {
119 | $this->isComplete = true;
120 | return;
121 | }
122 | }
123 | }
124 |
--------------------------------------------------------------------------------
/src/Service/AlgoliaPageCrawler.php:
--------------------------------------------------------------------------------
1 | element may contain other information which should
25 | * not be indexed.
26 | */
27 | class AlgoliaPageCrawler
28 | {
29 | use Configurable;
30 |
31 | private $item;
32 |
33 | private static $content_cutoff_bytes = 100000;
34 |
35 | /**
36 | * Defines the xpath selector for the first element of content
37 | * that should be indexed. If blank, defaults to the `main` element
38 | *
39 | * @config
40 | * @var string
41 | */
42 | private static $content_xpath_selector = '';
43 |
44 | /**
45 | * @config
46 | *
47 | * @var string
48 | */
49 | private static $content_element_tag = 'main';
50 |
51 |
52 | public function __construct($item)
53 | {
54 | $this->item = $item;
55 | }
56 |
57 | public function getMainContent(): string
58 | {
59 | if (!$this->item instanceof SiteTree) {
60 | return '';
61 | }
62 |
63 | $selector = $this->config()->get('content_xpath_selector');
64 | $useXpath = true;
65 |
66 | if (!$selector) {
67 | $useXpath = false;
68 | $selector = $this->config()->get('content_element_tag');
69 | }
70 |
71 | $originalStage = Versioned::get_stage();
72 | //Always set to live to ensure we don't pick up draft content in our render eg. draft elemental blocks
73 | Versioned::set_stage(Versioned::LIVE);
74 |
75 | // Enable frontend themes in order to correctly render the elements as
76 | // they would be for the frontend
77 | Config::nest();
78 | $oldThemes = SSViewer::get_themes();
79 | SSViewer::set_themes(SSViewer::config()->get('themes'));
80 |
81 | Requirements::clear();
82 |
83 | $controller = ModelAsController::controller_for($this->item);
84 | $current = Controller::curr();
85 |
86 | if ($current) {
87 | $controller->setRequest($current->getRequest());
88 | } else {
89 | $request = new HTTPRequest('GET', $this->item->Link());
90 | $request->setSession(new Session([]));
91 |
92 | $controller->setRequest($request);
93 | $controller->pushCurrent();
94 | }
95 |
96 | $page = '';
97 | $output = '';
98 |
99 | try {
100 | /** @var DBHTMLText $page */
101 | $page = $controller->render();
102 | if ($page) {
103 | libxml_use_internal_errors(true);
104 | $html5 = new HTML5();
105 |
106 | $dom = $html5->loadHTML($page->forTemplate());
107 |
108 | if ($useXpath) {
109 | $xpath = new DOMXPath($dom);
110 | $nodes = $xpath->query($selector);
111 | } else {
112 | $nodes = $dom->getElementsByTagName($selector);
113 | }
114 |
115 | if (isset($nodes[0])) {
116 | $output = $this->processMainContent($nodes[0]->nodeValue);
117 | }
118 | }
119 | } catch (Throwable $e) {
120 | Injector::inst()->get(LoggerInterface::class)->error($e);
121 | }
122 |
123 | SSViewer::set_themes($oldThemes);
124 | Requirements::restore();
125 | Config::unnest();
126 |
127 | Versioned::set_stage($originalStage);
128 |
129 | if ($this->config()->get('content_cutoff_bytes')) {
130 | $output = mb_strcut($output, 0, $this->config()->get('content_cutoff_bytes') - 1);
131 | }
132 |
133 | return $output;
134 | }
135 |
136 | /**
137 | * Process page DOM content
138 | *
139 | * @param string $content DOM node content
140 | */
141 | private function processMainContent($content): string
142 | {
143 | // Clean up the DOM content
144 | $content = preg_replace('/\s+/', ' ', $content);
145 | $content = trim($content);
146 |
147 | // set cutoff to allow room for other fields
148 | $cutoff = $this->config()->get('content_cutoff_bytes') - 20000;
149 |
150 | // If content is still too large, truncate it
151 | if (strlen($content) >= $cutoff) {
152 | $content = mb_strcut($content, 0, $cutoff);
153 | }
154 |
155 | return $content;
156 | }
157 | }
158 |
--------------------------------------------------------------------------------
/src/Jobs/AlgoliaReindexAllJob.php:
--------------------------------------------------------------------------------
1 | create(AlgoliaService::class);
55 | $task = new AlgoliaReindex();
56 |
57 | $this->totalSteps = 0;
58 | $this->currentStep = 0;
59 |
60 | $indexData = [];
61 |
62 | $filters = $this->config()->get('reindexing_default_filters');
63 | $batchSize = $task->config()->get('batch_size');
64 | $batching = $this->config()->get('use_batching');
65 |
66 | // find all classes we have to index and add them to the indexData map
67 | // in groups of batch size, this setup operation does the heavy lifting
68 | // and process simply handles one batch at a time.
69 | foreach ($algoliaService->indexes as $indexName => $index) {
70 | $classes = (isset($index['includeClasses'])) ? $index['includeClasses'] : null;
71 | $indexFilters = (isset($index['includeFilters'])) ? $index['includeFilters'] : null;
72 |
73 | if ($classes) {
74 | foreach ($classes as $class) {
75 | $filter = (isset($filters[$class])) ? $filters[$class] : '';
76 | $ids = $task->getItems($class, $filter, $indexFilters)->column('ID');
77 | if (count($ids)) {
78 | if ($batching && $batchSize > 1) {
79 | foreach (array_chunk($ids, $batchSize) as $chunk) {
80 | $indexData[] = [
81 | 'indexName' => $indexName,
82 | 'class' => $class,
83 | 'ids' => $chunk,
84 | ];
85 | }
86 | } else {
87 | foreach ($ids as $id) {
88 | $indexData[] = [
89 | 'indexName' => $indexName,
90 | 'class' => $class,
91 | 'id' => $id,
92 | ];
93 | }
94 | }
95 | $this->addMessage('[' . $indexName . '] Indexing ' . count($ids) . ' ' . $class . ' instances with filters: ' . ($filter ?: '(none)'));
96 | } else {
97 | $this->addMessage('[' . $indexName . '] 0 ' . $class . ' instances to index with filters: ' . ($filter ?: '(none) - skipping.'));
98 | }
99 | }
100 | }
101 | }
102 | $this->totalSteps += count($indexData);
103 | // Store in jobData to get written to the job descriptor in DB
104 | if (!$this->jobData) {
105 | $this->jobData = new stdClass();
106 | }
107 | $this->jobData->IndexData = $indexData;
108 | }
109 |
110 | /**
111 | * Index data is an array of steps to process, each step either looks like this with batching:
112 | * [
113 | * 'indexName' => string,
114 | * 'class' => string,
115 | * 'ids' => array of int,
116 | * ]
117 | * or this without batching:
118 | * [
119 | * 'indexName' => string,
120 | * 'class' => string,
121 | * 'id' => int,
122 | * ]
123 | * We process one step / batch / id per call.
124 | */
125 | public function process()
126 | {
127 | if ($this->currentStep >= $this->totalSteps) {
128 | $this->isComplete = true;
129 | $this->addMessage('Done!');
130 | return;
131 | }
132 | $indexData = isset($this->jobData->IndexData) ? $this->jobData->IndexData : null;
133 | if (!isset($indexData[$this->currentStep])) {
134 | $this->isComplete = true;
135 | $this->addMessage('Somehow we ran out of job data before all steps were processed. So we will assume we are done!');
136 | $this->addMessage('Dumping out the jop data for debug purposes: ' . json_encode($indexData));
137 | return;
138 | }
139 |
140 | $stepData = $indexData[$this->currentStep];
141 | $class = $stepData['class'];
142 |
143 | try {
144 | $task = new AlgoliaReindex();
145 |
146 | if (isset($stepData['ids'])) {
147 | $summary = $task->indexItems($stepData['indexName'], DataObject::get($class)->filter('ID', $stepData['ids']), false);
148 | $this->addMessage($summary);
149 | } else {
150 | $item = DataObject::get($class)->byID($stepData['id']);
151 | if ($item) {
152 | if (min($item->invokeWithExtensions('canIndexInAlgolia')) === false) {
153 | $this->addMessage('Skipped indexing ' . $class . ' ' . $item->ID);
154 | } elseif ($task->indexItem($item)) {
155 | $this->addMessage('Successfully indexed ' . $class . ' ' . $item->ID);
156 | } else {
157 | $this->addMessage('Error indexing ' . $class . ' ' . $item->ID);
158 | }
159 | } else {
160 | $this->addMessage('Error indexing ' . $class . ' ' . $stepData['id'] . ' - failed to load item from DB');
161 | }
162 | }
163 |
164 | $errors = $task->getErrors();
165 | } catch (Throwable $e) {
166 | $errors[] = $e->getMessage();
167 | }
168 |
169 | if (!empty($errors)) {
170 | $this->addMessage(implode(', ', $errors));
171 | $task->clearErrors();
172 | }
173 |
174 | $this->currentStep++;
175 | }
176 | }
177 |
--------------------------------------------------------------------------------
/src/Service/AlgoliaService.php:
--------------------------------------------------------------------------------
1 | client) {
38 | if (!$this->adminApiKey) {
39 | throw new Exception('No adminApiKey configured for ' . self::class);
40 | }
41 |
42 | if (!$this->applicationId) {
43 | throw new Exception('No applicationId configured for ' . self::class);
44 | }
45 |
46 | $this->client = SearchClient::create(
47 | $this->applicationId,
48 | $this->adminApiKey
49 | );
50 | }
51 |
52 | return $this->client;
53 | }
54 |
55 | /**
56 | * @return \Algolia\AlgoliaSearch\SearchClient
57 | */
58 | public function getSearchClient()
59 | {
60 | if (!$this->client) {
61 | if (!$this->searchApiKey) {
62 | throw new Exception('No searchApiKey configured for ' . self::class);
63 | }
64 |
65 | if (!$this->applicationId) {
66 | throw new Exception('No applicationId configured for ' . self::class);
67 | }
68 |
69 | $this->client = SearchClient::create(
70 | $this->applicationId,
71 | $this->searchApiKey
72 | );
73 | }
74 |
75 | return $this->client;
76 | }
77 |
78 | public function getIndexes($excludeReplicas = true)
79 | {
80 | if (!$excludeReplicas) {
81 | return $this->indexes;
82 | }
83 |
84 | $replicas = [];
85 | $output = [];
86 |
87 | foreach ($this->indexes as $indexName => $data) {
88 | if (isset($data['indexSettings']) && isset($data['indexSettings']['replicas'])) {
89 | foreach ($data['indexSettings']['replicas'] as $replicaName) {
90 | $replicas[$replicaName] = $replicaName;
91 | }
92 | }
93 | }
94 |
95 | foreach ($this->indexes as $indexName => $data) {
96 | if (in_array($indexName, $replicas)) {
97 | continue;
98 | }
99 |
100 | $output[$indexName] = $data;
101 | }
102 |
103 | return $output;
104 | }
105 |
106 |
107 | public function getIndexByName($name)
108 | {
109 | $indexes = $this->initIndexes();
110 |
111 | if (!isset($indexes[$name])) {
112 | throw new Exception(sprintf(
113 | 'Index ' . $name . ' not found, must be one of [%s]',
114 | implode(', ', array_keys($indexes))
115 | ));
116 | }
117 |
118 | return $indexes[$name];
119 | }
120 |
121 |
122 | /**
123 | * Returns an array of all the indexes which need the given item or item
124 | * class. If no item provided, returns a list of all the indexes defined.
125 | *
126 | * @param DataObject|string|null $item
127 | * @param bool $excludeReplicas
128 | *
129 | * @return \Algolia\AlgoliaSearch\SearchIndex[]
130 | */
131 | public function initIndexes($item = null, $excludeReplicas = true)
132 | {
133 | if (!Security::database_is_ready()) {
134 | return [];
135 | }
136 |
137 | try {
138 | $client = $this->getClient();
139 |
140 | if (!$client) {
141 | return [];
142 | }
143 | } catch (Throwable $e) {
144 | Injector::inst()->get(LoggerInterface::class)->error($e);
145 |
146 | if (Director::isDev()) {
147 | Debug::message($e->getMessage());
148 | }
149 |
150 | return [];
151 | }
152 | if (!$item) {
153 | if ($this->preloadedIndexes) {
154 | return $this->preloadedIndexes;
155 | }
156 |
157 | $indexes = $this->getIndexes($excludeReplicas);
158 |
159 | $this->preloadedIndexes = [];
160 |
161 | foreach ($indexes as $indexName => $data) {
162 | $this->preloadedIndexes[$indexName] = $client->initIndex($this->environmentizeIndex($indexName));
163 | }
164 |
165 | return $this->preloadedIndexes;
166 | }
167 |
168 | if (is_string($item)) {
169 | $item = Injector::inst()->get($item);
170 | } elseif (is_array($item)) {
171 | $item = Injector::inst()->get($item['objectClassName']);
172 | }
173 |
174 | $matches = [];
175 |
176 | $replicas = [];
177 |
178 | foreach ($this->indexes as $indexName => $data) {
179 | $classes = (isset($data['includeClasses'])) ? $data['includeClasses'] : null;
180 | $filter = (isset($data['includeFilter'])) ? $data['includeFilter'] : null;
181 |
182 | if ($classes) {
183 | foreach ($classes as $candidate) {
184 | if ($item instanceof $candidate) {
185 | if (method_exists($item, 'shouldIncludeInIndex') && !$item->shouldIncludeInIndex($indexName)) {
186 | continue;
187 | }
188 |
189 | if ($filter && isset($filter[$candidate])) {
190 | // check to see if this item matches the filter.
191 | $check = $candidate::get()->filter([
192 | 'ID' => $item->ID,
193 | ])->where($filter[$candidate])->first();
194 |
195 | if (!$check) {
196 | continue;
197 | }
198 | }
199 |
200 | $matches[] = $indexName;
201 |
202 | break;
203 | }
204 | }
205 | }
206 |
207 | if (isset($data['indexSettings']) && isset($data['indexSettings']['replicas'])) {
208 | foreach ($data['indexSettings']['replicas'] as $replicaName) {
209 | $replicas[$replicaName] = $replicaName;
210 | }
211 | }
212 | }
213 |
214 | $output = [];
215 |
216 | foreach ($matches as $index) {
217 | if (in_array($index, array_keys($replicas)) && $excludeReplicas) {
218 | continue;
219 | }
220 |
221 | $output[$index] = $client->initIndex($this->environmentizeIndex($index));
222 | }
223 |
224 | return $output;
225 | }
226 |
227 | /**
228 | * Prefixes the given indexName with the configured prefix, or environment
229 | * type.
230 | *
231 | * @param string $indexName
232 | *
233 | * @return string
234 | */
235 | public function environmentizeIndex($indexName)
236 | {
237 | $prefix = Environment::getEnv('ALGOLIA_PREFIX_INDEX_NAME');
238 |
239 | if ($prefix === false) {
240 | $prefix = Director::get_environment_type();
241 | }
242 |
243 | return sprintf("%s_%s", $prefix, $indexName);
244 | }
245 |
246 |
247 | /**
248 | * Sync setting from YAML configuration into Algolia.
249 | *
250 | * This runs automatically on dev/build operations.
251 | */
252 | public function syncSettings(): bool
253 | {
254 | $config = $this->indexes;
255 |
256 | if (!$config) {
257 | return false;
258 | }
259 |
260 | foreach ($config as $index => $data) {
261 | $indexName = $this->environmentizeIndex($index);
262 |
263 | if (isset($data['indexSettings'])) {
264 | $index = $this->getClient()->initIndex($indexName);
265 |
266 | if ($index) {
267 | try {
268 | // update any replica indexes with the environment
269 | if (isset($data['indexSettings']['replicas'])) {
270 | $data['indexSettings']['replicas'] = array_map(
271 | function ($replica) {
272 | return Director::get_environment_type() . '_' . $replica;
273 | },
274 | $data['indexSettings']['replicas']
275 | );
276 | }
277 |
278 | $index->setSettings($data['indexSettings']);
279 | } catch (Throwable $e) {
280 | Injector::inst()->get(LoggerInterface::class)->error($e);
281 |
282 |
283 | return false;
284 | }
285 | }
286 | }
287 | }
288 |
289 |
290 | return true;
291 | }
292 | }
293 |
--------------------------------------------------------------------------------
/src/Extensions/AlgoliaObjectExtension.php:
--------------------------------------------------------------------------------
1 | 'Datetime',
47 | 'AlgoliaError' => 'Varchar(200)',
48 | 'AlgoliaUUID' => 'Varchar(200)'
49 | ];
50 |
51 | /**
52 | * @return bool
53 | */
54 | public function indexEnabled(): bool
55 | {
56 | return $this->config()->get('enable_indexer') ? true : false;
57 | }
58 |
59 |
60 |
61 | /**
62 | * @param FieldList
63 | */
64 | public function updateCMSFields(FieldList $fields)
65 | {
66 | $fields->removeByName('AlgoliaIndexed');
67 | $fields->removeByName('AlgoliaUUID');
68 | $fields->removeByName('AlgoliaError');
69 | }
70 |
71 |
72 | /**
73 | * @param FieldList
74 | */
75 | public function updateSettingsFields(FieldList $fields)
76 | {
77 | if ($this->owner->indexEnabled()) {
78 | $fields->addFieldsToTab(
79 | 'Root.Search',
80 | [
81 | ReadonlyField::create('AlgoliaIndexed', _t(__CLASS__ . '.LastIndexed', 'Last indexed in Algolia'))
82 | ->setDescription($this->owner->AlgoliaError),
83 | ReadonlyField::create('AlgoliaUUID', _t(__CLASS__ . '.UUID', 'Algolia UUID'))
84 | ]
85 | );
86 | }
87 | }
88 |
89 | /**
90 | * Returns whether this object should be indexed into Algolia.
91 | */
92 | public function canIndexInAlgolia(): bool
93 | {
94 | if ($this->owner->hasField('ShowInSearch')) {
95 | return $this->owner->ShowInSearch;
96 | }
97 |
98 | return true;
99 | }
100 |
101 | /**
102 | * When publishing the page, push this data to Algolia Indexer. The data
103 | * which is sent to Algolia is the rendered template from the front end.
104 | */
105 | public function onAfterPublish()
106 | {
107 | if (min($this->owner->invokeWithExtensions('canIndexInAlgolia')) == false) {
108 | $this->owner->removeFromAlgolia();
109 | } else {
110 | // check to see if the classname changed, if it has then it might
111 | // need to be removed from other indexes before being re-added
112 | if ($this->owner->isChanged('ClassName')) {
113 | $this->owner->removeFromAlgolia();
114 | }
115 |
116 | $this->owner->indexInAlgolia();
117 | }
118 | }
119 |
120 | /**
121 | *
122 | */
123 | public function markAsRemovedFromAlgoliaIndex()
124 | {
125 | $this->touchAlgoliaIndexedDate(true);
126 |
127 | return $this->owner;
128 | }
129 |
130 | /**
131 | * Update the AlgoliaIndexed date for this object.
132 | */
133 | public function touchAlgoliaIndexedDate($isDeleted = false)
134 | {
135 | $newValue = $isDeleted ? 'null' : DB::get_conn()->now();
136 |
137 | $this->updateAlgoliaFields([
138 | 'AlgoliaIndexed' => $newValue,
139 | 'AlgoliaUUID' => "'" . $this->owner->AlgoliaUUID . "'",
140 | ]);
141 |
142 | return $this->owner;
143 | }
144 |
145 | /**
146 | * Update search metadata without triggering draft state etc
147 | */
148 | private function updateAlgoliaFields($fields)
149 | {
150 | $schema = DataObject::getSchema();
151 | $table = $schema->tableForField($this->owner->ClassName, 'AlgoliaIndexed');
152 |
153 | if ($table && count($fields)) {
154 | $sets = [];
155 |
156 | foreach ($fields as $field => $value) {
157 | $sets[] = "$field = $value";
158 | }
159 |
160 | $set = implode(', ', $sets);
161 | $query = sprintf('UPDATE %s SET %s WHERE ID = %s', $table, $set, $this->owner->ID);
162 |
163 | DB::query($query);
164 |
165 | if ($this->owner->hasExtension(Versioned::class) && $this->owner->hasStages()) {
166 | DB::query(
167 | sprintf(
168 | 'UPDATE %s_Live SET %s WHERE ID = %s',
169 | $table,
170 | $set,
171 | $this->owner->ID
172 | )
173 | );
174 | }
175 | }
176 | }
177 |
178 | /**
179 | * Index this record into Algolia or queue if configured to do so
180 | *
181 | * @return bool
182 | */
183 | public function indexInAlgolia(): bool
184 | {
185 | if ($this->owner->indexEnabled() && min($this->owner->invokeWithExtensions('canIndexInAlgolia')) == false) {
186 | return false;
187 | }
188 |
189 | if ($this->config()->get('use_queued_indexing')) {
190 | $indexJob = new AlgoliaIndexItemJob($this->owner->ClassName, $this->owner->ID);
191 | QueuedJobService::singleton()->queueJob($indexJob);
192 |
193 | return true;
194 | } else {
195 | return $this->doImmediateIndexInAlgolia();
196 | }
197 | }
198 |
199 | /**
200 | * Index this record into Algolia
201 | *
202 | * @return bool
203 | */
204 | public function doImmediateIndexInAlgolia(): bool
205 | {
206 | if ($this->owner->indexEnabled() && min($this->owner->invokeWithExtensions('canIndexInAlgolia')) == false) {
207 | return false;
208 | }
209 |
210 |
211 | $schema = DataObject::getSchema();
212 | $table = $schema->tableForField($this->owner->ClassName, 'AlgoliaError');
213 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
214 |
215 | try {
216 | if ($indexer->indexItem($this->owner)) {
217 | $this->touchAlgoliaIndexedDate();
218 |
219 | DB::query(
220 | sprintf(
221 | 'UPDATE %s SET AlgoliaError = \'\' WHERE ID = %s',
222 | $table,
223 | $this->owner->ID
224 | )
225 | );
226 |
227 | return true;
228 | } else {
229 | return false;
230 | }
231 | } catch (Throwable $e) {
232 | Injector::inst()->get(LoggerInterface::class)->error($e);
233 |
234 | DB::query(
235 | sprintf(
236 | 'UPDATE %s SET AlgoliaError = \'%s\' WHERE ID = %s',
237 | $table,
238 | Convert::raw2sql($e->getMessage()),
239 | $this->owner->ID
240 | )
241 | );
242 |
243 | $this->owner->AlgoliaError = $e->getMessage();
244 | }
245 |
246 | return false;
247 | }
248 |
249 | /**
250 | * When unpublishing this item, remove from Algolia
251 | */
252 | public function onAfterUnpublish()
253 | {
254 | if ($this->owner->indexEnabled()) {
255 | $this->removeFromAlgolia();
256 | }
257 | }
258 |
259 | /**
260 | * Remove this item from Algolia
261 | *
262 | * @return boolean
263 | */
264 | public function removeFromAlgolia(): bool
265 | {
266 | if (!$this->owner->AlgoliaUUID) {
267 | // Not in the index, so skipping
268 | return false;
269 | }
270 |
271 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
272 |
273 | if ($this->config()->get('use_queued_indexing')) {
274 | $indexDeleteJob = new AlgoliaDeleteItemJob($this->owner->getClassName(), $this->owner->AlgoliaUUID);
275 | QueuedJobService::singleton()->queueJob($indexDeleteJob);
276 |
277 | $this->markAsRemovedFromAlgoliaIndex();
278 | } else {
279 | try {
280 | $indexer->deleteItem($this->owner->getClassName(), $this->owner->AlgoliaUUID);
281 |
282 | $this->markAsRemovedFromAlgoliaIndex();
283 | } catch (Throwable $e) {
284 | Injector::inst()->get(LoggerInterface::class)->error($e);
285 |
286 | return false;
287 | }
288 | }
289 | return true;
290 | }
291 |
292 | public function onBeforeWrite()
293 | {
294 | if (!$this->owner->AlgoliaUUID) {
295 | $this->owner->assignAlgoliaUUID(false);
296 | }
297 | }
298 |
299 | public function assignAlgoliaUUID($writeImmediately = true)
300 | {
301 | $uuid = Uuid::uuid4();
302 | $value = $uuid->toString();
303 |
304 | $this->owner->AlgoliaUUID = $value;
305 |
306 | if ($writeImmediately) {
307 | $this->updateAlgoliaFields(['AlgoliaUUID' => "'$value'"]);
308 | }
309 | }
310 |
311 | /**
312 | * Before deleting this record ensure that it is removed from Algolia.
313 | */
314 | public function onBeforeDelete()
315 | {
316 | if ($this->owner->indexEnabled()) {
317 | $this->removeFromAlgolia();
318 | }
319 | }
320 |
321 | /**
322 | * Ensure each record has unique UUID
323 | */
324 | public function onBeforeDuplicate()
325 | {
326 | $this->owner->assignAlgoliaUUID(false);
327 | $this->owner->AlgoliaIndexed = null;
328 | $this->owner->AlgoliaError = null;
329 | }
330 |
331 | /**
332 | * @return array
333 | */
334 | public function getAlgoliaIndexes()
335 | {
336 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
337 |
338 | return $indexer->getService()->initIndexes($this->owner);
339 | }
340 | }
341 |
--------------------------------------------------------------------------------
/src/Tasks/AlgoliaReindex.php:
--------------------------------------------------------------------------------
1 | config()->get('reindexing_default_filters');
55 |
56 | if ($input->getOption('only')) {
57 | $targetClass = $input->getOption('only');
58 |
59 | if ($defaultFilters && isset($defaultFilters[$targetClass])) {
60 | $filter = $defaultFilters[$targetClass];
61 | }
62 | }
63 |
64 | if ($input->getOption('filter')) {
65 | $filter = $input->getOption('filter');
66 | }
67 |
68 | if (!$input->getOption('force') && !$filter) {
69 | $filter = 'AlgoliaIndexed IS NULL';
70 | }
71 |
72 | if ($input->getOption('subsite')) {
73 | $subsite = $input->getOption('subsite');
74 | }
75 |
76 | /** @var AlgoliaService */
77 | $algoliaService = Injector::inst()->create(AlgoliaService::class);
78 |
79 | if ($input->getOption('clear')) {
80 | $indexes = $algoliaService->initIndexes();
81 |
82 | foreach ($indexes as $indexName => $index) {
83 | $index->clearObjects();
84 | }
85 | }
86 |
87 | // find all classes we have to index and do so
88 | foreach ($algoliaService->indexes as $indexName => $index) {
89 | $environmentizedIndexName = $algoliaService->environmentizeIndex($indexName);
90 |
91 | $output->writeln('Updating index ' . $environmentizedIndexName);
92 |
93 | $classes = (isset($index['includeClasses'])) ? $index['includeClasses'] : null;
94 | $indexFilters = (isset($index['includeFilter'])) ? $index['includeFilter'] : [];
95 |
96 | if ($classes) {
97 | foreach ($classes as $candidate) {
98 | if ($targetClass && $targetClass !== $candidate) {
99 | // check to see if target class is a subclass of the candidate
100 | if (!is_subclass_of($targetClass, $candidate)) {
101 | continue;
102 | } else {
103 | $candidate = $targetClass;
104 | }
105 | }
106 |
107 |
108 | $items = $this->getItems($candidate, $filter, $indexFilters);
109 |
110 | if (!$subsite) {
111 | $items = $items->setDataQueryParam('Subsite.filter', false);
112 | }
113 |
114 | $filterLabel = implode(',', array_filter(array_merge([$filter], [$indexFilters[$candidate] ?? ''])));
115 |
116 | $output->writeln(sprintf(
117 | '| Found %s %s remaining to index %s',
118 | $items->count(),
119 | $candidate,
120 | $filterLabel ? 'which match filters ' . $filterLabel : ''
121 | ));
122 |
123 | if ($items->exists()) {
124 | $this->indexItems($indexName, $items, $output);
125 | }
126 | }
127 | }
128 | }
129 |
130 | return Command::SUCCESS;
131 | }
132 |
133 | public function getOptions(): array
134 | {
135 | return [
136 | new InputOption('only', null, InputOption::VALUE_OPTIONAL, 'Only index objects of this class'),
137 | new InputOption('filter', null, InputOption::VALUE_OPTIONAL, 'Filter to apply when fetching objects'),
138 | new InputOption('force', null, InputOption::VALUE_NONE, 'Force indexing of all objects'),
139 | new InputOption('subsite', null, InputOption::VALUE_OPTIONAL, 'Only index objects from this subsite'),
140 | new InputOption('clear', null, InputOption::VALUE_NONE, 'Clear all indexes before reindexing'),
141 | ];
142 | }
143 |
144 | /**
145 | * @param string $targetClass
146 | * @param string $filter
147 | * @param string[] $indexFilters
148 | *
149 | * @return \SilverStripe\ORM\DataList
150 | */
151 | public function getItems($targetClass, $filter = '', $indexFilters = [])
152 | {
153 | $inst = $targetClass::create();
154 |
155 | if ($inst->hasExtension(Versioned::class)) {
156 | $items = Versioned::get_by_stage($targetClass, 'Live', $filter);
157 | } else {
158 | $items = $inst::get();
159 |
160 | if ($filter) {
161 | $items = $items->where($filter);
162 | }
163 | }
164 |
165 | if (isset($indexFilters[$targetClass])) {
166 | $items = $items->where($indexFilters[$targetClass]);
167 | }
168 |
169 |
170 | return $items;
171 | }
172 |
173 |
174 | /**
175 | * @param DataObject $obj
176 | *
177 | * @return bool
178 | */
179 | public function indexItem($obj = null): bool
180 | {
181 | if (!$obj) {
182 | return false;
183 | } elseif (min($obj->invokeWithExtensions('canIndexInAlgolia')) === false) {
184 | return false;
185 | } else {
186 | if (!$obj->AlgoliaUUID) {
187 | $obj->assignAlgoliaUUID();
188 | }
189 |
190 | if ($obj->doImmediateIndexInAlgolia()) {
191 | return true;
192 | } else {
193 | return false;
194 | }
195 | }
196 | }
197 |
198 |
199 | /**
200 | * @param string $indexName
201 | * @param DataList? $items
202 | * @param PolyOutput $output;
203 | *
204 | * @return bool|string
205 | */
206 | public function indexItems($indexName, $items, PolyOutput $output)
207 | {
208 | $algoliaService = Injector::inst()->get(AlgoliaService::class);
209 | $count = 0;
210 | $skipped = 0;
211 | $total = ($items) ? $items->count() : 0;
212 | $batchSize = $this->config()->get('batch_size') ?? 25;
213 | $batchesTotal = ($total > 0) ? (ceil($total / $batchSize)) : 0;
214 | $indexer = Injector::inst()->get(AlgoliaIndexer::class);
215 | $pos = 0;
216 |
217 | if ($total < 1) {
218 | return false;
219 | }
220 |
221 | $currentBatches = [];
222 |
223 | for ($i = 0; $i < $batchesTotal; $i++) {
224 | $limitedSize = $items->sort('ID', 'DESC')->limit($batchSize, $i * $batchSize);
225 |
226 | foreach ($limitedSize as $item) {
227 | $pos++;
228 |
229 | if ($output) {
230 | if ($pos % 50 == 0) {
231 | $output->writeln(sprintf('[%s/%s]', $pos, $total));
232 | } else {
233 | $output->write('.');
234 | }
235 | }
236 |
237 | // fetch the actual instance
238 | $instance = DataObject::get_by_id($item->ClassName, $item->ID);
239 |
240 | if (!$instance || min($instance->invokeWithExtensions('canIndexInAlgolia')) == false) {
241 | $skipped++;
242 |
243 | continue;
244 | }
245 |
246 | // Set AlgoliaUUID, in case it wasn't previously set
247 | if (!$item->AlgoliaUUID) {
248 | $item->assignAlgoliaUUID();
249 | }
250 |
251 | $batchKey = get_class($item);
252 |
253 | if (!isset($currentBatches[$batchKey])) {
254 | $currentBatches[$batchKey] = [];
255 | }
256 |
257 | try {
258 | $data = $indexer->exportAttributesFromObject($item);
259 |
260 | if ($data instanceof Map) {
261 | $data = $data->toArray();
262 | }
263 |
264 | $currentBatches[$batchKey][] = $data;
265 | $item->touchAlgoliaIndexedDate();
266 | $count++;
267 | } catch (Throwable $e) {
268 | Injector::inst()->get(LoggerInterface::class)->warning($e->getMessage());
269 | }
270 |
271 | if (count($currentBatches[$batchKey]) >= $batchSize) {
272 | $this->indexBatch($indexName, $currentBatches[$batchKey]);
273 |
274 | unset($currentBatches[$batchKey]);
275 | }
276 |
277 | if ($output) {
278 | sleep(1);
279 | }
280 | }
281 | }
282 |
283 | foreach ($currentBatches as $class => $records) {
284 | if (count($currentBatches[$class]) > 0) {
285 | $this->indexBatch($indexName, $currentBatches[$class]);
286 |
287 | if ($output) {
288 | sleep(1);
289 | }
290 | }
291 | }
292 |
293 | $summary = sprintf(
294 | "%sNumber of objects indexed in %s: %s, Skipped %s",
295 | PHP_EOL,
296 | $indexName,
297 | $count,
298 | $skipped
299 | );
300 |
301 | if ($output) {
302 | $output->writeln($summary);
303 |
304 | $output->writeln(sprintf(
305 | "See index at " .
306 | "algolia.com/apps/%s/explorer/indices",
307 | $algoliaService->applicationId,
308 | $algoliaService->applicationId
309 | ));
310 | }
311 |
312 | return $summary;
313 | }
314 |
315 | /**
316 | * Index a batch of changes
317 | *
318 | * @param array $items
319 | *
320 | * @return bool
321 | */
322 | public function indexBatch($indexName, $items): bool
323 | {
324 | $service = Injector::inst()->create(AlgoliaService::class);
325 | $index = $service->getIndexByName($indexName);
326 |
327 | try {
328 | $result = $index->saveObjects($items, [
329 | 'autoGenerateObjectIDIfNotExist' => true
330 | ]);
331 |
332 | if (!$result->valid()) {
333 | return false;
334 | }
335 |
336 | return true;
337 | } catch (Throwable $e) {
338 | Injector::inst()->get(LoggerInterface::class)->error($e);
339 |
340 | if (Director::isDev()) {
341 | Debug::message($e->getMessage());
342 | }
343 |
344 | $this->errors[] = $e->getMessage();
345 |
346 | return false;
347 | }
348 | }
349 |
350 | /**
351 | * @return string[]
352 | */
353 | public function getErrors()
354 | {
355 | return $this->errors;
356 | }
357 |
358 | /**
359 | * @return $this
360 | */
361 | public function clearErrors()
362 | {
363 | $this->errors = [];
364 |
365 | return $this;
366 | }
367 | }
368 |
--------------------------------------------------------------------------------
/src/Service/AlgoliaIndexer.php:
--------------------------------------------------------------------------------
1 | getService()->initIndexes($item);
70 |
71 | try {
72 | $fields = $this->exportAttributesFromObject($item);
73 | } catch (Exception $e) {
74 | Injector::inst()->get(LoggerInterface::class)->error($e);
75 |
76 | return false;
77 | }
78 |
79 | if (method_exists($fields, 'toArray')) {
80 | $fields = $fields->toArray();
81 | }
82 |
83 | if ($searchIndexes) {
84 | $output = true;
85 | foreach ($searchIndexes as $searchIndex) {
86 | $result = $searchIndex->saveObject($fields, [
87 | 'autoGenerateObjectIDIfNotExist' => true
88 | ]);
89 |
90 | if (!$result->valid()) {
91 | $output = false;
92 | }
93 | }
94 |
95 | return $output;
96 | }
97 |
98 | return false;
99 | }
100 |
101 |
102 | public function getService(): AlgoliaService
103 | {
104 | return Injector::inst()->get(AlgoliaService::class);
105 | }
106 |
107 | /**
108 | * Index multiple items of the same class at a time.
109 | */
110 | public function indexItems(DataList $items): self
111 | {
112 | $sample = $items->first();
113 | $searchIndexes = $this->getService()->initIndexes($sample);
114 | $data = [];
115 |
116 | foreach ($items as $item) {
117 | $data[] = $this->exportAttributesFromObject($item)->toArray();
118 | }
119 |
120 | foreach ($searchIndexes as $searchIndex) {
121 | $searchIndex->saveObjects($data);
122 | }
123 |
124 | return $this;
125 | }
126 |
127 | /**
128 | * Generates a map of all the fields and values which will be sent. Two ways
129 | * to modifty the attributes sent to algolia. Either define the properties
130 | * via the config API
131 | *
132 | * ```
133 | * private static $algolia_index_fields = [
134 | * 'MyCustomField'
135 | * ];
136 | * ```
137 | *
138 | * Or, use exportObjectToAlgolia to return an Map. You can chose to include
139 | * the default fields or not.
140 | *
141 | * ```
142 | * class MyObject extends DataObject
143 | * {
144 | * public function exportObjectToAlgolia($data)
145 | * {
146 | * $data = array_merge($data, [
147 | * 'MyCustomField' => $this->MyCustomField()
148 | * ]);
149 | * $map = new Map(ArrayList::create());
150 | * foreach ($data as $k => $v) {
151 | * $map->push($k, $v);
152 | * }
153 | * return $map;
154 | * }
155 | * }
156 | * ```
157 | *
158 | * @param DataObject
159 | */
160 | public function exportAttributesFromObject($item): Map
161 | {
162 | $toIndex = [
163 | 'objectID' => $item->AlgoliaUUID,
164 | 'objectSilverstripeID' => $item->ID,
165 | 'objectIndexedTimestamp' => date('c'),
166 | 'objectTitle' => (string) $item->Title,
167 | 'objectClassName' => get_class($item),
168 | 'objectClassNameHierarchy' => array_values(ClassInfo::ancestry(get_class($item))),
169 | 'objectLastEdited' => $item->dbObject('LastEdited')->getTimestamp(),
170 | 'objectCreated' => $item->dbObject('Created')->getTimestamp()
171 | ];
172 |
173 | if ($item->hasMethod('AbsoluteLink') && !empty($item->AbsoluteLink())) {
174 | $link = $item->AbsoluteLink();
175 |
176 | if (!empty($link)) {
177 | $toIndex['objectLink'] = str_replace(['?stage=Stage', '?stage=Live'], '', $link);
178 | }
179 | } elseif ($item->hasMethod('Link') && !empty($item->Link())) {
180 | $link = $item->Link();
181 |
182 | if (!empty($link)) {
183 | $toIndex['objectLink'] = str_replace(['?stage=Stage', '?stage=Live'], '', $link);
184 | }
185 | }
186 |
187 | if ($item && $item->hasMethod('exportObjectToAlgolia')) {
188 | return $item->exportObjectToAlgolia($toIndex);
189 | }
190 |
191 | if ($this->config()->get('include_page_content')) {
192 | $toIndex['objectForTemplate'] =
193 | Injector::inst()->create(AlgoliaPageCrawler::class, $item)->getMainContent();
194 | }
195 |
196 | $item->invokeWithExtensions('onBeforeAttributesFromObject');
197 |
198 | $attributes = new Map(ArrayList::create());
199 |
200 | foreach ($toIndex as $k => $v) {
201 | $attributes->push($k, $v);
202 | }
203 |
204 | $specs = $item->config()->get('algolia_index_fields');
205 |
206 | if ($specs) {
207 | $attributes = $this->addSpecsToAttributes($item, $attributes, $specs);
208 | }
209 |
210 | $item->invokeWithExtensions('updateAlgoliaAttributes', $attributes);
211 |
212 | return $attributes;
213 | }
214 |
215 |
216 | public function addSpecsToAttributes($item, $attributes, $specs)
217 | {
218 | $maxFieldSize = $this->config()->get('max_field_size_bytes');
219 |
220 | foreach ($specs as $attributeName) {
221 | if (in_array($attributeName, $this->config()->get('attributes_blacklisted'))) {
222 | continue;
223 | }
224 |
225 | // fetch the db object, or fallback to the getters but prefer
226 | // the db object
227 | try {
228 | $dbField = $item->relObject($attributeName);
229 | } catch (LogicException $e) {
230 | $dbField = $item->{$attributeName};
231 | }
232 |
233 | if (!$dbField) {
234 | continue;
235 | }
236 |
237 | if (is_string($dbField) || is_array($dbField)) {
238 | $attributes->push($attributeName, $dbField);
239 | } elseif ($dbField instanceof DBForeignKey) {
240 | $attributes->push($attributeName, $dbField->Value);
241 | } elseif ($dbField->exists() || $dbField instanceof DBBoolean) {
242 | if ($dbField instanceof RelationList || $dbField instanceof DataObject) {
243 | // has-many, many-many, has-one
244 | $this->exportAttributesFromRelationship($item, $attributeName, $attributes);
245 | } else {
246 | // db-field, if it's a date then use the timestamp since we need it
247 | $hasContent = true;
248 |
249 | switch (get_class($dbField)) {
250 | case DBDate::class:
251 | case DBDatetime::class:
252 | $value = $dbField->getTimestamp();
253 | break;
254 | case DBBoolean::class:
255 | $value = $dbField->getValue();
256 | break;
257 | case DBHTMLText::class:
258 | $fieldData = $dbField->Plain();
259 | $fieldLength = mb_strlen($fieldData, '8bit');
260 |
261 | if ($fieldLength > $maxFieldSize) {
262 | $maxIterations = 100;
263 | $i = 0;
264 |
265 | while ($hasContent && $i < $maxIterations) {
266 | $block = mb_strcut(
267 | $fieldData,
268 | $i * $maxFieldSize,
269 | $maxFieldSize - 1
270 | );
271 |
272 | if ($block) {
273 | $attributes->push($attributeName . '_Block' . $i, $block);
274 | } else {
275 | $hasContent = false;
276 | }
277 |
278 | $i++;
279 | }
280 | } else {
281 | $value = $fieldData;
282 | }
283 | break;
284 | default:
285 | $value = @$dbField->forTemplate();
286 | }
287 |
288 | if ($hasContent) {
289 | $attributes->push($attributeName, $value);
290 | }
291 | }
292 | }
293 | }
294 |
295 | return $attributes;
296 | }
297 |
298 | /**
299 | * Retrieve all the attributes from the related object that we want to add
300 | * to this record.
301 | */
302 | public function exportAttributesFromRelationship(DataObject $item, string $relationship, Map $attributes): void
303 | {
304 | try {
305 | $data = [];
306 |
307 | $related = $item->relObject($relationship);
308 |
309 | if (!$related || !$related->exists()) {
310 | return;
311 | }
312 |
313 | if (is_iterable($related)) {
314 | foreach ($related as $relatedObj) {
315 | $relationshipAttributes = new Map(ArrayList::create());
316 | $relationshipAttributes->push('objectID', $relatedObj->ID);
317 | $relationshipAttributes->push('objectTitle', $relatedObj->Title);
318 |
319 | if ($item->hasMethod('updateAlgoliaRelationshipAttributes')) {
320 | $item->updateAlgoliaRelationshipAttributes($relationshipAttributes, $relatedObj);
321 | }
322 |
323 | $data[] = $relationshipAttributes->toArray();
324 | }
325 | } else {
326 | $relationshipAttributes = new Map(ArrayList::create());
327 | $relationshipAttributes->push('objectID', $related->ID);
328 | $relationshipAttributes->push('objectTitle', $related->Title);
329 |
330 | if ($item->hasMethod('updateAlgoliaRelationshipAttributes')) {
331 | $item->updateAlgoliaRelationshipAttributes($relationshipAttributes, $related);
332 | }
333 |
334 | $data = $relationshipAttributes->toArray();
335 | }
336 |
337 | $attributes->push($relationship, $data);
338 | } catch (Throwable $e) {
339 | Injector::inst()->get(LoggerInterface::class)->error($e);
340 | }
341 | }
342 |
343 | /**
344 | * Remove an item ID from the index. As this would usually be when an object
345 | * is deleted in Silverstripe we cannot rely on the object existing.
346 | *
347 | * @param string $itemClass
348 | * @param int $itemUUID
349 | */
350 | public function deleteItem($itemClass, $itemUUID)
351 | {
352 | if (!$itemUUID) {
353 | return false;
354 | }
355 |
356 | $searchIndexes = $this->getService()->initIndexes($itemClass);
357 |
358 | foreach ($searchIndexes as $key => $searchIndex) {
359 | try {
360 | $searchIndex->deleteObject($itemUUID);
361 | } catch (Throwable $e) {
362 | // do nothing
363 | }
364 | }
365 |
366 | return true;
367 | }
368 |
369 | /**
370 | * Generates a unique ID for this item. If using a single index with
371 | * different dataobjects such as products and pages they potentially would
372 | * have the same ID. Uses the classname and the ID.
373 | *
374 | * @deprecated
375 | * @param DataObject $item
376 | *
377 | * @return string
378 | */
379 | public function generateUniqueID($item)
380 | {
381 | return strtolower(str_replace('\\', '_', get_class($item)) . '_' . $item->ID);
382 | }
383 |
384 | /**
385 | * @param DataObject $item
386 | *
387 | * @return array
388 | */
389 | public function getObject($item)
390 | {
391 | $indexes = $this->getService()->initIndexes($item);
392 |
393 | if (!$item->AlgoliaUUID) {
394 | return [];
395 | }
396 |
397 | foreach ($indexes as $index) {
398 | try {
399 | $output = $index->getObject($item->AlgoliaUUID);
400 |
401 | if ($output) {
402 | return $output;
403 | }
404 | } catch (NotFoundException $ex) {
405 | }
406 | }
407 |
408 | return [];
409 | }
410 | }
411 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # :mag: Silverstripe Algolia Module
2 |
3 | [](https://codecov.io/gh/wilr/silverstripe-algolia)
4 | [](https://packagist.org/packages/wilr/silverstripe-algolia)
5 | [](LICENSE)
6 |
7 | ## Maintainer Contact
8 |
9 | - Will Rossiter (@wilr)
10 |
11 | ## Installation
12 |
13 | ```sh
14 | composer require "wilr/silverstripe-algolia"
15 | ```
16 |
17 | ## Features
18 |
19 | :ballot_box_with_check: Supports multiple indexes and saving records into
20 | multiple indexes.
21 |
22 | :ballot_box_with_check: Integrates into existing versioned workflow.
23 |
24 | :ballot_box_with_check: No dependencies on the CMS, supports any DataObject
25 | subclass.
26 |
27 | :ballot_box_with_check: Queued job support for offloading operations to Algolia.
28 |
29 | :ballot_box_with_check: Easily configure search configuration and indexes via
30 | YAML and PHP.
31 |
32 | :ballot_box_with_check: Indexes your webpage template so supports Elemental and
33 | custom fields out of the box
34 |
35 | ## Documentation
36 |
37 | Algolia’s search-as-a-service and full suite of APIs allow teams to easily
38 | develop tailored, fast Search and Discovery experiences that delight and
39 | convert.
40 |
41 | This module adds the ability to sync Silverstripe pages to a Algolia Index.
42 |
43 | Indexing and removing documents is done transparently for any objects which
44 | subclass `SiteTree` or by applying the
45 | `Wilr\SilverStripe\Algolia\Extensions\AlgoliaObjectExtension` to your
46 | DataObjects.
47 |
48 | ## :hammer_and_wrench: Setting Up
49 |
50 | First, sign up for Algolia.com account and install this module. Once installed,
51 | Configure the API keys via YAML (environment variables recommended).
52 |
53 | _app/\_config/algolia.yml_
54 |
55 | ```yml
56 | ---
57 | Name: algolia
58 | After: silverstripe-algolia
59 | ---
60 | SilverStripe\Core\Injector\Injector:
61 | Wilr\SilverStripe\Algolia\Service\AlgoliaService:
62 | properties:
63 | adminApiKey: "`ALGOLIA_ADMIN_API_KEY`"
64 | searchApiKey: "`ALGOLIA_SEARCH_API_KEY`"
65 | applicationId: "`ALGOLIA_SEARCH_APP_ID`"
66 | indexes:
67 | IndexName:
68 | includeClasses:
69 | - SilverStripe\CMS\Model\SiteTree
70 | indexSettings:
71 | attributesForFaceting:
72 | - "filterOnly(objectClassName)"
73 | ```
74 |
75 | Once the indexes and API keys are configured, run a `dev/build` to update the
76 | database and refresh the indexSettings. Alternatively you can run
77 | `AlgoliaConfigure` to manually rebuild the indexSettings.
78 |
79 | ### Configuring the index names
80 |
81 | This module will assume your indexes are setup as `dev_{IndexName}`,
82 | `test_{IndexName}` and `live_{IndexName}` where the result of your environment
83 | type is prefixed to the names listed in the main YAML config.
84 |
85 | If you explictly want to disable the environment prefix (or use a custom
86 | approach) use the `ALGOLIA_PREFIX_INDEX_NAME` environment variable.
87 |
88 | ```yml
89 | ALGOLIA_PREFIX_INDEX_NAME='dev_will'
90 | ```
91 |
92 | Or for testing with live data on dev use `ALGOLIA_PREFIX_INDEX_NAME='live'`
93 |
94 | ### Defining Replica Indexes
95 |
96 | If your search form provides a sort option (e.g latest or relevance) then you
97 | will be using replica indexes
98 | (https://www.algolia.com/doc/guides/managing-results/refine-results/sorting/how-to/creating-replicas/)
99 |
100 | These can be defined using the same YAML configuration.
101 |
102 | ```yml
103 | ---
104 | Name: algolia
105 | After: silverstripe-algolia
106 | ---
107 | SilverStripe\Core\Injector\Injector:
108 | Wilr\SilverStripe\Algolia\Service\AlgoliaService:
109 | properties:
110 | adminApiKey: "`ALGOLIA_ADMIN_API_KEY`"
111 | searchApiKey: "`ALGOLIA_SEARCH_API_KEY`"
112 | applicationId: "`ALGOLIA_SEARCH_APP_ID`"
113 | indexes:
114 | IndexName:
115 | includeClasses:
116 | - SilverStripe\CMS\Model\SiteTree
117 | indexSettings:
118 | attributesForFaceting:
119 | - "filterOnly(ObjectClassName)"
120 | replicas:
121 | - IndexName_Latest
122 | IndexName_Latest:
123 | indexSettings:
124 | ranking:
125 | - "desc(objectCreated)"
126 | - "typo"
127 | - "words"
128 | - "filters"
129 | - "proximity"
130 | - "attribute"
131 | - "exact"
132 | - "custom"
133 | ```
134 |
135 | ## Indexing
136 |
137 | If installing on a existing website run the `AlgoliaReindex` task (via CLI) to
138 | import existing data. This will batch import all the records from your database
139 | into the indexes configured above.
140 |
141 | ```sh
142 | ./vendor/bin/sake algolia-configure
143 | ./vendor/bin/sake algolia-index
144 | ```
145 |
146 | Individually records will be indexed automatically going forward via the
147 | `onAfterPublish` hook and removed via the `onAfterUnpublish` hook which is
148 | called when publishing or unpublishing a document. If your DataObject does not
149 | implement the `Versioned` extension you'll need to manage this state yourself by
150 | calling `$item->indexInAlgolia()` and `$item->removeFromAlgolia()`.
151 |
152 | `AlgoliaReindex` takes a number of arguments to allow for customisation of bulk
153 | indexing. For instance, if you have a large amount of JobVacancies to bulk
154 | import but only need the active ones you can trigger the task as follows:
155 |
156 | ```sh
157 | /vendor/bin/sake algolia-index --onlyClass="onlyClass=Vacancy" --filter="ExpiryDate>NOW()"
158 | ```
159 |
160 | If you do not have access to a CLI (i.e Silverstripe Cloud) then you can also
161 | bulk reindex via a queued job `AlgoliaReindexAllJob`.
162 |
163 | ### Optional
164 |
165 | `force` forces every Silverstripe record to be re-synced.
166 |
167 | ```sh
168 | ./vendor/bin/sake algolia-index --force
169 | ```
170 |
171 | `clear` truncates the search index before re-indexing.
172 |
173 | ```sh
174 | ./vendor/bin/sake algolia-index --clear
175 | ```
176 |
177 | ### Customising the indexed attributes (fields)
178 |
179 | By default only `ID`, `Title` and `Link`, `LastEdited` will be indexed from each
180 | record. To specify additional fields, define a `algolia_index_fields` config
181 | variable.
182 |
183 | ```php
184 | class MyPage extends Page {
185 | // ..
186 | private static $algolia_index_fields = [
187 | 'Content',
188 | 'MyCustomColumn',
189 | 'RelationshipName'
190 | ];
191 | }
192 | ```
193 |
194 | Or, you can define a `exportObjectToAlgolia` method on your object. This
195 | receives the default index fields and then allows you to add or remove fields as
196 | required
197 |
198 | ```php
199 | use SilverStripe\Model\List\ArrayList;
200 | use SilverStripe\Model\List\Map;
201 |
202 | class MyPage extends Page {
203 |
204 | public function exportObjectToAlgolia($data)
205 | {
206 | $data = array_merge($data, [
207 | 'MyCustomField' => $this->MyCustomField()
208 | ]);
209 |
210 | $map = new Map(ArrayList::create());
211 |
212 | foreach ($data as $k => $v) {
213 | $map->push($k, $v);
214 | }
215 |
216 | return $map;
217 | }
218 | }
219 | ```
220 |
221 | ### Customizing the indexed relationships
222 |
223 | Out of the box, the default is to push the ID and Title fields of any
224 | relationships (`$has_one`, `$has_many`, `$many_many`) into a field
225 | `relation{name}` with the record `ID` and `Title` as per the behaviour with
226 | records.
227 |
228 | Additional fields from the relationship can be indexed via a PHP function
229 |
230 | ```php
231 | public function updateAlgoliaRelationshipAttributes(\SilverStripe\ORM\Map $attributes, $related)
232 | {
233 | $attributes->push('CategoryName', $related->CategoryName);
234 | }
235 | ```
236 |
237 | ### Excluding an object from indexing
238 |
239 | Objects can define a `canIndexInAlgolia` method which should return false if the
240 | object should not be indexed in algolia.
241 |
242 | ```php
243 | public function canIndexInAlgolia(): bool
244 | {
245 | return ($this->Expired) ? false : true;
246 | }
247 | ```
248 |
249 | ### Queued Indexing
250 |
251 | To reduce the impact of waiting on a third-party service while publishing
252 | changes, this module utilizes the `queued-jobs` module for uploading index
253 | operations. The queuing feature can be disabled via the Config YAML.
254 |
255 | ```yaml
256 | Wilr\SilverStripe\Algolia\Extensions\AlgoliaObjectExtension:
257 | use_queued_indexing: false
258 | ```
259 |
260 | ## Displaying and fetching results
261 |
262 | For your website front-end you can use InstantSearch.js libraries if you wish,
263 | or to fetch a `PaginatedList` of results from Algolia, create a method on your
264 | `Controller` subclass to call `Wilr\SilverStripe\Algolia\Service\AlgoliaQuerier`
265 |
266 | ```php
267 | request->getVar('start') / $hitsPerPage);
278 |
279 | $results = Injector::inst()->get(AlgoliaQuerier::class)->fetchResults(
280 | 'indexName',
281 | $this->request->getVar('search'), [
282 | 'page' => $this->request->getVar('start') ? $paginatedPageNum : 0,
283 | 'hitsPerPage' => $hitsPerPage
284 | ]
285 | );
286 |
287 | return [
288 | 'Title' => 'Search Results',
289 | 'Results' => $results
290 | ];
291 | }
292 | }
293 | ```
294 |
295 | Or alternatively you can make use of JS Search SDK
296 | (https://www.algolia.com/doc/api-client/getting-started/install/javascript/)
297 |
298 | ## :mag: Inspect Object Fields
299 |
300 | To assist with debugging what fields will be pushed into Algolia and see what
301 | information is already in Algolia use the `AlgoliaInspect` BuildTask. This can
302 | be run via CLI
303 |
304 | ```
305 | ./vendor/bin/sake dev/tasks/AlgoliaInspect "class=Page&id=1"
306 | ```
307 |
308 | Will output the Algolia data structure for the Page with the ID of '1'.
309 |
310 | ## Elemental Support
311 |
312 | Out of the box this module scrapes the webpage's `main` HTML section and stores
313 | it in a `objectForTemplate` field in Algolia. This content is parsed via the
314 | `AlgoliaPageCrawler` class.
315 |
316 | ```html
317 |
318 | $ElementalArea
319 |
320 |
321 | ```
322 |
323 | If this behaviour is undesirable then it can be disabled via YAML.
324 |
325 | ```
326 | Wilr\SilverStripe\Algolia\Service\AlgoliaIndexer:
327 | include_page_content: false
328 | ```
329 |
330 | Or you can specify the HTML selector you do want to index using YAML. For
331 | instance to index any elements with a `data-index` attribute.
332 |
333 | ```
334 | Wilr\SilverStripe\Algolia\Service\AlgoliaPageCrawler:
335 | content_xpath_selector: '//[data-index]'
336 | ```
337 |
338 | ## Subsite support
339 |
340 | If you use the Silverstripe Subsite module to run multiple websites you can
341 | handle indexing in a couple ways:
342 |
343 | - Use separate indexes per site.
344 | - Use a single index, but add a `SubsiteID` field in Algolia.
345 |
346 | The decision to go either way depends on the nature of the websites and how
347 | related they are but separate indexes are highly recommended to prevent leaking
348 | information between websites and mucking up analytics and query suggestions.
349 |
350 | ### Subsite support with a single index
351 |
352 | If subsites are frequently being created then you may choose to prefer a single
353 | index since index names need to be controlled via YAML so any new subsite would
354 | require a code change.
355 |
356 | The key to this approach is added `SubsiteID` to the attributes for faceting
357 | and at the query time.
358 |
359 | Step 1. Add the field to Algolia
360 |
361 | ```yml
362 | SilverStripe\Core\Injector\Injector:
363 | Wilr\SilverStripe\Algolia\Service\AlgoliaService:
364 | properties:
365 | adminApiKey: "`ALGOLIA_ADMIN_API_KEY`"
366 | searchApiKey: "`ALGOLIA_SEARCH_API_KEY`"
367 | applicationId: "`ALGOLIA_SEARCH_APP_ID`"
368 | indexes:
369 | index_main_site:
370 | includeClasses:
371 | - SilverStripe\CMS\Model\SiteTree
372 | indexSettings:
373 | distinct: true
374 | attributeForDistinct: "objectLink"
375 | searchableAttributes:
376 | - objectTitle
377 | - objectContent
378 | - objectLink
379 | - Summary
380 | - objectForTemplate
381 | attributesForFaceting:
382 | - "filterOnly(objectClassName)"
383 | ***- "filterOnly(SubsiteID)"***
384 | ```
385 |
386 | Step 2. Expose the field on `SiteTree` via a DataExtension (make sure to apply the extension)
387 |
388 | ```php
389 | class SiteTreeExtension extends DataExtension
390 | {
391 | private static $algolia_index_fields = [
392 | 'SubsiteID'
393 | ];
394 | }
395 | ```
396 |
397 | Step 3. Filter by the Subsite ID in your results
398 |
399 | ```php
400 | use SilverStripe\Core\Injector\Injector;
401 | use Wilr\SilverStripe\Algolia\Service\AlgoliaQuerier;
402 |
403 | class PageController extends ContentController
404 | {
405 | public function results()
406 | {
407 | $hitsPerPage = 25;
408 | $paginatedPageNum = floor($this->request->getVar('start') / $hitsPerPage);
409 |
410 | $results = Injector::inst()->get(AlgoliaQuerier::class)->fetchResults(
411 | 'indexName',
412 | $this->request->getVar('search'), [
413 | 'page' => $this->request->getVar('start') ? $paginatedPageNum : 0,
414 | 'hitsPerPage' => $hitsPerPage,
415 | 'facetFilters' => [
416 | 'SubsiteID' => SubsiteState::singleton()->getSubsiteId()
417 | ]
418 | ]
419 | );
420 |
421 | return [
422 | 'Title' => 'Search Results',
423 | 'Results' => $results
424 | ];
425 | }
426 | }
427 | ```
428 |
429 | ### Subsite support with separate indexes
430 |
431 | Create multiple indexes in your config and use the `includeFilter` parameter to
432 | filter the records per index.
433 |
434 | The `includeFilter` should be in the format `{$Class}`: `{$WhereQuery}` where
435 | the `$WhereQuery` is a basic SQL statement performed by the ORM on the given
436 | class.
437 |
438 | ```yml
439 | SilverStripe\Core\Injector\Injector:
440 | Wilr\SilverStripe\Algolia\Service\AlgoliaService:
441 | properties:
442 | adminApiKey: "`ALGOLIA_ADMIN_API_KEY`"
443 | searchApiKey: "`ALGOLIA_SEARCH_API_KEY`"
444 | applicationId: "`ALGOLIA_SEARCH_APP_ID`"
445 | indexes:
446 | index_main_site:
447 | includeClasses:
448 | - SilverStripe\CMS\Model\SiteTree
449 | includeFilter:
450 | "SilverStripe\\CMS\\Model\\SiteTree": "SubsiteID = 0"
451 | indexSettings:
452 | distinct: true
453 | attributeForDistinct: "objectLink"
454 | searchableAttributes:
455 | - objectTitle
456 | - objectContent
457 | - objectLink
458 | - Summary
459 | - objectForTemplate
460 | attributesForFaceting:
461 | - "filterOnly(objectClassName)"
462 | index_subsite_pages:
463 | includeClasses:
464 | - SilverStripe\CMS\Model\SiteTree
465 | includeFilter:
466 | "SilverStripe\\CMS\\Model\\SiteTree": "SubsiteID > 0"
467 | indexSettings:
468 | distinct: true
469 | attributeForDistinct: "objectLink"
470 | searchableAttributes:
471 | - objectTitle
472 | - objectContent
473 | - objectLink
474 | - Summary
475 | - objectForTemplate
476 | attributesForFaceting:
477 | - "filterOnly(objectClassName)"
478 | ```
479 |
--------------------------------------------------------------------------------