├── .gitignore ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── composer.json ├── config └── scout.php └── src ├── BatchSearchable.php ├── Commands └── CheckBatchIndexStatus.php └── ServiceProvider.php /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor 2 | composer.lock -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [2.2.3] - 26-01-2024 9 | 10 | ### Changed 11 | 12 | - Fix getScoutKeyName 13 | 14 | ## [2.2.2] - 26-01-2024 15 | 16 | ### Changed 17 | 18 | - Fix broken cache handling from 2.1.0 19 | 20 | ## [2.2.1] - 26-01-2024 21 | 22 | ### Changed 23 | 24 | - Fix theoretical issue with using getScoutKeyName instead of getKeyName 25 | 26 | ## [2.2.0] - 26-01-2024 27 | 28 | ### Changed 29 | 30 | - NB! Reverts the changes published for ~24h in 2.1.1 which did not work as epected at all. 31 | - Fixed force deleted models not dispatched for removal 32 | 33 | ## [2.1.0] - 18-09-2023 34 | 35 | ### Changed 36 | 37 | - Laravel 10 compatibility 38 | 39 | ## [2.0.0] - 09-06-2022 40 | 41 | ### Changed 42 | 43 | - Renamed namespace from OptimistDigital to Outl1ne 44 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Outl1ne 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Scout Batch Searchable 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/outl1ne/laravel-scout-batch-searchable.svg?style=flat-square)](https://packagist.org/packages/outl1ne/laravel-scout-batch-searchable) 4 | [![Total Downloads](https://img.shields.io/packagist/dt/outl1ne/laravel-scout-batch-searchable.svg?style=flat-square)](https://packagist.org/packages/outl1ne/laravel-scout-batch-searchable) 5 | 6 | This [Laravel](https://laravel.com) package allows for batching of Scout updates. 7 | 8 | ## Requirements 9 | 10 | - Laravel Scout 9+ 11 | - Scheduler with cron 12 | 13 | ## Description 14 | 15 | This package provides a new trait `BatchSearchable` that should be used instead of the regular `Searchable` trait provided by Laravel Scout. 16 | 17 | Using that trait, all updates pushed through Scout to the search server (whether it be MeiliSearch, Algolia or whatever else), are batched together instead of being sent one-by-one. 18 | 19 | The updates are sent on two possible conditions: 20 | 21 | > Either `scout.batch_searchable_max_batch_size` (default 250) is exeeded 22 | 23 | or 24 | 25 | > `scout.batch_searchable_debounce_time_in_min` (default 1) minutes have passed from the last update to the pending queue 26 | 27 | The IDs of models that require updating are stored in the default cache layer using the `Cache` helper. 28 | 29 | The debounce check uses Laravel's Scheduler to schedule a job that checks through all the pending update queues and sees if the required time has passed. This requires that the system has a working cron setup that calls `schedule:run` every minute. 30 | 31 | ## Installation 32 | 33 | Install the package in a Laravel Nova project via Composer and run migrations: 34 | 35 | ```bash 36 | composer require outl1ne/laravel-scout-batch-searchable 37 | ``` 38 | 39 | ## Usage 40 | 41 | Where you previously used the Searchable trait, just use BatchSearchable instead: 42 | 43 | ```php 44 | use Outl1ne\ScoutBatchSearchable\BatchSearchable; 45 | 46 | class SomeModel extends Model { 47 | use BatchSearchable; 48 | } 49 | ``` 50 | 51 | ## Credits 52 | 53 | - [Tarvo Reinpalu](https://github.com/tarpsvo) 54 | 55 | ## License 56 | 57 | Laravel Scout Batch Searchable is open-sourced software licensed under the [MIT license](LICENSE.md). 58 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "outl1ne/laravel-scout-batch-searchable", 3 | "description": "This Laravel package adds a BatchSearchable trait to allow batching Scout updates.", 4 | "keywords": [ 5 | "laravel", 6 | "scout", 7 | "batch" 8 | ], 9 | "license": "MIT", 10 | "require": { 11 | "php": ">=7.3.0", 12 | "illuminate/support": ">=7.0", 13 | "laravel/scout": "^9.8.1|^10.0.0" 14 | }, 15 | "autoload": { 16 | "psr-4": { 17 | "Outl1ne\\ScoutBatchSearchable\\": "src/" 18 | } 19 | }, 20 | "extra": { 21 | "laravel": { 22 | "providers": [ 23 | "Outl1ne\\ScoutBatchSearchable\\ServiceProvider" 24 | ] 25 | } 26 | }, 27 | "config": { 28 | "sort-packages": true 29 | }, 30 | "minimum-stability": "dev", 31 | "prefer-stable": true 32 | } 33 | -------------------------------------------------------------------------------- /config/scout.php: -------------------------------------------------------------------------------- 1 | 'SCOUT_BATCH_SEARCHABLE_QUEUE', 5 | 'batch_searchable_max_batch_size' => 250, 6 | 'batch_searchable_debounce_time_in_min' => 1, 7 | ]; 8 | -------------------------------------------------------------------------------- /src/BatchSearchable.php: -------------------------------------------------------------------------------- 1 | queueMakeSearchable($this); 30 | }); 31 | 32 | BaseCollection::macro('searchableImmediately', function () use ($self) { 33 | $self->parentQueueMakeSearchable($this); 34 | }); 35 | 36 | BaseCollection::macro('unsearchable', function () use ($self) { 37 | $self->queueRemoveFromSearch($this); 38 | }); 39 | 40 | BaseCollection::macro('unsearchableImmediately', function () use ($self) { 41 | $self->parentQueueRemoveFromSearch($this); 42 | }); 43 | } 44 | 45 | /** 46 | * Dispatch the job to make the given models searchable. 47 | * 48 | * @param \Illuminate\Database\Eloquent\Collection $models 49 | * @return void 50 | */ 51 | public function queueMakeSearchable($models) 52 | { 53 | if ($models->isEmpty()) { 54 | return; 55 | } 56 | 57 | // Save the model ID's to cache 58 | $this->addToBatchingQueue($models, true); 59 | } 60 | 61 | /** 62 | * Dispatch the job to make the given models unsearchable. 63 | * 64 | * @param \Illuminate\Database\Eloquent\Collection $models 65 | * @return void 66 | */ 67 | public function queueRemoveFromSearch($models) 68 | { 69 | if ($models->isEmpty()) { 70 | return; 71 | } 72 | 73 | // Save the model ID's to cache 74 | $this->addToBatchingQueue($models, false); 75 | } 76 | 77 | private function addToBatchingQueue($models, $makeSearchable = true) 78 | { 79 | if ($models->isEmpty()) return; 80 | $className = get_class($models->first()); 81 | $modelIds = $models->pluck($models->first()->getKeyName())->toArray(); 82 | ServiceProvider::addBatchedModelClass($className); 83 | 84 | // Add IDs to the requested queue 85 | $cacheKey = $this->getCacheKey($className, $makeSearchable); 86 | $existingCacheValue = Cache::get($cacheKey) ?? ['updated_at' => Carbon::now(), 'models' => []]; 87 | 88 | $existingCacheValue['models'] = is_array($existingCacheValue['models']) ? $existingCacheValue['models'] : []; 89 | $newModelIds = array_unique(array_merge($existingCacheValue['models'], $modelIds)); 90 | $newCacheValue = ['updated_at' => Carbon::now(), 'models' => array_values($newModelIds)]; 91 | 92 | // Remove IDs from the opposite queue 93 | $opCacheKey = $this->getCacheKey($className, !$makeSearchable); 94 | $opExistingCacheValue = Cache::get($opCacheKey) ?? ['updated_at' => Carbon::now(), 'models' => []]; 95 | 96 | $opExistingCacheValue['models'] = is_array($opExistingCacheValue['models']) ? $opExistingCacheValue['models'] : []; 97 | $newOpModelIds = array_filter($opExistingCacheValue['models'], function ($id) use ($modelIds) { 98 | return !in_array($id, $modelIds); 99 | }); 100 | $newOpCacheValue = ['updated_at' => Carbon::now(), 'models' => array_values($newOpModelIds)]; 101 | 102 | // Store 103 | if (empty($newCacheValue['models'])) { 104 | Cache::forget($cacheKey); 105 | } else { 106 | Cache::put($cacheKey, $newCacheValue); 107 | } 108 | 109 | if (empty($newOpCacheValue['models'])) { 110 | Cache::forget($opCacheKey); 111 | } else { 112 | Cache::put($opCacheKey, $newOpCacheValue); 113 | } 114 | 115 | 116 | $this->checkBatchingStatusAndDispatchIfNecessaryFor($className, $makeSearchable); 117 | } 118 | 119 | public function checkBatchingStatusAndDispatchIfNecessary($className) 120 | { 121 | $this->checkBatchingStatusAndDispatchIfNecessaryFor($className, true); 122 | $this->checkBatchingStatusAndDispatchIfNecessaryFor($className, false); 123 | } 124 | 125 | private function checkBatchingStatusAndDispatchIfNecessaryFor($className, $makeSearchable = true) 126 | { 127 | $cacheKey = $this->getCacheKey($className, $makeSearchable); 128 | $cachedValue = Cache::get($cacheKey) ?? ['updated_at' => Carbon::now(), 'models' => []]; 129 | if (empty($cachedValue['models'])) return; 130 | 131 | $maxBatchSize = Config::get('scout.batch_searchable_max_batch_size', 250); 132 | $maxBatchSizeExceeded = sizeof($cachedValue['models']) >= $maxBatchSize; 133 | 134 | $maxTimeInMin = Config::get('scout.batch_searchable_debounce_time_in_min', 1); 135 | $maxTimePassed = Carbon::now()->diffInMinutes($cachedValue['updated_at'], $absolute = true) >= $maxTimeInMin; 136 | 137 | if ($maxBatchSizeExceeded || $maxTimePassed) { 138 | ServiceProvider::removeBatchedModelClass($className); 139 | Cache::forget($cacheKey); 140 | 141 | $fakeModel = new $className; 142 | $keyName = $fakeModel->getKeyName(); 143 | 144 | $models = collect(); 145 | 146 | if ($makeSearchable) { 147 | $models = method_exists($this, 'trashed') 148 | ? $className::withTrashed()->whereIn($keyName, $cachedValue['models'])->get() 149 | : $className::whereIn($keyName, $cachedValue['models'])->get(); 150 | } else { 151 | $models = collect($cachedValue['models'])->map(function ($id) use ($className) { 152 | $model = new $className; 153 | $model->{$model->getKeyName()} = $id; 154 | return $model; 155 | }); 156 | } 157 | 158 | return $makeSearchable 159 | ? $this->parentQueueMakeSearchable($models) 160 | : $this->parentQueueRemoveFromSearch($models); 161 | } 162 | } 163 | 164 | private function getCacheKey($className, $makeSearchable) 165 | { 166 | return $makeSearchable 167 | ? $this->getGenericCacheKey($className, 'MAKE_SEARCHABLE') 168 | : $this->getGenericCacheKey($className, 'REMOVE_FROM_SEARCH'); 169 | } 170 | 171 | private function getGenericCacheKey($className, $suffix) 172 | { 173 | $cacheKey = Config::get('scout.batch_searchable_cache_key', 'SCOUT_BATCH_SEARCHABLE_QUEUE'); 174 | $className = Str::upper(Str::snake(Str::replace('\\', '', $className))); 175 | return "{$cacheKey}_{$className}_{$suffix}"; 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /src/Commands/CheckBatchIndexStatus.php: -------------------------------------------------------------------------------- 1 | checkBatchingStatusAndDispatchIfNecessary($batchClass); 21 | } else { 22 | // Seems like the stored model has lost its BatchSearchable trait 23 | ServiceProvider::removeBatchedModelClass($batchClass); 24 | } 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/ServiceProvider.php: -------------------------------------------------------------------------------- 1 | mergeConfigFrom(__DIR__ . '/../config/scout.php', 'scout'); 18 | } 19 | 20 | /** 21 | * Perform post-registration booting of services. 22 | * 23 | * @return void 24 | */ 25 | public function boot() 26 | { 27 | if ($this->app->runningInConsole()) { 28 | $this->app->booted(function () { 29 | /** @var Schedule */ 30 | $schedule = $this->app->make(Schedule::class); 31 | $schedule->call(function () { 32 | $batchedModelsClasses = static::getBatchedModelsClasses(); 33 | foreach ($batchedModelsClasses as $batchClass) { 34 | $instantiatedBatchObject = new $batchClass; 35 | if (method_exists($instantiatedBatchObject, 'checkBatchingStatusAndDispatchIfNecessary')) { 36 | $instantiatedBatchObject->checkBatchingStatusAndDispatchIfNecessary($batchClass); 37 | } else { 38 | // Seems like the stored model has lost its BatchSearchable trait 39 | static::removeBatchedModelClass($batchClass); 40 | } 41 | } 42 | })->description('Scout Batch Searchable')->everyMinute(); 43 | }); 44 | } 45 | } 46 | 47 | public static function getBatchedModelsClasses() 48 | { 49 | return Cache::get('BATCH_SEARCHABLE_QUEUED_MODELS') ?? []; 50 | } 51 | 52 | public static function addBatchedModelClass($className) 53 | { 54 | $queuedModels = static::getBatchedModelsClasses(); 55 | $queuedModels[] = $className; 56 | $queuedModels = array_unique($queuedModels); 57 | Cache::put('BATCH_SEARCHABLE_QUEUED_MODELS', $queuedModels); 58 | return $queuedModels; 59 | } 60 | 61 | public static function removeBatchedModelClass($className) 62 | { 63 | $queuedModels = static::getBatchedModelsClasses(); 64 | $queuedModels = array_filter($queuedModels, function ($cls) use ($className) { 65 | return $cls !== $className; 66 | }); 67 | Cache::put('BATCH_SEARCHABLE_QUEUED_MODELS', $queuedModels); 68 | return $queuedModels; 69 | } 70 | } 71 | --------------------------------------------------------------------------------