├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── composer.json ├── config └── pastable.php ├── pint.json └── src ├── Commands ├── CopyPastableCommand.php ├── CutPastableCommand.php └── PastableCommand.php ├── Helper ├── PastableLogger.php └── PastableUtils.php ├── Jobs ├── CopyPastableJob.php ├── CutPastableJob.php ├── Middleware │ └── AtomicJob.php └── PastableJob.php ├── Models └── Traits │ ├── CopyPastable.php │ ├── CutPastable.php │ └── Pastable.php └── PastableServiceProvider.php /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to `laravel-pastable-model` will be documented in this file. 4 | 5 | ## 0.2.0 - 2024-03-25 6 | 7 | Bump PHP and Laravel to 11 8 | 9 | ### What's Changed 10 | 11 | * chore(deps): Bump actions/checkout from 3 to 4 by @dependabot in https://github.com/elipZis/laravel-pastable-model/pull/2 12 | * chore(deps): Bump aglipanci/laravel-pint-action from 2.3.0 to 2.3.1 by @dependabot in https://github.com/elipZis/laravel-pastable-model/pull/4 13 | * chore(deps): Bump ramsey/composer-install from 2 to 3 by @dependabot in https://github.com/elipZis/laravel-pastable-model/pull/5 14 | * chore(deps): Bump stefanzweifel/git-auto-commit-action from 4 to 5 by @dependabot in https://github.com/elipZis/laravel-pastable-model/pull/3 15 | * chore(deps): Bump dependabot/fetch-metadata from 1.6.0 to 2.0.0 by @dependabot in https://github.com/elipZis/laravel-pastable-model/pull/6 16 | 17 | **Full Changelog**: https://github.com/elipZis/laravel-pastable-model/compare/v0.1.0...v0.2.0 18 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) elipZis 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cut/Copy & Paste Laravel Eloquent models into another table 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/elipzis/laravel-pastable-model.svg?style=flat-square)](https://packagist.org/packages/elipzis/laravel-pastable-model) 4 | [![GitHub Tests Action Status](https://img.shields.io/github/actions/workflow/status/elipzis/laravel-pastable-model/run-tests.yml?branch=main&label=tests&style=flat-square)](https://github.com/elipzis/laravel-pastable-model/actions?query=workflow%3Arun-tests+branch%3Amain) 5 | [![GitHub Code Style Action Status](https://img.shields.io/github/actions/workflow/status/elipzis/laravel-pastable-model/fix-php-code-style-issues.yml?branch=main&label=code%20style&style=flat-square)](https://github.com/elipzis/laravel-pastable-model/actions?query=workflow%3A"Fix+PHP+code+style+issues"+branch%3Amain) 6 | [![Total Downloads](https://img.shields.io/packagist/dt/elipzis/laravel-pastable-model.svg?style=flat-square)](https://packagist.org/packages/elipzis/laravel-pastable-model) 7 | 8 | Enable your models to regularly cut/copy & paste their data into another table. 9 | 10 | - Cut & Paste or Copy & Paste 11 | - Scheduled Jobs available to regularly & asynchronously run 12 | - Cut & Paste in chunks, to split potential long-running processes 13 | - Store data e.g. into logging or daily tables and keep the production data clean 14 | 15 | ## Installation 16 | 17 | You can install the package via composer: 18 | 19 | ```bash 20 | composer require elipzis/laravel-pastable-model 21 | ``` 22 | 23 | You can publish the config file with: 24 | 25 | ```bash 26 | php artisan vendor:publish --tag="pastable-model-config" 27 | ``` 28 | 29 | This is the contents of the published config file: 30 | 31 | ```php 32 | return [ 33 | //The default cut&paste chunk size (limit) 34 | 'chunkSize' => 1000, 35 | //Auto-create tables, if not existing 36 | 'autoCreate' => false, 37 | //Enable detailed logging to any accepted and configured level 38 | 'logging' => [ 39 | 'enabled' => false, 40 | 'level' => null, 41 | ], 42 | ]; 43 | ``` 44 | 45 | ## Usage 46 | 47 | To make your model copy & pastable, add the trait `CopyPastable`. 48 | It will copy & paste your configured query result into the target table. 49 | 50 | ```php 51 | ... 52 | use ElipZis\Pastable\Models\Traits\CopyPastable; 53 | ... 54 | 55 | class YourModel extends Model { 56 | 57 | use CopyPastable; 58 | ... 59 | ``` 60 | 61 | To make your model cut & pastable, add the trait `CutPastable`. 62 | It will cut (delete) & paste your configured query result into the target table. 63 | If more rows than the chunk size (limit) are affected, it will respawn its own job until completed. 64 | 65 | ```php 66 | ... 67 | use ElipZis\Pastable\Models\Traits\CutPastable; 68 | ... 69 | 70 | class YourModel extends Model { 71 | 72 | use CutPastable; 73 | ... 74 | ``` 75 | 76 | ### Configuration 77 | 78 | To use any trait, you need to configure two settings: 79 | 80 | - the target table 81 | - the query to read its data from 82 | 83 | #### Target table _(mandatory)_ 84 | 85 | You must define the target table name. 86 | 87 | ```php 88 | ... 89 | 90 | class YourModel extends Model { 91 | 92 | ... 93 | protected string $pastableTable = 'log_something'; 94 | ... 95 | ``` 96 | 97 | or by overriding the getter function, to e.g. create dynamic table names 98 | 99 | ```php 100 | ... 101 | 102 | class YourModel extends Model { 103 | 104 | ... 105 | public function getPastableTable(): string 106 | { 107 | return 'log_something_' . Carbon::now()->format('Y_m_d'); 108 | } 109 | ... 110 | ``` 111 | 112 | If the table does not exist, you can use the configuration setting `autoCreate` and set it to `true` to have the system 113 | try to create the table from your query source. 114 | 115 | **It is recommended for you to create the table manually or via migration, as the automation is not fully tested and 116 | functional to any database system and table structure.** 117 | 118 | #### Query _(mandatory)_ 119 | 120 | You must define the query to use to read data and cut/copy & paste into the target table. 121 | 122 | ```php 123 | ... 124 | 125 | class YourModel extends Model { 126 | 127 | ... 128 | public function getPastableQuery(): Builder 129 | { 130 | return static::query()->where('created_at', '<=', now()->subDay()); 131 | } 132 | ... 133 | ``` 134 | 135 | You can use any query that returns a `Builder` object. 136 | 137 | In the case of cut & paste, the default `chunkSize` is used as a limiter. You can set your own limit by 138 | adding `->limit()` to the query or override the configuration setting in general. 139 | 140 | #### Connection _(optional)_ 141 | 142 | You can give a separate connection if you want the target table to be generated and filled in e.g. another database. 143 | 144 | ```php 145 | ... 146 | 147 | class YourModel extends Model { 148 | 149 | ... 150 | protected string $pastableConnection = 'logging'; 151 | ... 152 | ``` 153 | 154 | ### Run 155 | 156 | After implementation and configuration, you got three options to trigger the cut/copy & paste jobs: 157 | 158 | - Manually dispatching the jobs 159 | - Scheduled dispatch 160 | - Running a command to trigger it manually 161 | 162 | #### Scheduled 163 | 164 | The preferred way is to run the job on a schedule, configured via the Kernel, e.g. daily: 165 | 166 | ```php 167 | namespace App\Console; 168 | 169 | ... 170 | use ElipZis\Pastable\Jobs\PastableJob; 171 | ... 172 | 173 | class Kernel extends ConsoleKernel 174 | 175 | ... 176 | protected function schedule(Schedule $schedule) 177 | { 178 | ... 179 | $schedule->job(PastableJob::class)->daily(); 180 | ... 181 | } 182 | ... 183 | ``` 184 | 185 | #### Via Command 186 | 187 | You may also trigger the execution manually by using the command(s): 188 | 189 | - All cut/copy & pastable model classes: `php artisan pastable:all` 190 | - Only copy & pastable model classes: `php artisan pastable:copy` 191 | - Only cut & pastable model classes: `php artisan pastable:cut` 192 | 193 | #### Manual dispatch 194 | 195 | The final option is to trigger the job manually inside any of your functions, any logic, any application code: 196 | 197 | ```php 198 | ... 199 | use ElipZis\Pastable\Jobs\PastableJob; 200 | ... 201 | 202 | class YourClass 203 | 204 | ... 205 | protected function yourFunction() 206 | { 207 | ... 208 | PastableJob::dispatch(); 209 | ... 210 | } 211 | ... 212 | ``` 213 | 214 | ## Testing 215 | 216 | ```bash 217 | composer test 218 | ``` 219 | 220 | ## Notes 221 | 222 | This package is heavily inspired by two incredible resources: 223 | 224 | - [Laravel Prunable](https://laravel.com/docs/10.x/eloquent#pruning-models) 225 | - [Flare's cleaning big tables](https://flareapp.io/blog/7-how-to-safely-delete-records-in-massive-tables-on-aws-using-laravel) 226 | 227 | Kudos and Thanks to both for the inspiration. 228 | 229 | ## Changelog 230 | 231 | Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. 232 | 233 | ## Contributing 234 | 235 | Please see [CONTRIBUTING](.github/CONTRIBUTING.md) for details. 236 | 237 | ## Security Vulnerabilities 238 | 239 | Please review [our security policy](.github/SECURITY.md) on how to report security vulnerabilities. 240 | 241 | ## Credits 242 | 243 | - [elipZis GmbH](https://github.com/elipZis) 244 | - [NeA](https://github.com/nea) 245 | - [All Contributors](https://github.com/elipZis/laravel-pastable-model/contributors) 246 | 247 | ## License 248 | 249 | The MIT License (MIT). Please see [License File](LICENSE.md) for more information. 250 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "elipzis/laravel-pastable-model", 3 | "description": "Cut/Copy & Paste Laravel Eloquent model data into another table", 4 | "keywords": [ 5 | "elipZis", 6 | "laravel", 7 | "laravel-pastable-model", 8 | "model", 9 | "eloquent", 10 | "paste", 11 | "copy", 12 | "cut", 13 | "database" 14 | ], 15 | "homepage": "https://github.com/elipzis/laravel-pastable-model", 16 | "license": "MIT", 17 | "authors": [ 18 | { 19 | "name": "elipZis GmbH", 20 | "email": "contact@elipZis.com", 21 | "role": "Developer", 22 | "homepage": "https://elipZis.com" 23 | } 24 | ], 25 | "require": { 26 | "php": "^8.2", 27 | "spatie/laravel-package-tools": "^1.14.0", 28 | "illuminate/contracts": "^10.0|^11.0" 29 | }, 30 | "require-dev": { 31 | "laravel/pint": "^1.0", 32 | "nunomaduro/collision": "^7.8", 33 | "nunomaduro/larastan": "^2.0.1", 34 | "orchestra/testbench": "^8.8", 35 | "pestphp/pest": "^2.0", 36 | "pestphp/pest-plugin-arch": "^2.0", 37 | "pestphp/pest-plugin-laravel": "^2.0", 38 | "phpstan/extension-installer": "^1.1", 39 | "phpstan/phpstan-deprecation-rules": "^1.0", 40 | "phpstan/phpstan-phpunit": "^1.0", 41 | "spatie/laravel-ray": "^1.26" 42 | }, 43 | "autoload": { 44 | "psr-4": { 45 | "ElipZis\\Pastable\\": "src/" 46 | } 47 | }, 48 | "autoload-dev": { 49 | "psr-4": { 50 | "ElipZis\\Pastable\\Tests\\": "tests/" 51 | } 52 | }, 53 | "scripts": { 54 | "analyse": "vendor/bin/phpstan analyse", 55 | "test": "vendor/bin/pest", 56 | "test-coverage": "vendor/bin/pest --coverage", 57 | "format": "vendor/bin/pint" 58 | }, 59 | "config": { 60 | "sort-packages": true, 61 | "allow-plugins": { 62 | "pestphp/pest-plugin": true, 63 | "phpstan/extension-installer": true 64 | } 65 | }, 66 | "extra": { 67 | "laravel": { 68 | "providers": [ 69 | "ElipZis\\Pastable\\PastableServiceProvider" 70 | ] 71 | } 72 | }, 73 | "minimum-stability": "dev", 74 | "prefer-stable": true 75 | } 76 | -------------------------------------------------------------------------------- /config/pastable.php: -------------------------------------------------------------------------------- 1 | 1000, 7 | //Auto-create tables, if not existing 8 | 'autoCreate' => false, 9 | //Enable detailed logging to any accepted and configured level 10 | 'logging' => [ 11 | 'enabled' => false, 12 | 'level' => null, 13 | ], 14 | ]; 15 | -------------------------------------------------------------------------------- /pint.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "laravel", 3 | "rules": { 4 | "no_empty_phpdoc": false, 5 | "concat_space": { 6 | "spacing": "one" 7 | }, 8 | "not_operator_with_successor_space": false 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /src/Commands/CopyPastableCommand.php: -------------------------------------------------------------------------------- 1 | getPastableClasses(CopyPastable::class); 29 | foreach ($classes as $class) { 30 | CopyPastableJob::dispatch($class); 31 | } 32 | 33 | return self::SUCCESS; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Commands/CutPastableCommand.php: -------------------------------------------------------------------------------- 1 | getPastableClasses(CutPastable::class); 29 | foreach ($classes as $class) { 30 | CutPastableJob::dispatch($class); 31 | } 32 | 33 | return self::SUCCESS; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Commands/PastableCommand.php: -------------------------------------------------------------------------------- 1 | getNamespace(); 13 | 14 | return collect(File::allFiles(app_path()))->map(static function ($item) use ($appNamespace) { 15 | $rel = $item->getRelativePathName(); 16 | 17 | return sprintf('%s%s', $appNamespace, implode('\\', explode('/', substr($rel, 0, strrpos($rel, '.'))))); 18 | })->filter(fn ($class) => class_exists($class)) 19 | ->filter(fn ($class) => in_array($traitClass, class_uses_recursive($class))) 20 | ->all(); 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Jobs/CopyPastableJob.php: -------------------------------------------------------------------------------- 1 | log("Starting copy & pasting for class `{$this->class}` at {$now->toString()}"); 34 | 35 | try { 36 | $affected = (new $this->class)->copyAndPaste(); 37 | 38 | $this->log("Copy & pasted {$affected} entries for class `{$this->class}` at {$now->toString()}"); 39 | 40 | } catch (Throwable $t) { 41 | $this->log("Error while copy & pasting ({$t->getLine()}): {$t->getMessage()}"); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Jobs/CutPastableJob.php: -------------------------------------------------------------------------------- 1 | log("Starting cut & pasting for class `{$this->class}` at {$now->toString()}"); 35 | 36 | try { 37 | $affected = (new $this->class)->cutAndPaste(); 38 | 39 | $this->log("Cut & pasted {$affected} entries for class `{$this->class}` at {$now->toString()}"); 40 | 41 | //Self-dispatch as long as there is more 42 | if ($affected > 0) { 43 | static::dispatch($this->class); 44 | } 45 | } catch (Throwable $t) { 46 | $this->log("Error while cut & pasting: ({$t->getLine()}): {$t->getMessage()}"); 47 | } 48 | } 49 | 50 | /** 51 | * @return array 52 | */ 53 | public function middleware(): array 54 | { 55 | return [new AtomicJob($this->class)]; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Jobs/Middleware/AtomicJob.php: -------------------------------------------------------------------------------- 1 | lock(get_class($job) . '_' . $this->class . '_lock', $this->lockTime); 35 | 36 | if (!$lock->get()) { 37 | $job->delete(); 38 | 39 | return; 40 | } 41 | 42 | $next($job); 43 | $lock->release(); 44 | } catch (Throwable $t) { 45 | $next($job); 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Jobs/PastableJob.php: -------------------------------------------------------------------------------- 1 | getPastableClasses(CopyPastable::class); 28 | $cutClasses = $this->getPastableClasses(CutPastable::class); 29 | 30 | if (empty($copyClasses) && empty($cutClasses)) { 31 | $this->log('No pastable classes found.'); 32 | 33 | return; 34 | } 35 | 36 | $count = count($copyClasses) + count($cutClasses); 37 | $this->log("Found {$count} pastable classes."); 38 | 39 | foreach ($copyClasses as $copyClass) { 40 | //Copy & Paste every class 41 | CopyPastableJob::dispatch($copyClass); 42 | } 43 | 44 | foreach ($cutClasses as $cutClass) { 45 | //Cut & Paste every class 46 | CutPastableJob::dispatch($cutClass); 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Models/Traits/CopyPastable.php: -------------------------------------------------------------------------------- 1 | preparePastable($this->getPastableQuery()); 20 | $connection = $this->getPastableConnection(); 21 | $tableName = $this->getPastableTable(); 22 | 23 | //Copy and Paste 24 | $affected = DB::connection($connection)->table($tableName)->insertUsing($query->getQuery()->columns ?? [], $query); 25 | $this->log("Affected {$affected} rows"); 26 | 27 | return $affected; 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/Models/Traits/CutPastable.php: -------------------------------------------------------------------------------- 1 | getPastableQuery(), function (Builder $query) { 24 | $query->when(!$query->getQuery()->limit, function ($query) { 25 | $query->limit(config('pastable.chunkSize', 1000)); 26 | }); 27 | }); 28 | 29 | $query = $this->preparePastable($query); 30 | $connection = $this->getPastableConnection(); 31 | $tableName = $this->getPastableTable(); 32 | 33 | //Cut and Paste 34 | DB::beginTransaction(); 35 | try { 36 | $affected = DB::connection($connection)->table($tableName)->insertUsing($query->getQuery()->columns ?? [], $query); 37 | 38 | //Then delete (Cutting) 39 | in_array(SoftDeletes::class, class_uses_recursive(static::class)) 40 | ? $query->forceDelete() 41 | : $query->delete(); 42 | 43 | DB::commit(); 44 | } catch (Throwable $t) { 45 | DB::rollBack(); 46 | $this->log('Error while cut-paste: ' . $t->getMessage()); 47 | throw $t; 48 | } 49 | 50 | return $affected; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Models/Traits/Pastable.php: -------------------------------------------------------------------------------- 1 | getPastableTable(); 24 | $this->log("Checking for target table '{$tableName}'"); 25 | if (!$this->createTable($query)) { 26 | throw new Exception('[Pastable] Unable to find or create the target table: ' . $tableName); 27 | } 28 | 29 | $this->log("Using pastable query '{$query->toSql()}'"); 30 | 31 | return $query; 32 | } 33 | 34 | /** 35 | * Check if a table exists or is to be created 36 | */ 37 | protected function createTable(Builder $query): bool 38 | { 39 | $connection = $this->getPastableConnection(); 40 | $tableName = $this->getPastableTable(); 41 | 42 | //Does a target exist? 43 | if (!Schema::connection($connection)->hasTable($tableName)) { 44 | $this->log("Table `{$tableName}` does not exist"); 45 | 46 | //Should it be created? 47 | if (config('pastable.autoCreate', false)) { 48 | $this->log("Trying to create `{$tableName}` table"); 49 | 50 | //Create table 51 | Schema::connection($connection)->create($tableName, function (Blueprint $table) use ($query) { 52 | $columns = $query->getQuery()->columns ?? Schema::getColumnListing(static::getTable()); 53 | 54 | foreach ($columns as $column) { 55 | $type = Schema::getColumnType(static::getTable(), $column); 56 | 57 | //Resolve some types 58 | if (str_contains($type, 'int')) { 59 | $type = 'bigInteger'; //Always fall back to big ints to be sure 60 | } elseif ($type === 'datetime') { 61 | $type = 'dateTime'; 62 | } 63 | 64 | try { 65 | $this->log("Trying to call type function for column `{$column}` of type {$type}"); 66 | $table->{$type}($column); 67 | 68 | } catch (Throwable $t) { 69 | $this->log("Failed! Trying to add column `{$column}` of type {$type}"); 70 | $table->addColumn($type, $column); 71 | } 72 | } 73 | }); 74 | } else { 75 | return false; 76 | } 77 | } 78 | 79 | return true; 80 | } 81 | 82 | /** 83 | * Define a string describing the target table name 84 | * 85 | * @return string The target table name 86 | */ 87 | public function getPastableTable(): string 88 | { 89 | return property_exists(static::class, 'pastableTable') 90 | ? $this->pastableTable 91 | : throw new LogicException('[Pastable] Please set the `pastableTable` property or override `getPastableTable`!'); 92 | } 93 | 94 | /** 95 | * Define the connection you want to leverage, or default (null) 96 | * 97 | * @return ?string 98 | */ 99 | public function getPastableConnection(): ?string 100 | { 101 | return property_exists(static::class, 'pastableConnection') 102 | ? $this->pastableConnection 103 | : null; 104 | } 105 | 106 | /** 107 | * Get the pastable model query. 108 | * 109 | * @return Builder A query of to be selected and cut/copied data 110 | */ 111 | public function getPastableQuery(): Builder 112 | { 113 | throw new LogicException('[Pastable] Please implement the `getPastableQuery` function on your model.'); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/PastableServiceProvider.php: -------------------------------------------------------------------------------- 1 | name('laravel-pastable-model') 20 | ->hasConfigFile('pastable') 21 | ->hasCommand(PastableCommand::class); 22 | } 23 | } 24 | --------------------------------------------------------------------------------