├── LICENSE ├── README.md ├── composer.json ├── docker └── custom.cnf ├── examples ├── classicmodels.xlsx ├── classicmodels │ ├── customers.yaml │ ├── employees.yaml │ ├── offices.yaml │ ├── order_details.yaml │ ├── orders.yaml │ ├── payments.yaml │ ├── product_lines.yaml │ └── products.yaml ├── users.csv └── users │ └── users.yaml └── src ├── ChunkReadFilter.php ├── Console └── SeedCommand.php ├── Events └── Console.php ├── FileIterator.php ├── MySqlGrammar.php ├── Readers ├── Events │ ├── ChunkFinish.php │ ├── ChunkStart.php │ ├── FileFinish.php │ ├── FileSeed.php │ ├── FileStart.php │ ├── RowFinish.php │ ├── RowStart.php │ ├── SheetFinish.php │ └── SheetStart.php ├── HeaderImporter.php ├── PhpSpreadsheet │ ├── ChunkReadFilter.php │ ├── SourceChunk.php │ ├── SourceFile.php │ ├── SourceFileReadFilter.php │ ├── SourceHeader.php │ ├── SourceRow.php │ ├── SourceSheet.php │ └── SpreadsheetReader.php ├── RowImporter.php ├── Rows.php └── Types │ └── EmptyCell.php ├── SeederHelper.php ├── SeederMemoryHelper.php ├── SourceFileReadFilter.php ├── SpreadsheetSeeder.php ├── SpreadsheetSeederServiceProvider.php ├── SpreadsheetSeederSettings.php ├── Support ├── ColumnInfo.php ├── StrMacros.php └── Workaround │ └── RefreshDatabase │ └── RefreshDatabaseMySqlConnection.php └── Writers ├── Console └── ConsoleWriter.php ├── Database ├── DatabaseWriter.php ├── DestinationTable.php └── QueryLogMemoryLeakWorkaroundProvider.php └── Text ├── Markdown ├── MarkdownFormatter.php └── MarkdownWriter.php ├── TextOutputFileRepository.php ├── TextOutputWriter.php ├── TextTableFormatter.php ├── TextTableFormatterInterface.php └── Yaml ├── YamlFormatter.php └── YamlWriter.php /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright 2019 Brion Finlay 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | 24 | This work was inspired by and contains some code from laravel-csv-seeder 25 | https://github.com/jeroenzwart/laravel-csv-seeder 26 | 27 | The license for laravel-csv-seeder follows: 28 | 29 | MIT License 30 | 31 | Copyright (c) 2018 Jeroen Zwart 32 | 33 | Permission is hereby granted, free of charge, to any person obtaining a copy 34 | of this software and associated documentation files (the "Software"), to deal 35 | in the Software without restriction, including without limitation the rights 36 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 37 | copies of the Software, and to permit persons to whom the Software is 38 | furnished to do so, subject to the following conditions: 39 | 40 | The above copyright notice and this permission notice shall be included in all 41 | copies or substantial portions of the Software. 42 | 43 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 44 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 45 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 46 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 47 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 48 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 49 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Excel Seeder for Laravel 2 | [![PHPUnit tests](https://github.com/bfinlay/laravel-excel-seeder/actions/workflows/github-actions.yml/badge.svg?branch=master)](https://github.com/bfinlay/laravel-excel-seeder/actions/workflows/github-actions.yml) 3 | > #### Seed your database using CSV files, XLSX files, and more with Laravel 4 | 5 | With this package you can save time seeding your database. Instead of typing out seeder files, you can use CSV, XLSX, or any supported spreadsheet file format to load your project's database. There are configuration options available to control the insertion of data from your spreadsheet files. 6 | 7 | This project was forked from [laravel-csv-seeder](https://github.com/jeroenzwart/laravel-csv-seeder) and rewritten to support processing multiple input files and to use the [PhpSpreadsheet](https://github.com/PHPOffice/PhpSpreadsheet) library to support XLSX and other file formats. 8 | 9 | ### Features 10 | 11 | - Support CSV, XLS, XLSX, ODF, Gnumeric, XML, HTML, SLK files through [PhpSpreadsheet](https://github.com/PHPOffice/PhpSpreadsheet) library 12 | - Seed from multiple spreadsheet files per Laravel seeder class 13 | - Generate text output version of XLS spreadsheet for determining changes to XLS when branch merging 14 | - Automatically resolve CSV filename to table name. 15 | - Automatically resolve XLSX worksheet tabs to table names. 16 | - Automatically map CSV and XLSX headers to table column names. 17 | - Automatically determine delimiter for CSV files, including comma `,`, tab `\t`, pipe `|`, and semi-colon `;` 18 | - Skip seeding data columns by using a prefix character in the spreadsheet column header. 19 | - Hash values with a given array of column names. 20 | - Seed default values into table columns. 21 | - Adjust Laravel's timestamp at seeding. 22 | 23 | ### Scale 24 | This package has been used on CSV files with 5 million rows per file while maintaining flat memory usage (no memory leaks). 25 | 26 | ### Testing 27 | This package has PHPUnit tests run automatically by Github Actions. Tests are added as enhancements are made or as bugs are found and fixed. 28 | 29 | [![PHPUnit tests](https://github.com/bfinlay/laravel-excel-seeder/actions/workflows/github-actions.yml/badge.svg?branch=master)](https://github.com/bfinlay/laravel-excel-seeder/actions/workflows/github-actions.yml) 30 | This package is tested against the following Laravel versions 31 | * Laravel 5.8 32 | * Laravel 6.x 33 | * Laravel 7.x 34 | * Laravel 8.x 35 | * Laravel 9.x 36 | * Laravel 10.x 37 | * Laravel 11.x 38 | * Laravel 12.x 39 | 40 | ## Contents 41 | - [Installation](#installation) 42 | - [Simplest Usage](#simplest-usage) 43 | - [Basic Usage](#basic-usage) 44 | - [Seeding Individual Sheets](#seeding-individual-sheets) 45 | - [Markdown Diffs](#excel-text-output-for-branch-diffs) 46 | - [Configuration Settings](#configuration) 47 | - [Conversion Details](#details) 48 | - [Examples](#examples) 49 | - [License](#license) 50 | - [Changes](#changes) 51 | 52 | ## Installation 53 | ### Laravel 8.x, 9.x, 10.x, 11.x, 12.x 54 | - Require this package directly by `composer require --dev -W bfinlay/laravel-excel-seeder` 55 | - Or add this package in your composer.json and run `composer update` 56 | 57 | ``` 58 | "require-dev": { 59 | ... 60 | "bfinlay/laravel-excel-seeder": "^3.0", 61 | ... 62 | } 63 | ``` 64 | ### Laravel 5.8, 6.x, 7.x 65 | Laravel 5.8, 6.x, and 7.x require DBAL 2.x. Because DBAL is a `require-dev` dependency of laravel, its version 66 | constraint will not be resolved by composer when installing a child package. However, this is easy to solve by specifying DBAL 2.x as 67 | an additional dependency. 68 | 69 | Note that Laravel 5.8, 6.x, 7.x, 8.x, 9.x, and 10.x are EOL. See https://laravelversions.com/en. 70 | These versions will continue to be supported by this package for as long as reasonably possible, thanks to github actions performing the testing. 71 | 72 | To install for Laravel 5.8, 6.x, and 7.x: 73 | - Require this package directly by `composer require --dev -W bfinlay/laravel-excel-seeder` 74 | - Require the dbal package directly by `composer require --dev -W doctrine/dbal:^2.6` 75 | - Or add these packages in your composer.json and run `composer update` 76 | 77 | ``` 78 | "require-dev": { 79 | ... 80 | "bfinlay/laravel-excel-seeder": "^3.0", 81 | "doctrine/dbal": "^2.6" 82 | ... 83 | } 84 | ``` 85 | 86 | 87 | ## Simplest Usage 88 | In the simplest form, you can use the `bfinlay\SpreadsheetSeeder\SpreadsheetSeeder` 89 | as is and it will process all XLSX files in `/database/seeds/*.xlsx` and `/database/seeders/*.xlsx` (relative to Laravel project base path). 90 | 91 | Just add the SpreadsheetSeeder to be called in your `/database/seeds/DatabaseSeeder.php` (Laravel 5.8, 6.x, 7.x) or `/database/seeder/DatabaseSeeder.php` (Laravel 8.X and newer) class. 92 | 93 | ```php 94 | use Illuminate\Database\Seeder; 95 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 96 | 97 | class DatabaseSeeder extends Seeder 98 | { 99 | /** 100 | * Seed the application's database. 101 | * 102 | * @return void 103 | */ 104 | public function run() 105 | { 106 | $this->call([ 107 | SpreadsheetSeeder::class, 108 | ]); 109 | } 110 | } 111 | ``` 112 | 113 | Place your spreadsheets into the path `/database/seeds/` (Laravel 5.8, 6.x, 7.x) or `/database/seeders/` (Laravel 8.x and newer) of your Laravel project. 114 | 115 | With the default settings, the seeder makes certain assumptions about the XLSX files: 116 | * worksheet (tab) names match --> table names in database 117 | * worksheet (tab) has a header row and the column names match --> table column names in database 118 | * If there is only one worksheet in the XLSX workbook either the worksheet (tab) name or workbook filename must match a table in the database. 119 | 120 | 121 | An Excel example: 122 | 123 | | first_name | last_name | birthday | 124 | | ------------- | ------------- | ---------- | 125 | | Foo | Bar | 1970-01-01 | 126 | | John | Doe | 1980-01-01 | 127 | 128 | A CSV example: 129 | ``` 130 | first_name,last_name,birthday 131 | Foo,Bar,1970-01-01 132 | John,Doe,1980-01-01 133 | ``` 134 | 135 | 136 | ## Basic usage 137 | In most cases you will need to configure settings. 138 | Create a seed class that extends `bfinlay\SpreadsheetSeeder\SpreadsheetSeeder` and configure settings on your class. A seed class will look like this: 139 | ```php 140 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 141 | 142 | class UsersTableSeeder extends SpreadsheetSeeder 143 | { 144 | /** 145 | * Run the database seeds. 146 | * 147 | * @return void 148 | */ 149 | public function settings(SpreadsheetSeederSettings $set) 150 | { 151 | // By default, the seeder will process all XLSX files in /database/seeds/*.xlsx (relative to Laravel project base path) 152 | 153 | // Example setting 154 | $set->worksheetTableMapping = ['Sheet1' => 'first_table', 'Sheet2' => 'second_table']; 155 | } 156 | } 157 | ``` 158 | 159 | note: the older process of overloading `run()` still works for backwards compatibility. 160 | 161 | ## Seeding Individual Sheets 162 | By default, executing the `db:seed` Artisan command will seed all worksheets within a workbook. 163 | 164 | If you want to specify individual sheets to seed, you may use the `xl:seed` Artisan command 165 | with the `--sheet` option. You may specify multiple `--sheet` options. 166 | 167 | ``` 168 | php artisan xl:seed --sheet=users --sheet=posts 169 | ``` 170 | 171 | The above will run the `Database\Seeders\DatabaseSeeder` class, and for any SpreadsheetSeeders that are invoked 172 | will only seed sheets named `users` and `posts`. You may use the `--class` option to specify a specific seeder 173 | class to run individually 174 | 175 | ``` 176 | php artisan xl:seed --class=MySpreadsheetSeederClass --sheet=users --sheet=posts 177 | ``` 178 | 179 | If you want to run the default `SpreadsheetSeeder` class, you can specify `--class=#`. (The `#` resembles a spreadsheet.) 180 | 181 | ``` 182 | php artisan xl:seed --class=# --sheet=users --sheet=posts 183 | ``` 184 | 185 | For an easier syntax, you can also pass these as arguments and omit the --class and --seed. When using arguments, 186 | the first argument must be the class, and subsequent arguments will be sheets. 187 | 188 | ``` 189 | php artisan xl:seed # users posts 190 | ``` 191 | 192 | Important note: as with seeding traditional seeder classes individually, when seeding individual sheets if the truncate option is true, 193 | relations with cascade delete will also be deleted. 194 | 195 | ## Excel Text Output for Branch Diffs 196 | After running the database seeder, a subdirectory will be created using 197 | the same name as the input file. A text output file will be created 198 | for each worksheet using the worksheet name. This text file contains a text-based representation of each 199 | worksheet (tab) in the workbook and can be used to determine 200 | changes in the XLSX when merging branches from other contributors. 201 | 202 | Two formats are currently supported. The older format is 'markdown' and is the defualt for backward compatibility. 203 | The newer format is 'yaml' which is meant to work better with typical line-oriented diff software. 204 | 205 | Check this file into the repository so that it can serve as a basis for 206 | comparison. 207 | 208 | You will have to merge the XLSX spreadsheet manually. 209 | 210 | TextOutput can be disabled by setting `textOutput` to `FALSE` 211 | 212 | See [Text Output](#text-output) for more information. 213 | 214 | ## Configuration 215 | * [Add Columns](#add-columns) - array of column names that will be added in addition to those found in the worksheet 216 | * [Aliases](#column-aliases) - (global) map column names to alternative column names 217 | * [Batch Insert Size](#batch-insert-size) - number of rows to insert per batch 218 | * [Date Formats](#date-formats) - configure date formats by column when Carbon cannot automatically parse date 219 | * [Defaults](#defaults) - (global) map column names to default values 220 | * [Delimiter](#delimiter) - specify CSV delimiter (default: auto detect) 221 | * [Extension](#data-source-file-default-extension) - default file extension when directory is specified (default: xlsx) 222 | * [File](#data-source-file) - path or paths of data source files to process (default: /database/seeds/*.xlsx) 223 | * [Hashable](#hashable) - (global) array of column names hashed using Hash facade 224 | * [Header](#header) - (global) skip first row when true (default: true) 225 | * [Input Encodings](#input-encodings) - (global) array of possible encodings from input data source 226 | * [Limit](#limit) - (global) limit the maximum number of rows that will be loaded from a worksheet (default: no limit) 227 | * [Mapping](#column-mapping) - column "mapping"; array of column names to use as a header 228 | * [Offset](#offset) - (global) number of rows to skip at the start of the data source (default: 0) 229 | * [Output Encodings](#output-encodings) - (global) output encoding to database 230 | * [Parsers](#parsers) - (global) associative array of column names in the data source that should be parsed with the specified parser. 231 | * [Read Chunk Size](#read-chunk-size) - number of rows to read per chunk 232 | * [Skipper](#skipper) - (global) prefix string to indicate a column or worksheet should be skipped (default: "%") 233 | * [Skip Columns](#skip-columns) - array of column names that will be skipped in the worksheet. 234 | * [Skip Sheets](#skip-sheets) - array of worksheet names that will be skipped in the workbook. 235 | * [Tablename](#destination-table-name) - (legacy) table name to insert into database for single-sheet file 236 | * [Text Output](#text-output) - enable text markdown output (default: true) 237 | * [Text Output Path](#text-output-path) - path for text output 238 | * [Timestamps](#timestamps) - when true, set the Laravel timestamp columns 'created_at' and 'updated_at' with current date/time (default: true) 239 | * [Truncate](#truncate-destination-table) - truncate the table before seeding (default: true) 240 | * [Truncate Ignore Foreign Key Constraints](#truncate-destination-table-ignoring-foreign-key-constraints) - truncate the table before seeding (default: true) 241 | * [Unix Timestamps](#unix-timestamps) - interpret date/time values as unix timestamps instead of excel timestamps for specified columns (default: no columns) 242 | * [UUID](#uuid) - array of column names that the seeder will generate a UUID for. 243 | * [Validate](#validate) - map column names to laravel validation rules 244 | * [Worksheet Table Mapping](#worksheet-table-mapping) - map names of worksheets to table names 245 | 246 | ### Add Columns 247 | `$addColumns` *(array [])* 248 | 249 | This is an array of column names that will be column names in addition to 250 | those found in the worksheet. 251 | 252 | These additional columns will be processed the same ways as columns found 253 | in a worksheet. Cell values will be considered the same way as "empty" cells 254 | in the worksheet. These columns could be populated by parsers, defaults, or uuids. 255 | 256 | Example: ['uuid, 'column1', 'column2'] 257 | 258 | Default: [] 259 | 260 | ### Column Aliases 261 | `$aliases` *(array [])* 262 | 263 | This is an associative array to map the column names of the data source 264 | to alternative column names (aliases). 265 | 266 | Example: `['CSV Header 1' => 'Table Column 1', 'CSV Header 2' => 'Table Column 2']` 267 | 268 | Default: `[]` 269 | 270 | ### Batch Insert Size 271 | `$batchInsertSize` *(integer)* 272 | 273 | Number of rows to insert in a batch. 274 | 275 | Default: `5000` 276 | 277 | ### Date Formats 278 | `$dateFormats` *(array [])* 279 | 280 | This is an associative array mapping column names in the data source to 281 | date format strings that should be used by Carbon to parse the date. 282 | Information to construct date format strings is here: 283 | [https://www.php.net/manual/en/datetime.createfromformat.php](https://www.php.net/manual/en/datetime.createfromformat.php) 284 | 285 | When the destination column in the database table is a date time format, 286 | and the source data is a string, the seeder will use Carbon to parse the 287 | date format. In many cases Carbon can parse the date automatically 288 | without specifying the date format. 289 | 290 | When Carbon cannot parse the date automatically, map the column name in 291 | this array to the date format string. When a source column is mapped, 292 | Carbon will use the date format string instead of parsing automatically. 293 | 294 | If column mapping is used (see [mapping](#mapping)) the column name should match the 295 | value in the $mapping array instead of the value in the file, if any. 296 | 297 | Example: 298 | ``` 299 | [ 300 | 'order_date' => 'Y-m-d H:i:s.u+', // parses "2020-10-04 05:31:02.440000000" 301 | ] 302 | ``` 303 | 304 | Default: `[]` 305 | 306 | ### Defaults 307 | `$defaults` *(array [])* 308 | 309 | This is an associative array mapping column names in the data source to 310 | default values that will override any values in the datasource. 311 | 312 | Example: `['created_by' => 'seed', 'updated_by' => 'seed]` 313 | 314 | Default: `[]` 315 | 316 | ### Delimiter 317 | `$delimiter` *(string NULL)* 318 | 319 | The delimiter used in CSV, tab-separate-files, and other text delimited 320 | files. When this is not set, the phpspreadsheet library will 321 | automatically detect the text delimiter 322 | 323 | Default: `null` 324 | 325 | ### Data Source File Default Extension 326 | `$extension` *(string 'xlsx'*) 327 | 328 | The default extension used when a directory is specified in $this->file 329 | 330 | Default: `"xlsx"` 331 | 332 | ```php 333 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 334 | 335 | class UsersTableSeeder extends SpreadsheetSeeder 336 | { 337 | /** 338 | * Run the database seeds. 339 | * 340 | * @return void 341 | */ 342 | 343 | public function settings(SpreadsheetSeederSettings $set) 344 | { 345 | // specify relative to Laravel project base path 346 | // feature directories specified 347 | $set->file = [ 348 | '/database/seeds/feature1', 349 | '/database/seeds/feature2', 350 | '/database/seeds/feature3', 351 | ]; 352 | 353 | // process all xlsx and csv files in paths specified above 354 | $set->extension = ['xlsx', 'csv']; 355 | } 356 | } 357 | ``` 358 | 359 | ### Data Source File 360 | 361 | `$file` *(string*) or *(array []*) or *(Symfony\Component\Finder\Finder*) 362 | 363 | This value is the path of the Excel or CSV file used as the data 364 | source. This is a string or array[] and is list of files or directories 365 | to process, which can include wildcards. It can also be set to an instance 366 | of [Symfony Finder](https://symfony.com/doc/current/components/finder.html), 367 | which is a component that is already included with Laravel. 368 | 369 | By default, the seeder will process all XLSX files in /database/seeds (for Laravel 5.8 - 7.x) 370 | and /database/seeders (for Laravel 8.x and newer). 371 | 372 | The path is specified relative to the root of the project 373 | 374 | Default: `"/database/seeds/*.xlsx"` 375 | 376 | ```php 377 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 378 | 379 | class UsersTableSeeder extends SpreadsheetSeeder 380 | { 381 | /** 382 | * Run the database seeds. 383 | * 384 | * @return void 385 | */ 386 | public function settings(SpreadsheetSeederSettings $set) 387 | { 388 | // specify relative to Laravel project base path 389 | $set->file = [ 390 | '/database/seeds/file1.xlsx', 391 | '/database/seeds/file2.xlsx', 392 | '/database/seeds/seed*.xlsx', 393 | '/database/seeds/*.csv']; 394 | } 395 | } 396 | ``` 397 | 398 | This setting can also be configured to an instance of 399 | [Symfony Finder](https://symfony.com/doc/current/components/finder.html), 400 | which is a component that is already included with Laravel. 401 | 402 | When using Finder, the path is not relative to `base_path()` by default. 403 | To make the path relative to `base_path()` prepend it to the finder path. 404 | You could also use one of the other [Laravel path helpers](https://laravel.com/docs/master/helpers#method-base-path) . 405 | 406 | Example: 407 | ```php 408 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 409 | 410 | class UsersTableSeeder extends SpreadsheetSeeder 411 | { 412 | /** 413 | * Run the database seeds. 414 | * 415 | * @return void 416 | */ 417 | public function settings(SpreadsheetSeederSettings $set) 418 | { 419 | // specify relative to Laravel project base path 420 | $set->file = 421 | (new Finder) 422 | ->in(base_path() . '/database/seeds/') 423 | ->name('*.xlsx') 424 | ->notName('*customers*') 425 | ->sortByName(); 426 | } 427 | } 428 | ``` 429 | 430 | ### Hashable 431 | `$hashable` *(array ['password'])* 432 | 433 | This is an array of column names in the data source that should be hashed 434 | using Laravel's `Hash` facade. 435 | 436 | The hashing algorithm is configured in `config/hashing.php` per 437 | [https://laravel.com/docs/master/hashing](https://laravel.com/docs/master/hashing) 438 | 439 | Example: `['password']` 440 | 441 | Default: `[]` 442 | 443 | 444 | ### Header 445 | `$header` *(boolean TRUE)* 446 | 447 | If the data source has headers in the first row, setting this to true will 448 | skip the first row. 449 | 450 | Default: `TRUE` 451 | 452 | ### Input Encodings 453 | `$inputEncodings` *(array [])* 454 | 455 | Array of possible input encodings from input data source 456 | See [https://www.php.net/manual/en/mbstring.supported-encodings.php](https://www.php.net/manual/en/mbstring.supported-encodings.php) 457 | 458 | This value is used as the "from_encoding" parameter to mb_convert_encoding. 459 | If this is not specified, the internal encoding is used. 460 | 461 | Default: `[]` 462 | 463 | ### Limit 464 | `$limit` *(int*) 465 | 466 | Limit the maximum number of rows that will be loaded from a worksheet. 467 | This is useful in development to keep loading time fast. 468 | 469 | This can be used in conjunction with settings in the environment file or App::environment() (APP_ENV) to limit data rows in the development environment. 470 | 471 | Example: 472 | ```php 473 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 474 | 475 | class SalesTableSeeder extends SpreadsheetSeeder 476 | { 477 | /** 478 | * Run the database seeds. 479 | * 480 | * @return void 481 | */ 482 | public function settings(SpreadsheetSeederSettings $set) 483 | { 484 | $set->file = '/database/seeds/sales.xlsx'; 485 | if (App::environment('local')) 486 | $set->limit = 10000; 487 | } 488 | } 489 | ``` 490 | 491 | Default: null 492 | 493 | 494 | ### Column "Mapping" 495 | `$mapping` *(array [])* 496 | 497 | Backward compatibility to laravel-csv-seeder 498 | 499 | This is an array of column names that will be used as headers. 500 | 501 | If $this->header is true then the first row of data will be skipped. 502 | This allows existing headers in a CSV file to be overridden. 503 | 504 | This is called "Mapping" because its intended use is to map the fields of 505 | a CSV file without a header line to the columns of a database table. 506 | 507 | 508 | Example: `['Header Column 1', 'Header Column 2']` 509 | 510 | Default: `[]` 511 | 512 | ### Offset 513 | `$offset` *(integer)* 514 | 515 | Number of rows to skip at the start of the data source, excluding the 516 | header row. 517 | 518 | Default: `0` 519 | 520 | ### Output Encodings 521 | `$outputEncoding` *(string)* 522 | 523 | Output encoding to database 524 | See [https://www.php.net/manual/en/mbstring.supported-encodings.php](https://www.php.net/manual/en/mbstring.supported-encodings.php) 525 | 526 | This value is used as the "to_encoding" parameter to mb_convert_encoding. 527 | 528 | Default: `UTF-8` 529 | 530 | ### Parsers 531 | `$parsers` *(array ['column' => function($value) {}])* 532 | 533 | This is an associative array of column names in the data source that should be parsed 534 | with the specified parser. 535 | 536 | Example: 537 | ```php 538 | ['email' => function ($value) { 539 | return strtolower($value); 540 | }]; 541 | ``` 542 | 543 | Default: [] 544 | 545 | ### Read Chunk Size 546 | `$readChunkSize` *(integer)* 547 | 548 | Number of rows to read per chunk. 549 | 550 | Default: `5000` 551 | 552 | ### Skipper 553 | `$skipper` *(string %)* 554 | 555 | This is a string used as a prefix to indicate that a column in the data source 556 | should be skipped. For Excel workbooks, a worksheet prefixed with 557 | this string will also be skipped. The skipper prefix can be a 558 | multi-character string. 559 | 560 | - Example: Data source column `%id_copy` will be skipped with skipper set as `%` 561 | - Example: Data source column `#id_copy` will be skipped with skipper set as `#` 562 | - Example: Data source column `[skip]id_copy` will be skipped with skipper set as `[skip]` 563 | - Example: Worksheet `%worksheet1` will be skipped with skipper set as `%` 564 | 565 | Default: `"%"`; 566 | 567 | ### Skip Columns 568 | `$skipColumns` *(array [])* 569 | 570 | This is an array of column names that will be skipped in the worksheet. 571 | 572 | This can be used to skip columns in the same way as the skipper character, but 573 | without modifying the worksheet. 574 | 575 | Example: ['column1', 'column2'] 576 | 577 | Default: [] 578 | 579 | ### Skip Sheets 580 | `$skipSheets` *(array [])* 581 | 582 | This is an array of worksheet names that will be skipped in the workbook. 583 | 584 | This can be used to skip worksheets in the same way as the skipper character, 585 | but without modifying the workbook. 586 | 587 | Example: ['Sheet1', 'Sheet2'] 588 | 589 | Default: [] 590 | 591 | ### Destination Table Name 592 | `$tablename` *(string*) 593 | 594 | Backward compatibility to laravel-csv-seeder 595 | 596 | Table name to insert into in the database. If this is not set then 597 | the tablename is automatically resolved by the following rules: 598 | - if there is only 1 worksheet in a file and the worksheet is not the name of a table, use the base filename 599 | - otherwise use worksheet name 600 | 601 | Use worksheetTableMapping instead to map worksheet names to alternative 602 | table names 603 | 604 | Default: `null` 605 | 606 | ```php 607 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 608 | 609 | class UsersTableSeeder extends SpreadsheetSeeder 610 | { 611 | /** 612 | * Run the database seeds. 613 | * 614 | * @return void 615 | */ 616 | public function settings(SpreadsheetSeederSettings $set) 617 | { 618 | // specify relative to Laravel project base path 619 | // specify filename that is automatically dumped from an external process 620 | $set->file = '/database/seeds/autodump01234456789.xlsx'; // note: could alternatively be a csv 621 | 622 | // specify the table this is loaded into 623 | $set->tablename = 'sales'; 624 | 625 | // in this example, table truncation also needs to be disabled so previous sales records are not deleted 626 | $set->truncate = false; 627 | } 628 | } 629 | ``` 630 | 631 | ### Text Output 632 | `$textOutput` *(boolean)* or *(string*) or *(array []*) 633 | 634 | * Set to false to disable output of textual markdown tables. 635 | * `true` defaults to `'markdown'` output for backward compatibility. 636 | * `'markdown'` for markdown output 637 | * `'yaml'` for yaml output 638 | * `['markdown', 'yaml']` for both markdown and yaml output 639 | 640 | Default: `TRUE` 641 | 642 | ### Text Output Path 643 | `$textOutputPath` *(string)* 644 | Note: In development, subject to change 645 | 646 | Path for text output 647 | 648 | After processing a workbook, the seeder outputs a text format of 649 | the sheet to assist with diff and merge of the workbook. The default path 650 | is in the same path as the input workbook. Setting this path places the output 651 | files in a different location. 652 | 653 | Default: ""; 654 | 655 | ### Timestamps 656 | `$timestamps` *(string/boolean TRUE)* 657 | 658 | When `true`, set the Laravel timestamp columns 'created_at' and 'updated_at' 659 | with the current date/time. 660 | 661 | When `false`, the fields will be set to NULL 662 | 663 | Default: `true` 664 | 665 | ### Truncate Destination Table 666 | `$truncate` *(boolean TRUE)* 667 | 668 | Truncate the table before seeding. 669 | 670 | Default: `TRUE` 671 | 672 | Note: does not currently support array of table names to exclude 673 | 674 | See example for [tablename](#destination-table-name) above 675 | 676 | ### Truncate Destination Table Ignoring Foreign Key Constraints 677 | `$truncateIgnoreForeign` *(boolean TRUE)* 678 | 679 | Ignore foreign key constraints when truncating the table before seeding. 680 | 681 | When `false`, table will not be truncated if it violates foreign key integrity. 682 | 683 | Default: `TRUE` 684 | 685 | Note: does not currently support array of table names to exclude 686 | 687 | 688 | ### Unix Timestamps 689 | `$unixTimestamps` *(array [])* 690 | 691 | This is an array of column names that contain values that should be 692 | interpreted unix timestamps rather than excel timestamps. 693 | See [Conversions: Date/Time values](#datetime-values) 694 | 695 | If column mapping is used (see mapping) the column name should match the 696 | value in the $mapping array instead of the value in the file, if any. 697 | 698 | Note: this setting is currently global and applies to all files or 699 | worksheets that are processed. All columns with the specified name in all files 700 | or worksheets will be interpreted as unix timestamps. To apply differently to 701 | different files, process files with separate Seeder instances. 702 | 703 | Example: `['start_date', 'finish_date']`; 704 | 705 | Default: `[]` 706 | 707 | ### UUID 708 | `$uuid` *(array [])* 709 | 710 | This is an array of column names in the data source that the seeder will generate 711 | a UUID for. 712 | 713 | The UUID generated is a type 4 "Random" UUID using laravel Str::uuid() helper 714 | https://laravel.com/docs/10.x/helpers#method-str-uuid 715 | 716 | If the spreadsheet has the column and has a UUID value in the column, the seeder 717 | will use the UUID value from the spreadsheet. 718 | 719 | If the spreadsheet has any other value in the column or is empty, the seder will 720 | generate a new UUID value. 721 | 722 | If the spreadsheet does not have the column, use [$addColumns](#add-columns) to 723 | add the column, and also use $uuid (this setting) to generate a UUID for the added column. 724 | 725 | Example: ['uuid'] 726 | 727 | Default: [] 728 | 729 | ### Validate 730 | `$validate` *(array [])* 731 | 732 | This is an associative array mapping column names in the data source that 733 | should be validated to a Laravel Validator validation rule. 734 | The available validation rules are described here: 735 | [https://laravel.com/docs/master/validation#available-validation-rules](https://laravel.com/docs/master/validation#available-validation-rules) 736 | 737 | Example: 738 | ``` 739 | [ 740 | 'email' => 'unique:users,email_address', 741 | 'start_date' => 'required|date|after:tomorrow', 742 | 'finish_date' => 'required|date|after:start_date' 743 | ] 744 | ``` 745 | 746 | Default: `[]` 747 | 748 | ### Worksheet Table Mapping 749 | `$worksheetTableMapping` *(array [])* 750 | 751 | This is an associative array to map names of worksheets in an Excel file 752 | to table names. 753 | 754 | Excel worksheets have a 31 character limit. 755 | 756 | This is useful when the table name should be longer than the worksheet 757 | character limit. 758 | 759 | Example: `['Sheet1' => 'first_table', 'Sheet2' => 'second_table']` 760 | 761 | Default: `[]` 762 | 763 | ```php 764 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 765 | 766 | class UsersTableSeeder extends SpreadsheetSeeder 767 | { 768 | /** 769 | * Run the database seeds. 770 | * 771 | * @return void 772 | */ 773 | public function settings(SpreadsheetSeederSettings $set) 774 | { 775 | // specify the table this is loaded into 776 | $set->worksheetTableMapping = [ 777 | 'first_table_name_abbreviated' => 'really_rather_very_super_long_first_table_name', 778 | 'second_table_name_abbreviated' => 'really_rather_very_super_long_second_table_name' 779 | ]; 780 | } 781 | } 782 | ``` 783 | 784 | ## Details 785 | #### Null values 786 | - String conversions: 'null' is converted to `NULL`, 'true' is converted to `TRUE`, 'false' is converted to `FALSE` 787 | - 'null' strings converted to `NULL` are treated as explicit nulls. They are not subject to implicit conversions to default values. 788 | - Empty cells are set to the default value specified in the database table data definition, unless the entire row is empty 789 | - If the entire row consists of empty cells, the row is skipped. To intentionally insert a null row, put the string value 'null' in each cell 790 | 791 | #### Date/Time values 792 | When the destination table column is a date/time type, the cell value is converted to a Date/Time format. 793 | - If the value is numeric, it is assumed to be an excel date value 794 | - If the value is a string, it is parsed using [Carbon::parse](https://carbon.nesbot.com/docs/#api-instantiation) and formatted for the SQL query. 795 | - If the value is a unix timestamp, specify the column name with the [Unix Timestamps](#unix-timestamps) setting to convert it as a unix timestamp instead of an excel timestamp. 796 | 797 | ## Examples 798 | #### Table with specified timestamps and specified table name 799 | Use a specific timestamp for 'created_at' and 'updated_at' and also 800 | give the seeder a specific table name instead of using the CSV filename; 801 | 802 | ```php 803 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 804 | 805 | class UsersTableSeeder extends SpreadsheetSeeder 806 | { 807 | /** 808 | * Run the database seeds. 809 | * 810 | * @return void 811 | */ 812 | public function settings(SpreadsheetSeederSettings $set) 813 | { 814 | $set->file = '/database/seeds/csvs/users.csv'; 815 | $set->tablename = 'email_users'; 816 | $set->timestamps = '1970-01-01 00:00:00'; 817 | } 818 | } 819 | ``` 820 | 821 | #### Worksheet to Table Mapping 822 | Map the worksheet tab names to table names. 823 | 824 | Excel worksheet tabs have a 31 character limit. This is useful when the table name should be longer than the worksheet tab character limit. 825 | 826 | See [example](#worksheet-table-mapping) above 827 | 828 | #### Mapping 829 | Map the worksheet or CSV headers to table columns, with the following CSV; 830 | 831 | ##### XLSX 832 | | | | | 833 | |----| ------------- | ------------- | 834 | | 1 | Foo | Bar | 835 | | 2 | John | Doe | 836 | 837 | ##### CSV 838 | 1,Foo,Bar 839 | 2,John,Doe 840 | 841 | Example: 842 | ```php 843 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 844 | 845 | class UsersTableSeeder extends SpreadsheetSeeder 846 | { 847 | /** 848 | * Run the database seeds. 849 | * 850 | * @return void 851 | */ 852 | public function settings(SpreadsheetSeederSettings $set) 853 | { 854 | $set->file = '/database/seeds/users.xlsx'; 855 | $set->mapping = ['id', 'firstname', 'lastname']; 856 | $set->header = FALSE; 857 | } 858 | } 859 | ``` 860 | 861 | Note: this mapping is a legacy laravel-csv-seeder option. The mapping currently applies to all 862 | worksheets within a workbook, and is currently designed for single sheet workbooks 863 | and CSV files. 864 | 865 | There are two workarounds for mapping different column headers for different input files or worksheets: 866 | 1. add header columns to your multi-sheet workbooks 867 | 2. use CSVs or single-sheet workbooks and create a separate seeder for each that need different column mappings 868 | 869 | #### Aliases with defaults 870 | Seed a table with aliases and default values, like this; 871 | 872 | ```php 873 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 874 | 875 | class UsersTableSeeder extends SpreadsheetSeeder 876 | { 877 | /** 878 | * Run the database seeds. 879 | * 880 | * @return void 881 | */ 882 | public function settings(SpreadsheetSeederSettings $set) 883 | { 884 | $set->file = '/database/seeds/csvs/users.csv'; 885 | $set->aliases = ['csvColumnName' => 'table_column_name', 'foo' => 'bar']; 886 | $set->defaults = ['created_by' => 'seeder', 'updated_by' => 'seeder']; 887 | } 888 | } 889 | ``` 890 | 891 | #### Skipper 892 | Skip a worksheet in a workbook, or a column in an XLSX or CSV with a prefix. For example you use `id` in your worksheet which is only usable in your workbook. The worksheet file might look like the following: 893 | 894 | | %id | first_name | last_name | %id_copy | birthday | 895 | |-----| ------------- | ------------- | -------- | ---------- | 896 | | 1 | Foo | Bar | 1 | 1970-01-01 | 897 | | 2 | John | Doe | 2 | 1980-01-01 | 898 | 899 | The first and fourth value of each row will be skipped with seeding. The default prefix is '%' and changeable. In this example the skip prefix is changed to 'skip:' 900 | 901 | | skip:id | first_name | last_name | skip:id_copy | birthday | 902 | |---------| ------------- | ------------- | ------------ | ---------- | 903 | | 1 | Foo | Bar | 1 | 1970-01-01 | 904 | | 2 | John | Doe | 2 | 1980-01-01 | 905 | 906 | ```php 907 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 908 | 909 | class UsersTableSeeder extends SpreadsheetSeeder 910 | { 911 | /** 912 | * Run the database seeds. 913 | * 914 | * @return void 915 | */ 916 | public function settings(SpreadsheetSeederSettings $set) 917 | { 918 | $set->file = '/database/seeds/users.xlsx'; 919 | $set->skipper = 'skip:'; 920 | } 921 | } 922 | ``` 923 | 924 | To skip a worksheet in a workbook, prefix the worksheet name with '%' or the specified skipper prefix. 925 | 926 | #### Validate 927 | Validate each row of an XLSX or CSV like this; 928 | ```php 929 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 930 | 931 | class UsersTableSeeder extends SpreadsheetSeeder 932 | { 933 | /** 934 | * Run the database seeds. 935 | * 936 | * @return void 937 | */ 938 | public function settings(SpreadsheetSeederSettings $set) 939 | { 940 | $set->file = '/database/seeds/users.xlsx'; 941 | $set->validate = [ 'name' => 'required', 942 | 'email' => 'email', 943 | 'email_verified_at' => 'date_format:Y-m-d H:i:s', 944 | 'password' => ['required', Rule::notIn([' '])]]; 945 | } 946 | } 947 | ``` 948 | 949 | #### Hash 950 | Hash values when seeding an XLSX or CSV like this; 951 | ```php 952 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 953 | 954 | class UsersTableSeeder extends SpreadsheetSeeder 955 | { 956 | /** 957 | * Run the database seeds. 958 | * 959 | * @return void 960 | */ 961 | public function settings(SpreadsheetSeederSettings $set) 962 | { 963 | $set->file = '/database/seeds/users.xlsx'; 964 | $set->hashable = ['password']; 965 | } 966 | } 967 | ``` 968 | 969 | #### Input and Output Encodings 970 | The mb_convert_encodings function is used to convert encodings. 971 | * $this->inputEncodings is an array of possible input encodings. Default is `[]` which defaults to internal encoding. See [https://www.php.net/manual/en/mbstring.supported-encodings.php] 972 | * $this->outputEncoding is the output encoding. Default is 'UTF-8'; 973 | ```php 974 | use bfinlay\SpreadsheetSeeder\SpreadsheetSeeder; 975 | 976 | class UsersTableSeeder extends SpreadsheetSeeder 977 | { 978 | /** 979 | * Run the database seeds. 980 | * 981 | * @return void 982 | */ 983 | public function settings(SpreadsheetSeederSettings $set) 984 | { 985 | $set->file = '/database/seeds/users.xlsx'; 986 | $set->inputEncodings = ['UTF-8', 'ISO-8859-1']; 987 | $set->outputEncoding = 'UTF-8'; 988 | } 989 | } 990 | ``` 991 | 992 | #### Postgres Sequence Counters 993 | When using Postgres, Excel Seeder for Laravel will automatically update Postgres sequence counters for auto-incrementing id columns. 994 | 995 | MySQL automatically handles the sequence counter for its auto-incrementing columns. 996 | 997 | ## License 998 | Excel Seeder for Laravel is open-sourced software licensed under the MIT license. 999 | 1000 | ## Changes 1001 | #### 3.4.2 1002 | - Update to Laravel 12 PR #32 (contributed by @raarevalo96) 1003 | - Support running tests with PHPUnit 12 1004 | #### 3.4.1 1005 | - Fix issue #27. Add MariaDB containers and MariaDB tests. Improve test cases for empty values and explicit null values. 1006 | #### 3.4.0 1007 | - Add support for Laravel 11, issue #25 1008 | - initial fix for issue #26, incompatible updates in Orchestra test framework 1009 | #### 3.3.3 1010 | - Fix for RefreshDatabase, issue #19 1011 | #### 3.3.2 1012 | - Update tests for Laravel 10.x 1013 | #### 3.3.1 1014 | - Finish the tests for truncate tables on SQLite, MySQL, and Postgres. 1015 | #### 3.3.0 1016 | - Make Postgres Sequence Counters More Robust 1017 | #### 3.2.0 1018 | - add [parsers](#parsers) setting that enables closures to be run on columns 1019 | #### 3.1.0 1020 | - Add YAML [text output](#excel-text-output-for-branch-diffs) as an alternative to markdown, which is intended to work better with line-oriented diff editors 1021 | - Improve [settings configuration](#basic-usage) by adding settings method that passes the settings object and supports code completion, 1022 | instead of overriding run method and using magic methods that prevent code completion. 1023 | Original method is still supported for backward compatibility. 1024 | #### 3.0.0 1025 | - update composer.json to add support for Laravel 9.x and `doctrine\dbal` 3.x 1026 | - See [Installation](#installation) for new instructions to require DBAL 2.x package for Laravel 5.8, 6.x, 7.x legacy versions. 1027 | - Updated to 3.0.0 because DBAL update breaks backward compatibility of [Installation](#installation) process by requiring DBAL 2.x package for Laravel 5.8, 6.x, 7.x legacy versions. 1028 | #### 2.3.1 1029 | - fix bug #10 (contributed by @tezu35) 1030 | - add date time test cases pertaining to #10 1031 | - refactor code base to decouple header and row import transformers from spreadsheet reader 1032 | - implement github actions for automated testing of all supported php and laravel framework versions 1033 | #### 2.3.0 1034 | - refactor code base to decouple readers and writers and eliminate mediator 1035 | - add ability to set $this->file to an instance of Symfony Finder 1036 | - automatically update Postgres sequence numbers when using Postgres 1037 | - run tests on 5.8, 6.x, 7.x, 8.x, update composer.json, and document 1038 | #### 2.2.0 1039 | - added `xl:seed` command to specify individual sheets as suggested in issue #8 1040 | #### 2.1.15 1041 | - update truncate table to disable foreign key integrity constraints issue #8 1042 | #### 2.1.14 1043 | - fix for change to mb_convert_encoding in PHP 8 issue #7 (contributed by @mw7147) 1044 | #### 2.1.13 1045 | - fix bug in text output tables: deleted worksheets were not deleted from text output tables 1046 | - refactor text output tables 1047 | #### 2.1.12 1048 | - enhance progress messages to show progress for each chunk: number of rows processed, memory usage, and processing time 1049 | - fix memory leaks in laravel-excel-seeder 1050 | - fix memory leaks in laravel framework 1051 | - add configuration setting to specify date formats for columns that Carbon cannot automatically parse 1052 | - add unit tests for date parsing 1053 | #### 2.1.11 1054 | - improved date support 1055 | #### 2.1.10 1056 | - Add `limit` feature 1057 | - Organize documentation 1058 | - Add `limit` test 1059 | #### 2.1.9 1060 | - Fix bug: v2.1.8 table name is not determine properly when worksheet name is mapped 1061 | - Markdown output: save formulas and comments outside (to the right) of region defined by header columns 1062 | - Testing 1063 | - Add test for table name determination when worksheet names are mapped 1064 | - Refactor test namespaces to correspond to test names 1065 | - Move test-specific example data to laravel-excel-seeder-test-data 1066 | - Lock laravel-excel-seeder-test-data to specific version so that test data remains in-sync with package 1067 | #### 2.1.8 1068 | - Fixed "hashable" setting so that it works per documentation. Added "hashable" test. 1069 | - Added batchInsertSize setting to control batch size of insertions. Default 5000 rows. 1070 | - This will address `SQLSTATE[HY000]: General error: 7 number of parameters must be between 0 and 65535` 1071 | - Added chunked reading to read spreadsheet in chunks to conserve memory. Default 5000 rows. 1072 | - This will address out of memory errors when reading large worksheets 1073 | #### 2.1.7 1074 | - Added initial unit test cases for testing this package 1075 | - Refined auto-resolving of table name: 1076 | - if there is only 1 worksheet in a file and the worksheet is not the name of a table, use the base filename 1077 | - otherwise use worksheet name 1078 | - Implemented fix for issue in DBAL library that occurs when columns have upper case characters 1079 | #### 2.1.6 1080 | - Fix tablename setting not being used #5 (contributed by @MeowKim) 1081 | - Add setting to disable text output (default enabled) (contributed by @MeowKim) 1082 | #### 2.1.5 1083 | - Change method for text output markdown files. Create a directory with a separate markdown per sheet instead of one long file. 1084 | #### 2.1.4 1085 | - Fix bug where worksheet prefixed with skipper string was not skipped if it was the first worksheet in the workbook 1086 | #### 2.1.3 1087 | - Parameterize text table output to achieve different text table presentations 1088 | - Fix markdown issue where some tables with empty columns would not be rendered unless the outside column '|' symbols were present 1089 | #### 2.1.2 1090 | - Update text table output to output as markdown file 1091 | #### 2.1.1 1092 | - Fix bug with calling service container that prevented settings from being properly used 1093 | #### 2.1.0 1094 | - Refactor code for better separation of concerns and decrease coupling between classes 1095 | - Add feature to output textual representation of input source spreadsheets for diff 1096 | #### 2.0.6 1097 | - add input encodings and output encodings parameters 1098 | #### 2.0.5 1099 | - add tablesSeeded property to track which tables were seeded 1100 | #### 2.0.4 1101 | - add worksheet to table mapping for mapping worksheet tab names to different table names 1102 | - add example Excel spreadsheet '/database/seeds/xlsx/classicmodels.xlsx' 1103 | #### 2.0.3 1104 | - set default 'skipper' prefix to '%' 1105 | - recognize 'skipper' prefix strings greater than 1 character in length 1106 | #### 2.0.2 1107 | - skip rows that are entirely empty cells 1108 | - skip worksheet tabs that are prefixed with the skipper character. This allows for additional sheets to be used for documentation, alternative designs, or intermediate calculations. 1109 | - issue #2 - workaround to skip reading and calculating cells that are part of a skipped column. Common use case is using `=index(X:X,match(Y,Z:Z,0))` in a skipped column to verify foreign keys. 1110 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bfinlay/laravel-excel-seeder", 3 | "description": "Seed the database with Laravel using Excel, XLSX, XLS, CSV, ODS, Gnumeric, XML, HTML, SLK files", 4 | "homepage": "https://github.com/bfinlay/laravel-excel-seeder", 5 | "authors": [ 6 | { 7 | "name": "Brion Finlay" 8 | } 9 | ], 10 | "keywords": [ "laravel", "csv", "excel", "seed", "seeds", "seeder", "seeding", "open", "libre", "office", "openoffice", "libreoffice"], 11 | "license": "MIT", 12 | "minimum-stability": "stable", 13 | "prefer-stable": true, 14 | "require": { 15 | "php": ">=7.1.3", 16 | "laravel/framework": ">=5.8", 17 | "phpoffice/phpspreadsheet": "^1.10", 18 | "doctrine/dbal": "^2.6|^3.0", 19 | "composer/semver": "*" 20 | }, 21 | "repositories-disabled": [ 22 | { 23 | "type": "path", 24 | "url": "../laravel-excel-seeder-test-data" 25 | }, 26 | { 27 | "type": "path", 28 | "url": "~/development/laravel-expression-grammar" 29 | } 30 | ], 31 | "require-dev": { 32 | "orchestra/testbench": "*", 33 | "orchestra/testbench-core": "^3.0|^4.0|^5.0|~6.48.0|~7.41.0|~8.22.0|^9.0|^10.0", 34 | "bfinlay/laravel-excel-seeder-test-data": "3.4.1", 35 | "angel-source-labs/laravel-expression-grammar": "*" 36 | }, 37 | "autoload": { 38 | "psr-4": { 39 | "bfinlay\\SpreadsheetSeeder\\": "src/" 40 | } 41 | }, 42 | "autoload-dev": { 43 | "psr-4": { 44 | "bfinlay\\SpreadsheetSeeder\\Tests\\": "tests/" 45 | } 46 | }, 47 | "extra": { 48 | "laravel": { 49 | "providers": [ 50 | "bfinlay\\SpreadsheetSeeder\\SpreadsheetSeederServiceProvider" 51 | ] 52 | } 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /docker/custom.cnf: -------------------------------------------------------------------------------- 1 | [mysqld] 2 | max_allowed_packet=16M 3 | ; default-authentication-plugin=mysql_native_password 4 | authentication_policy='mysql_native_password' 5 | -------------------------------------------------------------------------------- /examples/classicmodels.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/bfinlay/laravel-excel-seeder/6881b68f95e5678431fcd1fc584518f876012aaf/examples/classicmodels.xlsx -------------------------------------------------------------------------------- /examples/classicmodels/employees.yaml: -------------------------------------------------------------------------------- 1 | Table: employees 2 | Header: 3 | - id 4 | - last_name 5 | - first_name 6 | - extension 7 | - email 8 | - office_id 9 | - '%office' 10 | - superior_id 11 | - '%superior' 12 | - job_title 13 | Rows: 14 | 1: 15 | id: 1002 16 | last_name: Murphy 17 | first_name: Diane 18 | extension: x5800 19 | email: 'dmurphy@classicmodelcars.com' 20 | office_id: 1 21 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 22 | superior_id: null 23 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 24 | job_title: President 25 | 10: '=A2+5000' 26 | 2: 27 | id: 1056 28 | last_name: Patterson 29 | first_name: Mary 30 | extension: x4611 31 | email: 'mpatterso@classicmodelcars.com' 32 | office_id: 1 33 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 34 | superior_id: 1002 35 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 36 | job_title: VP Sales 37 | 10: '=A3+5000' 38 | 3: 39 | id: 1076 40 | last_name: Firrelli 41 | first_name: Jeff 42 | extension: x9273 43 | email: 'jfirrelli@classicmodelcars.com' 44 | office_id: 1 45 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 46 | superior_id: 1002 47 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 48 | job_title: VP Marketing 49 | 10: '=A4+5000' 50 | 4: 51 | id: 1088 52 | last_name: Patterson 53 | first_name: William 54 | extension: x4871 55 | email: 'wpatterson@classicmodelcars.com' 56 | office_id: 6 57 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 58 | superior_id: 1056 59 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 60 | job_title: Sales Manager (APAC) 61 | 10: '=A5+5000' 62 | 5: 63 | id: 1102 64 | last_name: Bondur 65 | first_name: Gerard 66 | extension: x5408 67 | email: 'gbondur@classicmodelcars.com' 68 | office_id: 4 69 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 70 | superior_id: 1056 71 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 72 | job_title: Sale Manager (EMEA) 73 | 10: '=A6+5000' 74 | 6: 75 | id: 1143 76 | last_name: Bow 77 | first_name: Anthony 78 | extension: x5428 79 | email: 'abow@classicmodelcars.com' 80 | office_id: 1 81 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 82 | superior_id: 1056 83 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 84 | job_title: Sales Manager (NA) 85 | 10: '=A7+5000' 86 | 7: 87 | id: 1165 88 | last_name: Jennings 89 | first_name: Leslie 90 | extension: x3291 91 | email: 'ljennings@classicmodelcars.com' 92 | office_id: 1 93 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 94 | superior_id: 1143 95 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 96 | job_title: Sales Rep 97 | 10: '=A8+5000' 98 | 8: 99 | id: 1166 100 | last_name: Thompson 101 | first_name: Leslie 102 | extension: x4065 103 | email: 'lthompson@classicmodelcars.com' 104 | office_id: 1 105 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 106 | superior_id: 1143 107 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 108 | job_title: Sales Rep 109 | 10: '=A9+5000' 110 | 9: 111 | id: 1188 112 | last_name: Firrelli 113 | first_name: Julie 114 | extension: x2173 115 | email: 'jfirrelli@classicmodelcars.com' 116 | office_id: 2 117 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 118 | superior_id: 1143 119 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 120 | job_title: Sales Rep 121 | 10: '=A10+5000' 122 | 10: 123 | id: 1216 124 | last_name: Patterson 125 | first_name: Steve 126 | extension: x4334 127 | email: 'spatterson@classicmodelcars.com' 128 | office_id: 2 129 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 130 | superior_id: 1143 131 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 132 | job_title: Sales Rep 133 | 10: '=A11+5000' 134 | 11: 135 | id: 1286 136 | last_name: Tseng 137 | first_name: Foon Yue 138 | extension: x2248 139 | email: 'ftseng@classicmodelcars.com' 140 | office_id: 3 141 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 142 | superior_id: 1143 143 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 144 | job_title: Sales Rep 145 | 10: '=A12+5000' 146 | 12: 147 | id: 1323 148 | last_name: Vanauf 149 | first_name: George 150 | extension: x4102 151 | email: 'gvanauf@classicmodelcars.com' 152 | office_id: 3 153 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 154 | superior_id: 1143 155 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 156 | job_title: Sales Rep 157 | 10: '=A13+5000' 158 | 13: 159 | id: 1337 160 | last_name: Bondur 161 | first_name: Loui 162 | extension: x6493 163 | email: 'lbondur@classicmodelcars.com' 164 | office_id: 4 165 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 166 | superior_id: 1102 167 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 168 | job_title: Sales Rep 169 | 10: '=A14+5000' 170 | 14: 171 | id: 1370 172 | last_name: Hernandez 173 | first_name: Gerard 174 | extension: x2028 175 | email: 'ghernande@classicmodelcars.com' 176 | office_id: 4 177 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 178 | superior_id: 1102 179 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 180 | job_title: Sales Rep 181 | 10: '=A15+5000' 182 | 15: 183 | id: 1401 184 | last_name: Castillo 185 | first_name: Pamela 186 | extension: x2759 187 | email: 'pcastillo@classicmodelcars.com' 188 | office_id: 4 189 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 190 | superior_id: 1102 191 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 192 | job_title: Sales Rep 193 | 10: '=A16+5000' 194 | 16: 195 | id: 1501 196 | last_name: Bott 197 | first_name: Larry 198 | extension: x2311 199 | email: 'lbott@classicmodelcars.com' 200 | office_id: 7 201 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 202 | superior_id: 1102 203 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 204 | job_title: Sales Rep 205 | 10: '=A17+5000' 206 | 17: 207 | id: 1504 208 | last_name: Jones 209 | first_name: Barry 210 | extension: x102 211 | email: 'bjones@classicmodelcars.com' 212 | office_id: 7 213 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 214 | superior_id: 1102 215 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 216 | job_title: Sales Rep 217 | 10: '=A18+5000' 218 | 18: 219 | id: 1611 220 | last_name: Fixter 221 | first_name: Andy 222 | extension: x101 223 | email: 'afixter@classicmodelcars.com' 224 | office_id: 6 225 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 226 | superior_id: 1088 227 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 228 | job_title: Sales Rep 229 | 10: '=A19+5000' 230 | 19: 231 | id: 1612 232 | last_name: Marsh 233 | first_name: Peter 234 | extension: x102 235 | email: 'pmarsh@classicmodelcars.com' 236 | office_id: 6 237 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 238 | superior_id: 1088 239 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 240 | job_title: Sales Rep 241 | 10: '=A20+5000' 242 | 20: 243 | id: 1619 244 | last_name: King 245 | first_name: Tom 246 | extension: x103 247 | email: 'tking@classicmodelcars.com' 248 | office_id: 6 249 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 250 | superior_id: 1088 251 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 252 | job_title: Sales Rep 253 | 10: '=A21+5000' 254 | 21: 255 | id: 1621 256 | last_name: Nishi 257 | first_name: Mami 258 | extension: x101 259 | email: 'mnishi@classicmodelcars.com' 260 | office_id: 5 261 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 262 | superior_id: 1056 263 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 264 | job_title: Sales Rep 265 | 10: '=A22+5000' 266 | 22: 267 | id: 1625 268 | last_name: Kato 269 | first_name: Yoshimi 270 | extension: x102 271 | email: 'ykato@classicmodelcars.com' 272 | office_id: 5 273 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 274 | superior_id: 1621 275 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 276 | job_title: Sales Rep 277 | 10: '=A23+5000' 278 | 23: 279 | id: 1702 280 | last_name: Gerard 281 | first_name: Martin 282 | extension: x2312 283 | email: 'mgerard@classicmodelcars.com' 284 | office_id: 4 285 | '%office': '=INDEX(offices!B:B,MATCH(F:F,offices!A:A,0))' 286 | superior_id: 1102 287 | '%superior': '=INDEX(B:B,MATCH(H:H,A:A,0))' 288 | job_title: Sales Rep 289 | 10: '=A24+5000' 290 | RowCount: 23 291 | -------------------------------------------------------------------------------- /examples/classicmodels/offices.yaml: -------------------------------------------------------------------------------- 1 | Table: offices 2 | Header: 3 | - id 4 | - city 5 | - phone 6 | - address_line_1 7 | - address_line_2 8 | - state 9 | - country 10 | - postal_code 11 | - territory 12 | Rows: 13 | 1: 14 | id: 1 15 | city: San Francisco 16 | phone: +1 650 219 4782 17 | address_line_1: 100 Market Street 18 | address_line_2: Suite 300 19 | state: CA 20 | country: USA 21 | postal_code: 94080 22 | territory: NA 23 | 2: 24 | id: 2 25 | city: Boston 26 | phone: +1 215 837 0825 27 | address_line_1: 1550 Court Place 28 | address_line_2: Suite 102 29 | state: MA 30 | country: USA 31 | postal_code: 2107 32 | territory: NA 33 | 3: 34 | id: 3 35 | city: NYC 36 | phone: +1 212 555 3000 37 | address_line_1: 523 East 53rd Street 38 | address_line_2: apt. 5A 39 | state: NY 40 | country: USA 41 | postal_code: 10022 42 | territory: NA 43 | 4: 44 | id: 4 45 | city: Paris 46 | phone: +33 14 723 4404 47 | address_line_1: 43 Rue Jouffroy D'abbans 48 | address_line_2: 49 | state: 50 | country: France 51 | postal_code: 75017 52 | territory: EMEA 53 | 5: 54 | id: 5 55 | city: Tokyo 56 | phone: +81 33 224 5000 57 | address_line_1: '4-1 Kioicho' 58 | address_line_2: 59 | state: 'Chiyoda-Ku' 60 | country: Japan 61 | postal_code: '102-8578' 62 | territory: Japan 63 | 6: 64 | id: 6 65 | city: Sydney 66 | phone: +61 2 9264 2451 67 | address_line_1: '5-11 Wentworth Avenue' 68 | address_line_2: 'Floor #2' 69 | state: 70 | country: Australia 71 | postal_code: NSW 2010 72 | territory: APAC 73 | 7: 74 | id: 7 75 | city: London 76 | phone: +44 20 7877 2041 77 | address_line_1: 25 Old Broad Street 78 | address_line_2: Level 7 79 | state: 80 | country: UK 81 | postal_code: EC2N 1HN 82 | territory: EMEA 83 | RowCount: 7 84 | -------------------------------------------------------------------------------- /examples/classicmodels/product_lines.yaml: -------------------------------------------------------------------------------- 1 | Table: product_lines 2 | Header: 3 | - id 4 | - product_line 5 | - description 6 | Rows: 7 | 1: 8 | id: 1 9 | product_line: Classic Cars 10 | description: 'Attention car enthusiasts: Make your wildest car ownership dreams come true. Whether you are looking for classic muscle cars, dream sports cars or movie-inspired miniatures, you will find great choices in this category. These replicas feature superb attention to detail and craftsmanship and offer features such as working steering system, opening forward compartment, opening rear trunk with removable spare wheel, 4-wheel independent spring suspension, and so on. The models range in size from 1:10 to 1:24 scale and include numerous limited edition and several out-of-production vehicles. All models include a certificate of authenticity from their manufacturers and come fully assembled and ready for display in the home or office.' 11 | 2: 12 | id: 2 13 | product_line: Motorcycles 14 | description: 'Our motorcycles are state of the art replicas of classic as well as contemporary motorcycle legends such as Harley Davidson, Ducati and Vespa. Models contain stunning details such as official logos, rotating wheels, working kickstand, front suspension, gear-shift lever, footbrake lever, and drive chain. Materials used include diecast and plastic. The models range in size from 1:10 to 1:50 scale and include numerous limited edition and several out-of-production vehicles. All models come fully assembled and ready for display in the home or office. Most include a certificate of authenticity.' 15 | 3: 16 | id: 3 17 | product_line: Planes 18 | description: 'Unique, diecast airplane and helicopter replicas suitable for collections, as well as home, office or classroom decorations. Models contain stunning details such as official logos and insignias, rotating jet engines and propellers, retractable wheels, and so on. Most come fully assembled and with a certificate of authenticity from their manufacturers.' 19 | 4: 20 | id: 4 21 | product_line: Ships 22 | description: 'The perfect holiday or anniversary gift for executives, clients, friends, and family. These handcrafted model ships are unique, stunning works of art that will be treasured for generations! They come fully assembled and ready for display in the home or office. We guarantee the highest quality, and best value.' 23 | 5: 24 | id: 5 25 | product_line: Trains 26 | description: 'Model trains are a rewarding hobby for enthusiasts of all ages. Whether you''re looking for collectible wooden trains, electric streetcars or locomotives, you''ll find a number of great choices for any budget within this category. The interactive aspect of trains makes toy trains perfect for young children. The wooden train sets are ideal for children under the age of 5.' 27 | 6: 28 | id: 6 29 | product_line: Trucks and Buses 30 | description: 'The Truck and Bus models are realistic replicas of buses and specialized trucks produced from the early 1920s to present. The models range in size from 1:12 to 1:50 scale and include numerous limited edition and several out-of-production vehicles. Materials used include tin, diecast and plastic. All models include a certificate of authenticity from their manufacturers and are a perfect ornament for the home and office.' 31 | 7: 32 | id: 7 33 | product_line: Vintage Cars 34 | description: 'Our Vintage Car models realistically portray automobiles produced from the early 1900s through the 1940s. Materials used include Bakelite, diecast, plastic and wood. Most of the replicas are in the 1:18 and 1:24 scale sizes, which provide the optimum in detail and accuracy. Prices range from $30.00 up to $180.00 for some special limited edition replicas. All models include a certificate of authenticity from their manufacturers and come fully assembled and ready for display in the home or office.' 35 | RowCount: 7 36 | -------------------------------------------------------------------------------- /examples/users.csv: -------------------------------------------------------------------------------- 1 | id,name,email,email_verified_at,password,order 2 | 1,Foo,Foo@Bar.com,2019-01-23 21:38:54,password,10 3 | 5,John,John@Doe.com,2019-01-23 21:38:54,password,3 -------------------------------------------------------------------------------- /examples/users/users.yaml: -------------------------------------------------------------------------------- 1 | Table: users 2 | Header: 3 | - name 4 | - email 5 | - email_verified_at 6 | - password 7 | Rows: 8 | 1: 9 | name: Foo 10 | email: Foo@Bar.com 11 | email_verified_at: 2019-01-23 21:38:54 12 | password: password 13 | 2: 14 | name: John 15 | email: John@Doe.com 16 | email_verified_at: 2019-01-23 21:38:54 17 | password: password 18 | RowCount: 2 19 | -------------------------------------------------------------------------------- /src/ChunkReadFilter.php: -------------------------------------------------------------------------------- 1 | worksheetName = $worksheetName; 25 | } 26 | 27 | /** Set the list of rows that we want to read */ 28 | public function setRows($startRow, $chunkSize) { 29 | $this->startRow = $startRow; 30 | $this->endRow = $startRow + $chunkSize; 31 | } 32 | 33 | public function readCell($column, $row, $worksheetName = '') { 34 | // Only read the heading row, and the configured rows 35 | if ( 36 | $row >= $this->startRow && 37 | $row < $this->endRow && 38 | ($worksheetName == $this->worksheetName || 39 | $worksheetName == '') 40 | ) { 41 | return true; 42 | } 43 | return false; 44 | } 45 | } -------------------------------------------------------------------------------- /src/Console/SeedCommand.php: -------------------------------------------------------------------------------- 1 | argument('class'); 20 | if (is_null($class)) 21 | $class = $this->option('class'); 22 | else 23 | $this->input->setOption('class', $class); // support Laravel 6, 7 24 | 25 | if (($class === 'SpreadsheetSeeder' || $class === '#') && 26 | class_exists(SpreadsheetSeeder::class)) { 27 | $class = SpreadsheetSeeder::class; 28 | 29 | return $this->laravel->make($class) 30 | ->setContainer($this->laravel) 31 | ->setCommand($this); 32 | } 33 | 34 | return parent::getSeeder(); 35 | } 36 | 37 | protected function hasDefinedClassArgument($arguments) 38 | { 39 | foreach ($arguments as $argument) 40 | { 41 | if ( $argument[0] == 'class' ) return true; 42 | } 43 | 44 | return false; 45 | } 46 | 47 | protected function getArguments() 48 | { 49 | $arguments = parent::getArguments(); 50 | 51 | // Laravel 8 defines class as an argument. Laravel 5.8, 6, and 7 do not. 52 | if (! $this->hasDefinedClassArgument($arguments)) 53 | $arguments[] = ['class', InputArgument::OPTIONAL, 'The class name of the root seeder', null]; 54 | 55 | $arguments[] = ['sheet', InputArgument::IS_ARRAY | InputArgument::OPTIONAL, 'The name of the worksheet to seed', null]; 56 | 57 | return $arguments; 58 | } 59 | 60 | protected function getOptions() 61 | { 62 | return array_merge(parent::getOptions(), [ 63 | ['sheet', null, InputOption::VALUE_IS_ARRAY | InputOption::VALUE_OPTIONAL, 'The name of the worksheet to seed'], 64 | ]); 65 | } 66 | 67 | protected function setSheet() 68 | { 69 | $sheets = array_merge( 70 | Arr::wrap($this->argument('sheet')), 71 | Arr::wrap($this->option('sheet')) 72 | ); 73 | 74 | if (is_array($sheets) && count($sheets) > 0) { 75 | $settings = resolve(SpreadsheetSeederSettings::class); 76 | $settings->worksheets = $sheets; 77 | } 78 | } 79 | 80 | public function handle() 81 | { 82 | $this->setSheet(); 83 | return parent::handle(); // TODO: Change the autogenerated stub 84 | } 85 | } -------------------------------------------------------------------------------- /src/Events/Console.php: -------------------------------------------------------------------------------- 1 | message = $message; 31 | $this->level = $level; 32 | } 33 | } -------------------------------------------------------------------------------- /src/FileIterator.php: -------------------------------------------------------------------------------- 1 | settings = resolve(SpreadsheetSeederSettings::class); 26 | 27 | $flags = \FilesystemIterator::KEY_AS_FILENAME; 28 | 29 | $globs = $this->settings->file; 30 | if (! is_array($globs)) { 31 | $globs = [$globs]; 32 | } 33 | 34 | foreach ($globs as $glob) { 35 | if (is_dir($glob)) { 36 | $glob = dirname($glob) . "/*." . $this->settings->extension; 37 | } 38 | 39 | $it = new \GlobIterator(base_path($glob), $flags); 40 | $this->append($it); 41 | $this->count += $it->count(); 42 | } 43 | } 44 | 45 | public function valid() 46 | { 47 | while ( 48 | parent::valid() && 49 | $this->shouldSkip() 50 | ) 51 | { 52 | $this->next(); 53 | } 54 | 55 | return parent::valid(); 56 | } 57 | 58 | /** 59 | * Returns true if the file should be skipped. Currently this only checks for a leading "~" character in the 60 | * filename, which indicates that the file is an Excel temporary file. 61 | * 62 | * @return bool 63 | */ 64 | public function shouldSkip() { 65 | if (substr(parent::current()->getFilename(), 0, 1) === "~" ) return true; 66 | 67 | return false; 68 | } 69 | 70 | public function count() 71 | { 72 | return $this->count; 73 | } 74 | 75 | public function hasResults() 76 | { 77 | return $this->count() > 0; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/MySqlGrammar.php: -------------------------------------------------------------------------------- 1 | rows = $rows; 31 | $this->startRow = $rows->startRow; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Readers/Events/FileFinish.php: -------------------------------------------------------------------------------- 1 | file = $file; 26 | } 27 | } -------------------------------------------------------------------------------- /src/Readers/Events/RowFinish.php: -------------------------------------------------------------------------------- 1 | row = $row; 31 | $this->columns = $columns; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Readers/Events/RowStart.php: -------------------------------------------------------------------------------- 1 | row = $row; 31 | $this->columns = $columns; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Readers/Events/SheetFinish.php: -------------------------------------------------------------------------------- 1 | sheetName = $sheetName; 44 | $this->tableName = $tableName; 45 | $this->startRow = $startRow; 46 | $this->header = $header; 47 | } 48 | } -------------------------------------------------------------------------------- /src/Readers/HeaderImporter.php: -------------------------------------------------------------------------------- 1 | settings = resolve(SpreadsheetSeederSettings::class); 48 | } 49 | 50 | public function import(array $headerRow) 51 | { 52 | $this->headerRow = $headerRow; 53 | $this->makeHeader(); 54 | 55 | return $this->toArray(); 56 | } 57 | 58 | protected function makeHeader() 59 | { 60 | if (!empty($this->settings->mapping)) { 61 | $this->makeMappingHeader(); 62 | } else { 63 | $this->makeSheetHeader(); 64 | } 65 | $this->makeAdditionalColumns(); 66 | 67 | } 68 | 69 | protected function makeMappingHeader() 70 | { 71 | $this->rawColumns = $this->settings->mapping; 72 | foreach($this->rawColumns as $key => $value) { 73 | $this->columnNumbersByNameMap[$value] = $key; 74 | $this->columnNamesByNumberMap[$key] = $value; 75 | } 76 | } 77 | 78 | protected function makeSheetHeader() 79 | { 80 | foreach ($this->headerRow as $columnName) { 81 | $this->rawColumns[] = $columnName; 82 | if (!$this->skipColumn($columnName)) { 83 | $columnName = $this->columnAlias($columnName); 84 | $this->columnNumbersByNameMap[$columnName] = count($this->rawColumns) - 1; 85 | $this->columnNamesByNumberMap[count($this->rawColumns) - 1] = $columnName; 86 | } 87 | } 88 | } 89 | 90 | protected function makeAdditionalColumns() 91 | { 92 | foreach ($this->settings->addColumns as $columnName) { 93 | $this->rawColumns[] = $columnName; 94 | $this->columnNumbersByNameMap[$columnName] = count($this->rawColumns) - 1; 95 | $this->columnNamesByNumberMap[count($this->rawColumns) - 1] = $columnName; 96 | } 97 | } 98 | 99 | protected function columnAlias($columnName) 100 | { 101 | $columnName = isset($this->settings->aliases[$columnName]) ? $this->settings->aliases[$columnName] : $columnName; 102 | return $columnName; 103 | } 104 | 105 | protected function skipColumn($columnName) 106 | { 107 | return 108 | in_array($columnName, $this->settings->skipColumns) || 109 | $this->settings->skipper == substr($columnName, 0, strlen($this->settings->skipper)); 110 | } 111 | 112 | public function toArray() 113 | { 114 | return $this->columnNamesByNumberMap; 115 | } 116 | 117 | public function rawColumns() 118 | { 119 | return $this->rawColumns; 120 | } 121 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/ChunkReadFilter.php: -------------------------------------------------------------------------------- 1 | worksheetName = $worksheetName; 25 | } 26 | 27 | /** Set the list of rows that we want to read */ 28 | public function setRows($startRow, $chunkSize) { 29 | $this->startRow = $startRow; 30 | $this->endRow = $startRow + $chunkSize; 31 | } 32 | 33 | public function readCell($column, $row, $worksheetName = '') { 34 | // Only read the heading row, and the configured rows 35 | if ( 36 | $row >= $this->startRow && 37 | $row < $this->endRow && 38 | ($worksheetName == $this->worksheetName || 39 | $worksheetName == '') 40 | ) { 41 | return true; 42 | } 43 | return false; 44 | } 45 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SourceChunk.php: -------------------------------------------------------------------------------- 1 | worksheet = $worksheet; 47 | $this->settings = resolve(SpreadsheetSeederSettings::class); 48 | $this->tableName = $this->settings->tablename; 49 | $this->startRow = $startRow; 50 | $this->rowIterator = $this->worksheet->getRowIterator($this->startRow); 51 | $this->header = $header; 52 | } 53 | 54 | /** 55 | * @inheritDoc 56 | */ 57 | public function current() 58 | { 59 | return new SourceRow($this->rowIterator->current(), $this->header->toArray()); 60 | } 61 | 62 | /** 63 | * @inheritDoc 64 | */ 65 | public function next() 66 | { 67 | $this->rowIterator->next(); 68 | } 69 | 70 | /** 71 | * @inheritDoc 72 | */ 73 | public function key() 74 | { 75 | return $this->rowIterator->key(); 76 | } 77 | 78 | /** 79 | * @inheritDoc 80 | */ 81 | public function valid() 82 | { 83 | return $this->rowIterator->valid(); 84 | } 85 | 86 | /** 87 | * @inheritDoc 88 | */ 89 | public function rewind() 90 | { 91 | $this->rowIterator = $this->worksheet->getRowIterator($this->startRow); 92 | } 93 | 94 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SourceFile.php: -------------------------------------------------------------------------------- 1 | file = $file; 47 | $this->settings = resolve(SpreadsheetSeederSettings::class); 48 | 49 | if (!$this->shouldSkip()) $this->getSheetNames(); 50 | } 51 | 52 | /** 53 | * Returns true if the file should be skipped. Currently this only checks for a leading "~" character in the 54 | * filename, which indicates that the file is an Excel temporary file. 55 | * 56 | * @return bool 57 | */ 58 | public function shouldSkip() { 59 | if (substr($this->file->getFilename(), 0, 1) === "~" ) return true; 60 | 61 | return false; 62 | } 63 | 64 | public function getSheetNames() { 65 | if (!isset($this->sheetNames)) { 66 | $filename = $this->file->getPathname(); 67 | $this->fileType = IOFactory::identify($filename); 68 | $this->reader = IOFactory::createReader($this->fileType); 69 | if ($this->fileType == "Csv" && !empty($this->settings->delimiter)) { 70 | $this->reader->setDelimiter($this->settings->delimiter); 71 | } 72 | 73 | // fastest 74 | if (method_exists($this->reader, "listWorksheetNames")) { 75 | $this->sheetNames = $this->reader->listWorksheetNames($filename); 76 | } 77 | // slower 78 | else if (method_exists($this->reader, "listWorksheetInfo")) { 79 | /** 80 | * worksheet info array: 81 | * fake_names_100k.xlsx 82 | * - worksheetName = "fake_names_100k" 83 | * - lastColumnLetter = "AS" 84 | * - lastCoumnIndex = 44 85 | * - totalRows = "100001" 86 | * - totalColumns = 45 87 | */ 88 | $this->sheetNames = []; 89 | $worksheetInfo = $this->reader->listWorksheetInfo($filename); 90 | foreach ($worksheetInfo as $info) { 91 | $this->sheetNames[] = $info['worksheetName']; 92 | } 93 | } 94 | // slowest 95 | else { 96 | $this->reader->setReadFilter(new SourceFileReadFilter()); 97 | $workbook = $this->reader->load($filename); 98 | $this->sheetNames = $workbook->getSheetNames(); 99 | } 100 | } 101 | return $this->sheetNames; 102 | } 103 | 104 | /** 105 | * @inheritDoc 106 | */ 107 | public function current() 108 | { 109 | $sheetName = $this->sheetNames[$this->sheetIndex]; 110 | 111 | $sourceSheet = new SourceSheet($this->file->getPathname(), $this->fileType, $sheetName); 112 | if (count($this->sheetNames) == 1) { 113 | $sourceSheet->setSingleSheet(); 114 | } 115 | return $sourceSheet; 116 | } 117 | 118 | protected function shouldSkipSheet($sheetName) { 119 | return 120 | $this->hasSkipperPrefix($sheetName) || 121 | $this->isIncludedInSkipSheets($sheetName) || 122 | $this->isExcludedFromWorksheets($sheetName); 123 | } 124 | 125 | protected function hasSkipperPrefix($sheetName) 126 | { 127 | return $this->settings->skipper == substr($sheetName, 0, strlen($this->settings->skipper)); 128 | } 129 | 130 | protected function isIncludedInSkipSheets($sheetName) 131 | { 132 | return is_array($this->settings->skipSheets) && in_array($sheetName, $this->settings->skipSheets); 133 | } 134 | 135 | protected function isExcludedFromWorksheets($sheetName) 136 | { 137 | return 138 | is_array($this->settings->worksheets) && 139 | count($this->settings->worksheets) > 0 && 140 | ! in_array($sheetName, $this->settings->worksheets); 141 | } 142 | 143 | /** 144 | * @inheritDoc 145 | */ 146 | public function next() 147 | { 148 | $this->sheetIndex++; 149 | } 150 | 151 | /** 152 | * @inheritDoc 153 | */ 154 | public function key() 155 | { 156 | return $this->sheetIndex; 157 | } 158 | 159 | /** 160 | * @inheritDoc 161 | */ 162 | public function valid() 163 | { 164 | while ( 165 | $this->sheetIndex < count($this->sheetNames) && 166 | $this->shouldSkipSheet($this->sheetNames[$this->sheetIndex]) 167 | ) 168 | { 169 | $this->sheetIndex++; 170 | } 171 | 172 | return $this->sheetIndex < count($this->sheetNames); 173 | } 174 | 175 | /** 176 | * @inheritDoc 177 | */ 178 | public function rewind() 179 | { 180 | return $this->sheetIndex = 0; 181 | } 182 | 183 | public function getFilename() { 184 | return $this->file->getFilename(); 185 | } 186 | 187 | public function getPathname() { 188 | return $this->file->getPathname(); 189 | } 190 | 191 | public function getDelimiter() { 192 | return $this->reader->getDelimiter(); 193 | } 194 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SourceFileReadFilter.php: -------------------------------------------------------------------------------- 1 | sheetRow = $headerRow; 42 | $this->settings = resolve(SpreadsheetSeederSettings::class); 43 | $this->headerImporter = new HeaderImporter(); 44 | $this->makeHeader(); 45 | } 46 | 47 | private function makeHeader() 48 | { 49 | if (!empty($this->settings->mapping)) { 50 | $this->rawColumns = $this->settings->mapping; 51 | } else { 52 | $this->rawColumns = $this->readSheetHeader(); 53 | } 54 | 55 | $this->headerImporter->import($this->rawColumns); 56 | } 57 | 58 | private function readSheetHeader() { 59 | $rawColumns = []; 60 | 61 | foreach ($this->sheetRow->getCellIterator() as $cell) { 62 | $rawColumns[] = $cell->getCalculatedValue(); 63 | } 64 | 65 | return $rawColumns; 66 | } 67 | 68 | public function toArray() { 69 | return $this->headerImporter->toArray(); 70 | } 71 | 72 | public function rawColumns() { 73 | return $this->rawColumns; 74 | } 75 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SourceRow.php: -------------------------------------------------------------------------------- 1 | column name 56 | */ 57 | public function __construct(Row $row, $columnNames) 58 | { 59 | $this->sheetRow = $row; 60 | $this->columnNames = $columnNames; 61 | $this->rowImporter = new RowImporter($columnNames); 62 | $this->settings = resolve(SpreadsheetSeederSettings::class); 63 | $this->makeRow(); 64 | } 65 | 66 | public function toArray() { 67 | return $this->rowArray; 68 | } 69 | 70 | public function rawRow() { 71 | return $this->rawRowArray; 72 | } 73 | 74 | public function isValid() { 75 | return $this->rowImporter->isValid(); 76 | } 77 | 78 | private function makeRow() { 79 | $nullRow = true; 80 | $cellIterator = $this->sheetRow->getCellIterator(); 81 | $colIndex = 0; 82 | 83 | $row = []; 84 | 85 | /** @var Cell $cell */ 86 | foreach($cellIterator as $cell) { 87 | if (isset($this->columnNames[$colIndex])) { 88 | $row[$colIndex] = $cell->getCalculatedValue() ?? new EmptyCell(); 89 | } 90 | $this->rawRowArray[$colIndex] = $cell->getValue(); 91 | 92 | $colIndex++; 93 | } 94 | 95 | $this->rowArray = $this->rowImporter->import($row); 96 | } 97 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SourceSheet.php: -------------------------------------------------------------------------------- 1 | fileName = $fileName; 97 | $this->fileType = $fileType; 98 | $this->worksheetName = $worksheetName; 99 | $this->settings = resolve(SpreadsheetSeederSettings::class); 100 | $this->tableName = $this->settings->tablename; 101 | $this->rowOffset = $this->settings->offset + 1; 102 | 103 | $this->createReadFilter(); 104 | $this->createReader(); 105 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'construct sheet'); 106 | $this->loadHeader(); 107 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'load header'); 108 | 109 | $this->header = $this->constructHeaderRow(); 110 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'construct header'); 111 | } 112 | 113 | private function createReadFilter() { 114 | $this->readFilter = new ChunkReadFilter(); 115 | $this->chunkSize = $this->settings->readChunkSize; 116 | $this->chunkStartRow = $this->rowOffset; 117 | $this->readFilter->setWorksheet($this->worksheetName); 118 | } 119 | 120 | private function createReader() { 121 | $this->reader = IOFactory::createReader($this->fileType); 122 | if ($this->fileType == "Csv" && !empty($this->settings->delimiter)) { 123 | $this->reader->setDelimiter($this->settings->delimiter); 124 | } 125 | $this->reader->setReadFilter($this->readFilter); 126 | } 127 | 128 | private function loadHeader() { 129 | if (!$this->settings->header) return; 130 | 131 | $this->loadChunk(1,1); 132 | $this->rowOffset++; 133 | $this->chunkStartRow = $this->rowOffset; 134 | } 135 | 136 | private function loadChunk($startRow = null, $chunkSize = null) { 137 | if (is_null($startRow)) $startRow = $this->chunkStartRow; 138 | if (is_null($chunkSize)) $chunkSize = $this->chunkSize; 139 | 140 | if ($this->loadedChunk == $startRow) return; 141 | 142 | if (isset($this->worksheet)) $this->worksheet->disconnectCells(); 143 | unset($this->worksheet); 144 | $this->readFilter->setRows($startRow, $chunkSize); 145 | 146 | // reduces time from 10s to 4s on TablenameTest.test_table_name_is_worksheet_name (ClassicModelSeeder) 147 | $this->reader->setLoadSheetsOnly($this->worksheetName); 148 | 149 | $this->workbook = $this->reader->load($this->fileName); 150 | $this->worksheet = $this->workbook->setActiveSheetIndexByName($this->worksheetName); 151 | // This only returns the max row up to the chunk. The only way to get the max row appears to be to load the entire file. 152 | // $this->maxRow = $this->worksheet->getHighestDataRow(); 153 | $this->loadedChunk = $startRow; 154 | 155 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'load chunk'); 156 | } 157 | 158 | private function constructHeaderRow() { 159 | if ($this->settings->header == false) return null; // TODO adjust for mapping 160 | 161 | return new SourceHeader($this->worksheet->getRowIterator()->current(), $this->isCsv()); 162 | } 163 | 164 | public function setTableName($tableName) { 165 | $this->tableName = $tableName; 166 | } 167 | 168 | public function setFileType($fileType) { 169 | $this->fileType = $fileType; 170 | } 171 | 172 | public function getTableName() { 173 | if (isset($this->tableName)) { 174 | return $this->tableName; 175 | } 176 | else if (isset($this->settings->worksheetTableMapping[$this->worksheetName])) { 177 | $this->tableName = $this->settings->worksheetTableMapping[$this->worksheetName]; 178 | } 179 | else if ($this->isSingleSheet && !$this->titleIsTable()) { 180 | $this->tableName = pathinfo($this->fileName)["filename"]; 181 | } 182 | else { 183 | $this->tableName = $this->worksheetName; 184 | } 185 | 186 | return $this->tableName; 187 | } 188 | 189 | public function getHeader() { 190 | return $this->header; 191 | } 192 | 193 | public function isCsv() { 194 | return $this->fileType == "Csv"; 195 | } 196 | 197 | /** 198 | * @inheritDoc 199 | */ 200 | public function current() 201 | { 202 | $this->loadChunk(); 203 | return new SourceChunk($this->worksheet, $this->header, $this->chunkStartRow); 204 | } 205 | 206 | /** 207 | * @inheritDoc 208 | */ 209 | public function next() 210 | { 211 | $this->chunkStartRow += $this->chunkSize; 212 | $this->loadChunk(); 213 | } 214 | 215 | /** 216 | * @inheritDoc 217 | */ 218 | public function key() 219 | { 220 | return $this->chunkStartRow; 221 | } 222 | 223 | /** 224 | * @inheritDoc 225 | */ 226 | public function valid() 227 | { 228 | $this->loadChunk(); 229 | return $this->chunkStartRow <= $this->worksheet->getHighestDataRow(); 230 | } 231 | 232 | /** 233 | * @inheritDoc 234 | */ 235 | public function rewind() 236 | { 237 | $this->chunkStartRow = $this->rowOffset; 238 | $this->loadChunk(); 239 | } 240 | 241 | public function getTitle() { 242 | return $this->worksheet->getTitle(); 243 | } 244 | 245 | public function isUnnamed() { 246 | return $this->isCsv() || preg_match('/^Sheet[0-9]+$/', $this->getTitle()); 247 | } 248 | 249 | public function titleIsTable() { 250 | return DestinationTable::tableExists($this->getTitle()); 251 | } 252 | 253 | public function setSingleSheet($isSingle = true) { 254 | $this->isSingleSheet = $isSingle; 255 | } 256 | 257 | public function isSingleSheet() { 258 | return $this->isSingleSheet; 259 | } 260 | } -------------------------------------------------------------------------------- /src/Readers/PhpSpreadsheet/SpreadsheetReader.php: -------------------------------------------------------------------------------- 1 | settings = resolve(SpreadsheetSeederSettings::class); 52 | } 53 | 54 | /** 55 | * Run the class 56 | * 57 | * @return void 58 | */ 59 | public function boot() 60 | { 61 | Event::listen(FileSeed::class, [$this, 'handleFileSeed']); 62 | } 63 | 64 | /** 65 | * @param $fileSeed FileSeed 66 | */ 67 | public function handleFileSeed($fileSeed) 68 | { 69 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'seed'); 70 | 71 | $this->sourceFile = new SourceFile($fileSeed->file); 72 | 73 | foreach ($this->sourceFile as $this->sourceSheet) { 74 | $this->sheetStart(); 75 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'sheet start'); 76 | 77 | $this->checkColumns(); 78 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'check columns'); 79 | 80 | foreach ($this->sourceSheet as $this->sourceChunk) { 81 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'start chunk iterator'); 82 | 83 | $this->chunkStart(); 84 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'chunk start'); 85 | 86 | $this->processRows(); 87 | $this->chunkFinish(); 88 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'chunk finish'); 89 | 90 | if ($this->exceedsLimit()) break; 91 | 92 | $this->clearChunkMemory(); 93 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'processed'); 94 | } 95 | 96 | $this->sheetFinish(); 97 | } 98 | } 99 | 100 | private function clearChunkMemory() { 101 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'start clear chunk memory'); 102 | $this->rows = new Rows(); 103 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'finish clear chunk memory'); 104 | 105 | } 106 | 107 | private function checkColumns() 108 | { 109 | if ($this->sourceSheet->isCsv() && count($this->sourceSheet->getHeader()->toArray()) == 1) 110 | event(new Console('Found only one column in header. Maybe the delimiter set for the CSV is incorrect: [' . $this->sourceFile->getDelimiter() . ']')); 111 | } 112 | 113 | private function tableName() 114 | { 115 | return isset($this->settings->tablename) ? $this->settings->tablename : $this->sourceSheet->getTableName(); 116 | } 117 | 118 | private function sheetStart() 119 | { 120 | event(new SheetStart($this->sourceSheet->getTitle(), $this->tableName(), $this->sourceSheet->key(), $this->sourceSheet->getHeader()->rawColumns())); 121 | } 122 | 123 | /** 124 | * Process each row of the data source 125 | * 126 | * @return void 127 | */ 128 | private function processRows() 129 | { 130 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'start process rows'); 131 | foreach ($this->sourceChunk as $row) { 132 | $this->rowsProcessed++; 133 | $this->rows->countRowAsProcessed(); 134 | 135 | if (!$row->isValid()) { 136 | $this->rows->countRowAsSkipped(); 137 | continue; 138 | } 139 | 140 | $this->rows->rows[] = $row->toArray(); 141 | $this->rows->rawRows[] = $row->rawRow(); 142 | 143 | if ($this->exceedsLimit()) break; 144 | } 145 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'finish process rows'); 146 | } 147 | 148 | private function exceedsLimit() 149 | { 150 | return isset($this->settings->limit) && $this->rowsProcessed >= $this->settings->limit; 151 | } 152 | 153 | private function chunkStart() 154 | { 155 | $this->rows = new Rows(); 156 | $this->rows->startRow = $this->sourceChunk->key(); 157 | event(new ChunkStart($this->rows)); 158 | } 159 | 160 | private function chunkFinish() 161 | { 162 | event(new ChunkFinish($this->rows)); 163 | } 164 | 165 | private function sheetFinish() 166 | { 167 | event(new SheetFinish($this->sourceSheet->getTitle(), $this->tableName())); 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /src/Readers/RowImporter.php: -------------------------------------------------------------------------------- 1 | column name 44 | */ 45 | public function __construct($columnNames) 46 | { 47 | $this->columnNames = $columnNames; 48 | $this->settings = resolve(SpreadsheetSeederSettings::class); 49 | } 50 | 51 | public function import(array $row) 52 | { 53 | event(new RowStart($row, $this->columnNames)); 54 | $this->rowArray = []; 55 | 56 | foreach($row as $columnIndex => $value) { 57 | if (isset($this->columnNames[$columnIndex])) { 58 | if (!$this->isEmptyCell($value)) $this->nullRow = false; 59 | $columnName = $this->columnNames[$columnIndex]; 60 | $this->rowArray[$columnName] = $this->transformValue($columnName, $value); 61 | } 62 | } 63 | 64 | if ($this->isValid()) { 65 | $this->addTimestamps(); 66 | $this->addColumns(); 67 | } 68 | 69 | event(new RowFinish($this->rowArray, $this->columnNames)); 70 | return $this->rowArray; 71 | } 72 | 73 | /** 74 | * Returns true if all cells meet one of these conditions: 75 | * 1. Cell is primary key column 76 | * 2. Cell is an empty cell 77 | * 3. Cell is an empty string 78 | * 79 | * @param $row 80 | * @return void 81 | */ 82 | // public function shouldSkipEmptyRow($row) 83 | // { 84 | // $skipRow = false; 85 | // foreach($row as $columnIndex => $value) { 86 | // $skipRow = $skipRow && 87 | // $this->isEmptyCell($value) || 88 | // $columnIndex = // there is no way for the reader to know if a column is primary key 89 | // } 90 | // } 91 | 92 | public function isEmptyCell($value) 93 | { 94 | return is_null($value) || 95 | $value instanceof EmptyCell || 96 | ($this->settings->emptyStringIsEmptyCell && $value == ""); 97 | } 98 | 99 | public function isValid() 100 | { 101 | return !$this->nullRow && $this->validate(); 102 | } 103 | 104 | protected function transformValue($columnName, $value) 105 | { 106 | $value = $this->runParsers($columnName, $value); 107 | $value = $this->defaultValue($columnName, $value); 108 | $value = $this->transformEmptyValue($value); 109 | $value = $this->encode($value); 110 | $value = $this->hash($columnName, $value); 111 | $value = $this->uuid($columnName, $value); 112 | 113 | return $value; 114 | } 115 | 116 | protected function defaultValue($columnName, $value) 117 | { 118 | return isset($this->settings->defaults[$columnName]) ? $this->settings->defaults[$columnName] : $value; 119 | } 120 | 121 | protected function transformEmptyValue($value) 122 | { 123 | if($value instanceof EmptyCell) return $value; 124 | if(is_null($value)) return new EmptyCell(); 125 | 126 | if (!is_string($value)) return $value; 127 | 128 | if($this->settings->emptyStringIsEmptyCell && $value == "") return new EmptyCell(); 129 | 130 | if( strtoupper($value) == 'NULL' ) return NULL; 131 | if( strtoupper($value) == 'FALSE' ) return FALSE; 132 | if( strtoupper($value) == 'TRUE' ) return TRUE; 133 | 134 | return $value; 135 | } 136 | 137 | protected function encode($value) 138 | { 139 | if( is_string($value) ) 140 | $value = empty($this->settings->inputEncodings) ? 141 | mb_convert_encoding($value, $this->settings->outputEncoding) : 142 | mb_convert_encoding($value, $this->settings->outputEncoding, $this->settings->inputEncodings); 143 | return $value; 144 | } 145 | 146 | protected function hash($columnName, $value) 147 | { 148 | return in_array($columnName, $this->settings->hashable) ? Hash::make($value) : $value; 149 | } 150 | 151 | protected function uuid($columnName, $value) 152 | { 153 | if (!in_array($columnName, $this->settings->uuid)) return $value; 154 | 155 | return Str::isUuid($value) ? $value : Str::uuid(); 156 | } 157 | 158 | protected function runParsers($columnName, $value) 159 | { 160 | return array_key_exists($columnName, $this->settings->parsers) && is_callable($this->settings->parsers[$columnName]) ? 161 | $this->settings->parsers[$columnName]($value) : 162 | $value; 163 | } 164 | 165 | /** 166 | * Add timestamp to the processed row 167 | * 168 | * @return void 169 | */ 170 | protected function addTimestamps() 171 | { 172 | if( empty($this->settings->timestamps) ) return; 173 | 174 | $timestamp = date('Y-m-d H:i:s'); 175 | 176 | $this->rowArray[ 'created_at' ] = $timestamp; 177 | $this->rowArray[ 'updated_at' ] = $timestamp; 178 | } 179 | 180 | protected function addColumns() 181 | { 182 | foreach ($this->settings->addColumns as $column) { 183 | if (!isset($this->rowArray[$column])) $this->rowArray[$column] = $this->transformValue($column, null); 184 | } 185 | } 186 | 187 | protected function validate() 188 | { 189 | if( empty($this->settings->validate)) return true; 190 | 191 | $validator = Validator::make($this->rowArray, $this->settings->validate); 192 | 193 | if( $validator->fails() ) return FALSE; 194 | 195 | return TRUE; 196 | } 197 | } -------------------------------------------------------------------------------- /src/Readers/Rows.php: -------------------------------------------------------------------------------- 1 | rows) && empty($this->rawRows); 37 | } 38 | 39 | public function count() 40 | { 41 | return count($this->rows); 42 | } 43 | 44 | public function countRowAsProcessed() 45 | { 46 | $this->processedRows++; 47 | } 48 | 49 | public function countRowAsSkipped() 50 | { 51 | $this->skippedRows++; 52 | } 53 | } -------------------------------------------------------------------------------- /src/Readers/Types/EmptyCell.php: -------------------------------------------------------------------------------- 1 | 1024; $i++) { 20 | $sizeInBytes /= 1024; 21 | } 22 | 23 | return round($sizeInBytes, 2).' '.$units[$i]; 24 | } 25 | 26 | public static function getHumanReadableTime(float $timeInSeconds): string 27 | { 28 | $units = ['s', 'm', 'h', 'days']; 29 | $divisor = [60, 60, 24]; 30 | 31 | if ($timeInSeconds == 0) { 32 | return '0 '.$units[0]; 33 | } 34 | 35 | for ($i = 0; $timeInSeconds > $divisor[$i]; $i++) { 36 | $timeInSeconds /= $divisor[$i]; 37 | } 38 | 39 | return round($timeInSeconds, 2).' '.$units[$i]; 40 | } 41 | 42 | public static function memoryLog($message = '') { 43 | if (!self::$memoryLogEnabled) return; 44 | 45 | static $timer = null; 46 | 47 | $elapsed_time = isset($timer) ? microtime(true) - $timer : 0; 48 | error_log($message . ' ' . self::getHumanReadableSize(memory_get_usage()) . ' ' . self::getHumanReadableTime($elapsed_time)); 49 | $timer = microtime(true); 50 | } 51 | } -------------------------------------------------------------------------------- /src/SeederMemoryHelper.php: -------------------------------------------------------------------------------- 1 | 1024; $i++) { 20 | $sizeInBytes /= 1024; 21 | } 22 | 23 | return round($sizeInBytes, 2).' '.$units[$i]; 24 | } 25 | 26 | public static function getHumanReadableTime(float $timeInSeconds): string 27 | { 28 | $units = ['s', 'm', 'h', 'days']; 29 | $divisor = [60, 60, 24]; 30 | 31 | if ($timeInSeconds == 0) { 32 | return '0 '.$units[0]; 33 | } 34 | 35 | for ($i = 0; $timeInSeconds > $divisor[$i]; $i++) { 36 | $timeInSeconds /= $divisor[$i]; 37 | } 38 | 39 | return round($timeInSeconds, 2).' '.$units[$i]; 40 | } 41 | 42 | public static function measurements() { 43 | static $timer = null; 44 | 45 | $elapsed_time = isset($timer) ? microtime(true) - $timer : 0; 46 | $timer = microtime(true); 47 | return [ 48 | "memory" => self::getHumanReadableSize(memory_get_usage()), 49 | "time" => self::getHumanReadableTime($elapsed_time) 50 | ]; 51 | } 52 | 53 | public static function memoryLog($message = '') { 54 | if (!self::$memoryLogEnabled) return; 55 | 56 | $m = self::measurements(); 57 | error_log($message . ' ' . $m["memory"] . ' ' . $m["time"]); 58 | } 59 | } -------------------------------------------------------------------------------- /src/SourceFileReadFilter.php: -------------------------------------------------------------------------------- 1 | settings = $this->defaultSettings = $settings; 98 | $this->consoleWriter = $consoleWriter; 99 | $this->databaseWriter = $databaseWriter; 100 | $this->markdownWriter = $markdownWriter; 101 | $this->yamlWriter = $yamlWriter; 102 | $this->spreadsheetReader = $spreadsheetReader; 103 | 104 | } 105 | 106 | public function boot() 107 | { 108 | Event::listen(FileStart::class, [$this, 'handleFileStart']); 109 | Event::listen(SheetStart::class, [$this, 'handleSheetStart']); 110 | $this->consoleWriter->boot($this->command); 111 | $this->databaseWriter->boot(); 112 | $this->markdownWriter->boot(); 113 | $this->yamlWriter->boot(); 114 | $this->spreadsheetReader->boot(); 115 | } 116 | 117 | public function resetSettings() 118 | { 119 | App::forgetInstance(SpreadsheetSeederSettings::class); 120 | App::instance(SpreadsheetSeederSettings::class, $this->defaultSettings); 121 | $this->settings = App::make(SpreadsheetSeederSettings::class); 122 | $this->settings($this->settings); 123 | } 124 | 125 | public function settings(SpreadsheetSeederSettings $set) 126 | { 127 | /* do nothing unless overridden by subclass */ 128 | } 129 | 130 | /** 131 | * Run the class 132 | * 133 | * @return void 134 | */ 135 | public function run() 136 | { 137 | $this->settings($this->settings); 138 | 139 | $this->boot(); 140 | 141 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'start'); 142 | 143 | $finder = $this->finder(); 144 | 145 | if (!$finder->hasResults()) { 146 | event(new Console('No spreadsheet file given', 'error')); 147 | return; 148 | } 149 | 150 | foreach ($finder as $file) { 151 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'file'); 152 | event(new FileStart($file)); 153 | event(new FileSeed($file)); 154 | event(new FileFinish($file)); 155 | } 156 | 157 | $this->tablesSeeded = $this->databaseWriter->tablesSeeded; 158 | 159 | $this->cleanup(); 160 | } 161 | 162 | public function __set($name, $value) { 163 | $this->settings->$name = $value; 164 | } 165 | 166 | public function command() { 167 | return $this->command; 168 | } 169 | 170 | public function finder() 171 | { 172 | if ($this->activeFinder) return $this->activeFinder; 173 | 174 | if ($this->settings->file instanceof Finder) return $this->activeFinder = $this->settings->file; 175 | 176 | return $this->activeFinder = new FileIterator(); 177 | } 178 | 179 | /** 180 | * @param $fileStart FileStart 181 | */ 182 | public function handleFileStart($fileStart) 183 | { 184 | $this->fileName = $fileStart->file->getFilename(); 185 | $this->resetSettings(); 186 | } 187 | 188 | /** 189 | * @param $sheetStart SheetStart 190 | */ 191 | public function handleSheetStart($sheetStart) 192 | { 193 | $this->sheetName = $sheetStart->sheetName; 194 | $this->resetSettings(); 195 | } 196 | 197 | public function cleanup() 198 | { 199 | $events = [ 200 | Console::class, 201 | FileStart::class, 202 | FileSeed::class, 203 | FileFinish::class, 204 | SheetStart::class, 205 | SheetFinish::class, 206 | ChunkStart::class, 207 | ChunkFinish::class, 208 | ]; 209 | 210 | foreach ($events as $event) Event::forget($event); 211 | } 212 | } 213 | -------------------------------------------------------------------------------- /src/SpreadsheetSeederServiceProvider.php: -------------------------------------------------------------------------------- 1 | SpreadsheetSeederSettings::class, 38 | ]; 39 | 40 | /** 41 | * Register services. 42 | * 43 | * @return void 44 | */ 45 | public function register() 46 | { 47 | $this->app->singleton(SeedCommand::class, function ($app) { 48 | return new SeedCommand($app['db']); 49 | }); 50 | 51 | StrMacros::registerSupportMacros(); 52 | } 53 | 54 | /** 55 | * Bootstrap services. 56 | * 57 | * @return void 58 | */ 59 | public function boot() 60 | { 61 | $this->bindGrammarClasses(); 62 | $this->commands([ 63 | SeedCommand::class, 64 | ]); 65 | } 66 | 67 | /** 68 | * Get the services provided by the provider. 69 | * 70 | * @return array 71 | */ 72 | public function provides() 73 | { 74 | return array(); 75 | } 76 | 77 | protected function getMySqlConnectionClass() 78 | { 79 | $phpversion = explode("-", phpversion()); 80 | if ( 81 | app()->runningUnitTests() && 82 | Semver::satisfies(app()->version(), "^6.0|^7.0|^8.0") && 83 | Semver::satisfies($phpversion[0], "^8.0") 84 | ) 85 | return RefreshDatabaseMySqlConnection::class; 86 | 87 | return MySqlConnection::class; 88 | } 89 | 90 | protected function bindGrammarClasses() 91 | { 92 | $connections = [ 93 | 'mysql' => [ 94 | 'connection' => $this->getMySqlConnectionClass(), 95 | 'schemaGrammar' => MySqlGrammar::class, 96 | ], 97 | ]; 98 | 99 | foreach($connections as $driver => $class) { 100 | Connection::resolverFor($driver, function($pdo, $database = '', $tablePrefix = '', array $config = []) use ($driver, $class) { 101 | $connection = new $class['connection']($pdo, $database, $tablePrefix, $config); 102 | 103 | // In Laravel versions 11 and below, Illuminate\Database\Grammar does not expect any arguments 104 | // in the constructor. In Laravel 12, it does. We need to check the version and pass the arguments 105 | // if the version is 12 or above. 106 | 107 | if (Semver::satisfies(app()->version(), '<12.0')) { 108 | $connection->setSchemaGrammar(new $class['schemaGrammar']()); 109 | } else { 110 | $connection->setSchemaGrammar(new $class['schemaGrammar']($connection)); 111 | } 112 | 113 | return $connection; 114 | }); 115 | } 116 | } 117 | } -------------------------------------------------------------------------------- /src/SpreadsheetSeederSettings.php: -------------------------------------------------------------------------------- 1 | 'Table Column 1', 'CSV Header 2' => 'Table Column 2'] 43 | * 44 | * Default: [] 45 | * 46 | */ 47 | public $aliases = []; 48 | 49 | /* 50 | * -------------------------------------------------------------------------- 51 | * Batch Insert Size 52 | * -------------------------------------------------------------------------- 53 | * 54 | * Number of rows to insert per batch 55 | * 56 | * 57 | * Default: 5000; 58 | * 59 | */ 60 | public $batchInsertSize = 5000; 61 | 62 | /* 63 | * -------------------------------------------------------------------------- 64 | * Defaults 65 | * -------------------------------------------------------------------------- 66 | * 67 | * This is an associative array mapping column names in the data source to 68 | * default values that will override any values in the datasource. 69 | * 70 | * Note: this setting is currently global and applies to all files or 71 | * worksheets that are processed. To apply differently to 72 | * different files, process files with separate Seeder instances. 73 | * 74 | * Example: ['created_by' => 'seed', 'updated_by' => 'seed] 75 | * 76 | * Default: [] 77 | * 78 | */ 79 | public $defaults = []; 80 | 81 | /* 82 | * -------------------------------------------------------------------------- 83 | * Delimiter 84 | * -------------------------------------------------------------------------- 85 | * 86 | * The delimiter used in CSV, tab-separate-files, and other text delimited 87 | * files. When this is not set, the phpspreadsheet library will 88 | * automatically detect the text delimiter 89 | * 90 | * Default: null 91 | * 92 | */ 93 | public $delimiter = null; 94 | 95 | /* 96 | * -------------------------------------------------------------------------- 97 | * Date Formats 98 | * -------------------------------------------------------------------------- 99 | * 100 | * This is an associative array mapping column names in the data source to 101 | * date format strings that should be used by Carbon to parse the date. 102 | * Information to construct date format strings is here: 103 | * https://www.php.net/manual/en/datetime.createfromformat.php 104 | * 105 | * When the destination column in the database table is a date time format, 106 | * and the source data is a string, the seeder will use Carbon to parse the 107 | * date format. In many cases Carbon can parse the date automatically 108 | * without specifying the date format. 109 | * 110 | * When Carbon cannot parse the date automatically, map the column name in 111 | * this array to the date format string. When a source column is mapped, 112 | * Carbon will use the date format string instead of parsing automatically. 113 | * 114 | * If column mapping is used (see mapping) the column name should match the 115 | * value in the $mapping array instead of the value in the file, if any. 116 | * 117 | * Note: this setting is currently global and applies to all files or 118 | * worksheets that are processed. All columns with the specified name in all files 119 | * or worksheets will have the validation rule applied. To apply differently to 120 | * different files, process files with separate Seeder instances. 121 | * 122 | * Example: [ 123 | * 'order_date' => 'Y-m-d H:i:s.u+', // parses "2020-10-04 05:31:02.440000000" 124 | * ] 125 | * 126 | * Default: [] 127 | * 128 | */ 129 | public $dateFormats = []; 130 | 131 | /* 132 | * -------------------------------------------------------------------------- 133 | * Empty String is Empty Cell 134 | * -------------------------------------------------------------------------- 135 | * 136 | * If a cell contains an empty string `""` treat it as an empty cell 137 | * 138 | * Default: "true" 139 | * 140 | */ 141 | public $emptyStringIsEmptyCell = true; 142 | 143 | /* 144 | * -------------------------------------------------------------------------- 145 | * Data Source File Default Extension 146 | * -------------------------------------------------------------------------- 147 | * 148 | * The default extension used when a directory is specified in $this->file 149 | * 150 | * Default: "xlsx" 151 | * 152 | */ 153 | public $extension = "xlsx"; 154 | 155 | /* 156 | * -------------------------------------------------------------------------- 157 | * Data Source File 158 | * -------------------------------------------------------------------------- 159 | * 160 | * This value is the path of the Excel or CSV file used as the data 161 | * source. This is a string or array[] and is list of files or directories 162 | * to process, which can include wildcards. 163 | * 164 | * The path is specified relative to the root of the project 165 | * 166 | * Default: "/database/seeds/*.xlsx" 167 | * 168 | */ 169 | public $file = [ 170 | "/database/seeds/*.xlsx", 171 | "/database/seeders/*.xlsx" 172 | ]; 173 | 174 | /* 175 | * -------------------------------------------------------------------------- 176 | * Hashable 177 | * -------------------------------------------------------------------------- 178 | * 179 | * This is an array of column names in the data source that should be hashed 180 | * using Laravel's `Hash` facade. 181 | * 182 | * The hashing algorithm is configured in `config/hashing.php` per 183 | * https://laravel.com/docs/master/hashing 184 | * 185 | * Note: this setting is currently global and applies to all files or 186 | * worksheets that are processed. All columns with the specified name in all files 187 | * or worksheets will have hashing applied. To apply differently to 188 | * different files, process files with separate Seeder instances. 189 | * 190 | * Example: ['password', 'salt'] 191 | * 192 | * Default: [] 193 | * 194 | */ 195 | public $hashable = []; 196 | 197 | /* 198 | * -------------------------------------------------------------------------- 199 | * Header 200 | * -------------------------------------------------------------------------- 201 | * 202 | * If the data source has headers in the first row, setting this to true will 203 | * skip the first row. 204 | * 205 | * Default: TRUE 206 | * 207 | */ 208 | public $header = TRUE; 209 | 210 | /* 211 | * -------------------------------------------------------------------------- 212 | * Input Encodings 213 | * -------------------------------------------------------------------------- 214 | * 215 | * Array of possible input encodings from input data source 216 | * See https://www.php.net/manual/en/mbstring.supported-encodings.php 217 | * 218 | * This value is used as the "from_encoding" parameter to mb_convert_encoding. 219 | * If this is not specified, the internal encoding is used. 220 | * 221 | * Default: [] 222 | * 223 | */ 224 | public $inputEncodings = []; 225 | 226 | 227 | /* 228 | * -------------------------------------------------------------------------- 229 | * Limit 230 | * -------------------------------------------------------------------------- 231 | * 232 | * Limit the maximum number of rows that will be loaded from a worksheet. 233 | * This is useful in development to keep loading time fast. 234 | * 235 | * Default: null 236 | * 237 | */ 238 | public $limit = null; 239 | 240 | /* 241 | * -------------------------------------------------------------------------- 242 | * Column "Mapping" 243 | * -------------------------------------------------------------------------- 244 | * Backward compatibility to laravel-csv-seeder 245 | * 246 | * This is an array of column names that will be used as headers. 247 | * 248 | * If $this->header is true then the first row of data will be skipped. 249 | * This allows existing headers in a CSV file to be overridden. 250 | * 251 | * This is called "Mapping" because its intended use is to map the fields of 252 | * a CSV file without a header line to the columns of a database table. 253 | * 254 | * Note: this setting is currently global and applies to all files or 255 | * worksheets that are processed. To apply differently to different files, 256 | * process files with separate Seeder instances. 257 | * 258 | * Example: ['Header Column 1', 'Header Column 2'] 259 | * 260 | * Default: [] 261 | * 262 | */ 263 | public $mapping = []; 264 | 265 | /* 266 | * -------------------------------------------------------------------------- 267 | * Offset 268 | * -------------------------------------------------------------------------- 269 | * 270 | * Number of rows to skip at the start of the data source, excluding the 271 | * header row. 272 | * 273 | * Default: 0 274 | * 275 | */ 276 | public $offset = 0; 277 | 278 | /* 279 | * -------------------------------------------------------------------------- 280 | * Output Encodings 281 | * -------------------------------------------------------------------------- 282 | * 283 | * Output encoding to database 284 | * See https://www.php.net/manual/en/mbstring.supported-encodings.php 285 | * 286 | * This value is used as the "to_encoding" parameter to mb_convert_encoding. 287 | * 288 | * Default: "UTF-8"; 289 | * 290 | */ 291 | public $outputEncoding = "UTF-8"; 292 | 293 | /* 294 | * -------------------------------------------------------------------------- 295 | * Parsers 296 | * -------------------------------------------------------------------------- 297 | * 298 | * This is an associative array of column names in the data source that should be parsed 299 | * with the specified parser. 300 | * 301 | * Note: this setting is currently global and applies to all files or 302 | * worksheets that are processed. All columns with the specified name in all files 303 | * or worksheets will have hashing applied. To apply differently to 304 | * different files, process files with separate Seeder instances. 305 | * 306 | * Example: ['email' => function ($value) { 307 | * return strtolower($value); 308 | * }]; 309 | * 310 | * Default: [] 311 | * 312 | */ 313 | public $parsers = []; 314 | 315 | /* 316 | * -------------------------------------------------------------------------- 317 | * Read Chunk Size 318 | * -------------------------------------------------------------------------- 319 | * 320 | * Number of rows to read per chunk 321 | * 322 | * 323 | * Default: 5000; 324 | * 325 | */ 326 | public $readChunkSize = 5000; 327 | 328 | /* 329 | * -------------------------------------------------------------------------- 330 | * Skipper 331 | * -------------------------------------------------------------------------- 332 | * 333 | * This is a string used as a prefix to indicate that a column in the data source 334 | * should be skipped. For Excel workbooks, a worksheet prefixed with 335 | * this string will also be skipped. The skipper prefix can be a 336 | * multi-character string. 337 | * 338 | * Example: Data source column '%id_copy' will be skipped with skipper set as '%' 339 | * Example: Data source column '#id_copy' will be skipped with skipper set as '#' 340 | * Example: Data source column '[skip]id_copy' will be skipped with skipper set as '[skip]' 341 | * Example: Worksheet '%worksheet1' will be skipped with skipper set as '%' 342 | * 343 | * Default: "%"; 344 | * 345 | */ 346 | public $skipper = "%"; 347 | 348 | /** 349 | * -------------------------------------------------------------------------- 350 | * Skip Columns 351 | * -------------------------------------------------------------------------- 352 | * 353 | * This is an array of column names that will be skipped in the worksheet. 354 | * 355 | * This can be used to skip columns in the same way as the skipper character, 356 | * but without modifying the worksheet. 357 | * 358 | * Example: ['column1', 'column2'] 359 | * 360 | * Default: [] 361 | * 362 | * @var array 363 | */ 364 | public $skipColumns = []; 365 | 366 | /** 367 | * -------------------------------------------------------------------------- 368 | * Skip Sheets 369 | * -------------------------------------------------------------------------- 370 | * 371 | * This is an array of worksheet names that will be skipped in the workbook. 372 | * 373 | * This can be used to skip worksheets in the same way as the skipper character, 374 | * but without modifying the workbook. 375 | * 376 | * Example: ['Sheet1', 'Sheet2'] 377 | * 378 | * Default: [] 379 | * 380 | * @var array 381 | */ 382 | public $skipSheets = []; 383 | 384 | /* 385 | * -------------------------------------------------------------------------- 386 | * Table Name 387 | * -------------------------------------------------------------------------- 388 | * Backward compatibility to laravel-csv-seeder 389 | * 390 | * Table name to insert into in the database. If this is not set then it 391 | * uses the name of the current CSV filename 392 | * 393 | * Use worksheetTableMapping instead to map worksheet names to alternative 394 | * table names 395 | * 396 | * Default: null 397 | * 398 | */ 399 | public $tablename = null; 400 | 401 | /* 402 | * -------------------------------------------------------------------------- 403 | * Text Output 404 | * -------------------------------------------------------------------------- 405 | * 406 | * Configure the format for text output. 407 | * 408 | * "false": text output is disabled. 409 | * "true": text output is in markdown format for backward compatibility. 410 | * "markdown": test output is in markdown format. 411 | * "yaml": text output is in yaml format. 412 | * ["markdown", "yaml"]: text output is produced in both markdown and yaml format. 413 | * 414 | * 415 | * 416 | * Default: "true"; 417 | * 418 | */ 419 | public $textOutput = true; 420 | 421 | /** 422 | * returns canonicalized collection of textOutput settings. 423 | * if $textOutput is an array that contains true or false, that overrides other settings. 424 | * 425 | * @return Collection|\Illuminate\Support\Traits\EnumeratesValues 426 | */ 427 | public function textOutput() 428 | { 429 | $formats = Collection::wrap($this->textOutput); 430 | if ($formats->contains('false')) return Collection::make(['false']); 431 | if ($formats->contains('true')) return Collection::make(['markdown']); 432 | return $formats; 433 | } 434 | 435 | /* 436 | * -------------------------------------------------------------------------- 437 | * Text Output Table File Extension 438 | * -------------------------------------------------------------------------- 439 | * 440 | * Extension for text output table 441 | * 442 | * After processing a workbook, the seeder outputs a text format of 443 | * the sheet to assist with diff and merge of the workbook. The default format 444 | * is markdown 'md' which will render the text as tables in markdown viewers 445 | * like github. This can be changed by setting this attribute. 446 | * 447 | * Default: "md"; 448 | * 449 | */ 450 | public $textOutputFileExtension = "md"; 451 | 452 | /* 453 | * -------------------------------------------------------------------------- 454 | * Text Output Path 455 | * -------------------------------------------------------------------------- 456 | * 457 | * Path for text output 458 | * 459 | * After processing a workbook, the seeder outputs a text format of 460 | * the sheet to assist with diff and merge of the workbook. The default format 461 | * is markdown 'md' which will render the text as tables in markdown viewers 462 | * like github. This can be changed by setting this attribute. 463 | * 464 | * Default: ""; 465 | * 466 | */ 467 | public $textOutputPath = ''; 468 | 469 | /* 470 | * -------------------------------------------------------------------------- 471 | * Timestamps 472 | * -------------------------------------------------------------------------- 473 | * 474 | * When `true`, set the Laravel timestamp columns 'created_at' and 'updated_at' 475 | * with the current date/time. 476 | * 477 | * When `false`, the fields will be set to NULL 478 | * 479 | * Default: true 480 | * 481 | */ 482 | public $timestamps = true; 483 | 484 | /* 485 | * -------------------------------------------------------------------------- 486 | * Truncate Destination Table 487 | * -------------------------------------------------------------------------- 488 | * 489 | * Truncate the table before seeding. 490 | * 491 | * Default: TRUE 492 | * 493 | * Note: does not currently support array of table names to exclude 494 | * 495 | */ 496 | public $truncate = TRUE; 497 | 498 | /* 499 | * -------------------------------------------------------------------------- 500 | * Truncate Destination Table Ignoring Foreign Key Constraints 501 | * -------------------------------------------------------------------------- 502 | * 503 | * Ignore foreign key constraints when truncating the table before seeding. 504 | * 505 | * Default: TRUE 506 | * 507 | * Note: does not currently support array of table names to exclude 508 | * 509 | */ 510 | public $truncateIgnoreForeign = TRUE; 511 | 512 | /* 513 | * -------------------------------------------------------------------------- 514 | * Unix Timestamps 515 | * -------------------------------------------------------------------------- 516 | * 517 | * This is an array of column names that contain values that should be 518 | * interpreted unix timestamps rather than excel timestamps. 519 | * 520 | * If column mapping is used (see mapping) the column name should match the 521 | * value in the $mapping array instead of the value in the file, if any. 522 | * 523 | * Note: this setting is currently global and applies to all files or 524 | * worksheets that are processed. All columns with the specified name in all files 525 | * or worksheets will be interpreted as unix timestamps. To apply differently to 526 | * different files, process files with separate Seeder instances. 527 | * 528 | * Example: ['start_date', 'finish_date']; 529 | * 530 | * Default: [] 531 | * 532 | */ 533 | public $unixTimestamps = []; 534 | 535 | /* 536 | * -------------------------------------------------------------------------- 537 | * UUID 538 | * -------------------------------------------------------------------------- 539 | * 540 | * This is an array of column names in the data source that the seeder will 541 | * generate a UUID for. 542 | * 543 | * The UUID generated is a type 4 "Random" UUID using laravel Str::uuid() helper 544 | * https://laravel.com/docs/10.x/helpers#method-str-uuid 545 | * 546 | * Note: this setting is currently global and applies to all files or 547 | * worksheets that are processed. All columns with the specified name in all files 548 | * or worksheets will have hashing applied. To apply differently to 549 | * different files, process files with separate Seeder instances. 550 | * 551 | * Example: ['uuid'] 552 | * 553 | * Default: [] 554 | * 555 | */ 556 | public $uuid = []; 557 | 558 | /* 559 | * -------------------------------------------------------------------------- 560 | * Validate 561 | * -------------------------------------------------------------------------- 562 | * 563 | * This is an associative array mapping column names in the data source that 564 | * should be validated to a Laravel Validator validation rule. 565 | * The available validation rules are described here: 566 | * https://laravel.com/docs/master/validation#available-validation-rules 567 | * 568 | * Note: this setting is currently global and applies to all files or 569 | * worksheets that are processed. All columns with the specified name in all files 570 | * or worksheets will have the validation rule applied. To apply differently to 571 | * different files, process files with separate Seeder instances. 572 | * 573 | * Example: [ 574 | * 'email' => 'unique:users,email_address', 575 | * 'start_date' => 'required|date|after:tomorrow', 576 | * 'finish_date' => 'required|date|after:start_date' 577 | * ] 578 | * 579 | * Default: [] 580 | * 581 | */ 582 | public $validate = []; 583 | 584 | /* 585 | * -------------------------------------------------------------------------- 586 | * Worksheet Table Mapping 587 | * -------------------------------------------------------------------------- 588 | * 589 | * This is an associative array to map names of worksheets in an Excel file 590 | * to table names. 591 | * 592 | * Excel worksheets have a 31 character limit. 593 | * 594 | * This is useful when the table name should be longer than the worksheet 595 | * character limit. 596 | * 597 | * Example: ['Sheet1' => 'first_table', 'Sheet2' => 'second_table'] 598 | * 599 | * Default: [] 600 | * 601 | */ 602 | public $worksheetTableMapping = []; 603 | 604 | /* 605 | * -------------------------------------------------------------------------- 606 | * Worksheets 607 | * -------------------------------------------------------------------------- 608 | * 609 | * If array is not empty, only worksheets matching entries in the array will be seeded. 610 | * 611 | * Example: ['Sheet1', 'Sheet2'] 612 | * 613 | * Default: [] 614 | * 615 | */ 616 | public $worksheets = []; 617 | } 618 | -------------------------------------------------------------------------------- /src/Support/ColumnInfo.php: -------------------------------------------------------------------------------- 1 | fromDoctrine($column); 23 | if (is_array($column)) $this->fromLaravel($column); 24 | } 25 | 26 | public function fromDoctrine(Column $column) 27 | { 28 | $this->name = $column->getName(); 29 | $this->type_name = $this->type = $column->getType()->getName(); 30 | $this->nullable = ! $column->getNotnull(); 31 | $this->default = $column->getDefault(); 32 | $this->autoIncrement = $column->getAutoincrement(); 33 | $this->comment = $column->getComment(); 34 | } 35 | 36 | public function fromLaravel($column) 37 | { 38 | $this->name = $column["name"]; 39 | $this->type_name = $column["type_name"] ?? null; 40 | $this->type = $column["type"] ?? null; 41 | $this->collation = $column["collation"] ?? null; 42 | $this->nullable = $column["nullable"] ?? null; 43 | $this->default = $column["default"] ?? null; 44 | $this->autoIncrement = $column["auto_increment"] ?? null; 45 | $this->comment = $column["comment"] ?? null; 46 | $this->generation = $column["generation"] ?? null; 47 | 48 | if (is_string($this->default)) { 49 | $this->default = Str::between($this->default, "'", "'"); 50 | // $this->default = Str::replaceLast('::bpchar', '', $this->default); 51 | // $this->default = Str::unwrap($this->default, "'"); 52 | } 53 | } 54 | 55 | public function getName() 56 | { 57 | return $this->name; 58 | } 59 | 60 | public function getType() 61 | { 62 | return $this->type_name; 63 | } 64 | 65 | public function getNullable() 66 | { 67 | return $this->nullable; 68 | } 69 | 70 | public function getDefault() 71 | { 72 | return $this->default; 73 | } 74 | 75 | public function getAutoIncrement() 76 | { 77 | return $this->autoIncrement; 78 | } 79 | 80 | public function getComment() 81 | { 82 | return $this->comment; 83 | } 84 | } -------------------------------------------------------------------------------- /src/Support/StrMacros.php: -------------------------------------------------------------------------------- 1 | 0; 81 | }); 82 | } 83 | 84 | public static function registerTrimMacro() 85 | { 86 | if (method_exists(Str::class, "trim")) return; 87 | 88 | /** 89 | * Remove all whitespace from both ends of a string. 90 | * 91 | * @param string $value 92 | * @param string|null $charlist 93 | * @return string 94 | */ 95 | Str::macro('trim', function ($value, $charlist = null) { 96 | if ($charlist === null) { 97 | return preg_replace('~^[\s\x{FEFF}\x{200B}\x{200E}]+|[\s\x{FEFF}\x{200B}\x{200E}]+$~u', '', $value) ?? trim($value); 98 | } 99 | 100 | return trim($value, $charlist); 101 | }); 102 | } 103 | } -------------------------------------------------------------------------------- /src/Support/Workaround/RefreshDatabase/RefreshDatabaseMySqlConnection.php: -------------------------------------------------------------------------------- 1 | getPdo(); 23 | 24 | if ($pdo->inTransaction()) { 25 | $pdo->rollBack(); 26 | } 27 | } elseif ($this->queryGrammar->supportsSavepoints()) { 28 | $this->getPdo()->exec( 29 | $this->queryGrammar->compileSavepointRollBack('trans'.($toLevel + 1)) 30 | ); 31 | } 32 | } 33 | } -------------------------------------------------------------------------------- /src/Writers/Console/ConsoleWriter.php: -------------------------------------------------------------------------------- 1 | setCommand($command); 67 | } 68 | 69 | public function setCommand(Command $command) 70 | { 71 | $this->command = $command; 72 | } 73 | 74 | public function boot(Command $command = null) 75 | { 76 | if (isset($command)) $this->setCommand($command); 77 | Event::listen(Console::class, [$this, 'handleConsole']); 78 | Event::listen(FileStart::class, [$this, 'handleFileStart']); 79 | Event::listen(SheetStart::class, [$this, 'handleSheetStart']); 80 | Event::listen(ChunkStart::class, [$this, 'handleChunkStart']); 81 | Event::listen(ChunkFinish::class, [$this, 'handleChunkFinish']); 82 | Event::listen(SheetFinish::class, [$this, 'handleSheetFinish']); 83 | } 84 | 85 | /** 86 | * Logging 87 | * 88 | * @param string $message 89 | * @param string $level 90 | * @return void 91 | */ 92 | public function console( $message, $level = FALSE ) 93 | { 94 | if( $level ) $message = '<'.$level.'>'.$message.''; 95 | 96 | $this->command->line( 'SpreadsheetSeeder: '.$message ); 97 | } 98 | 99 | /** 100 | * Logging 101 | * 102 | * @param Console $consoleEvent 103 | * @return void 104 | */ 105 | public function handleConsole( $consoleEvent ) 106 | { 107 | $this->console($consoleEvent->message, $consoleEvent->level); 108 | } 109 | 110 | public function handleFileStart(FileStart $fileStart) 111 | { 112 | $this->file = $fileStart->file; 113 | } 114 | 115 | public function currentRowMessage() 116 | { 117 | $m = SeederMemoryHelper::measurements(); 118 | $message = "File: " . $this->file->getFilename() . " Sheet: " . $this->sheetName . " Row: " . $this->currentRow . " " . $m["memory"] . " " . $m["time"]; 119 | $this->console($message, "info"); 120 | SeederMemoryHelper::memoryLog($message); 121 | } 122 | 123 | public function handleSheetStart(SheetStart $sheetStart) 124 | { 125 | $this->sheetName = $sheetStart->sheetName; 126 | $this->tableName = $sheetStart->tableName; 127 | $this->currentRow = $sheetStart->startRow; 128 | 129 | $this->currentRowMessage(); 130 | } 131 | 132 | public function handleChunkStart(ChunkStart $chunkStart) 133 | { 134 | $this->currentRow = $chunkStart->startRow; 135 | 136 | $this->chunkCount++; 137 | if ($this->chunkCount > 1) { 138 | $this->currentRowMessage(); 139 | } 140 | } 141 | 142 | public function handleChunkFinish(ChunkFinish $chunkFinish) 143 | { 144 | $this->rowsInserted += $chunkFinish->rows->count(); 145 | $this->rowsProcessed += $chunkFinish->rows->processedRows; 146 | } 147 | 148 | public function handleSheetFinish(SheetFinish $sheetFinish) 149 | { 150 | $this->console($this->rowsInserted . ' of ' . $this->rowsProcessed . ' rows has been seeded in table "' . $this->tableName . '"'); 151 | 152 | $this->rowsProcessed = 0; 153 | $this->rowsInserted = 0; 154 | $this->chunkCount = 0; 155 | } 156 | 157 | } -------------------------------------------------------------------------------- /src/Writers/Database/DatabaseWriter.php: -------------------------------------------------------------------------------- 1 | queryLogMemoryLeakWorkaroundProvider = (new QueryLogMemoryLeakWorkaroundProvider()); 41 | $this->queryLogMemoryLeakWorkaroundProvider->boot(); 42 | 43 | Event::listen(FileStart::class, [$this, 'handleFileStart']); 44 | Event::listen(SheetStart::class, [$this, 'handleSheetStart']); 45 | Event::listen(ChunkFinish::class, [$this, 'handleChunkFinish']); 46 | Event::listen(SheetFinish::class, [$this, 'handleSheetFinish']); 47 | } 48 | 49 | /** 50 | * @param $fileStart FileStart 51 | */ 52 | public function handleFileStart($fileStart) 53 | { 54 | $this->fileName = $fileStart->file->getFilename(); 55 | $this->tablesSeeded = []; 56 | } 57 | 58 | /** 59 | * @param $sheetStart SheetStart 60 | */ 61 | public function handleSheetStart($sheetStart) 62 | { 63 | $this->seedTable = new DestinationTable($sheetStart->tableName); 64 | $this->sheetName = $sheetStart->sheetName; 65 | 66 | if (!$this->seedTable->exists()) { 67 | event(new Console('Table "' . $sheetStart->tableName . '" could not be found in database', 'error')); 68 | return; 69 | } 70 | } 71 | 72 | /** 73 | * @param $chunkFinish ChunkFinish 74 | */ 75 | public function handleChunkFinish($chunkFinish) 76 | { 77 | $this->insertRows($chunkFinish->rows); 78 | } 79 | 80 | /** 81 | * @param $sheetFinish SheetFinish 82 | */ 83 | public function handleSheetFinish($sheetFinish) 84 | { 85 | $this->tablesSeeded[] = $sheetFinish->tableName; 86 | $this->updatePostgresSeqCounters($sheetFinish->tableName); 87 | SeederMemoryHelper::memoryLog(__METHOD__ . '::' . __LINE__ . ' ' . 'processed'); 88 | } 89 | 90 | /** 91 | * Insert rows into table 92 | * 93 | * @param $rows Rows 94 | * 95 | * @return void 96 | */ 97 | private function insertRows($rows) 98 | { 99 | if ($rows->isEmpty()) return; 100 | 101 | try { 102 | $this->seedTable->insertRows($rows->rows); 103 | } catch (Exception $e) { 104 | $message = 'Rows of the file "' . $this->fileName . '" sheet "' . $this->sheetName . '" has failed to insert in table "' . $this->seedTable->getName() . '": ' . $e->getMessage(); 105 | event(new Console($message, 'error')); 106 | 107 | throw(new Exception($message)); 108 | } 109 | } 110 | 111 | /** 112 | * @param $table 113 | * @return void 114 | * @throws \Doctrine\DBAL\Exception 115 | */ 116 | public function updatePostgresSeqCounters($table) 117 | { 118 | if (!DB::connection()->getQueryGrammar() instanceof PostgresGrammar) { 119 | return; 120 | } 121 | 122 | foreach($this->getSequencesForTable($table) as $column => $sequence) { 123 | $result = DB::select("select setval('{$sequence}', max(\"{$column}\")) from {$table}"); 124 | } 125 | } 126 | 127 | /** 128 | * @param string $table 129 | * @param string | string[] $columns 130 | * @return \Doctrine\DBAL\Schema\Sequence[] 131 | * @throws \Doctrine\DBAL\Exception 132 | */ 133 | public static function getSequencesForTable(string $table) 134 | { 135 | $sequences = DB::table('information_schema.columns') 136 | ->select("table_name", "column_name", "column_default") 137 | ->whereRaw("table_name=? and column_default like ?", [$table, "nextval%"]) 138 | ->get() 139 | ->mapWithKeys(function ($value, $key) { 140 | return [$value->column_name => Str::between($value->column_default, "'", "'")]; 141 | }); 142 | 143 | return $sequences; 144 | } 145 | } -------------------------------------------------------------------------------- /src/Writers/Database/DestinationTable.php: -------------------------------------------------------------------------------- 1 | name = $name; 56 | $this->settings = resolve(SpreadsheetSeederSettings::class); 57 | 58 | if ($this->exists() && $this->settings->truncate) $this->truncate(); 59 | $this->loadColumns(); 60 | } 61 | 62 | public function getName() { 63 | return $this->name; 64 | } 65 | 66 | public function exists() { 67 | if (isset($this->exists)) return $this->exists; 68 | $this->exists = self::tableExists( $this->name ); 69 | 70 | return $this->exists; 71 | } 72 | 73 | public static function tableExists($name) 74 | { 75 | return DB::getSchemaBuilder()->hasTable( $name ); 76 | } 77 | 78 | public function truncate() { 79 | $ignoreForeign = $this->settings->truncateIgnoreForeign; 80 | 81 | if( $ignoreForeign ) Schema::disableForeignKeyConstraints(); 82 | 83 | DB::table( $this->name )->truncate(); 84 | 85 | if( $ignoreForeign ) Schema::enableForeignKeyConstraints(); 86 | } 87 | 88 | protected function loadColumns() { 89 | if (! isset($this->columns)) { 90 | $this->columns = DB::getSchemaBuilder()->getColumnListing( $this->name ); 91 | $connection = DB::getSchemaBuilder()->getConnection(); 92 | 93 | $schemaBuilder = Schema::getFacadeRoot(); 94 | if (method_exists($schemaBuilder, "getColumns")) { 95 | $columns = Schema::getColumns($this->name); 96 | } 97 | else { 98 | $columns = DB::getSchemaBuilder()->getConnection()->getDoctrineSchemaManager()->listTableColumns($this->name); 99 | } 100 | /* 101 | * Doctrine DBAL 2.11.x-dev does not return the column name as an index in the case of mixed case (or uppercase?) column names 102 | * In sqlite in-memory database, DBAL->listTableColumns() uses the lowercase version of the column name as a column index 103 | * In postgres, it uses the lowercase version of the mixed-case column name and places '"' around the name (for the mixed-case name only) 104 | * The solution here is to iterate through the columns to retrieve the column name and use that to build a new index. 105 | */ 106 | $this->columnInfo = []; 107 | foreach($columns as $column) { 108 | $c = new ColumnInfo($column); 109 | $this->columnInfo[$c->getName()] = $c; 110 | } 111 | 112 | /* 113 | * The primaryKey is used by isPrimaryColumn() which is not currently used. This may support future features like 114 | * ignoring the primary key column in determining if a row in a spreadsheet is empty and should be skipped. 115 | * getIndexes() is only available in Laravel 10.x and up. TODO find alternative for older Laravel versions. 116 | */ 117 | if (method_exists($schemaBuilder, "getIndexes")) { 118 | $indexes = DB::getSchemaBuilder()->getIndexes($this->name); 119 | $indexes = collect($indexes); 120 | $this->primaryKey = $indexes->first(function($value, $key) { 121 | return $value['primary'] == true; 122 | }); 123 | } 124 | } 125 | } 126 | 127 | public function getColumns() { 128 | return $this->columns; 129 | } 130 | 131 | public function isPrimaryColumn($columnName) 132 | { 133 | return $this->primaryKey['columns'][0] == $columnName; 134 | } 135 | 136 | public function getColumnType($name) { 137 | return $this->columnInfo[$name]->getType(); 138 | } 139 | 140 | public function columnExists($columnName) { 141 | return in_array($columnName, $this->columns); 142 | } 143 | 144 | private function transformEmptyCellValue($columnName, $value) { 145 | if ($value instanceof EmptyCell) { 146 | $value = $this->defaultValue($columnName); 147 | } 148 | return $value; 149 | } 150 | 151 | private function transformDateCellValue($columnName, $value) { 152 | if (is_null($value)) { 153 | return null; 154 | } 155 | 156 | if ($this->isDateColumn($columnName)) { 157 | 158 | if (is_numeric($value)) { 159 | if (in_array($columnName, $this->settings->unixTimestamps)) 160 | $date = Carbon::parse($value); 161 | else 162 | $date = Carbon::parse(Date::excelToDateTimeObject($value)); 163 | } 164 | else { 165 | if (isset($this->settings->dateFormats[$columnName])) { 166 | $date = Carbon::createFromFormat($this->settings->dateFormats[$columnName], $value); 167 | } 168 | else { 169 | $date = Carbon::parse($value); 170 | } 171 | } 172 | 173 | return $date->format('Y-m-d H:i:s.u'); 174 | } 175 | return $value; 176 | } 177 | 178 | private function checkRows($rows) { 179 | foreach ($rows as $row) { 180 | $tableRow = []; 181 | foreach ($row as $column => $value) { 182 | if ($this->columnExists($column)) { 183 | // note: empty values are transformed into their defaults in order to do batch inserts. 184 | // laravel doesn't support DEFAULT keyword for insertion 185 | $tableRow[$column] = $this->transformEmptyCellValue($column, $value); 186 | $tableRow[$column] = $this->transformDateCellValue($column, $tableRow[$column]); 187 | } 188 | } 189 | $this->rows[] = $tableRow; 190 | } 191 | } 192 | 193 | public function insertRows($rows) { 194 | if( empty($rows) ) return; 195 | 196 | $this->checkRows($rows); 197 | 198 | $offset = 0; 199 | while ($offset < count($this->rows)) { 200 | $batchRows = array_slice($this->rows, $offset, $this->settings->batchInsertSize); 201 | 202 | DB::table($this->name)->insert($batchRows); 203 | 204 | $offset += $this->settings->batchInsertSize; 205 | } 206 | $this->rows = []; 207 | } 208 | 209 | private function isDateColumn($column) { 210 | $c = $this->columnInfo[$column]; 211 | 212 | // if column is date or time type return 213 | $dateColumnTypes = ['date', 'date_immutable', 'datetime', 'datetime_immutable', 'datetimez', 'datetimez_immutable', 'time', 'time_immutable', 'dateinterval', 'timestamp']; 214 | return in_array($c->getType(), $dateColumnTypes); 215 | } 216 | 217 | public function isNumericColumn($column) 218 | { 219 | $c = $this->columnInfo[$column]; 220 | 221 | $numericTypes = ['smallint', 'integer', 'bigint', 'tinyint', 'decimal', 'float', 'numeric']; 222 | if (in_array($c->getType(), $numericTypes)) return true; 223 | return false; 224 | 225 | } 226 | 227 | public function defaultValue($column) { 228 | $c = $this->columnInfo[$column]; 229 | 230 | // MariaDB returns 'NULL' instead of null like mysql, sqlite, and postgres 231 | $isNull = is_null($c->getDefault()) || $c->getDefault() === 'NULL'; 232 | 233 | // return default value for column if set 234 | if (! $isNull ) { 235 | if ($this->isNumericColumn($column)) return intval($c->getDefault()); 236 | return $c->getDefault(); 237 | } 238 | 239 | // if column is auto-incrementing return null and let database set the value 240 | if ($c->getAutoIncrement()) return null; 241 | 242 | // if column accepts null values, return null 243 | if ($c->getNullable()) return null; 244 | 245 | // if column is numeric, return 0 246 | if ($this->isNumericColumn($column)) return 0; 247 | 248 | // if column is date or time type return 249 | if ($this->isDateColumn($column)) { 250 | if ($this->settings->timestamps) return date('Y-m-d H:i:s.u'); 251 | else return 0; 252 | } 253 | 254 | // if column is boolean return false 255 | if ($c->getType() == "boolean") return false; 256 | 257 | // else return empty string 258 | return ""; 259 | } 260 | } 261 | -------------------------------------------------------------------------------- /src/Writers/Database/QueryLogMemoryLeakWorkaroundProvider.php: -------------------------------------------------------------------------------- 1 | disableQueryLog() resolves many situations. 13 | * However, some packages ignore this setting, including: 14 | * 1. spatie/ignition, which was included with laravel versions between 6.x and 9.x 15 | * 2. telescope 16 | * 17 | * One technique for disabling query logging for these packages is to disable the event dispatcher for the db connection. 18 | * However, disabling the event dispatcher is not compatible with the implementation of RefreshDatabase, which itself 19 | * disables the querylog before beginning a transaction, and then assumes that it can restore the previous dispatcher 20 | * without checking to see if it was enabled. 21 | * 22 | * The workaround for RefreshDatabase is to disable the event dispatcher at the start of a file, and enable it again 23 | * at the end of the file. 24 | * 25 | * 26 | */ 27 | class QueryLogMemoryLeakWorkaroundProvider 28 | { 29 | protected $dbConnectionEventDispatcher; 30 | 31 | public function boot() { 32 | DB::connection()->disableQueryLog(); 33 | $this->dbConnectionEventDispatcher = DB::connection()->getEventDispatcher(); 34 | 35 | Event::listen(FileStart::class, [$this, 'disableEventDispatcher']); 36 | Event::listen(FileFinish::class, [$this, 'enableEventDispatcher']); 37 | } 38 | 39 | public function disableEventDispatcher() 40 | { 41 | DB::connection()->unsetEventDispatcher(); 42 | } 43 | 44 | public function enableEventDispatcher() 45 | { 46 | DB::connection()->setEventDispatcher($this->dbConnectionEventDispatcher); 47 | } 48 | 49 | } -------------------------------------------------------------------------------- /src/Writers/Text/Markdown/MarkdownFormatter.php: -------------------------------------------------------------------------------- 1 | settings = $settings; 27 | $this->writer = new TextOutputWriter("md", $markdownFormatter); 28 | } 29 | 30 | public function boot() 31 | { 32 | if (!$this->settings->textOutput()->contains('markdown')) return; 33 | 34 | $this->writer->boot(); 35 | } 36 | } -------------------------------------------------------------------------------- /src/Writers/Text/TextOutputFileRepository.php: -------------------------------------------------------------------------------- 1 | spreadsheetSeederSettings = App::make(SpreadsheetSeederSettings::class); 51 | $this->sourcePathname = $sourcePathname; 52 | $this->extension = ltrim($outputExtension, ".*"); 53 | $this->createPath(); 54 | } 55 | 56 | public function openSheet($sheetName) 57 | { 58 | if (! $this->isSheetActive( $sheetName )) { 59 | $this->closeSheet(); 60 | $this->sheet = $sheetName; 61 | } 62 | } 63 | 64 | public function write($string) 65 | { 66 | return $this->file()->fwrite($string); 67 | } 68 | 69 | 70 | public function closeSheet() 71 | { 72 | unset($this->sheet); 73 | if (isset($this->file)) $this->file->fflush(); 74 | $this->file = null; 75 | } 76 | 77 | protected function isSheetActive($name) 78 | { 79 | return isset($this->sheet) && $this->sheet == $name; 80 | } 81 | 82 | protected function isOutputPathSet() 83 | { 84 | return !empty($this->spreadsheetSeederSettings->textOutputPath); 85 | } 86 | 87 | protected function pathName() 88 | { 89 | if (isset($this->_pathName)) return $this->_pathName; 90 | 91 | $this->_pathName = ''; 92 | $path_parts = pathinfo($this->sourcePathname); 93 | if ($this->isOutputPathSet()) 94 | $this->_pathName = $this->spreadsheetSeederSettings->textOutputPath; 95 | elseif (strlen($path_parts['dirname']) > 0) 96 | $this->_pathName = $path_parts['dirname'] . '/'; 97 | $this->_pathName = $this->_pathName . $path_parts['filename']; 98 | return $this->_pathName; 99 | } 100 | 101 | protected function createPath() 102 | { 103 | $mkdirResult = false; 104 | if (!(is_dir($this->pathName()))) { 105 | $mkdirResult = mkdir($this->pathName(), 0777, true); 106 | } 107 | 108 | $glob = $this->pathName() . "/*.$this->extension"; 109 | array_map('unlink', glob($glob)); 110 | } 111 | 112 | protected function filename() 113 | { 114 | return $this->pathName() . '/' . $this->sheet . '.' . $this->extension; 115 | } 116 | 117 | protected function file() 118 | { 119 | return $this->file ?? $this->file = new \SplFileObject($this->filename(), 'w'); 120 | } 121 | } -------------------------------------------------------------------------------- /src/Writers/Text/TextOutputWriter.php: -------------------------------------------------------------------------------- 1 | extension = $extension; 37 | $this->formatter = $textTableFormatter; 38 | } 39 | 40 | public function boot() 41 | { 42 | Event::listen(FileStart::class, [$this, 'handleFileStart']); 43 | Event::listen(SheetStart::class, [$this, 'handleSheetStart']); 44 | Event::listen(ChunkFinish::class, [$this, 'handleChunkFinish']); 45 | Event::listen(SheetFinish::class, [$this, 'handleSheetFinish']); 46 | } 47 | 48 | /** 49 | * @param $fileStart FileStart 50 | */ 51 | public function handleFileStart($fileStart) 52 | { 53 | $this->repository = new TextOutputFileRepository($fileStart->file, $this->extension); 54 | } 55 | 56 | /** 57 | * @param $sheetStart SheetStart 58 | */ 59 | public function handleSheetStart($sheetStart) 60 | { 61 | $this->repository->openSheet($sheetStart->tableName); 62 | $this->repository->write($this->formatter->tableName($sheetStart->tableName)); 63 | $this->repository->write($this->formatter->header($sheetStart->header)); 64 | } 65 | 66 | /** 67 | * @param $chunkFinish ChunkFinish 68 | */ 69 | public function handleChunkFinish($chunkFinish) 70 | { 71 | $this->repository->write($this->formatter->rows($chunkFinish->rows->rawRows)); 72 | } 73 | 74 | /** 75 | * @param $sheetFinish SheetFinish 76 | */ 77 | public function handleSheetFinish($sheetFinish) 78 | { 79 | $this->repository->write($this->formatter->footer()); 80 | $this->repository->closeSheet(); 81 | } 82 | } -------------------------------------------------------------------------------- /src/Writers/Text/TextTableFormatter.php: -------------------------------------------------------------------------------- 1 | header = $header; 113 | $this->isHeaderWritten = false; 114 | return ""; 115 | } 116 | public function tableHeader() : string 117 | { 118 | if ($this->isHeaderWritten) return ""; 119 | $this->columnWidths(); 120 | $this->isHeaderWritten = true; 121 | return $this->headerColumns(); 122 | } 123 | 124 | public function rows($rows) :string 125 | { 126 | $out = ""; 127 | $this->rows = $rows; 128 | $this->rowCount += count($rows); 129 | $out .= $this->tableHeader(); 130 | $this->columnWidthsFromRows(); 131 | $out .= $this->formatRowsAsTable(); 132 | unset($this->rows); 133 | return $out; 134 | } 135 | 136 | public function footer() : string 137 | { 138 | return'(' . $this->rowCount . " rows)\n\n"; 139 | } 140 | 141 | public function tableName($tableName) : string 142 | { 143 | $out = ""; 144 | $this->tableName = $tableName; 145 | $out .= $this->tableName . "\n"; 146 | $border = str_repeat('=', strlen($this->tableName)); 147 | $out .= $border . "\n\n"; 148 | return $out; 149 | } 150 | 151 | protected function headerColumns() : string 152 | { 153 | $out = ""; 154 | 155 | foreach ($this->header as $index => $columnName) { 156 | $columnHeader = str_pad($columnName, $this->columnWidths[$index] + $this->columnPadding, " ", STR_PAD_BOTH); 157 | $columnSeparator = ($index > 0) ? $columnSeparator = $this->headerColumnSeparator : $this->headerOutsideLeftColumnSeparator; 158 | $out .= $columnSeparator . $columnHeader; 159 | } 160 | $out .= $this->headerOutsideRightColumnSeparator . "\n"; 161 | 162 | foreach ($this->header as $index => $columnName) { 163 | $columnHeader = str_repeat($this->headerUnderlineCharacter, $this->columnWidths[$index] + $this->columnPadding); 164 | $columnSeparator = ($index > 0) ? $columnSeparator = $this->headerUnderlineColumnSeparator : $columnSeparator = $this->headerUnderlineOutsideLeftColumnSeparator; 165 | $out .= $columnSeparator . $columnHeader; 166 | } 167 | $out .= $this->headerUnderlineOutsideRightColumnSeparator . "\n"; 168 | return $out; 169 | } 170 | 171 | protected function formatRowsAsTable() { 172 | $out = ""; 173 | foreach ($this->rows as $row) { 174 | foreach ($row as $index => $value) { 175 | $valueCell = str_pad($value, $this->columnWidths[$index]); 176 | $columnSeparator = ($index > 0) ? $columnSeparator = $this->rowColumnSeparator : $columnSeparator = $this->rowOutsideLeftColumnSeparator; 177 | $out .= $columnSeparator . ' ' . $valueCell . ' '; 178 | } 179 | $out .= $this->rowOutsideRightColumnSeparator . "\n"; 180 | } 181 | 182 | return $out; 183 | } 184 | 185 | protected function columnWidths() { 186 | foreach ($this->header as $index => $columnName) { 187 | $this->columnWidths[$index] = max(strlen($columnName),1); 188 | } 189 | 190 | $this->columnWidthsFromRows(); 191 | } 192 | 193 | protected function columnWidthsFromRows() { 194 | if(is_null($this->rows)) return; 195 | foreach ($this->rows as $row) { 196 | foreach ($row as $index => $value) { 197 | if (!isset($this->columnWidths[$index]) || strlen($value) > $this->columnWidths[$index]) 198 | $this->columnWidths[$index] = strlen($value); 199 | } 200 | } 201 | } 202 | } -------------------------------------------------------------------------------- /src/Writers/Text/TextTableFormatterInterface.php: -------------------------------------------------------------------------------- 1 | header = $header; 33 | $out .= "Header:\n"; 34 | foreach($this->header as $header) 35 | { 36 | $out .= " - " . $this->quote($header) . "\n"; 37 | } 38 | $out .= "Rows:\n"; 39 | return $out; 40 | } 41 | 42 | public function tableName($tableName) : string 43 | { 44 | $this->tableName = $tableName; 45 | return "Table: " . $this->tableName . "\n"; 46 | } 47 | 48 | protected function columnName($index) 49 | { 50 | if (isset($this->header[$index])) return $this->quote($this->header[$index]); 51 | return $index; 52 | } 53 | 54 | public function rows($rows) : string 55 | { 56 | $out = ""; 57 | foreach ($rows as $row) { 58 | $this->rowCount++; 59 | $out .= " " . $this->rowCount . ":\n"; 60 | foreach ($row as $index => $value) { 61 | $out .= " " . $this->columnName($index) . ": " . $this->quote($value) . "\n"; 62 | } 63 | } 64 | 65 | return $out; 66 | } 67 | 68 | public function footer() : string 69 | { 70 | $out = 'RowCount: ' . $this->rowCount . "\n"; 71 | $this->rowCount = 0; 72 | return $out; 73 | } 74 | 75 | protected function quote($string) : string 76 | { 77 | if (is_null($string)) return ""; 78 | $special = [':', '{', '}', '[', ']', ',', '&', '*', '#', '?', '|', '-', '<', ">", '=', '!', '%', '@', "\\"]; 79 | if (Str::contains($string, $special)) { 80 | return "'" . str_replace("'", "''", $string) . "'"; 81 | } 82 | if ($string == 'Yes') return "'Yes'"; 83 | if ($string == 'No') return "'No'"; 84 | return $string; 85 | } 86 | } -------------------------------------------------------------------------------- /src/Writers/Text/Yaml/YamlWriter.php: -------------------------------------------------------------------------------- 1 | settings = $settings; 27 | $this->writer = new TextOutputWriter("yaml", $yamlFormatter); 28 | } 29 | 30 | public function boot() 31 | { 32 | if (!$this->settings->textOutput()->contains('yaml')) return; 33 | 34 | $this->writer->boot(); 35 | } 36 | } --------------------------------------------------------------------------------