├── .gitignore ├── composer.json ├── readme.md ├── src └── Crockett │ └── CsvSeeder │ ├── CsvSeeder.php │ └── CsvSeederServiceProvider.php └── tests ├── CsvTest.php ├── README.md ├── csvs ├── users.csv └── users_with_ignored_column.csv └── migrations └── 2014_10_12_000000_create_users_table.php /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | /vendor 3 | composer.phar 4 | composer.lock 5 | .DS_Store -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "crockett/csv-seeder", 3 | "description": "Database seeding using CSV files", 4 | "keywords": [ 5 | "laravel", 6 | "csv", 7 | "seed", 8 | "seeds", 9 | "seeder", 10 | "seeding" 11 | ], 12 | "license": "MIT", 13 | "authors": [ 14 | { 15 | "name": "Andy Crockett", 16 | "email": "andyhcrockett@gmail.com" 17 | } 18 | ], 19 | "require": { 20 | "php": ">=5.4.0", 21 | "laravel/framework": ">=5.0.0" 22 | }, 23 | "autoload": { 24 | "psr-0": { 25 | "Crockett\\CsvSeeder\\": "src/" 26 | } 27 | }, 28 | "minimum-stability": "dev" 29 | } 30 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## CSV Seeder 2 | 3 | This package is intended to minimize the time and hassle spent importing CSV-based data. By making a few assumptions about your average CSV file, most users won't need any configuration to start seeding. For those that do, the available configuration options offer plenty of control over your CSV and how the data is inserted into the database. 4 | 5 | 6 | ### Overview 7 | - [Key Features](#key-features) 8 | - [Installation](#installation) 9 | - [Setup](#setup) 10 | - [Usage Examples](#usage-examples) 11 | - [Mismatched Columns](#mismatched-columns) 12 | - [Insert Callbacks](#insert-callbacks) 13 | - [Configuration](#configuration) 14 | - [More Examples](#more-examples) 15 | - [License](#license) 16 | 17 | 18 | ### Key Features 19 | 20 | - Automatic mapping of CSV headers to the columns in your DB table. 21 | - Aliases allow you to easily adjust a CSV column's name before it's inserted. 22 | - Insert callbacks can be used to directly manipulate the CSV data before it's inserted. 23 | - ORM support - when using a model, common attributes such as `$guarded` and `$fillable` are applied to the CSV. 24 | 25 | 26 | ## Installation 27 | 28 | Require this package in your composer.json and run `composer update` 29 | 30 | "crockett/csv-seeder": "1.1.*" 31 | 32 | Or just require it directly `composer require crockett/csv-seeder` 33 | 34 | ## Setup 35 | 36 | Here is a typical, single CSV seeder setup: 37 | 38 | use Crockett\CsvSeeder\CsvSeeder; 39 | 40 | class UsersTableSeeder extends CsvSeeder { 41 | 42 | public function __construct() 43 | { 44 | $this->filename = base_path('path/to/csv/users.csv'); 45 | $this->table = 'users'; 46 | } 47 | 48 | public function run() 49 | { 50 | // runs the seeder - alternatively, you could call $this->runSeeder(); for the same result 51 | parent::run(); 52 | } 53 | } 54 | 55 | If you want to seed multiple CSVs in the same seeder, you could do something like this: 56 | 57 | use Crockett\CsvSeeder\CsvSeeder; 58 | 59 | class UsersTableSeeder extends CsvSeeder { 60 | 61 | public function run() 62 | { 63 | // seed the users table 64 | $this->filename = base_path('path/to/csv/users.csv'); 65 | $this->table = 'users'; 66 | parent::run(); 67 | 68 | // seed the posts table 69 | $this->filename = base_path('path/to/csv/posts.csv'); 70 | $this->table = 'posts'; 71 | parent::run(); 72 | } 73 | } 74 | 75 | As you can imagine, that can get messy very fast. Instead, you could use the helper method `seedFromCSV()` which is just a cleaner way to define your parameters and run the seeder in one go: 76 | 77 | use Crockett\CsvSeeder\CsvSeeder; 78 | 79 | class UsersTableSeeder extends CsvSeeder { 80 | 81 | public function run() 82 | { 83 | // seed the users table 84 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 85 | 86 | // seed the posts table 87 | $this->seedFromCSV(base_path('path/to/posts.csv'), 'posts'); 88 | } 89 | } 90 | 91 | 92 | ## Usage Examples 93 | 94 | Given the following CSV and database table: 95 | 96 | // users.csv 97 | first_name,last_name,birth_date,password,favorite_color 98 | Joe,Bar,2000-02-10,joePassword,red 99 | Jim,Foo,1990-02-10,jimPassword,blue 100 | Foo,Bar,1980-02-10,fooPassword,green 101 | 102 | // users DB table 103 | id, first_name, last_name, birth_date, password, favorite_color 104 | 105 | You can run the seeder with no further setup: 106 | 107 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 108 | 109 | You could even go a step further and omit the table name, as the CSV filename is the same as the table name. `CsvSeeder` will automatically try to resolve table and column names when they're not defined. If your CSV doesn't have a header row, you'll need to manually define a `$mapping`, as described in the next section. 110 | 111 | ### Mismatched columns 112 | 113 | Unless you have complete control over your CSVs, the headers won't always match up with your DB columns. For example: 114 | 115 | // users.csv 116 | first_name, last_name, birth_date, password, favorite_color 117 | 118 | // users DB table 119 | id, first_name, last_name, age, password 120 | 121 | In this case, you can define `$aliases` to rename the `birth_date` column to `age` before it's inserted: 122 | 123 | $this->aliases = [ 124 | 'birth_date' => 'age' 125 | ]; 126 | 127 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 128 | 129 | Alternatively, you can manually define a `$mapping` for your CSV. A mapping allows you to explicitly choose and rename CSV columns. For example: 130 | 131 | // users.csv 132 | first_name, last_name, birth_date, password, favorite_color 133 | 134 | // users DB table 135 | id, first_name, last_name, color, password 136 | 137 | // users seeder 138 | $this->mapping = [ 139 | 0 => 'first_name', 140 | 1 => 'last_name', 141 | 3 => 'password', 142 | 4 => 'color', // renamed from favorite_color 143 | ]; 144 | 145 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 146 | 147 | When you define a `$mapping`, a header row on your CSV is *not* required. In all other cases, `CsvSeeder` will assume your header row is the first row after `$offset_rows`. 148 | 149 | ### Insert Callbacks 150 | 151 | In some cases, you'll need to manipulate the CSV data directly before it's inserted to the database. Using an `$insert_callback`, it couldn't be easier! Everytime a `$chunk` of rows is read from the CSV, it's passed to the default `$insert_callback`. All you need to do is define your own callback to override it. 152 | 153 | Here we'll iterate over individual rows in the chunk and insert them using `Model::create()`: 154 | 155 | $this->insert_callback = function ($chunk) { 156 | foreach($chunk as $row) { 157 | \App\User::create($row->toArray()); 158 | } 159 | }; 160 | 161 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 162 | 163 | Note, `$chunk` and `$row` are instances of `\Illuminate\Support\Collection` so you can easily manipulate and filter the rows and columns: 164 | 165 | $this->insert_callback = function ($chunk) { 166 | foreach($chunk as $row) { 167 | $user_data = $row->only('first_name', 'last_name', 'password')->toArray(); 168 | \App\User::create($user_data); 169 | } 170 | }; 171 | 172 | $this->seedFromCSV(base_path('path/to/users.csv'), 'users'); 173 | 174 | 175 | ## Configuration 176 | 177 | - `table` (string) Database table to insert into. 178 | - `model` (string) Instead of a table name, you can pass an ORM model name. 179 | - `model_guard` (bool true) - Respect model attributes such as $fillable and $guarded when resolving table columns with a model. 180 | - `filename` (string) The path to the CSV file. 181 | - `delimiter` (string ,) The CSV field delimiter. 182 | - `offset_rows` (int 0) How many rows at the start of the CSV to skip. 183 | - `skip_header_row` (bool true) Automatically skip the first row if it's determined to be the header. Setting `offset_rows` higher than 0 bypasses this. 184 | - `mapping` (array) Associative array of csvColumnIndex => csvColumnName. See examples for details. If not specified, the first row (after offset) of the CSV will be used as the mapping. 185 | - `aliases` (array) Associative array of csvColumnName => aliasColumnName. See examples for details. Allows for flexible CSV column names. 186 | - `hashable` (string|array 'password') Hashes the specified field(s) using `bcrypt`. Useful if you are importing users and need their passwords hashed. Note: This is EXTREMELY SLOW, large CSVs will take time to import. 187 | - `insert_chunk_size` (int 50) An insert callback will trigger every `insert_chunk_size` rows while reading the CSV. 188 | - `insert_callback` (callable) - Override the default insert callback with your own. Callback must accept a `Collection` of rows ($chunk). 189 | - `console_logs` (bool true) - Show messages in the console. (neglible performance impact) 190 | - `write_logs` (bool false) - Write messages to logs. (recommended off for large CSVs) 191 | - `disable_query_log` (bool true) - Disable the query log. (recommended on for large CSVs) 192 | - `log_prefix` (string) - Customize the log messages 193 | 194 | 195 | ## More Examples 196 | 197 | CSV with pipe delimited values: 198 | 199 | public function __construct() 200 | { 201 | $this->table = 'users'; 202 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 203 | $this->delimiter = '|'; 204 | } 205 | 206 | Specifying which CSV columns to import: 207 | 208 | public function __construct() 209 | { 210 | $this->table = 'users'; 211 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 212 | $this->mapping = [ 213 | 0 => 'first_name', 214 | 1 => 'last_name', 215 | 5 => 'age', 216 | ]; 217 | } 218 | 219 | Using a model instead of a table: 220 | 221 | public function __construct() 222 | { 223 | $this->model = \App\User::class; 224 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 225 | // optionally, disable the $model_guard to ignore your model's guarded/fillable attributes 226 | $this->model_guard = false; 227 | } 228 | 229 | Skipping the first row of your CSV (Note: If the first row after the offset isn't the header row, a mapping must be defined): 230 | 231 | public function __construct() 232 | { 233 | $this->table = 'users'; 234 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 235 | $this->offset_rows = 1; 236 | $this->mapping = [ 237 | 0 => 'first_name', 238 | 1 => 'last_name', 239 | 2 => 'password', 240 | ]; 241 | } 242 | 243 | Aliasing a CSV column: 244 | 245 | public function __construct() 246 | { 247 | $this->table = 'users'; 248 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 249 | $this->aliases = [ 250 | 'age' => 'date_of_birth', 251 | ]; 252 | } 253 | 254 | Aliasing a CSV column defined in `$mapping`: 255 | 256 | public function __construct() 257 | { 258 | $this->table = 'users'; 259 | $this->filename = base_path('database/seeds/csvs/your_csv.csv'); 260 | $this->mapping = [ 261 | 0 => 'first_name', 262 | 1 => 'last_name', 263 | 5 => 'birth_date', // in the CSV file, this column is named 'age' 264 | ]; 265 | $this->aliases = [ 266 | 'birth_date' => 'date_of_birth', 267 | ]; 268 | } 269 | 270 | Check out the source of `Crockett\CsvSeeder\CsvSeeder` for more complete information about the available methods. 271 | 272 | ## License 273 | 274 | CsvSeeder is open-sourced software licensed under the [MIT license](http://opensource.org/licenses/MIT) 275 | -------------------------------------------------------------------------------- /src/Crockett/CsvSeeder/CsvSeeder.php: -------------------------------------------------------------------------------- 1 | 'first_column_name', 76 | * 2 => 'third_column_name', 77 | * 3 => 'fourth_column_name', 78 | * ]; 79 | * 80 | * @var array 81 | */ 82 | public $mapping = []; 83 | 84 | /** 85 | * Aliases CSV headers to differently named DB columns. 86 | * 87 | * Useful when $this->mapping is being resolved automatically and you're 88 | * reading a CSV with headers named differently from your DB columns. 89 | * 90 | * For example, to alias a CSV column named "email_address" to a DB column named "email": 91 | * [ 92 | * 'email_address' => 'email' 93 | * // 'csv_header' => 'alias_name' 94 | * ]; 95 | * 96 | * @var array 97 | */ 98 | public $aliases = []; 99 | 100 | /** 101 | * Specifies DB columns that should have their values hashed prior to insertion. 102 | * Override this as needed. 103 | * 104 | * If you set any $aliases, be sure to use the aliased DB column name. 105 | * 106 | * @var array|string 107 | */ 108 | 109 | public $hashable = 'password'; 110 | 111 | /** 112 | * A SQL INSERT query will execute every time this number of rows are read from the CSV. 113 | * Without this, large INSERTS will fail silently. 114 | * 115 | * @var int 116 | */ 117 | public $insert_chunk_size = 50; 118 | 119 | /** 120 | * A closure that takes an array of CSV rows ($chunk) and inserts them into the DB. 121 | * Use to override the default insertion behavior. 122 | * 123 | * Example: 124 | * function ($chunk) { 125 | * // insert $rows individually with model::create() 126 | * foreach($chunk as $row) { 127 | * YourModel::create($row); 128 | * } 129 | * } 130 | * 131 | * @var closure|callable|null 132 | */ 133 | public $insert_callback = null; 134 | 135 | /** 136 | * Show messages in the console 137 | * 138 | * @var bool 139 | */ 140 | public $console_logs = true; 141 | 142 | /** 143 | * Write messages to laravel.log 144 | * 145 | * @var bool 146 | */ 147 | public $write_logs = false; 148 | 149 | /** 150 | * Enables or disables query logging. Recommended for large CSVs. 151 | * 152 | * @var bool 153 | */ 154 | public $disable_query_log = true; 155 | 156 | /** 157 | * The prefix for log messages 158 | * 159 | * @var string 160 | */ 161 | public $log_prefix = ''; 162 | 163 | /** 164 | * Holder for columns read from the DB table 165 | * 166 | * @var array 167 | */ 168 | private $table_columns; 169 | 170 | /** 171 | * CsvSeeder constructor. 172 | */ 173 | public function __construct( 174 | $filename = null, 175 | $table = null, 176 | $model = null, 177 | $delimiter = null, 178 | $mapping = null, 179 | $aliases = null, 180 | $insert_callback = null 181 | ) { 182 | if (!is_null($filename)) { 183 | $this->seedFromCSV($filename, $table, $model, $delimiter, $mapping, $aliases, $insert_callback); 184 | } 185 | } 186 | 187 | /** 188 | * Run DB seed 189 | */ 190 | public function run() 191 | { 192 | $this->runSeeder(); 193 | } 194 | 195 | 196 | public function seedFromCSV( 197 | $filename = null, 198 | $table = null, 199 | $model = null, 200 | $delimiter = null, 201 | $aliases = null, 202 | $mapping = null, 203 | $insert_callback = null 204 | ) { 205 | $this->filename = $filename ?: $this->filename; 206 | $this->table = $table ?: $this->table; 207 | $this->model = $model ?: $this->model; 208 | $this->delimiter = $delimiter ?: $this->delimiter; 209 | $this->aliases = $aliases ?: $this->aliases; 210 | $this->mapping = $mapping ?: $this->mapping; 211 | $this->insert_callback = $insert_callback ?: $this->insert_callback ; 212 | 213 | $this->runSeeder(); 214 | } 215 | 216 | public function runSeeder() 217 | { 218 | // abort for missing filename 219 | if (empty( $this->filename )) { 220 | $this->log('CSV filename was not specified.', 'critical'); 221 | $this->console('CSV filename was not specified.', 'error'); 222 | 223 | return; 224 | } 225 | 226 | // resolve the model 227 | if (!empty( $this->model )) { 228 | if ($this->resolveModel() === false) { 229 | $this->log("$this->model could not be resolved.", 'warning'); 230 | $this->console("$this->model could not be resolved.", 'error'); 231 | // continue, despite the model 232 | } 233 | } 234 | 235 | // resolve the table name or abort 236 | if ($this->resolveTable() === false) { 237 | $this->log("Table could not be resolved or does not exist.", 'warning'); 238 | $this->console('Table could not be resolved or does not exist. Try setting it manually.', 'error'); 239 | 240 | return; 241 | } 242 | 243 | // update the log_prefix with the table name 244 | $this->log_prefix = $this->log_prefix . "$this->table: "; 245 | 246 | // load the allowed table columns or abort if there are none 247 | if ($this->resolveTableColumns() === false) { 248 | $this->log('Unable to resolve DB columns', 'critical'); 249 | $this->console('Unable to resolve DB columns.', 'error'); 250 | 251 | return; 252 | }; 253 | 254 | // convert hashable to array 255 | if (is_string($this->hashable)) { 256 | $this->hashable = [$this->hashable]; 257 | } 258 | 259 | // disable query log 260 | if ($this->disable_query_log) { 261 | DB::disableQueryLog(); 262 | } 263 | 264 | // read and parse the CSV, seeding the database 265 | $this->parseCSV(); 266 | 267 | // reset seeder for another run 268 | $this->resetSeeder(); 269 | } 270 | 271 | /** 272 | * Reset the seeder for another use 273 | */ 274 | public function resetSeeder() 275 | { 276 | $this->filename = null; 277 | $this->model = null; 278 | $this->table = null; 279 | $this->aliases = []; 280 | $this->mapping = []; 281 | $this->hashable = []; 282 | $this->delimiter = ','; 283 | $this->offset_rows = 0; 284 | $this->log_prefix = ''; 285 | 286 | $this->insert_chunk_size = 50; 287 | $this->insert_callback = null; 288 | } 289 | 290 | /** 291 | * Opens a CSV file and returns it as a resource 292 | * 293 | * @param $filename 294 | * 295 | * @return FALSE|resource 296 | */ 297 | public function openCSV($filename) 298 | { 299 | if (!file_exists($filename) || !is_readable($filename)) { 300 | return false; 301 | } 302 | 303 | // check if file is gzipped 304 | $file_info = finfo_open(FILEINFO_MIME_TYPE); 305 | $file_mime_type = finfo_file($file_info, $filename); 306 | finfo_close($file_info); 307 | $gzipped = strcmp($file_mime_type, "application/x-gzip") == 0; 308 | 309 | $handle = $gzipped ? gzopen($filename, 'r') : fopen($filename, 'r'); 310 | 311 | return $handle; 312 | } 313 | 314 | /** 315 | * Parse rows from the CSV and pass chunks of rows to the insert function 316 | */ 317 | public function parseCSV() 318 | { 319 | $handle = $this->openCSV($this->filename); 320 | 321 | // abort for bad CSV 322 | if ($handle === false) { 323 | $this->console( 324 | "CSV file {$this->filename} does not exist or is not readable.", 'error'); 325 | 326 | return; 327 | } 328 | 329 | $row_count = 0; 330 | $skipped = 0; // rows that were skipped 331 | $failed = 0; // chunk inserts that failed 332 | $chunk = new Collection(); // accumulator for rows until the chunk_limit is reached 333 | $mapping = empty( $this->mapping ) ? [] : $this->cleanMapping($this->mapping); 334 | $offset = $this->offset_rows; 335 | 336 | while (( $row = fgetcsv($handle, 0, $this->delimiter) ) !== false) { 337 | 338 | if ($row_count == 0 && $offset == 0) { 339 | // Resolve mapping from the first row 340 | if (empty( $mapping )) { 341 | $mapping = $this->cleanMapping($row); 342 | } 343 | 344 | // Automagically skip the header row 345 | if (!empty( $mapping ) && $this->skip_header_row) { 346 | if ($this->isHeaderRow($row, $mapping)) { 347 | $offset ++; 348 | } 349 | } 350 | } 351 | 352 | // Skip the offset rows 353 | while ($offset > 0) { 354 | $offset --; 355 | continue 2; 356 | } 357 | 358 | // Resolve mapping using the first row after offset 359 | if (empty( $mapping )) { 360 | $mapping = $this->cleanMapping($row); 361 | // abort if mapping empty 362 | if (empty( $mapping )) { 363 | $this->console("The mapping columns do not exist on the DB table.", 'error'); 364 | 365 | return; 366 | } 367 | } 368 | 369 | $row = $this->parseRow($row, $mapping); 370 | 371 | // Insert only non-empty rows from the csv file 372 | if ($row->isEmpty()) { 373 | $skipped ++; 374 | continue; 375 | } 376 | 377 | $chunk->push($row); 378 | 379 | // Chunk size reached, insert and clear the chunk 380 | if (count($chunk) >= $this->insert_chunk_size) { 381 | if (!$this->insert($chunk)) $failed ++; 382 | $chunk = new Collection(); 383 | } 384 | 385 | $row_count ++; 386 | } 387 | 388 | // convert failed chunks to failed rows 389 | $failed = $failed * $this->insert_chunk_size; 390 | 391 | // Insert any leftover rows from the last chunk 392 | if (count($chunk) > 0) { 393 | if (!$this->insert($chunk)) $failed += count($chunk); 394 | } 395 | 396 | fclose($handle); 397 | 398 | // log results to console 399 | $log = 'Imported ' . ( $row_count - $skipped - $failed ) . ' of ' . $row_count . ' rows. '; 400 | if ($skipped > 0) $log .= $skipped . " empty rows. "; 401 | if ($failed > 0) $log .= "" . $failed . " failed rows."; 402 | 403 | $this->console($log); 404 | } 405 | 406 | /** 407 | * Insert a chunk of rows into the DB 408 | * 409 | * @param Collection $chunk 410 | * 411 | * @return bool TRUE on success else FALSE 412 | */ 413 | public function insert(Collection $chunk) 414 | { 415 | $callback = $this->getInsertCallback(); 416 | 417 | try { 418 | call_user_func($callback, $chunk); 419 | } catch (\Exception $e) { 420 | $this->log("Chunk insert failed:\n" . $e->getMessage(), 'critical'); 421 | 422 | return false; 423 | } 424 | 425 | return true; 426 | } 427 | 428 | /** 429 | * Resolve the function that inserts chunks into the database or returns the default behavior. 430 | * 431 | * @returns closure|callable 432 | */ 433 | public function getInsertCallback() 434 | { 435 | return is_object($this->insert_callback) 436 | ? $this->insert_callback 437 | : function (Collection $chunk) { 438 | if (empty( $this->model )) { 439 | // use DB table insert method 440 | DB::table($this->table)->insert($chunk->toArray()); 441 | } else { 442 | // use model insert method 443 | $model = $this->resolveModel(); 444 | $model->insert($chunk->toArray()); 445 | } 446 | }; 447 | } 448 | 449 | /** 450 | * Strips UTF-8 BOM characters from a string 451 | * 452 | * @param $string 453 | * 454 | * @return string 455 | */ 456 | public function stripUtf8Bom($string) 457 | { 458 | $bom = pack('H*', 'EFBBBF'); 459 | $string = preg_replace("/^$bom/", '', $string); 460 | 461 | return $string; 462 | } 463 | 464 | /** 465 | * Truncate a table (optionally ignore foreign keys) 466 | */ 467 | public function truncateTable($ignore_foreign_keys = false) 468 | { 469 | if (empty( $this->table )) { 470 | if ($this->resolveTable() === false) { 471 | $this->log('Unable to truncate table: Table not specified.', 'warning'); 472 | $this->console('Unable to truncate table: Table not specified.', 'error'); 473 | 474 | return; 475 | }; 476 | } 477 | 478 | if ($ignore_foreign_keys) { 479 | DB::statement('SET FOREIGN_KEY_CHECKS=0;'); 480 | } 481 | 482 | DB::table($this->table)->truncate(); 483 | 484 | if ($ignore_foreign_keys) { 485 | DB::statement('SET FOREIGN_KEY_CHECKS=1;'); 486 | } 487 | } 488 | 489 | /** 490 | * Check if the column values in $row are the same as the column names in $mapping. 491 | */ 492 | protected function isHeaderRow(array $row, array $mapping) 493 | { 494 | $is_header_row = true; 495 | 496 | foreach ($mapping as $index => $column) { 497 | if (array_key_exists($index, $row)) { 498 | if ($row[$index] != $column) { 499 | $is_header_row = false; 500 | } 501 | } 502 | } 503 | 504 | return $is_header_row; 505 | } 506 | 507 | /** 508 | * Parse a CSV row into a DB insertable array 509 | * 510 | * @param array $row List of CSV columns 511 | * @param array $mapping Array of csvCol => dbCol 512 | * 513 | * @return Collection 514 | */ 515 | protected function parseRow(array $row, array $mapping) 516 | { 517 | $columns = new Collection(); 518 | // apply mapping to a given row 519 | foreach ($mapping as $csv_index => $column_name) { 520 | $column_value = ( array_key_exists($csv_index, $row) && isset( $row[$csv_index] ) && $row[$csv_index] !== '') 521 | ? $row[$csv_index] 522 | : null; 523 | $columns->put($column_name, $column_value); 524 | } 525 | 526 | $columns = $this->aliasColumns($columns); 527 | 528 | $columns = $this->hashColumns($columns); 529 | 530 | return $columns; 531 | } 532 | 533 | /** 534 | * Remove columns in the mapping that don't exist in the DB table 535 | * 536 | * @param array $mapping 537 | * 538 | * @return array 539 | */ 540 | protected function cleanMapping(array $mapping) 541 | { 542 | $columns = $mapping; 543 | if (isset($columns[0])) // only needed if we use first column of line 544 | $columns[0] = $this->stripUtf8Bom($columns[0]); 545 | 546 | // Cull columns that don't exist in the database or were guarded by the model 547 | foreach ($columns as $index => $column) { 548 | // apply column alias 549 | $column = $this->aliasColumn($column); 550 | if (array_search($column, $this->table_columns) === false) { 551 | array_pull($columns, $index); 552 | } 553 | } 554 | 555 | return $columns; 556 | } 557 | 558 | /** 559 | * Apply alias to a single column 560 | */ 561 | protected function aliasColumn($column) 562 | { 563 | return is_array($this->aliases) && array_key_exists($column, $this->aliases) 564 | ? $this->aliases[$column] 565 | : $column; 566 | } 567 | 568 | /** 569 | * Apply aliases to a group of columns 570 | */ 571 | protected function aliasColumns(Collection $columns) 572 | { 573 | if (is_array($this->aliases) && !empty( $this->aliases )) { 574 | foreach ($this->aliases as $csv_column => $alias_column) { 575 | if ($columns->has($csv_column)) { 576 | $columns->put($alias_column, $columns->get($csv_column)); 577 | $columns->pull($csv_column); 578 | } 579 | } 580 | } 581 | 582 | return $columns; 583 | } 584 | 585 | /** 586 | * Hash any hashable columns 587 | */ 588 | protected function hashColumns(Collection $columns) 589 | { 590 | if (is_array($this->hashable) && !empty( $this->hashable )) { 591 | foreach ($this->hashable as $hashable) { 592 | if ($columns->contains($hashable)) { 593 | $columns->put($hashable, bcrypt($columns[$hashable])); 594 | } 595 | } 596 | } 597 | 598 | return $columns; 599 | } 600 | 601 | /** 602 | * Apply model attributes like $fillable and $guarded to an array of columns 603 | * 604 | * @param array $columns 605 | * 606 | * @return array 607 | */ 608 | protected function guardColumns(array $columns) 609 | { 610 | if (!$this->model_guard || empty( $this->model )) { 611 | return $columns; 612 | } 613 | 614 | $model = $this->resolveModel(); 615 | 616 | // filter out columns not allowed by the $fillable attribute 617 | if (method_exists($model, 'getFillable')) { 618 | if (!empty( $fillable = $model->getFillable() )) { 619 | foreach ($columns as $index => $column) { 620 | if (array_search($column, $fillable) === false) { 621 | array_pull($columns, $index); 622 | } 623 | } 624 | }; 625 | } 626 | 627 | return $columns; 628 | } 629 | 630 | /** 631 | * Returns a new model instance 632 | */ 633 | protected function resolveModel($parameters = []) 634 | { 635 | try { 636 | $model = app($this->model, $parameters); 637 | } catch (\Exception $e) { 638 | return false; 639 | } 640 | 641 | return $model; 642 | } 643 | 644 | /** 645 | * Tries to resolve the table name using the filename 646 | */ 647 | protected function resolveTable() 648 | { 649 | // try to resolve using model 650 | if (empty( $this->table ) && !empty( $this->model )) { 651 | $model = $this->resolveModel(); 652 | if ($model !== false) { 653 | $this->table = method_exists($model, 'getTable') 654 | ? $model->getTable() 655 | : null; 656 | } 657 | } 658 | 659 | // try to resolve using filename 660 | if (empty( $this->table ) && !empty( $this->filename )) { 661 | $file = explode('/', $this->filename); 662 | $file = explode('.', $file[count($file) - 1]); 663 | 664 | $this->table = $file[0]; 665 | 666 | $this->console('Table name "' . $this->table . '" resolved from CSV filename'); 667 | } 668 | 669 | return DB::getSchemaBuilder()->hasTable($this->table); 670 | } 671 | 672 | /** 673 | * Resolves allowed columns for the table. Applies model guard if available. 674 | */ 675 | protected function resolveTableColumns() 676 | { 677 | // get every column that exists on the table 678 | $columns = DB::getSchemaBuilder()->getColumnListing($this->table); 679 | 680 | // Run the model guard on the columns 681 | $columns = $this->guardColumns($columns); 682 | 683 | $this->table_columns = $columns; 684 | 685 | return !empty( $columns ); 686 | } 687 | 688 | /** 689 | * Show a message in the console 690 | */ 691 | protected function console($message, $style = null) 692 | { 693 | if ($this->console_logs === false) return; 694 | 695 | $message = $style ? "<$style>$message" : $message; 696 | 697 | $this->command->line('CSVSeeder: ' . $this->log_prefix . $message); 698 | } 699 | 700 | /** 701 | * Write a message to the logs using Laravel's Log helper 702 | */ 703 | protected function log($message, $level = 'info') 704 | { 705 | if ($this->write_logs === false) return; 706 | 707 | logger()->log($level, 'CSVSeeder: ' . $this->log_prefix . $message); 708 | } 709 | } 710 | -------------------------------------------------------------------------------- /src/Crockett/CsvSeeder/CsvSeederServiceProvider.php: -------------------------------------------------------------------------------- 1 | package('crockett/csv-seeder'); 25 | } 26 | 27 | /** 28 | * Register the service provider. 29 | * 30 | * @return void 31 | */ 32 | public function register() 33 | { 34 | // 35 | } 36 | 37 | /** 38 | * Get the services provided by the provider. 39 | * 40 | * @return array 41 | */ 42 | public function provides() 43 | { 44 | return array(); 45 | } 46 | 47 | } -------------------------------------------------------------------------------- /tests/CsvTest.php: -------------------------------------------------------------------------------- 1 | artisan('migrate', [ 12 | '--path' => 'vendor/Crockett/csv-seeder/tests/migrations', 13 | ]); 14 | 15 | $this->beforeApplicationDestroyed(function () { 16 | $this->artisan('migrate:rollback'); 17 | }); 18 | } 19 | 20 | /** 21 | * Setup the test environment. 22 | * 23 | * @return void 24 | */ 25 | public function setUp() 26 | { 27 | parent::setUp(); 28 | 29 | // Use an in-memory DB 30 | $this->app['config']->set('database.default', 'csvSeederTest'); 31 | $this->app['config']->set('database.connections.csvSeederTest', [ 32 | 'driver' => 'sqlite', 33 | 'database' => ':memory:', 34 | 'prefix' => '', 35 | ]); 36 | } 37 | 38 | public function testBOMIsStripped() 39 | { 40 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 41 | 42 | $bomString = chr(239) . chr(187) . chr(191) . "foo"; 43 | $nonBomString = "my non bom string"; 44 | 45 | // Test a BOM string 46 | $expected = "foo"; 47 | $actual = $seeder->stripUtf8Bom($bomString); 48 | $this->assertEquals($expected, $actual); 49 | 50 | // Test a non BOM string 51 | $expected = $nonBomString; 52 | $actual = $seeder->stripUtf8Bom($nonBomString); 53 | $this->assertEquals($expected, $actual); 54 | } 55 | 56 | public function testMappings() 57 | { 58 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 59 | $row = [1, 'ignored', 'first', 'last']; 60 | 61 | // Test no skipped columns 62 | $mapping = [ 63 | 0 => 'id', 64 | 1 => 'ignored', 65 | 2 => 'first_name', 66 | 3 => 'last_name', 67 | ]; 68 | $actual = $seeder->readRow($row, $mapping); 69 | $expected = [ 70 | 'id' => 1, 71 | 'ignored' => 'ignored', 72 | 'first_name' => 'first', 73 | 'last_name' => 'last', 74 | ]; 75 | $this->assertEquals($expected, $actual); 76 | 77 | // Test a skipped column 78 | $mapping = [ 79 | 0 => 'id', 80 | 2 => 'first_name', 81 | 3 => 'last_name', 82 | ]; 83 | $actual = $seeder->readRow($row, $mapping); 84 | $expected = [ 85 | 'id' => 1, 86 | 'first_name' => 'first', 87 | 'last_name' => 'last', 88 | ]; 89 | $this->assertEquals($expected, $actual); 90 | 91 | // Test a non-existant column 92 | $mapping = [ 93 | 0 => 'id', 94 | 2 => 'first_name', 95 | 99 => 'last_name', 96 | ]; 97 | $actual = $seeder->readRow($row, $mapping); 98 | $expected = [ 99 | 'id' => 1, 100 | 'first_name' => 'first', 101 | 'last_name' => null, 102 | ]; 103 | $this->assertEquals($expected, $actual); 104 | } 105 | 106 | public function testCanOpenCSV() 107 | { 108 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 109 | 110 | // Test an openable CSV 111 | $expected = "resource"; 112 | $actual = $seeder->openCSV(__DIR__.'/csvs/users.csv'); 113 | $this->assertInternalType($expected, $actual); 114 | 115 | // Test a non-openable CSV 116 | $expected = FALSE; 117 | $actual = $seeder->openCSV(__DIR__.'/csvs/csv_that_does_not_exist.csv'); 118 | $this->assertEquals($expected, $actual); 119 | } 120 | 121 | public function testImport() 122 | { 123 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 124 | $seeder->table = 'users'; 125 | $seeder->filename = __DIR__.'/csvs/users.csv'; 126 | $seeder->hashable = ''; 127 | $seeder->run(); 128 | 129 | // Make sure the rows imported 130 | $this->seeInDatabase('users', [ 131 | 'id' => 1, 132 | 'first_name' => 'Abe', 133 | 'last_name' => 'Abeson', 134 | 'email' => 'abe.abeson@foo.com', 135 | 'age' => 50, 136 | ]); 137 | $this->seeInDatabase('users', [ 138 | 'id' => 3, 139 | 'first_name' => 'Charly', 140 | 'last_name' => 'Charlyson', 141 | 'email' => 'charly.charlyson@foo.com', 142 | 'age' => 52, 143 | ]); 144 | } 145 | 146 | public function testIgnoredColumnImport() 147 | { 148 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 149 | $seeder->table = 'users'; 150 | $seeder->filename = __DIR__.'/csvs/users_with_ignored_column.csv'; 151 | $seeder->hashable = ''; 152 | $seeder->run(); 153 | 154 | // Make sure the rows imported 155 | $this->seeInDatabase('users', [ 156 | 'id' => 1, 157 | 'first_name' => 'Abe', 158 | 'last_name' => 'Abeson', 159 | 'email' => 'abe.abeson@foo.com', 160 | 'age' => 50, 161 | ]); 162 | $this->seeInDatabase('users', [ 163 | 'id' => 3, 164 | 'first_name' => 'Charly', 165 | 'last_name' => 'Charlyson', 166 | 'email' => 'charly.charlyson@foo.com', 167 | 'age' => 52, 168 | ]); 169 | } 170 | 171 | public function testHash() 172 | { 173 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 174 | $seeder->table = 'users'; 175 | $seeder->filename = __DIR__.'/csvs/users.csv'; 176 | 177 | // Assert unhashed passwords 178 | $seeder->hashable = ''; 179 | $seeder->run(); 180 | $this->seeInDatabase('users', [ 181 | 'id' => 1, 182 | 'password' => 'abeabeson', 183 | ]); 184 | 185 | // Reset users table 186 | DB::table('users')->truncate(); 187 | 188 | // Assert hashed passwords 189 | $seeder->hashable = 'password'; 190 | $seeder->run(); 191 | // Row 1 should still be in DB... 192 | $this->seeInDatabase('users', [ 193 | 'id' => 1, 194 | ]); 195 | // ... But passwords were hashed 196 | $this->missingFromDatabase('users', [ 197 | 'id' => 1, 198 | 'password' => 'abeabeson', 199 | ]); 200 | } 201 | 202 | public function testOffset() 203 | { 204 | $seeder = new \Crockett\CsvSeeder\CsvSeeder; 205 | $seeder->table = 'users'; 206 | $seeder->filename = __DIR__.'/csvs/users.csv'; 207 | $seeder->hashable = ''; 208 | $seeder->offset_rows = 4; 209 | $seeder->mapping = [ 210 | 0 => 'id', 211 | 1 => 'first_name', 212 | 6 => 'age', 213 | ]; 214 | $seeder->run(); 215 | 216 | // Assert offset occurred 217 | $this->missingFromDatabase('users', [ 218 | 'id' => 1, 219 | ]); 220 | 221 | // Assert mapping worked 222 | $this->seeInDatabase('users', [ 223 | 'id' => 5, 224 | 'first_name' => 'Echo', 225 | 'last_name' => '', 226 | 'age' => 54 227 | ]); 228 | } 229 | } -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | To test, open *phpunit.xml* in your laravel installations root directory 4 | and add to the `testsuites` section: 5 | 6 | 7 | ./vendor/crockett/csv-seeder/tests/ 8 | 9 | 10 | Then run `phpunit` -------------------------------------------------------------------------------- /tests/csvs/users.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,password,address,age 2 | 1,Abe,Abeson,abe.abeson@foo.com,abeabeson,123 Abe street,50 3 | 2,Betty,Bettyson,betty.bettyson@foo.com,bettybettyson,123 Betty street,51 4 | 3,Charly,Charlyson,charly.charlyson@foo.com,charlycharlyson,123 Charly street,52 5 | 4,Delta,Deltason,delta.deltason@foo.com,deltadeltason,123 Delta street,53 6 | 5,Echo,Echoson,echo.echoson@foo.com,echoechoson,123 Echo street,54 -------------------------------------------------------------------------------- /tests/csvs/users_with_ignored_column.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,password,address,age,foo 2 | 1,Abe,Abeson,abe.abeson@foo.com,abeabeson,123 Abe street,50,abelony 3 | 2,Betty,Bettyson,betty.bettyson@foo.com,bettybettyson,123 Betty street,51,bettlelony 4 | 3,Charly,Charlyson,charly.charlyson@foo.com,charlycharlyson,123 Charly street,52,charlelony 5 | 4,Delta,Deltason,delta.deltason@foo.com,deltadeltason,123 Delta street,53,deltalony 6 | 5,Echo,Echoson,echo.echoson@foo.com,echoechoson,123 Echo street,54,echolony -------------------------------------------------------------------------------- /tests/migrations/2014_10_12_000000_create_users_table.php: -------------------------------------------------------------------------------- 1 | increments('id'); 17 | $table->string('first_name')->default(''); 18 | $table->string('last_name')->default(''); 19 | $table->string('email')->default(''); 20 | $table->string('password')->default(''); 21 | $table->string('address')->default(''); 22 | $table->integer('age')->default(0); 23 | }); 24 | } 25 | 26 | /** 27 | * Reverse the migrations. 28 | * 29 | * @return void 30 | */ 31 | public function down() 32 | { 33 | Schema::drop('users'); 34 | } 35 | } --------------------------------------------------------------------------------