├── .phpcs.xml
├── LICENSE.md
├── composer.json
├── phpstan.neon
├── readme.md
└── src
    ├── CsvSeeder.php
    └── CsvSeederServiceProvider.php


/.phpcs.xml:
--------------------------------------------------------------------------------
 1 | <?xml version="1.0"?>
 2 | <ruleset name="PSR12 Coding Standards">
 3 |     <description>A custom set of code standard rules</description>
 4 | 
 5 |     <rule ref="PSR12"/>
 6 | 
 7 |     <rule ref="PSR1.Methods.CamelCapsMethodName.NotCamelCaps">
 8 |         <!--
 9 |         Exceptions to the PSR-2 guidelines as per our Developer Guide:
10 |         https://octobercms.com/help/guidelines/developer#psr-exceptions
11 |         -->
12 |         <exclude-pattern>*/tests/*\.php</exclude-pattern>
13 |     </rule>
14 | 
15 |     <rule ref="PSR1.Classes.ClassDeclaration.MissingNamespace">
16 |         <!--
17 |         Migrations and tests do not need a namespace defined
18 |         -->
19 |         <exclude-pattern>*/tests/migrations/*\.php</exclude-pattern>
20 |     </rule>
21 | 
22 |     <file>src</file>
23 | 
24 |     <exclude-pattern>*/vendor/*</exclude-pattern>
25 |     <exclude-pattern>*/coverage/*</exclude-pattern>
26 | </ruleset>


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (C) 2013 Mior Muhammad Zaki <http://git.io/crynobone>
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of
 6 | this software and associated documentation files (the "Software"), to deal in
 7 | the Software without restriction, including without limitation the rights to
 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
 9 | the Software, and to permit persons to whom the Software is furnished to do so,
10 | subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "flynsarmy/csv-seeder",
 3 |     "description": "Allows seeding of the database with CSV files",
 4 |     "keywords": ["laravel", "csv", "seed", "seeds", "seeding"],
 5 |     "license": "MIT",
 6 |     "authors": [
 7 |         {
 8 |             "name": "Flyn San",
 9 |             "email": "flynsarmy@gmail.com"
10 |         }
11 |     ],
12 |     "require": {
13 |         "php": ">=7.4",
14 |         "illuminate/support": ">=4.1.0"
15 |     },
16 |     "autoload": {
17 |         "psr-4": {
18 |             "Flynsarmy\\CsvSeeder\\": "src/"
19 |         }
20 |     },
21 |     "autoload-dev": {
22 |         "psr-4": {
23 |             "Flynsarmy\\CsvSeeder\\Tests\\": "tests"
24 |         }
25 |     },
26 |     "extra": {
27 |         "laravel": {
28 |             "providers": [
29 |                 "Flynsarmy\\CsvSeeder\\CsvSeederServiceProvider"
30 |             ]
31 |         }
32 |     },
33 |     "minimum-stability": "dev",
34 |     "require-dev": {
35 |         "orchestra/testbench": "6.*",
36 |         "squizlabs/php_codesniffer": "3.*",
37 |         "nunomaduro/larastan": "^0.6.0@dev"
38 |     },
39 |     "scripts": {
40 |         "phpstan": "php -d memory_limit=-1 ./vendor/bin/phpstan analyse",
41 |         "phpcbf": "vendor/bin/phpcbf --standard=./.phpcs.xml ./",
42 |         "phpcs":  "vendor/bin/phpcs  -s --standard=./.phpcs.xml ./",
43 |         "phpunit": "vendor/bin/phpunit ./tests",
44 |         "coverage": "vendor/bin/phpunit tests --coverage-html coverage --whitelist src/",
45 |         "lint": "vendor/bin/parallel-lint --exclude vendor .",
46 |         "test": [
47 |             "composer validate --strict",
48 |             "@phpcs",
49 |             "@phpstan",
50 |             "@phpunit"
51 |         ]
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/phpstan.neon:
--------------------------------------------------------------------------------
1 | includes:
2 |     - ./vendor/nunomaduro/larastan/extension.neon
3 | parameters:
4 |     level: 5
5 |     inferPrivatePropertyTypeFromConstructor: true
6 |     paths:
7 |         - src


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
  1 | ## CSV Seeder
  2 | 
  3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/flynsarmy/csv-seeder.svg?style=flat-square)](https://packagist.org/packages/flynsarmy/csv-seeder)
  4 | [![Software License](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat-square)](LICENSE.md)
  5 | ![Build Status](https://github.com/Flynsarmy/laravel-csv-seeder/workflows/CI/badge.svg)
  6 | [![Quality Score](https://scrutinizer-ci.com/g/Flynsarmy/laravel-csv-seeder/badges/quality-score.png)](https://scrutinizer-ci.com/g/flynsarmy/laravel-csv-seeder)
  7 | [![Total Downloads](https://img.shields.io/packagist/dt/flynsarmy/csv-seeder?style=flat-square)](https://packagist.org/packages/flynsarmy/csv-seeder)
  8 |                     
  9 | 
 10 | ### Seed your database with CSV files
 11 | 
 12 | This package allows CSV based seeds.
 13 | 
 14 | 
 15 | ### Installation
 16 | 
 17 | Require this package in your composer.json and run composer update (or run `composer require flynsarmy/csv-seeder:2.*` directly):
 18 | 
 19 | **For PHP 7.4+**
 20 | 
 21 | ```json
 22 | "flynsarmy/csv-seeder": "2.0.*"
 23 | ```
 24 | 
 25 | **For older PHP versions**
 26 | 
 27 | ```json
 28 | "flynsarmy/csv-seeder": "1.*"
 29 | ```
 30 | 
 31 | ### Usage
 32 | 
 33 | Your CSV's header row should match the DB columns you wish to import. IE to import *id* and *name* columns, your CSV should look like:
 34 | 
 35 | ```csv
 36 | id,name
 37 | 1,Foo
 38 | 2,Bar
 39 | ```
 40 | 
 41 | Seed classes must extend `Flynsarmy\CsvSeeder\CsvSeeder`, they must define the destination database table and CSV file path, and finally they must call `parent::run()` like so:
 42 | 
 43 | ```php
 44 | use Flynsarmy\CsvSeeder\CsvSeeder;
 45 | 
 46 | class StopsTableSeeder extends CsvSeeder {
 47 | 
 48 | 	public function __construct()
 49 | 	{
 50 | 		$this->table = 'your_table';
 51 | 		$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
 52 | 	}
 53 | 
 54 | 	public function run()
 55 | 	{
 56 | 		// Recommended when importing larger CSVs
 57 | 		DB::disableQueryLog();
 58 | 
 59 | 		// Uncomment the below to wipe the table clean before populating
 60 | 		DB::table($this->table)->truncate();
 61 | 
 62 | 		parent::run();
 63 | 	}
 64 | }
 65 | ```
 66 | 
 67 | Drop your CSV into */database/seeds/csvs/your_csv.csv* or whatever path you specify in your constructor above.
 68 | 
 69 | ### Configuration
 70 | 
 71 | In addition to setting the database table and CSV filename, the following configuration options are available. They can be set in your class constructor:
 72 | 
 73 |  - `connection` (string '') Connection to use for inserts. Leave empty for default connection.
 74 |  - `insert_chunk_size` (int 500) An SQL insert statement will trigger every `insert_chunk_size` number of rows while reading the CSV
 75 |  - `csv_delimiter` (string ,) The CSV field delimiter.
 76 |  - `hashable` (array [password]) List of fields to be hashed before import, useful if you are importing users and need their passwords hashed. Uses `Hash::make()`. Note: This is EXTREMELY SLOW. If you have a lot of rows in your CSV your import will take quite a long time.
 77 |  - `offset_rows` (int 0) How many rows at the start of the CSV to ignore. Warning: If used, you probably want to set a mapping as your header row in the CSV will be skipped.
 78 |  - `mapping` (array []) Associative array of csvCol => dbCol. See examples section for details. If not specified, the first row (after offset) of the CSV will be used as the mapping.
 79 |  - `should_trim` (bool false) Whether to trim the data in each cell of the CSV during import.
 80 |  - `timestamps` (bool false) Whether or not to add *created_at* and *updated_at* columns on import.
 81 |    - `created_at` (string current time in ISO 8601 format) Only used if `timestamps` is `true`
 82 |    - `updated_at` (string current time in ISO 8601 format) Only used if `timestamps` is `true`
 83 | 
 84 | 
 85 | ### Examples 
 86 | CSV with pipe delimited values:
 87 | 
 88 | ```php
 89 | public function __construct()
 90 | {
 91 | 	$this->table = 'users';
 92 | 	$this->csv_delimiter = '|';
 93 | 	$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
 94 | }
 95 | ```
 96 | 
 97 | Specifying which CSV columns to import:
 98 | 
 99 | ```php
100 | public function __construct()
101 | {
102 | 	$this->table = 'users';
103 | 	$this->csv_delimiter = '|';
104 | 	$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
105 | 	$this->mapping = [
106 | 	    0 => 'first_name',
107 | 	    1 => 'last_name',
108 | 	    5 => 'age',
109 | 	];
110 | }
111 | ```
112 | 
113 | Trimming the whitespace from the imported data:
114 | 
115 | ```php
116 | public function __construct()
117 | {
118 | 	$this->table = 'users';
119 | 	$this->csv_delimiter = '|';
120 | 	$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
121 | 	$this->mapping = [
122 | 	    0 => 'first_name',
123 | 	    1 => 'last_name',
124 | 	    5 => 'age',
125 | 	];
126 | 	$this->should_trim = true;
127 | }
128 | ```
129 | 
130 | Skipping the CSV header row (Note: A mapping is required if this is done):
131 | 
132 | ```php
133 | public function __construct()
134 | {
135 | 	$this->table = 'users';
136 | 	$this->csv_delimiter = '|';
137 | 	$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
138 | 	$this->offset_rows = 1;
139 | 	$this->mapping = [
140 | 	    0 => 'first_name',
141 | 	    1 => 'last_name',
142 | 	    2 => 'password',
143 | 	];
144 | 	$this->should_trim = true;
145 | }
146 | ```
147 | 
148 | Specifying the DB connection to use:
149 | 
150 | ```php
151 | public function __construct()
152 | {
153 | 	$this->table = 'users';
154 | 	$this->connection = 'my_connection';
155 | 	$this->filename = base_path().'/database/seeds/csvs/your_csv.csv';
156 | }
157 | ```
158 | 
159 | ### Migration Guide
160 | 
161 | #### 2.0
162 | 
163 | - `$seeder->hashable` is now an `array` of columns rather than a single column name. Wrap your old string value in `[]`.
164 | 
165 | ### License
166 | 
167 | CsvSeeder is open-sourced software licensed under the [MIT license](http://opensource.org/licenses/MIT)
168 | 


--------------------------------------------------------------------------------
/src/CsvSeeder.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | 
  3 | namespace Flynsarmy\CsvSeeder;
  4 | 
  5 | use App;
  6 | use DB;
  7 | use Exception;
  8 | use Log;
  9 | use Carbon\Carbon;
 10 | use Illuminate\Database\Seeder;
 11 | use Illuminate\Database\Schema;
 12 | use Illuminate\Support\Facades\Hash;
 13 | 
 14 | /**
 15 |  * Taken from http://laravelsnippets.com/snippets/seeding-database-with-csv-files-cleanly
 16 |  * and modified to include insert chunking
 17 |  */
 18 | class CsvSeeder extends Seeder
 19 | {
 20 |     /**
 21 |      * DB table name
 22 |      *
 23 |      * @var string
 24 |      */
 25 |     public string $table = '';
 26 | 
 27 |     /**
 28 |      * CSV filename
 29 |      *
 30 |      * @var string
 31 |      */
 32 |     public string $filename = '';
 33 | 
 34 |     /**
 35 |      * DB connection to use. Leave empty for default connection
 36 |      */
 37 |     public string $connection = '';
 38 | 
 39 |     /**
 40 |      * DB fields to be hashed before import, For example a password field.
 41 |      */
 42 |     public array $hashable = ['password'];
 43 | 
 44 |     /**
 45 |      * An SQL INSERT query will execute every time this number of rows
 46 |      * are read from the CSV. Without this, large INSERTS will silently
 47 |      * fail.
 48 |      */
 49 |     public int $insert_chunk_size = 50;
 50 | 
 51 |     /**
 52 |      * CSV delimiter (defaults to ,)
 53 |      */
 54 |     public string $csv_delimiter = ',';
 55 | 
 56 |     /**
 57 |      * Number of rows to skip at the start of the CSV
 58 |      */
 59 |     public int $offset_rows = 0;
 60 | 
 61 |     /**
 62 |      * Can be used to tell the import to trim any leading or trailing white space from the column;
 63 |      */
 64 |     public bool $should_trim = false;
 65 | 
 66 |     /**
 67 |      * Add created_at and updated_at to rows
 68 |      */
 69 |     public bool $timestamps = false;
 70 |     /**
 71 |      * created_at and updated_at values to be added to each row. Only used if
 72 |      * $this->timestamps is true
 73 |      */
 74 |     public string $created_at = '';
 75 |     public string $updated_at = '';
 76 | 
 77 |     /**
 78 |      * The mapping of CSV to DB column. If not specified manually, the first
 79 |      * row (after offset_rows) of your CSV will be read as your DB columns.
 80 |      *
 81 |      * Mappings take the form of csvColNumber => dbColName.
 82 |      *
 83 |      * IE to read the first, third and fourth columns of your CSV only, use:
 84 |      * array(
 85 |      *   0 => id,
 86 |      *   2 => name,
 87 |      *   3 => description,
 88 |      * )
 89 |      */
 90 |     public array $mapping = [];
 91 | 
 92 | 
 93 |     /**
 94 |      * Run DB seed
 95 |      */
 96 |     public function run()
 97 |     {
 98 |         // Cache created_at and updated_at if we need to
 99 |         if ($this->timestamps) {
100 |             if (!$this->created_at) {
101 |                 $this->created_at = Carbon::now()->toIso8601String();
102 |             }
103 |             if (!$this->updated_at) {
104 |                 $this->updated_at = Carbon::now()->toIso8601String();
105 |             }
106 |         }
107 | 
108 |         $this->seedFromCSV($this->filename, $this->csv_delimiter);
109 |     }
110 | 
111 |     /**
112 |      * Strip UTF-8 BOM characters from the start of a string
113 |      *
114 |      * @param  string $text
115 |      * @return string       String with BOM stripped
116 |      */
117 |     public function stripUtf8Bom(string $text): string
118 |     {
119 |         $bom = pack('H*', 'EFBBBF');
120 |         $text = preg_replace("/^$bom/", '', $text);
121 | 
122 |         return $text;
123 |     }
124 | 
125 |     /**
126 |      * Opens a CSV file and returns it as a resource
127 |      *
128 |      * @param string $filename
129 |      * @return FALSE|resource
130 |      */
131 |     public function openCSV(string $filename)
132 |     {
133 |         if (!file_exists($filename) || !is_readable($filename)) {
134 |             Log::error("CSV insert failed: CSV " . $filename . " does not exist or is not readable.");
135 |             return false;
136 |         }
137 | 
138 |         // check if file is gzipped
139 |         $finfo = finfo_open(FILEINFO_MIME_TYPE);
140 |         $file_mime_type = finfo_file($finfo, $filename);
141 |         finfo_close($finfo);
142 |         $gzipped = strcmp($file_mime_type, "application/x-gzip") == 0;
143 | 
144 |         $handle = $gzipped ? gzopen($filename, 'r') : fopen($filename, 'r');
145 | 
146 |         return $handle;
147 |     }
148 | 
149 |     /**
150 |      * Reads all rows of a given CSV and imports the data.
151 |      *
152 |      * @param string $filename
153 |      * @param string $deliminator
154 |      * @return bool  Whether or not the import completed successfully.
155 |      * @throws Exception
156 |      */
157 |     public function seedFromCSV(string $filename, string $deliminator = ","): bool
158 |     {
159 |         $handle = $this->openCSV($filename);
160 | 
161 |         // CSV doesn't exist or couldn't be read from.
162 |         if ($handle === false) {
163 |             throw new Exception("CSV insert failed: CSV " . $filename . " does not exist or is not readable.");
164 |         }
165 | 
166 |         $success = true;
167 |         $row_count = 0;
168 |         $data = [];
169 |         $mapping = $this->mapping ?: [];
170 |         $offset = $this->offset_rows;
171 | 
172 |         if ($mapping) {
173 |             $this->hashable = $this->removeUnusedHashColumns($mapping);
174 |         }
175 | 
176 |         while (($row = fgetcsv($handle, 0, $deliminator)) !== false) {
177 |             // Offset the specified number of rows
178 | 
179 |             while ($offset-- > 0) {
180 |                 continue 2;
181 |             }
182 | 
183 |             // No mapping specified - the first row will be used as the mapping
184 |             // ie it's a CSV title row. This row won't be inserted into the DB.
185 |             if (!$mapping) {
186 |                 $mapping = $this->createMappingFromRow($row);
187 |                 $this->hashable = $this->removeUnusedHashColumns($mapping);
188 |                 continue;
189 |             }
190 | 
191 |             $row = $this->readRow($row, $mapping);
192 | 
193 |             // insert only non-empty rows from the csv file
194 |             if (empty($row)) {
195 |                 continue;
196 |             }
197 | 
198 |             $data[$row_count] = $row;
199 | 
200 |             // Chunk size reached, insert
201 |             if (++$row_count == $this->insert_chunk_size) {
202 |                 $success = $success && $this->insert($data);
203 |                 $row_count = 0;
204 |                 // clear the data array explicitly when it was inserted so
205 |                 // that nothing is left, otherwise a leftover scenario can
206 |                 // cause duplicate inserts
207 |                 $data = [];
208 |             }
209 |         }
210 | 
211 |         // Insert any leftover rows
212 |         //check if the data array explicitly if there are any values left to be inserted, if insert them
213 |         if (count($data)) {
214 |             $success = $success && $this->insert($data);
215 |         }
216 | 
217 |         fclose($handle);
218 | 
219 |         return $success;
220 |     }
221 | 
222 |     /**
223 |      * Creates a CSV->DB column mapping from the given CSV row.
224 |      *
225 |      * @param array $row  List of DB columns to insert into
226 |      * @return array  List of DB fields to insert into
227 |      */
228 |     public function createMappingFromRow(array $row): array
229 |     {
230 |         $mapping = $row;
231 |         $mapping[0] = $this->stripUtf8Bom($mapping[0]);
232 | 
233 |         // skip csv columns that don't exist in the database
234 |         foreach ($mapping as $index => $fieldname) {
235 |             if (!DB::connection($this->connection)->getSchemaBuilder()->hasColumn($this->table, $fieldname)) {
236 |                 if (isset($mapping[$index])) {
237 |                     unset($mapping[$index]);
238 |                 }
239 |             }
240 |         }
241 | 
242 |         return $mapping;
243 |     }
244 | 
245 |     /**
246 |      * Removes fields from the hashable array that don't exist in our mapping.
247 |      *
248 |      * This function acts as a performance enhancement - we don't want
249 |      * to search for hashable columns on every row imported when we already
250 |      * know they don't exist.
251 |      *
252 |      * @param array $mapping
253 |      * @return array
254 |      */
255 |     public function removeUnusedHashColumns(array $mapping)
256 |     {
257 |         $hashables = $this->hashable;
258 | 
259 |         foreach ($hashables as $key => $field) {
260 |             if (!in_array($field, $mapping)) {
261 |                 unset($hashables[$key]);
262 |             }
263 |         }
264 | 
265 |         return $hashables;
266 |     }
267 | 
268 |     /**
269 |      * Read a CSV row into a DB insertable array
270 |      *
271 |      * @param array $row        A row of data to read
272 |      * @param array $mapping    Array of csvCol => dbCol
273 |      * @return array
274 |      */
275 |     public function readRow(array $row, array $mapping): array
276 |     {
277 |         $row_values = [];
278 | 
279 |         foreach ($mapping as $csvCol => $dbCol) {
280 |             if (!isset($row[$csvCol]) || $row[$csvCol] === '') {
281 |                 $row_values[$dbCol] = null;
282 |             } else {
283 |                 $row_values[$dbCol] = $this->should_trim ? trim($row[$csvCol]) : $row[$csvCol];
284 |             }
285 |         }
286 | 
287 |         if (!empty($this->hashable)) {
288 |             foreach ($this->hashable as $columnToHash) {
289 |                 if (isset($row_values[$columnToHash])) {
290 |                     $row_values[$columnToHash] = Hash::make($row_values[$columnToHash]);
291 |                 }
292 |             }
293 |         }
294 | 
295 |         if ($this->timestamps) {
296 |             $row_values['created_at'] = $this->created_at;
297 |             $row_values['updated_at'] = $this->updated_at;
298 |         }
299 | 
300 |         return $row_values;
301 |     }
302 | 
303 |     /**
304 |      * Seed a given set of data to the DB
305 |      *
306 |      * @param array $seedData
307 |      * @return bool   TRUE on success else FALSE
308 |      */
309 |     public function insert(array $seedData): bool
310 |     {
311 |         try {
312 |             DB::connection($this->connection)->table($this->table)->insert($seedData);
313 |         } catch (\Exception $e) {
314 |             Log::error("CSV insert failed: " . $e->getMessage() . " - CSV " . $this->filename);
315 |             return false;
316 |         }
317 | 
318 |         return true;
319 |     }
320 | }
321 | 


--------------------------------------------------------------------------------
/src/CsvSeederServiceProvider.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 | namespace Flynsarmy\CsvSeeder;
 4 | 
 5 | use Illuminate\Support\ServiceProvider;
 6 | 
 7 | class CsvSeederServiceProvider extends ServiceProvider
 8 | {
 9 | 
10 |     /**
11 |      * Indicates if loading of the provider is deferred.
12 |      *
13 |      * @var bool
14 |      */
15 |     protected $defer = false;
16 | 
17 |     /**
18 |      * Bootstrap the application events.
19 |      *
20 |      * @return void
21 |      */
22 |     public function boot()
23 |     {
24 |         //
25 |     }
26 | 
27 |     /**
28 |      * Register the service provider.
29 |      *
30 |      * @return void
31 |      */
32 |     public function register()
33 |     {
34 |         //
35 |     }
36 | 
37 |     /**
38 |      * Get the services provided by the provider.
39 |      *
40 |      * @return array
41 |      */
42 |     public function provides()
43 |     {
44 |         return [];
45 |     }
46 | }
47 | 


--------------------------------------------------------------------------------