├── .gitignore ├── phpstan.neon.dist ├── src ├── Casts │ ├── EmptyValuesToNull.php │ └── NumericValues.php ├── SimpleCsv.php ├── SimpleCsvServiceProvider.php └── SimpleCsvService.php ├── split-csv.sh ├── tests ├── LazyGenerator.php ├── TestCase.php └── Unit │ ├── ProviderTest.php │ └── DefaultTest.php ├── .github └── workflows │ └── ci.yaml ├── LICENSE ├── phpunit.xml ├── composer.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /.phpunit.result.cache 3 | /composer.lock 4 | /clover.xml 5 | /build 6 | /vendor 7 | .phpunit.cache 8 | -------------------------------------------------------------------------------- /phpstan.neon.dist: -------------------------------------------------------------------------------- 1 | includes: 2 | - vendor/larastan/larastan/extension.neon 3 | parameters: 4 | level: 5 5 | paths: 6 | - src -------------------------------------------------------------------------------- /src/Casts/EmptyValuesToNull.php: -------------------------------------------------------------------------------- 1 | $value){ 10 | if(empty($value)){ 11 | $item[$key] = null; 12 | } 13 | } 14 | return $item; 15 | } 16 | } -------------------------------------------------------------------------------- /split-csv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | _PATH=$1 3 | _CHUNKS=$2 4 | _INDEX=1 5 | _HEADER=$(head -1 "${_PATH}") 6 | _NAME=$(basename "${_PATH}" .csv) 7 | _DIR=$(dirname -- "${_PATH}") 8 | 9 | split -l "${_CHUNKS}" "${_PATH}" _CHUNKS 10 | 11 | for _CHUNK in _CHUNKS* 12 | do 13 | if (( ${_INDEX} > 1 )); then 14 | echo ${_HEADER} > ${_DIR}/${_NAME}-chunk-${_INDEX}.csv 15 | fi 16 | cat ${_CHUNK} >> ${_DIR}/${_NAME}-chunk-${_INDEX}.csv 17 | rm ${_CHUNK} 18 | ((_INDEX++)) 19 | done -------------------------------------------------------------------------------- /src/Casts/NumericValues.php: -------------------------------------------------------------------------------- 1 | $value){ 10 | if(is_numeric($value)){ 11 | if(str_contains($value, '.')){ 12 | $item[$key] = floatval($value); 13 | continue; 14 | } 15 | $item[$key] = intval($value); 16 | } 17 | } 18 | return $item; 19 | } 20 | } -------------------------------------------------------------------------------- /tests/LazyGenerator.php: -------------------------------------------------------------------------------- 1 | SimpleCsv::class, 29 | ]; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/SimpleCsvServiceProvider.php: -------------------------------------------------------------------------------- 1 | app->bind('simple-csv', function () { 17 | return new SimpleCsvService( 18 | config('simple-csv.delimiter', SimpleCsvService::DELIMITER), 19 | config('simple-csv.enclosure', SimpleCsvService::ENCLOSURE), 20 | config('simple-csv.escape', SimpleCsvService::ESCAPE) 21 | ); 22 | }); 23 | } 24 | 25 | /** 26 | * Bootstrap any application services. 27 | * @return void 28 | */ 29 | public function boot() 30 | { 31 | 32 | } 33 | 34 | /** 35 | * The services provided. 36 | * 37 | * @return array 38 | */ 39 | public function provides() 40 | { 41 | return ['simple-csv']; 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/ci.yaml: -------------------------------------------------------------------------------- 1 | name: ci 2 | env: 3 | XDEBUG_MODE: 'coverage' 4 | on: 5 | push: 6 | branches: 7 | - master 8 | - dev 9 | paths-ignore: 10 | - 'README.md' 11 | - 'LICENSE' 12 | pull_request: 13 | branches: 14 | - master 15 | paths-ignore: 16 | - 'README.md' 17 | - 'LICENSE' 18 | jobs: 19 | build: 20 | runs-on: ubuntu-latest 21 | steps: 22 | - uses: actions/checkout@v1 23 | with: 24 | fetch-depth: 1 25 | - name: Cache Composer 26 | uses: actions/cache@v4 27 | with: 28 | path: vendor 29 | key: ${{ runner.OS }}-build-${{ hashFiles('**/composer.lock') }} 30 | - name: Composer Dependencies 31 | run: composer install --no-ansi --no-interaction --no-scripts --no-suggest --no-progress --prefer-dist 32 | - name: Lint 33 | run: composer lint 34 | - name: Unit Tests 35 | run: composer test 36 | - name: Codecov 37 | uses: codecov/codecov-action@v1.0.5 38 | with: 39 | token: ${{ secrets.CODECOV_TOKEN }} 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 Dan Alvidrez 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | ./tests/Unit 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | src/ 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "bayareawebpro/laravel-simple-csv", 3 | "description": "A simple CSV importer/ exporter for Laravel.", 4 | "license": "MIT", 5 | "authors": [ 6 | { 7 | "name": "Daniel Alvidrez", 8 | "email": "dan@bayareawebpro.com" 9 | } 10 | ], 11 | "require": { 12 | "php": "^7.2|^8.0", 13 | "illuminate/support": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0|^12.0" 14 | }, 15 | "require-dev": { 16 | "orchestra/testbench": "^4.0|^5.0|^6.0|^7.0|^8.0|^9.0|^10.0", 17 | "phpunit/phpunit": "^8.0|^9.0|^10.0|^11.0", 18 | "larastan/larastan": "^1.0|^2.0" 19 | }, 20 | "autoload": { 21 | "psr-4": { 22 | "BayAreaWebPro\\SimpleCsv\\": "src/" 23 | } 24 | }, 25 | "autoload-dev": { 26 | "psr-4": { 27 | "BayAreaWebPro\\SimpleCsv\\Tests\\": "tests" 28 | } 29 | }, 30 | "scripts": { 31 | "test": "XDEBUG_MODE=coverage vendor/bin/phpunit", 32 | "lint": "vendor/bin/phpstan analyse" 33 | }, 34 | "extra": { 35 | "laravel": { 36 | "providers": [ 37 | "BayAreaWebPro\\SimpleCsv\\SimpleCsvServiceProvider" 38 | ], 39 | "aliases": { 40 | "SimpleCsv": "BayAreaWebPro\\SimpleCsv\\SimpleCsvFacade" 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/Unit/ProviderTest.php: -------------------------------------------------------------------------------- 1 | assertInstanceOf(SimpleCsvServiceProvider::class, $this->app->getProvider(SimpleCsvServiceProvider::class), 'Provider is registered with container.'); 14 | } 15 | 16 | public function test_container_can_resolve_instance() 17 | { 18 | $this->assertInstanceOf(SimpleCsvService::class, $this->app->make('simple-csv'), 'Container can make instance of service.'); 19 | } 20 | 21 | public function test_facade_can_resolve_instance() 22 | { 23 | $this->assertInstanceOf(SimpleCsvService::class, \SimpleCsv::getFacadeRoot(), 'Facade can make instance of service.'); 24 | } 25 | 26 | public function test_service_can_be_resolved() 27 | { 28 | $csv = app('simple-csv'); 29 | $this->assertInstanceOf(SimpleCsvService::class, $csv); 30 | } 31 | 32 | public function test_declares_provided() 33 | { 34 | $this->assertTrue(in_array('simple-csv', 35 | collect(app()->getProviders(SimpleCsvServiceProvider::class))->first()->provides()) 36 | ); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/SimpleCsvService.php: -------------------------------------------------------------------------------- 1 | delimiter = $delimiter; 29 | $this->enclosure = $enclosure; 30 | $this->escape = $escape; 31 | $this->headers = null; 32 | $this->file = null; 33 | } 34 | 35 | public function import(string $path, array $casts = []): LazyCollection 36 | { 37 | $this->openFileObject($path); 38 | $this->headers = array_values($this->getLine()); 39 | 40 | $instance = LazyCollection::make(function () { 41 | while ($this->file->valid() && $line = $this->getLine()) { 42 | if (!$this->isInValidLine($line)) { 43 | yield array_combine($this->headers, $line); 44 | } 45 | } 46 | $this->resetState(); 47 | }); 48 | 49 | if(count($casts)){ 50 | foreach($casts as $caster){ 51 | $instance = $instance->map(App::make($caster)); 52 | } 53 | } 54 | 55 | return $instance; 56 | } 57 | 58 | protected function resetState(): void 59 | { 60 | $this->headers = null; 61 | $this->file = null; 62 | } 63 | 64 | protected function isInValidLine(array $line): bool 65 | { 66 | return count($line) === 1 && is_null($line[0]); 67 | } 68 | 69 | public function export($collection, string $path): self 70 | { 71 | if (!file_exists($path)) touch($path); 72 | $this->openFileObject($path, 'w'); 73 | $this->writeLines($collection); 74 | $this->resetState(); 75 | return $this; 76 | } 77 | 78 | public function download($collection, string $filename, $headers = []): StreamedResponse 79 | { 80 | return response()->streamDownload(function () use ($collection) { 81 | $this->openFileObject('php://output', 'w'); 82 | $this->writeLines($collection); 83 | $this->resetState(); 84 | }, $filename, array_merge([ 85 | 'Content-Type' => 'text/csv', 86 | ], $headers)); 87 | } 88 | 89 | protected function getLine(): array 90 | { 91 | return $this->file->fgetcsv($this->delimiter, $this->enclosure, $this->escape); 92 | } 93 | 94 | protected function writeLine(array $line): void 95 | { 96 | $this->file->fputcsv($line, $this->delimiter, $this->enclosure, $this->escape); 97 | } 98 | 99 | protected function flattenRow($entry): array 100 | { 101 | return is_object($entry) && method_exists($entry, 'toArray') ? $entry->toArray() : (array)$entry; 102 | } 103 | 104 | protected function openFileObject(string $path, string $mode = 'r'): void 105 | { 106 | $this->file = new \SplFileObject($path, $mode); 107 | } 108 | 109 | protected function writeLines($collection): void 110 | { 111 | if ( 112 | !$collection instanceof Iterator && 113 | !$collection instanceof Collection && 114 | !$collection instanceof LazyCollection && 115 | !is_array($collection) 116 | ) { 117 | throw new Exception("Non-Iterable Object cannot be iterated."); 118 | } 119 | foreach ($collection as $entry) { 120 | if (!$this->headers) { 121 | $this->headers = array_keys($this->flattenRow($entry)); 122 | $this->writeLine($this->headers); 123 | } 124 | $this->writeLine(array_values($this->flattenRow($entry))); 125 | } 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /tests/Unit/DefaultTest.php: -------------------------------------------------------------------------------- 1 | (string)$faker->uuid, 27 | 'name' => (string)$faker->name, 28 | 'email' => (string)$faker->email, 29 | 'empty' => null, 30 | 'float' => 3.14, 31 | 'int' => 256, 32 | ]; 33 | }); 34 | } 35 | 36 | private function getRandomStoragePath() 37 | { 38 | File::cleanDirectory(storage_path()); 39 | return storage_path(Str::random(16) . '.csv'); 40 | } 41 | 42 | public function test_import_casts() 43 | { 44 | $items = $this->getCollectionData(5)->toArray(); 45 | 46 | $path = $this->getRandomStoragePath(); 47 | 48 | SimpleCsv::export($items, $path); 49 | 50 | $this->assertFileExists($path); 51 | 52 | $items = SimpleCsv::import($path, [ 53 | EmptyValuesToNull::class, 54 | NumericValues::class 55 | ]); 56 | 57 | foreach($items as $item){ 58 | $this->assertIsFloat($item['float']); 59 | $this->assertNull($item['empty']); 60 | $this->assertIsInt($item['int']); 61 | } 62 | } 63 | 64 | public function test_export_from_iterables() 65 | { 66 | $items = $this->getCollectionData(10)->toArray(); 67 | 68 | // Array 69 | $pathA = $this->getRandomStoragePath(); 70 | SimpleCsv::export($items, $pathA); 71 | 72 | $this->assertFileExists($pathA); 73 | foreach ($items as $item) { 74 | $this->assertStringContainsString($item['uuid'], File::get($pathA)); 75 | } 76 | 77 | // Collection 78 | $pathB = $this->getRandomStoragePath(); 79 | SimpleCsv::export(Collection::make($items), $pathB); 80 | 81 | $this->assertFileExists($pathB); 82 | foreach ($items as $item) { 83 | $this->assertStringContainsString($item['uuid'], File::get($pathB)); 84 | } 85 | } 86 | 87 | public function test_export_files_and_restore() 88 | { 89 | $items = $this->getCollectionData()->toArray(); 90 | 91 | $collectionLazy = LazyCollection::make($items); 92 | 93 | $path = $this->getRandomStoragePath(); 94 | 95 | SimpleCsv::export($collectionLazy, $path); 96 | 97 | $this->assertFileExists($path); 98 | 99 | $fileData = File::get($path); 100 | foreach ($items as $item) { 101 | $this->assertStringContainsString($item['email'], $fileData); 102 | } 103 | 104 | $decoded = SimpleCsv::import($path); 105 | $this->assertInstanceOf(LazyCollection::class, $decoded); 106 | 107 | foreach ($decoded as $decodedItem) { 108 | $this->assertStringContainsString($decodedItem['email'], $fileData); 109 | } 110 | } 111 | 112 | public function test_can_download_streams() 113 | { 114 | $items = $this->getCollectionData()->toArray(); 115 | 116 | $collectionLazy = LazyCollection::make($items); 117 | 118 | $response = SimpleCsv::download($collectionLazy, 'download.csv'); 119 | 120 | $this->assertInstanceOf(StreamedResponse::class, $response); 121 | 122 | ob_start(); 123 | $response->sendContent(); 124 | $data = (string)ob_get_clean(); 125 | 126 | $this->assertNotEmpty($data); 127 | 128 | foreach ($items as $item) { 129 | $this->assertStringContainsString($item['email'], $data); 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Simple CSV 2 | 3 | ![](https://github.com/bayareawebpro/laravel-simple-csv/workflows/ci/badge.svg) 4 | ![](https://codecov.io/gh/bayareawebpro/laravel-simple-csv/branch/master/graph/badge.svg) 5 | ![](https://img.shields.io/github/v/release/bayareawebpro/laravel-simple-csv.svg) 6 | ![](https://img.shields.io/packagist/dt/bayareawebpro/laravel-simple-csv.svg) 7 | ![](https://img.shields.io/badge/License-MIT-success.svg) 8 | 9 | > https://packagist.org/packages/bayareawebpro/laravel-simple-csv 10 | 11 | ## Features 12 | - Import to LazyCollection. 13 | - Export from Collection, LazyCollection, Iterable, Generator, Array. 14 | - Low(er) Memory Consumption by use of LazyCollection Generators. 15 | - Uses Native PHP SplFileObject. 16 | - Facade Included. 17 | 18 | ## Installation 19 | Require the package and Laravel will Auto-Discover the Service Provider. 20 | 21 | ``` 22 | composer require bayareawebpro/laravel-simple-csv 23 | ``` 24 | 25 | ## Usage: 26 | 27 | Invokable classes can be passed to the import method allowing you to customize 28 | how each row is processed. Two classes to handle numerics 29 | and null values have been supplied. 30 | 31 | ```php 32 | use BayAreaWebPro\SimpleCsv\SimpleCsv; 33 | use BayAreaWebPro\SimpleCsv\Casts\EmptyValuesToNull; 34 | use BayAreaWebPro\SimpleCsv\Casts\NumericValues; 35 | 36 | $lazyCsvCollection = SimpleCsv::import(storage_path('collection.csv'), [ 37 | EmptyValuesToNull::class, 38 | NumericValues::class, 39 | ]); 40 | ``` 41 | 42 | ### Invokable Classes 43 | 44 | **Dependency Injection:** Invokable classes can typehint required dependencies in a 45 | constructor method when defined. 46 | 47 | ```php 48 | $value){ 60 | if(in_array($key, ['created_at', 'updated_at'])){ 61 | $item[$key] = Carbon::parse($value); 62 | } 63 | } 64 | return $item; 65 | } 66 | } 67 | ``` 68 | 69 | ### Export to File 70 | ```php 71 | use BayAreaWebPro\SimpleCsv\SimpleCsv; 72 | 73 | // Collection 74 | SimpleCsv::export( 75 | Collection::make(...), 76 | storage_path('collection.csv') 77 | ); 78 | 79 | // LazyCollection 80 | SimpleCsv::export( 81 | LazyCollection::make(...), 82 | storage_path('collection.csv') 83 | ); 84 | 85 | // Generator (Cursor) 86 | SimpleCsv::export( 87 | User::query()->where(...)->limit(500)->cursor(), 88 | storage_path('collection.csv') 89 | ); 90 | 91 | // Array 92 | SimpleCsv::export( 93 | [...], 94 | storage_path('collection.csv') 95 | ); 96 | ``` 97 | 98 | ### Export Download Stream 99 | 100 | ```php 101 | use BayAreaWebPro\SimpleCsv\SimpleCsv; 102 | 103 | return SimpleCsv::download([...], 'download.csv'); 104 | ``` 105 | 106 | #### Override Options 107 | ```php 108 | use Illuminate\Support\Facades\Config; 109 | 110 | Config::set('simple-csv.delimiter', ...); 111 | Config::set('simple-csv.enclosure', ...); 112 | Config::set('simple-csv.escape', ...); 113 | ``` 114 | 115 | ## Or, Create a Config File 116 | 117 | `config/simple-csv.php` 118 | 119 | ```php 120 | return [ 121 | 'delimiter' => '?', 122 | 'enclosure' => '?', 123 | 'escape' => '?', 124 | ]; 125 | ``` 126 | 127 | ## File Splitting Utility 128 | A file splitting utility has been included that will break large CSV files into chunks 129 | (while retaining column headers) which you can move/delete after importing. 130 | This can help with automating the import of large data sets. 131 | 132 | Tip: Find your Bash Shell Binary Path: `which sh` 133 | 134 | ``` 135 | /bin/sh vendor/bayareawebpro/laravel-simple-csv/split-csv.sh /Projects/laravel/storage/big-file.csv 5000 136 | 137 | File Output: 138 | /Projects/laravel/storage/big-file-chunk-1.csv (chunk of 5000) 139 | /Projects/laravel/storage/big-file-chunk-2.csv (chunk of 5000) 140 | /Projects/laravel/storage/big-file-chunk-3.csv (chunk of 5000) 141 | etc... 142 | ``` 143 | 144 | ## Speed Tips 145 | - Using Lazy Collections is the preferred method. 146 | - Using the queue worker, you can import a several thousand rows at a time without much impact. 147 | - Be sure to use "Database Transactions" and "Timeout Detection" to insure safe imports. 148 | - [Article: How to Insert & Update Many at Once](https://medium.com/@danielalvidrez/laravel-query-builder-macros-fe176d34135e) --------------------------------------------------------------------------------