├── .styleci.yml ├── .gitignore ├── CHANGELOG.md ├── tests ├── resources │ ├── without-header.csv │ └── with-header.csv └── CollectionTest.php ├── .phpunit.xml ├── composer.json ├── LICENSE.md ├── CONTRIBUTING.md ├── README.md └── src └── CsvCollection.php /.styleci.yml: -------------------------------------------------------------------------------- 1 | preset: laravel 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | .vscode 3 | build 4 | composer.lock 5 | docs 6 | vendor 7 | coverage 8 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## 0.1.0 4 | 5 | ### Added 6 | - Initial version 7 | 8 | -------------------------------------------------------------------------------- /tests/resources/without-header.csv: -------------------------------------------------------------------------------- 1 | 1,Wait,Blampy,wblampy0@addtoany.com,Male,93.96.91.96 2 | 2,Pattie,Brute,pbrute1@loc.gov,Male,39.85.71.104 3 | 3,Joanie,Petrollo,jpetrollo2@comsenz.com,Female,157.92.43.130 4 | 4,Mufinella,Colwell,mcolwell3@foxnews.com,Female,156.25.135.88 5 | 5,Eirena,Siemens,esiemens4@cargocollective.com,Female,148.45.47.113 6 | 6,Dolly,Butters,dbutters5@wordpress.org,Female,180.210.94.47 7 | 7,Margery,MacCaig,mmaccaig6@netlog.com,Female,116.245.1.205 8 | 8,Garv,Schleicher,gschleicher7@baidu.com,Male,188.84.122.179 9 | 9,Kennedy,Crowther,kcrowther8@unicef.org,Male,205.169.102.225 10 | 10,Wendi,Hairon,whairon9@dyndns.org,Female,65.207.28.53 11 | -------------------------------------------------------------------------------- /.phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 10 | 11 | 12 | ./tests/ 13 | 14 | 15 | 16 | 17 | src/ 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /tests/resources/with-header.csv: -------------------------------------------------------------------------------- 1 | id,first_name,last_name,email,gender,ip_address 2 | 1,Dolley,Songer,dsonger0@businessweek.com,Female,120.107.3.146 3 | 2,Aylmar,Rakes,arakes1@facebook.com,Male,181.227.141.140 4 | 3,Agace,Beecham,abeecham2@gov.uk,Female,220.225.204.176 5 | 4,Aimee,McGinty,amcginty3@edublogs.org,Female,144.153.43.146 6 | 5,Issie,Trevithick,itrevithick4@skyrock.com,Female,251.177.50.163 7 | 6,Crissie,Merington,cmerington5@nba.com,Female,230.170.133.12 8 | 7,Gage,De Rye Barrett,gderyebarrett6@fc2.com,Male,97.97.211.201 9 | 8,Helene,Hawley,hhawley7@example.com,Female,223.206.155.97 10 | 9,Harrie,Soutar,hsoutar8@cbsnews.com,Female,115.41.88.193 11 | 10,Bryn,Neve,bneve9@technorati.com,Female,47.72.114.149 12 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dutchcodingcompany/csv-collection", 3 | "description": "Read and write large csv files using Laravel's Illuminate Collections", 4 | "keywords": [ 5 | "dutch coding company", 6 | "dcc", 7 | "csv collection", 8 | "csv", 9 | "collection", 10 | "laravel", 11 | "illuminate" 12 | ], 13 | "scripts": { 14 | "test": "./vendor/bin/phpunit ./tests" 15 | }, 16 | "require": { 17 | "php": "^7.4|^8.0", 18 | "illuminate/collections": "^6.0|^7.0|^8.0|^9.0|^10.0|^11.0|^12.0" 19 | }, 20 | "require-dev": { 21 | "phpunit/phpunit": "*" 22 | }, 23 | "autoload": { 24 | "psr-4": { 25 | "DutchCodingCompany\\CsvCollection\\": "src/", 26 | "Tests\\": "tests/" 27 | } 28 | }, 29 | "config": { 30 | "sort-packages": true 31 | }, 32 | "homepage": "https://dutchcodingcompany.com", 33 | "license": "MIT", 34 | "minimum-stability": "dev", 35 | "prefer-stable": true 36 | } 37 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) Dutch Coding Company BV 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /tests/CollectionTest.php: -------------------------------------------------------------------------------- 1 | options(['header' => false]) 20 | ->open(__DIR__ . '/resources/without-header.csv'); 21 | 22 | $this->assertSame($collection->first(), [ 23 | 0 => '1', 24 | 1 => 'Wait', 25 | 2 => 'Blampy', 26 | 3 => 'wblampy0@addtoany.com', 27 | 4 => 'Male', 28 | 5 => '93.96.91.96', 29 | ]); 30 | 31 | $this->assertSame(10, $collection->count()); 32 | } 33 | 34 | public function testCanOpenWithHeader(): void 35 | { 36 | $collection = CsvCollection::make() 37 | ->open(__DIR__ . '/resources/with-header.csv'); 38 | 39 | $this->assertSame($collection->first(), [ 40 | 'id' => '1', 41 | 'first_name' => 'Dolley', 42 | 'last_name' => 'Songer', 43 | 'email' => 'dsonger0@businessweek.com', 44 | 'gender' => 'Female', 45 | 'ip_address' => '120.107.3.146', 46 | ]); 47 | 48 | $this->assertSame(10, $collection->count()); 49 | } 50 | 51 | public function testCanSaveWithoutHeader(): void 52 | { 53 | $path = __DIR__ . '/temporary/save-without-header.csv'; 54 | 55 | $data = [ 56 | 'Wait', 57 | 'Blampy', 58 | ]; 59 | 60 | $collection = CsvCollection::make([$data]) 61 | ->options(['header' => false]) 62 | ->save($path) 63 | ->open($path); 64 | 65 | $this->assertSame($collection->first(), $data); 66 | $this->assertSame(1, $collection->count()); 67 | 68 | unlink($path); 69 | } 70 | 71 | public function testCanSaveWithHeader(): void 72 | { 73 | $path = __DIR__ . '/temporary/save-with-header.csv'; 74 | 75 | $data = [ 76 | 'first_name' => 'Wait', 77 | 'last_name' => 'Blampy', 78 | ]; 79 | 80 | $collection = CsvCollection::make([$data]) 81 | ->options(['header' => true]) 82 | ->save($path) 83 | ->open($path); 84 | 85 | $this->assertSame($collection->first(), $data); 86 | $this->assertSame(1, $collection->count()); 87 | 88 | unlink($path); 89 | } 90 | 91 | protected function tearDown(): void 92 | { 93 | rmdir(__DIR__ . '/temporary'); 94 | 95 | parent::tearDown(); 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are **welcome** and will be fully **credited**. 4 | 5 | Please read and understand the contribution guide before creating an issue or pull request. 6 | 7 | ## Etiquette 8 | 9 | This project is open source, and as such, the maintainers give their free time to build and maintain the source code 10 | held within. They make the code freely available in the hope that it will be of use to other developers. It would be 11 | extremely unfair for them to suffer abuse or anger for their hard work. 12 | 13 | Please be considerate towards maintainers when raising issues or presenting pull requests. Let's show the 14 | world that developers are civilized and selfless people. 15 | 16 | It's the duty of the maintainer to ensure that all submissions to the project are of sufficient 17 | quality to benefit the project. Many developers have different skillsets, strengths, and weaknesses. Respect the maintainer's decision, and do not be upset or abusive if your submission is not used. 18 | 19 | ## Viability 20 | 21 | When requesting or submitting new features, first consider whether it might be useful to others. Open 22 | source projects are used by many developers, who may have entirely different needs to your own. Think about 23 | whether or not your feature is likely to be used by other users of the project. 24 | 25 | ## Procedure 26 | 27 | Before filing an issue: 28 | 29 | - Attempt to replicate the problem, to ensure that it wasn't a coincidental incident. 30 | - Check to make sure your feature suggestion isn't already present within the project. 31 | - Check the pull requests tab to ensure that the bug doesn't have a fix in progress. 32 | - Check the pull requests tab to ensure that the feature isn't already in progress. 33 | 34 | Before submitting a pull request: 35 | 36 | - Check the codebase to ensure that your feature doesn't already exist. 37 | - Check the pull requests to ensure that another person hasn't already submitted the feature or fix. 38 | 39 | ## Requirements 40 | 41 | If the project maintainer has any additional requirements, you will find them listed here. 42 | 43 | - **[PSR-2 Coding Standard](https://github.com/php-fig/fig-standards/blob/master/accepted/PSR-2-coding-style-guide.md)** - The easiest way to apply the conventions is to install [PHP Code Sniffer](https://pear.php.net/package/PHP_CodeSniffer). 44 | 45 | - **Add tests!** - Your patch won't be accepted if it doesn't have tests. 46 | 47 | - **Document any change in behaviour** - Make sure the `README.md` and any other relevant documentation are kept up-to-date. 48 | 49 | - **Consider our release cycle** - We try to follow [SemVer v2.0.0](https://semver.org/). Randomly breaking public APIs is not an option. 50 | 51 | - **One pull request per feature** - If you want to do more than one thing, send multiple pull requests. 52 | 53 | - **Send coherent history** - Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please [squash them](https://www.git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Changing-Multiple-Commit-Messages) before submitting. 54 | 55 | **Happy coding**! 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSV Collection 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/dutchcodingcompany/csv-collection.svg?style=flat-square)](https://packagist.org/packages/dutchcodingcompany/csv-collection) 4 | [![Total Downloads](https://img.shields.io/packagist/dt/dutchcodingcompany/csv-collection.svg?style=flat-square)](https://packagist.org/packages/dutchcodingcompany/csv-collection) 5 | 6 | This package provides a simple but powerful way to read and write large CSV files using the power of Laravel's lazy 7 | collections. 8 | 9 | ## Installation 10 | 11 | You can install the package via composer: 12 | 13 | ```bash 14 | composer require dutchcodingcompany/csv-collection 15 | ``` 16 | 17 | ## Usage 18 | 19 | You may create a collection using the `new` keyword or the `make` method. 20 | 21 | ```php 22 | CsvCollection::make(); 23 | ``` 24 | 25 | This gives you access to all [Collection](https://laravel.com/docs/8.x/collections#available-methods) 26 | and [Lazy Collection](https://laravel.com/docs/8.x/collections#lazy-collection-methods) methods. 27 | 28 | ### Open 29 | 30 | To open a file and load it's content into a new collection you may use the `open` method on the collection. 31 | 32 | ```php 33 | use DutchCodingCompany\CsvCollection\CsvCollection; 34 | 35 | CsvCollection::make() 36 | ->open('path/to/file.csv') 37 | ->count(); 38 | ``` 39 | 40 | ### Save 41 | 42 | To save the collection items to a file you may use the `save` method on the collection. 43 | 44 | ```php 45 | use DutchCodingCompany\CsvCollection\CsvCollection; 46 | 47 | CsvCollection::make(static function () { 48 | yield [ 49 | 'key' => 'value', 50 | ]; 51 | }) 52 | ->save('path/to/file.csv'); 53 | ``` 54 | 55 | #### Model exports 56 | 57 | When exporting models a memory efficient method is to lazily iterate through the models and `yield` it's content. 58 | 59 | ```php 60 | use DutchCodingCompany\CsvCollection\CsvCollection; 61 | 62 | CsvCollection::make(static function () { 63 | $models = Model::query()->lazy(); 64 | 65 | foreach ($models as $model){ 66 | yield $model->only([ 67 | 'id', 68 | // 69 | ]); 70 | } 71 | }) 72 | ->save('path/to/file.csv'); 73 | ``` 74 | 75 | ### Options 76 | 77 | The following options are available to suit your needs: 78 | 79 | - `header`, default: `true` 80 | - `delimiter`, default: `,` 81 | - `enclosure`, default: `"` 82 | - `escape`, default: `\\` 83 | 84 | These options could be passed to the `open` and `save` methods, be set using the `options` method, or be set as the 85 | global default using the static `defaults` method. 86 | 87 | The delimiter can be detected for a file by using the `detectDelimiter` method like this: 88 | 89 | ``` 90 | CsvCollection::detectDelimiter($path); 91 | ``` 92 | 93 | #### Header 94 | 95 | When using a header, lines will contain an associated array. Otherwise, lines will contain an indexed array. 96 | 97 | ```php 98 | // Without header 99 | [ 100 | 0 => 'John', 101 | 1 => 'Doe', 102 | ] 103 | 104 | // With header 105 | [ 106 | 'first_name' => 'John', 107 | 'last_name' => 'Doe', 108 | ] 109 | ``` 110 | 111 | _**Note**: When saving a collection to a file the keys of the first element in the collection will be used as the 112 | header._ 113 | 114 | ## Testing 115 | 116 | ```bash 117 | composer test 118 | ``` 119 | 120 | ## Credits 121 | 122 | - [Bjorn Voesten](https://github.com/bjornvoesten) 123 | - [Dutch Coding Company](https://github.com/dutchcodingcompany) 124 | - [All contributors](https://github.com/dutchcodingcompany/csv-collection/graphs/contributors) 125 | 126 | ## License 127 | 128 | The MIT License (MIT). Please see [License File](LICENSE.md) for more information. 129 | -------------------------------------------------------------------------------- /src/CsvCollection.php: -------------------------------------------------------------------------------- 1 | ',', 16 | 'enclosure' => '"', 17 | 'escape' => '\\', 18 | 'header' => true, 19 | ]; 20 | 21 | /** 22 | * The collection's options. 23 | * 24 | * @var array 25 | */ 26 | public array $options = []; 27 | 28 | /** 29 | * Create a new csv collection instance. 30 | * 31 | * @param mixed $source 32 | * @return void 33 | */ 34 | public function __construct($source = null) 35 | { 36 | parent::__construct($source); 37 | 38 | $this->options(static::$defaults); 39 | } 40 | 41 | /** 42 | * Load the csv file items into a new collection. 43 | * 44 | * @param string $file 45 | * @param array $options 46 | * @return static 47 | */ 48 | public function open(string $file, array $options = []): self 49 | { 50 | $options = array_merge( 51 | $this->options, $options 52 | ); 53 | 54 | return static::make(static function () use ($file, $options) { 55 | $resource = fopen($file, 'r'); 56 | 57 | $read = static fn() => fgetcsv( 58 | $resource, 0, 59 | $options['delimiter'], 60 | $options['enclosure'], 61 | $options['escape'], 62 | ); 63 | 64 | $header = null; 65 | 66 | // Loop over the rows and yield them into a generator. 67 | while (($line = $read()) !== false) { 68 | if (! ($options['header'])) { 69 | yield $line; 70 | continue; 71 | } 72 | 73 | if (! $header) { 74 | $header = $line; 75 | continue; 76 | } 77 | 78 | yield array_combine($header, $line); 79 | } 80 | 81 | fclose($resource); 82 | }); 83 | } 84 | 85 | /** 86 | * Save the collection items to the csv file. 87 | * 88 | * @param string|null $file 89 | * @param array $options 90 | * @return $this 91 | */ 92 | public function save(string $file, array $options = []): self 93 | { 94 | $options = array_merge( 95 | $this->options, $options 96 | ); 97 | 98 | $resource = fopen($file, 'w'); 99 | 100 | $write = static fn(array $line) => fputcsv( 101 | $resource, $line, 102 | $options['delimiter'], 103 | $options['enclosure'], 104 | $options['escape'], 105 | ); 106 | 107 | if ($options['header']) { 108 | $write(array_keys($this->first())); 109 | } 110 | 111 | $this->each($write); 112 | 113 | fclose($resource); 114 | 115 | return $this; 116 | } 117 | 118 | /** 119 | * Push an item into the collection and save the item to the csv file. 120 | * 121 | * @param string|null $file 122 | * @param array $line 123 | * @param array $options 124 | * @return \App\CsvCollection 125 | */ 126 | public function push(string $file, array $line, array $options = []): self 127 | { 128 | $options = array_merge( 129 | $this->options, $options 130 | ); 131 | 132 | $resource = fopen($file, 'a'); 133 | 134 | // Lock the file. 135 | if (! flock($resource, LOCK_EX)) { 136 | throw new IOException("Could not lock file"); 137 | } 138 | 139 | $write = static fn(array $line) => fputcsv( 140 | $resource, $line, 141 | $options['delimiter'], 142 | $options['enclosure'], 143 | $options['escape'], 144 | ); 145 | 146 | if ($options['header'] && $this->open($file)->count() === 0) { 147 | $write(array_keys($line)); 148 | } 149 | 150 | $write($line); 151 | 152 | // Unlock the file. 153 | flock($resource, LOCK_UN); 154 | fclose($resource); 155 | 156 | return $this->open($file, $options); 157 | } 158 | 159 | /** 160 | * @param string $file Path to the CSV file 161 | * @return string Delimiter 162 | */ 163 | public static function detectDelimiter(string $file): string 164 | { 165 | $delimiters = [";" => 0, "," => 0, "\t" => 0, "|" => 0]; 166 | 167 | $handle = fopen($file, "r"); 168 | $firstLine = fgets($handle); 169 | fclose($handle); 170 | foreach ($delimiters as $delimiter => &$count) { 171 | $count = count(str_getcsv($firstLine, $delimiter)); 172 | } 173 | 174 | return array_search(max($delimiters), $delimiters, true); 175 | } 176 | 177 | /** 178 | * Set the collection's options. 179 | * 180 | * @param array $options 181 | * @return static 182 | */ 183 | public function options(array $options): self 184 | { 185 | $this->options = array_merge( 186 | $this->options, $options 187 | ); 188 | 189 | return $this; 190 | } 191 | 192 | /** 193 | * Set the collection's default options. 194 | * 195 | * @param array $options 196 | * @return void 197 | */ 198 | public static function defaults(array $options): void 199 | { 200 | static::$defaults = array_merge( 201 | static::$defaults, $options 202 | ); 203 | } 204 | } 205 | --------------------------------------------------------------------------------