├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── doc.go ├── easycsv.go ├── easycsv_test.go ├── encode.go ├── encode_test.go ├── example_test.go ├── examples ├── example.ipynb ├── noheader.csv └── withheader.csv ├── go.mod ├── go.sum ├── issues_test.go ├── option.go ├── option_test.go └── testdata ├── issue1.csv ├── sample.csv └── sample.tsv /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.dll 4 | *.so 5 | *.dylib 6 | 7 | # Test binary, build with `go test -c` 8 | *.test 9 | 10 | # Output of the go coverage tool, specifically when used with LiteIDE 11 | *.out 12 | 13 | # Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 14 | .glide/ 15 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - "1.13.x" 4 | - "1.14.x" 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # easycsv [![Build Status](https://travis-ci.org/yunabe/easycsv.svg?branch=master)](https://travis-ci.org/yunabe/easycsv) [![Go Report Card](https://goreportcard.com/badge/github.com/yunabe/easycsv)](https://goreportcard.com/report/github.com/yunabe/easycsv) [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/yunabe/easycsv-binder/master?filepath=example.ipynb) 2 | 3 | easycsv package provides API to read CSV files in Go (golang) easily. 4 | 5 | # Installation 6 | 7 | ``` 8 | go get -u github.com/yunabe/easycsv 9 | ``` 10 | 11 | # Features 12 | 13 | - You can read CSV files with less boilerplate code because `easycsv` provides a consice error API. 14 | - `easycsv` automatically converts CSV rows into your custom structs. 15 | - Of course, you can handle TSV and other CSV-like formats by customizing `easycsv.Reader`. 16 | 17 | # Links 18 | 19 | - [yunabe/easycsv pkg.go.dev](https://pkg.go.dev/github.com/yunabe/easycsv) 20 | - [yunabe/easycsv GitHub](https://github.com/yunabe/easycsv) 21 | 22 | # Quick Tour 23 | 24 | ## Read a CSV file to a struct 25 | 26 | ```golang 27 | r := easycsv.NewReaderFile("testdata/sample.csv") 28 | var entry struct { 29 | Name string `index:"0"` 30 | Age int `index:"1"` 31 | } 32 | for r.Read(&entry) { 33 | fmt.Print(entry) 34 | } 35 | if err := r.Done(); err != nil { 36 | log.Fatalf("Failed to read a CSV file: %v", err) 37 | } 38 | ``` 39 | 40 | ## Read a CSV file with Loop 41 | 42 | ```golang 43 | r := easycsv.NewReaderFile("testdata/sample.csv") 44 | err := r.Loop(func(entry *struct { 45 | Name string `index:"0"` 46 | Age int `index:"1"` 47 | }) error { 48 | fmt.Print(entry) 49 | return nil 50 | }) 51 | if err != nil { 52 | log.Fatalf("Failed to read a CSV file: %v", err) 53 | } 54 | ``` 55 | 56 | # Usages 57 | 58 | ## NewReader 59 | 60 | The core component of easycsv is [`Reader`](https://godoc.org/github.com/yunabe/easycsv#Reader). 61 | You can create a new `Reader` instance from `io.Reader`, `io.ReadCloser` and a file path. 62 | 63 | - [NewReader](https://godoc.org/github.com/yunabe/easycsv#NewReader) 64 | - Create `easycsv.Reader` from `io.Reader`. 65 | - [NewReadCloser](https://godoc.org/github.com/yunabe/easycsv#NewReadCloser) 66 | - Create `easycsv.Reader` from `io.ReadCloser`. 67 | - [NewReaderFile](https://godoc.org/github.com/yunabe/easycsv#NewReaderFile) 68 | - Create `easycsv.Reader` from a file path. 69 | 70 | The Reader created by NewReadCloser and NewReaderFile closes the file automatically when the Reader is finished. 71 | So you do not need to close files manually and you can omit error handling code for closing files. 72 | 73 | ## Read 74 | 75 | There are three methods to read CSV with `easycsv.Reader`. Read, Loop and ReadAll. 76 | We are looking into [`Read`](https://godoc.org/github.com/yunabe/easycsv#Reader.Read) method first, which is the most basic and naive way to read CSV with Reader. 77 | 78 | ```golang 79 | func (r *Reader) Read(e interface{}) bool 80 | ``` 81 | 82 | [`Read`](https://godoc.org/github.com/yunabe/easycsv#Reader.Read) receives a pointer to a struct (e.g. `*myStruct`) or a pointer to a slice of a primitive type (e.g. `*[]int`). 83 | If it reads a new row from CSV successufly, it stores the row into `e` and returns `true`. 84 | If `Reader` reaches to `EOF` or it fails to read a new row for some reasons, it returns `false`. 85 | `Read` returns `false` for various reasons. To check the reason, you have to call `Done()` subsequently. 86 | `Done` returns an error if `Read` encountered an error. 87 | `Done` returns `nil` if `Read` returned `false` because it reached to `EOF`. 88 | 89 | You can pass two types of pointers to Read. A pointer of a struct (e.g. `*myStruct`) or a pointer of a slice of primitive typs (e.g. `*[]int`). Passing a pointer of a struct is more convenient. 90 | When you use a struct, you need to specify how to map CSV columns to the struct's field using struct field's tags. 91 | Here are examples: 92 | 93 | ```golang 94 | var entry struct { 95 | Name string `index:"0"` 96 | Age int `index:"1"` 97 | } 98 | 99 | var entry struct { 100 | Name string `name:"name"` 101 | Age int `name:"age"` 102 | } 103 | ``` 104 | 105 | You can use `index` tag or `name` tag to specify the mapping. 106 | When `index` is used, Read maps `index`-th (0-based) column to the field. 107 | In the first example, the frist column is mapped to Name and the second column is mapped to Age. 108 | When `name` is used, Read uses the first line of CSV as a header with column names and maps columns to fields based on the column names in the header. In the second example, give that the content of CSV is the following, 109 | 110 | ```csv 111 | age,name 112 | 10,Alice 113 | 20,Bob 114 | ``` 115 | 116 | the frist column is mapped to Age and the second column is mapped to Name. So `{Alice 10}` and `{Bob 20}` are stored to the struct respectively. You can not use both `index` tag and `name` tag in the same struct. Read reports an error in that case. 117 | 118 | If you pass a pointer to a slice to Read, Read converts CSV row into the slice and fills it to the argument. 119 | If the argument `e` is invalid, Read returns false immediately and the reason of the error is reported by `Done()`. 120 | 121 | The conversion from CSV row (string) to the given field type (int, float32, bool, etc...) is handled in Reader automatically. If you want to customize the conversion, see [Custom encoding](#custom-encoding) section below. 122 | 123 | When you read CSV with `Read` methods, you have to always call `Done()` subsequently to (1) check the error and (2) close the file behind the Reader. If you forget to call `Done()`, the error will be completely gone. 124 | Do not forget to call `Done` after `Read`. 125 | 126 | ## Loop 127 | 128 | ```golang 129 | func (r *Reader) Loop(body interface{}) (err error) 130 | ``` 131 | 132 | [Loop](https://godoc.org/github.com/yunabe/easycsv#Loop) reads CSV line by line and executes `body` with a line everytime it reads a line. 133 | `body` must be a function that receives a struct (e.g. `myStruct`), a pointer of a struct (e.g. `*myStruct`) or a slice of primitives (e.g. `[]int`). 134 | A line of CSV is automatically converted to the argument of `body` when Loop reads the line and passed to `body`. 135 | Also, `body` must be a function that returns `bool`, `error` or no return value. 136 | If `body` is a function that returns `bool`, Loop stops reading CSV at the line where `body` returns false. 137 | If `body` is a function that returns `error`, Loop stops reading CSV when `body` retruns an error. 138 | Loop does not stop until it reached to the end if `body` has no return value. 139 | If `body` retuns an error, Loop quits and reports the error. 140 | 141 | When `Loop` ends, it invokes `Done` and closes internal files automatically. 142 | So, you do not need to call Done after Loop. 143 | Loop returns the first error if it encounters errors. It returns `nil` if everything goes well. 144 | Do not forget to handle the error returned by Loop. 145 | 146 | The example below shows how to use Loop with a function which returns `error`. 147 | This code reads CSV until Loop reaches to EOF or an entry with Age < 0 is found in the CSV. 148 | 149 | ```golang 150 | err := r.Loop(func(entry *struct { 151 | Name string `index:"0"` 152 | Age int `index:"1"` 153 | }) error { 154 | fmt.Println(entry) 155 | if Age < 0 { 156 | return errors.New("Age mustn't be negative") 157 | } 158 | }) 159 | if err != nil { 160 | log.Fatalf("Failed to read a CSV file: %v", err) 161 | } 162 | ``` 163 | 164 | ## ReadAll 165 | 166 | ```golang 167 | func (r *Reader) ReadAll(s interface{}) (err error) 168 | ``` 169 | 170 | [ReadAll](https://godoc.org/github.com/yunabe/easycsv#ReadAll) reads a CSV input to the end and convert all rows into the slice passed as an argument. 171 | The argument `s` must be a pointer of a slice of a struct (`*[]myStruct`) or a pointer of a slice of a slice (`*[][]int`). 172 | Aside from that, the same rules of Read are applied to ReadAll. You need to specify how to map columns to struct fields using struct field's tag. 173 | 174 | Like `Loop`, you do not need to call `Done` after ReadAll. ReadAll returns the first error if it encounters errors. It returns `nil` if everything goes well. 175 | Do not forget to handle the error returned by ReadAll. 176 | 177 | ```golang 178 | var entry []struct { 179 | Name string `index:"0"` 180 | Age int `index:"1"` 181 | } 182 | err := r.ReadAll(&entry); 183 | ``` 184 | 185 | # Option 186 | 187 | To control the behavior of Reader, you can pass Option to NewReader methods. 188 | 189 | NewReader methods receive Option as a variadic parameter `opts`. `opts` is a variadic parameter so that we can omit `opts` from parameters when we call NewReader methods without changing Option. 190 | Thus, you don't need to pass multiple Option to NewReader methods although you can pass as many Option as you want. 191 | 192 | ## Comma 193 | 194 | Like [csv.Reader](https://golang.org/pkg/encoding/csv/#Reader) in the standard library, you can change the deliminator of CSV by specifying `Comma` option. For example, if you set `'\t'` to Comma, Reader reads a file as a TSV file. 195 | 196 | ## Comment 197 | 198 | Comment, if not 0, is the comment character. Lines beginning with the character without preceding whitespace are ignored. 199 | 200 | ## FieldsPerRecord 201 | 202 | In the standard library [csv.Reader](https://golang.org/pkg/encoding/csv/#Reader), an option `FieldsPerRecord` is available to define the number of fields allowed per CSV record. If you set a value that is not 0 to `FieldsPerRecord`, this option will be updated. 203 | 204 | # Customizing decoders 205 | 206 | By default, easycsv converts strings in CSV to integers, floats and bool automatically based on the types of struct fields and slices. 207 | 208 | - Integers are parsed with `strconv.ParseInt` and unsigned integers are parsed with `strconv.ParseUint`. 209 | easycsv parses inputs as decimals by default. But it parses inputs as hex if inputs have `"0x"` prefix and 210 | as octal if inputs have `"0"` prefix (`"0xff"` → 255, `"077"` → 63). 211 | - Floats are parsed with `strconv.ParseFloat`. 212 | - bool is parsed with `strconv.ParseBool`. 213 | 214 | You can customize how to decode strings in CSV to values by specifying `enc` attribute to struct fields. 215 | 216 | ## Predefined encoding 217 | 218 | easycsv has three predefined custom encoding for integers. 219 | 220 | - `deci` - Parses inputs as decimal integers even if the inputs are prefixed with `"0"`. 221 | - `hex` - Parses inputs as hex integers. 222 | - `oct`- Parses inputs as oct integers. 223 | 224 | ## Custom encoding 225 | 226 | Also, you can use custom encodings in easycsv. 227 | 228 | To use custom encodings: 229 | 230 | - Define a func that convert strings to your custom types. This func must receive a string and returns (custom-type, error). 231 | - Register the func to Option.Decoders. 232 | - Specify the registered func name with `enc` struct-field attribute. 233 | 234 | ```golang 235 | r := NewReader(bytes.NewBufferString("name,birthday\nAlice,1980-12-30\nBob,1975-06-09"), 236 | Option{ 237 | Decoders: map[string]interface{}{ 238 | "date": func(s string) (time.Time, error) { 239 | return time.Parse("2006-01-02", s) 240 | }, 241 | }, 242 | }) 243 | var entry struct { 244 | Name string `name:"name"` 245 | Birth time.Time `name:"birthday" enc:"date"` 246 | } 247 | for r.Read(&entry) { 248 | fmt.Print(entry) 249 | } 250 | if err := r.Done(); err != nil { 251 | fmt.Printf("Failed: %v\n", err) 252 | } 253 | // Output: {Alice 1980-12-30 00:00:00 +0000 UTC}{Bob 1975-06-09 00:00:00 +0000 UTC} 254 | ``` 255 | 256 | ## Customizing decoders for types 257 | 258 | You can also define how to convert strings into specific types in easycsv by using Option.TypeDecoders option. Option.TypeDecoders is similar to Option.Decoders. The key is `reflect.Type` and the value is a function to convert strings to the specific type. 259 | Reader uses the functions registered to Option.TypeDecoders instead of default converters when it converts rows in CSV into those types. 260 | 261 | The following example shows how to define a converter for `time.Time` with Option.TypeDecoders. 262 | 263 | ```golang 264 | r := NewReader(bytes.NewReader([]byte("2017-01-02,2016-02-03\n2015-03-04,2014-04-05")), 265 | Option{TypeDecoders: map[reflect.Type]interface{}{ 266 | reflect.TypeOf(time.Time{}): func(s string) (time.Time, error) { 267 | return time.Parse("2006-01-02", s) 268 | }, 269 | }}) 270 | var entry []time.Time 271 | for r.Read(&entry) { 272 | for _, e := range entry { 273 | fmt.Print(e.Format("2006/1/2"), ";") 274 | } 275 | } 276 | if err := r.Done(); err != nil { 277 | fmt.Print(err) 278 | } 279 | ``` 280 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | // Package easycsv provides API to read CSV files with less boilerplate code in Go. 2 | // 3 | // Overview guideline: https://github.com/yunabe/easycsv/blob/master/README.md 4 | package easycsv 5 | -------------------------------------------------------------------------------- /easycsv.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "encoding/csv" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "os" 9 | "reflect" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | // Break is the error returned by the callback passed to Loop to terminate the loop. 15 | var Break = errors.New("break") 16 | 17 | // Reader provides a convenient interface for reading csv. 18 | type Reader struct { 19 | // csv.Reader. To read content from csv, use readLine. 20 | csv *csv.Reader 21 | closer io.Closer 22 | done bool 23 | // An error occurred while processing csv. io.EOF is stored when csv is reached to the end. 24 | err error 25 | opt Option 26 | 27 | // Used from readLine. 28 | lineno int 29 | firstLine []string 30 | cur []string 31 | } 32 | 33 | func newCSVReader(r io.Reader, opt Option) *csv.Reader { 34 | cr := csv.NewReader(r) 35 | if opt.Comma != 0 { 36 | cr.Comma = opt.Comma 37 | } 38 | if opt.Comment != 0 { 39 | cr.Comment = opt.Comment 40 | } 41 | if opt.LazyQuotes { 42 | cr.LazyQuotes = opt.LazyQuotes 43 | } 44 | if opt.FieldsPerRecord != 0 { 45 | cr.FieldsPerRecord = opt.FieldsPerRecord 46 | } 47 | 48 | return cr 49 | } 50 | 51 | // NewReader returns a new Reader to read CSV from r. 52 | func NewReader(r io.Reader, opts ...Option) *Reader { 53 | opt, err := mergeOptions(opts) 54 | if err != nil { 55 | return &Reader{err: err} 56 | } 57 | rd := Reader{ 58 | csv: newCSVReader(r, opt), 59 | opt: opt, 60 | } 61 | return &rd 62 | } 63 | 64 | // NewReadCloser returns a new Reader to read CSV from r. 65 | // Reader instantiated with NewReadCloser closes r automatically when Done() is called. 66 | func NewReadCloser(r io.ReadCloser, opts ...Option) *Reader { 67 | opt, err := mergeOptions(opts) 68 | if err != nil { 69 | return &Reader{err: err} 70 | } 71 | return &Reader{ 72 | csv: newCSVReader(r, opt), 73 | opt: opt, 74 | closer: r, 75 | } 76 | } 77 | 78 | // NewReaderFile returns a new Reader to read CSV from the file path. 79 | func NewReaderFile(path string, opts ...Option) *Reader { 80 | f, err := os.Open(path) 81 | if err == nil { 82 | return NewReadCloser(f, opts...) 83 | } 84 | return &Reader{err: err} 85 | } 86 | 87 | // readLine reads a line from r.csv and update r.err, r.cur, r.lineno and r.firstLine. 88 | // readLine does not update r.err. io.EOF is returned when csv reached to the end. 89 | func (r *Reader) readLine() { 90 | line, err := r.csv.Read() 91 | if err != nil { 92 | r.err = err 93 | return 94 | } 95 | r.cur = line 96 | r.lineno++ 97 | if r.lineno == 1 { 98 | r.firstLine = line 99 | } 100 | } 101 | 102 | var errorType = reflect.TypeOf((*error)(nil)).Elem() 103 | 104 | // Loop reads from r until an error or EOF and invokes body everytime it reads a line. 105 | // body should be a function which has no return value or returns bool or error. 106 | // If body returns false or an error, Loop stops reading r. 107 | // If body does not have a return value, Loop does not stop until an error or EOF. 108 | // If body returns error and you want to terminate Loop without reporting an error, returns easycsv.Break in body. 109 | // 110 | // Also, body must receive one argument. The argument must be a pointer to a struct, a struct or a pointer to a slice. 111 | // The line of csv is automatically converted to the struct or the slice based on the rule described above. 112 | // 113 | // Loop returns an error if it encounters an error and exits the loop. 114 | func (r *Reader) Loop(body interface{}) (err error) { 115 | defer func() { err = r.Done() }() 116 | if r.err != nil { 117 | return 118 | } 119 | if body == nil { 120 | r.err = errors.New("The argument of Loop must not be nil.") 121 | return 122 | } 123 | v := reflect.TypeOf(body) 124 | if v.Kind() != reflect.Func { 125 | r.err = fmt.Errorf("The argument of Loop must be func but got %v", v.Kind()) 126 | return 127 | } 128 | if v.NumIn() != 1 || v.NumOut() > 1 { 129 | r.err = fmt.Errorf("The function passed to Loop must receive one argument and return one or zero value") 130 | return 131 | } 132 | if v.NumOut() > 0 { 133 | if out := v.Out(0); out.Kind() != reflect.Bool && out != errorType { 134 | r.err = fmt.Errorf("The function passed to Loop must return error or bool") 135 | return 136 | } 137 | } 138 | in := v.In(0) 139 | var inStruct reflect.Type 140 | if in.Kind() == reflect.Struct { 141 | inStruct = in 142 | } else if in.Kind() == reflect.Ptr && in.Elem().Kind() == reflect.Struct { 143 | inStruct = in.Elem() 144 | } else if in.Kind() == reflect.Slice { 145 | inStruct = in 146 | } else { 147 | r.err = fmt.Errorf("The function passed to Loop must receive a struct, a pointer to a struct or a slice") 148 | return 149 | } 150 | if in.Kind() != reflect.Slice { 151 | numf := inStruct.NumField() 152 | if numf == 0 { 153 | r.err = errors.New("The struct passed to Loop must have at least one field") 154 | return 155 | } 156 | } 157 | dec, err := newDecoder(r.opt, inStruct) 158 | if err != nil { 159 | r.err = err 160 | return 161 | } 162 | if dec.needHeader() { 163 | if r.lineno == 0 { 164 | // Loop quits immediately if the csv is empty. 165 | r.readLine() 166 | if r.err != nil { 167 | return 168 | } 169 | } 170 | err = dec.consumeHeader(r.firstLine) 171 | if err != nil { 172 | r.err = err 173 | return 174 | } 175 | } 176 | for { 177 | r.readLine() 178 | if r.err != nil { 179 | break 180 | } 181 | p := reflect.New(inStruct) 182 | if err := dec.decode(r.cur, p); err != nil { 183 | r.err = err 184 | break 185 | } 186 | arg := p 187 | if in.Kind() == reflect.Struct || in.Kind() == reflect.Slice { 188 | arg = p.Elem() 189 | } 190 | rets := reflect.ValueOf(body).Call([]reflect.Value{arg}) 191 | if len(rets) == 0 { 192 | continue 193 | } 194 | ret := rets[0] 195 | if ret.Kind() == reflect.Bool { 196 | if ret.Bool() { 197 | continue 198 | } else { 199 | break 200 | } 201 | } 202 | if ret.IsNil() { 203 | // body returned nil error. 204 | continue 205 | } 206 | err = rets[0].Interface().(error) 207 | if err == nil { 208 | panic("err must not be nil if I understand reflect spec correctly") 209 | } 210 | if err != Break { 211 | r.err = err 212 | } 213 | break 214 | } 215 | return 216 | } 217 | 218 | // Read reads one line from csv and store values in the line to e. 219 | // The argument e must be a pointer to a struct or a pointer to a slice. 220 | // Read returns false when it encounters an error or EOF. 221 | func (r *Reader) Read(e interface{}) bool { 222 | if r.err != nil { 223 | return false 224 | } 225 | if e == nil { 226 | r.err = errors.New("The argument of Loop must not be nil.") 227 | return false 228 | } 229 | t := reflect.TypeOf(e) 230 | if t.Kind() != reflect.Ptr { 231 | r.err = fmt.Errorf("The argument of Read must be a pointer to a struct or a pointer to a slice, but got %v", t.Kind()) 232 | return false 233 | } 234 | if t.Elem().Kind() != reflect.Struct && t.Elem().Kind() != reflect.Slice { 235 | r.err = fmt.Errorf("The argument of Read must be a pointer to a struct or a pointer to a slice, but got a pointer to %v", t.Elem().Kind()) 236 | return false 237 | } 238 | decoder, err := newDecoder(r.opt, t.Elem()) 239 | if err != nil { 240 | r.err = err 241 | return false 242 | } 243 | if decoder.needHeader() { 244 | if r.lineno == 0 { 245 | // Loop quits immediately if the csv is empty. 246 | r.readLine() 247 | if r.err != nil { 248 | return false 249 | } 250 | } 251 | if err := decoder.consumeHeader(r.firstLine); err != nil { 252 | r.err = err 253 | return false 254 | } 255 | } 256 | r.readLine() 257 | if r.err != nil { 258 | return false 259 | } 260 | // TODO: Append the line number to the error message. 261 | r.err = decoder.decode(r.cur, reflect.ValueOf(e)) 262 | return r.err == nil 263 | } 264 | 265 | // ReadAll reads all rows from csv and store it into the slice s. 266 | // s must be a pointer to a slice of a struct (e.g. *[]entry) or a pointer to a slice of primitive types (e.g. *[][]int). 267 | // ReadAll reports an error if it encounters an error while reading the input. 268 | // Also, ReadAll closes the file behind r automatically. 269 | func (r *Reader) ReadAll(s interface{}) (err error) { 270 | defer func() { err = r.Done() }() 271 | // TODO: Consolidate code with Read. 272 | if s == nil { 273 | r.err = errors.New("The argument of ReadAll must not be nil.") 274 | return 275 | } 276 | t := reflect.TypeOf(s) 277 | if t.Kind() != reflect.Ptr || t.Elem().Kind() != reflect.Slice { 278 | r.err = fmt.Errorf("The argument of ReadAll must be a pointer to a slice of a slice or a pointer to a slice of a struct, but got %v", t) 279 | return 280 | } 281 | et := t.Elem().Elem() 282 | if et.Kind() != reflect.Struct && et.Kind() != reflect.Slice { 283 | r.err = fmt.Errorf("The argument of ReadAll must be a pointer to a slice of a slice or a pointer to a slice of a struct, but got %v", t) 284 | return 285 | } 286 | decoder, err := newDecoder(r.opt, et) 287 | if err != nil { 288 | r.err = err 289 | return 290 | } 291 | if decoder.needHeader() { 292 | if r.lineno == 0 { 293 | // Loop quits immediately if the csv is empty. 294 | r.readLine() 295 | if r.err != nil { 296 | return 297 | } 298 | } 299 | decoder.consumeHeader(r.firstLine) 300 | } 301 | for { 302 | r.readLine() 303 | if r.err != nil { 304 | return 305 | } 306 | p := reflect.New(et) 307 | v := reflect.ValueOf(s).Elem() 308 | err := decoder.decode(r.cur, p) 309 | v.Set(reflect.Append(v, p.Elem())) 310 | if err != nil { 311 | r.err = err 312 | break 313 | } 314 | // TODO: Append the line number to the error message. 315 | } 316 | return 317 | } 318 | 319 | func (r *Reader) nonEOFError() error { 320 | if r.err == nil || r.err == io.EOF { 321 | return nil 322 | } 323 | return r.err 324 | } 325 | 326 | // Done returns the first non-EOF error that was encountered by the Reader. 327 | // Done also closes the internal Closer if the Reader is instantiated with NewReaderCloser. 328 | // 329 | // You need to use Done when you read CSV with Read method to check errors and close files behind. 330 | // You don't use Done when you read CSV with ReadAll and Loop because they call Done internally. 331 | func (r *Reader) Done() error { 332 | if r.done { 333 | return r.nonEOFError() 334 | } 335 | r.done = true 336 | if r.closer != nil { 337 | if cerr := r.closer.Close(); r.err == nil || r.err == io.EOF { 338 | r.err = cerr 339 | } 340 | } 341 | return r.nonEOFError() 342 | } 343 | 344 | // DoneDefer does the same thing as Done does. But it outputs an error to the argument. 345 | // DoneDefer does not overwrite an error if an error is already stored in err. 346 | // DoneDefer is useful to call Done from defer statement. 347 | func (r *Reader) DoneDefer(err *error) { 348 | e := r.Done() 349 | if *err == nil && e != nil { 350 | *err = e 351 | } 352 | } 353 | 354 | // LineNumber returns the current position of r in the input (1-based line number). 355 | // The behavior of LineNumber is undefined if it is called before Read operations or 356 | // after r reached to EOF or an error. 357 | func (r *Reader) LineNumber() int { 358 | return r.lineno 359 | } 360 | 361 | type rowDecoder interface { 362 | decode(s []string, out reflect.Value) error 363 | needHeader() bool 364 | consumeHeader([]string) error 365 | } 366 | 367 | func validateCustomConverter(conv interface{}, enc string, field reflect.StructField, errs *[]string) bool { 368 | convType := reflect.TypeOf(conv) 369 | if convType.Kind() != reflect.Func { 370 | *errs = append(*errs, fmt.Sprintf("The custom decoder for Encoding %q must be a function", enc)) 371 | return false 372 | } 373 | ok := true 374 | if numin := convType.NumIn(); numin != 1 { 375 | *errs = append(*errs, fmt.Sprintf("The custom decoder for Encoding %q must receive an arg, but receives %d args", enc, numin)) 376 | ok = false 377 | } else if convType.In(0).Kind() != reflect.String { 378 | *errs = append(*errs, fmt.Sprintf("The custom decoder for Encoding %q must receive a string, but receives %v", enc, convType.In(0))) 379 | ok = false 380 | } 381 | // TODO: Supports custom decoders that does not return an error. 382 | if numout := convType.NumOut(); numout != 2 { 383 | *errs = append(*errs, fmt.Sprintf("The custom decoder for Encoding %q must return two values, but returns %d values", enc, numout)) 384 | ok = false 385 | } else { 386 | if convType.Out(0) != field.Type { 387 | *errs = append(*errs, fmt.Sprintf("The type of field %q is %v, but enc %q returns %q", field.Name, field.Type, enc, convType.Out(0))) 388 | ok = false 389 | } 390 | if convType.Out(1) != errorType { 391 | *errs = append(*errs, fmt.Sprintf("The second return value of the custom decoder for %q must be error", enc)) 392 | ok = false 393 | } 394 | } 395 | return ok 396 | } 397 | 398 | func parseStructTag( 399 | opt Option, 400 | field reflect.StructField, 401 | fieldIdx int, 402 | nameMap map[string]int, 403 | idxMap map[int]int, 404 | converters *[]reflect.Value, 405 | errors *[]string) { 406 | tag := field.Tag 407 | name := tag.Get("name") 408 | index := tag.Get("index") 409 | if name == "" && index == "" { 410 | *errors = append(*errors, fmt.Sprintf("Please specify name or index to the struct field: %s", field.Name)) 411 | return 412 | } 413 | if name != "" && index != "" { 414 | *errors = append(*errors, fmt.Sprintf("Please specify name or index to the struct field: %s", field.Name)) 415 | return 416 | } 417 | var conv interface{} 418 | enc := tag.Get("enc") 419 | if enc != "" { 420 | if opt.Decoders != nil && opt.Decoders[enc] != nil { 421 | conv = opt.Decoders[enc] 422 | if !validateCustomConverter(conv, enc, field, errors) { 423 | conv = nil 424 | } 425 | } else { 426 | pre := predefinedDecoders[enc] 427 | // TODO: Test these errors. 428 | if pre != nil { 429 | conv = pre(field.Type) 430 | if conv == nil { 431 | *errors = append(*errors, fmt.Sprintf("Encoding %q does not support %v", enc, field.Type)) 432 | } 433 | } else { 434 | *errors = append(*errors, fmt.Sprintf("Encoding %q is not defined", enc)) 435 | return 436 | } 437 | } 438 | } 439 | if conv == nil { 440 | var err error 441 | conv, err = createConverterFromType(opt, field.Type) 442 | if err != nil { 443 | *errors = append(*errors, err.Error()) 444 | } 445 | } 446 | if conv == nil { 447 | *errors = append(*errors, fmt.Sprintf("Unexpected field type for %s: %s", field.Name, field.Type)) 448 | return 449 | } 450 | *converters = append(*converters, reflect.ValueOf(conv)) 451 | if name != "" { 452 | nameMap[name] = fieldIdx 453 | return 454 | } 455 | i, err := strconv.Atoi(index) 456 | if err != nil || i < 0 { 457 | *errors = append(*errors, fmt.Sprintf("Failed to parse index of field %s: %q", field.Name, index)) 458 | return 459 | } 460 | idxMap[i] = fieldIdx 461 | } 462 | 463 | func newDecoder(opt Option, t reflect.Type) (rowDecoder, error) { 464 | if t.Kind() == reflect.Struct { 465 | return newStructDecoder(opt, t) 466 | } else if t.Kind() == reflect.Slice { 467 | return newSliceDecoder(opt, t) 468 | } 469 | panic("newDecoder must be called with struct or slice.") 470 | } 471 | 472 | func newSliceDecoder(opt Option, t reflect.Type) (rowDecoder, error) { 473 | elem := t.Elem() 474 | c, err := createConverterFromType(opt, elem) 475 | if err != nil { 476 | return nil, err 477 | } 478 | if c == nil { 479 | return nil, fmt.Errorf("Failed to create a converter for %v", t) 480 | } 481 | return &sliceRowDecoder{ 482 | elemType: elem, 483 | converter: reflect.ValueOf(c), 484 | }, nil 485 | } 486 | 487 | type sliceRowDecoder struct { 488 | elemType reflect.Type 489 | converter reflect.Value 490 | } 491 | 492 | func (d *sliceRowDecoder) needHeader() bool { return false } 493 | func (d *sliceRowDecoder) consumeHeader([]string) error { return nil } 494 | func (d *sliceRowDecoder) decode(s []string, out reflect.Value) error { 495 | slicePtr := reflect.New(reflect.SliceOf(d.elemType)) 496 | for _, e := range s { 497 | rets := d.converter.Call([]reflect.Value{reflect.ValueOf(e)}) 498 | if len(rets) != 2 { 499 | panic("converter must return two values.") 500 | } 501 | if !rets[1].IsNil() { 502 | return rets[1].Interface().(error) 503 | } 504 | slicePtr.Elem().Set(reflect.Append(slicePtr.Elem(), rets[0])) 505 | } 506 | out.Elem().Set(slicePtr.Elem()) 507 | return nil 508 | } 509 | 510 | func newStructDecoder(opt Option, t reflect.Type) (rowDecoder, error) { 511 | if t.NumField() == 0 { 512 | return nil, errors.New("The struct has no field") 513 | } 514 | v := reflect.New(t).Elem() 515 | var unexported []string 516 | for i := 0; i < v.NumField(); i++ { 517 | if !v.Field(i).CanSet() { 518 | unexported = append(unexported, t.Field(i).Name) 519 | } 520 | } 521 | if unexported != nil { 522 | return nil, fmt.Errorf("The struct passed to Loop must not have unexported fields: %s", strings.Join(unexported, ", ")) 523 | } 524 | 525 | var tagErrors []string 526 | nameMap := make(map[string]int) 527 | idxMap := make(map[int]int) 528 | var converters []reflect.Value 529 | for i := 0; i < t.NumField(); i++ { 530 | f := t.Field(i) 531 | parseStructTag(opt, f, i, nameMap, idxMap, &converters, &tagErrors) 532 | } 533 | if len(nameMap) != 0 && len(idxMap) != 0 { 534 | tagErrors = append(tagErrors, "Fields with name and fields with index are mixed") 535 | } 536 | if tagErrors != nil { 537 | return nil, errors.New(strings.Join(tagErrors, "\n")) 538 | } 539 | if len(converters) != t.NumField() { 540 | panic("converters size mismatch") 541 | } 542 | if len(nameMap) != 0 { 543 | idxMap = nil 544 | } else { 545 | nameMap = nil 546 | } 547 | return &structRowDecoder{ 548 | structType: t, 549 | converters: converters, 550 | names: nameMap, 551 | indice: idxMap, 552 | opt: opt, 553 | }, nil 554 | } 555 | 556 | type structRowDecoder struct { 557 | structType reflect.Type 558 | converters []reflect.Value 559 | names map[string]int 560 | indice map[int]int 561 | opt Option 562 | } 563 | 564 | func (d *structRowDecoder) consumeHeader(header []string) error { 565 | indice := make(map[int]int) 566 | for i, col := range header { 567 | idx, ok := d.names[col] 568 | if !ok { 569 | continue 570 | } 571 | indice[i] = idx 572 | delete(d.names, col) 573 | } 574 | d.indice = indice 575 | if len(d.names) != 0 { 576 | var unused []string 577 | for n := range d.names { 578 | unused = append(unused, n) 579 | } 580 | return fmt.Errorf("%s did not appear in the first line", strings.Join(unused, ", ")) 581 | } 582 | d.names = nil 583 | return nil 584 | } 585 | 586 | func (d *structRowDecoder) decode(row []string, out reflect.Value) error { 587 | // TODO: Reset with zero first. 588 | for i, j := range d.indice { 589 | if i >= len(row) { 590 | if d.opt.FieldsPerRecord < 0 { 591 | continue 592 | } 593 | return fmt.Errorf("Accessed index %d though the size of the row is %d", i, len(row)) 594 | } 595 | rets := d.converters[j].Call([]reflect.Value{reflect.ValueOf(row[i])}) 596 | if len(rets) != 2 { 597 | panic("converter must return two values.") 598 | } 599 | if !rets[1].IsNil() { 600 | return rets[1].Interface().(error) 601 | } 602 | out.Elem().Field(j).Set(rets[0]) 603 | } 604 | return nil 605 | } 606 | 607 | func (d *structRowDecoder) needHeader() bool { 608 | return d.names != nil 609 | } 610 | -------------------------------------------------------------------------------- /easycsv_test.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "io" 7 | "reflect" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/google/go-cmp/cmp" 12 | ) 13 | 14 | // noDiff compares got and want and show an error if there is a diff. 15 | func noDiff(t *testing.T, name string, got, want interface{}) { 16 | t.Helper() 17 | if diff := cmp.Diff(want, got); diff != "" { 18 | t.Errorf("%s mismatch (-want +got):\n%s", name, diff) 19 | } 20 | } 21 | 22 | func TestLoopNil(t *testing.T) { 23 | f := bytes.NewReader([]byte("")) 24 | r := NewReader(f) 25 | err := r.Loop(nil) 26 | if err == nil || !strings.Contains(err.Error(), "must not be nil") { 27 | t.Errorf("Unexpected error: %v", err) 28 | } 29 | } 30 | 31 | func TestReadNil(t *testing.T) { 32 | f := bytes.NewReader([]byte("")) 33 | r := NewReader(f) 34 | ok := r.Read(nil) 35 | if ok { 36 | t.Fatal("Read returned true unexpectedly") 37 | } 38 | if err := r.Done(); err == nil || !strings.Contains(err.Error(), "must not be nil") { 39 | t.Errorf("Unexpected eror: %v", err) 40 | } 41 | } 42 | 43 | type fakeCloser struct { 44 | reader io.Reader 45 | err error 46 | closed bool 47 | } 48 | 49 | func (c *fakeCloser) Close() error { 50 | c.closed = true 51 | return c.err 52 | } 53 | 54 | func (c *fakeCloser) Read(p []byte) (int, error) { 55 | return c.reader.Read(p) 56 | } 57 | 58 | func TestCloser(t *testing.T) { 59 | c := &fakeCloser{} 60 | r := NewReadCloser(c) 61 | if err := r.Done(); err != nil { 62 | t.Fatalf("Done failed: %v", err) 63 | } 64 | if !c.closed { 65 | t.Error("c is not closed") 66 | } 67 | } 68 | 69 | func TestCloserWithError(t *testing.T) { 70 | c := &fakeCloser{ 71 | reader: bytes.NewBufferString(""), 72 | } 73 | c.err = errors.New("Close Error") 74 | r := NewReadCloser(c) 75 | var unused []string 76 | if ok := r.Read(&unused); ok { 77 | t.Errorf("r.Read() must return false for a empty input") 78 | } 79 | if err := r.Done(); err != c.err { 80 | t.Errorf("Unexpected error: %v", err) 81 | } 82 | if !c.closed { 83 | t.Error("c is not closed") 84 | } 85 | } 86 | 87 | func TestCloserEOFAndError(t *testing.T) { 88 | c := &fakeCloser{} 89 | c.err = errors.New("Close Error") 90 | r := NewReadCloser(c) 91 | if err := r.Done(); err != c.err { 92 | t.Errorf("Unexpected error: %v", err) 93 | } 94 | if !c.closed { 95 | t.Error("c is not closed") 96 | } 97 | } 98 | 99 | func TestCloserDontOverwriteError(t *testing.T) { 100 | c := &fakeCloser{} 101 | c.err = errors.New("Close Error") 102 | r := NewReadCloser(c) 103 | anotherErr := errors.New("Another error") 104 | r.err = anotherErr 105 | if err := r.Done(); err != anotherErr { 106 | t.Errorf("Unexpected error: %v", err) 107 | } 108 | if !c.closed { 109 | t.Error("c is not closed") 110 | } 111 | } 112 | 113 | func TestNewReaderFile(t *testing.T) { 114 | r := NewReaderFile("testing/notexist.csv") 115 | ok := r.Read(nil) 116 | if ok { 117 | t.Error("r.Read returned true unexpectedly") 118 | } 119 | if err := r.Done(); err == nil || !strings.Contains(err.Error(), "no such file") { 120 | t.Errorf("Unexpected error: %v", err) 121 | } 122 | } 123 | 124 | func TestLoop(t *testing.T) { 125 | f := bytes.NewReader([]byte("10,1.2,alpha\n20,2.3,beta\n30,3.4,gamma")) 126 | r := NewReader(f) 127 | var ints []int 128 | var floats []float32 129 | var strs []string 130 | err := r.Loop(func(e struct { 131 | Int int `index:"0"` 132 | Float float32 `index:"1"` 133 | Str string `index:"2"` 134 | }) error { 135 | ints = append(ints, e.Int) 136 | floats = append(floats, e.Float) 137 | strs = append(strs, e.Str) 138 | return nil 139 | }) 140 | if err != nil { 141 | t.Fatalf("Loop failed: %v", err) 142 | } 143 | noDiff(t, "ints", ints, []int{10, 20, 30}) 144 | noDiff(t, "floats", floats, []float32{1.2, 2.3, 3.4}) 145 | noDiff(t, "strs", strs, []string{"alpha", "beta", "gamma"}) 146 | } 147 | 148 | func TestLoopPointer(t *testing.T) { 149 | f := bytes.NewReader([]byte("10,1.2\n20,2.3\n30,3.4")) 150 | r := NewReader(f) 151 | var ints []int 152 | var floats []float32 153 | err := r.Loop(func(e *struct { 154 | Int int `index:"0"` 155 | Float float32 `index:"1"` 156 | }) error { 157 | ints = append(ints, e.Int) 158 | floats = append(floats, e.Float) 159 | return nil 160 | }) 161 | if err != nil { 162 | t.Fatalf("Loop failed: %v", err) 163 | } 164 | wantInt := []int{10, 20, 30} 165 | wantFloat := []float32{1.2, 2.3, 3.4} 166 | noDiff(t, "ints", ints, wantInt) 167 | noDiff(t, "floats", floats, wantFloat) 168 | } 169 | 170 | func TestLoopWithName(t *testing.T) { 171 | f := bytes.NewReader([]byte("int,float,str\n10,1.2,alpha\n20,2.3,beta\n30,3.4,gamma")) 172 | r := NewReader(f) 173 | var ints []int 174 | var floats []float32 175 | var strs []string 176 | err := r.Loop(func(e struct { 177 | Int int `name:"int"` 178 | Float float32 `name:"float"` 179 | Str string `name:"str"` 180 | }) error { 181 | ints = append(ints, e.Int) 182 | floats = append(floats, e.Float) 183 | strs = append(strs, e.Str) 184 | return nil 185 | }) 186 | if err != nil { 187 | t.Fatalf("Loop failed: %v", err) 188 | } 189 | wantInt := []int{10, 20, 30} 190 | wantFloat := []float32{1.2, 2.3, 3.4} 191 | wantStr := []string{"alpha", "beta", "gamma"} 192 | noDiff(t, "ints", ints, wantInt) 193 | noDiff(t, "floats", floats, wantFloat) 194 | noDiff(t, "strs", strs, wantStr) 195 | } 196 | 197 | func TestLoopIndexOutOfRange(t *testing.T) { 198 | f := bytes.NewReader([]byte("10,1.2\n20,2.3")) 199 | r := NewReader(f) 200 | err := r.Loop(func(e struct { 201 | Int int `index:"0"` 202 | Float float32 `index:"2"` 203 | }) error { 204 | t.Error("The callback of Look is invoked unexpectedly") 205 | return nil 206 | }) 207 | if err == nil || err.Error() != "Accessed index 2 though the size of the row is 2" { 208 | t.Errorf("Unexpected error: %v", err) 209 | } 210 | } 211 | 212 | func TestLoopMissingColumn(t *testing.T) { 213 | f := bytes.NewReader([]byte("a,b\n10,1.2")) 214 | r := NewReader(f) 215 | err := r.Loop(func(e struct { 216 | Int int `name:"a"` 217 | Float float32 `name:"c"` 218 | }) error { 219 | t.Error("The callback of Look is invoked unexpectedly") 220 | return nil 221 | }) 222 | if err == nil || err.Error() != "c did not appear in the first line" { 223 | t.Errorf("Unexpected error: %v", err) 224 | } 225 | } 226 | 227 | func TestLoopWithSlice(t *testing.T) { 228 | f := bytes.NewReader([]byte("10,20\n30,40")) 229 | r := NewReader(f) 230 | var rows [][]int 231 | err := r.Loop(func(row []int) error { 232 | rows = append(rows, row) 233 | return nil 234 | }) 235 | if err != nil { 236 | t.Fatalf("Loop failed: %v", err) 237 | } 238 | want := [][]int{{10, 20}, {30, 40}} 239 | noDiff(t, "rows", rows, want) 240 | } 241 | 242 | func TestLoopBreak(t *testing.T) { 243 | f := bytes.NewReader([]byte("10,20\n30,40")) 244 | r := NewReader(f) 245 | var rows [][]int 246 | err := r.Loop(func(row []int) error { 247 | rows = append(rows, row) 248 | return Break 249 | }) 250 | if err != nil { 251 | t.Fatalf("Loop failed: %v", err) 252 | } 253 | want := [][]int{{10, 20}} 254 | noDiff(t, "rows", rows, want) 255 | } 256 | 257 | func TestLoopBreakWithError(t *testing.T) { 258 | f := bytes.NewReader([]byte("10,20\n30,40")) 259 | r := NewReader(f) 260 | e := errors.New("error") 261 | var rows [][]int 262 | err := r.Loop(func(row []int) error { 263 | rows = append(rows, row) 264 | return e 265 | }) 266 | if err != e { 267 | t.Fatalf("Loop returned an unexpected error: %v", err) 268 | } 269 | want := [][]int{{10, 20}} 270 | noDiff(t, "rows", rows, want) 271 | } 272 | 273 | func TestLoopNoReturn(t *testing.T) { 274 | r := NewReader(bytes.NewReader([]byte("10,20\n30,40"))) 275 | var rows [][]int 276 | err := r.Loop(func(row []int) { 277 | rows = append(rows, row) 278 | }) 279 | if err != nil { 280 | t.Fatalf("Loop failed: %v", err) 281 | } 282 | want := [][]int{{10, 20}, {30, 40}} 283 | noDiff(t, "rows", rows, want) 284 | } 285 | 286 | func TestLoopReturnBool(t *testing.T) { 287 | r := NewReader(bytes.NewReader([]byte("10,20\n30,40"))) 288 | var rows [][]int 289 | err := r.Loop(func(row []int) bool { 290 | rows = append(rows, row) 291 | return false 292 | }) 293 | if err != nil { 294 | t.Fatalf("Loop returned an unexpected error: %v", err) 295 | } 296 | want := [][]int{{10, 20}} 297 | noDiff(t, "rows", rows, want) 298 | } 299 | 300 | func TestRead(t *testing.T) { 301 | f := bytes.NewReader([]byte("10,1.2\n20,2.3\n30,3.4")) 302 | r := NewReader(f) 303 | var ints []int 304 | var floats []float32 305 | var e struct { 306 | Int int `index:"0"` 307 | Float float32 `index:"1"` 308 | } 309 | for r.Read(&e) { 310 | ints = append(ints, e.Int) 311 | floats = append(floats, e.Float) 312 | } 313 | if err := r.Done(); err != nil { 314 | t.Fatalf("Failed to read: %v", err) 315 | } 316 | wantInt := []int{10, 20, 30} 317 | wantFloat := []float32{1.2, 2.3, 3.4} 318 | noDiff(t, "ints", ints, wantInt) 319 | noDiff(t, "floats", floats, wantFloat) 320 | } 321 | 322 | func TestReadWithName(t *testing.T) { 323 | f := bytes.NewReader([]byte("a,b\n10,1.2\n20,2.3\n30,3.4")) 324 | r := NewReader(f) 325 | var ints []int 326 | var floats []float32 327 | var e struct { 328 | Int int `name:"a"` 329 | Float float32 `name:"b"` 330 | } 331 | for r.Read(&e) { 332 | ints = append(ints, e.Int) 333 | floats = append(floats, e.Float) 334 | } 335 | if err := r.Done(); err != nil { 336 | t.Fatalf("Failed to read: %v", err) 337 | } 338 | wantInt := []int{10, 20, 30} 339 | wantFloat := []float32{1.2, 2.3, 3.4} 340 | noDiff(t, "ints", ints, wantInt) 341 | noDiff(t, "floats", floats, wantFloat) 342 | } 343 | 344 | func TestReadIndexOutOfRange(t *testing.T) { 345 | f := bytes.NewReader([]byte("10,1.2\n20,2.3")) 346 | r := NewReader(f) 347 | var e struct { 348 | Int int `index:"0"` 349 | Float float32 `index:"2"` 350 | } 351 | for r.Read(&e) { 352 | t.Errorf("r.Read returned true unexpectedly with %#v", e) 353 | } 354 | err := r.Done() 355 | if err == nil || err.Error() != "Accessed index 2 though the size of the row is 2" { 356 | t.Errorf("Unexpected error: %v", err) 357 | } 358 | } 359 | 360 | func TestReadMissingColumn(t *testing.T) { 361 | f := bytes.NewReader([]byte("a,c\n10,1.2")) 362 | r := NewReader(f) 363 | var e struct { 364 | Int int `name:"a"` 365 | Float float32 `name:"b"` 366 | } 367 | for r.Read(&e) { 368 | t.Errorf("r.Read returned true unexpectedly with %#v", e) 369 | } 370 | err := r.Done() 371 | if err == nil || err.Error() != "b did not appear in the first line" { 372 | t.Errorf("Unexpected error: %v", err) 373 | } 374 | } 375 | 376 | func TestReadExtraColumn(t *testing.T) { 377 | f := bytes.NewReader([]byte("a,b,c\n10,1.2")) 378 | r := NewReader(f) 379 | var e struct { 380 | Int int `name:"a"` 381 | Float float32 `name:"b"` 382 | String string `name:"c"` 383 | } 384 | for r.Read(&e) { 385 | t.Errorf("r.Read returned true unexpectedly with %#v", e) 386 | } 387 | err := r.Done() 388 | if err == nil || err.Error() != "record on line 2: wrong number of fields" { 389 | t.Errorf("Unexpected error: %v", err) 390 | } 391 | 392 | f = bytes.NewReader([]byte("a,b,c\n20,2.4")) 393 | r = NewReader(f, Option{ 394 | FieldsPerRecord: -1, 395 | }) 396 | for r.Read(&e) { 397 | // read is successful in this case 398 | } 399 | err = r.Done() 400 | if err != nil { 401 | t.Errorf("Unexpected error: %v", err) 402 | } 403 | noDiff(t, "ints", []int{e.Int}, []int{20}) 404 | noDiff(t, "floats", []float32{e.Float}, []float32{2.4}) 405 | noDiff(t, "ints", []string{e.String}, []string{""}) 406 | } 407 | 408 | func TestReadWithSlice(t *testing.T) { 409 | f := bytes.NewReader([]byte("10,20\n30,40")) 410 | r := NewReader(f) 411 | var rows [][]int 412 | var row []int 413 | for r.Read(&row) { 414 | rows = append(rows, row) 415 | } 416 | if err := r.Done(); err != nil { 417 | t.Fatalf("Failed to read: %v", err) 418 | } 419 | want := [][]int{{10, 20}, {30, 40}} 420 | noDiff(t, "rows", rows, want) 421 | } 422 | 423 | func TestReadAllStruct(t *testing.T) { 424 | f := bytes.NewReader([]byte("10,2.3\n30,4.5")) 425 | r := NewReader(f) 426 | type entry struct { 427 | Int int `index:"0"` 428 | Float float32 `index:"1"` 429 | } 430 | var s []entry 431 | if err := r.ReadAll(&s); err != nil { 432 | t.Fatalf("ReadAll failed: %v", err) 433 | } 434 | want := []entry{{Int: 10, Float: 2.3}, {Int: 30, Float: 4.5}} 435 | noDiff(t, "s", s, want) 436 | } 437 | 438 | func TestReadAllSlice(t *testing.T) { 439 | f := bytes.NewReader([]byte("10,20\n30,40")) 440 | r := NewReader(f) 441 | var s [][]int 442 | if err := r.ReadAll(&s); err != nil { 443 | t.Fatalf("ReadAll failed: %v", err) 444 | } 445 | want := [][]int{{10, 20}, {30, 40}} 446 | noDiff(t, "s", s, want) 447 | } 448 | 449 | func TestEncTag(t *testing.T) { 450 | f := bytes.NewReader([]byte("10,10,010\n20,20,020")) 451 | r := NewReader(f) 452 | var ints0 []int 453 | var ints1 []int 454 | var ints2 []int 455 | var e struct { 456 | Int0 int `index:"0" enc:"hex"` 457 | Int1 int `index:"1" enc:"oct"` 458 | Int2 int `index:"2" enc:"deci"` 459 | } 460 | for r.Read(&e) { 461 | ints0 = append(ints0, e.Int0) 462 | ints1 = append(ints1, e.Int1) 463 | ints2 = append(ints2, e.Int2) 464 | } 465 | if err := r.Done(); err != nil { 466 | t.Fatalf("Failed to read: %v", err) 467 | } 468 | wantInt0 := []int{16, 32} 469 | wantInt1 := []int{8, 16} 470 | wantInt2 := []int{10, 20} 471 | noDiff(t, "ints0", ints0, wantInt0) 472 | noDiff(t, "ints1", ints1, wantInt1) 473 | noDiff(t, "ints2", ints2, wantInt2) 474 | } 475 | 476 | func TestNewDecoder(t *testing.T) { 477 | d, err := newDecoder(Option{}, reflect.TypeOf(struct { 478 | Name int `name:"name"` 479 | Age int `name:"age"` 480 | }{})) 481 | if err != nil { 482 | t.Fatalf("Failed to create a decoder: %v", err) 483 | } 484 | if !d.needHeader() { 485 | t.Error("needHeader() returned false") 486 | } 487 | } 488 | 489 | func TestNewDecoder_IndexAndName(t *testing.T) { 490 | _, err := newDecoder(Option{}, reflect.TypeOf(struct { 491 | Name int `name:"name"` 492 | Age int `index:"0"` 493 | }{})) 494 | if err == nil || err.Error() != "Fields with name and fields with index are mixed" { 495 | t.Errorf("Unexpected error: %v", err) 496 | } 497 | } 498 | 499 | func TestNewDecoder_NoStructTag(t *testing.T) { 500 | _, err := newDecoder(Option{}, reflect.TypeOf(struct { 501 | Name int 502 | Age int 503 | }{})) 504 | if err == nil || err.Error() != "Please specify name or index to the struct field: Name\nPlease specify name or index to the struct field: Age" { 505 | t.Errorf("Unexpected error: %v", err) 506 | } 507 | } 508 | 509 | func TestNewDecoder_InvalidIndex(t *testing.T) { 510 | _, err := newDecoder(Option{}, reflect.TypeOf(struct { 511 | Name int `index:"-1"` 512 | Age int `index:"hello"` 513 | }{})) 514 | if err == nil || err.Error() != "Failed to parse index of field Name: \"-1\"\nFailed to parse index of field Age: \"hello\"" { 515 | t.Errorf("Unexpected error: %q", err) 516 | } 517 | } 518 | 519 | func TestLineNumber(t *testing.T) { 520 | f := bytes.NewReader([]byte("10,1.2\n20,2.3\n30,3.4")) 521 | r := NewReader(f) 522 | var ints []int 523 | var lineno []int 524 | err := r.Loop(func(e struct { 525 | Int int `index:"0"` 526 | }) error { 527 | ints = append(ints, e.Int) 528 | lineno = append(lineno, r.LineNumber()) 529 | return nil 530 | }) 531 | if err != nil { 532 | t.Fatalf("Loop failed: %v", err) 533 | } 534 | wantInt := []int{10, 20, 30} 535 | wantLineno := []int{1, 2, 3} 536 | noDiff(t, "ints", ints, wantInt) 537 | noDiff(t, "lineno", lineno, wantLineno) 538 | } 539 | -------------------------------------------------------------------------------- /encode.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strconv" 7 | ) 8 | 9 | var predefinedDecoders = map[string]func(t reflect.Type) interface{}{ 10 | "hex": func(t reflect.Type) interface{} { 11 | return createIntConverter(t, 16) 12 | }, 13 | "oct": func(t reflect.Type) interface{} { 14 | return createIntConverter(t, 8) 15 | }, 16 | "deci": func(t reflect.Type) interface{} { 17 | return createIntConverter(t, 10) 18 | }, 19 | } 20 | 21 | func createIntConverter(t reflect.Type, base int) interface{} { 22 | switch t.Kind() { 23 | case reflect.Int: 24 | return func(s string) (int, error) { 25 | i, err := strconv.ParseInt(s, base, 0) 26 | return int(i), err 27 | } 28 | case reflect.Int8: 29 | return func(s string) (int8, error) { 30 | i, err := strconv.ParseInt(s, base, 8) 31 | return int8(i), err 32 | } 33 | case reflect.Int16: 34 | return func(s string) (int16, error) { 35 | i, err := strconv.ParseInt(s, base, 16) 36 | return int16(i), err 37 | } 38 | case reflect.Int32: 39 | return func(s string) (int32, error) { 40 | i, err := strconv.ParseInt(s, base, 32) 41 | return int32(i), err 42 | } 43 | case reflect.Int64: 44 | return func(s string) (int64, error) { 45 | i, err := strconv.ParseInt(s, base, 64) 46 | return int64(i), err 47 | } 48 | case reflect.Uint: 49 | return func(s string) (uint, error) { 50 | i, err := strconv.ParseUint(s, base, 0) 51 | return uint(i), err 52 | } 53 | case reflect.Uint8: 54 | return func(s string) (uint8, error) { 55 | i, err := strconv.ParseUint(s, base, 8) 56 | return uint8(i), err 57 | } 58 | case reflect.Uint16: 59 | return func(s string) (uint16, error) { 60 | i, err := strconv.ParseUint(s, base, 16) 61 | return uint16(i), err 62 | } 63 | case reflect.Uint32: 64 | return func(s string) (uint32, error) { 65 | i, err := strconv.ParseUint(s, base, 32) 66 | return uint32(i), err 67 | } 68 | case reflect.Uint64: 69 | return func(s string) (uint64, error) { 70 | i, err := strconv.ParseUint(s, base, 32) 71 | return uint64(i), err 72 | } 73 | default: 74 | return nil 75 | } 76 | } 77 | 78 | func validateTypeDecoder(t reflect.Type, conv interface{}) error { 79 | convT := reflect.TypeOf(conv) 80 | if convT.Kind() != reflect.Func { 81 | return fmt.Errorf("The decoder for %v must be a function but %v", t, convT) 82 | } 83 | if convT.NumIn() != 1 || convT.NumOut() != 2 { 84 | return fmt.Errorf("The decoder for %v must receive one arguments and returns two values", t) 85 | } 86 | if convT.In(0).Kind() != reflect.String { 87 | return fmt.Errorf("The decoder for %v must receive a string as the first arg, but receives %v", t, convT.In(0)) 88 | } 89 | if convT.Out(0) != t || convT.Out(1) != errorType { 90 | return fmt.Errorf("The decoder for %v must return (%v, error), but returned (%v, %v)", 91 | t, t, convT.Out(0), convT.Out(1)) 92 | } 93 | return nil 94 | } 95 | 96 | func createConverterFromType(opt Option, t reflect.Type) (interface{}, error) { 97 | if opt.TypeDecoders != nil { 98 | if conv, ok := opt.TypeDecoders[t]; ok { 99 | if err := validateTypeDecoder(t, conv); err != nil { 100 | return nil, err 101 | } 102 | return conv, nil 103 | } 104 | } 105 | return createDefaultConverter(t), nil 106 | } 107 | 108 | func createDefaultConverter(t reflect.Type) interface{} { 109 | c := createIntConverter(t, 0) 110 | if c != nil { 111 | return c 112 | } 113 | switch t.Kind() { 114 | case reflect.Float32: 115 | return func(s string) (float32, error) { 116 | f, err := strconv.ParseFloat(s, 32) 117 | return float32(f), err 118 | } 119 | case reflect.Float64: 120 | return func(s string) (float64, error) { 121 | f, err := strconv.ParseFloat(s, 64) 122 | return float64(f), err 123 | } 124 | case reflect.Bool: 125 | return strconv.ParseBool 126 | case reflect.String: 127 | return func(s string) (string, error) { 128 | return s, nil 129 | } 130 | default: 131 | return nil 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /encode_test.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | "testing" 7 | ) 8 | 9 | func TestConverterInt(t *testing.T) { 10 | r := NewReader(bytes.NewBufferString("10,0xff,017")) 11 | var got []int 12 | ok := r.Read(&got) 13 | if !ok { 14 | t.Fatal("Read returned false unexpectedly") 15 | } 16 | want := []int{10, 255, 15} 17 | noDiff(t, "Read()", got, want) 18 | } 19 | 20 | func TestConverterInvalidWithSlice(t *testing.T) { 21 | r := NewReader(bytes.NewBufferString("hello")) 22 | var row []int 23 | ok := r.Read(&row) 24 | if ok { 25 | t.Error("Read returned true unexpectedly") 26 | } 27 | if err := r.Done(); err == nil || !strings.Contains(err.Error(), "parsing \"hello\"") { 28 | t.Errorf("Unexpected error: %v", err) 29 | } 30 | } 31 | 32 | func TestConverterInvalidWithStruct(t *testing.T) { 33 | r := NewReader(bytes.NewBufferString("hello")) 34 | var row struct { 35 | Int int `index:"0"` 36 | } 37 | ok := r.Read(&row) 38 | if ok { 39 | t.Error("Read returned true unexpectedly") 40 | } 41 | if err := r.Done(); err == nil || !strings.Contains(err.Error(), "parsing \"hello\"") { 42 | t.Errorf("Unexpected error: %v", err) 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /example_test.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "log" 7 | "reflect" 8 | "time" 9 | ) 10 | 11 | func ExampleReader_read() { 12 | r := NewReaderFile("testdata/sample.csv") 13 | var entry struct { 14 | Name string `index:"0"` 15 | Age int `index:"1"` 16 | } 17 | for r.Read(&entry) { 18 | fmt.Print(entry) 19 | } 20 | if err := r.Done(); err != nil { 21 | log.Fatalf("Failed to read a CSV file: %v", err) 22 | } 23 | // Output: {Alice 10}{Bob 20} 24 | } 25 | 26 | func ExampleReader_loop() { 27 | r := NewReaderFile("testdata/sample.csv") 28 | err := r.Loop(func(entry *struct { 29 | Name string `index:"0"` 30 | Age int `index:"1"` 31 | }) error { 32 | fmt.Print(entry) 33 | return nil 34 | }) 35 | if err != nil { 36 | log.Fatalf("Failed to read a CSV file: %v", err) 37 | } 38 | // Output: &{Alice 10}&{Bob 20} 39 | } 40 | 41 | func ExampleReader_readAll() { 42 | r := NewReaderFile("testdata/sample.csv") 43 | var entry []struct { 44 | Name string `index:"0"` 45 | Age int `index:"1"` 46 | } 47 | if err := r.ReadAll(&entry); err != nil { 48 | log.Fatalf("Failed to read a CSV file: %v", err) 49 | } 50 | fmt.Println(entry) 51 | // Output: [{Alice 10} {Bob 20}] 52 | } 53 | 54 | func ExampleReader_tSV() { 55 | r := NewReaderFile("testdata/sample.tsv", Option{ 56 | Comma: '\t', 57 | }) 58 | var entry struct { 59 | Name string `index:"0"` 60 | Age int `index:"1"` 61 | } 62 | for r.Read(&entry) { 63 | fmt.Print(entry) 64 | } 65 | if err := r.Done(); err != nil { 66 | log.Fatalf("Failed to read a CSV file: %v", err) 67 | } 68 | // Output: {Alice 10}{Bob 20} 69 | } 70 | 71 | func ExampleReader_encodings() { 72 | r := NewReader(bytes.NewReader([]byte("010,010,010"))) 73 | var entry struct { 74 | Deci int `index:"0" enc:"deci"` 75 | Oct int `index:"1" enc:"oct"` 76 | Hex int `index:"2" enc:"hex"` 77 | } 78 | for r.Read(&entry) { 79 | fmt.Print(entry) 80 | } 81 | if err := r.Done(); err != nil { 82 | log.Fatalf("Failed to read: %v", err) 83 | } 84 | // Output: {10 8 16} 85 | } 86 | 87 | func ExampleReader_typeEncodings() { 88 | r := NewReader(bytes.NewReader([]byte("2017-01-02,2016-02-03\n2015-03-04,2014-04-05")), 89 | Option{TypeDecoders: map[reflect.Type]interface{}{ 90 | reflect.TypeOf(time.Time{}): func(s string) (time.Time, error) { 91 | return time.Parse("2006-01-02", s) 92 | }, 93 | }}) 94 | var entry []time.Time 95 | for r.Read(&entry) { 96 | for _, e := range entry { 97 | fmt.Print(e.Format("2006/1/2"), ";") 98 | } 99 | } 100 | if err := r.Done(); err != nil { 101 | log.Fatalf("Failed to read: %v", err) 102 | } 103 | // Output: 2017/1/2;2016/2/3;2015/3/4;2014/4/5; 104 | } 105 | 106 | func ExampleReader_LineNumber_reader() { 107 | r := NewReaderFile("testdata/sample.csv") 108 | var entry struct { 109 | Name string `index:"0"` 110 | Age int `index:"1"` 111 | } 112 | bob := "Bob" 113 | lino := 0 114 | for r.Read(&entry) { 115 | if entry.Name == bob { 116 | lino = r.LineNumber() 117 | } 118 | } 119 | if lino > 0 { 120 | fmt.Printf("Found %s at line %d", bob, lino) 121 | } 122 | // Output: Found Bob at line 2 123 | } 124 | 125 | func ExampleReader_DoneDefer() { 126 | f := func() (err error) { 127 | r := NewReaderFile("testdata/sample.csv") 128 | defer r.DoneDefer(&err) 129 | var entry struct { 130 | Name string `index:"3"` 131 | } 132 | // This fails with a index-out-of-range error. 133 | for r.Read(&entry) { 134 | err = fmt.Errorf("This block must not be executed") 135 | } 136 | return 137 | } 138 | err := f() 139 | if err != nil { 140 | fmt.Printf("Failed: %v", err) 141 | } 142 | // Output: Failed: Accessed index 3 though the size of the row is 2 143 | } 144 | 145 | func ExampleOption_decoders() { 146 | r := NewReader(bytes.NewBufferString("name,birthday\nAlice,1980-12-30\nBob,1975-06-09"), 147 | Option{ 148 | Decoders: map[string]interface{}{ 149 | "date": func(s string) (time.Time, error) { 150 | return time.Parse("2006-01-02", s) 151 | }, 152 | }, 153 | }) 154 | var entry struct { 155 | Name string `name:"name"` 156 | Birth time.Time `name:"birthday" enc:"date"` 157 | } 158 | for r.Read(&entry) { 159 | fmt.Print(entry) 160 | } 161 | if err := r.Done(); err != nil { 162 | log.Fatalf("Failed: %v\n", err) 163 | } 164 | // Output: {Alice 1980-12-30 00:00:00 +0000 UTC}{Bob 1975-06-09 00:00:00 +0000 UTC} 165 | } 166 | -------------------------------------------------------------------------------- /examples/example.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": null, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import (\n", 10 | " \"fmt\"\n", 11 | " \"log\"\n", 12 | "\n", 13 | " \"github.com/yunabe/easycsv\"\n", 14 | ")" 15 | ] 16 | }, 17 | { 18 | "cell_type": "markdown", 19 | "metadata": {}, 20 | "source": [ 21 | "# Read\n", 22 | "[`Read`](https://godoc.org/github.com/yunabe/easycsv#Reader.Read) with `index` field tags." 23 | ] 24 | }, 25 | { 26 | "cell_type": "code", 27 | "execution_count": null, 28 | "metadata": {}, 29 | "outputs": [ 30 | { 31 | "name": "stdout", 32 | "output_type": "stream", 33 | "text": [ 34 | "entry: name = alice, age = 12\n", 35 | "entry: name = bob, age = 34\n" 36 | ] 37 | } 38 | ], 39 | "source": [ 40 | "{\n", 41 | " r := easycsv.NewReaderFile(\"noheader.csv\")\n", 42 | " var entry struct {\n", 43 | " Name string `index:\"0\"`\n", 44 | " Age int64 `index:\"1\"`\n", 45 | " }\n", 46 | " for r.Read(&entry) {\n", 47 | " fmt.Printf(\"entry: name = %s, age = %d\\n\", entry.Name, entry.Age)\n", 48 | " }\n", 49 | " if err := r.Done(); err != nil {\n", 50 | " log.Printf(\"failed to read: %v\", err)\n", 51 | " }\n", 52 | "}" 53 | ] 54 | }, 55 | { 56 | "cell_type": "markdown", 57 | "metadata": {}, 58 | "source": [ 59 | "[`Loop`](https://godoc.org/github.com/yunabe/easycsv#Reader.Loop) with `name` field tags." 60 | ] 61 | }, 62 | { 63 | "cell_type": "code", 64 | "execution_count": null, 65 | "metadata": {}, 66 | "outputs": [ 67 | { 68 | "name": "stdout", 69 | "output_type": "stream", 70 | "text": [ 71 | "entry: name = alice, age = 12\n", 72 | "entry: name = bob, age = 34\n" 73 | ] 74 | } 75 | ], 76 | "source": [ 77 | "{\n", 78 | " r := easycsv.NewReaderFile(\"withheader.csv\")\n", 79 | " err := r.Loop(func(entry struct {\n", 80 | " Name string `name:\"name\"`\n", 81 | " Age int64 `name:\"age\"`\n", 82 | " }) {\n", 83 | " fmt.Printf(\"entry: name = %s, age = %d\\n\", entry.Name, entry.Age)\n", 84 | " })\n", 85 | " if err != nil {\n", 86 | " log.Printf(\"failed to read: %v\", err)\n", 87 | " }\n", 88 | "}" 89 | ] 90 | } 91 | ], 92 | "metadata": { 93 | "kernelspec": { 94 | "display_name": "Go (lgo)", 95 | "language": "go", 96 | "name": "lgo" 97 | }, 98 | "language_info": { 99 | "file_extension": "", 100 | "mimetype": "", 101 | "name": "go", 102 | "version": "" 103 | } 104 | }, 105 | "nbformat": 4, 106 | "nbformat_minor": 2 107 | } 108 | -------------------------------------------------------------------------------- /examples/noheader.csv: -------------------------------------------------------------------------------- 1 | alice,12 2 | bob,34 3 | -------------------------------------------------------------------------------- /examples/withheader.csv: -------------------------------------------------------------------------------- 1 | name,age 2 | alice,12 3 | bob,34 4 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/yunabe/easycsv 2 | 3 | go 1.14 4 | 5 | require github.com/google/go-cmp v0.4.0 6 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= 2 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 3 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 4 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 5 | -------------------------------------------------------------------------------- /issues_test.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestIssue1Fixed(t *testing.T) { 9 | // https://github.com/yunabe/easycsv/issues/1 10 | r := NewReaderFile("testdata/issue1.csv", Option{ 11 | Decoders: map[string]interface{}{ 12 | "date": func(s string) (time.Time, error) { 13 | return time.Parse("2006-01-02", s) 14 | }, 15 | }, 16 | }) 17 | var entry struct { 18 | Name string `name:"name"` 19 | Birth time.Time `name:"birthday" enc:"date"` 20 | } 21 | var names []string 22 | var births []string 23 | for r.Read(&entry) { 24 | names = append(names, entry.Name) 25 | births = append(births, entry.Birth.Format("2006/01/02")) 26 | } 27 | if err := r.Done(); err != nil { 28 | t.Fatalf("Failed to read: %v", err) 29 | } 30 | wantNames := []string{"Alice", "Bob"} 31 | wantBirths := []string{"1980/12/30", "1975/06/09"} 32 | noDiff(t, "names", names, wantNames) 33 | noDiff(t, "births", births, wantBirths) 34 | } 35 | -------------------------------------------------------------------------------- /option.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "errors" 5 | "reflect" 6 | ) 7 | 8 | // Option specifies the spec of Reader. 9 | type Option struct { 10 | // Comma is the field delimiter. 11 | // For exampoe, if '\t' is set to Comma, Reader reads files as TSV files. 12 | Comma rune 13 | // Comment, if not 0, is the comment character. Lines beginning with the character without preceding whitespace are ignored. 14 | Comment rune 15 | // Allow lazy parsing of quotes, default to false 16 | LazyQuotes bool 17 | // If negative, Reader does not check the number of fields per record 18 | // If 0, this option does not update Reader 19 | // If positive, Reader requires all records to have this number of fields 20 | FieldsPerRecord int 21 | // Decoders is the map to define custom encodings. 22 | Decoders map[string]interface{} 23 | // Custom decoders to parse specific types. 24 | TypeDecoders map[reflect.Type]interface{} 25 | 26 | // TODO: Support AutoIndex 27 | AutoIndex bool 28 | // TODO: Support AutoName 29 | AutoName bool 30 | } 31 | 32 | func (a *Option) mergeOption(b Option) { 33 | if b.Comma != 0 { 34 | a.Comma = b.Comma 35 | } 36 | if b.Comment != 0 { 37 | a.Comment = b.Comment 38 | } 39 | if b.AutoIndex { 40 | a.AutoIndex = true 41 | } 42 | if b.AutoName { 43 | a.AutoName = true 44 | } 45 | if b.LazyQuotes { 46 | a.LazyQuotes = b.LazyQuotes 47 | } 48 | if b.FieldsPerRecord != 0 { 49 | a.FieldsPerRecord = b.FieldsPerRecord 50 | } 51 | if b.Decoders != nil { 52 | if a.Decoders == nil { 53 | a.Decoders = make(map[string]interface{}) 54 | } 55 | for name, dec := range b.Decoders { 56 | a.Decoders[name] = dec 57 | } 58 | } 59 | if b.TypeDecoders != nil { 60 | if a.TypeDecoders == nil { 61 | a.TypeDecoders = make(map[reflect.Type]interface{}) 62 | } 63 | for t, dec := range b.TypeDecoders { 64 | a.TypeDecoders[t] = dec 65 | } 66 | } 67 | } 68 | 69 | func (a *Option) validate() error { 70 | if a.AutoIndex && a.AutoName { 71 | return errors.New("You can not set both AutoIndex and AutoName to easycsv.Reader.") 72 | } 73 | return nil 74 | } 75 | 76 | func mergeOptions(opts []Option) (Option, error) { 77 | var opt Option 78 | for _, o := range opts { 79 | opt.mergeOption(o) 80 | } 81 | return opt, opt.validate() 82 | } 83 | -------------------------------------------------------------------------------- /option_test.go: -------------------------------------------------------------------------------- 1 | package easycsv 2 | 3 | import ( 4 | "bytes" 5 | "reflect" 6 | "strings" 7 | "testing" 8 | "time" 9 | 10 | "github.com/google/go-cmp/cmp" 11 | ) 12 | 13 | func TestReadTSV(t *testing.T) { 14 | f := bytes.NewBufferString("1\t2\n3\t4\n") 15 | r := NewReader(f, Option{ 16 | Comma: '\t', 17 | }) 18 | var got [][]int 19 | if err := r.ReadAll(&got); err != nil { 20 | t.Fatalf("Failed to read: %v", err) 21 | } 22 | want := [][]int{{1, 2}, {3, 4}} 23 | noDiff(t, "ReadAll() with tsv", got, want) 24 | } 25 | 26 | func TestSkipComment(t *testing.T) { 27 | f := bytes.NewBufferString("1,2\n#3,4\n5,6") 28 | r := NewReader(f, Option{ 29 | Comment: '#', 30 | }) 31 | var got [][]int 32 | if err := r.ReadAll(&got); err != nil { 33 | t.Fatalf("Failed to read: %v", err) 34 | } 35 | want := [][]int{{1, 2}, {5, 6}} 36 | noDiff(t, "ReadAll() with Comment", got, want) 37 | } 38 | 39 | func TestLazyQuotes(t *testing.T) { 40 | f := bytes.NewBufferString("1,2,3,\"\"4\",5") 41 | r := NewReader(f, Option{ 42 | LazyQuotes: true, 43 | Comment: '#', 44 | }) 45 | var got [][]string 46 | if err := r.ReadAll(&got); err != nil { 47 | t.Fatalf("Failed to read: %v", err) 48 | } 49 | want := [][]string{{"1", "2", "3", "\"4", "5"}} 50 | noDiff(t, "ReadAll() with LazyQuotes", got, want) 51 | } 52 | 53 | func TestFieldsPerRecord(t *testing.T) { 54 | csvContents := "1,2,3,4,5\n6,7,8" 55 | f := bytes.NewBufferString(csvContents) 56 | r := NewReader(f) 57 | var got [][]int 58 | if err := r.ReadAll(&got); err == nil { 59 | t.Fatal("Should have failed to read with error 'record on line 2: wrong number of fields'") 60 | } 61 | 62 | f = bytes.NewBufferString(csvContents) 63 | r = NewReader(f, Option{ 64 | FieldsPerRecord: -1, 65 | }) 66 | got = [][]int{} 67 | if err := r.ReadAll(&got); err != nil { 68 | t.Fatalf("Failed to read: %v", err) 69 | } 70 | want := [][]int{{1, 2, 3, 4, 5}, {6, 7, 8}} 71 | noDiff(t, "ReadAll() with FieldsPerRecord", got, want) 72 | } 73 | 74 | func TestOptionWithNewReadCloser(t *testing.T) { 75 | f := &fakeCloser{ 76 | reader: bytes.NewBufferString("1\t2\n3\t4\n"), 77 | } 78 | r := NewReadCloser(f, Option{ 79 | Comma: '\t', 80 | }) 81 | var got [][]int 82 | if err := r.ReadAll(&got); err != nil { 83 | t.Fatalf("Failed to read: %v", err) 84 | } 85 | want := [][]int{{1, 2}, {3, 4}} 86 | noDiff(t, "results", got, want) 87 | if !f.closed { 88 | t.Error("f is not closed") 89 | } 90 | } 91 | 92 | func TestCustomDecoder(t *testing.T) { 93 | f := bytes.NewBufferString("hello,2010-11-12\nworld,2012-01-02") 94 | r := NewReader(f, Option{ 95 | Decoders: map[string]interface{}{ 96 | "custom": func(s string) (string, error) { return "[" + s + "]", nil }, 97 | "date": func(s string) (time.Time, error) { 98 | return time.Parse("2006-01-02", s) 99 | }, 100 | }, 101 | }) 102 | var msgs []string 103 | var dates []string 104 | err := r.Loop(func(e struct { 105 | Msg string `index:"0" enc:"custom"` 106 | Date time.Time `index:"1" enc:"date"` 107 | }) error { 108 | msgs = append(msgs, e.Msg) 109 | dates = append(dates, e.Date.Format("2006/1/2")) 110 | return nil 111 | }) 112 | if err != nil { 113 | t.Fatalf("Loop failed: %v", err) 114 | } 115 | msgsWant := []string{"[hello]", "[world]"} 116 | datesWant := []string{"2010/11/12", "2012/1/2"} 117 | if diff := cmp.Diff(msgsWant, msgs); diff != "" { 118 | t.Errorf("mismatch of msgs (-want +got):\n%s", diff) 119 | } 120 | if diff := cmp.Diff(datesWant, dates); diff != "" { 121 | t.Errorf("mismatch of dates (-want +got):\n%s", diff) 122 | } 123 | } 124 | 125 | func TestCustomDecoder_wrongType(t *testing.T) { 126 | f := bytes.NewBufferString("hello,2010-11-12\nworld,2012-01-02") 127 | r := NewReader(f, Option{ 128 | Decoders: map[string]interface{}{ 129 | "enc0": nil, 130 | "enc1": 10, 131 | "enc2": func() {}, 132 | "enc3": func(n int) (float32, bool) { return 1.0, true }, 133 | }, 134 | }) 135 | err := r.Loop(func(e struct { 136 | F0 string `index:"0" enc:"enc0"` 137 | F1 string `index:"0" enc:"enc1"` 138 | F2 string `index:"0" enc:"enc2"` 139 | F3 string `index:"0" enc:"enc3"` 140 | }) error { 141 | t.Error("Loop read an entry unexpectedly") 142 | return nil 143 | }) 144 | if err == nil { 145 | t.Errorf("Loop above must fail") 146 | } 147 | expectedErrors := []string{ 148 | "Encoding \"enc0\" is not defined", 149 | "The custom decoder for Encoding \"enc1\" must be a function", 150 | "The custom decoder for Encoding \"enc2\" must receive an arg, but receives 0 args", 151 | "The custom decoder for Encoding \"enc2\" must return two values, but returns 0 values", 152 | "The custom decoder for Encoding \"enc3\" must receive a string, but receives int", 153 | "The type of field \"F3\" is string, but enc \"enc3\" returns \"float32\"", 154 | "The second return value of the custom decoder for \"enc3\" must be error", 155 | } 156 | if err.Error() != strings.Join(expectedErrors, "\n") { 157 | t.Errorf("Unexpected errors: %v", err) 158 | } 159 | } 160 | 161 | func TestTypeDecoders(t *testing.T) { 162 | f := bytes.NewBufferString("2013-01-02,2010-11-12\n2015-11-19,2012-01-02") 163 | r := NewReader(f, Option{ 164 | TypeDecoders: map[reflect.Type]interface{}{ 165 | reflect.TypeOf(time.Time{}): func(s string) (time.Time, error) { 166 | return time.Parse("2006-01-02", s) 167 | }, 168 | }, 169 | }) 170 | var entry struct { 171 | Date0 time.Time `index:"0"` 172 | Date1 time.Time `index:"1"` 173 | } 174 | var all []string 175 | for r.Read(&entry) { 176 | all = append(all, entry.Date0.Format("2006/1/2")) 177 | all = append(all, entry.Date1.Format("Jan 2, 2006")) 178 | } 179 | if err := r.Done(); err != nil { 180 | t.Fatalf("Failed to Done: %v", err) 181 | } 182 | want := []string{"2013/1/2", "Nov 12, 2010", "2015/11/19", "Jan 2, 2012"} 183 | noDiff(t, "all", all, want) 184 | } 185 | 186 | func TestTypeDecodersWithSlice(t *testing.T) { 187 | f := bytes.NewBufferString("2013-01-02,2010-11-12\n2015-11-19,2012-01-02") 188 | r := NewReader(f, Option{ 189 | TypeDecoders: map[reflect.Type]interface{}{ 190 | reflect.TypeOf(time.Time{}): func(s string) (time.Time, error) { 191 | return time.Parse("2006-01-02", s) 192 | }, 193 | }, 194 | }) 195 | var row []time.Time 196 | var all []string 197 | for r.Read(&row) { 198 | for _, e := range row { 199 | all = append(all, e.Format("2006/1/2")) 200 | } 201 | } 202 | if err := r.Done(); err != nil { 203 | t.Fatalf("Failed to Done: %v", err) 204 | } 205 | want := []string{"2013/1/2", "2010/11/12", "2015/11/19", "2012/1/2"} 206 | noDiff(t, "all", all, want) 207 | } 208 | 209 | func TestTypeDecodersErrors(t *testing.T) { 210 | tests := []struct { 211 | decoder interface{} 212 | suberr string 213 | }{ 214 | { 215 | decoder: "decoder", 216 | suberr: "must be a function but string", 217 | }, { 218 | decoder: func(s string) (int, error) { 219 | return 0, nil 220 | }, 221 | suberr: "but returned (int, error)", 222 | }, { 223 | decoder: func(s string) time.Time { 224 | return time.Now() 225 | }, 226 | suberr: "must receive one arguments and returns two values", 227 | }, { 228 | decoder: func(i int) (time.Time, error) { 229 | return time.Now(), nil 230 | }, 231 | suberr: "must receive a string as the first arg, but receives int", 232 | }, 233 | } 234 | for _, test := range tests { 235 | f := bytes.NewBufferString("2013-01-02,2010-11-12\n2015-11-19,2012-01-02") 236 | r := NewReader(f, Option{ 237 | TypeDecoders: map[reflect.Type]interface{}{ 238 | reflect.TypeOf(time.Time{}): test.decoder, 239 | }, 240 | }) 241 | var row []time.Time 242 | var all []string 243 | for r.Read(&row) { 244 | for _, e := range row { 245 | all = append(all, e.Format("2006/1/2")) 246 | } 247 | } 248 | err := r.Done() 249 | if err == nil { 250 | t.Error("Done() returned nil unexpectedly") 251 | } 252 | if !strings.Contains(err.Error(), test.suberr) { 253 | t.Errorf("%q does not contains %q", err.Error(), test.suberr) 254 | } 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /testdata/issue1.csv: -------------------------------------------------------------------------------- 1 | name,birthday 2 | Alice,1980-12-30 3 | Bob,1975-06-09 4 | -------------------------------------------------------------------------------- /testdata/sample.csv: -------------------------------------------------------------------------------- 1 | Alice,10 2 | Bob,20 -------------------------------------------------------------------------------- /testdata/sample.tsv: -------------------------------------------------------------------------------- 1 | Alice 10 2 | Bob 20 3 | --------------------------------------------------------------------------------