├── .gitignore ├── go.mod ├── util_test.go ├── Makefile ├── .codecov.yml ├── types.go ├── reflect.go ├── LICENSE ├── util.go ├── .github └── workflows │ └── go.yml ├── reflect_test.go ├── inline_column_test.go ├── processors.go ├── .golangci.yml ├── go.sum ├── tag.go ├── tag_test.go ├── processors_test.go ├── csvlib_test.go ├── inline_column.go ├── errors_render_test.go ├── errors_render_as_csv_test.go ├── errors_test.go ├── csvlib.go ├── validator.go ├── validator_test.go ├── data_test.go ├── decode.go ├── encode.go ├── README.md ├── errors.go ├── errors_render.go ├── docs ├── ENCODING.md └── DECODING.md ├── errors_render_as_csv.go ├── encoder.go ├── encoder_test.go └── decoder.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test 9 | *.test 10 | *.out 11 | 12 | # Dependency 13 | vendor/ 14 | 15 | # Goland, vscode, OS 16 | .idea 17 | .vscode 18 | .DS_Store 19 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tiendc/go-csvlib 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/hashicorp/go-multierror v1.1.1 7 | github.com/stretchr/testify v1.9.0 8 | github.com/tiendc/gofn v1.11.0 9 | ) 10 | 11 | require ( 12 | github.com/davecgh/go-spew v1.1.1 // indirect 13 | github.com/hashicorp/errwrap v1.1.0 // indirect 14 | github.com/pmezard/go-difflib v1.0.0 // indirect 15 | github.com/tiendc/go-rflutil v0.0.0-20231112145832-693b7b74d697 // indirect 16 | gopkg.in/yaml.v3 v3.0.1 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /util_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_validateHeader(t *testing.T) { 10 | assert.Nil(t, validateHeader([]string{"col1", "col2", "col3"})) 11 | assert.Nil(t, validateHeader([]string{"col1", "col2", "Col1"})) 12 | 13 | assert.ErrorIs(t, validateHeader([]string{"col1", "col2", "col3 "}), ErrHeaderColumnInvalid) 14 | assert.ErrorIs(t, validateHeader([]string{"col1", "col2", "col1"}), ErrHeaderColumnDuplicated) 15 | } 16 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: lint test 2 | 3 | prepare: 4 | @curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.60.3 5 | 6 | build: 7 | @go build -v ./... 8 | 9 | test: 10 | @go test -cover -v ./... 11 | 12 | cover: 13 | @go test -race -coverprofile=coverage.txt -coverpkg=./... ./... 14 | @go tool cover -html=coverage.txt -o coverage.html 15 | 16 | lint: 17 | golangci-lint --timeout=5m0s run -v ./... 18 | 19 | bench: 20 | go test -benchmem -count 100 -bench . 21 | 22 | mod: 23 | go mod tidy && go mod vendor 24 | -------------------------------------------------------------------------------- /.codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | range: 80..100 3 | round: down 4 | precision: 2 5 | 6 | status: 7 | project: # measuring the overall project coverage 8 | default: # context, you can create multiple ones with custom titles 9 | enabled: yes # must be yes|true to enable this status 10 | target: 85% # specify the target coverage for each commit status 11 | # option: "auto" (must increase from parent commit or pull request base) 12 | # option: "X%" a static target percentage to hit 13 | if_not_found: success # if parent is not found report status as success, error, or failure 14 | if_ci_failed: error # if ci fails report status as success, error, or failure 15 | -------------------------------------------------------------------------------- /types.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | type Int interface { 4 | int | int8 | int16 | int32 | int64 5 | } 6 | 7 | type IntEx interface { 8 | ~int | ~int8 | ~int16 | ~int32 | ~int64 9 | } 10 | 11 | type UInt interface { 12 | uint | uint8 | uint16 | uint32 | uint64 13 | } 14 | 15 | type UIntEx interface { 16 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 17 | } 18 | 19 | type Float interface { 20 | float32 | float64 21 | } 22 | 23 | type FloatEx interface { 24 | ~float32 | ~float64 25 | } 26 | 27 | type Number interface { 28 | Int | UInt | Float 29 | } 30 | 31 | type NumberEx interface { 32 | IntEx | UIntEx | FloatEx 33 | } 34 | 35 | type String interface { 36 | string 37 | } 38 | 39 | type StringEx interface { 40 | ~string 41 | } 42 | 43 | type LTComparable interface { 44 | Int | IntEx | UInt | UIntEx | Float | FloatEx | String | StringEx 45 | } 46 | -------------------------------------------------------------------------------- /reflect.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | func isKindOrPtrOf(t reflect.Type, kinds ...reflect.Kind) bool { 8 | k := t.Kind() 9 | if k == reflect.Pointer { 10 | k = t.Elem().Kind() 11 | } 12 | for _, kk := range kinds { 13 | if k == kk { 14 | return true 15 | } 16 | } 17 | return false 18 | } 19 | 20 | func indirectType(t reflect.Type) reflect.Type { 21 | for t.Kind() == reflect.Pointer { 22 | return t.Elem() 23 | } 24 | return t 25 | } 26 | 27 | func indirectValue(v reflect.Value) reflect.Value { 28 | for v.Kind() == reflect.Pointer { 29 | return v.Elem() 30 | } 31 | return v 32 | } 33 | 34 | func initAndIndirectValue(v reflect.Value) reflect.Value { 35 | if v.Kind() == reflect.Pointer { 36 | if v.IsNil() { 37 | // NOTE: v.CanSet must return true in order to call v.Set 38 | v.Set(reflect.New(v.Type().Elem())) 39 | } 40 | return v.Elem() 41 | } 42 | return v 43 | } 44 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 tiendc 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /util.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | "text/template" 8 | 9 | "github.com/hashicorp/go-multierror" 10 | ) 11 | 12 | func validateHeader(header []string) error { 13 | mapCheckUniq := make(map[string]struct{}, len(header)) 14 | for _, h := range header { 15 | hh := strings.TrimSpace(h) 16 | if h != hh || len(hh) == 0 { 17 | return fmt.Errorf("%w: \"%s\" invalid", ErrHeaderColumnInvalid, h) 18 | } 19 | if _, ok := mapCheckUniq[hh]; ok { 20 | return fmt.Errorf("%w: \"%s\" duplicated", ErrHeaderColumnDuplicated, h) 21 | } 22 | mapCheckUniq[hh] = struct{}{} 23 | } 24 | return nil 25 | } 26 | 27 | func processTemplate(templ string, params ParameterMap) (detail string, retErr error) { 28 | detail = templ 29 | t, err := template.New("error").Parse(detail) 30 | if err != nil { 31 | retErr = multierror.Append(retErr, err) 32 | return 33 | } 34 | 35 | buf := bytes.NewBuffer(make([]byte, 0, 100)) //nolint:mnd 36 | err = t.Execute(buf, params) 37 | if err != nil { 38 | retErr = multierror.Append(retErr, err) 39 | } else { 40 | detail = buf.String() 41 | } 42 | return 43 | } 44 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | branches: ['*'] 6 | tags: ['v*'] 7 | pull_request: 8 | branches: ['*'] 9 | 10 | jobs: 11 | 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | go: ["1.18.x", "1.22.x", "1.23.x"] 17 | include: 18 | - go: 1.23.x 19 | latest: true 20 | 21 | steps: 22 | - name: Setup Go 23 | uses: actions/setup-go@v5 24 | with: 25 | go-version: ${{ matrix.go }} 26 | 27 | - name: Checkout code 28 | uses: actions/checkout@v4 29 | 30 | - name: Load cached dependencies 31 | uses: actions/cache@v4 32 | with: 33 | path: ~/go/pkg/mod 34 | key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} 35 | restore-keys: | 36 | ${{ runner.os }}-go- 37 | 38 | - name: Download Dependencies 39 | run: make prepare 40 | 41 | - name: Lint 42 | run: make lint 43 | 44 | - name: Test 45 | run: make cover 46 | 47 | - name: Upload coverage to codecov.io 48 | uses: codecov/codecov-action@v4 49 | with: 50 | token: ${{ secrets.CODECOV_TOKEN }} 51 | -------------------------------------------------------------------------------- /reflect_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_isKindOrPtrOf(t *testing.T) { 11 | var v int 12 | assert.True(t, isKindOrPtrOf(reflect.TypeOf(v), reflect.Int)) 13 | assert.True(t, isKindOrPtrOf(reflect.TypeOf(&v), reflect.Int)) 14 | assert.False(t, isKindOrPtrOf(reflect.TypeOf(&v), reflect.String)) 15 | } 16 | 17 | func Test_indirectType(t *testing.T) { 18 | var v string 19 | assert.Equal(t, reflect.TypeOf(""), indirectType(reflect.TypeOf(v))) 20 | assert.Equal(t, reflect.TypeOf(""), indirectType(reflect.TypeOf(&v))) 21 | } 22 | 23 | func Test_indirectValue(t *testing.T) { 24 | v := "abc" 25 | assert.Equal(t, "abc", indirectValue(reflect.ValueOf(v)).String()) 26 | assert.Equal(t, "abc", indirectValue(reflect.ValueOf(&v)).String()) 27 | } 28 | 29 | func Test_initAndIndirectValue(t *testing.T) { 30 | v := "abc" 31 | assert.Equal(t, "abc", initAndIndirectValue(reflect.ValueOf(v)).String()) 32 | assert.Equal(t, "abc", initAndIndirectValue(reflect.ValueOf(&v)).String()) 33 | 34 | var v2 int 35 | assert.Equal(t, int64(0), initAndIndirectValue(reflect.ValueOf(v2)).Int()) 36 | assert.Equal(t, int64(0), initAndIndirectValue(reflect.ValueOf(&v2)).Int()) 37 | 38 | s := []*string{nil} 39 | assert.Equal(t, "", initAndIndirectValue(reflect.ValueOf(s).Index(0)).String()) 40 | } 41 | -------------------------------------------------------------------------------- /inline_column_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | // func Test_parseInlineColumnDynamicType(t *testing.T) { 4 | // parent := &decodeColumnMeta{} 5 | // _, err := parseInlineColumnDynamicType(reflect.TypeOf([]string{}), parent) 6 | // assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 7 | // 8 | // type DynInline1 struct { 9 | // } 10 | // _, err = parseInlineColumnDynamicType(reflect.TypeOf(DynInline1{}), parent) 11 | // assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 12 | // 13 | // type DynInline2 struct { 14 | // Header []int 15 | // } 16 | // _, err = parseInlineColumnDynamicType(reflect.TypeOf(DynInline2{}), parent) 17 | // assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 18 | // 19 | // type DynInline3 struct { 20 | // Header []string 21 | // } 22 | // _, err = parseInlineColumnDynamicType(reflect.TypeOf(DynInline3{}), parent) 23 | // assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 24 | // 25 | // type DynInline4 struct { 26 | // Header []string 27 | // Values [10]int32 28 | // } 29 | // _, err = parseInlineColumnDynamicType(reflect.TypeOf(DynInline4{}), parent) 30 | // assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 31 | // 32 | // type DynInlineOK struct { 33 | // Header []string 34 | // Values []int32 35 | // } 36 | // typ, err := parseInlineColumnDynamicType(reflect.TypeOf(DynInlineOK{}), parent) 37 | // assert.Nil(t, err) 38 | // assert.Equal(t, reflect.TypeOf(int32(0)), typ) 39 | // } 40 | -------------------------------------------------------------------------------- /processors.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/tiendc/gofn" 7 | ) 8 | 9 | var ( 10 | // ProcessorTrim trims space of string 11 | ProcessorTrim = strings.TrimSpace 12 | // ProcessorTrimPrefix trims prefix from a string 13 | ProcessorTrimPrefix = strings.TrimPrefix 14 | // ProcessorTrimSuffix trims suffix from a string 15 | ProcessorTrimSuffix = strings.TrimSuffix 16 | // ProcessorReplace replaces a substring in a string 17 | ProcessorReplace = strings.Replace 18 | // ProcessorReplaceAll replaces all occurrences of a substring in a string 19 | ProcessorReplaceAll = strings.ReplaceAll 20 | // ProcessorLower converts a string to lowercase 21 | ProcessorLower = strings.ToLower 22 | // ProcessorUpper converts a string to uppercase 23 | ProcessorUpper = strings.ToUpper 24 | // ProcessorNumberGroup formats a number with grouping its digits 25 | ProcessorNumberGroup = gofn.NumberFmtGroup 26 | // ProcessorNumberUngroup ungroups number digits 27 | ProcessorNumberUngroup = gofn.NumberFmtUngroup 28 | ) 29 | 30 | // ProcessorNumberGroupComma formats a number with grouping its digits by comma 31 | func ProcessorNumberGroupComma(s string) string { 32 | return gofn.NumberFmtGroup(s, '.', ',') 33 | } 34 | 35 | // ProcessorNumberUngroupComma ungroups number digits by comma 36 | func ProcessorNumberUngroupComma(s string) string { 37 | return gofn.NumberFmtUngroup(s, ',') 38 | } 39 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | linters-settings: 2 | funlen: 3 | lines: 100 4 | statements: 80 5 | gci: 6 | sections: 7 | - standard 8 | - default 9 | - prefix(github.com/tiendc/go-csvlib) 10 | gocyclo: 11 | min-complexity: 20 12 | goimports: 13 | local-prefixes: github.com/golangci/golangci-lint 14 | lll: 15 | line-length: 120 16 | misspell: 17 | locale: US 18 | 19 | linters: 20 | enable: 21 | - bodyclose 22 | - contextcheck 23 | - dogsled 24 | - errcheck 25 | - errname 26 | - errorlint 27 | - exhaustive 28 | - copyloopvar 29 | - forbidigo 30 | - forcetypeassert 31 | - funlen 32 | - gci 33 | - gocognit 34 | - goconst 35 | - gocritic 36 | - gocyclo 37 | - err113 38 | - gofmt 39 | - goimports 40 | - mnd 41 | - gosec 42 | - gosimple 43 | - govet 44 | - ineffassign 45 | - lll 46 | - misspell 47 | - nakedret 48 | - nestif 49 | - nilerr 50 | - rowserrcheck 51 | - staticcheck 52 | - stylecheck 53 | - typecheck 54 | - unconvert 55 | - unparam 56 | - unused 57 | - whitespace 58 | 59 | issues: 60 | exclude-rules: 61 | - path: _test\.go 62 | linters: 63 | - funlen 64 | - contextcheck 65 | - staticcheck 66 | - gocyclo 67 | - gocognit 68 | - err113 69 | - forcetypeassert 70 | - wrapcheck 71 | - gomnd 72 | - errorlint 73 | - unused -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 4 | github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= 5 | github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= 6 | github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= 7 | github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 11 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 12 | github.com/tiendc/go-rflutil v0.0.0-20231112145832-693b7b74d697 h1:BYWZUvxBkpnlC4MywWhO1bEch5L6cCc0t5FNVSUjZps= 13 | github.com/tiendc/go-rflutil v0.0.0-20231112145832-693b7b74d697/go.mod h1:nSMBac9C+G4b8nvxSgPZ0rmhrLonJ5ZdknynSKxQhL8= 14 | github.com/tiendc/gofn v1.11.0 h1:rjXJ2tZ6L96ICwBkbVv0ubDOvrGRo1QMXhhq90xZTBw= 15 | github.com/tiendc/gofn v1.11.0/go.mod h1:uevHlES37QrasSvoZxBUcooejk7QfvBCfrJ809b+giA= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 17 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 18 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 19 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | -------------------------------------------------------------------------------- /tag.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | ) 8 | 9 | type tagDetail struct { 10 | name string 11 | prefix string 12 | ignored bool 13 | empty bool 14 | omitEmpty bool 15 | optional bool 16 | inline bool 17 | } 18 | 19 | func parseTag(tagName string, field reflect.StructField) (*tagDetail, error) { 20 | tagValue, ok := field.Tag.Lookup(tagName) 21 | if !ok { 22 | return nil, nil 23 | } 24 | 25 | tag := &tagDetail{} 26 | tags := strings.Split(tagValue, ",") 27 | if len(tags) == 1 && tags[0] == "" { 28 | tag.name = field.Name 29 | tag.empty = true 30 | } else { 31 | switch tags[0] { 32 | case "-": 33 | tag.ignored = true 34 | case "": 35 | tag.name = field.Name 36 | default: 37 | tag.name = tags[0] 38 | } 39 | 40 | for _, tagOpt := range tags[1:] { 41 | switch { 42 | case tagOpt == "optional": 43 | tag.optional = true 44 | case tagOpt == "omitempty": 45 | tag.omitEmpty = true 46 | case tagOpt == "inline": 47 | tag.inline = true 48 | case strings.HasPrefix(tagOpt, "prefix="): 49 | tag.prefix = tagOpt[len("prefix="):] 50 | } 51 | } 52 | } 53 | 54 | // Validation: struct field unexported 55 | if !tag.ignored && !field.IsExported() { 56 | return nil, fmt.Errorf("%w: struct field %s unexported", ErrTagOptionInvalid, field.Name) 57 | } 58 | // Validation: only inline column can have prefix 59 | if tag.prefix != "" && !tag.inline { 60 | return nil, fmt.Errorf("%w: prefix tag is only accepted for inline column", ErrTagOptionInvalid) 61 | } 62 | // Validation: inline column must not be optional 63 | if tag.inline && tag.optional { 64 | return nil, fmt.Errorf("%w: inline column must not be optional", ErrTagOptionInvalid) 65 | } 66 | 67 | return tag, nil 68 | } 69 | -------------------------------------------------------------------------------- /tag_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func Test_parseTag(t *testing.T) { 11 | type Item struct { 12 | Col0 bool 13 | Col1 int `csv:"col1,optional"` 14 | Col2 *int `csv:"col2,omitempty"` 15 | Col3 string `csv:"-"` 16 | Col4 string `csv:""` 17 | Col5 string `csv:",unsupported"` 18 | Col6 InlineColumn[int] `csv:"col6,inline,prefix=xyz"` 19 | col7 int32 `csv:"col7"` // nolint: unused 20 | } 21 | structType := reflect.TypeOf(Item{}) 22 | 23 | col0, _ := structType.FieldByName("Col0") 24 | tag0, err := parseTag(DefaultTagName, col0) 25 | assert.Nil(t, err) 26 | assert.Nil(t, tag0) 27 | 28 | col1, _ := structType.FieldByName("Col1") 29 | tag1, err := parseTag(DefaultTagName, col1) 30 | assert.Nil(t, err) 31 | assert.True(t, tag1.name == "col1" && tag1.optional) 32 | 33 | col2, _ := structType.FieldByName("Col2") 34 | tag2, err := parseTag(DefaultTagName, col2) 35 | assert.Nil(t, err) 36 | assert.True(t, tag2.name == "col2" && tag2.omitEmpty) 37 | 38 | col3, _ := structType.FieldByName("Col3") 39 | tag3, err := parseTag(DefaultTagName, col3) 40 | assert.Nil(t, err) 41 | assert.True(t, tag3.ignored) 42 | 43 | col4, _ := structType.FieldByName("Col4") 44 | tag4, err := parseTag(DefaultTagName, col4) 45 | assert.Nil(t, err) 46 | assert.True(t, tag4.name == "Col4" && tag4.empty) 47 | 48 | col5, _ := structType.FieldByName("Col5") 49 | tag5, err := parseTag(DefaultTagName, col5) 50 | assert.Nil(t, err) 51 | assert.True(t, tag5.name == "Col5") 52 | 53 | col6, _ := structType.FieldByName("Col6") 54 | tag6, err := parseTag(DefaultTagName, col6) 55 | assert.Nil(t, err) 56 | assert.True(t, tag6.name == "col6" && tag6.inline && tag6.prefix == "xyz") 57 | 58 | col7, _ := structType.FieldByName("col7") 59 | _, err = parseTag(DefaultTagName, col7) 60 | assert.ErrorIs(t, err, ErrTagOptionInvalid) 61 | } 62 | -------------------------------------------------------------------------------- /processors_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_ProcessorTrim(t *testing.T) { 10 | assert.Equal(t, "", ProcessorTrim("")) 11 | assert.Equal(t, "abc", ProcessorTrim("\n abc\t")) 12 | } 13 | 14 | func Test_ProcessorTrimPrefix(t *testing.T) { 15 | assert.Equal(t, "", ProcessorTrimPrefix("", "abc")) 16 | assert.Equal(t, "bc\t", ProcessorTrimPrefix("\n abc\t", "\n a")) 17 | } 18 | 19 | func Test_ProcessorTrimSuffix(t *testing.T) { 20 | assert.Equal(t, "", ProcessorTrimSuffix("", "abc")) 21 | assert.Equal(t, "\n a", ProcessorTrimSuffix("\n abc\t", "bc\t")) 22 | } 23 | 24 | func Test_ProcessorReplace(t *testing.T) { 25 | assert.Equal(t, "", ProcessorReplace("", "abc", "xyz", 1)) 26 | assert.Equal(t, "xyzxyz", ProcessorReplace("abcxyz", "abc", "xyz", 1)) 27 | } 28 | 29 | func Test_ProcessorReplaceAll(t *testing.T) { 30 | assert.Equal(t, "", ProcessorReplaceAll("", "abc", "xyz")) 31 | assert.Equal(t, "xyzxyzxyz", ProcessorReplaceAll("abcxyzabc", "abc", "xyz")) 32 | } 33 | 34 | func Test_ProcessorLower(t *testing.T) { 35 | assert.Equal(t, "", ProcessorLower("")) 36 | assert.Equal(t, "abc123", ProcessorLower("aBc123")) 37 | } 38 | 39 | func Test_ProcessorUpper(t *testing.T) { 40 | assert.Equal(t, "", ProcessorUpper("")) 41 | assert.Equal(t, "ABC123", ProcessorUpper("aBc123")) 42 | } 43 | 44 | func Test_ProcessorNumberGroup(t *testing.T) { 45 | assert.Equal(t, "", ProcessorNumberGroup("", '.', ',')) 46 | assert.Equal(t, "aBc123", ProcessorNumberGroup("aBc123", '.', ',')) 47 | assert.Equal(t, "123", ProcessorNumberGroup("123", '.', ',')) 48 | assert.Equal(t, "12,345,678", ProcessorNumberGroup("12345678", '.', ',')) 49 | } 50 | 51 | func Test_ProcessorNumberUngroup(t *testing.T) { 52 | assert.Equal(t, "", ProcessorNumberUngroup("", ',')) 53 | assert.Equal(t, "aBc123", ProcessorNumberUngroup("aBc,123", ',')) 54 | assert.Equal(t, "123", ProcessorNumberUngroup("123", ',')) 55 | assert.Equal(t, "1234567.8", ProcessorNumberUngroup("12,3456,7.8", ',')) 56 | } 57 | 58 | func Test_ProcessorNumberGroupComma(t *testing.T) { 59 | assert.Equal(t, "", ProcessorNumberGroupComma("")) 60 | assert.Equal(t, "aBc123", ProcessorNumberGroupComma("aBc123")) 61 | assert.Equal(t, "123", ProcessorNumberGroupComma("123")) 62 | assert.Equal(t, "12,345,678", ProcessorNumberGroupComma("12345678")) 63 | } 64 | 65 | func Test_ProcessorNumberUngroupComma(t *testing.T) { 66 | assert.Equal(t, "", ProcessorNumberUngroupComma("")) 67 | assert.Equal(t, "aBc123", ProcessorNumberUngroupComma("aBc,123")) 68 | assert.Equal(t, "123", ProcessorNumberUngroupComma("123")) 69 | assert.Equal(t, "1234567.8", ProcessorNumberUngroupComma("12,3456,7.8")) 70 | } 71 | -------------------------------------------------------------------------------- /csvlib_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/tiendc/gofn" 9 | ) 10 | 11 | func Test_Unmarshal(t *testing.T) { 12 | type Item struct { 13 | ColX bool `csv:",optional"` 14 | ColY bool `csv:"-"` 15 | Col1 int `csv:"col1"` 16 | Col2 float32 `csv:"col2"` 17 | } 18 | 19 | t.Run("#1: success", func(t *testing.T) { 20 | data := gofn.MultilineString( 21 | `col1,col2 22 | 1,2.123 23 | 100,200`) 24 | 25 | var v []Item 26 | ret, err := Unmarshal([]byte(data), &v) 27 | assert.Nil(t, err) 28 | assert.Equal(t, 3, ret.TotalRow()) 29 | assert.Equal(t, []string{"ColX"}, ret.MissingOptionalColumns()) 30 | assert.Equal(t, []Item{{Col1: 1, Col2: 2.123}, {Col1: 100, Col2: 200}}, v) 31 | }) 32 | } 33 | 34 | func Test_Marshal(t *testing.T) { 35 | type Item struct { 36 | ColX bool `csv:",optional"` 37 | ColY bool 38 | Col1 int `csv:"col1"` 39 | Col2 float32 `csv:"col2"` 40 | } 41 | 42 | t.Run("#1: success", func(t *testing.T) { 43 | v := []Item{ 44 | {Col1: 1, Col2: 2.123}, 45 | {Col1: 100, Col2: 200}, 46 | } 47 | data, err := Marshal(v) 48 | assert.Nil(t, err) 49 | assert.Equal(t, gofn.MultilineString( 50 | `ColX,col1,col2 51 | false,1,2.123 52 | false,100,200 53 | `), string(data)) 54 | }) 55 | } 56 | 57 | func Test_GetHeaderDetails(t *testing.T) { 58 | t.Run("#1: success", func(t *testing.T) { 59 | type Item struct { 60 | Col1 int `csv:"col1,omitempty"` 61 | Col2 *string `csv:"col2,optional"` 62 | Col3 bool `csv:"-"` 63 | Col4 float32 64 | Col5 InlineColumn[int] `csv:"col5,inline"` 65 | } 66 | details, err := GetHeaderDetails(Item{}, "csv") 67 | assert.Nil(t, err) 68 | assert.Equal(t, []ColumnDetail{ 69 | {Name: "col1", DataType: reflect.TypeOf(int(0)), OmitEmpty: true}, 70 | {Name: "col2", DataType: reflect.TypeOf(gofn.New("")), Optional: true}, 71 | {Name: "col5", DataType: reflect.TypeOf(InlineColumn[int]{}), Inline: true}, 72 | }, details) 73 | }) 74 | 75 | t.Run("#2: invalid type", func(t *testing.T) { 76 | _, err := GetHeaderDetails("abc", "csv") 77 | assert.ErrorIs(t, err, ErrTypeInvalid) 78 | }) 79 | } 80 | 81 | func Test_GetHeader(t *testing.T) { 82 | t.Run("#1: success", func(t *testing.T) { 83 | type Item struct { 84 | Col1 int `csv:"col1,omitempty"` 85 | Col2 *string `csv:"col2,optional"` 86 | Col3 bool `csv:"-"` 87 | Col4 float32 88 | Col5 InlineColumn[int] `csv:"col5,inline"` 89 | } 90 | header, err := GetHeader(Item{}, "csv") 91 | assert.Nil(t, err) 92 | assert.Equal(t, []string{"col1", "col2", "col5"}, header) 93 | }) 94 | 95 | t.Run("#2: invalid type", func(t *testing.T) { 96 | _, err := GetHeader(0, "csv") 97 | assert.ErrorIs(t, err, ErrTypeInvalid) 98 | }) 99 | } 100 | -------------------------------------------------------------------------------- /inline_column.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "reflect" 5 | ) 6 | 7 | type inlineColumnStructType int8 8 | 9 | const ( 10 | inlineColumnStructDynamic = inlineColumnStructType(1) 11 | inlineColumnStructFixed = inlineColumnStructType(2) 12 | 13 | dynamicInlineColumnHeader = "Header" 14 | dynamicInlineColumnValues = "Values" 15 | ) 16 | 17 | // InlineColumn represents inline columns of type `T` 18 | type InlineColumn[T any] struct { 19 | Header []string 20 | Values []T 21 | } 22 | 23 | // inlineColumnMeta metadata of inline columns 24 | type inlineColumnMeta struct { 25 | headerText []string 26 | inlineType inlineColumnStructType 27 | targetField reflect.StructField 28 | dataType reflect.Type 29 | 30 | // columnCurrIndex current processing column (used for dynamic inline columns) 31 | columnCurrIndex int 32 | } 33 | 34 | func (m *inlineColumnMeta) decodePrepareForNextRow() { 35 | m.columnCurrIndex = -1 36 | } 37 | 38 | func (m *inlineColumnMeta) decodeInitInlineStruct(inlineStruct reflect.Value) { 39 | if inlineStruct.Kind() == reflect.Pointer { 40 | inlineStruct.Set(reflect.New(inlineStruct.Type().Elem())) // No need to check Nil as this is decoding 41 | inlineStruct = inlineStruct.Elem() 42 | } 43 | 44 | switch m.inlineType { 45 | case inlineColumnStructFixed: 46 | // Do nothing 47 | case inlineColumnStructDynamic: 48 | if m.columnCurrIndex != -1 { 49 | return 50 | } 51 | numCols := len(m.headerText) 52 | inlineStruct.FieldByName(dynamicInlineColumnHeader).Set(reflect.ValueOf(m.headerText)) 53 | 54 | columnValues := inlineStruct.Field(m.targetField.Index[0]) 55 | columnValues.Set(reflect.MakeSlice(reflect.SliceOf(m.dataType), numCols, numCols)) 56 | m.columnCurrIndex = 0 57 | } 58 | } 59 | 60 | func (m *inlineColumnMeta) decodeGetColumnValue(inlineStruct reflect.Value) reflect.Value { 61 | if inlineStruct.Kind() == reflect.Pointer { 62 | inlineStruct.Set(reflect.New(inlineStruct.Type().Elem())) // No need to check Nil as this is decoding 63 | inlineStruct = inlineStruct.Elem() 64 | } 65 | 66 | switch m.inlineType { 67 | case inlineColumnStructFixed: 68 | return inlineStruct.Field(m.targetField.Index[0]) 69 | case inlineColumnStructDynamic: 70 | if m.columnCurrIndex == -1 { 71 | m.decodeInitInlineStruct(inlineStruct) 72 | } 73 | colVal := inlineStruct.Field(m.targetField.Index[0]).Index(m.columnCurrIndex) 74 | m.columnCurrIndex++ 75 | return colVal 76 | } 77 | return reflect.Value{} 78 | } 79 | 80 | func (m *inlineColumnMeta) encodePrepareForNextRow() { 81 | m.columnCurrIndex = 0 82 | } 83 | 84 | func (m *inlineColumnMeta) encodeGetColumnValue(inlineStruct reflect.Value) reflect.Value { 85 | if inlineStruct.Kind() == reflect.Pointer { 86 | inlineStruct = inlineStruct.Elem() 87 | if !inlineStruct.IsValid() { 88 | return reflect.Value{} 89 | } 90 | } 91 | 92 | switch m.inlineType { 93 | case inlineColumnStructFixed: 94 | return inlineStruct.Field(m.targetField.Index[0]) 95 | case inlineColumnStructDynamic: 96 | colVal := inlineStruct.Field(m.targetField.Index[0]).Index(m.columnCurrIndex) 97 | m.columnCurrIndex++ 98 | return colVal 99 | } 100 | return reflect.Value{} 101 | } 102 | -------------------------------------------------------------------------------- /errors_render_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/tiendc/gofn" 9 | ) 10 | 11 | func Test_ErrorRender(t *testing.T) { 12 | // CSV error has 2 row errors 13 | csvErr := NewErrors() 14 | csvErr.totalRow = 100 15 | rowErr1 := NewRowErrors(10, 12) 16 | rowErr2 := NewRowErrors(20, 22) 17 | csvErr.Add(rowErr1, rowErr2) 18 | 19 | // First row error has 2 cell errors and a common error 20 | cellErr11 := NewCellError(ErrValidationStrLen, 0, "Name") 21 | cellErr11.SetLocalizationKey("ERR_NAME_TOO_LONG") 22 | cellErr11.value = "David David David" 23 | _ = cellErr11.WithParam("MinLen", 1).WithParam("MaxLen", 10) 24 | 25 | cellErr12 := NewCellError(ErrValidationRange, 1, "Age") 26 | cellErr12.SetLocalizationKey("ERR_AGE_OUT_OF_RANGE") 27 | cellErr12.value = "101" 28 | _ = cellErr12.WithParam("MinValue", 1).WithParam("MaxValue", 100) 29 | 30 | cellErr13 := NewCellError(ErrDecodeQuoteInvalid, -1, "") // error not relate to any column 31 | rowErr1.Add(cellErr11, cellErr12, cellErr13) 32 | 33 | // Second row error has 2 other cell errors 34 | cellErr21 := NewCellError(ErrValidationStrLen, 0, "Name") 35 | cellErr22 := NewCellError(ErrValidationRange, 1, "Age") 36 | rowErr2.Add(cellErr21, cellErr22) 37 | 38 | // A common error (unexpected) 39 | csvErr.Add(ErrTypeUnsupported) 40 | 41 | t.Run("#1: default rendering", func(t *testing.T) { 42 | r, err := NewRenderer(csvErr) 43 | assert.Nil(t, err) 44 | msg, _, err := r.Render() 45 | assert.Nil(t, err) 46 | assert.Equal(t, gofn.MultilineString( 47 | `Error content: TotalRow: 100, TotalRowError: 2, TotalCellError: 5, TotalError: 6 48 | Row 10 (line 12): ERR_NAME_TOO_LONG, ERR_AGE_OUT_OF_RANGE, ErrDecodeQuoteInvalid 49 | Row 20 (line 22): ErrValidation: StrLen, ErrValidation: Range 50 | ErrTypeUnsupported`), msg) 51 | }) 52 | 53 | t.Run("#2: translate en_US", func(t *testing.T) { 54 | r, err := NewRenderer(csvErr, func(cfg *ErrorRenderConfig) { 55 | cfg.LocalizationFunc = localizeEnUs 56 | }) 57 | assert.Nil(t, err) 58 | msg, _, err := r.Render() 59 | assert.Nil(t, err) 60 | // nolint: lll 61 | assert.Equal(t, gofn.MultilineString( 62 | `Error content: TotalRow: 100, TotalRowError: 2, TotalCellError: 5, TotalError: 6 63 | Row 10 (line 12): 'David David David' at column 0 - Name length must be from 1 to 10, '101' at column 1 - Age must be from 1 to 100, ErrDecodeQuoteInvalid 64 | Row 20 (line 22): ErrValidation: StrLen, ErrValidation: Range 65 | ErrTypeUnsupported`), msg) 66 | }) 67 | 68 | t.Run("#3: translate vi_VN", func(t *testing.T) { 69 | r, err := NewRenderer(csvErr, func(cfg *ErrorRenderConfig) { 70 | cfg.LocalizationFunc = localizeViVn 71 | cfg.CellRenderFunc = func(rowErr *RowErrors, cellErr *CellError, params ParameterMap) (string, bool) { 72 | if errors.Is(cellErr, ErrDecodeQuoteInvalid) { 73 | return "nội dung bị bao sai (quote)", true 74 | } 75 | return "", true 76 | } 77 | }) 78 | assert.Nil(t, err) 79 | msg, _, err := r.Render() 80 | assert.Nil(t, err) 81 | // nolint: lll 82 | assert.Equal(t, gofn.MultilineString( 83 | `Error content: TotalRow: 100, TotalRowError: 2, TotalCellError: 5, TotalError: 6 84 | Row 10 (line 12): 'David David David' at column 0 - Tên phải dài từ 1 đến 10 ký tự, '101' at column 1 - Tuổi phải từ 1 đến 100, nội dung bị bao sai (quote) 85 | Row 20 (line 22): ErrValidation: StrLen, ErrValidation: Range 86 | ErrTypeUnsupported`), msg) 87 | }) 88 | } 89 | -------------------------------------------------------------------------------- /errors_render_as_csv_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/tiendc/gofn" 9 | ) 10 | 11 | func Test_ErrorRenderAsCSV(t *testing.T) { 12 | // CSV error has 2 row errors 13 | csvErr := NewErrors() 14 | csvErr.totalRow = 200 15 | csvErr.header = []string{"Name", "Age", "Address"} 16 | 17 | rowErr1 := NewRowErrors(10, 12) 18 | rowErr2 := NewRowErrors(20, 22) 19 | csvErr.Add(rowErr1, rowErr2) 20 | 21 | // First row error has 2 cell errors and an unexpected error 22 | cellErr11 := NewCellError(ErrValidationStrLen, 0, "Name") 23 | cellErr11.SetLocalizationKey("ERR_NAME_TOO_LONG") 24 | cellErr11.value = "David David David" 25 | _ = cellErr11.WithParam("MinLen", 1).WithParam("MaxLen", 10) 26 | 27 | cellErr12 := NewCellError(ErrValidationRange, 1, "Age") 28 | cellErr12.SetLocalizationKey("ERR_AGE_OUT_OF_RANGE") 29 | cellErr12.value = "101" 30 | _ = cellErr12.WithParam("MinValue", 1).WithParam("MaxValue", 100) 31 | 32 | cellErr13 := NewCellError(ErrDecodeQuoteInvalid, -1, "") // error not relate to any column 33 | rowErr1.Add(cellErr11, cellErr12, cellErr13) 34 | 35 | // Second row error has 2 other cell errors 36 | cellErr21 := NewCellError(ErrValidationStrLen, 0, "Name") 37 | cellErr22 := NewCellError(ErrValidationRange, 1, "Age") 38 | rowErr2.Add(cellErr21, cellErr22) 39 | 40 | // An unexpected error 41 | csvErr.Add(ErrTypeUnsupported) 42 | 43 | t.Run("#1: default rendering", func(t *testing.T) { 44 | r, err := NewCSVRenderer(csvErr) 45 | assert.Nil(t, err) 46 | msg, _, err := r.RenderAsString() 47 | assert.Nil(t, err) 48 | assert.Equal(t, gofn.MultilineString( 49 | `Row,Line,CommonError,Name,Age,Address 50 | 10,12,ErrDecodeQuoteInvalid,ERR_NAME_TOO_LONG,ERR_AGE_OUT_OF_RANGE, 51 | 20,22,,ErrValidation: StrLen,ErrValidation: Range, 52 | `), msg) 53 | }) 54 | 55 | t.Run("#2: translate en_US", func(t *testing.T) { 56 | r, err := NewCSVRenderer(csvErr, func(cfg *CSVRenderConfig) { 57 | cfg.LocalizationFunc = localizeEnUs 58 | }) 59 | assert.Nil(t, err) 60 | msg, _, err := r.RenderAsString() 61 | assert.Nil(t, err) 62 | // nolint: lll 63 | assert.Equal(t, gofn.MultilineString( 64 | `Row,Line,CommonError,Name,Age,Address 65 | 10,12,ErrDecodeQuoteInvalid,'David David David' at column 0 - Name length must be from 1 to 10,'101' at column 1 - Age must be from 1 to 100, 66 | 20,22,,ErrValidation: StrLen,ErrValidation: Range, 67 | `), msg) 68 | }) 69 | 70 | t.Run("#3: translate vi_VN", func(t *testing.T) { 71 | r, err := NewCSVRenderer(csvErr, func(cfg *CSVRenderConfig) { 72 | cfg.LocalizationFunc = localizeViVn 73 | cfg.CellRenderFunc = func(rowErr *RowErrors, cellErr *CellError, params ParameterMap) (string, bool) { 74 | if errors.Is(cellErr, ErrDecodeQuoteInvalid) { 75 | return "nội dung bị bao sai (quote)", true 76 | } 77 | return "", true 78 | } 79 | cfg.RenderRowNumberColumnIndex = 1 80 | cfg.RenderCommonErrorColumnIndex = 0 81 | cfg.RenderLineNumberColumnIndex = -1 82 | cfg.HeaderRenderFunc = func(header []string, params ParameterMap) { 83 | header[cfg.RenderRowNumberColumnIndex] = "ROW" 84 | header[cfg.RenderCommonErrorColumnIndex] = "COMMONERR" 85 | } 86 | }) 87 | assert.Nil(t, err) 88 | msg, _, err := r.RenderAsString() 89 | assert.Nil(t, err) 90 | // nolint: lll 91 | assert.Equal(t, gofn.MultilineString( 92 | `COMMONERR,ROW,Name,Age,Address 93 | nội dung bị bao sai (quote),10,'David David David' at column 0 - Tên phải dài từ 1 đến 10 ký tự,'101' at column 1 - Tuổi phải từ 1 đến 100, 94 | ,20,ErrValidation: StrLen,ErrValidation: Range, 95 | `), msg) 96 | }) 97 | } 98 | -------------------------------------------------------------------------------- /errors_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | var ( 11 | errTest1 = errors.New("test error 1") 12 | errTest2 = errors.New("test error 2") 13 | errTest3 = errors.New("test error 3") 14 | 15 | errCell1 = NewCellError(errTest1, 0, "column-1") 16 | errCell2 = NewCellError(errTest2, 1, "column-2") 17 | 18 | errRow1 = func() *RowErrors { 19 | e := NewRowErrors(1, 11) 20 | e.Add(errTest1) 21 | e.Add(errCell1) 22 | return e 23 | }() 24 | errRow2 = func() *RowErrors { 25 | e := NewRowErrors(2, 22) 26 | e.Add(errTest2) 27 | e.Add(errTest3) 28 | e.Add(errCell2) 29 | return e 30 | }() 31 | ) 32 | 33 | func TestErrors(t *testing.T) { 34 | e := NewErrors() 35 | assert.Equal(t, 0, e.TotalRow()) 36 | assert.Equal(t, "", e.Error()) 37 | assert.False(t, e.HasError()) 38 | assert.Equal(t, 0, e.TotalError()) 39 | assert.Equal(t, 0, e.TotalRowError()) 40 | assert.Equal(t, 0, e.TotalCellError()) 41 | 42 | e.Add(errTest1) 43 | e.Add(errRow1) 44 | assert.Equal(t, 1, e.TotalRowError()) 45 | assert.Equal(t, 1, e.TotalCellError()) 46 | assert.Equal(t, 3, e.TotalError()) // errRow1 has 2 inner errors 47 | } 48 | 49 | func TestErrors_Is(t *testing.T) { 50 | e := NewErrors() 51 | assert.False(t, errors.Is(e, errTest1)) 52 | assert.False(t, errors.Is(e, errRow1)) 53 | assert.False(t, errors.Is(e, errCell1)) 54 | 55 | e.Add(errTest1) 56 | e.Add(errRow1) 57 | assert.True(t, errors.Is(e, errTest1)) 58 | assert.True(t, errors.Is(e, errRow1)) 59 | assert.True(t, errors.Is(e, errCell1)) // as errRow1 contains errCell1 60 | assert.False(t, errors.Is(e, errCell2)) 61 | assert.False(t, errors.Is(e, errRow2)) 62 | } 63 | 64 | func TestRowErrors(t *testing.T) { 65 | e := NewRowErrors(1, 11) 66 | assert.Equal(t, 1, e.Row()) 67 | assert.Equal(t, 11, e.Line()) 68 | assert.Equal(t, "", e.Error()) 69 | assert.False(t, e.HasError()) 70 | assert.Equal(t, 0, e.TotalError()) 71 | assert.Equal(t, 0, e.TotalCellError()) 72 | 73 | e.Add(errTest1) 74 | e.Add(errCell1) 75 | assert.Equal(t, 2, e.TotalError()) 76 | assert.Equal(t, "test error 1, test error 1", e.Error()) 77 | assert.Equal(t, 1, e.TotalCellError()) 78 | } 79 | 80 | func TestRowErrors_Is(t *testing.T) { 81 | e := NewRowErrors(1, 11) 82 | assert.False(t, errors.Is(e, errTest1)) 83 | assert.False(t, errors.Is(e, errRow1)) 84 | assert.False(t, errors.Is(e, errCell1)) 85 | 86 | e.Add(errTest1) 87 | e.Add(errCell1) 88 | assert.True(t, errors.Is(e, errTest1)) 89 | assert.True(t, errors.Is(e, errCell1)) 90 | assert.False(t, errors.Is(e, errCell2)) 91 | } 92 | 93 | func TestCellError(t *testing.T) { 94 | e1 := NewCellError(nil, 1, "column-1") 95 | assert.Equal(t, "column-1", e1.Header()) 96 | assert.Equal(t, 1, e1.Column()) 97 | assert.Equal(t, "", e1.Error()) 98 | assert.False(t, e1.HasError()) 99 | 100 | e2 := NewCellError(errTest2, 2, "column-2") 101 | assert.Equal(t, errTest2.Error(), e2.Error()) 102 | assert.Equal(t, "", e2.LocalizationKey()) 103 | 104 | e2.SetLocalizationKey("local-key") 105 | assert.Equal(t, "local-key", e2.LocalizationKey()) 106 | 107 | _ = e2.WithParam("k", 1) 108 | assert.Equal(t, 1, e2.fields["k"]) 109 | } 110 | 111 | func TestCellError_Is(t *testing.T) { 112 | assert.False(t, errors.Is(NewCellError(nil, 1, "column-1"), errTest1)) 113 | assert.False(t, errors.Is(NewCellError(errTest1, 1, "column-1"), errTest2)) 114 | assert.True(t, errors.Is(NewCellError(errTest1, 1, "column-1"), errTest1)) 115 | assert.True(t, errors.Is(NewCellError(errRow1, 1, "column-1"), errTest1)) 116 | assert.True(t, errors.Is(NewCellError(errRow1, 1, "column-1"), errCell1)) 117 | } 118 | -------------------------------------------------------------------------------- /csvlib.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "fmt" 7 | "reflect" 8 | ) 9 | 10 | const ( 11 | DefaultTagName = "csv" 12 | ) 13 | 14 | // Reader reader object interface required by the lib to read CSV data. 15 | // Should use csv.Reader from the built-in package "encoding/csv". 16 | type Reader interface { 17 | Read() ([]string, error) 18 | } 19 | 20 | // Writer writer object interface required by the lib to write CSV data to. 21 | // Should use csv.Writer from the built-in package "encoding/csv". 22 | type Writer interface { 23 | Write(record []string) error 24 | } 25 | 26 | // CSVUnmarshaler unmarshaler interface for decoding custom type 27 | type CSVUnmarshaler interface { 28 | UnmarshalCSV([]byte) error 29 | } 30 | 31 | // CSVMarshaler marshaler interface for encoding custom type 32 | type CSVMarshaler interface { 33 | MarshalCSV() ([]byte, error) 34 | } 35 | 36 | // DecodeFunc decode function for a given cell text 37 | type DecodeFunc func(text string, v reflect.Value) error 38 | 39 | // EncodeFunc encode function for a given Go value 40 | type EncodeFunc func(v reflect.Value, omitempty bool) (string, error) 41 | 42 | // ProcessorFunc function to transform cell value before decoding or after encoding 43 | type ProcessorFunc func(s string) string 44 | 45 | // ValidatorFunc function to validate the values of decoded cells 46 | type ValidatorFunc func(v any) error 47 | 48 | type ParameterMap map[string]any 49 | 50 | // LocalizationFunc function to translate message into a specific language 51 | type LocalizationFunc func(key string, params ParameterMap) (string, error) 52 | 53 | // OnCellErrorFunc function to be called when error happens on decoding cell value 54 | type OnCellErrorFunc func(e *CellError) 55 | 56 | // ColumnDetail details of a column parsed from a struct tag 57 | type ColumnDetail struct { 58 | Name string 59 | Optional bool 60 | OmitEmpty bool 61 | Inline bool 62 | DataType reflect.Type 63 | } 64 | 65 | // Unmarshal convenient method to decode CVS data into a slice of structs 66 | func Unmarshal(data []byte, v any, options ...DecodeOption) (*DecodeResult, error) { 67 | decoder := NewDecoder(csv.NewReader(bytes.NewReader(data)), options...) 68 | return decoder.Decode(v) 69 | } 70 | 71 | // Marshal convenient method to encode a slice of structs into CSV format 72 | func Marshal(v any, options ...EncodeOption) ([]byte, error) { 73 | var buf bytes.Buffer 74 | w := csv.NewWriter(&buf) 75 | encoder := NewEncoder(w, options...) 76 | if err := encoder.Encode(v); err != nil { 77 | return nil, err 78 | } 79 | w.Flush() 80 | if err := w.Error(); err != nil { 81 | return nil, err 82 | } 83 | return buf.Bytes(), nil 84 | } 85 | 86 | // GetHeaderDetails get CSV header details from the given struct type 87 | func GetHeaderDetails(v any, tagName string) (columnDetails []ColumnDetail, err error) { 88 | t := reflect.TypeOf(v) 89 | t = indirectType(t) 90 | if t.Kind() != reflect.Struct { 91 | return nil, fmt.Errorf("%w: must be struct", ErrTypeInvalid) 92 | } 93 | numFields := t.NumField() 94 | for i := 0; i < numFields; i++ { 95 | field := t.Field(i) 96 | tag, _ := parseTag(tagName, field) 97 | if tag == nil || tag.ignored { 98 | continue 99 | } 100 | columnDetails = append(columnDetails, ColumnDetail{ 101 | Name: tag.name, 102 | Optional: tag.optional, 103 | OmitEmpty: tag.omitEmpty, 104 | Inline: tag.inline, 105 | DataType: field.Type, 106 | }) 107 | } 108 | return 109 | } 110 | 111 | // GetHeader get CSV header from the given struct 112 | func GetHeader(v any, tagName string) ([]string, error) { 113 | details, err := GetHeaderDetails(v, tagName) 114 | if err != nil { 115 | return nil, err 116 | } 117 | header := make([]string, 0, len(details)) 118 | for i := range details { 119 | header = append(header, details[i].Name) 120 | } 121 | return header, nil 122 | } 123 | -------------------------------------------------------------------------------- /validator.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | "unicode/utf8" 8 | "unsafe" 9 | ) 10 | 11 | // ValidatorLT validates a value to be less than the given value 12 | func ValidatorLT[T LTComparable](val T) ValidatorFunc { 13 | return func(v any) error { 14 | v1, ok := v.(T) 15 | if !ok { 16 | return errValidationConversion(v, v1) 17 | } 18 | if v1 < val { 19 | return nil 20 | } 21 | return ErrValidationLT 22 | } 23 | } 24 | 25 | // ValidatorLTE validates a value to be less than or equal to the given value 26 | func ValidatorLTE[T LTComparable](val T) ValidatorFunc { 27 | return func(v any) error { 28 | v1, ok := v.(T) 29 | if !ok { 30 | return errValidationConversion(v, v1) 31 | } 32 | if v1 <= val { 33 | return nil 34 | } 35 | return ErrValidationLTE 36 | } 37 | } 38 | 39 | // ValidatorGT validates a value to be greater than the given value 40 | func ValidatorGT[T LTComparable](val T) ValidatorFunc { 41 | return func(v any) error { 42 | v1, ok := v.(T) 43 | if !ok { 44 | return errValidationConversion(v, v1) 45 | } 46 | if v1 > val { 47 | return nil 48 | } 49 | return ErrValidationGT 50 | } 51 | } 52 | 53 | // ValidatorGTE validates a value to be greater than or equal to the given value 54 | func ValidatorGTE[T LTComparable](val T) ValidatorFunc { 55 | return func(v any) error { 56 | v1, ok := v.(T) 57 | if !ok { 58 | return errValidationConversion(v, v1) 59 | } 60 | if v1 >= val { 61 | return nil 62 | } 63 | return ErrValidationGTE 64 | } 65 | } 66 | 67 | // ValidatorRange validates a value to be in the given range (min and max are inclusive) 68 | func ValidatorRange[T LTComparable](min, max T) ValidatorFunc { 69 | return func(v any) error { 70 | v1, ok := v.(T) 71 | if !ok { 72 | return errValidationConversion(v, v1) 73 | } 74 | if min <= v1 && v1 <= max { 75 | return nil 76 | } 77 | return ErrValidationRange 78 | } 79 | } 80 | 81 | // ValidatorIN validates a value to be one of the specific values 82 | func ValidatorIN[T LTComparable](vals ...T) ValidatorFunc { 83 | return func(v any) error { 84 | v1, ok := v.(T) 85 | if !ok { 86 | return errValidationConversion(v, v1) 87 | } 88 | for _, val := range vals { 89 | if v1 == val { 90 | return nil 91 | } 92 | } 93 | return ErrValidationIN 94 | } 95 | } 96 | 97 | // ValidatorStrLen validates a string to have length in the given range. 98 | // Pass argument -1 to skip the equivalent validation. 99 | func ValidatorStrLen[T StringEx](minLen, maxLen int, lenFuncs ...func(s string) int) ValidatorFunc { 100 | return func(v any) error { 101 | s, ok := v.(T) 102 | if !ok { 103 | return errValidationConversion(v, s) 104 | } 105 | lenFunc := utf8.RuneCountInString 106 | if len(lenFuncs) > 0 { 107 | lenFunc = lenFuncs[0] 108 | } 109 | length := lenFunc(*(*string)(unsafe.Pointer(&s))) 110 | if (minLen == -1 || minLen <= length) && (maxLen == -1 || length <= maxLen) { 111 | return nil 112 | } 113 | return ErrValidationStrLen 114 | } 115 | } 116 | 117 | // ValidatorStrPrefix validates a string to have prefix matching the given one 118 | func ValidatorStrPrefix[T StringEx](prefix string) ValidatorFunc { 119 | return func(v any) error { 120 | s, ok := v.(T) 121 | if !ok { 122 | return errValidationConversion(v, s) 123 | } 124 | if strings.HasPrefix(*(*string)(unsafe.Pointer(&s)), prefix) { 125 | return nil 126 | } 127 | return ErrValidationStrPrefix 128 | } 129 | } 130 | 131 | // ValidatorStrSuffix validates a string to have suffix matching the given one 132 | func ValidatorStrSuffix[T StringEx](suffix string) ValidatorFunc { 133 | return func(v any) error { 134 | s, ok := v.(T) 135 | if !ok { 136 | return errValidationConversion(v, s) 137 | } 138 | if strings.HasSuffix(*(*string)(unsafe.Pointer(&s)), suffix) { 139 | return nil 140 | } 141 | return ErrValidationStrSuffix 142 | } 143 | } 144 | 145 | func errValidationConversion[T any](v1 any, v2 T) error { 146 | return fmt.Errorf("%w: (%v -> %v)", ErrValidationConversion, reflect.TypeOf(v1), reflect.TypeOf(v2)) 147 | } 148 | -------------------------------------------------------------------------------- /validator_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_ValidatorLT(t *testing.T) { 10 | assert.Nil(t, ValidatorLT(100)(99)) 11 | assert.Nil(t, ValidatorLT(100)(0)) 12 | assert.ErrorIs(t, ValidatorLT(100)("abc"), ErrValidationConversion) 13 | assert.ErrorIs(t, ValidatorLT(100)(100), ErrValidationLT) 14 | assert.ErrorIs(t, ValidatorLT(100)(101), ErrValidation) 15 | } 16 | 17 | func Test_ValidatorLTE(t *testing.T) { 18 | assert.Nil(t, ValidatorLTE(int64(0))(int64(0))) 19 | assert.Nil(t, ValidatorLTE(int64(0))(int64(-1))) 20 | assert.ErrorIs(t, ValidatorLTE(100)(true), ErrValidationConversion) 21 | assert.ErrorIs(t, ValidatorLTE(int64(0))(int64(1)), ErrValidationLTE) 22 | assert.ErrorIs(t, ValidatorLTE(int64(0))(int64(100)), ErrValidation) 23 | } 24 | 25 | func Test_ValidatorGT(t *testing.T) { 26 | assert.Nil(t, ValidatorGT(100)(101)) 27 | assert.Nil(t, ValidatorGT(100)(10000)) 28 | assert.ErrorIs(t, ValidatorGT(100)(int8(1)), ErrValidationConversion) 29 | assert.ErrorIs(t, ValidatorGT(100)(100), ErrValidationGT) 30 | assert.ErrorIs(t, ValidatorGT(100)(99), ErrValidation) 31 | } 32 | 33 | func Test_ValidatorGTE(t *testing.T) { 34 | assert.Nil(t, ValidatorGTE(int64(0))(int64(0))) 35 | assert.Nil(t, ValidatorGTE(int64(0))(int64(1))) 36 | assert.ErrorIs(t, ValidatorGTE(100)(int32(1)), ErrValidationConversion) 37 | assert.ErrorIs(t, ValidatorGTE(int64(0))(int64(-1)), ErrValidationGTE) 38 | assert.ErrorIs(t, ValidatorGTE(int64(0))(int64(-10)), ErrValidation) 39 | } 40 | 41 | func Test_ValidatorRange(t *testing.T) { 42 | assert.Nil(t, ValidatorRange(0, 10)(0)) 43 | assert.Nil(t, ValidatorRange(0, 10)(10)) 44 | assert.ErrorIs(t, ValidatorRange(0, 10)(int32(1)), ErrValidationConversion) 45 | assert.ErrorIs(t, ValidatorRange("a", "g")("h"), ErrValidationRange) 46 | assert.ErrorIs(t, ValidatorRange("a", "g")("0bc"), ErrValidation) 47 | } 48 | 49 | func Test_ValidatorIN(t *testing.T) { 50 | assert.Nil(t, ValidatorIN("a", "b", "c")("b")) 51 | assert.Nil(t, ValidatorIN("a", "b", "")("")) 52 | assert.ErrorIs(t, ValidatorIN("a", "b", "")(1), ErrValidationConversion) 53 | assert.ErrorIs(t, ValidatorIN("a", "b", "")("c"), ErrValidationIN) 54 | assert.ErrorIs(t, ValidatorIN("a", "b", "")("d"), ErrValidation) 55 | } 56 | 57 | func Test_ValidatorStrLen(t *testing.T) { 58 | lenFn := func(s string) int { return len(s) } 59 | assert.Nil(t, ValidatorStrLen[string](0, 5)("abc")) 60 | assert.Nil(t, ValidatorStrLen[string](0, 5)("")) 61 | assert.Nil(t, ValidatorStrLen[string](0, 5, lenFn)("abc")) 62 | assert.Nil(t, ValidatorStrLen[StrType](0, 5)(StrType("abc"))) 63 | assert.ErrorIs(t, ValidatorStrLen[string](0, 5)(StrType("abc")), ErrValidationConversion) 64 | assert.ErrorIs(t, ValidatorStrLen[string](1, 5)(""), ErrValidationStrLen) 65 | assert.ErrorIs(t, ValidatorStrLen[string](0, 5)("abc123"), ErrValidation) 66 | assert.ErrorIs(t, ValidatorStrLen[string](0, 5, lenFn)("abc123"), ErrValidation) 67 | assert.ErrorIs(t, ValidatorStrLen[StrType](0, 5)(StrType("abc123")), ErrValidation) 68 | } 69 | 70 | func Test_ValidatorStrPrefix(t *testing.T) { 71 | assert.Nil(t, ValidatorStrPrefix[string]("a")("abc")) 72 | assert.Nil(t, ValidatorStrPrefix[string](" a")(" abc")) 73 | assert.Nil(t, ValidatorStrPrefix[StrType](" a")(StrType(" abc"))) 74 | assert.ErrorIs(t, ValidatorStrPrefix[string]("x")(StrType("abc")), ErrValidationConversion) 75 | assert.ErrorIs(t, ValidatorStrPrefix[string]("x")("abc"), ErrValidationStrPrefix) 76 | assert.ErrorIs(t, ValidatorStrPrefix[string]("x")("abc"), ErrValidation) 77 | assert.ErrorIs(t, ValidatorStrPrefix[StrType]("x")(StrType("abc123")), ErrValidationStrPrefix) 78 | } 79 | 80 | func Test_ValidatorStrSuffix(t *testing.T) { 81 | assert.Nil(t, ValidatorStrSuffix[string]("c")("abc")) 82 | assert.Nil(t, ValidatorStrSuffix[string]("c ")(" abc ")) 83 | assert.Nil(t, ValidatorStrSuffix[StrType]("c ")(StrType("abc "))) 84 | assert.ErrorIs(t, ValidatorStrSuffix[string]("x")(StrType("abc")), ErrValidationConversion) 85 | assert.ErrorIs(t, ValidatorStrSuffix[string]("x")("abc"), ErrValidationStrSuffix) 86 | assert.ErrorIs(t, ValidatorStrSuffix[string]("x")("abc"), ErrValidation) 87 | assert.ErrorIs(t, ValidatorStrSuffix[StrType]("x")(StrType("abc123")), ErrValidationStrSuffix) 88 | } 89 | -------------------------------------------------------------------------------- /data_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | type StrType string 10 | 11 | type StrUpperType string 12 | 13 | func (s *StrUpperType) UnmarshalText(b []byte) error { 14 | *s = StrUpperType(strings.ToUpper(string(b))) 15 | return nil 16 | } 17 | 18 | func (s StrUpperType) MarshalText() ([]byte, error) { 19 | return []byte(strings.ToUpper(string(s))), nil 20 | } 21 | 22 | type StrLowerType string 23 | 24 | func (s *StrLowerType) UnmarshalCSV(b []byte) error { 25 | *s = StrLowerType(strings.ToLower(string(b))) 26 | return nil 27 | } 28 | 29 | func (s StrLowerType) MarshalCSV() ([]byte, error) { 30 | return []byte(strings.ToLower(string(s))), nil 31 | } 32 | 33 | type SaleStats struct { 34 | ID int `csv:"id"` 35 | Name string `csv:"name"` 36 | Year int `csv:"year"` 37 | QuarterCount QuarterSaleCount `csv:"quarter_count,inline"` 38 | QuarterAmount QuarterSaleAmount `csv:"quarter_amount,inline"` 39 | TotalCount int64 `csv:"total_count"` 40 | TotalAmount int64 `csv:"total_amount"` 41 | Success bool `csv:"success"` 42 | } 43 | 44 | type QuarterSaleCount struct { 45 | Q1 int `csv:"q1-count"` 46 | Q2 int `csv:"q2-count"` 47 | Q3 int `csv:"q3-count"` 48 | Q4 int `csv:"q4-count"` 49 | } 50 | 51 | type QuarterSaleAmount struct { 52 | Q1 int64 `csv:"q1-amount"` 53 | Q2 int64 `csv:"q2-amount"` 54 | Q3 int64 `csv:"q3-amount"` 55 | Q4 int64 `csv:"q4-amount"` 56 | } 57 | 58 | type SaleStats2 struct { 59 | ID int `csv:"id"` 60 | Name string `csv:"name"` 61 | Year int `csv:"year"` 62 | QuarterCount InlineColumn[int] `csv:"quarter_count,inline"` 63 | QuarterAmount InlineColumn[int64] `csv:"quarter_amount,inline"` 64 | TotalCount int64 `csv:"total_count"` 65 | TotalAmount int64 `csv:"total_amount"` 66 | Success bool `csv:"success"` 67 | } 68 | 69 | var ( 70 | mapLanguageEn = map[string]string{ 71 | "col1": "col-1", 72 | "col2": "col-2", 73 | "col3": "col-3", 74 | "col4": "col-4", 75 | "col5": "col-5", 76 | "colX": "col-X", 77 | "colY": "col-Y", 78 | "colZ": "col-Z", 79 | "Col1": "Col-1", 80 | "Col2": "Col-2", 81 | "Col3": "Col-3", 82 | "Col4": "Col-4", 83 | "Col5": "Col-5", 84 | "ColX": "Col-X", 85 | "ColY": "Col-Y", 86 | "ColZ": "Col-Z", 87 | 88 | "sub": "sub", 89 | "sub_col1": "sub-col-1", 90 | "sub_col2": "sub-col-2", 91 | "sub_col3": "sub-col-3", 92 | "sub_col4": "sub-col-4", 93 | "sub_col5": "sub-col-5", 94 | 95 | "ERR_HEADER_FORMAT": "Error content: TotalRow: {{.TotalRow}}, TotalRowError: {{.TotalRowError}}, " + 96 | "TotalCellError: {{.TotalCellError}}, TotalError: {{.TotalError}}", 97 | "ERR_ROW_FORMAT": "Row {{.Row}} (at line {{.Line}}): {{.ErrorContent}}", 98 | 99 | "ERR_NAME_TOO_LONG": "'{{.Value}}' at column {{.Column}} - Name length must be from {{.MinLen}} to {{.MaxLen}}", 100 | "ERR_AGE_OUT_OF_RANGE": "'{{.Value}}' at column {{.Column}} - Age must be from {{.MinValue}} to {{.MaxValue}}", 101 | "ERR_AGE_INVALID": "'{{.Value}}' at column {{.Column}} - Age must be a number", 102 | } 103 | 104 | mapLanguageVi = map[string]string{ 105 | "col1": "cột-1", 106 | "col2": "cột-2", 107 | "col3": "cột-3", 108 | "col4": "cột-4", 109 | "col5": "cột-5", 110 | "colX": "cột-X", 111 | "colY": "cột-Y", 112 | "colZ": "cột-Z", 113 | "Col1": "Cột-1", 114 | "Col2": "Cột-2", 115 | "Col3": "Cột-3", 116 | "Col4": "Cột-4", 117 | "Col5": "Cột-5", 118 | "ColX": "Cột-X", 119 | "ColY": "Cột-Y", 120 | "ColZ": "Cột-Z", 121 | 122 | "sub": "phụ", 123 | "sub_col1": "cột-phụ-1", 124 | "sub_col2": "cột-phụ-2", 125 | "sub_col3": "cột-phụ-3", 126 | "sub_col4": "cột-phụ-4", 127 | "sub_col5": "cột-phụ-5", 128 | 129 | "ERR_HEADER_FORMAT": "Error details: Số hàng: {{.TotalRow}}, Số hàng bị lỗi: {{.TotalRowError}}, " + 130 | "Số ô bị lỗi: {{.TotalCellError}}, Tổng lỗi: {{.TotalError}}", 131 | "ERR_ROW_FORMAT": "Hàng {{.Row}} (dòng {{.Line}}): {{.ErrorDetails}}", 132 | 133 | "ERR_NAME_TOO_LONG": "'{{.Value}}' at column {{.Column}} - Tên phải dài từ {{.MinLen}} đến {{.MaxLen}} ký tự", 134 | "ERR_AGE_OUT_OF_RANGE": "'{{.Value}}' at column {{.Column}} - Tuổi phải từ {{.MinValue}} đến {{.MaxValue}}", 135 | "ERR_AGE_INVALID": "'{{.Value}}' at column {{.Column}} - Tuổi phải là dạng số", 136 | } 137 | 138 | errKeyNotFound = errors.New("key not found") 139 | ) 140 | 141 | func localizeViVn(k string, params ParameterMap) (string, error) { 142 | s, ok := mapLanguageVi[k] 143 | if !ok { 144 | return "", fmt.Errorf("%w: '%s'", errKeyNotFound, k) 145 | } 146 | return processTemplate(s, params) 147 | } 148 | 149 | func localizeEnUs(k string, params ParameterMap) (string, error) { 150 | s, ok := mapLanguageEn[k] 151 | if !ok { 152 | return "", fmt.Errorf("%w: '%s'", errKeyNotFound, k) 153 | } 154 | return processTemplate(s, params) 155 | } 156 | 157 | func localizeFail(k string, params ParameterMap) (string, error) { 158 | return "", fmt.Errorf("%w: '%s'", errKeyNotFound, k) 159 | } 160 | -------------------------------------------------------------------------------- /decode.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "encoding" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | ) 9 | 10 | var ( 11 | textUnmarshaler = reflect.TypeOf((*encoding.TextUnmarshaler)(nil)).Elem() 12 | csvUnmarshaler = reflect.TypeOf((*CSVUnmarshaler)(nil)).Elem() 13 | ) 14 | 15 | func getDecodeFunc(typ reflect.Type) (DecodeFunc, error) { 16 | if typ.Implements(csvUnmarshaler) { 17 | return decodeCSVUnmarshaler, nil 18 | } 19 | if reflect.PointerTo(typ).Implements(csvUnmarshaler) { 20 | return decodePtrCSVUnmarshaler, nil 21 | } 22 | if typ.Implements(textUnmarshaler) { 23 | return decodeTextUnmarshaler, nil 24 | } 25 | if reflect.PointerTo(typ).Implements(textUnmarshaler) { 26 | return decodePtrTextUnmarshaler, nil 27 | } 28 | return getDecodeFuncBaseType(typ) 29 | } 30 | 31 | func getDecodeFuncBaseType(typ reflect.Type) (DecodeFunc, error) { 32 | typeIsPtr := false 33 | if typ.Kind() == reflect.Pointer { 34 | typeIsPtr = true 35 | typ = typ.Elem() 36 | } 37 | switch typ.Kind() { // nolint: exhaustive 38 | case reflect.String: 39 | if typeIsPtr { 40 | return decodePtrStr, nil 41 | } 42 | return decodeStr, nil 43 | case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8: 44 | if typeIsPtr { 45 | return decodePtrIntFunc(typ.Bits()), nil 46 | } 47 | return decodeIntFunc(typ.Bits()), nil 48 | case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8: 49 | if typeIsPtr { 50 | return decodePtrUintFunc(typ.Bits()), nil 51 | } 52 | return decodeUintFunc(typ.Bits()), nil 53 | case reflect.Bool: 54 | if typeIsPtr { 55 | return decodePtrBool, nil 56 | } 57 | return decodeBool, nil 58 | case reflect.Float32, reflect.Float64: 59 | if typeIsPtr { 60 | return decodePtrFloatFunc(typ.Bits()), nil 61 | } 62 | return decodeFloatFunc(typ.Bits()), nil 63 | case reflect.Interface: 64 | if typeIsPtr { 65 | return decodePtrInterface, nil 66 | } 67 | return decodeInterface, nil 68 | default: 69 | return nil, fmt.Errorf("%w: %v", ErrTypeUnsupported, typ.Kind()) 70 | } 71 | } 72 | 73 | func decodeTextUnmarshaler(s string, v reflect.Value) error { 74 | initAndIndirectValue(v) 75 | return v.Interface().(encoding.TextUnmarshaler).UnmarshalText([]byte(s)) // nolint: forcetypeassert 76 | } 77 | 78 | func decodePtrTextUnmarshaler(s string, v reflect.Value) error { 79 | if v.CanAddr() { 80 | return decodeTextUnmarshaler(s, v.Addr()) 81 | } 82 | // Fallback to process the value dynamically 83 | decodeFn, err := getDecodeFuncBaseType(v.Type()) 84 | if err != nil { 85 | return err 86 | } 87 | return decodeFn(s, v) 88 | } 89 | 90 | func decodeCSVUnmarshaler(s string, v reflect.Value) error { 91 | initAndIndirectValue(v) 92 | return v.Interface().(CSVUnmarshaler).UnmarshalCSV([]byte(s)) // nolint: forcetypeassert 93 | } 94 | 95 | func decodePtrCSVUnmarshaler(s string, v reflect.Value) error { 96 | if v.CanAddr() { 97 | return decodeCSVUnmarshaler(s, v.Addr()) 98 | } 99 | // Fallback to process the value dynamically 100 | decodeFn, err := getDecodeFuncBaseType(v.Type()) 101 | if err != nil { 102 | return err 103 | } 104 | return decodeFn(s, v) 105 | } 106 | 107 | func decodeStr(s string, v reflect.Value) error { 108 | v.SetString(s) 109 | return nil 110 | } 111 | 112 | func decodePtrStr(s string, v reflect.Value) error { 113 | return decodeStr(s, initAndIndirectValue(v)) 114 | } 115 | 116 | func decodeBool(s string, v reflect.Value) error { 117 | b, err := strconv.ParseBool(s) 118 | if err != nil { 119 | return fmt.Errorf("%w: %v (%s)", ErrDecodeValueType, v.Type(), s) 120 | } 121 | v.SetBool(b) 122 | return nil 123 | } 124 | 125 | func decodePtrBool(s string, v reflect.Value) error { 126 | return decodeBool(s, initAndIndirectValue(v)) 127 | } 128 | 129 | func decodeInt(s string, v reflect.Value, bits int) error { 130 | n, err := strconv.ParseInt(s, 10, bits) 131 | if err != nil { 132 | return fmt.Errorf("%w: %v (%s)", ErrDecodeValueType, v.Type(), s) 133 | } 134 | v.SetInt(n) 135 | return nil 136 | } 137 | 138 | func decodeIntFunc(bits int) DecodeFunc { 139 | return func(s string, v reflect.Value) error { 140 | return decodeInt(s, v, bits) 141 | } 142 | } 143 | 144 | func decodePtrIntFunc(bits int) DecodeFunc { 145 | return func(s string, v reflect.Value) error { 146 | return decodeInt(s, initAndIndirectValue(v), bits) 147 | } 148 | } 149 | 150 | func decodeUint(s string, v reflect.Value, bits int) error { 151 | n, err := strconv.ParseUint(s, 10, bits) 152 | if err != nil { 153 | return fmt.Errorf("%w: %v (%s)", ErrDecodeValueType, v.Type(), s) 154 | } 155 | v.SetUint(n) 156 | return nil 157 | } 158 | 159 | func decodeUintFunc(bits int) DecodeFunc { 160 | return func(s string, v reflect.Value) error { 161 | return decodeUint(s, v, bits) 162 | } 163 | } 164 | 165 | func decodePtrUintFunc(bits int) DecodeFunc { 166 | return func(s string, v reflect.Value) error { 167 | return decodeUint(s, initAndIndirectValue(v), bits) 168 | } 169 | } 170 | 171 | func decodeFloat(s string, v reflect.Value, bits int) error { 172 | n, err := strconv.ParseFloat(s, bits) 173 | if err != nil { 174 | return fmt.Errorf("%w: %v (%s)", ErrDecodeValueType, v.Type(), s) 175 | } 176 | v.SetFloat(n) 177 | return nil 178 | } 179 | 180 | func decodeFloatFunc(bits int) DecodeFunc { 181 | return func(s string, v reflect.Value) error { 182 | return decodeFloat(s, v, bits) 183 | } 184 | } 185 | 186 | func decodePtrFloatFunc(bits int) DecodeFunc { 187 | return func(s string, v reflect.Value) error { 188 | return decodeFloat(s, initAndIndirectValue(v), bits) 189 | } 190 | } 191 | 192 | func decodeInterface(s string, v reflect.Value) error { 193 | v.Set(reflect.ValueOf(s)) 194 | return nil 195 | } 196 | 197 | func decodePtrInterface(s string, v reflect.Value) error { 198 | initAndIndirectValue(v).Set(reflect.ValueOf(s)) 199 | return nil 200 | } 201 | -------------------------------------------------------------------------------- /encode.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "encoding" 5 | "fmt" 6 | "reflect" 7 | "strconv" 8 | ) 9 | 10 | var ( 11 | textMarshaler = reflect.TypeOf((*encoding.TextMarshaler)(nil)).Elem() 12 | csvMarshaler = reflect.TypeOf((*CSVMarshaler)(nil)).Elem() 13 | ) 14 | 15 | func getEncodeFunc(typ reflect.Type) (EncodeFunc, error) { 16 | if typ.Implements(csvMarshaler) { 17 | return encodeCSVMarshaler, nil 18 | } 19 | if reflect.PointerTo(typ).Implements(csvMarshaler) { 20 | return encodePtrCSVMarshaler, nil 21 | } 22 | if typ.Implements(textMarshaler) { 23 | return encodeTextMarshaler, nil 24 | } 25 | if reflect.PointerTo(typ).Implements(textMarshaler) { 26 | return encodePtrTextMarshaler, nil 27 | } 28 | return getEncodeFuncBaseType(typ) 29 | } 30 | 31 | func getEncodeFuncBaseType(typ reflect.Type) (EncodeFunc, error) { 32 | typeIsPtr := false 33 | if typ.Kind() == reflect.Pointer { 34 | typeIsPtr = true 35 | typ = typ.Elem() 36 | } 37 | switch typ.Kind() { // nolint: exhaustive 38 | case reflect.String: 39 | if typeIsPtr { 40 | return encodePtrStr, nil 41 | } 42 | return encodeStr, nil 43 | case reflect.Int, reflect.Int64, reflect.Int32, reflect.Int16, reflect.Int8: 44 | if typeIsPtr { 45 | return encodePtrInt, nil 46 | } 47 | return encodeInt, nil 48 | case reflect.Uint, reflect.Uint64, reflect.Uint32, reflect.Uint16, reflect.Uint8: 49 | if typeIsPtr { 50 | return encodePtrUint, nil 51 | } 52 | return encodeUint, nil 53 | case reflect.Bool: 54 | if typeIsPtr { 55 | return encodePtrBool, nil 56 | } 57 | return encodeBool, nil 58 | case reflect.Float32, reflect.Float64: 59 | if typeIsPtr { 60 | return encodePtrFloatFunc(typ.Bits()), nil 61 | } 62 | return encodeFloatFunc(typ.Bits()), nil 63 | case reflect.Interface: 64 | if typeIsPtr { 65 | return encodePtrInterface, nil 66 | } 67 | return encodeInterface, nil 68 | default: 69 | return nil, fmt.Errorf("%w: %v", ErrTypeUnsupported, typ.Kind()) 70 | } 71 | } 72 | 73 | func encodeCSVMarshaler(v reflect.Value, _ bool) (string, error) { 74 | if !v.IsValid() { 75 | return "", nil 76 | } 77 | if v.Kind() == reflect.Pointer && v.IsNil() { 78 | return "", nil 79 | } 80 | b, err := v.Interface().(CSVMarshaler).MarshalCSV() 81 | if err != nil { 82 | return "", fmt.Errorf("%w: %v", ErrEncodeValueType, v.Type()) 83 | } 84 | return string(b), nil 85 | } 86 | 87 | func encodePtrCSVMarshaler(v reflect.Value, omitempty bool) (string, error) { 88 | if v.CanAddr() { 89 | return encodeCSVMarshaler(v.Addr(), omitempty) 90 | } 91 | // Fallback to process the value dynamically 92 | encodeFn, err := getEncodeFuncBaseType(v.Type()) 93 | if err != nil { 94 | return "", err 95 | } 96 | return encodeFn(v, omitempty) 97 | } 98 | 99 | func encodeTextMarshaler(v reflect.Value, _ bool) (string, error) { 100 | if !v.IsValid() { 101 | return "", nil 102 | } 103 | if v.Kind() == reflect.Pointer && v.IsNil() { 104 | return "", nil 105 | } 106 | b, err := v.Interface().(encoding.TextMarshaler).MarshalText() 107 | if err != nil { 108 | return "", fmt.Errorf("%w: %v", ErrEncodeValueType, v.Type()) 109 | } 110 | return string(b), nil 111 | } 112 | 113 | func encodePtrTextMarshaler(v reflect.Value, omitempty bool) (string, error) { 114 | if v.CanAddr() { 115 | return encodeTextMarshaler(v.Addr(), omitempty) 116 | } 117 | // Fallback to process the value dynamically 118 | encodeFn, err := getEncodeFuncBaseType(v.Type()) 119 | if err != nil { 120 | return "", err 121 | } 122 | return encodeFn(v, omitempty) 123 | } 124 | 125 | func encodeStr(v reflect.Value, _ bool) (string, error) { 126 | return v.String(), nil 127 | } 128 | 129 | func encodePtrStr(v reflect.Value, _ bool) (string, error) { 130 | v = v.Elem() 131 | if !v.IsValid() { 132 | return "", nil 133 | } 134 | return v.String(), nil 135 | } 136 | 137 | func encodeBool(v reflect.Value, omitempty bool) (string, error) { 138 | t := v.Bool() 139 | if !t && omitempty { 140 | return "", nil 141 | } 142 | return strconv.FormatBool(t), nil 143 | } 144 | 145 | func encodePtrBool(v reflect.Value, omitempty bool) (string, error) { 146 | v = v.Elem() 147 | if !v.IsValid() { 148 | return "", nil 149 | } 150 | return encodeBool(v, omitempty) 151 | } 152 | 153 | func encodeInt(v reflect.Value, omitempty bool) (string, error) { 154 | n := v.Int() 155 | if n == 0 && omitempty { 156 | return "", nil 157 | } 158 | return strconv.FormatInt(n, 10), nil 159 | } 160 | 161 | func encodePtrInt(v reflect.Value, omitempty bool) (string, error) { 162 | v = v.Elem() 163 | if !v.IsValid() { 164 | return "", nil 165 | } 166 | return encodeInt(v, omitempty) 167 | } 168 | 169 | func encodeUint(v reflect.Value, omitempty bool) (string, error) { 170 | n := v.Uint() 171 | if n == 0 && omitempty { 172 | return "", nil 173 | } 174 | return strconv.FormatUint(n, 10), nil 175 | } 176 | 177 | func encodePtrUint(v reflect.Value, omitempty bool) (string, error) { 178 | v = v.Elem() 179 | if !v.IsValid() { 180 | return "", nil 181 | } 182 | return encodeUint(v, omitempty) 183 | } 184 | 185 | func encodeFloat(v reflect.Value, omitempty bool, bits int) (string, error) { 186 | f := v.Float() 187 | if f == 0 && omitempty { 188 | return "", nil 189 | } 190 | return strconv.FormatFloat(f, 'f', -1, bits), nil 191 | } 192 | 193 | func encodePtrFloat(v reflect.Value, omitempty bool, bits int) (string, error) { 194 | v = v.Elem() 195 | if !v.IsValid() { 196 | return "", nil 197 | } 198 | return encodeFloat(v, omitempty, bits) 199 | } 200 | 201 | func encodeFloatFunc(bits int) EncodeFunc { 202 | return func(v reflect.Value, omitempty bool) (string, error) { 203 | return encodeFloat(v, omitempty, bits) 204 | } 205 | } 206 | 207 | func encodePtrFloatFunc(bits int) EncodeFunc { 208 | return func(v reflect.Value, omitempty bool) (string, error) { 209 | return encodePtrFloat(v, omitempty, bits) 210 | } 211 | } 212 | 213 | func encodeInterface(v reflect.Value, omitempty bool) (string, error) { 214 | val := v.Elem() 215 | if !val.IsValid() { 216 | return "", nil 217 | } 218 | encodeFn, err := getEncodeFunc(val.Type()) 219 | if err != nil { 220 | return "", err 221 | } 222 | return encodeFn(val, omitempty) 223 | } 224 | 225 | func encodePtrInterface(v reflect.Value, omitempty bool) (string, error) { 226 | val := v.Elem() 227 | if !val.IsValid() { 228 | return "", nil 229 | } 230 | val = val.Elem() 231 | if !val.IsValid() { 232 | return "", nil 233 | } 234 | encodeFn, err := getEncodeFunc(val.Type()) 235 | if err != nil { 236 | return "", err 237 | } 238 | return encodeFn(val, omitempty) 239 | } 240 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Go Version][gover-img]][gover] [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov] [![GoReport][rpt-img]][rpt] 2 | 3 | # High level CSV library for Go 1.18+ 4 | 5 | This is a library for decoding and encoding CSV at high level as it provides convenient methods and many configuration options to process the data the way you expect. 6 | 7 | This library is inspired by the project https://github.com/jszwec/csvutil. 8 | 9 | ## Functionalities 10 | 11 | **Decoding** 12 | - Decode CSV data into Go struct 13 | - Support Go interface `encoding.TextUnmarshaler` (with function `UnmarshalText`) 14 | - Support custom interface `CSVUnmarshaler` (with function `UnmarshalCSV`) 15 | - Ability to continue decoding when error occurs (collect all errors at once) 16 | - Ability to perform custom preprocessor functions on cell data before decoding 17 | - Ability to perform custom validator functions on cell data after decoding 18 | - Ability to decode dynamic columns into Go struct field (inline columns) 19 | - Support rendering the result errors into human-readable content (row-by-row text and CSV) 20 | - Support localization to render the result errors into a specific language 21 | 22 | **Encoding** 23 | - Encode Go struct into CSV data 24 | - Support Go interface `encoding.TextMarshaler` (with function `MarshalText`) 25 | - Support custom interface `CSVMarshaler` (with function `MarshalCSV`) 26 | - Ability to perform custom postprocessor functions on cell data after encoding 27 | - Ability to encode dynamic columns defined via inner Go struct (inline columns) 28 | - Ability to localize the header into a specific language 29 | 30 | ## Installation 31 | 32 | ```shell 33 | go get github.com/tiendc/go-csvlib 34 | ``` 35 | 36 | ## Usage 37 | 38 | - [Decoding](docs/DECODING.md) 39 | - [Encoding](docs/ENCODING.md) 40 | 41 | ## Benchmarks 42 | 43 | ### csvlib vs csvutil vs gocsv vs easycsv 44 | 45 | [Benchmark code](https://gist.github.com/tiendc/c394677a846233bf8de819da3bb7093c) 46 | 47 | ### Unmarshal 48 | 49 | ``` 50 | BenchmarkUnmarshal/csvlib.Unmarshal/100_records 51 | BenchmarkUnmarshal/csvlib.Unmarshal/100_records-10 21572 55520 ns/op 52 | BenchmarkUnmarshal/csvlib.Unmarshal/1000_records 53 | BenchmarkUnmarshal/csvlib.Unmarshal/1000_records-10 2641 455794 ns/op 54 | BenchmarkUnmarshal/csvlib.Unmarshal/10000_records 55 | BenchmarkUnmarshal/csvlib.Unmarshal/10000_records-10 253 4716323 ns/op 56 | BenchmarkUnmarshal/csvlib.Unmarshal/100000_records 57 | BenchmarkUnmarshal/csvlib.Unmarshal/100000_records-10 26 44519502 ns/op 58 | 59 | BenchmarkUnmarshal/csvutil.Unmarshal/100_records 60 | BenchmarkUnmarshal/csvutil.Unmarshal/100_records-10 27848 42927 ns/op 61 | BenchmarkUnmarshal/csvutil.Unmarshal/1000_records 62 | BenchmarkUnmarshal/csvutil.Unmarshal/1000_records-10 2952 405309 ns/op 63 | BenchmarkUnmarshal/csvutil.Unmarshal/10000_records 64 | BenchmarkUnmarshal/csvutil.Unmarshal/10000_records-10 296 4059881 ns/op 65 | BenchmarkUnmarshal/csvutil.Unmarshal/100000_records 66 | BenchmarkUnmarshal/csvutil.Unmarshal/100000_records-10 28 40531973 ns/op 67 | 68 | BenchmarkUnmarshal/gocsv.Unmarshal/100_records 69 | BenchmarkUnmarshal/gocsv.Unmarshal/100_records-10 9830 118919 ns/op 70 | BenchmarkUnmarshal/gocsv.Unmarshal/1000_records 71 | BenchmarkUnmarshal/gocsv.Unmarshal/1000_records-10 1022 1164278 ns/op 72 | BenchmarkUnmarshal/gocsv.Unmarshal/10000_records 73 | BenchmarkUnmarshal/gocsv.Unmarshal/10000_records-10 86 12609154 ns/op 74 | BenchmarkUnmarshal/gocsv.Unmarshal/100000_records 75 | BenchmarkUnmarshal/gocsv.Unmarshal/100000_records-10 9 119912333 ns/op 76 | 77 | BenchmarkUnmarshal/easycsv.ReadAll/100_records 78 | BenchmarkUnmarshal/easycsv.ReadAll/100_records-10 3831 315302 ns/op 79 | BenchmarkUnmarshal/easycsv.ReadAll/1000_records 80 | BenchmarkUnmarshal/easycsv.ReadAll/1000_records-10 384 3083931 ns/op 81 | BenchmarkUnmarshal/easycsv.ReadAll/10000_records 82 | BenchmarkUnmarshal/easycsv.ReadAll/10000_records-10 34 31440493 ns/op 83 | BenchmarkUnmarshal/easycsv.ReadAll/100000_records 84 | BenchmarkUnmarshal/easycsv.ReadAll/100000_records-10 4 324321531 ns/op 85 | ``` 86 | 87 | ### Marshal 88 | 89 | ``` 90 | BenchmarkMarshal/csvlib.Marshal/100_records 91 | BenchmarkMarshal/csvlib.Marshal/100_records-10 19753 58890 ns/op 92 | BenchmarkMarshal/csvlib.Marshal/1000_records 93 | BenchmarkMarshal/csvlib.Marshal/1000_records-10 2149 554537 ns/op 94 | BenchmarkMarshal/csvlib.Marshal/10000_records 95 | BenchmarkMarshal/csvlib.Marshal/10000_records-10 214 5575920 ns/op 96 | BenchmarkMarshal/csvlib.Marshal/100000_records 97 | BenchmarkMarshal/csvlib.Marshal/100000_records-10 19 55735281 ns/op 98 | 99 | BenchmarkMarshal/csvutil.Marshal/100_records 100 | BenchmarkMarshal/csvutil.Marshal/100_records-10 24388 48931 ns/op 101 | BenchmarkMarshal/csvutil.Marshal/1000_records 102 | BenchmarkMarshal/csvutil.Marshal/1000_records-10 2557 467704 ns/op 103 | BenchmarkMarshal/csvutil.Marshal/10000_records 104 | BenchmarkMarshal/csvutil.Marshal/10000_records-10 256 4720885 ns/op 105 | BenchmarkMarshal/csvutil.Marshal/100000_records 106 | BenchmarkMarshal/csvutil.Marshal/100000_records-10 22 48627754 ns/op 107 | 108 | BenchmarkMarshal/gocsv.Marshal/100_records 109 | BenchmarkMarshal/gocsv.Marshal/100_records-10 13254 90873 ns/op 110 | BenchmarkMarshal/gocsv.Marshal/1000_records 111 | BenchmarkMarshal/gocsv.Marshal/1000_records-10 1294 898938 ns/op 112 | BenchmarkMarshal/gocsv.Marshal/10000_records 113 | BenchmarkMarshal/gocsv.Marshal/10000_records-10 132 9017481 ns/op 114 | BenchmarkMarshal/gocsv.Marshal/100000_records 115 | BenchmarkMarshal/gocsv.Marshal/100000_records-10 12 90260420 ns/op 116 | ``` 117 | 118 | ## Contributing 119 | 120 | - You are welcome to make pull requests for new functions and bug fixes. 121 | 122 | ## Authors 123 | 124 | - Dao Cong Tien ([tiendc](https://github.com/tiendc)) 125 | 126 | ## License 127 | 128 | - [MIT License](LICENSE) 129 | 130 | [doc-img]: https://pkg.go.dev/badge/github.com/tiendc/go-csvlib 131 | [doc]: https://pkg.go.dev/github.com/tiendc/go-csvlib 132 | [gover-img]: https://img.shields.io/badge/Go-%3E%3D%201.18-blue 133 | [gover]: https://img.shields.io/badge/Go-%3E%3D%201.18-blue 134 | [ci-img]: https://github.com/tiendc/go-csvlib/actions/workflows/go.yml/badge.svg 135 | [ci]: https://github.com/tiendc/go-csvlib/actions/workflows/go.yml 136 | [cov-img]: https://codecov.io/gh/tiendc/go-csvlib/branch/main/graph/badge.svg 137 | [cov]: https://codecov.io/gh/tiendc/go-csvlib 138 | [rpt-img]: https://goreportcard.com/badge/github.com/tiendc/go-csvlib 139 | [rpt]: https://goreportcard.com/report/github.com/tiendc/go-csvlib 140 | -------------------------------------------------------------------------------- /errors.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | ) 7 | 8 | var ( 9 | ErrTypeInvalid = errors.New("ErrTypeInvalid") 10 | ErrTypeUnsupported = errors.New("ErrTypeUnsupported") 11 | ErrTypeUnmatched = errors.New("ErrTypeUnmatched") 12 | ErrValueNil = errors.New("ErrValueNil") 13 | ErrAlreadyFailed = errors.New("ErrAlreadyFailed") 14 | ErrFinished = errors.New("ErrFinished") 15 | ErrUnexpected = errors.New("ErrUnexpected") 16 | 17 | ErrTagOptionInvalid = errors.New("ErrTagOptionInvalid") 18 | ErrConfigOptionInvalid = errors.New("ErrConfigOptionInvalid") 19 | ErrLocalization = errors.New("ErrLocalization") 20 | 21 | ErrHeaderColumnInvalid = errors.New("ErrHeaderColumnInvalid") 22 | ErrHeaderColumnUnrecognized = errors.New("ErrHeaderColumnUnrecognized") 23 | ErrHeaderColumnRequired = errors.New("ErrHeaderColumnRequired") 24 | ErrHeaderColumnDuplicated = errors.New("ErrHeaderColumnDuplicated") 25 | ErrHeaderColumnOrderInvalid = errors.New("ErrHeaderColumnOrderInvalid") 26 | ErrHeaderDynamicTypeInvalid = errors.New("ErrHeaderDynamicTypeInvalid") 27 | ErrHeaderDynamicNotAllowNoHeaderMode = errors.New("ErrHeaderDynamicNotAllowNoHeaderMode") 28 | ErrHeaderDynamicRequireColumnOrder = errors.New("ErrHeaderDynamicRequireColumnOrder") 29 | ErrHeaderDynamicNotAllowUnrecognizedColumns = errors.New("ErrHeaderDynamicNotAllowUnrecognizedColumns") 30 | ErrHeaderDynamicNotAllowLocalizedHeader = errors.New("ErrHeaderDynamicNotAllowLocalizedHeader") 31 | 32 | ErrValidationConversion = errors.New("ErrValidationConversion") 33 | ErrValidation = errors.New("ErrValidation") 34 | ErrValidationLT = fmt.Errorf("%w: LT", ErrValidation) 35 | ErrValidationLTE = fmt.Errorf("%w: LTE", ErrValidation) 36 | ErrValidationGT = fmt.Errorf("%w: GT", ErrValidation) 37 | ErrValidationGTE = fmt.Errorf("%w: GTE", ErrValidation) 38 | ErrValidationRange = fmt.Errorf("%w: Range", ErrValidation) 39 | ErrValidationIN = fmt.Errorf("%w: IN", ErrValidation) 40 | ErrValidationStrLen = fmt.Errorf("%w: StrLen", ErrValidation) 41 | ErrValidationStrPrefix = fmt.Errorf("%w: StrPrefix", ErrValidation) 42 | ErrValidationStrSuffix = fmt.Errorf("%w: StrSuffix", ErrValidation) 43 | 44 | ErrDecodeValueType = errors.New("ErrDecodeValueType") 45 | ErrDecodeRowFieldCount = errors.New("ErrDecodeRowFieldCount") 46 | ErrDecodeQuoteInvalid = errors.New("ErrDecodeQuoteInvalid") 47 | 48 | ErrEncodeValueType = errors.New("ErrEncodeValueType") 49 | ) 50 | 51 | // Errors represents errors returned by the encoder or decoder 52 | type Errors struct { // nolint: errname 53 | errs []error 54 | totalRow int 55 | header []string 56 | } 57 | 58 | // NewErrors creates a new Errors object 59 | func NewErrors() *Errors { 60 | return &Errors{} 61 | } 62 | 63 | // TotalRow gets total rows of CSV data 64 | func (e *Errors) TotalRow() int { 65 | return e.totalRow 66 | } 67 | 68 | // Header gets list of column headers 69 | func (e *Errors) Header() []string { 70 | return e.header 71 | } 72 | 73 | // Error implements Go error interface 74 | func (e *Errors) Error() string { 75 | return getErrorMsg(e.errs) 76 | } 77 | 78 | // HasError checks if there is at least one error in the list 79 | func (e *Errors) HasError() bool { 80 | return len(e.errs) > 0 81 | } 82 | 83 | // TotalRowError gets the total number of error of rows 84 | func (e *Errors) TotalRowError() int { 85 | c := 0 86 | for _, e := range e.errs { 87 | if _, ok := e.(*RowErrors); ok { // nolint: errorlint 88 | c++ 89 | } 90 | } 91 | return c 92 | } 93 | 94 | // TotalCellError gets the total number of error of cells 95 | func (e *Errors) TotalCellError() int { 96 | c := 0 97 | for _, e := range e.errs { 98 | if rowErr, ok := e.(*RowErrors); ok { // nolint: errorlint 99 | c += rowErr.TotalCellError() 100 | } 101 | } 102 | return c 103 | } 104 | 105 | // TotalError gets the total number of errors including row errors and cell errors 106 | func (e *Errors) TotalError() int { 107 | c := 0 108 | for _, e := range e.errs { 109 | if rowErr, ok := e.(*RowErrors); ok { // nolint: errorlint 110 | c += rowErr.TotalError() 111 | } else { 112 | c++ 113 | } 114 | } 115 | return c 116 | } 117 | 118 | // Add appends errors to the list 119 | func (e *Errors) Add(errs ...error) { 120 | e.errs = append(e.errs, errs...) 121 | } 122 | 123 | // Is checks if there is at least an error in the list kind of the specified error 124 | func (e *Errors) Is(err error) bool { 125 | for _, er := range e.errs { 126 | if errors.Is(er, err) { 127 | return true 128 | } 129 | } 130 | return false 131 | } 132 | 133 | // Unwrap implements Go error unwrap function 134 | func (e *Errors) Unwrap() []error { 135 | return e.errs 136 | } 137 | 138 | // RowErrors data structure of error of a row 139 | type RowErrors struct { // nolint: errname 140 | errs []error 141 | row int 142 | line int 143 | } 144 | 145 | // NewRowErrors creates a new RowErrors 146 | func NewRowErrors(row, line int) *RowErrors { 147 | return &RowErrors{row: row, line: line} 148 | } 149 | 150 | // Row gets the row contains the error 151 | func (e *RowErrors) Row() int { 152 | return e.row 153 | } 154 | 155 | // Line gets the line contains the error (line equals to row in most cases) 156 | func (e *RowErrors) Line() int { 157 | return e.line 158 | } 159 | 160 | // Error implements Go error interface 161 | func (e *RowErrors) Error() string { 162 | return getErrorMsg(e.errs) 163 | } 164 | 165 | // HasError checks if there is at least one error in the list 166 | func (e *RowErrors) HasError() bool { 167 | return len(e.errs) > 0 168 | } 169 | 170 | // TotalError gets the total number of errors 171 | func (e *RowErrors) TotalError() int { 172 | return len(e.errs) 173 | } 174 | 175 | // TotalCellError gets the total number of error of cells 176 | func (e *RowErrors) TotalCellError() int { 177 | c := 0 178 | for _, e := range e.errs { 179 | if _, ok := e.(*CellError); ok { // nolint: errorlint 180 | c++ 181 | } 182 | } 183 | return c 184 | } 185 | 186 | // Add appends errors to the list 187 | func (e *RowErrors) Add(errs ...error) { 188 | e.errs = append(e.errs, errs...) 189 | } 190 | 191 | // Is checks if there is at least an error in the list kind of the specified error 192 | func (e *RowErrors) Is(err error) bool { 193 | for _, er := range e.errs { 194 | if errors.Is(er, err) { 195 | return true 196 | } 197 | } 198 | return false 199 | } 200 | 201 | // Unwrap implements Go error unwrap function 202 | func (e *RowErrors) Unwrap() []error { 203 | return e.errs 204 | } 205 | 206 | // CellError data structure of error of a cell 207 | type CellError struct { 208 | err error 209 | fields map[string]any 210 | localizationKey string 211 | 212 | column int 213 | header string 214 | value string 215 | } 216 | 217 | // NewCellError creates a new CellError 218 | func NewCellError(err error, column int, header string) *CellError { 219 | return &CellError{err: err, column: column, header: header, fields: map[string]any{}} 220 | } 221 | 222 | // Error implements Go error interface 223 | func (e *CellError) Error() string { 224 | if e.err == nil { 225 | return "" 226 | } 227 | return e.err.Error() 228 | } 229 | 230 | // Column gets the column of the cell 231 | func (e *CellError) Column() int { 232 | return e.column 233 | } 234 | 235 | // Header gets the header of the column 236 | func (e *CellError) Header() string { 237 | return e.header 238 | } 239 | 240 | // Value gets the value of the cell 241 | func (e *CellError) Value() string { 242 | return e.value 243 | } 244 | 245 | // HasError checks if the error contains an error 246 | func (e *CellError) HasError() bool { 247 | return e.err != nil 248 | } 249 | 250 | // Is checks if the inner error is kind of the specified error 251 | func (e *CellError) Is(err error) bool { 252 | return errors.Is(e.err, err) 253 | } 254 | 255 | // Unwrap implements Go error unwrap function 256 | func (e *CellError) Unwrap() error { 257 | return e.err 258 | } 259 | 260 | // WithParam sets a param of error 261 | func (e *CellError) WithParam(k string, v any) *CellError { 262 | e.fields[k] = v 263 | return e 264 | } 265 | 266 | // LocalizationKey gets localization key of error 267 | func (e *CellError) LocalizationKey() string { 268 | return e.localizationKey 269 | } 270 | 271 | // SetLocalizationKey sets localization key of error 272 | func (e *CellError) SetLocalizationKey(k string) { 273 | e.localizationKey = k 274 | } 275 | 276 | func getErrorMsg(errs []error) string { 277 | s := "" 278 | for i, e := range errs { 279 | if i > 0 { 280 | s += ", " 281 | } 282 | s += e.Error() 283 | } 284 | return s 285 | } 286 | -------------------------------------------------------------------------------- /errors_render.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/hashicorp/go-multierror" 7 | "github.com/tiendc/gofn" 8 | ) 9 | 10 | const ( 11 | newLine = "\n" 12 | ) 13 | 14 | type ErrorRenderConfig struct { 15 | // HeaderFormatKey header format string. 16 | // You can use a localization key as the value to force the renderer to translate the key first. 17 | // If the translation fails, the original value is used for next step. 18 | // 19 | // For example, header format key can be: 20 | // - "HEADER_FORMAT_KEY" (a localization key you define in your localization data such as a json file, 21 | // HEADER_FORMAT_KEY = "CSV decoding result: total errors is {{.TotalError}}") 22 | // - "CSV decoding result: total errors is {{.TotalError}}" (direct string) 23 | // 24 | // Supported params: 25 | // {{.TotalRow}} - number of rows in the CSV data 26 | // {{.TotalRowError}} - number of rows have error 27 | // {{.TotalCellError}} - number of cells have error 28 | // {{.TotalError}} - number of errors 29 | // 30 | // Extra params: 31 | // {{.CrLf}} - line break 32 | // {{.Tab}} - tab character 33 | HeaderFormatKey string 34 | 35 | // RowFormatKey format string for each row. 36 | // Similar to the header format key, this can be a localization key or a direct string. 37 | // 38 | // For example, row format key can be: 39 | // - "ROW_FORMAT_KEY" (a localization key you define in your localization data such as a json file) 40 | // - "Row {{.Row}} (line {{.Line}}): {{.Error}}" (direct string) 41 | // 42 | // Supported params: 43 | // {{.Row}} - row index (1-based, row 1 can be the header row if present) 44 | // {{.Line}} - line of row in source file (can be -1 if undetected) 45 | // {{.Error}} - error content of the row which is a list of cell errors 46 | RowFormatKey string 47 | 48 | // RowSeparator separator to join row error details, normally a row is in a separated line 49 | RowSeparator string 50 | 51 | // CellSeparator separator to join cell error details within a row, normally a comma (`,`) 52 | CellSeparator string 53 | 54 | // LineBreak custom new line character (default is `\n`) 55 | LineBreak string 56 | 57 | // LocalizeCellFields localize cell's fields before rendering the cell error (default is `true`) 58 | LocalizeCellFields bool 59 | 60 | // LocalizeCellHeader localize cell header before rendering the cell error (default is `true`) 61 | LocalizeCellHeader bool 62 | 63 | // Params custom params user wants to send to the localization (optional) 64 | Params ParameterMap 65 | 66 | // LocalizationFunc function to translate message (optional) 67 | LocalizationFunc LocalizationFunc 68 | 69 | // CellRenderFunc custom render function for rendering a cell error (optional). 70 | // The func can return ("", false) to skip rendering the cell error, return ("", true) to let the 71 | // renderer continue using its solution, and return ("", true) to override the value. 72 | // 73 | // Supported params: 74 | // {{.Column}} - column index (0-based) 75 | // {{.ColumnHeader}} - column name 76 | // {{.Value}} - cell value 77 | // {{.Error}} - error detail which is result of calling err.Error() 78 | // 79 | // Use cellErr.WithParam() to add more extra params 80 | CellRenderFunc func(*RowErrors, *CellError, ParameterMap) (string, bool) 81 | 82 | // CommonErrorRenderFunc renders common error (not RowErrors, CellError) (optional) 83 | CommonErrorRenderFunc func(error, ParameterMap) (string, error) 84 | } 85 | 86 | func defaultRenderConfig() *ErrorRenderConfig { 87 | return &ErrorRenderConfig{ 88 | HeaderFormatKey: "Error content: TotalRow: {{.TotalRow}}, TotalRowError: {{.TotalRowError}}, " + 89 | "TotalCellError: {{.TotalCellError}}, TotalError: {{.TotalError}}", 90 | RowFormatKey: "Row {{.Row}} (line {{.Line}}): {{.Error}}", 91 | 92 | RowSeparator: newLine, 93 | CellSeparator: ", ", 94 | LineBreak: newLine, 95 | 96 | LocalizeCellFields: true, 97 | LocalizeCellHeader: true, 98 | } 99 | } 100 | 101 | // SimpleRenderer a simple implementation of error renderer which can produce a text message 102 | // for the input errors. 103 | type SimpleRenderer struct { 104 | cfg *ErrorRenderConfig 105 | sourceErr *Errors 106 | transErr error 107 | } 108 | 109 | // NewRenderer creates a new SimpleRenderer 110 | func NewRenderer(err *Errors, options ...func(*ErrorRenderConfig)) (*SimpleRenderer, error) { 111 | cfg := defaultRenderConfig() 112 | for _, opt := range options { 113 | opt(cfg) 114 | } 115 | return &SimpleRenderer{cfg: cfg, sourceErr: err}, nil 116 | } 117 | 118 | // Render renders Errors object as text. 119 | // 120 | // Sample output: 121 | // 122 | // There are 5 total errors in your CSV file 123 | // Row 20 (line 21): column 2: invalid type (Int), column 4: value (12345) too big 124 | // Row 30 (line 33): column 2: invalid type (Int), column 4: value (12345) too big, column 6: unexpected 125 | // Row 35 (line 38): column 2: invalid type (Int), column 4: value (12345) too big 126 | // Row 40 (line 44): column 2: invalid type (Int), column 4: value (12345) too big 127 | // Row 41 (line 50): invalid number of columns (10) 128 | func (r *SimpleRenderer) Render() (msg string, transErr error, err error) { 129 | cfg := r.cfg 130 | errs := r.sourceErr.Unwrap() 131 | content := make([]string, 0, len(errs)+1) 132 | params := gofn.MapUpdate(ParameterMap{ 133 | "CrLf": cfg.LineBreak, 134 | "Tab": "\t", 135 | 136 | "TotalRow": r.sourceErr.TotalRow(), 137 | "TotalError": r.sourceErr.TotalError(), 138 | "TotalRowError": r.sourceErr.TotalRowError(), 139 | "TotalCellError": r.sourceErr.TotalCellError(), 140 | }, cfg.Params) 141 | 142 | // Header line 143 | if cfg.HeaderFormatKey != "" { 144 | header := r.localizeKeySkipError(cfg.HeaderFormatKey, params) 145 | if header != "" { 146 | content = append(content, header) 147 | } 148 | } 149 | 150 | // Body part (simply each RowErrors object is rendered as a line) 151 | for _, err := range errs { 152 | var detail string 153 | if rowErr, ok := err.(*RowErrors); ok { // nolint: errorlint 154 | detail = r.renderRow(rowErr, params) 155 | } else { 156 | detail = r.renderCommonError(err, params) 157 | } 158 | if detail != "" { 159 | content = append(content, detail) 160 | } 161 | } 162 | 163 | return strings.Join(content, cfg.RowSeparator), r.transErr, nil 164 | } 165 | 166 | func (r *SimpleRenderer) renderRow(rowErr *RowErrors, exparams ParameterMap) string { 167 | cfg := r.cfg 168 | errs := rowErr.Unwrap() 169 | content := make([]string, 0, len(errs)) 170 | 171 | params := gofn.MapUpdate(ParameterMap{}, exparams) 172 | params["Row"] = rowErr.Row() 173 | params["Line"] = rowErr.Line() 174 | 175 | for _, err := range errs { 176 | var detail string 177 | if cellErr, ok := err.(*CellError); ok { // nolint: errorlint 178 | detail = r.renderCell(rowErr, cellErr, params) 179 | } else { 180 | detail = r.renderCommonError(err, params) 181 | } 182 | if detail != "" { 183 | content = append(content, detail) 184 | } 185 | } 186 | 187 | if cfg.RowFormatKey != "" { 188 | params["Error"] = strings.Join(content, cfg.CellSeparator) 189 | return r.localizeKeySkipError(cfg.RowFormatKey, params) 190 | } 191 | return "" 192 | } 193 | 194 | func (r *SimpleRenderer) renderCell(rowErr *RowErrors, cellErr *CellError, exparams ParameterMap) string { 195 | params := gofn.MapUpdate(ParameterMap{}, exparams) 196 | params = gofn.MapUpdate(params, r.renderCellFields(cellErr, params)) 197 | params["Column"] = cellErr.Column() 198 | params["ColumnHeader"] = r.renderCellHeader(cellErr, params) 199 | params["Value"] = cellErr.Value() 200 | params["Error"] = cellErr.Error() 201 | 202 | if r.cfg.CellRenderFunc != nil { 203 | msg, flag := r.cfg.CellRenderFunc(rowErr, cellErr, exparams) 204 | if !flag { 205 | return "" 206 | } 207 | if msg != "" { 208 | return msg 209 | } 210 | } 211 | 212 | locKey := cellErr.LocalizationKey() 213 | if locKey == "" { 214 | locKey = cellErr.Error() 215 | } 216 | return r.localizeKeySkipError(locKey, params) 217 | } 218 | 219 | func (r *SimpleRenderer) renderCellFields(cellErr *CellError, params ParameterMap) ParameterMap { 220 | if !r.cfg.LocalizeCellFields { 221 | return cellErr.fields 222 | } 223 | result := make(ParameterMap, len(cellErr.fields)) 224 | for k, v := range cellErr.fields { 225 | vAsStr, ok := v.(string) 226 | if !ok { 227 | result[k] = v 228 | continue 229 | } 230 | if translated, err := r.localizeKey(vAsStr, params); err != nil { 231 | result[k] = v 232 | } else { 233 | result[k] = translated 234 | } 235 | } 236 | return result 237 | } 238 | 239 | func (r *SimpleRenderer) renderCellHeader(cellErr *CellError, params ParameterMap) string { 240 | if !r.cfg.LocalizeCellHeader { 241 | return cellErr.Header() 242 | } 243 | return r.localizeKeySkipError(cellErr.Header(), params) 244 | } 245 | 246 | func (r *SimpleRenderer) renderCommonError(err error, params ParameterMap) string { 247 | if r.cfg.CommonErrorRenderFunc == nil { 248 | return r.localizeKeySkipError(err.Error(), params) 249 | } 250 | msg, err := r.cfg.CommonErrorRenderFunc(err, params) 251 | if err != nil { 252 | r.transErr = multierror.Append(r.transErr, err) 253 | } 254 | return msg 255 | } 256 | 257 | func (r *SimpleRenderer) localizeKey(key string, params ParameterMap) (string, error) { 258 | if r.cfg.LocalizationFunc == nil { 259 | return processTemplate(key, params) 260 | } 261 | msg, err := r.cfg.LocalizationFunc(key, params) 262 | if err != nil { 263 | err = multierror.Append(ErrLocalization, err) 264 | r.transErr = multierror.Append(r.transErr, err) 265 | return "", err 266 | } 267 | return msg, nil 268 | } 269 | 270 | func (r *SimpleRenderer) localizeKeySkipError(key string, params ParameterMap) string { 271 | s, err := r.localizeKey(key, params) 272 | if err == nil || r.cfg.LocalizationFunc == nil { 273 | return s 274 | } 275 | s, _ = processTemplate(key, params) 276 | return s 277 | } 278 | -------------------------------------------------------------------------------- /docs/ENCODING.md: -------------------------------------------------------------------------------- 1 | # Encoding CSV 2 | 3 | ## Index 4 | - [First example](#first-example) 5 | - [No header mode](#no-header-mode) 6 | - [Postprocessor](#postprocessor) 7 | - [Skip columns](#skip-columns) 8 | - [Fixed inline columns](#fixed-inline-columns) 9 | - [Dynamic inline columns](#dynamic-inline-columns) 10 | - [Custom marshaler](#custom-marshaler) 11 | - [Custom column delimiter](#custom-column-delimiter) 12 | - [Encode one-by-one](#encode-one-by-one) 13 | - [Header localization](#header-localization) 14 | 15 | ## Content 16 | 17 | ### First example 18 | 19 | [Playground](https://go.dev/play/p/SVTP87loZ-g) 20 | 21 | ```go 22 | type Student struct { 23 | Name string `csv:"name"` 24 | Birthdate time.Time `csv:"birthdate"` 25 | Address string `csv:"address,omitempty"` 26 | } 27 | 28 | students := []Student{ 29 | {Name: "jerry", Birthdate: time.Date(1990, time.January, 1, 0, 0, 0, 0, time.UTC), Address: "tokyo"}, 30 | {Name: "tom", Birthdate: time.Date(1989, time.November, 11, 0, 0, 0, 0, time.UTC), Address: "new york"}, 31 | } 32 | data, err := csvlib.Marshal(students) 33 | if err != nil { 34 | fmt.Println("error:", err) 35 | } 36 | fmt.Println(string(data)) 37 | 38 | // Output: 39 | // name,birthdate,address 40 | // jerry,1990-01-01T00:00:00Z,tokyo 41 | // tom,1989-11-11T00:00:00Z,new york 42 | ``` 43 | 44 | ### No header mode 45 | 46 | [Playground](https://go.dev/play/p/qkfXClVUO5e) 47 | 48 | ```go 49 | type Student struct { 50 | Name string `csv:"name"` 51 | Birthdate time.Time `csv:"birthdate"` 52 | Address string `csv:"address,omitempty"` 53 | } 54 | 55 | students := []Student{ 56 | {Name: "jerry", Birthdate: time.Date(1990, time.January, 1, 0, 0, 0, 0, time.UTC), Address: "tokyo"}, 57 | {Name: "tom", Birthdate: time.Date(1989, time.November, 11, 0, 0, 0, 0, time.UTC), Address: "new york"}, 58 | } 59 | data, err := csvlib.Marshal(students, func(cfg *csvlib.EncodeConfig) { 60 | cfg.NoHeaderMode = true 61 | }) 62 | if err != nil { 63 | fmt.Println("error:", err) 64 | } 65 | fmt.Println(string(data)) 66 | 67 | // Output: 68 | // jerry,1990-01-01T00:00:00Z,tokyo 69 | // tom,1989-11-11T00:00:00Z,new york 70 | ``` 71 | 72 | ### Postprocessor 73 | 74 | - Postprocessor functions will be called after Go values are encoded into CSV string. 75 | 76 | [Playground](https://go.dev/play/p/MlppfYm-xey) 77 | 78 | ```go 79 | type Student struct { 80 | Name string `csv:"name"` 81 | Age int `csv:"age"` 82 | Address string `csv:"address,omitempty"` 83 | } 84 | 85 | students := []Student{ 86 | {Name: "jerry ", Age: 20, Address: "tokyo"}, 87 | {Name: " tom ", Age: 19, Address: "new york"}, 88 | } 89 | data, err := csvlib.Marshal(students, func(cfg *csvlib.EncodeConfig) { 90 | cfg.ConfigureColumn("name", func(cfg *csvlib.EncodeColumnConfig) { 91 | cfg.PostprocessorFuncs = []csvlib.ProcessorFunc{csvlib.ProcessorTrim, csvlib.ProcessorUpper} 92 | }) 93 | }) 94 | if err != nil { 95 | fmt.Println("error:", err) 96 | } 97 | fmt.Println(string(data)) 98 | 99 | // Output: 100 | // name,age,address 101 | // JERRY,20,tokyo 102 | // TOM,19,new york 103 | ``` 104 | 105 | ### Skip columns 106 | 107 | - To skip encoding a column, you can omit the `csv` tag from the struct field or use `csv:"-"`. You can also use an equivalent configuration option for the column. 108 | 109 | [Playground](https://go.dev/play/p/NHzpm9o-y6N) 110 | 111 | ```go 112 | type Student struct { 113 | Name string `csv:"name"` 114 | Age int `csv:"-"` 115 | Address string `csv:"address,omitempty"` 116 | } 117 | 118 | students := []Student{ 119 | {Name: "jerry", Age: 20, Address: "tokyo"}, 120 | {Name: "tom", Age: 19, Address: "new york"}, 121 | } 122 | data, err := csvlib.Marshal(students, func(cfg *csvlib.EncodeConfig) { 123 | cfg.ConfigureColumn("address", func(cfg *csvlib.EncodeColumnConfig) { 124 | cfg.Skip = true 125 | }) 126 | }) 127 | if err != nil { 128 | fmt.Println("error:", err) 129 | } 130 | fmt.Println(string(data)) 131 | 132 | // Output: 133 | // name 134 | // jerry 135 | // tom 136 | ``` 137 | 138 | ### Fixed inline columns 139 | 140 | - Fixed inline columns are represented by an inner struct. `prefix` can be set for inline columns. 141 | 142 | [Playground](https://go.dev/play/p/kenlqLQ9p75) 143 | 144 | ```go 145 | type Marks struct { 146 | Math int `csv:"math"` 147 | Chemistry int `csv:"-"` 148 | Physics int `csv:"physics"` 149 | } 150 | type Student struct { 151 | Name string `csv:"name"` 152 | Age int `csv:"age,omitempty"` 153 | Address string `csv:"address,optional"` 154 | Marks Marks `csv:"marks,inline,prefix=mark_"` 155 | } 156 | 157 | students := []Student{ 158 | {Name: "jerry", Age: 20, Address: "tokyo", Marks: Marks{Math: 9, Chemistry: 8, Physics: 7}}, 159 | {Name: "tom", Age: 0, Address: "new york", Marks: Marks{Math: 7, Chemistry: 8, Physics: 9}}, 160 | } 161 | data, err := csvlib.Marshal(students) 162 | if err != nil { 163 | fmt.Println("error:", err) 164 | } 165 | fmt.Println(string(data)) 166 | 167 | // Output: 168 | // name,age,address,mark_math,mark_physics 169 | // jerry,20,tokyo,9,7 170 | // tom,,new york,7,9 171 | ``` 172 | 173 | ### Dynamic inline columns 174 | 175 | - Dynamic inline columns are represented by an inner struct with variable number of columns. `prefix` can also be set for them. 176 | 177 | [Playground](https://go.dev/play/p/BaMNjFj5Kr0) 178 | 179 | ```go 180 | type Student struct { 181 | Name string `csv:"name"` 182 | Age int `csv:"age,omitempty"` 183 | Address string `csv:"address,optional"` 184 | Marks csvlib.InlineColumn[int] `csv:"marks,inline,prefix=mark_"` 185 | } 186 | 187 | marksHeader := []string{"math", "chemistry", "physics"} 188 | students := []Student{ 189 | {Name: "jerry", Age: 20, Address: "tokyo", Marks: csvlib.InlineColumn[int]{Header: marksHeader, Values: []int{9, 8, 7}}}, 190 | {Name: "tom", Age: 0, Address: "new york", Marks: csvlib.InlineColumn[int]{Header: marksHeader, Values: []int{7, 8, 9}}}, 191 | } 192 | data, err := csvlib.Marshal(students) 193 | if err != nil { 194 | fmt.Println("error:", err) 195 | } 196 | fmt.Println(string(data)) 197 | 198 | // Output: 199 | // name,age,address,mark_math,mark_chemistry,mark_physics 200 | // jerry,20,tokyo,9,8,7 201 | // tom,,new york,7,8,9 202 | ``` 203 | 204 | ### Custom marshaler 205 | 206 | - Any user custom type can be decoded by implementing either `encoding.TextMarshaler` or `csvlib.CSVMarshaler` or both. `csvlib.CSVMarshaler` has higher priority. 207 | 208 | [Playground](https://go.dev/play/p/iSfxQSkUCax) 209 | 210 | ```go 211 | type BirthDate time.Time 212 | 213 | func (d BirthDate) MarshalCSV() ([]byte, error) { 214 | return []byte(time.Time(d).Format("2006-01-02")), nil // RFC3339 215 | } 216 | ``` 217 | ```go 218 | type Student struct { 219 | Name string `csv:"name"` 220 | Birthdate BirthDate `csv:"birthdate"` 221 | Address string `csv:"address,omitempty"` 222 | } 223 | 224 | students := []Student{ 225 | {Name: "jerry", Birthdate: BirthDate(time.Date(1990, time.January, 1, 0, 0, 0, 0, time.UTC)), Address: "tokyo"}, 226 | {Name: "tom", Birthdate: BirthDate(time.Date(1989, time.November, 11, 0, 0, 0, 0, time.UTC)), Address: "new york"}, 227 | } 228 | data, err := csvlib.Marshal(students) 229 | if err != nil { 230 | fmt.Println("error:", err) 231 | } 232 | fmt.Println(string(data)) 233 | 234 | // Output: 235 | // name,birthdate,address 236 | // jerry,1990-01-01,tokyo 237 | // tom,1989-11-11,new york 238 | ``` 239 | 240 | ### Custom column delimiter 241 | 242 | - By default, the encoder uses comma as the delimiter, if you want to encode custom one, use `csv.Writer` from the built-in package `encoding/csv` as the input writer. 243 | 244 | [Playground](https://go.dev/play/p/WxOStP5cb6G) 245 | 246 | ```go 247 | type Student struct { 248 | Name string `csv:"name"` 249 | Age int `csv:"age"` 250 | Address string `csv:"address"` 251 | } 252 | 253 | students := []Student{ 254 | {Name: "jerry", Age: 20, Address: "tokyo"}, 255 | {Name: "tom", Age: 19, Address: "new york"}, 256 | } 257 | 258 | var buf bytes.Buffer 259 | writer := csv.NewWriter(&buf) 260 | writer.Comma = '\t' 261 | 262 | err := csvlib.NewEncoder(writer).Encode(students) 263 | if err != nil { 264 | fmt.Println("error:", err) 265 | } 266 | writer.Flush() 267 | fmt.Println(buf.String()) 268 | 269 | // Output: 270 | // name age address 271 | // jerry 20 tokyo 272 | // tom 19 new york 273 | ``` 274 | 275 | ### Encode one-by-one 276 | 277 | [Playground](https://go.dev/play/p/aQWk3qqlBVm) 278 | 279 | ```go 280 | type Student struct { 281 | Name string `csv:"name"` 282 | Age int `csv:"age"` 283 | Address string `csv:"address"` 284 | } 285 | 286 | students := []Student{ 287 | {Name: "jerry", Age: 20, Address: "tokyo"}, 288 | {Name: "tom", Age: 19, Address: "new york"}, 289 | } 290 | var buf bytes.Buffer 291 | writer := csv.NewWriter(&buf) 292 | encoder := csvlib.NewEncoder(writer) 293 | 294 | for _, student := range students { 295 | if err := encoder.EncodeOne(student); err != nil { 296 | fmt.Println("error:", err) 297 | break 298 | } 299 | } 300 | encoder.Finish() 301 | writer.Flush() 302 | fmt.Println(buf.String()) 303 | 304 | // Output: 305 | // name,age,address 306 | // jerry,20,tokyo 307 | // tom,19,new york 308 | ``` 309 | 310 | ### Header localization 311 | 312 | - This functionality allows to encode CSV data with header translated into a specific language. 313 | 314 | [Playground](https://go.dev/play/p/jH7joZPgWJO) 315 | 316 | ```go 317 | // Sample localization data (you can define them somewhere else such as in a json file) 318 | var ( 319 | mapLanguageEn = map[string]string{ 320 | "name": "name", 321 | "age": "age", 322 | "address": "address", 323 | } 324 | 325 | mapLanguageVi = map[string]string{ 326 | "name": "tên", 327 | "age": "tuổi", 328 | "address": "địa chỉ", 329 | } 330 | ) 331 | 332 | func localizeViVn(k string, params csvlib.ParameterMap) (string, error) { 333 | return mapLanguageVi[k], nil 334 | } 335 | 336 | func localizeEnUs(k string, params csvlib.ParameterMap) (string, error) { 337 | return mapLanguageEn[k], nil 338 | } 339 | ``` 340 | 341 | ```go 342 | type Student struct { 343 | Name string `csv:"name"` 344 | Age int `csv:"age"` 345 | Address string `csv:"address,omitempty"` 346 | } 347 | 348 | students := []Student{ 349 | {Name: "jerry", Age: 20, Address: "tokyo"}, 350 | {Name: "tom", Age: 19, Address: "new york"}, 351 | } 352 | data, err := csvlib.Marshal(students, func(cfg *csvlib.EncodeConfig) { 353 | cfg.LocalizeHeader = true 354 | cfg.LocalizationFunc = localizeViVn 355 | }) 356 | if err != nil { 357 | fmt.Println("error:", err) 358 | } 359 | fmt.Println(string(data)) 360 | 361 | // Output: 362 | // tên,tuổi,địa chỉ 363 | // jerry,20,tokyo 364 | // tom,19,new york 365 | ``` 366 | -------------------------------------------------------------------------------- /errors_render_as_csv.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "sort" 7 | "strconv" 8 | "strings" 9 | 10 | "github.com/hashicorp/go-multierror" 11 | "github.com/tiendc/gofn" 12 | ) 13 | 14 | type CSVRenderConfig struct { 15 | // CellSeparator separator to join cell error details within a row, normally a comma (`,`) 16 | CellSeparator string 17 | 18 | // LineBreak custom new line character (default is `\n`) 19 | LineBreak string 20 | 21 | // RenderHeader whether render header row or not 22 | RenderHeader bool 23 | 24 | // RenderRowNumberColumnIndex index of `row` column to render, set `-1` to not render it (default is `0`) 25 | RenderRowNumberColumnIndex int 26 | 27 | // RenderLineNumberColumnIndex index of `line` column to render, set `-1` to not render it (default is `-1`) 28 | RenderLineNumberColumnIndex int 29 | 30 | // RenderCommonErrorColumnIndex index of `common error` column to render, set `-1` to not render it 31 | // (default is `1`) 32 | RenderCommonErrorColumnIndex int 33 | 34 | // LocalizeCellFields localize cell's fields before rendering the cell error (default is `true`) 35 | LocalizeCellFields bool 36 | 37 | // LocalizeCellHeader localize cell header before rendering the cell error (default is `true`) 38 | LocalizeCellHeader bool 39 | 40 | // Params custom params user wants to send to the localization (optional) 41 | Params ParameterMap 42 | 43 | // LocalizationFunc function to translate message (optional) 44 | LocalizationFunc LocalizationFunc 45 | 46 | // HeaderRenderFunc custom render function for rendering header row (optional) 47 | HeaderRenderFunc func([]string, ParameterMap) 48 | 49 | // CellRenderFunc custom render function for rendering a cell error (optional). 50 | // The func can return ("", false) to skip rendering the cell error, return ("", true) to let the 51 | // renderer continue using its solution, and return ("", true) to override the value. 52 | // 53 | // Supported params: 54 | // {{.Column}} - column index (0-based) 55 | // {{.ColumnHeader}} - column name 56 | // {{.Value}} - cell value 57 | // {{.Error}} - error detail which is result of calling err.Error() 58 | // 59 | // Use cellErr.WithParam() to add more extra params 60 | CellRenderFunc func(*RowErrors, *CellError, ParameterMap) (string, bool) 61 | 62 | // CommonErrorRenderFunc renders common error (not RowErrors, CellError) (optional) 63 | CommonErrorRenderFunc func(error, ParameterMap) (string, error) 64 | } 65 | 66 | func defaultCSVRenderConfig() *CSVRenderConfig { 67 | return &CSVRenderConfig{ 68 | CellSeparator: ", ", 69 | LineBreak: newLine, 70 | 71 | RenderHeader: true, 72 | RenderRowNumberColumnIndex: 0, 73 | RenderLineNumberColumnIndex: 1, 74 | RenderCommonErrorColumnIndex: 2, //nolint:mnd 75 | 76 | LocalizeCellFields: true, 77 | LocalizeCellHeader: true, 78 | } 79 | } 80 | 81 | // CSVRenderer an implementation of error renderer which can produce messages 82 | // for the input errors as CSV output data. 83 | type CSVRenderer struct { 84 | cfg *CSVRenderConfig 85 | sourceErr *Errors 86 | transErr error 87 | numColumns int 88 | startCellErrIndex int 89 | data [][]string 90 | } 91 | 92 | // NewCSVRenderer creates a new CSVRenderer 93 | func NewCSVRenderer(err *Errors, options ...func(*CSVRenderConfig)) (*CSVRenderer, error) { 94 | cfg := defaultCSVRenderConfig() 95 | for _, opt := range options { 96 | opt(cfg) 97 | } 98 | // Validate/Correct the base columns to render 99 | baseColumns := make([]*int, 0, 3) //nolint:mnd 100 | if cfg.RenderRowNumberColumnIndex >= 0 { 101 | baseColumns = append(baseColumns, &cfg.RenderRowNumberColumnIndex) 102 | } 103 | if cfg.RenderLineNumberColumnIndex >= 0 { 104 | baseColumns = append(baseColumns, &cfg.RenderLineNumberColumnIndex) 105 | } 106 | if cfg.RenderCommonErrorColumnIndex >= 0 { 107 | baseColumns = append(baseColumns, &cfg.RenderCommonErrorColumnIndex) 108 | } 109 | sort.Slice(baseColumns, func(i, j int) bool { 110 | return *baseColumns[i] < *baseColumns[j] 111 | }) 112 | for i := range baseColumns { 113 | *baseColumns[i] = i 114 | } 115 | 116 | return &CSVRenderer{cfg: cfg, sourceErr: err}, nil 117 | } 118 | 119 | // Render renders Errors object as CSV rows data 120 | func (r *CSVRenderer) Render() (data [][]string, transErr error, err error) { 121 | cfg := r.cfg 122 | r.startCellErrIndex = 0 123 | if cfg.RenderRowNumberColumnIndex >= 0 { 124 | r.startCellErrIndex++ 125 | } 126 | if cfg.RenderLineNumberColumnIndex >= 0 { 127 | r.startCellErrIndex++ 128 | } 129 | if cfg.RenderCommonErrorColumnIndex >= 0 { 130 | r.startCellErrIndex++ 131 | } 132 | 133 | r.numColumns = len(r.sourceErr.Header()) + r.startCellErrIndex 134 | errs := r.sourceErr.Unwrap() 135 | r.data = make([][]string, 0, len(errs)+1) 136 | 137 | params := gofn.MapUpdate(ParameterMap{ 138 | "CrLf": cfg.LineBreak, 139 | "Tab": "\t", 140 | 141 | "TotalRow": r.sourceErr.TotalRow(), 142 | "TotalError": r.sourceErr.TotalError(), 143 | "TotalRowError": r.sourceErr.TotalRowError(), 144 | "TotalCellError": r.sourceErr.TotalCellError(), 145 | }, cfg.Params) 146 | 147 | // Render header row 148 | r.renderHeader(params) 149 | 150 | // Render rows content 151 | for _, err := range errs { 152 | if rowErr, ok := err.(*RowErrors); ok { // nolint: errorlint 153 | rowContent := r.renderRow(rowErr, params) 154 | r.data = append(r.data, rowContent) 155 | } else { 156 | _ = r.renderCommonError(err, params) 157 | } 158 | } 159 | return r.data, r.transErr, nil 160 | } 161 | 162 | // RenderAsString renders the input as CSV string 163 | func (r *CSVRenderer) RenderAsString() (msg string, transErr error, err error) { 164 | csvData, transErr, err := r.Render() 165 | if err != nil { 166 | return "", transErr, err 167 | } 168 | buf := bytes.NewBuffer(make([]byte, 0, r.estimateCSVBuffer(csvData))) 169 | w := csv.NewWriter(buf) 170 | if err = w.WriteAll(csvData); err != nil { 171 | return "", transErr, err 172 | } 173 | w.Flush() 174 | return buf.String(), transErr, nil 175 | } 176 | 177 | // RenderTo renders the input as CSV string and writes it to the writer 178 | func (r *CSVRenderer) RenderTo(w Writer) (transErr error, err error) { 179 | csvData, transErr, err := r.Render() 180 | if err != nil { 181 | return transErr, err 182 | } 183 | writeAll, canWriteAll := w.(interface{ WriteAll([][]string) error }) 184 | if canWriteAll { 185 | return transErr, writeAll.WriteAll(csvData) 186 | } 187 | for _, row := range csvData { 188 | err = w.Write(row) 189 | if err != nil { 190 | return transErr, err 191 | } 192 | } 193 | return transErr, nil 194 | } 195 | 196 | func (r *CSVRenderer) renderHeader(exparams ParameterMap) { 197 | if !r.cfg.RenderHeader { 198 | return 199 | } 200 | cfg := r.cfg 201 | header := make([]string, r.numColumns) 202 | if cfg.RenderRowNumberColumnIndex >= 0 { 203 | header[cfg.RenderRowNumberColumnIndex] = "Row" 204 | } 205 | if cfg.RenderLineNumberColumnIndex >= 0 { 206 | header[cfg.RenderLineNumberColumnIndex] = "Line" 207 | } 208 | if cfg.RenderCommonErrorColumnIndex >= 0 { 209 | header[cfg.RenderCommonErrorColumnIndex] = "CommonError" 210 | } 211 | for i := r.startCellErrIndex; i < r.numColumns; i++ { 212 | header[i] = r.sourceErr.header[i-r.startCellErrIndex] 213 | } 214 | 215 | if cfg.HeaderRenderFunc != nil { 216 | cfg.HeaderRenderFunc(header, exparams) 217 | } 218 | r.data = append(r.data, header) 219 | } 220 | 221 | func (r *CSVRenderer) renderRow(rowErr *RowErrors, exparams ParameterMap) []string { 222 | cfg := r.cfg 223 | content := make([]string, r.numColumns) 224 | 225 | if cfg.RenderRowNumberColumnIndex >= 0 { 226 | content[cfg.RenderRowNumberColumnIndex] = strconv.FormatInt(int64(rowErr.row), 10) 227 | } 228 | if cfg.RenderLineNumberColumnIndex >= 0 { 229 | content[cfg.RenderLineNumberColumnIndex] = strconv.FormatInt(int64(rowErr.line), 10) 230 | } 231 | 232 | errs := rowErr.Unwrap() 233 | mapErrByIndex := make(map[int][]string, r.numColumns) 234 | params := gofn.MapUpdate(ParameterMap{}, exparams) 235 | params["Row"] = rowErr.Row() 236 | params["Line"] = rowErr.Line() 237 | 238 | for _, err := range errs { 239 | if cellErr, ok := err.(*CellError); ok { // nolint: errorlint 240 | detail := r.renderCell(rowErr, cellErr, params) 241 | colIndex := cellErr.column + r.startCellErrIndex 242 | if cellErr.column == -1 { 243 | colIndex = cfg.RenderCommonErrorColumnIndex 244 | } 245 | if listItems, ok := mapErrByIndex[colIndex]; ok { 246 | mapErrByIndex[colIndex] = append(listItems, detail) 247 | } else { 248 | mapErrByIndex[colIndex] = []string{detail} 249 | } 250 | continue 251 | } 252 | // Common error 253 | detail := r.renderCommonError(err, params) 254 | if listItems, ok := mapErrByIndex[cfg.RenderCommonErrorColumnIndex]; ok { 255 | mapErrByIndex[cfg.RenderCommonErrorColumnIndex] = append(listItems, detail) 256 | } else { 257 | mapErrByIndex[cfg.RenderCommonErrorColumnIndex] = []string{detail} 258 | } 259 | } 260 | 261 | for index, items := range mapErrByIndex { 262 | content[index] = strings.Join(items, cfg.CellSeparator) 263 | } 264 | return content 265 | } 266 | 267 | func (r *CSVRenderer) renderCell(rowErr *RowErrors, cellErr *CellError, exparams ParameterMap) string { 268 | params := gofn.MapUpdate(ParameterMap{}, exparams) 269 | params = gofn.MapUpdate(params, r.renderCellFields(cellErr, params)) 270 | params["Column"] = cellErr.Column() 271 | params["ColumnHeader"] = r.renderCellHeader(cellErr, params) 272 | params["Value"] = cellErr.Value() 273 | params["Error"] = cellErr.Error() 274 | 275 | if r.cfg.CellRenderFunc != nil { 276 | msg, flag := r.cfg.CellRenderFunc(rowErr, cellErr, exparams) 277 | if !flag { 278 | return "" 279 | } 280 | if msg != "" { 281 | return msg 282 | } 283 | } 284 | 285 | locKey := cellErr.LocalizationKey() 286 | if locKey == "" { 287 | locKey = cellErr.Error() 288 | } 289 | return r.localizeKeySkipError(locKey, params) 290 | } 291 | 292 | func (r *CSVRenderer) renderCellFields(cellErr *CellError, params ParameterMap) ParameterMap { 293 | if !r.cfg.LocalizeCellFields { 294 | return cellErr.fields 295 | } 296 | result := make(ParameterMap, len(cellErr.fields)) 297 | for k, v := range cellErr.fields { 298 | vAsStr, ok := v.(string) 299 | if !ok { 300 | result[k] = v 301 | continue 302 | } 303 | if translated, err := r.localizeKey(vAsStr, params); err != nil { 304 | result[k] = v 305 | } else { 306 | result[k] = translated 307 | } 308 | } 309 | return result 310 | } 311 | 312 | func (r *CSVRenderer) renderCellHeader(cellErr *CellError, params ParameterMap) string { 313 | if !r.cfg.LocalizeCellHeader { 314 | return cellErr.Header() 315 | } 316 | return r.localizeKeySkipError(cellErr.Header(), params) 317 | } 318 | 319 | func (r *CSVRenderer) renderCommonError(err error, params ParameterMap) string { 320 | if r.cfg.CommonErrorRenderFunc == nil { 321 | return r.localizeKeySkipError(err.Error(), params) 322 | } 323 | msg, err := r.cfg.CommonErrorRenderFunc(err, params) 324 | if err != nil { 325 | r.transErr = multierror.Append(r.transErr, err) 326 | } 327 | return msg 328 | } 329 | 330 | func (r *CSVRenderer) localizeKey(key string, params ParameterMap) (string, error) { 331 | if r.cfg.LocalizationFunc == nil { 332 | return processTemplate(key, params) 333 | } 334 | msg, err := r.cfg.LocalizationFunc(key, params) 335 | if err != nil { 336 | err = multierror.Append(ErrLocalization, err) 337 | r.transErr = multierror.Append(r.transErr, err) 338 | return "", err 339 | } 340 | return msg, nil 341 | } 342 | 343 | func (r *CSVRenderer) localizeKeySkipError(key string, params ParameterMap) string { 344 | s, err := r.localizeKey(key, params) 345 | if err == nil || r.cfg.LocalizationFunc == nil { 346 | return s 347 | } 348 | s, _ = processTemplate(key, params) 349 | return s 350 | } 351 | 352 | func (r *CSVRenderer) estimateCSVBuffer(data [][]string) int { 353 | if len(data) <= 1 { 354 | return 512 //nolint:mnd 355 | } 356 | row := data[1] 357 | rowSz := 0 358 | for _, v := range row { 359 | rowSz += len(v) 360 | } 361 | return (rowSz + len(row)) * len(data) 362 | } 363 | -------------------------------------------------------------------------------- /encoder.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "fmt" 5 | "reflect" 6 | "strings" 7 | 8 | "github.com/hashicorp/go-multierror" 9 | "github.com/tiendc/gofn" 10 | ) 11 | 12 | // EncodeConfig configuration for encoding Go structs as CSV data 13 | type EncodeConfig struct { 14 | // TagName tag name to parse the struct (default is `csv`) 15 | TagName string 16 | 17 | // NoHeaderMode indicates whether to write header or not (default is `false`) 18 | NoHeaderMode bool 19 | 20 | // LocalizeHeader indicates whether to localize the header or not (default is `false`) 21 | LocalizeHeader bool 22 | 23 | // LocalizationFunc localization function, required when LocalizeHeader is true 24 | LocalizationFunc LocalizationFunc 25 | 26 | // columnConfigMap a map consists of configuration for specific columns (optional) 27 | columnConfigMap map[string]*EncodeColumnConfig 28 | } 29 | 30 | func defaultEncodeConfig() *EncodeConfig { 31 | return &EncodeConfig{ 32 | TagName: DefaultTagName, 33 | } 34 | } 35 | 36 | // ConfigureColumn configures encoding for a column by name 37 | func (c *EncodeConfig) ConfigureColumn(name string, fn func(*EncodeColumnConfig)) { 38 | if c.columnConfigMap == nil { 39 | c.columnConfigMap = map[string]*EncodeColumnConfig{} 40 | } 41 | columnCfg, ok := c.columnConfigMap[name] 42 | if !ok { 43 | columnCfg = defaultEncodeColumnConfig() 44 | c.columnConfigMap[name] = columnCfg 45 | } 46 | fn(columnCfg) 47 | } 48 | 49 | // EncodeColumnConfig configuration for encoding a specific column 50 | type EncodeColumnConfig struct { 51 | // Skip whether skip encoding the column or not (this is equivalent to use `csv:"-"` in struct tag) 52 | // (default is `false`) 53 | Skip bool 54 | 55 | // EncodeFunc custom encode function (optional) 56 | EncodeFunc EncodeFunc 57 | 58 | // PostprocessorFuncs a list of functions will be called after encoding a cell value (optional) 59 | PostprocessorFuncs []ProcessorFunc 60 | } 61 | 62 | func defaultEncodeColumnConfig() *EncodeColumnConfig { 63 | return &EncodeColumnConfig{} 64 | } 65 | 66 | // EncodeOption function to modify encoding config 67 | type EncodeOption func(cfg *EncodeConfig) 68 | 69 | // Encoder data structure of the default encoder 70 | type Encoder struct { 71 | w Writer 72 | cfg *EncodeConfig 73 | err error 74 | finished bool 75 | itemType reflect.Type 76 | headerWritten bool 77 | hasDynamicInlineColumns bool 78 | hasFixedInlineColumns bool 79 | colsMeta []*encodeColumnMeta 80 | } 81 | 82 | // NewEncoder creates a new Encoder object 83 | func NewEncoder(w Writer, options ...EncodeOption) *Encoder { 84 | cfg := defaultEncodeConfig() 85 | for _, opt := range options { 86 | opt(cfg) 87 | } 88 | return &Encoder{ 89 | w: w, 90 | cfg: cfg, 91 | } 92 | } 93 | 94 | // Encode encode input data stored in the given variable. 95 | // The input var must be a slice, e.g. `[]Student` or `[]*Student`. 96 | func (e *Encoder) Encode(v any) error { 97 | if e.finished { 98 | return ErrFinished 99 | } 100 | if e.err != nil { 101 | return ErrAlreadyFailed 102 | } 103 | 104 | val := reflect.ValueOf(v) 105 | if e.itemType == nil { 106 | if err := e.prepareEncode(val); err != nil { 107 | e.err = err 108 | return err 109 | } 110 | } else { 111 | itemType, err := e.parseInputVar(val) 112 | if err != nil { 113 | return err 114 | } 115 | if itemType != e.itemType { 116 | return fmt.Errorf("%w: %v (expect %v)", ErrTypeUnmatched, itemType, e.itemType) 117 | } 118 | } 119 | 120 | totalRow := val.Len() 121 | itemKindIsPtr := e.itemType.Kind() == reflect.Pointer 122 | for row := 0; row < totalRow; row++ { 123 | rowVal := val.Index(row) 124 | if itemKindIsPtr { 125 | if rowVal.IsNil() { 126 | continue 127 | } 128 | rowVal = rowVal.Elem() 129 | } 130 | if err := e.encodeRow(rowVal); err != nil { 131 | e.err = err 132 | break 133 | } 134 | } 135 | return e.err 136 | } 137 | 138 | // EncodeOne encode single object into a single CSV row 139 | func (e *Encoder) EncodeOne(v any) error { 140 | if e.finished { 141 | return ErrFinished 142 | } 143 | if e.err != nil { 144 | return ErrAlreadyFailed 145 | } 146 | 147 | rowVal := reflect.ValueOf(v) 148 | itemType := rowVal.Type() 149 | if !isKindOrPtrOf(itemType, reflect.Struct) { 150 | return fmt.Errorf("%w: must be a struct", ErrTypeInvalid) 151 | } 152 | if e.itemType == nil { 153 | slice := reflect.MakeSlice(reflect.SliceOf(itemType), 1, 1) 154 | slice.Index(0).Set(rowVal) 155 | err := e.prepareEncode(slice) 156 | if err != nil { 157 | e.err = err 158 | return err 159 | } 160 | } else if itemType != e.itemType { 161 | return fmt.Errorf("%w: %v (expect %v)", ErrTypeUnmatched, itemType, e.itemType) 162 | } 163 | 164 | if err := e.encodeRow(rowVal); err != nil { 165 | e.err = err 166 | return err 167 | } 168 | return nil 169 | } 170 | 171 | // Finish encoding, after calling this func, you can't encode more 172 | func (e *Encoder) Finish() error { 173 | e.finished = true 174 | return e.err 175 | } 176 | 177 | func (e *Encoder) prepareEncode(v reflect.Value) error { 178 | if e.itemType != nil { 179 | return fmt.Errorf("%w: item type already parsed", ErrUnexpected) 180 | } 181 | itemType, err := e.parseInputVar(v) 182 | if err != nil { 183 | return err 184 | } 185 | e.itemType = itemType 186 | 187 | if err = e.validateConfig(); err != nil { 188 | return err 189 | } 190 | 191 | if err = e.parseColumnsMeta(itemType, v); err != nil { 192 | return err 193 | } 194 | 195 | if err = e.buildColumnEncoders(); err != nil { 196 | return err 197 | } 198 | 199 | if err = e.encodeHeader(); err != nil { 200 | return err 201 | } 202 | return nil 203 | } 204 | 205 | func (e *Encoder) encodeHeader() error { 206 | if e.headerWritten { 207 | return fmt.Errorf("%w: header already encoded", ErrUnexpected) 208 | } 209 | record := make([]string, 0, len(e.colsMeta)) 210 | for _, colMeta := range e.colsMeta { 211 | if colMeta.skipColumn { 212 | continue 213 | } 214 | record = append(record, colMeta.headerText) 215 | } 216 | if err := validateHeader(record); err != nil { 217 | return err 218 | } 219 | if e.cfg.NoHeaderMode { 220 | e.headerWritten = true 221 | return nil 222 | } else { 223 | err := e.w.Write(record) 224 | e.headerWritten = err == nil 225 | return err 226 | } 227 | } 228 | 229 | func (e *Encoder) encodeRow(rowVal reflect.Value) error { 230 | colsMeta := e.colsMeta 231 | if e.hasDynamicInlineColumns || e.hasFixedInlineColumns { 232 | for _, colMeta := range colsMeta { 233 | if colMeta.inlineColumnMeta != nil { 234 | colMeta.inlineColumnMeta.encodePrepareForNextRow() 235 | } 236 | } 237 | } 238 | 239 | record := make([]string, 0, len(colsMeta)) 240 | for _, colMeta := range colsMeta { 241 | if colMeta.skipColumn { 242 | continue 243 | } 244 | colVal := colMeta.getColumnValue(rowVal) 245 | if !colVal.IsValid() { 246 | record = append(record, "") 247 | continue 248 | } 249 | text, err := colMeta.encodeFunc(colVal, colMeta.omitEmpty) 250 | if err != nil { 251 | return err 252 | } 253 | for _, fn := range colMeta.postprocessorFuncs { 254 | text = fn(text) 255 | } 256 | record = append(record, text) 257 | } 258 | return e.w.Write(record) 259 | } 260 | 261 | func (e *Encoder) parseInputVar(v reflect.Value) (itemType reflect.Type, err error) { 262 | kind := v.Kind() 263 | if kind != reflect.Slice && kind != reflect.Array { 264 | err = fmt.Errorf("%w: %v", ErrTypeInvalid, kind) 265 | return 266 | } 267 | if v.IsNil() { 268 | err = fmt.Errorf("%w: must be non-nil pointer", ErrValueNil) 269 | return 270 | } 271 | 272 | typ := v.Type() 273 | itemType = typ.Elem() // E.g. val: []Item, typ: []Item, itemType: Item 274 | 275 | if indirectType(itemType).Kind() != reflect.Struct { 276 | err = fmt.Errorf("%w: %v", ErrTypeInvalid, itemType.Kind()) 277 | return 278 | } 279 | return 280 | } 281 | 282 | func (e *Encoder) validateConfig() error { 283 | if e.cfg.LocalizeHeader && e.cfg.LocalizationFunc == nil { 284 | return fmt.Errorf("%w: localization function required", ErrConfigOptionInvalid) 285 | } 286 | return nil 287 | } 288 | 289 | func (e *Encoder) parseColumnsMeta(itemType reflect.Type, val reflect.Value) error { 290 | colsMeta, err := e.parseColumnsMetaFromStructType(itemType, val) 291 | if err != nil { 292 | return err 293 | } 294 | 295 | if err = e.validateColumnsMeta(colsMeta); err != nil { 296 | return err 297 | } 298 | 299 | e.colsMeta = colsMeta 300 | return nil 301 | } 302 | 303 | func (e *Encoder) validateColumnsMeta(colsMeta []*encodeColumnMeta) error { 304 | cfg := e.cfg 305 | // Make sure all column options valid 306 | for colKey := range cfg.columnConfigMap { 307 | if !gofn.ContainBy(colsMeta, func(colMeta *encodeColumnMeta) bool { 308 | return colMeta.headerKey == colKey || colMeta.parentKey == colKey 309 | }) { 310 | return fmt.Errorf("%w: column \"%s\" not found", ErrConfigOptionInvalid, colKey) 311 | } 312 | } 313 | 314 | // Make sure all columns are unique 315 | if err := e.validateHeaderUniqueness(colsMeta); err != nil { 316 | return err 317 | } 318 | return nil 319 | } 320 | 321 | func (e *Encoder) parseColumnsMetaFromStructType(itemType reflect.Type, val reflect.Value) ( 322 | colsMeta []*encodeColumnMeta, err error) { 323 | cfg := e.cfg 324 | 325 | // Get first row data from the input 326 | firstRowVal := reflect.Value{} 327 | if val.Len() > 0 { 328 | firstRowVal = val.Index(0) 329 | } 330 | if firstRowVal.IsValid() && firstRowVal.Kind() == reflect.Pointer { 331 | firstRowVal = firstRowVal.Elem() 332 | } 333 | 334 | itemType = indirectType(itemType) 335 | numFields := itemType.NumField() 336 | for i := 0; i < numFields; i++ { 337 | field := itemType.Field(i) 338 | tag, err := parseTag(cfg.TagName, field) 339 | if err != nil { 340 | return nil, err 341 | } 342 | if tag == nil || tag.ignored { 343 | continue 344 | } 345 | 346 | colMeta := &encodeColumnMeta{ 347 | column: len(colsMeta), 348 | headerKey: tag.name, 349 | headerText: tag.name, 350 | prefix: tag.prefix, 351 | omitEmpty: tag.omitEmpty, 352 | targetField: field, 353 | } 354 | if tag.inline { 355 | inlineColsMeta, err := e.parseInlineColumn(field, colMeta, firstRowVal) 356 | if err != nil { 357 | return nil, err 358 | } 359 | colsMeta = append(colsMeta, inlineColsMeta...) 360 | continue 361 | } 362 | 363 | colMeta.copyConfig(cfg.columnConfigMap[colMeta.headerKey]) 364 | if err = colMeta.localizeHeader(cfg); err != nil { 365 | return nil, err 366 | } 367 | 368 | colsMeta = append(colsMeta, colMeta) 369 | } 370 | 371 | for i, colMeta := range colsMeta { 372 | colMeta.column = i 373 | } 374 | return colsMeta, err 375 | } 376 | 377 | func (e *Encoder) parseInlineColumn(field reflect.StructField, parentCol *encodeColumnMeta, firstRowVal reflect.Value) ( 378 | colsMeta []*encodeColumnMeta, err error) { 379 | if firstRowVal.IsValid() { 380 | inlineStruct := firstRowVal.Field(field.Index[0]) 381 | inlineColumnsMeta, err := e.parseInlineColumnDynamicType(inlineStruct, parentCol) 382 | if err == nil { 383 | e.hasDynamicInlineColumns = true 384 | return inlineColumnsMeta, nil 385 | } 386 | } 387 | inlineColumnsMeta, err := e.parseInlineColumnFixedType(field.Type, parentCol) 388 | if err == nil && len(inlineColumnsMeta) > 0 { 389 | e.hasFixedInlineColumns = true 390 | return inlineColumnsMeta, nil 391 | } 392 | return nil, fmt.Errorf("%w: %v", ErrHeaderDynamicTypeInvalid, field.Type) 393 | } 394 | 395 | func (e *Encoder) parseInlineColumnFixedType(typ reflect.Type, parent *encodeColumnMeta) ([]*encodeColumnMeta, error) { 396 | cfg := e.cfg 397 | typ = indirectType(typ) 398 | if typ.Kind() != reflect.Struct { 399 | return nil, fmt.Errorf("%w: not struct type", ErrHeaderDynamicTypeInvalid) 400 | } 401 | numFields := typ.NumField() 402 | colsMeta := make([]*encodeColumnMeta, 0, numFields) 403 | for i := 0; i < numFields; i++ { 404 | field := typ.Field(i) 405 | tag, err := parseTag(cfg.TagName, field) 406 | if err != nil { 407 | return nil, err 408 | } 409 | if tag == nil || tag.ignored { 410 | continue 411 | } 412 | 413 | headerKey := parent.prefix + tag.name 414 | colMeta := &encodeColumnMeta{ 415 | column: len(colsMeta), 416 | headerKey: headerKey, 417 | headerText: headerKey, 418 | parentKey: parent.headerKey, 419 | targetField: parent.targetField, 420 | inlineColumnMeta: &inlineColumnMeta{ 421 | inlineType: inlineColumnStructFixed, 422 | targetField: field, 423 | dataType: field.Type, 424 | }, 425 | } 426 | 427 | columnCfg := cfg.columnConfigMap[headerKey] 428 | if columnCfg == nil { 429 | columnCfg = cfg.columnConfigMap[colMeta.parentKey] 430 | } 431 | colMeta.copyConfig(columnCfg) 432 | if err = colMeta.localizeHeader(cfg); err != nil { 433 | return nil, err 434 | } 435 | 436 | colsMeta = append(colsMeta, colMeta) 437 | } 438 | return colsMeta, nil 439 | } 440 | 441 | func (e *Encoder) parseInlineColumnDynamicType(inlineStruct reflect.Value, parent *encodeColumnMeta) ( 442 | []*encodeColumnMeta, error) { 443 | cfg := e.cfg 444 | inlineStruct = indirectValue(inlineStruct) 445 | if !inlineStruct.IsValid() { 446 | return []*encodeColumnMeta{}, nil 447 | } 448 | if inlineStruct.Kind() != reflect.Struct { 449 | return nil, fmt.Errorf("%w: not struct type", ErrHeaderDynamicTypeInvalid) 450 | } 451 | headerField := inlineStruct.FieldByName(dynamicInlineColumnHeader) 452 | if !headerField.IsValid() { 453 | return nil, fmt.Errorf("%w: field Header not found", ErrHeaderDynamicTypeInvalid) 454 | } 455 | if headerField.Type() != reflect.TypeOf([]string{}) { 456 | return nil, fmt.Errorf("%w: field Header not []string", ErrHeaderDynamicTypeInvalid) 457 | } 458 | 459 | valuesField, ok := inlineStruct.Type().FieldByName(dynamicInlineColumnValues) 460 | if !ok { 461 | return nil, fmt.Errorf("%w: field Values not found", ErrHeaderDynamicTypeInvalid) 462 | } 463 | if valuesField.Type.Kind() != reflect.Slice { 464 | return nil, fmt.Errorf("%w: field Values not slice", ErrHeaderDynamicTypeInvalid) 465 | } 466 | 467 | dataType := valuesField.Type.Elem() 468 | header, _ := headerField.Interface().([]string) 469 | 470 | colsMeta := make([]*encodeColumnMeta, 0, len(header)) 471 | inlineColumnMeta := &inlineColumnMeta{ 472 | headerText: header, 473 | inlineType: inlineColumnStructDynamic, 474 | targetField: valuesField, 475 | dataType: dataType, 476 | } 477 | for _, h := range header { 478 | headerKey := parent.prefix + h 479 | colMeta := *parent 480 | colMeta.headerKey = headerKey 481 | colMeta.headerText = headerKey 482 | colMeta.parentKey = parent.headerKey 483 | colMeta.inlineColumnMeta = inlineColumnMeta 484 | 485 | columnCfg := cfg.columnConfigMap[colMeta.headerKey] 486 | if columnCfg == nil { 487 | columnCfg = cfg.columnConfigMap[colMeta.parentKey] 488 | } 489 | colMeta.copyConfig(columnCfg) 490 | 491 | // Try to localize header (ignore the error when fail) 492 | _ = colMeta.localizeHeader(cfg) 493 | 494 | colsMeta = append(colsMeta, &colMeta) 495 | } 496 | 497 | return colsMeta, nil 498 | } 499 | 500 | func (e *Encoder) buildColumnEncoders() error { 501 | for _, colMeta := range e.colsMeta { 502 | if colMeta.encodeFunc != nil { 503 | continue 504 | } 505 | dataType := colMeta.targetField.Type 506 | if colMeta.inlineColumnMeta != nil { 507 | dataType = colMeta.inlineColumnMeta.dataType 508 | } 509 | encodeFunc, err := getEncodeFunc(dataType) 510 | if err != nil { 511 | return err 512 | } 513 | colMeta.encodeFunc = encodeFunc 514 | } 515 | return nil 516 | } 517 | 518 | // validateHeaderUniqueness validate to make sure header columns are unique 519 | func (e *Encoder) validateHeaderUniqueness(colsMeta []*encodeColumnMeta) error { 520 | mapCheckUniq := make(map[string]struct{}, len(colsMeta)) 521 | for _, colMeta := range colsMeta { 522 | h := colMeta.headerKey 523 | hh := strings.TrimSpace(h) 524 | if h != hh || len(hh) == 0 { 525 | return fmt.Errorf("%w: \"%s\" invalid", ErrHeaderColumnInvalid, h) 526 | } 527 | isDynamicInline := colMeta.inlineColumnMeta != nil && 528 | colMeta.inlineColumnMeta.inlineType == inlineColumnStructDynamic 529 | if _, ok := mapCheckUniq[hh]; ok && !isDynamicInline { 530 | return fmt.Errorf("%w: \"%s\" duplicated", ErrHeaderColumnDuplicated, h) 531 | } 532 | mapCheckUniq[hh] = struct{}{} 533 | } 534 | return nil 535 | } 536 | 537 | type encodeColumnMeta struct { 538 | column int 539 | headerKey string 540 | headerText string 541 | parentKey string 542 | prefix string 543 | omitEmpty bool 544 | skipColumn bool 545 | 546 | targetField reflect.StructField 547 | inlineColumnMeta *inlineColumnMeta 548 | 549 | encodeFunc EncodeFunc 550 | postprocessorFuncs []ProcessorFunc 551 | } 552 | 553 | func (m *encodeColumnMeta) localizeHeader(cfg *EncodeConfig) error { 554 | if cfg.LocalizeHeader { 555 | headerText, err := cfg.LocalizationFunc(m.headerKey, nil) 556 | if err != nil { 557 | return multierror.Append(ErrLocalization, err) 558 | } 559 | m.headerText = headerText 560 | } 561 | return nil 562 | } 563 | 564 | func (m *encodeColumnMeta) copyConfig(columnCfg *EncodeColumnConfig) { 565 | if columnCfg == nil { 566 | return 567 | } 568 | m.skipColumn = columnCfg.Skip 569 | m.encodeFunc = columnCfg.EncodeFunc 570 | m.postprocessorFuncs = columnCfg.PostprocessorFuncs 571 | } 572 | 573 | func (m *encodeColumnMeta) getColumnValue(rowVal reflect.Value) reflect.Value { 574 | colVal := rowVal.Field(m.targetField.Index[0]) 575 | if m.inlineColumnMeta != nil { 576 | colVal = m.inlineColumnMeta.encodeGetColumnValue(colVal) 577 | } 578 | return colVal 579 | } 580 | -------------------------------------------------------------------------------- /docs/DECODING.md: -------------------------------------------------------------------------------- 1 | # Decoding CSV 2 | 3 | ## Index 4 | - [First example](#first-example) 5 | - [Preprocessor and Validator](#preprocessor-and-validator) 6 | - [When StopOnError is false](#when-stoponerror-is-false) 7 | - [Optional and Unrecognized columns](#optional-and-unrecognized-columns) 8 | - [Allow unordered header columns](#allow-unordered-header-columns) 9 | - [Fixed inline columns](#fixed-inline-columns) 10 | - [Dynamic inline columns](#dynamic-inline-columns) 11 | - [Custom unmarshaler](#custom-unmarshaler) 12 | - [Custom column delimiter](#custom-column-delimiter) 13 | - [Decode one-by-one](#decode-one-by-one) 14 | - [Header localization](#header-localization) 15 | - [Render error as human-readable format](#render-error-as-human-readable-format) 16 | 17 | ## Content 18 | 19 | ### First example 20 | 21 | [Playground](https://go.dev/play/p/tUw9SfaD8QK) 22 | 23 | ```go 24 | data := []byte(` 25 | name,birthdate,address 26 | jerry,1990-01-01T15:00:00Z, 27 | tom,1989-11-11T00:00:00Z,new york`) 28 | 29 | type Student struct { 30 | Name string `csv:"name"` 31 | Birthdate time.Time `csv:"birthdate"` 32 | Address string `csv:"address,omitempty"` 33 | } 34 | 35 | var students []Student 36 | result, err := csvlib.Unmarshal(data, &students) 37 | if err != nil { 38 | fmt.Println("error:", err) 39 | } 40 | 41 | fmt.Printf("%+v\n", *result) 42 | for _, u := range students { 43 | fmt.Printf("%+v\n", u) 44 | } 45 | 46 | // Output: 47 | // {totalRow:3 unrecognizedColumns:[] missingOptionalColumns:[]} 48 | // {Name:jerry Birthdate:1990-01-01 15:00:00 +0000 UTC Address:} 49 | // {Name:tom Birthdate:1989-11-11 00:00:00 +0000 UTC Address:new york} 50 | ``` 51 | 52 | ### Preprocessor and Validator 53 | 54 | - Preprocessor functions will be called before cell values are decoded and validator functions will be called after that. 55 | 56 | [Playground](https://go.dev/play/p/Ndn2sj4030c) 57 | 58 | ```go 59 | data := []byte(` 60 | name,age,address 61 | jerry ,20, 62 | tom ,26,new york`) 63 | 64 | type Student struct { 65 | Name string `csv:"name"` 66 | Age int `csv:"age"` 67 | Address string `csv:"address,omitempty"` 68 | } 69 | 70 | var students []Student 71 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 72 | cfg.ConfigureColumn("name", func(cfg *csvlib.DecodeColumnConfig) { 73 | cfg.TrimSpace = true // you can use csvlib.ProcessorTrim as an alternative 74 | cfg.PreprocessorFuncs = []csvlib.ProcessorFunc{csvlib.ProcessorUpper} 75 | }) 76 | 77 | cfg.ConfigureColumn("age", func(cfg *csvlib.DecodeColumnConfig) { 78 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(20, 30)} 79 | }) 80 | }) 81 | if err != nil { 82 | fmt.Println("error:", err) 83 | } 84 | 85 | fmt.Printf("%+v\n", *result) 86 | for _, u := range students { 87 | fmt.Printf("%+v\n", u) 88 | } 89 | 90 | // Success: when use ValidatorRange(20, 30) as above 91 | // {Name:JERRY Age:20 Address:} 92 | // {Name:TOM Age:26 Address:new york} 93 | 94 | // Failure: when use ValidatorRange(10, 20) instead 95 | // error: ErrValidation: Range 96 | ``` 97 | 98 | ### When StopOnError is false 99 | 100 | - When set `StopOnError = false`, the decoding will continue to process the data even when errors occur. You can handle all errors of the process at once. 101 | 102 | [Playground](https://go.dev/play/p/mV1UX69mHWS) 103 | 104 | ```go 105 | data := []byte(` 106 | name,age,address 107 | jerry,30, 108 | tom,26,new york`) 109 | 110 | type Student struct { 111 | Name string `csv:"name"` 112 | Age int `csv:"age"` 113 | Address string `csv:"address,omitempty"` 114 | } 115 | 116 | var students []Student 117 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 118 | cfg.StopOnError = false 119 | cfg.ConfigureColumn("age", func(cfg *csvlib.DecodeColumnConfig) { 120 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(10, 20)} 121 | }) 122 | }) 123 | if err != nil { 124 | csvErr := err.(*csvlib.Errors) 125 | fmt.Println("error 1:", csvErr.Unwrap()[0]) 126 | fmt.Println("error 2:", csvErr.Unwrap()[1]) 127 | } 128 | 129 | // Output: 130 | // error 1: ErrValidation: Range 131 | // error 2: ErrValidation: Range 132 | ``` 133 | 134 | ### Optional and Unrecognized columns 135 | 136 | - Optional column is a column which is defined in the struct tag but not exist in the input data 137 | - Unrecognized column is a column which exists in the input data but is not defined in the struct tag 138 | 139 | [Playground](https://go.dev/play/p/OA8Jh3Y4aaD) 140 | 141 | ```go 142 | data := []byte(` 143 | name,age,mark 144 | jerry,20,10 145 | tom,26,9`) 146 | 147 | type Student struct { 148 | Name string `csv:"name"` 149 | Age int `csv:"age"` 150 | Address string `csv:"address,optional"` // without `optional` tag, it will fail to decode 151 | } 152 | 153 | var students []Student 154 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 155 | cfg.AllowUnrecognizedColumns = true // without this config, it will fail to decode 156 | }) 157 | if err != nil { 158 | fmt.Println("error:", err) 159 | } 160 | 161 | fmt.Printf("%+v\n", *result) 162 | for _, u := range students { 163 | fmt.Printf("%+v\n", u) 164 | } 165 | 166 | // Output: 167 | // {totalRow:3 unrecognizedColumns:[mark] missingOptionalColumns:[address]} 168 | // {Name:jerry Age:20 Address:} 169 | // {Name:tom Age:26 Address:} 170 | ``` 171 | 172 | ### Allow unordered header columns 173 | 174 | - By default, the header order in the input data must match the order defined in the struct tag. 175 | 176 | [Playground](https://go.dev/play/p/3va5HYWacPO) 177 | 178 | ```go 179 | // `address` appears before `age` 180 | data := []byte(` 181 | name,address,age 182 | jerry,tokyo,10 183 | tom,new york,9`) 184 | 185 | type Student struct { 186 | Name string `csv:"name"` 187 | Age int `csv:"age"` 188 | Address string `csv:"address,optional"` 189 | } 190 | 191 | var students []Student 192 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 193 | cfg.RequireColumnOrder = false // without this config, it will fail to decode 194 | }) 195 | if err != nil { 196 | fmt.Println("error:", err) 197 | } 198 | 199 | fmt.Printf("%+v\n", *result) 200 | for _, u := range students { 201 | fmt.Printf("%+v\n", u) 202 | } 203 | 204 | // Output: 205 | // {Name:jerry Age:10 Address:tokyo} 206 | // {Name:tom Age:9 Address:new york} 207 | ``` 208 | 209 | ### Fixed inline columns 210 | 211 | - Fixed inline columns are represented by an inner struct. `prefix` can be set for inline columns. 212 | 213 | [Playground](https://go.dev/play/p/ouZDv4zjfSA) 214 | 215 | ```go 216 | data := []byte(` 217 | name,age,mark_math,mark_chemistry,mark_physics 218 | jerry,20,9,8,7 219 | tom,19,7,8,9`) 220 | 221 | type Marks struct { 222 | Math int `csv:"math"` 223 | Chemistry int `csv:"chemistry"` 224 | Physics int `csv:"physics"` 225 | } 226 | type Student struct { 227 | Name string `csv:"name"` 228 | Age int `csv:"age"` 229 | Address string `csv:"address,optional"` 230 | Marks Marks `csv:"marks,inline,prefix=mark_"` 231 | } 232 | 233 | var students []Student 234 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 235 | cfg.ConfigureColumn("mark_math", func(cfg *csvlib.DecodeColumnConfig) { // `math` field will be decoded with these configurations 236 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(0, 10)} 237 | }) 238 | cfg.ConfigureColumn("marks", func(cfg *csvlib.DecodeColumnConfig) { // `chemistry` and `physics` fields will be decoded with these configurations 239 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(1, 10)} 240 | }) 241 | }) 242 | if err != nil { 243 | fmt.Println("error:", err) 244 | } 245 | 246 | fmt.Printf("%+v\n", *result) 247 | for _, u := range students { 248 | fmt.Printf("%+v\n", u) 249 | } 250 | 251 | // Output: 252 | // {Name:jerry Age:20 Address: Marks:{Math:9 Chemistry:8 Physics:7}} 253 | // {Name:tom Age:19 Address: Marks:{Math:7 Chemistry:8 Physics:9}} 254 | ``` 255 | 256 | ### Dynamic inline columns 257 | 258 | - Dynamic inline columns are represented by an inner struct with variable number of columns. `prefix` can also be set for them. 259 | 260 | [Playground](https://go.dev/play/p/5SCRzMp6NZP) 261 | 262 | ```go 263 | data := []byte(` 264 | name,age,mark_math,mark_chemistry,mark_physics 265 | jerry,20,9,8,7 266 | tom,19,7,8,9`) 267 | 268 | type Student struct { 269 | Name string `csv:"name"` 270 | Age int `csv:"age"` 271 | Address string `csv:"address,optional"` 272 | Marks csvlib.InlineColumn[int] `csv:"marks,inline,prefix=mark_"` 273 | } 274 | 275 | var students []Student 276 | result, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 277 | cfg.ConfigureColumn("marks", func(cfg *csvlib.DecodeColumnConfig) { // all inline columns are validated by this 278 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(1, 10)} 279 | }) 280 | }) 281 | if err != nil { 282 | fmt.Println("error:", err) 283 | } 284 | 285 | fmt.Printf("%+v\n", *result) 286 | for _, u := range students { 287 | fmt.Printf("%+v\n", u) 288 | } 289 | 290 | // Output: 291 | // {Name:jerry Age:20 Address: Marks:{Header:[mark_math mark_chemistry mark_physics] Values:[9 8 7]}} 292 | // {Name:tom Age:19 Address: Marks:{Header:[mark_math mark_chemistry mark_physics] Values:[7 8 9]}} 293 | ``` 294 | 295 | ### Custom unmarshaler 296 | 297 | - Any user custom type can be decoded by implementing either `encoding.TextUnmarshaler` or `csvlib.CSVUnmarshaler` or both. `csvlib.CSVUnmarshaler` has higher priority. 298 | 299 | [Playground](https://go.dev/play/p/g_QmAN6di-9) 300 | 301 | ```go 302 | type BirthDate time.Time 303 | 304 | func (d *BirthDate) UnmarshalCSV(data []byte) error { 305 | t, err := time.Parse("2006-1-2", string(data)) // RFC3339 format 306 | if err != nil { 307 | return err 308 | } 309 | *d = BirthDate(t) 310 | return nil 311 | } 312 | 313 | func (d BirthDate) String() string { // this is for displaying the result only 314 | return time.Time(d).Format("2006-01-02") 315 | } 316 | ``` 317 | ```go 318 | data := []byte(` 319 | name,birthdate 320 | jerry,1990-01-01 321 | tom,1989-11-11`) 322 | 323 | type Student struct { 324 | Name string `csv:"name"` 325 | BirthDate BirthDate `csv:"birthdate"` 326 | } 327 | 328 | var students []Student 329 | result, err := csvlib.Unmarshal(data, &students) 330 | if err != nil { 331 | fmt.Println("error:", err) 332 | } 333 | 334 | fmt.Printf("%+v\n", *result) 335 | for _, u := range students { 336 | fmt.Printf("%+v\n", u) 337 | } 338 | 339 | // Output: 340 | // {Name:jerry BirthDate:1990-01-01} 341 | // {Name:tom BirthDate:1989-11-11} 342 | ``` 343 | 344 | ### Custom column delimiter 345 | 346 | - By default, the decoder detects columns via comma delimiter, if you want to decode custom one, use `csv.Reader` from the built-in package `encoding/csv` as the input reader. 347 | 348 | [Playground](https://go.dev/play/p/HgxcyzeHho6) 349 | 350 | ```go 351 | data := []byte( 352 | "name\tage\taddress\n" + 353 | "jerry\t10\ttokyo\n" + 354 | "tom\t9\tnew york\n", 355 | ) 356 | 357 | type Student struct { 358 | Name string `csv:"name"` 359 | Age int `csv:"age"` 360 | Address string `csv:"address,optional"` 361 | } 362 | 363 | var students []Student 364 | reader := csv.NewReader(bytes.NewReader(data)) 365 | reader.Comma = '\t' 366 | _, err := csvlib.NewDecoder(reader).Decode(&students) 367 | if err != nil { 368 | fmt.Println("error:", err) 369 | } 370 | 371 | for _, u := range students { 372 | fmt.Printf("%+v\n", u) 373 | } 374 | 375 | // Output: 376 | // {Name:jerry Age:10 Address:tokyo} 377 | // {Name:tom Age:9 Address:new york} 378 | ``` 379 | 380 | ### Decode one-by-one 381 | 382 | [Playground](https://go.dev/play/p/81jLLnGbENa) 383 | 384 | ```go 385 | data := []byte(` 386 | name,age,address 387 | jerry,20, 388 | tom,26,new york`) 389 | 390 | type Student struct { 391 | Name string `csv:"name"` 392 | Age int `csv:"age"` 393 | Address string `csv:"address"` 394 | } 395 | 396 | reader := csv.NewReader(bytes.NewReader(data)) 397 | decoder := csvlib.NewDecoder(reader) 398 | for { 399 | var student Student 400 | err := decoder.DecodeOne(&student) 401 | if err != nil { 402 | if errors.Is(err, csvlib.ErrFinished) { 403 | break 404 | } 405 | fmt.Println("error:", err) 406 | break 407 | } 408 | fmt.Printf("%+v\n", student) 409 | } 410 | 411 | result, err := decoder.Finish() // err is *csvlib.Errors if it's not nil 412 | fmt.Printf("%+v\n", *result) 413 | 414 | // Output: 415 | // {Name:jerry Age:20 Address:} 416 | // {Name:tom Age:26 Address:new york} 417 | // {totalRow:3 unrecognizedColumns:[] missingOptionalColumns:[]} 418 | ``` 419 | 420 | ### Header localization 421 | 422 | - This functionality allows to decode multiple input data with header translated into specific language 423 | 424 | [Playground](https://go.dev/play/p/Tf-unVfBnYH) 425 | 426 | ```go 427 | // Sample localization data (you can define them somewhere else such as in a json file) 428 | var ( 429 | mapLanguageEn = map[string]string{ 430 | "name": "name", 431 | "age": "age", 432 | "address": "address", 433 | } 434 | 435 | mapLanguageVi = map[string]string{ 436 | "name": "tên", 437 | "age": "tuổi", 438 | "address": "địa chỉ", 439 | } 440 | ) 441 | 442 | func localizeViVn(k string, params csvlib.ParameterMap) (string, error) { 443 | return mapLanguageVi[k], nil 444 | } 445 | 446 | func localizeEnUs(k string, params csvlib.ParameterMap) (string, error) { 447 | return mapLanguageEn[k], nil 448 | } 449 | ``` 450 | 451 | ```go 452 | dataEn := []byte(` 453 | name,age,address 454 | jerry,20,tokyo 455 | tom,19,new york`) 456 | 457 | type Student struct { 458 | Name string `csv:"name"` // this tag is now used as localization key to find the real header 459 | Age int `csv:"age"` 460 | Address string `csv:"address"` 461 | } 462 | 463 | var students []Student 464 | result, _ := csvlib.Unmarshal(dataEn, &students, func(cfg *csvlib.DecodeConfig) { 465 | cfg.ParseLocalizedHeader = true 466 | cfg.LocalizationFunc = localizeEnUs 467 | }) 468 | 469 | fmt.Printf("%+v\n", *result) 470 | for _, u := range students { 471 | fmt.Printf("%+v\n", u) 472 | } 473 | 474 | // Output: 475 | // {Name:jerry Age:20 Address:tokyo} 476 | // {Name:tom Age:19 Address:new york} 477 | 478 | dataVi := []byte(` 479 | tên,tuổi,địa chỉ 480 | jerry,20,tokyo 481 | tom,19,new york`) 482 | 483 | result, _ = csvlib.Unmarshal(dataVi, &students, func(cfg *csvlib.DecodeConfig) { 484 | cfg.ParseLocalizedHeader = true 485 | cfg.LocalizationFunc = localizeViVn 486 | }) 487 | 488 | fmt.Printf("%+v\n", *result) 489 | for _, u := range students { 490 | fmt.Printf("%+v\n", u) 491 | } 492 | 493 | // Output: the same 494 | // {Name:jerry Age:20 Address:tokyo} 495 | // {Name:tom Age:19 Address:new york} 496 | ``` 497 | 498 | ### Render error as human-readable format 499 | 500 | - Decoding errors can be rendered as a more human-readable content such as text or CSV. 501 | 502 | [Playground](https://go.dev/play/p/cTNGAUlP7s_T) 503 | 504 | ```go 505 | data := []byte(` 506 | name,age,address 507 | jerry,20,tokyo 508 | tom,26,new york 509 | tintin from paris,32,paris 510 | jj,40,`) 511 | 512 | type Student struct { 513 | Name string `csv:"name"` 514 | Age int `csv:"age"` 515 | Address string `csv:"address"` 516 | } 517 | 518 | var students []Student 519 | _, err := csvlib.Unmarshal(data, &students, func(cfg *csvlib.DecodeConfig) { 520 | cfg.StopOnError = false 521 | cfg.DetectRowLine = true 522 | cfg.ConfigureColumn("name", func(cfg *csvlib.DecodeColumnConfig) { 523 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorStrLen[string](3, 10)} 524 | cfg.OnCellErrorFunc = func(e *csvlib.CellError) { 525 | if errors.Is(e, csvlib.ErrValidationStrLen) { 526 | e.SetLocalizationKey("Column {{.Column}} - '{{.Value}}': Name length must be from {{.MinLen}} to {{.MaxLen}}") 527 | e.WithParam("MinLen", 3).WithParam("MaxLen", 10) 528 | } 529 | } 530 | }) 531 | cfg.ConfigureColumn("age", func(cfg *csvlib.DecodeColumnConfig) { 532 | cfg.ValidatorFuncs = []csvlib.ValidatorFunc{csvlib.ValidatorRange(10, 30)} 533 | cfg.OnCellErrorFunc = func(e *csvlib.CellError) { 534 | if errors.Is(e, csvlib.ErrValidationRange) { 535 | e.SetLocalizationKey("Column {{.Column}} - '{{.Value}}': Age must be from {{.MinVal}} to {{.MaxVal}}") 536 | e.WithParam("MinVal", 10).WithParam("MaxVal", 30) 537 | } 538 | } 539 | }) 540 | }) 541 | if err != nil { 542 | renderer, _ := csvlib.NewRenderer(err.(*csvlib.Errors)) 543 | msg, transErr, _ := renderer.Render() 544 | fmt.Println(transErr) // Translation error can be ignored if that is not important to you 545 | fmt.Println(msg) 546 | } 547 | 548 | // Output: 549 | // Error content: TotalRow: 5, TotalRowError: 2, TotalCellError: 4, TotalError: 4 550 | // Row 4 (line 5): Column 0 - 'tintin from paris': Name length must be from 3 to 10, Column 1 - '32': Age must be from 10 to 30 551 | // Row 5 (line 6): Column 0 - 'jj': Name length must be from 3 to 10, Column 1 - '40': Age must be from 10 to 30 552 | ``` 553 | 554 | - Render error as CSV content. 555 | 556 | [Playground](https://go.dev/play/p/7sJTKNSrOYl) 557 | 558 | ```go 559 | renderer, _ := csvlib.NewCSVRenderer(err.(*csvlib.Errors), func(cfg *csvlib.CSVRenderConfig) { 560 | cfg.CellSeparator = "\n" 561 | }) 562 | msg, _, _ := renderer.RenderAsString() 563 | fmt.Println(msg) 564 | 565 | // Output: 566 | // |------|--------|--------------|--------------------------------------------------------|----------------------------------|----------| 567 | // | Row | Line | CommonError | name | age | address | 568 | // |------|--------|--------------|--------------------------------------------------------|----------------------------------|----------| 569 | // | 4 | 5 | | 'tintin from paris': Name length must be from 3 to 10 | '32': Age must be from 10 to 30 | | 570 | // |------|--------|--------------|--------------------------------------------------------|----------------------------------|----------| 571 | // | 5 | 6 | | 'jj': Name length must be from 3 to 10 | '40': Age must be from 10 to 30 | | 572 | // |------|--------|--------------|--------------------------------------------------------|----------------------------------|----------| 573 | ``` -------------------------------------------------------------------------------- /encoder_test.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/tiendc/gofn" 10 | ) 11 | 12 | func doEncode(v any, options ...EncodeOption) ([]byte, error) { 13 | var buf bytes.Buffer 14 | w := csv.NewWriter(&buf) 15 | encoder := NewEncoder(w, options...) 16 | if err := encoder.Encode(v); err != nil { 17 | return nil, err 18 | } 19 | w.Flush() 20 | if err := w.Error(); err != nil { 21 | return nil, err 22 | } 23 | return buf.Bytes(), nil 24 | } 25 | 26 | func makeEncoder(options ...EncodeOption) (*Encoder, *csv.Writer, *bytes.Buffer) { 27 | var buf bytes.Buffer 28 | w := csv.NewWriter(&buf) 29 | e := NewEncoder(w, options...) 30 | return e, w, &buf 31 | } 32 | 33 | func Test_Encode_configOption(t *testing.T) { 34 | type Item struct { 35 | ColX bool `csv:",optional"` 36 | ColY bool 37 | Col1 int16 `csv:"col1"` 38 | Col2 StrType `csv:"col2"` 39 | } 40 | 41 | t.Run("#1: column option not found", func(t *testing.T) { 42 | v := []Item{} 43 | _, err := doEncode(v, func(cfg *EncodeConfig) { 44 | cfg.ConfigureColumn("ColX", func(cfg *EncodeColumnConfig) {}) 45 | cfg.ConfigureColumn("col1", func(cfg *EncodeColumnConfig) {}) 46 | cfg.ConfigureColumn("colX", func(cfg *EncodeColumnConfig) {}) 47 | }) 48 | assert.ErrorIs(t, err, ErrConfigOptionInvalid) 49 | }) 50 | 51 | t.Run("#2: localize header without localization func", func(t *testing.T) { 52 | v := []Item{} 53 | _, err := doEncode(v, func(cfg *EncodeConfig) { 54 | cfg.LocalizeHeader = true 55 | }) 56 | assert.ErrorIs(t, err, ErrConfigOptionInvalid) 57 | }) 58 | 59 | t.Run("#3: invalid input var", func(t *testing.T) { 60 | var v []Item 61 | _, err := doEncode(v) 62 | assert.ErrorIs(t, err, ErrValueNil) 63 | }) 64 | 65 | t.Run("#4: invalid input var", func(t *testing.T) { 66 | v := []string{} 67 | _, err := doEncode(&v) 68 | assert.ErrorIs(t, err, ErrTypeInvalid) 69 | }) 70 | 71 | t.Run("#5: invalid input var", func(t *testing.T) { 72 | var v Item 73 | _, err := doEncode(&v) 74 | assert.ErrorIs(t, err, ErrTypeInvalid) 75 | }) 76 | } 77 | 78 | func Test_Encode_withHeader(t *testing.T) { 79 | type Item struct { 80 | ColX bool `csv:",optional"` 81 | ColY bool 82 | Col1 int `csv:"col1"` 83 | Col2 float32 `csv:"col2"` 84 | } 85 | 86 | t.Run("#1: column optional", func(t *testing.T) { 87 | v := []Item{ 88 | {Col1: 1, Col2: 2.123}, 89 | {Col1: 100, Col2: 200}, 90 | } 91 | data, err := doEncode(v) 92 | assert.Nil(t, err) 93 | assert.Equal(t, gofn.MultilineString( 94 | `ColX,col1,col2 95 | false,1,2.123 96 | false,100,200 97 | `), string(data)) 98 | }) 99 | 100 | t.Run("#2: no header mode", func(t *testing.T) { 101 | v := []Item{ 102 | {Col1: 1, Col2: 2.123}, 103 | {Col1: 100, Col2: 200}, 104 | } 105 | data, err := doEncode(v, func(cfg *EncodeConfig) { 106 | cfg.NoHeaderMode = true 107 | }) 108 | assert.Nil(t, err) 109 | assert.Equal(t, gofn.MultilineString( 110 | `false,1,2.123 111 | false,100,200 112 | `), string(data)) 113 | }) 114 | } 115 | 116 | func Test_Encode_withPostprocessor(t *testing.T) { 117 | type Item struct { 118 | ColX bool `csv:",optional,omitempty"` 119 | ColY bool 120 | Col1 int `csv:"col1"` 121 | Col2 string `csv:"col2"` 122 | } 123 | 124 | t.Run("#1: trim/upper specific column after encoding", func(t *testing.T) { 125 | v := []Item{ 126 | {Col1: 1, Col2: "\tabcXYZ "}, 127 | {Col1: 100, Col2: " xYz123 "}, 128 | } 129 | data, err := doEncode(v, func(cfg *EncodeConfig) { 130 | cfg.ConfigureColumn("col2", func(cfg *EncodeColumnConfig) { 131 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorTrim, ProcessorUpper} 132 | }) 133 | }) 134 | assert.Nil(t, err) 135 | assert.Equal(t, gofn.MultilineString( 136 | `ColX,col1,col2 137 | ,1,ABCXYZ 138 | ,100,XYZ123 139 | `), string(data)) 140 | }) 141 | 142 | t.Run("#2: add comma to numbers after encoding", func(t *testing.T) { 143 | type Item struct { 144 | ColX bool `csv:",optional,omitempty"` 145 | ColY bool 146 | Col1 int `csv:"col1"` 147 | Col2 float64 `csv:"col2"` 148 | } 149 | v := []Item{ 150 | {Col1: 12345, Col2: 1234.1234567}, 151 | {Col1: 1234567, Col2: 1.1234}, 152 | } 153 | data, err := doEncode(v, func(cfg *EncodeConfig) { 154 | cfg.ConfigureColumn("col1", func(cfg *EncodeColumnConfig) { 155 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorNumberGroupComma} 156 | }) 157 | cfg.ConfigureColumn("col2", func(cfg *EncodeColumnConfig) { 158 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorNumberGroupComma} 159 | }) 160 | }) 161 | assert.Nil(t, err) 162 | assert.Equal(t, gofn.MultilineString( 163 | `ColX,col1,col2 164 | ,"12,345","1,234.1234567" 165 | ,"1,234,567",1.1234 166 | `), string(data)) 167 | }) 168 | } 169 | 170 | func Test_Encode_withSpecialChars(t *testing.T) { 171 | type Item struct { 172 | ColX bool `csv:",optional,omitempty"` 173 | ColY bool 174 | Col1 int `csv:"col1"` 175 | Col2 string `csv:"col2"` 176 | } 177 | 178 | t.Run("#1: values have CRLF and TAB", func(t *testing.T) { 179 | v := []Item{ 180 | {Col1: 1, Col2: " a\tbc\nXYZ "}, 181 | {Col1: 100, Col2: " xYz\n123 "}, 182 | } 183 | data, err := doEncode(v, func(cfg *EncodeConfig) { 184 | cfg.ConfigureColumn("col2", func(cfg *EncodeColumnConfig) { 185 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorTrim, ProcessorUpper} 186 | }) 187 | }) 188 | assert.Nil(t, err) 189 | assert.Equal(t, gofn.MultilineString( 190 | `ColX,col1,col2 191 | ,1,"A BC 192 | XYZ" 193 | ,100,"XYZ 194 | 123" 195 | `), string(data)) 196 | }) 197 | 198 | t.Run("#2: values have bare double-quote and single quote", func(t *testing.T) { 199 | v := []Item{ 200 | {Col1: 1, Col2: "abc\"XYZ"}, 201 | {Col1: 100, Col2: "xYz'123"}, 202 | } 203 | data, err := doEncode(v) 204 | assert.Nil(t, err) 205 | assert.Equal(t, gofn.MultilineString( 206 | `ColX,col1,col2 207 | ,1,"abc""XYZ" 208 | ,100,xYz'123 209 | `), string(data)) 210 | }) 211 | 212 | t.Run("#3: values have valid double-quote", func(t *testing.T) { 213 | v := []Item{ 214 | {Col1: 1, Col2: "\"abcXYZ\""}, 215 | {Col1: 100, Col2: "xYz'123"}, 216 | } 217 | data, err := doEncode(v) 218 | assert.Nil(t, err) 219 | assert.Equal(t, gofn.MultilineString( 220 | `ColX,col1,col2 221 | ,1,"""abcXYZ""" 222 | ,100,xYz'123 223 | `), string(data)) 224 | }) 225 | } 226 | 227 | func Test_Encode_withOmitEmpty(t *testing.T) { 228 | type Item struct { 229 | ColX bool `csv:",optional,omitempty"` 230 | Col1 int `csv:"col1,omitempty"` 231 | Col2 string `csv:"col2,omitempty"` 232 | Col3 *int `csv:"col3,omitempty"` 233 | } 234 | 235 | t.Run("#1: success", func(t *testing.T) { 236 | v := []*Item{ 237 | {}, 238 | nil, 239 | {Col3: gofn.New(0)}, 240 | {Col1: 123}, 241 | } 242 | data, err := doEncode(v) 243 | assert.Nil(t, err) 244 | assert.Equal(t, gofn.MultilineString( 245 | `ColX,col1,col2,col3 246 | ,,, 247 | ,,, 248 | ,123,, 249 | `), string(data)) 250 | }) 251 | } 252 | 253 | func Test_Encode_multipleCalls(t *testing.T) { 254 | type Item struct { 255 | ColY bool 256 | Col1 int `csv:"col1"` 257 | Col2 float32 `csv:"col2"` 258 | } 259 | type Item2 struct { 260 | ColY bool 261 | Col1 int `csv:"col1"` 262 | Col2 float32 `csv:"col2"` 263 | } 264 | 265 | t.Run("#1: encode multiple times", func(t *testing.T) { 266 | v := []Item{ 267 | {Col1: 1, Col2: 1.12345}, 268 | {Col1: 2, Col2: 6.543210}, 269 | } 270 | e, w, buf := makeEncoder() 271 | err := e.Encode(v) 272 | w.Flush() 273 | assert.Nil(t, err) 274 | assert.Equal(t, gofn.MultilineString( 275 | `col1,col2 276 | 1,1.12345 277 | 2,6.54321 278 | `), buf.String()) 279 | 280 | // Second call 281 | v = []Item{ 282 | {Col1: 3, Col2: 1.12345}, 283 | {Col1: 4, Col2: 6.543}, 284 | } 285 | err = e.Encode(v) 286 | w.Flush() 287 | assert.Nil(t, err) 288 | assert.Equal(t, gofn.MultilineString( 289 | `col1,col2 290 | 1,1.12345 291 | 2,6.54321 292 | 3,1.12345 293 | 4,6.543 294 | `), buf.String()) 295 | 296 | // Finish encoding, then try to encode more 297 | _ = e.Finish() 298 | err = e.Encode(v) 299 | assert.ErrorIs(t, err, ErrFinished) 300 | }) 301 | 302 | t.Run("#2: encode different types of data", func(t *testing.T) { 303 | v := []Item{ 304 | {Col1: 1, Col2: 1.12345}, 305 | {Col1: 2, Col2: 6.543210}, 306 | } 307 | e, w, buf := makeEncoder() 308 | err := e.Encode(v) 309 | w.Flush() 310 | assert.Nil(t, err) 311 | assert.Equal(t, gofn.MultilineString( 312 | `col1,col2 313 | 1,1.12345 314 | 2,6.54321 315 | `), buf.String()) 316 | 317 | // Second call use different data type 318 | v2 := []Item2{ 319 | {Col1: 3, Col2: 1.12345}, 320 | {Col1: 4, Col2: 6.543}, 321 | } 322 | err = e.Encode(v2) 323 | assert.ErrorIs(t, err, ErrTypeUnmatched) 324 | }) 325 | } 326 | 327 | func Test_Encode_withFixedInlineColumn(t *testing.T) { 328 | type Sub struct { 329 | ColZ bool `csv:",optional"` 330 | ColY bool 331 | Col1 int16 `csv:"sub1"` 332 | Col2 string `csv:"sub2,optional"` 333 | } 334 | type Item struct { 335 | ColX bool `csv:",optional,omitempty"` 336 | ColY bool 337 | Col1 int `csv:"col1"` 338 | Sub1 Sub `csv:"sub1,inline"` 339 | Col2 string `csv:"col2"` 340 | } 341 | type Item2 struct { 342 | ColX bool `csv:",optional,omitempty"` 343 | ColY bool 344 | Col1 int `csv:"col1"` 345 | Sub1 *Sub `csv:"sub1,inline"` 346 | Col2 string `csv:"col2"` 347 | } 348 | 349 | t.Run("#1: success", func(t *testing.T) { 350 | v := []Item{ 351 | {Col1: 1, Sub1: Sub{Col1: 111}, Col2: "abcxyz123"}, 352 | {Col1: 1000, Sub1: Sub{Col1: 222}, Col2: "abc123"}, 353 | } 354 | data, err := doEncode(v) 355 | assert.Nil(t, err) 356 | assert.Equal(t, gofn.MultilineString( 357 | `ColX,col1,ColZ,sub1,sub2,col2 358 | ,1,false,111,,abcxyz123 359 | ,1000,false,222,,abc123 360 | `), string(data)) 361 | }) 362 | 363 | t.Run("#2: success with pointer type", func(t *testing.T) { 364 | v := []Item2{ 365 | {Col1: 1, Sub1: &Sub{Col1: 111}, Col2: "abcxyz123"}, 366 | {Col1: 1000, Sub1: &Sub{Col1: 222}, Col2: "abc123"}, 367 | } 368 | data, err := doEncode(v) 369 | assert.Nil(t, err) 370 | assert.Equal(t, gofn.MultilineString( 371 | `ColX,col1,ColZ,sub1,sub2,col2 372 | ,1,false,111,,abcxyz123 373 | ,1000,false,222,,abc123 374 | `), string(data)) 375 | }) 376 | 377 | t.Run("#3: with no header mode", func(t *testing.T) { 378 | v := []Item{ 379 | {Col1: 1, Sub1: Sub{Col1: 111}, Col2: "abcxyz123"}, 380 | {Col1: 1000, Sub1: Sub{Col1: 222}, Col2: "abc123"}, 381 | } 382 | data, err := doEncode(v, func(cfg *EncodeConfig) { 383 | cfg.NoHeaderMode = true 384 | }) 385 | assert.Nil(t, err) 386 | assert.Equal(t, gofn.MultilineString( 387 | `,1,false,111,,abcxyz123 388 | ,1000,false,222,,abc123 389 | `), string(data)) 390 | }) 391 | 392 | t.Run("#4: with column prefix", func(t *testing.T) { 393 | type Item struct { 394 | ColX bool `csv:",optional,omitempty"` 395 | ColY bool 396 | Col1 int `csv:"col1"` 397 | Sub1 *Sub `csv:"sub1,inline,prefix=sub_"` 398 | Col2 string `csv:"col2"` 399 | } 400 | v := []Item{ 401 | {Col1: 1, Sub1: &Sub{Col1: 111}, Col2: "abcxyz123"}, 402 | {Col1: 1000, Sub1: &Sub{Col1: 222}, Col2: "abc123"}, 403 | } 404 | data, err := doEncode(v) 405 | assert.Nil(t, err) 406 | assert.Equal(t, gofn.MultilineString( 407 | `ColX,col1,sub_ColZ,sub_sub1,sub_sub2,col2 408 | ,1,false,111,,abcxyz123 409 | ,1000,false,222,,abc123 410 | `), string(data)) 411 | }) 412 | 413 | t.Run("#5: invalid inline column", func(t *testing.T) { 414 | type Item struct { 415 | ColX bool `csv:",optional"` 416 | ColY bool 417 | Col1 int `csv:"col1,inline"` 418 | Col2 string `csv:"col2"` 419 | } 420 | 421 | v := []Item{} 422 | _, err := doEncode(v) 423 | assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 424 | }) 425 | 426 | t.Run("#6: with prefix and custom postprocessor", func(t *testing.T) { 427 | type Item struct { 428 | Col1 int `csv:"col1"` 429 | Sub1 Sub `csv:"sub1,inline,prefix=sub_"` 430 | Col2 string `csv:"col2"` 431 | } 432 | 433 | v := []Item{ 434 | {Col1: 1, Sub1: Sub{Col1: 12345, Col2: "abC"}, Col2: "abcxyz123"}, 435 | {Col1: 1000, Sub1: Sub{Col1: 4321, Col2: "xYz123"}, Col2: "abc123"}, 436 | } 437 | data, err := doEncode(v, func(cfg *EncodeConfig) { 438 | cfg.ConfigureColumn("sub_sub1", func(cfg *EncodeColumnConfig) { 439 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorNumberGroupComma} 440 | }) 441 | cfg.ConfigureColumn("sub1", func(cfg *EncodeColumnConfig) { 442 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorUpper} 443 | }) 444 | }) 445 | assert.Nil(t, err) 446 | assert.Equal(t, gofn.MultilineString( 447 | `col1,sub_ColZ,sub_sub1,sub_sub2,col2 448 | 1,FALSE,"12,345",ABC,abcxyz123 449 | 1000,FALSE,"4,321",XYZ123,abc123 450 | `), string(data)) 451 | }) 452 | } 453 | 454 | func Test_Encode_withDynamicInlineColumn(t *testing.T) { 455 | type Item struct { 456 | ColX bool `csv:",optional"` 457 | ColY bool `csv:"-"` 458 | Col1 int `csv:"col1"` 459 | Sub1 InlineColumn[int] `csv:"sub1,inline"` 460 | Col2 *string `csv:"col2"` 461 | ColZ bool `csv:",optional"` 462 | } 463 | type Item2 struct { 464 | ColX bool `csv:",optional"` 465 | ColY bool `csv:"-"` 466 | Col1 int `csv:"col1"` 467 | Sub1 *InlineColumn[int] `csv:"sub1,inline"` 468 | Col2 *string `csv:"col2"` 469 | ColZ bool `csv:",optional"` 470 | } 471 | 472 | t.Run("#1: success", func(t *testing.T) { 473 | header := []string{"sub1", "sub2"} 474 | v := []Item{ 475 | {Col1: 1, Sub1: InlineColumn[int]{Header: header, Values: []int{111, 11}}, Col2: gofn.New("abcxyz123")}, 476 | {Col1: 1000, Sub1: InlineColumn[int]{Header: header, Values: []int{222, 22}}, Col2: gofn.New("abc123")}, 477 | } 478 | data, err := doEncode(v) 479 | assert.Nil(t, err) 480 | assert.Equal(t, gofn.MultilineString( 481 | `ColX,col1,sub1,sub2,col2,ColZ 482 | false,1,111,11,abcxyz123,false 483 | false,1000,222,22,abc123,false 484 | `), string(data)) 485 | }) 486 | 487 | t.Run("#2: success with using pointer", func(t *testing.T) { 488 | header := []string{"sub1", "sub2"} 489 | v := []Item2{ 490 | {Col1: 1, Sub1: &InlineColumn[int]{Header: header, Values: []int{111, 11}}, Col2: gofn.New("abcxyz123")}, 491 | {Col1: 1000, Sub1: &InlineColumn[int]{Header: header, Values: []int{222, 22}}, Col2: gofn.New("abc123")}, 492 | } 493 | data, err := doEncode(v) 494 | assert.Nil(t, err) 495 | assert.Equal(t, gofn.MultilineString( 496 | `ColX,col1,sub1,sub2,col2,ColZ 497 | false,1,111,11,abcxyz123,false 498 | false,1000,222,22,abc123,false 499 | `), string(data)) 500 | }) 501 | 502 | t.Run("#3: no header mode", func(t *testing.T) { 503 | header := []string{"sub1", "sub2"} 504 | v := []Item{ 505 | {Col1: 1, Sub1: InlineColumn[int]{Header: header, Values: []int{111, 11}}, Col2: gofn.New("abcxyz123")}, 506 | {Col1: 1000, Sub1: InlineColumn[int]{Header: header, Values: []int{222, 22}}, Col2: gofn.New("abc123")}, 507 | } 508 | data, err := doEncode(v, func(cfg *EncodeConfig) { 509 | cfg.NoHeaderMode = true 510 | }) 511 | assert.Nil(t, err) 512 | assert.Equal(t, gofn.MultilineString( 513 | `false,1,111,11,abcxyz123,false 514 | false,1000,222,22,abc123,false 515 | `), string(data)) 516 | }) 517 | 518 | t.Run("#4: with column prefix", func(t *testing.T) { 519 | type Item struct { 520 | ColX bool `csv:",optional"` 521 | ColY bool `csv:"-"` 522 | Col1 *int `csv:"col1"` 523 | Sub1 InlineColumn[int] `csv:"sub1,inline,prefix=sub_"` 524 | Col2 string `csv:"col2"` 525 | } 526 | 527 | header := []string{"col1", "col2"} 528 | v := []Item{ 529 | {Col1: gofn.New(1), Sub1: InlineColumn[int]{Header: header, Values: []int{1234, 11}}, Col2: "abcxyz123"}, 530 | {Col1: gofn.New(1000), Sub1: InlineColumn[int]{Header: header, Values: []int{12345, 22}}, Col2: "abc123"}, 531 | } 532 | data, err := doEncode(v, func(cfg *EncodeConfig) { 533 | cfg.LocalizeHeader = true 534 | cfg.LocalizationFunc = localizeViVn 535 | cfg.ConfigureColumn("sub_col1", func(cfg *EncodeColumnConfig) { 536 | cfg.PostprocessorFuncs = []ProcessorFunc{ProcessorNumberGroupComma} 537 | }) 538 | }) 539 | assert.Nil(t, err) 540 | assert.Equal(t, gofn.MultilineString( 541 | `Cột-X,cột-1,cột-phụ-1,cột-phụ-2,cột-2 542 | false,1,"1,234",11,abcxyz123 543 | false,1000,"12,345",22,abc123 544 | `), string(data)) 545 | }) 546 | 547 | t.Run("#5: invalid dynamic inline type", func(t *testing.T) { 548 | type InlineColumn2[T any] struct { 549 | Header []string 550 | } 551 | type Item struct { 552 | Col1 int `csv:"col1"` 553 | Sub1 InlineColumn2[int] `csv:"sub1,inline"` 554 | Col2 string `csv:"col2"` 555 | } 556 | 557 | header := []string{"sub1", "sub2"} 558 | v := []*Item{ 559 | {Col1: 1, Sub1: InlineColumn2[int]{Header: header}, Col2: "aBc123"}, 560 | {Col1: 2, Sub1: InlineColumn2[int]{Header: header}, Col2: "xyzZ"}, 561 | } 562 | _, err := doEncode(v) 563 | assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 564 | }) 565 | 566 | t.Run("#6: invalid dynamic inline type", func(t *testing.T) { 567 | type InlineColumn2[T any] struct { 568 | Header []string 569 | Values *[]T 570 | } 571 | type Item struct { 572 | Col1 int `csv:"col1"` 573 | Sub1 InlineColumn2[int] `csv:"sub1,inline"` 574 | Col2 string `csv:"col2"` 575 | } 576 | 577 | header := []string{"sub1", "sub2"} 578 | v := []*Item{ 579 | {Col1: 1, Sub1: InlineColumn2[int]{Header: header}, Col2: "aBc123"}, 580 | {Col1: 2, Sub1: InlineColumn2[int]{Header: header}, Col2: "xyzZ"}, 581 | } 582 | _, err := doEncode(v) 583 | assert.ErrorIs(t, err, ErrHeaderDynamicTypeInvalid) 584 | }) 585 | } 586 | 587 | func Test_Encode_withLocalization(t *testing.T) { 588 | type Item struct { 589 | ColX bool `csv:",optional"` 590 | ColY bool `csv:"-"` 591 | Col1 int16 `csv:"col1"` 592 | Col2 StrType `csv:"col2"` 593 | } 594 | 595 | t.Run("#1: localization fails", func(t *testing.T) { 596 | v := []Item{} 597 | _, err := doEncode(v, func(cfg *EncodeConfig) { 598 | cfg.LocalizeHeader = true 599 | cfg.LocalizationFunc = localizeFail 600 | }) 601 | assert.ErrorIs(t, err, ErrLocalization) 602 | assert.ErrorIs(t, err, errKeyNotFound) 603 | }) 604 | 605 | t.Run("#2: no header mode, empty data", func(t *testing.T) { 606 | v := []Item{} 607 | data, err := doEncode(v, func(cfg *EncodeConfig) { 608 | cfg.NoHeaderMode = true 609 | cfg.LocalizeHeader = true 610 | cfg.LocalizationFunc = localizeEnUs 611 | }) 612 | assert.Nil(t, err) 613 | assert.Equal(t, "", string(data)) 614 | }) 615 | 616 | t.Run("#3: no header mode, have data", func(t *testing.T) { 617 | v := []Item{ 618 | {Col1: 111}, 619 | } 620 | data, err := doEncode(v, func(cfg *EncodeConfig) { 621 | cfg.NoHeaderMode = true 622 | cfg.LocalizeHeader = true 623 | cfg.LocalizationFunc = localizeEnUs 624 | }) 625 | assert.Nil(t, err) 626 | assert.Equal(t, gofn.MultilineString( 627 | `false,111, 628 | `), string(data)) 629 | }) 630 | } 631 | 632 | func Test_Encode_withCustomMarshaler(t *testing.T) { 633 | t.Run("#1: no encode func matching", func(t *testing.T) { 634 | type Item struct { 635 | Col1 int `csv:"col1"` 636 | Col2 float32 `csv:"col2"` 637 | Col3 map[int]string `csv:"col3"` 638 | } 639 | 640 | v := []Item{} 641 | _, err := doEncode(v) 642 | assert.ErrorIs(t, err, ErrTypeUnsupported) 643 | }) 644 | 645 | t.Run("#2: ptr of type implements MarshalText/MarshalCSV", func(t *testing.T) { 646 | type Item struct { 647 | ColX bool `csv:",optional"` 648 | ColY bool 649 | Col1 int `csv:"col1"` 650 | Col2 StrUpperType `csv:"col2"` 651 | Col3 StrLowerType `csv:"col3"` 652 | } 653 | 654 | v := []Item{ 655 | {Col1: 1, Col2: "aBcXyZ123", Col3: "aA"}, 656 | {Col1: 1000, Col2: "aBc123", Col3: "bB"}, 657 | } 658 | data, err := doEncode(v) 659 | assert.Nil(t, err) 660 | assert.Equal(t, gofn.MultilineString( 661 | `ColX,col1,col2,col3 662 | false,1,ABCXYZ123,aa 663 | false,1000,ABC123,bb 664 | `), string(data)) 665 | }) 666 | 667 | t.Run("#3: type implements MarshalText/MarshalCSV", func(t *testing.T) { 668 | type Item struct { 669 | ColX bool `csv:",optional"` 670 | ColY bool 671 | Col1 int `csv:"col1"` 672 | Col2 *StrUpperType `csv:"col2"` 673 | Col3 *StrLowerType `csv:"col3"` 674 | } 675 | 676 | v := []Item{ 677 | {Col1: 1, Col2: gofn.New[StrUpperType]("aBcXyZ123"), Col3: gofn.New[StrLowerType]("aA")}, 678 | {Col1: 1000, Col2: gofn.New[StrUpperType]("aBc123"), Col3: gofn.New[StrLowerType]("bB")}, 679 | } 680 | data, err := doEncode(v) 681 | assert.Nil(t, err) 682 | assert.Equal(t, gofn.MultilineString( 683 | `ColX,col1,col2,col3 684 | false,1,ABCXYZ123,aa 685 | false,1000,ABC123,bb 686 | `), string(data)) 687 | }) 688 | } 689 | 690 | func Test_Encode_specialCases(t *testing.T) { 691 | type Item struct { 692 | ColX bool `csv:",optional"` 693 | ColY bool 694 | Col1 int `csv:"col1"` 695 | Col2 float32 `csv:"col2"` 696 | } 697 | 698 | t.Run("#1: no input data", func(t *testing.T) { 699 | v := []Item{} 700 | data, err := doEncode(v) 701 | assert.Nil(t, err) 702 | assert.Equal(t, 0, len(v)) 703 | assert.Equal(t, gofn.MultilineString( 704 | `ColX,col1,col2 705 | `), string(data)) 706 | }) 707 | 708 | t.Run("#2: no input data and no header mode", func(t *testing.T) { 709 | v := []Item{} 710 | data, err := doEncode(v, func(cfg *EncodeConfig) { 711 | cfg.NoHeaderMode = true 712 | }) 713 | assert.Nil(t, err) 714 | assert.Equal(t, "", string(data)) 715 | }) 716 | 717 | t.Run("#3: duplicated column from struct", func(t *testing.T) { 718 | type Item2 struct { 719 | ColX bool `csv:",optional"` 720 | ColY bool 721 | Col1 int `csv:"col1"` 722 | Col2 float32 `csv:"col2"` 723 | Col3 int `csv:"col1"` 724 | } 725 | v := []Item2{} 726 | _, err := doEncode(v) 727 | assert.ErrorIs(t, err, ErrHeaderColumnDuplicated) 728 | }) 729 | 730 | t.Run("#4: invalid header (contains space)", func(t *testing.T) { 731 | type Item struct { 732 | ColX bool `csv:",optional"` 733 | ColY bool 734 | Col1 int `csv:"col1 "` 735 | Col2 float32 `csv:" col2"` 736 | } 737 | v := []Item{} 738 | _, err := doEncode(v) 739 | assert.ErrorIs(t, err, ErrHeaderColumnInvalid) 740 | }) 741 | 742 | t.Run("#5: item type is pointer and nil item is ignored", func(t *testing.T) { 743 | v := []*Item{ 744 | {Col1: 1, Col2: 2.123}, 745 | nil, 746 | {Col1: 100, Col2: 200}, 747 | } 748 | data, err := doEncode(v) 749 | assert.Nil(t, err) 750 | assert.Equal(t, gofn.MultilineString( 751 | `ColX,col1,col2 752 | false,1,2.123 753 | false,100,200 754 | `), string(data)) 755 | }) 756 | } 757 | 758 | func Test_Encode_specialTypes(t *testing.T) { 759 | t.Run("#1: any type", func(t *testing.T) { 760 | type Item struct { 761 | Col1 int `csv:"col1"` 762 | Col2 any `csv:"col2"` 763 | } 764 | 765 | v := []Item{ 766 | {Col1: 1, Col2: 2.123}, 767 | {Col1: 2, Col2: "200"}, 768 | {Col1: 3, Col2: true}, 769 | {Col1: 4, Col2: nil}, 770 | } 771 | data, err := doEncode(v) 772 | assert.Nil(t, err) 773 | assert.Equal(t, gofn.MultilineString( 774 | `col1,col2 775 | 1,2.123 776 | 2,200 777 | 3,true 778 | 4, 779 | `), string(data)) 780 | }) 781 | 782 | t.Run("#2: ptr to any type", func(t *testing.T) { 783 | type Item struct { 784 | Col1 int `csv:"col1"` 785 | Col2 *any `csv:"col2"` 786 | } 787 | 788 | v := []*Item{ 789 | {Col1: 1, Col2: gofn.New[any](2.123)}, 790 | nil, 791 | {Col1: 100, Col2: gofn.New[any]("200")}, 792 | {Col1: 100, Col2: gofn.New[any](true)}, 793 | } 794 | data, err := doEncode(v) 795 | assert.Nil(t, err) 796 | assert.Equal(t, gofn.MultilineString( 797 | `col1,col2 798 | 1,2.123 799 | 100,200 800 | 100,true 801 | `), string(data)) 802 | }) 803 | 804 | t.Run("#3: all base types", func(t *testing.T) { 805 | type Item struct { 806 | Col1 int `csv:"c1"` 807 | Col2 *int `csv:"c2"` 808 | Col3 int8 `csv:"c3"` 809 | Col4 *int8 `csv:"c4"` 810 | Col5 int16 `csv:"c5"` 811 | Col6 *int16 `csv:"c6"` 812 | Col7 int32 `csv:"c7"` 813 | Col8 *int32 `csv:"c8"` 814 | Col9 int64 `csv:"c9"` 815 | Col10 *int64 `csv:"c10"` 816 | Col11 uint `csv:"c11"` 817 | Col12 *uint `csv:"c12"` 818 | Col13 uint8 `csv:"c13"` 819 | Col14 *uint8 `csv:"c14"` 820 | Col15 uint16 `csv:"c15"` 821 | Col16 *uint16 `csv:"c16"` 822 | Col17 uint32 `csv:"c17"` 823 | Col18 *uint32 `csv:"c18"` 824 | Col19 uint64 `csv:"c19"` 825 | Col20 *uint64 `csv:"c20"` 826 | 827 | Col21 float32 `csv:"c21"` 828 | Col22 *float32 `csv:"c22"` 829 | Col23 float64 `csv:"c23"` 830 | Col24 *float64 `csv:"c24"` 831 | 832 | Col25 bool `csv:"c25"` 833 | Col26 *bool `csv:"c26"` 834 | Col27 string `csv:"c27"` 835 | Col28 *string `csv:"c28"` 836 | } 837 | 838 | v := []Item{ 839 | {-1, gofn.New(-1), int8(-1), gofn.New(int8(-1)), int16(-1), gofn.New(int16(-1)), 840 | int32(-1), gofn.New(int32(-1)), int64(-1), gofn.New(int64(-1)), uint(1), gofn.New(uint(1)), 841 | uint8(1), gofn.New(uint8(1)), uint16(1), gofn.New(uint16(1)), uint32(1), gofn.New(uint32(1)), 842 | uint64(1), gofn.New(uint64(1)), float32(1), gofn.New(float32(1)), float64(1), gofn.New(float64(1)), 843 | false, gofn.New(false), "abc", gofn.New("123")}, 844 | } 845 | data, err := doEncode(v) 846 | assert.Nil(t, err) 847 | assert.Equal(t, gofn.MultilineString( 848 | `c1,c2,c3,c4,c5,c6,c7,c8,c9,c10,c11,c12,c13,c14,c15,c16,c17,c18,c19,c20,c21,c22,c23,c24,c25,c26,c27,c28 849 | -1,-1,-1,-1,-1,-1,-1,-1,-1,-1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,false,false,abc,123 850 | `), string(data)) 851 | }) 852 | } 853 | 854 | func Test_EncodeOne(t *testing.T) { 855 | type Item struct { 856 | ColY bool 857 | Col1 int `csv:"col1,omitempty"` 858 | Col2 *float32 `csv:"col2,omitempty"` 859 | } 860 | type Item2 struct { 861 | ColY bool 862 | Col1 int `csv:"col1,omitempty"` 863 | Col2 float32 `csv:"col2,omitempty"` 864 | } 865 | 866 | t.Run("#1: encode one multiple times", func(t *testing.T) { 867 | e, w, buf := makeEncoder() 868 | err := e.EncodeOne(Item{Col1: 1, Col2: gofn.New[float32](1.12345)}) 869 | w.Flush() 870 | assert.Nil(t, err) 871 | assert.Equal(t, gofn.MultilineString( 872 | `col1,col2 873 | 1,1.12345 874 | `), buf.String()) 875 | 876 | // Second call 877 | err = e.EncodeOne(Item{Col1: 2, Col2: gofn.New[float32](0.0)}) 878 | w.Flush() 879 | assert.Nil(t, err) 880 | assert.Equal(t, gofn.MultilineString( 881 | `col1,col2 882 | 1,1.12345 883 | 2, 884 | `), buf.String()) 885 | 886 | // Third call 887 | err = e.EncodeOne(Item{Col1: 0, Col2: gofn.New[float32](2.22)}) 888 | w.Flush() 889 | assert.Nil(t, err) 890 | assert.Equal(t, gofn.MultilineString( 891 | `col1,col2 892 | 1,1.12345 893 | 2, 894 | ,2.22 895 | `), buf.String()) 896 | 897 | // Encode a different data type 898 | err = e.EncodeOne(Item2{Col1: 11, Col2: 11}) 899 | assert.ErrorIs(t, err, ErrTypeUnmatched) 900 | 901 | // Finish encoding, then try to encode more 902 | _ = e.Finish() 903 | err = e.EncodeOne(Item{Col1: 0}) 904 | assert.ErrorIs(t, err, ErrFinished) 905 | }) 906 | } 907 | -------------------------------------------------------------------------------- /decoder.go: -------------------------------------------------------------------------------- 1 | package csvlib 2 | 3 | import ( 4 | "encoding/csv" 5 | "errors" 6 | "fmt" 7 | "io" 8 | "reflect" 9 | "strings" 10 | 11 | "github.com/hashicorp/go-multierror" 12 | "github.com/tiendc/gofn" 13 | ) 14 | 15 | // DecodeConfig configuration for decoding CSV data as structs 16 | type DecodeConfig struct { 17 | // TagName tag name to parse the struct (default is `csv`) 18 | TagName string 19 | 20 | // NoHeaderMode indicates the input data have no header (default is `false`) 21 | NoHeaderMode bool 22 | 23 | // StopOnError when error occurs, stop the processing (default is `true`) 24 | StopOnError bool 25 | 26 | // TrimSpace trim all cell values before processing (default is `false`) 27 | TrimSpace bool 28 | 29 | // RequireColumnOrder order of columns defined in struct must match the order of columns 30 | // in the input data (default is "true") 31 | RequireColumnOrder bool 32 | 33 | // ParseLocalizedHeader header in the input data is localized (default is `false`) 34 | // 35 | // For example: 36 | // type Student struct { 37 | // Name string `csv:"name"` -> `name` is header key now, the actual header is localized based on the key 38 | // Age int `csv:"age"` 39 | // } 40 | ParseLocalizedHeader bool 41 | 42 | // AllowUnrecognizedColumns allow a column in the input data but not in the struct tag definition 43 | // (default is "false") 44 | AllowUnrecognizedColumns bool 45 | 46 | // TreatIncorrectStructureAsError treat incorrect data structure as error (default is `true`) 47 | // 48 | // For example: header has 5 columns, if there is a row having 6 columns, it will be treated as error 49 | // and the decoding process will stop even StopOnError flag is false. 50 | TreatIncorrectStructureAsError bool 51 | 52 | // DetectRowLine detect exact lines of rows (default is `false`) 53 | // 54 | // If turn this flag on, the input reader should be an instance of "encoding/csv" Reader 55 | // as this lib uses Reader.FieldPos() function to get the line of a row. 56 | DetectRowLine bool 57 | 58 | // LocalizationFunc localization function, required when ParseLocalizedHeader is true 59 | LocalizationFunc LocalizationFunc 60 | 61 | // columnConfigMap a map consists of configuration for specific columns 62 | columnConfigMap map[string]*DecodeColumnConfig 63 | } 64 | 65 | func defaultDecodeConfig() *DecodeConfig { 66 | return &DecodeConfig{ 67 | TagName: DefaultTagName, 68 | StopOnError: true, 69 | RequireColumnOrder: true, 70 | TreatIncorrectStructureAsError: true, 71 | } 72 | } 73 | 74 | func (c *DecodeConfig) ConfigureColumn(name string, fn func(*DecodeColumnConfig)) { 75 | if c.columnConfigMap == nil { 76 | c.columnConfigMap = map[string]*DecodeColumnConfig{} 77 | } 78 | columnCfg, ok := c.columnConfigMap[name] 79 | if !ok { 80 | columnCfg = defaultDecodeColumnConfig() 81 | c.columnConfigMap[name] = columnCfg 82 | } 83 | fn(columnCfg) 84 | } 85 | 86 | // DecodeColumnConfig configuration for decoding a specific column 87 | type DecodeColumnConfig struct { 88 | // TrimSpace if `true` and DecodeConfig.TrimSpace is `false`, only trim space this column 89 | // (default is "false") 90 | TrimSpace bool 91 | 92 | // StopOnError if `true` and DecodeConfig.StopOnError is `false`, only stop when error occurs 93 | // within this column processing (default is "false") 94 | StopOnError bool 95 | 96 | // DecodeFunc custom decode function (optional) 97 | DecodeFunc DecodeFunc 98 | 99 | // PreprocessorFuncs a list of functions will be called before decoding a cell value (optional) 100 | PreprocessorFuncs []ProcessorFunc 101 | 102 | // ValidatorFuncs a list of functions will be called after decoding (optional) 103 | ValidatorFuncs []ValidatorFunc 104 | 105 | // OnCellErrorFunc function will be called every time an error happens when decode a cell. 106 | // This func can be helpful to set localization key and additional params for the error 107 | // to localize the error message later on. (optional) 108 | OnCellErrorFunc OnCellErrorFunc 109 | } 110 | 111 | func defaultDecodeColumnConfig() *DecodeColumnConfig { 112 | return &DecodeColumnConfig{} 113 | } 114 | 115 | // DecodeOption function to modify decoding config 116 | type DecodeOption func(cfg *DecodeConfig) 117 | 118 | // DecodeResult decoding result 119 | type DecodeResult struct { 120 | totalRow int 121 | unrecognizedColumns []string 122 | missingOptionalColumns []string 123 | } 124 | 125 | func (r *DecodeResult) TotalRow() int { 126 | return r.totalRow 127 | } 128 | 129 | func (r *DecodeResult) UnrecognizedColumns() []string { 130 | return r.unrecognizedColumns 131 | } 132 | 133 | func (r *DecodeResult) MissingOptionalColumns() []string { 134 | return r.missingOptionalColumns 135 | } 136 | 137 | // Decoder data structure of the default decoder 138 | type Decoder struct { 139 | r Reader 140 | cfg *DecodeConfig 141 | err *Errors 142 | result *DecodeResult 143 | finished bool 144 | rowsData []*rowData 145 | itemType reflect.Type 146 | shouldStop bool 147 | hasDynamicInlineColumns bool 148 | hasFixedInlineColumns bool 149 | colsMeta []*decodeColumnMeta 150 | } 151 | 152 | // NewDecoder creates a new Decoder object 153 | func NewDecoder(r Reader, options ...DecodeOption) *Decoder { 154 | cfg := defaultDecodeConfig() 155 | for _, opt := range options { 156 | opt(cfg) 157 | } 158 | return &Decoder{ 159 | r: r, 160 | cfg: cfg, 161 | err: NewErrors(), 162 | } 163 | } 164 | 165 | // Decode decode input data and store the result in the given variable. 166 | // The input var must be a pointer to a slice, e.g. `*[]Student` (recommended) or `*[]*Student`. 167 | func (d *Decoder) Decode(v any) (*DecodeResult, error) { 168 | if d.finished { 169 | return nil, ErrFinished 170 | } 171 | if d.shouldStop { 172 | return nil, ErrAlreadyFailed 173 | } 174 | 175 | val := reflect.ValueOf(v) 176 | if d.itemType == nil { 177 | if err := d.prepareDecode(val); err != nil { 178 | d.err.Add(err) 179 | d.shouldStop = true 180 | return nil, d.err 181 | } 182 | } else { 183 | itemType, err := d.parseOutputVar(val) 184 | if err != nil { 185 | return nil, err 186 | } 187 | if itemType != d.itemType { 188 | return nil, fmt.Errorf("%w: %v (expect %v)", ErrTypeUnmatched, itemType, d.itemType) 189 | } 190 | } 191 | 192 | outSlice := reflect.MakeSlice(val.Type().Elem(), len(d.rowsData), len(d.rowsData)) 193 | itemKindIsPtr := d.itemType.Kind() == reflect.Pointer 194 | row := 0 195 | for !d.shouldStop && len(d.rowsData) > 0 { 196 | // Reduce memory consumption by splitting the source data into chunks (10000 items each) 197 | // After each chunk is processed, resize the slice to allow Go to free the memory when necessary 198 | chunkSz := gofn.Min(10000, len(d.rowsData)) //nolint:mnd 199 | chunk := d.rowsData[0:chunkSz] 200 | d.rowsData = d.rowsData[chunkSz:] 201 | 202 | for _, rowData := range chunk { 203 | rowVal := outSlice.Index(row) 204 | row++ 205 | if itemKindIsPtr { 206 | rowVal.Set(reflect.New(d.itemType.Elem())) 207 | rowVal = rowVal.Elem() 208 | } 209 | if err := d.decodeRow(rowData, rowVal); err != nil { 210 | d.err.Add(err) 211 | if d.cfg.StopOnError || d.shouldStop { 212 | d.shouldStop = true 213 | break 214 | } 215 | } 216 | } 217 | } 218 | 219 | if d.err.HasError() { 220 | return d.result, d.err 221 | } 222 | val.Elem().Set(outSlice) 223 | d.finished = len(d.rowsData) == 0 224 | return d.result, nil 225 | } 226 | 227 | // DecodeOne decode the next one row data. 228 | // The input var must be a pointer to a struct (e.g. *Student). 229 | // This func returns error of the current row processing only, after finishing the last row decoding, 230 | // call Finish() to get the overall result and error. 231 | func (d *Decoder) DecodeOne(v any) error { 232 | if d.finished { 233 | return ErrFinished 234 | } 235 | if d.shouldStop { 236 | return ErrAlreadyFailed 237 | } 238 | 239 | rowVal := reflect.ValueOf(v) 240 | rowVal, itemType, err := d.parseOutputVarOne(rowVal) 241 | if err != nil { 242 | return err 243 | } 244 | if d.itemType == nil { 245 | if err := d.prepareDecode(reflect.New(reflect.SliceOf(itemType))); err != nil { 246 | d.err.Add(err) 247 | d.shouldStop = true 248 | return err 249 | } 250 | } else { 251 | if itemType != d.itemType { 252 | return fmt.Errorf("%w: %v (expect %v)", ErrTypeUnmatched, itemType, d.itemType) 253 | } 254 | } 255 | 256 | if len(d.rowsData) == 0 { 257 | d.finished = true 258 | return ErrFinished 259 | } 260 | rowData := d.rowsData[0] 261 | d.rowsData = d.rowsData[1:] 262 | err = d.decodeRow(rowData, rowVal) 263 | if err != nil { 264 | d.err.Add(err) 265 | if d.cfg.StopOnError { 266 | d.shouldStop = true 267 | } 268 | } 269 | d.finished = len(d.rowsData) == 0 270 | return err 271 | } 272 | 273 | // Finish decoding, after calling this func, you can't decode more even there is data 274 | func (d *Decoder) Finish() (*DecodeResult, error) { 275 | d.finished = true 276 | if d.err.HasError() { 277 | return d.result, d.err 278 | } 279 | return d.result, nil 280 | } 281 | 282 | // prepareDecode prepare for decoding by parsing the struct tags and build column decoders. 283 | // This step is performed one time only before the first row decoding. 284 | func (d *Decoder) prepareDecode(v reflect.Value) error { 285 | d.result = &DecodeResult{} 286 | itemType, err := d.parseOutputVar(v) 287 | if err != nil { 288 | return err 289 | } 290 | d.itemType = itemType 291 | 292 | if err = d.validateConfig(); err != nil { 293 | return err 294 | } 295 | 296 | if err = d.parseColumnsMeta(itemType); err != nil { // typ: []Item, itemTyp: Item 297 | return err 298 | } 299 | 300 | if err = d.buildColumnDecoders(); err != nil { 301 | return err 302 | } 303 | 304 | if err = d.readRowData(); err != nil { 305 | return err 306 | } 307 | 308 | totalRow := len(d.rowsData) 309 | if !d.cfg.NoHeaderMode { 310 | totalRow++ 311 | } 312 | d.result.totalRow = totalRow 313 | d.err.totalRow = totalRow 314 | for _, colMeta := range d.colsMeta { 315 | d.err.header = append(d.err.header, colMeta.headerText) 316 | } 317 | return nil 318 | } 319 | 320 | // decodeRow decode row data and write the result to the row target value 321 | // `rowVal` is normally a slice item at a specific index 322 | // nolint: gocyclo,gocognit 323 | func (d *Decoder) decodeRow(rowData *rowData, rowVal reflect.Value) error { 324 | cfg, colsMeta := d.cfg, d.colsMeta 325 | if rowData.err != nil { 326 | rowErr := NewRowErrors(rowData.row, rowData.line) 327 | rowErr.Add(d.handleCellError(rowData.err, "", nil)) 328 | return rowErr 329 | } 330 | 331 | if d.hasDynamicInlineColumns || d.hasFixedInlineColumns { 332 | for _, colMeta := range colsMeta { 333 | if colMeta.inlineColumnMeta != nil { 334 | colMeta.inlineColumnMeta.decodePrepareForNextRow() 335 | } 336 | } 337 | } 338 | 339 | var cellErrs []error 340 | for col, cellText := range rowData.records { 341 | colMeta := colsMeta[col] 342 | if colMeta.unrecognized { 343 | continue 344 | } 345 | if cfg.TrimSpace || colMeta.trimSpace { 346 | cellText = strings.TrimSpace(cellText) 347 | } 348 | for _, fn := range colMeta.preprocessorFuncs { 349 | cellText = fn(cellText) 350 | } 351 | 352 | outVal := rowVal.Field(colMeta.targetField.Index[0]) 353 | if colMeta.inlineColumnMeta != nil { 354 | outVal = colMeta.inlineColumnMeta.decodeGetColumnValue(outVal) 355 | } 356 | 357 | var errs []error 358 | hasDecodeErr := false 359 | if !colMeta.omitempty || cellText != "" { 360 | if err := colMeta.decodeFunc(cellText, outVal); err != nil { 361 | errs = []error{err} 362 | hasDecodeErr = true 363 | } 364 | } 365 | if !hasDecodeErr && len(colMeta.validatorFuncs) > 0 { 366 | errs = d.validateParsedCell(outVal, colMeta) 367 | } 368 | for _, err := range errs { 369 | cellErrs = append(cellErrs, d.handleCellError(err, rowData.records[col], colMeta)) 370 | if cfg.StopOnError || colMeta.stopOnError { 371 | d.shouldStop = true 372 | break 373 | } 374 | } 375 | } 376 | if len(cellErrs) > 0 { 377 | rowErr := NewRowErrors(rowData.row, rowData.line) 378 | rowErr.Add(cellErrs...) 379 | return rowErr 380 | } 381 | return nil 382 | } 383 | 384 | // validateParsedCell validate a cell value after decoding 385 | func (d *Decoder) validateParsedCell(v reflect.Value, colMeta *decodeColumnMeta) []error { 386 | var errs []error 387 | vAsIface := v.Interface() 388 | for _, validatorFunc := range colMeta.validatorFuncs { 389 | err := validatorFunc(vAsIface) 390 | if err != nil { 391 | if _, ok := err.(*CellError); !ok { // nolint: errorlint 392 | err = NewCellError(err, colMeta.column, colMeta.headerText) 393 | } 394 | errs = append(errs, err) 395 | if d.cfg.StopOnError || colMeta.stopOnError { 396 | return errs 397 | } 398 | } 399 | } 400 | return errs 401 | } 402 | 403 | // handleCellError build cell error for the given error and call the onCellErrorFunc 404 | func (d *Decoder) handleCellError(err error, value string, colMeta *decodeColumnMeta) error { 405 | cellErr, ok := err.(*CellError) // nolint: errorlint 406 | if !ok { 407 | if colMeta != nil { 408 | cellErr = NewCellError(err, colMeta.column, colMeta.headerText) 409 | } else { 410 | // This is error that not relate to any column (e.g. RwoFieldCount error) 411 | cellErr = NewCellError(err, -1, "") 412 | } 413 | } 414 | cellErr.value = value 415 | if colMeta != nil && colMeta.onCellErrorFunc != nil { 416 | colMeta.onCellErrorFunc(cellErr) 417 | } 418 | return cellErr 419 | } 420 | 421 | // parseOutputVar parse and validate the input var 422 | func (d *Decoder) parseOutputVar(v reflect.Value) (itemType reflect.Type, err error) { 423 | if v.Kind() != reflect.Pointer || v.IsNil() { 424 | err = fmt.Errorf("%w: %v", ErrTypeInvalid, v.Kind()) 425 | return 426 | } 427 | 428 | typ := v.Type().Elem() // E.g. []Item 429 | switch typ.Kind() { // nolint: exhaustive 430 | case reflect.Slice, reflect.Array: 431 | default: 432 | err = fmt.Errorf("%w: %v", ErrTypeInvalid, typ.Kind()) 433 | return 434 | } 435 | 436 | itemType = typ.Elem() 437 | if indirectType(itemType).Kind() != reflect.Struct { 438 | err = fmt.Errorf("%w: %v", ErrTypeInvalid, itemType.Kind()) 439 | return 440 | } 441 | return 442 | } 443 | 444 | func (d *Decoder) parseOutputVarOne(v reflect.Value) (val reflect.Value, itemType reflect.Type, err error) { 445 | itemType = v.Type() 446 | if itemType.Kind() != reflect.Pointer || itemType.Elem().Kind() != reflect.Struct { 447 | return reflect.Value{}, nil, fmt.Errorf("%w: must be a pointer to struct", ErrTypeInvalid) 448 | } 449 | itemType = itemType.Elem() 450 | if v.IsNil() { 451 | return reflect.Value{}, nil, fmt.Errorf("%w: must be a non-nil pointer", ErrValueNil) 452 | } 453 | val = v.Elem() 454 | return 455 | } 456 | 457 | // readRowData read data of all rows from the input to struct type. 458 | // If you use `encoding/csv` Reader, we can determine the lines of rows (via Reader.FieldPos func). 459 | // Otherwise, `line` will be set to `-1` which mean undetected. 460 | func (d *Decoder) readRowData() error { 461 | cfg, r := d.cfg, d.r 462 | row := 2 463 | if d.cfg.NoHeaderMode { 464 | row = 1 465 | } 466 | getLine, ableToGetLine := r.(interface { 467 | FieldPos(field int) (line, column int) // Reader from "encoding/csv" provides this func 468 | }) 469 | if !cfg.DetectRowLine { 470 | ableToGetLine = false 471 | getLine = nil 472 | } 473 | rowDataItems := make([]*rowData, 0, 10000) //nolint:mnd 474 | 475 | for ; ; row++ { 476 | records, err := r.Read() 477 | line := -1 478 | if err == nil { 479 | if ableToGetLine { 480 | line, _ = getLine.FieldPos(0) 481 | } 482 | rowDataItems = append(rowDataItems, &rowData{ 483 | records: records, 484 | line: line, 485 | row: row, 486 | }) 487 | continue 488 | } 489 | if errors.Is(err, io.EOF) { 490 | break 491 | } 492 | if errors.Is(err, csv.ErrFieldCount) { 493 | err = fmt.Errorf("%w: row %d", ErrDecodeRowFieldCount, row) 494 | if cfg.TreatIncorrectStructureAsError || cfg.StopOnError { 495 | return err 496 | } 497 | if ableToGetLine { 498 | line, _ = getLine.FieldPos(0) 499 | } 500 | rowDataItems = append(rowDataItems, &rowData{row: row, line: line, err: err}) 501 | continue 502 | } 503 | if errors.Is(err, csv.ErrQuote) || errors.Is(err, csv.ErrBareQuote) { 504 | err = fmt.Errorf("%w: row %d", ErrDecodeQuoteInvalid, row) 505 | if cfg.TreatIncorrectStructureAsError || cfg.StopOnError { 506 | return err 507 | } 508 | // NOTE: it seems when invalid quote, calling getLine will panic 509 | rowDataItems = append(rowDataItems, &rowData{row: row, line: line, err: err}) 510 | continue 511 | } 512 | return err 513 | } 514 | 515 | d.rowsData = rowDataItems 516 | return nil 517 | } 518 | 519 | // parseColumnsMeta parse struct metadata 520 | func (d *Decoder) parseColumnsMeta(itemType reflect.Type) error { 521 | cfg, result := d.cfg, d.result 522 | fileHeader, err := d.readFileHeader() 523 | if err != nil { 524 | return err 525 | } 526 | 527 | colsMetaFromStruct, err := d.parseColumnsMetaFromStructType(itemType, fileHeader) 528 | if err != nil { 529 | return err 530 | } 531 | if len(fileHeader) == 0 { 532 | d.colsMeta = colsMetaFromStruct 533 | return nil 534 | } 535 | 536 | mapColMetaFromStruct := make(map[string]*decodeColumnMeta, len(colsMetaFromStruct)) 537 | for _, colMeta := range colsMetaFromStruct { 538 | mapColMetaFromStruct[colMeta.headerText] = colMeta 539 | } 540 | 541 | colsMeta := make([]*decodeColumnMeta, 0, len(fileHeader)) 542 | for _, headerText := range fileHeader { 543 | colMeta := mapColMetaFromStruct[headerText] 544 | if colMeta == nil { 545 | if !cfg.AllowUnrecognizedColumns { 546 | return fmt.Errorf("%w: \"%s\"", ErrHeaderColumnUnrecognized, headerText) 547 | } 548 | colMeta = &decodeColumnMeta{ 549 | headerKey: headerText, 550 | headerText: headerText, 551 | unrecognized: true, 552 | } 553 | } 554 | colMeta.column = len(colsMeta) 555 | colsMeta = append(colsMeta, colMeta) 556 | } 557 | 558 | mapColMeta := make(map[string]*decodeColumnMeta, len(colsMeta)) 559 | for _, colMeta := range colsMeta { 560 | mapColMeta[colMeta.headerText] = colMeta 561 | if colMeta.unrecognized { 562 | result.unrecognizedColumns = append(result.unrecognizedColumns, colMeta.headerText) 563 | } 564 | } 565 | for _, colMeta := range colsMetaFromStruct { 566 | if _, ok := mapColMeta[colMeta.headerText]; !ok { 567 | if !colMeta.optional { 568 | return fmt.Errorf("%w: \"%s\"", ErrHeaderColumnRequired, colMeta.headerText) 569 | } 570 | result.missingOptionalColumns = append(result.missingOptionalColumns, colMeta.headerText) 571 | } 572 | } 573 | 574 | if cfg.RequireColumnOrder { 575 | if err = d.validateHeaderOrder(colsMeta, colsMetaFromStruct); err != nil { 576 | return err 577 | } 578 | } 579 | 580 | if err = d.validateColumnsMeta(colsMeta, colsMetaFromStruct); err != nil { 581 | return err 582 | } 583 | 584 | d.colsMeta = colsMeta 585 | return nil 586 | } 587 | 588 | // validateColumnsMeta validate struct metadata 589 | func (d *Decoder) validateColumnsMeta(colsMeta, colsMetaFromStruct []*decodeColumnMeta) error { 590 | cfg := d.cfg 591 | // Make sure all column options valid 592 | for colKey := range cfg.columnConfigMap { 593 | if !gofn.ContainBy(colsMetaFromStruct, func(colMeta *decodeColumnMeta) bool { 594 | return colMeta.headerKey == colKey || colMeta.parentKey == colKey 595 | }) { 596 | return fmt.Errorf("%w: column \"%s\" not found", ErrConfigOptionInvalid, colKey) 597 | } 598 | } 599 | 600 | // Make sure all columns are unique 601 | if err := d.validateHeaderUniqueness(colsMeta); err != nil { 602 | return err 603 | } 604 | return nil 605 | } 606 | 607 | func (d *Decoder) readFileHeader() (fileHeader []string, err error) { 608 | if !d.cfg.NoHeaderMode { 609 | fileHeader, err = d.r.Read() 610 | if err != nil { 611 | return nil, err 612 | } 613 | } 614 | if err = validateHeader(fileHeader); err != nil { 615 | return nil, err 616 | } 617 | return 618 | } 619 | 620 | func (d *Decoder) parseColumnsMetaFromStructType(itemType reflect.Type, fileHeader []string) ( 621 | colsMeta []*decodeColumnMeta, err error) { 622 | cfg := d.cfg 623 | itemType = indirectType(itemType) 624 | numFields := itemType.NumField() 625 | for i := 0; i < numFields; i++ { 626 | field := itemType.Field(i) 627 | tag, err := parseTag(cfg.TagName, field) 628 | if err != nil { 629 | return nil, err 630 | } 631 | if tag == nil || tag.ignored { 632 | continue 633 | } 634 | 635 | colMeta := &decodeColumnMeta{ 636 | column: len(colsMeta), 637 | headerKey: tag.name, 638 | headerText: tag.name, 639 | prefix: tag.prefix, 640 | optional: tag.optional, 641 | omitempty: tag.omitEmpty, 642 | targetField: field, 643 | } 644 | 645 | if tag.inline { 646 | inlineColumnsMeta, err := d.parseInlineColumn(field, colMeta) 647 | if err != nil { 648 | return nil, err 649 | } 650 | colsMeta = append(colsMeta, inlineColumnsMeta...) 651 | continue 652 | } 653 | 654 | colMeta.copyConfig(cfg.columnConfigMap[colMeta.headerKey]) 655 | if err = colMeta.localizeHeader(cfg); err != nil { 656 | return nil, err 657 | } 658 | 659 | colsMeta = append(colsMeta, colMeta) 660 | } 661 | if err = d.validateHeaderUniqueness(colsMeta); err != nil { 662 | return nil, err 663 | } 664 | 665 | if d.hasFixedInlineColumns || d.hasDynamicInlineColumns { 666 | if err = d.validateConfigOnInlineColumns(fileHeader); err != nil { 667 | return nil, err 668 | } 669 | // Parse dynamic inline columns based on file header 670 | if d.hasDynamicInlineColumns { 671 | colsMeta, err = d.parseDynamicInlineColumns(colsMeta, fileHeader) 672 | if err != nil { 673 | return nil, err 674 | } 675 | } 676 | } 677 | 678 | // Correct column index (0-index) 679 | for i, colMeta := range colsMeta { 680 | colMeta.column = i 681 | } 682 | return colsMeta, err 683 | } 684 | 685 | func (d *Decoder) parseInlineColumn(field reflect.StructField, parentCol *decodeColumnMeta) ( 686 | colsMeta []*decodeColumnMeta, err error) { 687 | inlineColumnsMeta, err := d.parseInlineColumnDynamicType(field.Type, parentCol) 688 | if err == nil { 689 | d.hasDynamicInlineColumns = true 690 | return inlineColumnsMeta, nil 691 | } 692 | inlineColumnsMeta, err = d.parseInlineColumnFixedType(field.Type, parentCol) 693 | if err == nil && len(inlineColumnsMeta) > 0 { 694 | d.hasFixedInlineColumns = true 695 | return inlineColumnsMeta, nil 696 | } 697 | return nil, fmt.Errorf("%w: %v", ErrHeaderDynamicTypeInvalid, field.Type) 698 | } 699 | 700 | func (d *Decoder) parseInlineColumnFixedType(typ reflect.Type, parent *decodeColumnMeta) ([]*decodeColumnMeta, error) { 701 | cfg := d.cfg 702 | typ = indirectType(typ) 703 | if typ.Kind() != reflect.Struct { 704 | return nil, fmt.Errorf("%w: not struct type", ErrHeaderDynamicTypeInvalid) 705 | } 706 | numFields := typ.NumField() 707 | colsMeta := make([]*decodeColumnMeta, 0, numFields) 708 | for i := 0; i < numFields; i++ { 709 | field := typ.Field(i) 710 | tag, err := parseTag(cfg.TagName, field) 711 | if err != nil { 712 | return nil, err 713 | } 714 | if tag == nil || tag.ignored { 715 | continue 716 | } 717 | 718 | headerKey := parent.prefix + tag.name 719 | colMeta := &decodeColumnMeta{ 720 | column: len(colsMeta), 721 | headerKey: headerKey, 722 | headerText: headerKey, 723 | parentKey: parent.headerKey, 724 | optional: tag.optional, 725 | omitempty: tag.omitEmpty, 726 | targetField: parent.targetField, 727 | inlineColumnMeta: &inlineColumnMeta{ 728 | inlineType: inlineColumnStructFixed, 729 | targetField: field, 730 | dataType: field.Type, 731 | }, 732 | } 733 | 734 | columnCfg := cfg.columnConfigMap[colMeta.headerKey] 735 | if columnCfg == nil { 736 | columnCfg = cfg.columnConfigMap[colMeta.parentKey] 737 | } 738 | colMeta.copyConfig(columnCfg) 739 | if err = colMeta.localizeHeader(cfg); err != nil { 740 | return nil, err 741 | } 742 | 743 | colsMeta = append(colsMeta, colMeta) 744 | } 745 | return colsMeta, nil 746 | } 747 | 748 | func (d *Decoder) parseInlineColumnDynamicType(typ reflect.Type, parent *decodeColumnMeta) ( 749 | []*decodeColumnMeta, error) { 750 | cfg := d.cfg 751 | typ = indirectType(typ) 752 | if typ.Kind() != reflect.Struct { 753 | return nil, fmt.Errorf("%w: not struct type", ErrHeaderDynamicTypeInvalid) 754 | } 755 | headerField, ok := typ.FieldByName(dynamicInlineColumnHeader) 756 | if !ok { 757 | return nil, fmt.Errorf("%w: field Header not found", ErrHeaderDynamicTypeInvalid) 758 | } 759 | if headerField.Type != reflect.TypeOf([]string{}) { 760 | return nil, fmt.Errorf("%w: field Header not []string", ErrHeaderDynamicTypeInvalid) 761 | } 762 | 763 | valuesField, ok := typ.FieldByName(dynamicInlineColumnValues) 764 | if !ok { 765 | return nil, fmt.Errorf("%w: field Values not found", ErrHeaderDynamicTypeInvalid) 766 | } 767 | if valuesField.Type.Kind() != reflect.Slice { 768 | return nil, fmt.Errorf("%w: field Values not slice", ErrHeaderDynamicTypeInvalid) 769 | } 770 | 771 | dataType := valuesField.Type.Elem() 772 | colMeta := *parent 773 | colMeta.headerKey = parent.prefix + parent.headerKey 774 | colMeta.parentKey = parent.headerKey 775 | colMeta.inlineColumnMeta = &inlineColumnMeta{ 776 | inlineType: inlineColumnStructDynamic, 777 | targetField: valuesField, 778 | dataType: dataType, 779 | } 780 | 781 | columnCfg := cfg.columnConfigMap[colMeta.headerKey] 782 | if columnCfg == nil { 783 | columnCfg = cfg.columnConfigMap[colMeta.parentKey] 784 | } 785 | colMeta.copyConfig(columnCfg) 786 | 787 | return []*decodeColumnMeta{&colMeta}, nil 788 | } 789 | 790 | func (d *Decoder) parseDynamicInlineColumns(colsMetaFromStruct []*decodeColumnMeta, fileHeader []string) ( 791 | []*decodeColumnMeta, error) { 792 | newColsMetaFromStruct := make([]*decodeColumnMeta, 0, len(colsMetaFromStruct)*2) //nolint:mnd 793 | fileHeaderIndex := 0 794 | for i, colMetaFromStruct := range colsMetaFromStruct { 795 | if colMetaFromStruct.inlineColumnMeta == nil { 796 | colOptional := colMetaFromStruct.optional 797 | expectHeader := colMetaFromStruct.headerText 798 | if fileHeaderIndex < len(fileHeader) && fileHeader[fileHeaderIndex] == expectHeader { 799 | newColsMetaFromStruct = append(newColsMetaFromStruct, colMetaFromStruct) 800 | fileHeaderIndex++ 801 | continue 802 | } 803 | if colOptional { 804 | newColsMetaFromStruct = append(newColsMetaFromStruct, colMetaFromStruct) 805 | continue 806 | } 807 | return nil, fmt.Errorf("%w: \"%s\"", ErrHeaderDynamicNotAllowUnrecognizedColumns, expectHeader) 808 | } 809 | 810 | inlineColumnMeta := colMetaFromStruct.inlineColumnMeta 811 | for j := fileHeaderIndex; j < len(fileHeader); j++ { 812 | if (i+1) < len(colsMetaFromStruct) && colsMetaFromStruct[i+1].headerText == fileHeader[j] { 813 | break 814 | } 815 | newColMeta := *colMetaFromStruct 816 | newColMeta.headerText = fileHeader[j] 817 | inlineColumnMeta.headerText = append(inlineColumnMeta.headerText, newColMeta.headerText) 818 | newColsMetaFromStruct = append(newColsMetaFromStruct, &newColMeta) 819 | fileHeaderIndex++ 820 | } 821 | } 822 | return newColsMetaFromStruct, nil 823 | } 824 | 825 | // buildColumnDecoders build decoders for each column type. 826 | // If the type of column is determined, e.g. `int`, the decode function for that will be determined at 827 | // the prepare step, and it will be fast at decoding. If it is `interface`, the decode function will parse 828 | // the actual type at decoding, and it will be slower. 829 | func (d *Decoder) buildColumnDecoders() error { 830 | for _, colMeta := range d.colsMeta { 831 | if colMeta.decodeFunc != nil || colMeta.unrecognized { 832 | continue 833 | } 834 | dataType := colMeta.targetField.Type 835 | if colMeta.inlineColumnMeta != nil { 836 | dataType = colMeta.inlineColumnMeta.dataType 837 | } 838 | decodeFunc, err := getDecodeFunc(dataType) 839 | if err != nil { 840 | return err 841 | } 842 | colMeta.decodeFunc = decodeFunc 843 | } 844 | return nil 845 | } 846 | 847 | // validateHeaderUniqueness validate to make sure header columns are unique 848 | func (d *Decoder) validateHeaderUniqueness(colsMeta []*decodeColumnMeta) error { 849 | mapCheckUniq := make(map[string]struct{}, len(colsMeta)) 850 | for _, colMeta := range colsMeta { 851 | h := colMeta.headerKey 852 | hh := strings.TrimSpace(h) 853 | if h != hh || len(hh) == 0 { 854 | return fmt.Errorf("%w: \"%s\" invalid", ErrHeaderColumnInvalid, h) 855 | } 856 | isDynamicInline := colMeta.inlineColumnMeta != nil && 857 | colMeta.inlineColumnMeta.inlineType == inlineColumnStructDynamic 858 | if _, ok := mapCheckUniq[hh]; ok && !isDynamicInline { 859 | return fmt.Errorf("%w: \"%s\" duplicated", ErrHeaderColumnDuplicated, h) 860 | } 861 | mapCheckUniq[hh] = struct{}{} 862 | } 863 | return nil 864 | } 865 | 866 | // validateHeaderOrder validate to make sure the order of columns in the struct must match 867 | // the order of columns in the input data 868 | func (d *Decoder) validateHeaderOrder(colsMeta, colsMetaFromStruct []*decodeColumnMeta) error { 869 | mapColMeta := make(map[string]*decodeColumnMeta, len(colsMeta)) 870 | for _, colMeta := range colsMeta { 871 | mapColMeta[colMeta.headerText] = colMeta 872 | } 873 | 874 | header := make([]string, 0, len(colsMeta)) 875 | for _, colMeta := range colsMeta { 876 | if colMeta.unrecognized { 877 | continue 878 | } 879 | header = append(header, colMeta.headerText) 880 | } 881 | 882 | headerFromStruct := make([]string, 0, len(colsMetaFromStruct)) 883 | for _, colMeta := range colsMetaFromStruct { 884 | if colMeta.optional && mapColMeta[colMeta.headerText] == nil { 885 | continue 886 | } 887 | headerFromStruct = append(headerFromStruct, colMeta.headerText) 888 | } 889 | 890 | if !reflect.DeepEqual(header, headerFromStruct) { 891 | return fmt.Errorf("%w: %v (expect %v)", ErrHeaderColumnOrderInvalid, header, headerFromStruct) 892 | } 893 | return nil 894 | } 895 | 896 | // validateConfig validate the configuration sent from user 897 | func (d *Decoder) validateConfig() error { 898 | if d.cfg.ParseLocalizedHeader && d.cfg.LocalizationFunc == nil { 899 | return fmt.Errorf("%w: localization function required", ErrConfigOptionInvalid) 900 | } 901 | 902 | return nil 903 | } 904 | 905 | // validateConfigOnInlineColumns validate the configuration on inline columns 906 | func (d *Decoder) validateConfigOnInlineColumns(fileHeader []string) error { 907 | cfg := d.cfg 908 | // If file has inline columns, there are some restrictions 909 | if cfg.NoHeaderMode || len(fileHeader) == 0 { 910 | return ErrHeaderDynamicNotAllowNoHeaderMode 911 | } 912 | if d.hasDynamicInlineColumns && !cfg.RequireColumnOrder { 913 | return ErrHeaderDynamicRequireColumnOrder 914 | } 915 | if d.hasDynamicInlineColumns && cfg.AllowUnrecognizedColumns { 916 | return ErrHeaderDynamicNotAllowUnrecognizedColumns 917 | } 918 | if d.hasDynamicInlineColumns && cfg.ParseLocalizedHeader { 919 | return ErrHeaderDynamicNotAllowLocalizedHeader 920 | } 921 | return nil 922 | } 923 | 924 | // rowData input data of each row 925 | // `line` can be different from `row`, as a row can be in multiple rows and empty lines are skipped 926 | type rowData struct { 927 | records []string 928 | line int 929 | row int 930 | err error 931 | } 932 | 933 | // decodeColumnMeta metadata for decoding a specific column 934 | type decodeColumnMeta struct { 935 | column int 936 | headerKey string 937 | headerText string 938 | parentKey string 939 | prefix string 940 | optional bool 941 | unrecognized bool 942 | omitempty bool 943 | trimSpace bool 944 | stopOnError bool 945 | 946 | targetField reflect.StructField 947 | inlineColumnMeta *inlineColumnMeta 948 | 949 | decodeFunc DecodeFunc 950 | preprocessorFuncs []ProcessorFunc 951 | validatorFuncs []ValidatorFunc 952 | onCellErrorFunc OnCellErrorFunc 953 | } 954 | 955 | func (m *decodeColumnMeta) localizeHeader(cfg *DecodeConfig) error { 956 | if cfg.ParseLocalizedHeader { 957 | headerText, err := cfg.LocalizationFunc(m.headerKey, nil) 958 | if err != nil { 959 | return multierror.Append(ErrLocalization, err) 960 | } 961 | m.headerText = headerText 962 | } 963 | return nil 964 | } 965 | 966 | func (m *decodeColumnMeta) copyConfig(columnCfg *DecodeColumnConfig) { 967 | if columnCfg == nil { 968 | return 969 | } 970 | m.trimSpace = columnCfg.TrimSpace 971 | m.stopOnError = columnCfg.StopOnError 972 | m.decodeFunc = columnCfg.DecodeFunc 973 | m.validatorFuncs = columnCfg.ValidatorFuncs 974 | m.preprocessorFuncs = columnCfg.PreprocessorFuncs 975 | m.onCellErrorFunc = columnCfg.OnCellErrorFunc 976 | } 977 | --------------------------------------------------------------------------------