├── version.go ├── .travis.yml ├── go.mod ├── string_array.go ├── .gitignore ├── go.sum ├── dist.sh ├── LICENSE ├── README.md ├── main_test.go └── main.go /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const VERSION = "1.2.1" 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.x 4 | sudo: false 5 | notifications: 6 | email: false 7 | 8 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/jehiah/json2csv 2 | 3 | go 1.13 4 | 5 | require github.com/stretchr/testify v1.4.0 6 | -------------------------------------------------------------------------------- /string_array.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | type StringArray []string 9 | 10 | func (a *StringArray) Set(s string) error { 11 | for _, ss := range strings.Split(s, ",") { 12 | *a = append(*a, ss) 13 | } 14 | return nil 15 | } 16 | 17 | func (a *StringArray) String() string { 18 | return fmt.Sprint(*a) 19 | } 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | json2csv 2 | dist 3 | build 4 | 5 | # from https://github.com/github/gitignore/blob/master/Go.gitignore 6 | 7 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 8 | *.o 9 | *.a 10 | *.so 11 | 12 | # Folders 13 | _obj 14 | _test 15 | 16 | # Architecture specific extensions/prefixes 17 | *.[568vq] 18 | [568vq].out 19 | 20 | *.cgo1.go 21 | *.cgo2.c 22 | _cgo_defun.c 23 | _cgo_gotypes.go 24 | _cgo_export.* 25 | 26 | _testmain.go 27 | 28 | *.exe -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 4 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 5 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 6 | github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= 7 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 8 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 9 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 10 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 11 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 12 | -------------------------------------------------------------------------------- /dist.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # build binary distributions for linux/amd64 and darwin/amd64 4 | set -e 5 | 6 | DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" 7 | echo "working dir $DIR" 8 | 9 | echo "... running tests" 10 | go test ./... || exit 1 11 | 12 | arch=$(go env GOARCH) 13 | version=$(cat $DIR/version.go | grep "const VERSION" | awk '{print $NF}' | sed 's/"//g') 14 | goversion=$(go version | awk '{print $3}') 15 | 16 | for os in linux darwin; do 17 | echo "... building v$version for $os/$arch" 18 | BUILD=$(mktemp -d -t json2csv) 19 | TARGET="json2csv-$version.$os-$arch.$goversion" 20 | mkdir -p $BUILD/$TARGET 21 | GOOS=$os GOARCH=$arch CGO_ENABLED=0 go build -o $BUILD/$TARGET/json2csv 22 | 23 | pushd $BUILD >/dev/null 24 | tar czvf $TARGET.tar.gz $TARGET 25 | if [ -e $DIR/dist/$TARGET.tar.gz ]; then 26 | echo "... WARNING overwriting dist/$TARGET.tar.gz" 27 | fi 28 | 29 | mkdir -p $DIR/dist 30 | mv $TARGET.tar.gz $DIR/dist/ 31 | echo "... built dist/$TARGET.tar.gz" 32 | popd >/dev/null 33 | done -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Permission is hereby granted, free of charge, to any person obtaining a copy 2 | of this software and associated documentation files (the "Software"), to deal 3 | in the Software without restriction, including without limitation the rights 4 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 5 | copies of the Software, and to permit persons to whom the Software is 6 | furnished to do so, subject to the following conditions: 7 | 8 | The above copyright notice and this permission notice shall be included in 9 | all copies or substantial portions of the Software. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 13 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 14 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 15 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 16 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 17 | THE SOFTWARE. 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | json2csv 2 | ======== 3 | 4 | Converts a stream of newline separated json data to csv format. 5 | 6 | [![Build Status](https://travis-ci.org/jehiah/json2csv.png?branch=master)](https://travis-ci.org/jehiah/json2csv) [![GitHub release](https://img.shields.io/github/release/jehiah/json2csv.svg)](https://github.com/jehiah/json2csv/releases/latest) 7 | 8 | 9 | Installation 10 | ============ 11 | 12 | pre-built binaries are available under [releases](https://github.com/jehiah/json2csv/releases). 13 | 14 | If you have a working golang install, you can use `go install`. 15 | 16 | ```bash 17 | go install github.com/jehiah/json2csv@latest 18 | ``` 19 | 20 | Usage 21 | ===== 22 | 23 | ``` 24 | usage: json2csv 25 | -k fields,and,nested.fields,to,output 26 | -i /path/to/input.json (optional; default is stdin) 27 | -o /path/to/output.csv (optional; default is stdout) 28 | --version 29 | -p print csv header row 30 | -h This help 31 | ``` 32 | 33 | To convert: 34 | 35 | ```json 36 | {"user": {"name":"jehiah", "password": "root"}, "remote_ip": "127.0.0.1", "dt" : "[20/Aug/2010:01:12:44 -0400]"} 37 | {"user": {"name":"jeroenjanssens", "password": "123"}, "remote_ip": "192.168.0.1", "dt" : "[20/Aug/2010:01:12:44 -0400]"} 38 | {"user": {"name":"unknown", "password": ""}, "remote_ip": "76.216.210.0", "dt" : "[20/Aug/2010:01:12:45 -0400]"} 39 | ``` 40 | 41 | to: 42 | 43 | ``` 44 | "jehiah","127.0.0.1" 45 | "jeroenjanssens","192.168.0.1" 46 | "unknown","76.216.210.0" 47 | ``` 48 | 49 | you would either 50 | 51 | ```bash 52 | json2csv -k user.name,remote_ip -i input.json -o output.csv 53 | ``` 54 | 55 | or 56 | 57 | ```bash 58 | cat input.json | json2csv -k user.name,remote_ip > output.csv 59 | ``` 60 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestGetTopic(t *testing.T) { 15 | log.SetOutput(ioutil.Discard) 16 | defer log.SetOutput(os.Stdout) 17 | 18 | reader := bytes.NewBufferString(`{"a": 1, "b": "asdf\n"} 19 | {"a" : null}`) 20 | buf := bytes.NewBuffer([]byte{}) 21 | writer := csv.NewWriter(buf) 22 | 23 | json2csv(reader, writer, []string{"a", "c"}, false) 24 | 25 | output := buf.String() 26 | assert.Equal(t, output, "1,\n,\n") 27 | } 28 | 29 | func TestGetLargeInt(t *testing.T) { 30 | log.SetOutput(ioutil.Discard) 31 | defer log.SetOutput(os.Stdout) 32 | 33 | reader := bytes.NewBufferString(`{"a": 1356998399}`) 34 | buf := bytes.NewBuffer([]byte{}) 35 | writer := csv.NewWriter(buf) 36 | 37 | json2csv(reader, writer, []string{"a"}, false) 38 | 39 | output := buf.String() 40 | assert.Equal(t, output, "1356998399\n") 41 | } 42 | 43 | func TestGetFloat(t *testing.T) { 44 | log.SetOutput(ioutil.Discard) 45 | defer log.SetOutput(os.Stdout) 46 | 47 | reader := bytes.NewBufferString(`{"a": 1356998399.32}`) 48 | buf := bytes.NewBuffer([]byte{}) 49 | writer := csv.NewWriter(buf) 50 | 51 | json2csv(reader, writer, []string{"a"}, false) 52 | 53 | output := buf.String() 54 | assert.Equal(t, output, "1356998399.320000\n") 55 | } 56 | 57 | func TestGetNested(t *testing.T) { 58 | log.SetOutput(ioutil.Discard) 59 | defer log.SetOutput(os.Stdout) 60 | 61 | reader := bytes.NewBufferString(`{"a": {"b": "asdf"}}`) 62 | buf := bytes.NewBuffer([]byte{}) 63 | writer := csv.NewWriter(buf) 64 | 65 | json2csv(reader, writer, []string{"a.b"}, false) 66 | 67 | output := buf.String() 68 | assert.Equal(t, output, "asdf\n") 69 | } 70 | 71 | func TestHeader(t *testing.T) { 72 | log.SetOutput(ioutil.Discard) 73 | defer log.SetOutput(os.Stdout) 74 | 75 | reader := bytes.NewBufferString(`{"a": "b"} 76 | {"a": "c"}`) 77 | buf := bytes.NewBuffer([]byte{}) 78 | writer := csv.NewWriter(buf) 79 | 80 | json2csv(reader, writer, []string{"a"}, true) 81 | 82 | output := buf.String() 83 | assert.Equal(t, output, "a\nb\nc\n") 84 | } 85 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/csv" 6 | "encoding/json" 7 | "flag" 8 | "fmt" 9 | "io" 10 | "log" 11 | "math" 12 | "os" 13 | "runtime" 14 | "strings" 15 | "unicode/utf8" 16 | ) 17 | 18 | type LineReader interface { 19 | ReadBytes(delim byte) (line []byte, err error) 20 | } 21 | 22 | func main() { 23 | inputFile := flag.String("i", "", "/path/to/input.json (optional; default is stdin)") 24 | outputFile := flag.String("o", "", "/path/to/output.csv (optional; default is stdout)") 25 | outputDelim := flag.String("d", ",", "delimiter used for output values") 26 | showVersion := flag.Bool("version", false, "print version string") 27 | printHeader := flag.Bool("p", false, "prints header to output") 28 | keys := StringArray{} 29 | flag.Var(&keys, "k", "fields to output") 30 | flag.Parse() 31 | 32 | if *showVersion { 33 | fmt.Printf("json2csv v%s (built w/%s)\n", VERSION, runtime.Version()) 34 | return 35 | } 36 | 37 | var reader *bufio.Reader 38 | var writer *csv.Writer 39 | if *inputFile != "" { 40 | file, err := os.OpenFile(*inputFile, os.O_RDONLY, 0600) 41 | if err != nil { 42 | log.Printf("Error %s opening input file %v", err, *inputFile) 43 | os.Exit(1) 44 | } 45 | reader = bufio.NewReader(file) 46 | } else { 47 | reader = bufio.NewReader(os.Stdin) 48 | } 49 | 50 | if *outputFile != "" { 51 | file, err := os.OpenFile(*outputFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) 52 | if err != nil { 53 | log.Printf("Error %s opening output file %v", err, *outputFile) 54 | os.Exit(1) 55 | } 56 | writer = csv.NewWriter(file) 57 | } else { 58 | writer = csv.NewWriter(os.Stdout) 59 | } 60 | 61 | delim, _ := utf8.DecodeRuneInString(*outputDelim) 62 | writer.Comma = delim 63 | 64 | json2csv(reader, writer, keys, *printHeader) 65 | } 66 | 67 | func get_value(data map[string]interface{}, keyparts []string) string { 68 | if len(keyparts) > 1 { 69 | subdata, _ := data[keyparts[0]].(map[string]interface{}) 70 | return get_value(subdata, keyparts[1:]) 71 | } else if v, ok := data[keyparts[0]]; ok { 72 | switch v.(type) { 73 | case nil: 74 | return "" 75 | case float64: 76 | f, _ := v.(float64) 77 | if math.Mod(f, 1.0) == 0.0 { 78 | return fmt.Sprintf("%d", int(f)) 79 | } else { 80 | return fmt.Sprintf("%f", f) 81 | } 82 | default: 83 | return fmt.Sprintf("%+v", v) 84 | } 85 | } 86 | 87 | return "" 88 | } 89 | 90 | func json2csv(r LineReader, w *csv.Writer, keys []string, printHeader bool) { 91 | var line []byte 92 | var err error 93 | line_count := 0 94 | 95 | var expanded_keys [][]string 96 | for _, key := range keys { 97 | expanded_keys = append(expanded_keys, strings.Split(key, ".")) 98 | } 99 | 100 | for { 101 | if err == io.EOF { 102 | return 103 | } 104 | line, err = r.ReadBytes('\n') 105 | if err != nil { 106 | if err != io.EOF { 107 | log.Printf("Input ERROR: %s", err) 108 | break 109 | } 110 | } 111 | line_count++ 112 | if len(line) == 0 { 113 | continue 114 | } 115 | 116 | if printHeader { 117 | w.Write(keys) 118 | w.Flush() 119 | printHeader = false 120 | } 121 | 122 | var data map[string]interface{} 123 | err = json.Unmarshal(line, &data) 124 | if err != nil { 125 | log.Printf("ERROR Decoding JSON at line %d: %s\n%s", line_count, err, line) 126 | continue 127 | } 128 | 129 | var record []string 130 | for _, expanded_key := range expanded_keys { 131 | record = append(record, get_value(data, expanded_key)) 132 | } 133 | 134 | w.Write(record) 135 | w.Flush() 136 | } 137 | } 138 | --------------------------------------------------------------------------------