├── go.mod ├── .gitignore ├── reader.go ├── LICENSE ├── writer.go ├── writer_test.go ├── reader_test.go └── README.md /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/simonfrey/jsonl 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | .idea 18 | -------------------------------------------------------------------------------- /reader.go: -------------------------------------------------------------------------------- 1 | package jsonl 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | ) 9 | 10 | type Reader struct { 11 | r io.Reader 12 | scanner *bufio.Scanner 13 | } 14 | 15 | func NewReader(r io.Reader) Reader { 16 | scanner := bufio.NewScanner(r) 17 | scanner.Split(bufio.ScanLines) 18 | 19 | return Reader{ 20 | r: r, 21 | scanner: scanner, 22 | } 23 | } 24 | 25 | func (r Reader) Close() error { 26 | if c, ok := r.r.(io.ReadCloser); ok { 27 | return c.Close() 28 | } 29 | return fmt.Errorf("given reader is no ReadCloser") 30 | } 31 | 32 | func (r Reader) ReadSingleLine(output interface{}) error { 33 | ok := r.scanner.Scan() 34 | if !ok { 35 | return fmt.Errorf("could not read from scanner. Scanner done") 36 | } 37 | 38 | return json.Unmarshal(r.scanner.Bytes(), output) 39 | } 40 | 41 | func (r Reader) ReadLines(callback func(data []byte) error) error { 42 | for r.scanner.Scan() { 43 | err := callback(r.scanner.Bytes()) 44 | if err != nil { 45 | return fmt.Errorf("error in callback: %w", err) 46 | } 47 | } 48 | return nil 49 | } 50 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 VitalFrog 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /writer.go: -------------------------------------------------------------------------------- 1 | package jsonl 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | ) 9 | 10 | type Writer struct { 11 | w io.Writer 12 | prefix string 13 | } 14 | 15 | type WriterOption func(w *Writer) 16 | 17 | func WithPrefix(prefix string) WriterOption { 18 | return func(w *Writer) { 19 | w.prefix = prefix 20 | } 21 | } 22 | 23 | func NewWriter(w io.Writer, opts ...WriterOption) Writer { 24 | wr := Writer{ 25 | w: w, 26 | } 27 | for _, opt := range opts { 28 | opt(&wr) 29 | } 30 | return wr 31 | } 32 | 33 | func (w Writer) Close() error { 34 | if c, ok := w.w.(io.WriteCloser); ok { 35 | return c.Close() 36 | } 37 | return fmt.Errorf("given writer is no WriteCloser") 38 | } 39 | 40 | func (w Writer) Write(data interface{}) error { 41 | j, err := json.Marshal(data) 42 | if err != nil { 43 | return fmt.Errorf("could not json marshal data: %w", err) 44 | } 45 | 46 | if w.prefix != "" { 47 | _, err = w.w.Write([]byte(w.prefix)) 48 | if err != nil { 49 | return fmt.Errorf("could not write prefix to underlying io.Writer: %w", err) 50 | } 51 | } 52 | 53 | _, err = w.w.Write(j) 54 | if err != nil { 55 | return fmt.Errorf("could not write json data to underlying io.Writer: %w", err) 56 | } 57 | 58 | _, err = w.w.Write([]byte("\n")) 59 | if err != nil { 60 | return fmt.Errorf("could not write newline to underlying io.Writer: %w", err) 61 | } 62 | 63 | if f, ok := w.w.(http.Flusher); ok { 64 | // If http writer, flush as well 65 | f.Flush() 66 | } 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /writer_test.go: -------------------------------------------------------------------------------- 1 | package jsonl_test 2 | 3 | import ( 4 | "bytes" 5 | "github.com/simonfrey/jsonl" 6 | "testing" 7 | ) 8 | 9 | type T1 struct { 10 | Type string `json:"type"` 11 | Count int `json:"am"` 12 | } 13 | type T2 struct { 14 | Type string `json:"type"` 15 | Comment string `json:"am"` 16 | } 17 | 18 | func TestWriter_Write(t *testing.T) { 19 | expectedResponse := "{\"type\":\"T1\",\"am\":2}\n" 20 | 21 | buff := bytes.Buffer{} 22 | 23 | w := jsonl.NewWriter(&buff) 24 | 25 | err := w.Write(T1{ 26 | Type: "T1", 27 | Count: 2, 28 | }) 29 | if err != nil { 30 | t.Fatalf("could not write T1: %s", err) 31 | } 32 | 33 | if buff.String() != expectedResponse { 34 | t.Fatalf("Response %q is not as expected %q", buff.String(), expectedResponse) 35 | } 36 | } 37 | 38 | func TestWriter_WriteMultipleLines(t *testing.T) { 39 | expectedResponse := "{\"type\":\"T1\",\"am\":2}\n{\"type\":\"T2\",\"am\":\"I am T2\"}\n{\"type\":\"T1\",\"am\":9999}\n" 40 | 41 | buff := bytes.Buffer{} 42 | 43 | w := jsonl.NewWriter(&buff) 44 | 45 | err := w.Write(T1{ 46 | Type: "T1", 47 | Count: 2, 48 | }) 49 | if err != nil { 50 | t.Fatalf("could not write T1: %s", err) 51 | } 52 | err = w.Write(T2{ 53 | Type: "T2", 54 | Comment: "I am T2", 55 | }) 56 | if err != nil { 57 | t.Fatalf("could not write T2: %s", err) 58 | } 59 | err = w.Write(T1{ 60 | Type: "T1", 61 | Count: 9999, 62 | }) 63 | if err != nil { 64 | t.Fatalf("could not write T1: %s", err) 65 | } 66 | 67 | if buff.String() != expectedResponse { 68 | t.Fatalf("Response %q is not as expected %q", buff.String(), expectedResponse) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /reader_test.go: -------------------------------------------------------------------------------- 1 | package jsonl_test 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "github.com/simonfrey/jsonl" 8 | "strings" 9 | "testing" 10 | ) 11 | 12 | func TestReader_ReadSingleLine(t *testing.T) { 13 | input := "{\"type\":\"T1\",\"am\":2}\n" 14 | 15 | r := jsonl.NewReader(strings.NewReader(input)) 16 | 17 | t1 := T1{} 18 | err := r.ReadSingleLine(&t1) 19 | if err != nil { 20 | t.Fatalf("could not ReadSingleLine: %s", err) 21 | } 22 | 23 | if t1.Count != 2 { 24 | t.Fatalf("read invalid count. Got %d but exepcted %d", t1.Count, 2) 25 | } 26 | } 27 | 28 | func TestReader_ReadLines(t *testing.T) { 29 | input := "{\"type\":\"T1\",\"am\":2}\n{\"type\":\"T2\",\"am\":\"I am T2\"}\n{\"type\":\"T1\",\"am\":9999}\n" 30 | r := jsonl.NewReader(strings.NewReader(input)) 31 | 32 | output := "" 33 | err := r.ReadLines(func(data []byte) error { 34 | switch { 35 | case bytes.Contains(data, []byte(`"T1"`)): 36 | // T1 struct type 37 | t := T1{} 38 | err := json.Unmarshal(data, &t) 39 | if err != nil { 40 | return fmt.Errorf("could not unmarshal into T1: %w", err) 41 | } 42 | output += fmt.Sprintf("%T:%d|", t, t.Count) 43 | case bytes.Contains(data, []byte(`"T2"`)): 44 | // T2 struct type 45 | t := T2{} 46 | err := json.Unmarshal(data, &t) 47 | if err != nil { 48 | return fmt.Errorf("could not unmarshal into T2: %w", err) 49 | } 50 | output += fmt.Sprintf("%T:%s|", t, t.Comment) 51 | } 52 | return nil 53 | }) 54 | if err != nil { 55 | t.Fatalf("could not read lines: %s", err) 56 | } 57 | 58 | if output != "jsonl_test.T1:2|jsonl_test.T2:I am T2|jsonl_test.T1:9999|" { 59 | t.Fatalf("did get wrong response: %q", output) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # JSON Lines (JSONL) golang library 2 | 3 | This library provides you with a Reader and a Writer for the JSONL format. It automatically flushes underlying 4 | `http.ResponseWriter` if applicable. 5 | 6 | No external dependencies are required for this to run. 7 | 8 | > JSON Lines is a convenient format for storing structured data that may be processed one record at a time. It works well with unix-style text processing tools and shell pipelines. It's a great format for log files. It's also a flexible format for passing messages between cooperating processes. 9 | 10 | Source: [jsonlines.org](https://jsonlines.org/) 11 | 12 | ## Examples 13 | 14 | *Dropping errors to keep code example shorter. You should definitely check them!* 15 | 16 | ### Write JSONL (JSON Lines) 17 | 18 | The library automatically does the json marshaling for you on `Write(in interface{})` 19 | 20 | ```go 21 | package main 22 | 23 | import( 24 | "github.com/simonfrey/jsonl" 25 | "bytes" 26 | ) 27 | 28 | func main(){ 29 | buff := bytes.Buffer{} 30 | 31 | w := jsonl.NewWriter(&buff) 32 | w.Write("Hello") 33 | w.Write("World") 34 | w.Write(42) 35 | 36 | fmt.Println(buff.String()) 37 | // Output: 38 | // "Hello"\n"World"\n42\n 39 | } 40 | ``` 41 | 42 | ### Read JSONL (JSON Lines) 43 | 44 | More interesting than writing into JSONL is reading it and working with it. This library provides you with two functions 45 | to do so. 46 | 47 | #### Read Single Line and unmarshal into struct 48 | 49 | If you exactly know what you expect you can directly read a single line from the data into a struct. The library does 50 | the json unmarshaling for you. 51 | 52 | ```go 53 | package main 54 | 55 | import( 56 | "github.com/simonfrey/jsonl" 57 | "strings" 58 | ) 59 | 60 | func main(){ 61 | input := "\"Hello\"\n\"World\"\n42\n" 62 | 63 | r := jsonl.NewReader(strings.NewReader(input)) 64 | 65 | outString := "" 66 | r.ReadSingleLine(&outString) 67 | 68 | fmt.Println(outString) 69 | // Output: 70 | // Hello 71 | } 72 | ``` 73 | 74 | #### Read multiple entries with type detection 75 | 76 | This is my favorite use case of JSONL. Reading different JSON types from one stream. This allows to build a form of streaming 77 | data interface from the server (which sends JSONL with above writer function) 78 | 79 | As you see for stricter type safety I use a dedicated `Type` field in the structs, so we can do a string/byte comparison 80 | to figure out what type we are using. If you types do have unique fields you could also check for those. 81 | 82 | In the following example you learn how you can read JSONL typesafe in golang. 83 | 84 | ```go 85 | package main 86 | 87 | import( 88 | "github.com/simonfrey/jsonl" 89 | "strings" 90 | ) 91 | 92 | 93 | type T1 struct { 94 | Type string `json:"type"` 95 | Count int `json:"am"` 96 | } 97 | type T2 struct { 98 | Type string `json:"type"` 99 | Comment string `json:"am"` 100 | } 101 | 102 | 103 | func main(){ 104 | input := "{\"type\":\"T1\",\"am\":2}\n{\"type\":\"T2\",\"am\":\"I am T2\"}\n{\"type\":\"T1\",\"am\":9999}\n" 105 | 106 | r := jsonl.NewReader(strings.NewReader(input)) 107 | r.ReadLines(func(data []byte) error { 108 | switch { 109 | case bytes.Contains(data, []byte(`"T1"`)): 110 | // T1 struct type 111 | t := T1{} 112 | err := json.Unmarshal(data, &t) 113 | if err != nil { 114 | return fmt.Errorf("could not unmarshal into T1: %w", err) 115 | } 116 | fmt.Printf("%T: %d\n", t, t.Count) 117 | case bytes.Contains(data, []byte(`"T2"`)): 118 | // T2 struct type 119 | t := T2{} 120 | err := json.Unmarshal(data, &t) 121 | if err != nil { 122 | return fmt.Errorf("could not unmarshal into T2: %w", err) 123 | } 124 | fmt.Printf("%T: %s\n", t, t.Comment) 125 | output += fmt.Sprintf("%T:%s|", t, t.Comment) 126 | } 127 | return nil 128 | }) 129 | 130 | // Output: 131 | // main.T1: 2 132 | // main.T2: I am T2 133 | // main.T1: 9999 134 | } 135 | ``` 136 | 137 | 138 | ## License 139 | 140 | MIT 141 | --------------------------------------------------------------------------------