├── .gitignore ├── AUTHORS ├── examples ├── net │ ├── client.go │ └── server.go ├── file │ ├── reader.go │ └── writer.go ├── bufferedEncoder │ └── bufferedEncoder.go ├── decodeRecord │ └── decodeRecord.go ├── encodeRecord │ └── encodeRecord.go ├── generic_datum │ └── generic_datum.go ├── nestedRecords │ └── nestedRecords.go ├── nestedRecordsMultipleDefinitions │ └── nestedRecordsMultipleDefinitions.go └── blocktick │ └── pipe.go ├── ocf.go ├── name.go ├── helpers_test.go ├── name_test.go ├── encoder.go ├── decoder.go ├── record_test.go ├── ocf_reader_test.go ├── ocf_writer_test.go ├── record.go ├── LICENSE ├── ocf_reader.go ├── README.md ├── ocf_writer.go ├── codec.go └── codec_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.test 2 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Goavro was originally created during the Fall of 2014 at LinkedIn, 2 | Corp., in New York City, New York, USA. 3 | 4 | The following persons, listed in alphabetical order, have participated 5 | with goavro development by contributing code and test cases. 6 | 7 | Fellyn Silliman 8 | Karrick McDermott 9 | Michael Johnson 10 | 11 | A big thank you to these persons who provided testing and amazing 12 | feedback to goavro during its initial implementation: 13 | 14 | Dennis Ordanov 15 | Thomas Desrosiers 16 | 17 | Also a big thank you is extended to our supervisors who supported our 18 | efforts to bring goavro to the open source community: 19 | 20 | Greg Leffler 21 | Nick Berry 22 | -------------------------------------------------------------------------------- /examples/net/client.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/linkedin/goavro" 24 | "log" 25 | "net" 26 | ) 27 | 28 | func main() { 29 | conn, err := net.Dial("tcp", "127.0.0.1:8080") 30 | if err != nil { 31 | log.Fatal(err) 32 | } 33 | fr, err := goavro.NewReader(goavro.FromReader(conn)) 34 | if err != nil { 35 | log.Fatal("cannot create Reader: ", err) 36 | } 37 | defer func() { 38 | if err := fr.Close(); err != nil { 39 | log.Fatal(err) 40 | } 41 | }() 42 | 43 | for fr.Scan() { 44 | datum, err := fr.Read() 45 | if err != nil { 46 | log.Println("cannot read datum: ", err) 47 | continue 48 | } 49 | fmt.Println("RECORD: ", datum) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /examples/file/reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/linkedin/goavro" 24 | "io" 25 | "log" 26 | "os" 27 | ) 28 | 29 | func main() { 30 | if len(os.Args) > 1 { 31 | for i, arg := range os.Args { 32 | if i == 0 { 33 | continue 34 | } 35 | fh, err := os.Open(arg) 36 | if err != nil { 37 | log.Fatal(err) 38 | } 39 | dumpReader(fh) 40 | fh.Close() 41 | } 42 | } else { 43 | dumpReader(os.Stdin) 44 | } 45 | } 46 | 47 | func dumpReader(r io.Reader) { 48 | fr, err := goavro.NewReader(goavro.BufferFromReader(r)) 49 | if err != nil { 50 | log.Fatal(err) 51 | } 52 | defer func() { 53 | if err := fr.Close(); err != nil { 54 | log.Fatal(err) 55 | } 56 | }() 57 | 58 | for fr.Scan() { 59 | datum, err := fr.Read() 60 | if err != nil { 61 | log.Println(err) 62 | continue 63 | } 64 | fmt.Println(datum) 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /ocf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | const ( 22 | magicBytes = "Obj\x01" 23 | syncLength = 16 24 | metadataSchema = `{"type":"map","values":"bytes"}` 25 | ) 26 | 27 | // Compression codecs that Reader and Writer instances can process. 28 | const ( 29 | CompressionNull = "null" 30 | CompressionDeflate = "deflate" 31 | CompressionSnappy = "snappy" 32 | ) 33 | 34 | var ( 35 | metadataCodec Codec 36 | ) 37 | 38 | func init() { 39 | metadataCodec, _ = NewCodec(metadataSchema) 40 | } 41 | 42 | // IsCompressionCodecSupported returns true if and only if the specified codec 43 | // string is supported by this library. 44 | func IsCompressionCodecSupported(someCodec string) bool { 45 | switch someCodec { 46 | case CompressionNull, CompressionDeflate, CompressionSnappy: 47 | return true 48 | default: 49 | return false 50 | } 51 | } 52 | 53 | // Datum binds together a piece of data and any error resulting from 54 | // either reading or writing that datum. 55 | type Datum struct { 56 | Value interface{} 57 | Err error 58 | } 59 | -------------------------------------------------------------------------------- /examples/bufferedEncoder/bufferedEncoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "github.com/linkedin/goavro" 25 | "io" 26 | "log" 27 | ) 28 | 29 | func main() { 30 | bits, err := bufferedEncoder(`"string"`, "filibuster") 31 | if err != nil { 32 | log.Fatal(err) 33 | } 34 | expected := []byte("\x14filibuster") 35 | if bytes.Compare(bits, expected) != 0 { 36 | log.Fatalf("Actual: %#v; Expected: %#v", bits, expected) 37 | } 38 | } 39 | 40 | func bufferedEncoder(someSchemaJSON string, datum interface{}) (bits []byte, err error) { 41 | bb := new(bytes.Buffer) 42 | defer func() { 43 | bits = bb.Bytes() 44 | }() 45 | 46 | var c goavro.Codec 47 | c, err = goavro.NewCodec(someSchemaJSON) 48 | if err != nil { 49 | return 50 | } 51 | err = encodeWithBufferedWriter(c, bb, datum) 52 | return 53 | } 54 | 55 | func encodeWithBufferedWriter(c goavro.Codec, w io.Writer, datum interface{}) error { 56 | bw := bufio.NewWriter(w) 57 | err := c.Encode(bw, datum) 58 | if err != nil { 59 | return err 60 | } 61 | return bw.Flush() 62 | } 63 | -------------------------------------------------------------------------------- /examples/decodeRecord/decodeRecord.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/linkedin/goavro" 25 | "log" 26 | ) 27 | 28 | func main() { 29 | recordSchemaJSON := ` 30 | { 31 | "type": "record", 32 | "name": "comments", 33 | "doc:": "A basic schema for storing blog comments", 34 | "namespace": "com.example", 35 | "fields": [ 36 | { 37 | "doc": "Name of user", 38 | "type": "string", 39 | "name": "username" 40 | }, 41 | { 42 | "doc": "The content of the user's message", 43 | "type": "string", 44 | "name": "comment" 45 | }, 46 | { 47 | "doc": "Unix epoch time in milliseconds", 48 | "type": "long", 49 | "name": "timestamp" 50 | } 51 | ] 52 | } 53 | ` 54 | codec, err := goavro.NewCodec(recordSchemaJSON) 55 | if err != nil { 56 | log.Fatal(err) 57 | } 58 | encoded := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08") 59 | bb := bytes.NewBuffer(encoded) 60 | decoded, err := codec.Decode(bb) 61 | 62 | fmt.Println(decoded) // default String() representation is JSON 63 | 64 | // but direct access to data is provided 65 | record := decoded.(*goavro.Record) 66 | fmt.Println("Record Name:", record.Name) 67 | fmt.Println("Record Fields:") 68 | for i, field := range record.Fields { 69 | fmt.Println(" field", i, field.Name, ":", field.Datum) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /examples/encodeRecord/encodeRecord.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "github.com/linkedin/goavro" 24 | "log" 25 | ) 26 | 27 | func main() { 28 | recordSchemaJSON := ` 29 | { 30 | "type": "record", 31 | "name": "comments", 32 | "doc:": "A basic schema for storing blog comments", 33 | "namespace": "com.example", 34 | "fields": [ 35 | { 36 | "doc": "Name of user", 37 | "type": "string", 38 | "name": "username" 39 | }, 40 | { 41 | "doc": "The content of the user's message", 42 | "type": "string", 43 | "name": "comment" 44 | }, 45 | { 46 | "doc": "Unix epoch time in milliseconds", 47 | "type": "long", 48 | "name": "timestamp" 49 | } 50 | ] 51 | } 52 | ` 53 | someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchemaJSON)) 54 | if err != nil { 55 | log.Fatal(err) 56 | } 57 | // identify field name to set datum for 58 | someRecord.Set("username", "Aquaman") 59 | someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 60 | // you can fully qualify the field name 61 | someRecord.Set("com.example.timestamp", int64(1082196484)) 62 | 63 | codec, err := goavro.NewCodec(recordSchemaJSON) 64 | if err != nil { 65 | log.Fatal(err) 66 | } 67 | 68 | bb := new(bytes.Buffer) 69 | if err = codec.Encode(bb, someRecord); err != nil { 70 | log.Fatal(err) 71 | } 72 | 73 | actual := bb.Bytes() 74 | expected := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08") 75 | if bytes.Compare(actual, expected) != 0 { 76 | log.Printf("Actual: %#v; Expected: %#v", actual, expected) 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /examples/net/server.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "github.com/linkedin/goavro" 23 | "log" 24 | "net" 25 | ) 26 | 27 | const recordSchema = ` 28 | { 29 | "type": "record", 30 | "name": "comments", 31 | "doc:": "A basic schema for storing blog comments", 32 | "namespace": "com.example", 33 | "fields": [ 34 | { 35 | "doc": "Name of user", 36 | "type": "string", 37 | "name": "username" 38 | }, 39 | { 40 | "doc": "The content of the user's message", 41 | "type": "string", 42 | "name": "comment" 43 | }, 44 | { 45 | "doc": "Unix epoch time in milliseconds", 46 | "type": "long", 47 | "name": "timestamp" 48 | } 49 | ] 50 | } 51 | ` 52 | 53 | var ( 54 | codec goavro.Codec 55 | ) 56 | 57 | func init() { 58 | var err error 59 | // If you want speed, create the codec one time for each 60 | // schema and reuse it to create multiple Writer instances. 61 | codec, err = goavro.NewCodec(recordSchema) 62 | if err != nil { 63 | log.Fatal(err) 64 | } 65 | } 66 | 67 | func main() { 68 | ln, err := net.Listen("tcp", ":8080") 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | for { 73 | conn, err := ln.Accept() 74 | if err != nil { 75 | log.Fatal(err) 76 | } 77 | go serveClient(conn, codec) 78 | } 79 | } 80 | 81 | func serveClient(conn net.Conn, codec goavro.Codec) { 82 | fw, err := codec.NewWriter( 83 | goavro.Compression(goavro.CompressionDeflate), 84 | goavro.ToWriter(conn)) 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | defer fw.Close() 89 | 90 | // create a record that matches the schema we want to encode 91 | someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema)) 92 | if err != nil { 93 | log.Fatal(err) 94 | } 95 | // identify field name to set datum for 96 | someRecord.Set("username", "Aquaman") 97 | someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 98 | // you can fully qualify the field name 99 | someRecord.Set("com.example.timestamp", int64(1082196484)) 100 | fw.Write(someRecord) 101 | 102 | // create another record 103 | if someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)); err != nil { 104 | log.Fatal(err) 105 | } 106 | someRecord.Set("username", "Batman") 107 | someRecord.Set("comment", "Who are all of these crazies?") 108 | someRecord.Set("com.example.timestamp", int64(1427383430)) 109 | fw.Write(someRecord) 110 | } 111 | -------------------------------------------------------------------------------- /examples/file/writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/linkedin/goavro" 24 | "io" 25 | "log" 26 | "os" 27 | ) 28 | 29 | const innerSchema = ` 30 | { 31 | "type": "record", 32 | "name": "user", 33 | "namespace": "com.example", 34 | "doc": "User information", 35 | "fields": [ 36 | { 37 | "type": "string", 38 | "name": "account", 39 | "doc": "The user's account name" 40 | }, 41 | { 42 | "type": "long", 43 | "name": "creationDate", 44 | "doc": "Unix epoch time in milliseconds" 45 | } 46 | ] 47 | } 48 | ` 49 | 50 | var ( 51 | outerSchema string 52 | codec goavro.Codec 53 | ) 54 | 55 | func init() { 56 | outerSchema = fmt.Sprintf(` 57 | { 58 | "type": "record", 59 | "name": "comments", 60 | "doc:": "A basic schema for storing blog comments", 61 | "namespace": "com.example", 62 | "fields": [ 63 | { 64 | "name": "user", 65 | "type": %s 66 | }, 67 | { 68 | "doc": "The content of the user's message", 69 | "type": "string", 70 | "name": "comment" 71 | }, 72 | { 73 | "doc": "Unix epoch time in milliseconds", 74 | "type": "long", 75 | "name": "timestamp" 76 | } 77 | ] 78 | } 79 | `, innerSchema) 80 | 81 | var err error 82 | // If you want speed, create the codec one time for each 83 | // schema and reuse it to create multiple Writer instances. 84 | codec, err = goavro.NewCodec(outerSchema) 85 | if err != nil { 86 | log.Fatal(err) 87 | } 88 | } 89 | 90 | func main() { 91 | switch len(os.Args) { 92 | case 1: 93 | dumpWriter(os.Stdout, codec) 94 | case 2: 95 | fh, err := os.Create(os.Args[1]) 96 | if err != nil { 97 | log.Fatal(err) 98 | } 99 | dumpWriter(fh, codec) 100 | fh.Close() 101 | default: 102 | fmt.Fprintf(os.Stderr, "usage: %s [filename]\n", os.Args[0]) 103 | os.Exit(2) 104 | } 105 | } 106 | 107 | func dumpWriter(w io.Writer, codec goavro.Codec) { 108 | fw, err := codec.NewWriter( 109 | goavro.Compression(goavro.CompressionDeflate), 110 | goavro.ToWriter(w)) 111 | if err != nil { 112 | log.Fatal(err) 113 | } 114 | defer fw.Close() 115 | 116 | // If we want to encode data, we need to put it in an actual 117 | // goavro.Record instance corresponding to the schema we wish 118 | // to encode against. 119 | // 120 | // NewRecord will create a goavro.Record instance 121 | // corresponding to the specified schema. 122 | innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema)) 123 | if err != nil { 124 | log.Fatal(err) 125 | } 126 | innerRecord.Set("account", "Aquaman") 127 | innerRecord.Set("creationDate", int64(1082196484)) 128 | 129 | // We create both an innerRecord and an outerRecord. 130 | outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema)) 131 | if err != nil { 132 | log.Fatal(err) 133 | } 134 | // innerRecord is a completely seperate record instance from 135 | // outerRecord. Once we have an innerRecord instance it can be 136 | // assigned to the appropriate Datum item of the outerRecord. 137 | outerRecord.Set("user", innerRecord) 138 | // Other fields are set on the outerRecord. 139 | outerRecord.Set("comment", "The Atlantic is oddly cold this morning!") 140 | outerRecord.Set("timestamp", int64(1427255074)) 141 | fw.Write(outerRecord) 142 | } 143 | -------------------------------------------------------------------------------- /examples/generic_datum/generic_datum.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/linkedin/goavro" 25 | "log" 26 | ) 27 | 28 | var ( 29 | outerSchema, innerSchema string 30 | ) 31 | 32 | func init() { 33 | innerSchema = ` 34 | { 35 | "type": "record", 36 | "name": "TestRecord2", 37 | "fields": [ 38 | { 39 | "name": "stringValue", 40 | "type": "string" 41 | }, 42 | { 43 | "name": "intValue", 44 | "type": "int" 45 | } 46 | ] 47 | } 48 | ` 49 | outerSchema = fmt.Sprintf(` 50 | { 51 | "type": "record", 52 | "name": "TestRecord", 53 | "fields": [ 54 | { 55 | "name": "value", 56 | "type": "int" 57 | }, 58 | { 59 | "name": "rec", 60 | "type": { 61 | "type": "array", 62 | "items": %s 63 | } 64 | } 65 | ] 66 | } 67 | `, innerSchema) 68 | } 69 | 70 | func main() { 71 | innerRecords := make([]interface{}, 0) 72 | // make first inner record 73 | innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema)) 74 | if err != nil { 75 | log.Fatalf("cannot create innerRecord: %v", err) 76 | } 77 | if err = innerRecord.Set("stringValue", "Hello"); err != nil { 78 | log.Fatal(err) 79 | } 80 | if err = innerRecord.Set("intValue", int32(1)); err != nil { 81 | log.Fatal(err) 82 | } 83 | innerRecords = append(innerRecords, innerRecord) 84 | // make another inner record 85 | innerRecord, _ = goavro.NewRecord(goavro.RecordSchema(innerSchema)) 86 | innerRecord.Set("stringValue", "World") 87 | innerRecord.Set("intValue", int32(2)) 88 | innerRecords = append(innerRecords, innerRecord) 89 | // make outer record 90 | outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema)) 91 | if err != nil { 92 | log.Fatalf("cannot create outerRecord: %v", err) 93 | } 94 | outerRecord.Set("value", int32(3)) 95 | outerRecord.Set("rec", innerRecords) 96 | // make a codec 97 | c, err := goavro.NewCodec(outerSchema) 98 | if err != nil { 99 | log.Fatal(err) 100 | } 101 | // encode outerRecord to io.Writer (here, a bytes.Buffer) 102 | bb := new(bytes.Buffer) 103 | err = c.Encode(bb, outerRecord) 104 | if err != nil { 105 | log.Fatal(err) 106 | } 107 | // decode bytes 108 | decoded, err := c.Decode(bytes.NewReader(bb.Bytes())) 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | decodedRecord, ok := decoded.(*goavro.Record) 113 | if !ok { 114 | log.Fatalf("expected *goavro.Record; received: %T", decoded) 115 | } 116 | decodedValue, err := decodedRecord.Get("value") 117 | if err != nil { 118 | log.Fatal(err) 119 | } 120 | if decodedValue != int32(3) { 121 | log.Printf("Actual: %#v; Expected: %#v\n", decodedValue, int32(3)) 122 | } 123 | fmt.Printf("Read a value: %d\n", decodedValue) 124 | rec, err := decodedRecord.Get("rec") 125 | if err != nil { 126 | log.Fatal(err) 127 | } 128 | decodedArray := rec.([]interface{}) 129 | if len(decodedArray) != 2 { 130 | log.Fatalf("Actual: %#v; Expected: %#v\n", len(decodedArray), 2) 131 | } 132 | for index, decodedSubRecord := range decodedArray { 133 | r := decodedSubRecord.(*goavro.Record) 134 | sv, err := r.Get("stringValue") 135 | if err != nil { 136 | log.Fatal(err) 137 | } 138 | iv, err := r.Get("intValue") 139 | if err != nil { 140 | log.Fatal(err) 141 | } 142 | fmt.Printf("Read a subrecord %d string value: %s\n", index, sv) 143 | fmt.Printf("Read a subrecord %d int value: %d\n", index, iv) 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /name.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "fmt" 23 | "strings" 24 | ) 25 | 26 | const ( 27 | nullNamespace = "" 28 | ) 29 | 30 | type name struct { 31 | n string // name 32 | ns string // namespace 33 | ens string // enclosing namespace 34 | } 35 | 36 | type nameSetter func(*name) error 37 | 38 | func newName(setters ...nameSetter) (*name, error) { 39 | var err error 40 | n := &name{} 41 | for _, setter := range setters { 42 | if err = setter(n); err != nil { 43 | return nil, err 44 | } 45 | } 46 | // if name contains dot, then ignore namespace and enclosing namespace 47 | if !strings.ContainsRune(n.n, '.') { 48 | if n.ns != "" { 49 | n.n = n.ns + "." + n.n 50 | } else if n.ens != "" { 51 | n.n = n.ens + "." + n.n 52 | } 53 | } 54 | return n, nil 55 | } 56 | 57 | func nameSchema(schema map[string]interface{}) nameSetter { 58 | return func(n *name) error { 59 | val, ok := schema["name"] 60 | if !ok { 61 | return fmt.Errorf("ought to have name key") 62 | } 63 | n.n, ok = val.(string) 64 | if !ok || len(n.n) == 0 { 65 | return fmt.Errorf("name ought to be non-empty string: %T", n) 66 | } 67 | if val, ok := schema["namespace"]; ok { 68 | n.ns, ok = val.(string) 69 | if !ok { 70 | return fmt.Errorf("namespace ought to be a string: %T", n) 71 | } 72 | } 73 | return nil 74 | } 75 | } 76 | 77 | // ErrInvalidName is returned when a Codec cannot be created due to 78 | // invalid name format. 79 | type ErrInvalidName struct { 80 | Message string 81 | } 82 | 83 | func (e ErrInvalidName) Error() string { 84 | return "The name portion of a fullname, record field names, and enum symbols must " + e.Message 85 | } 86 | 87 | func isRuneInvalidForFirstCharacter(r rune) bool { 88 | if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || r == '_' { 89 | return false 90 | } 91 | return true 92 | } 93 | 94 | func isRuneInvalidForOtherCharacters(r rune) bool { 95 | if r >= '0' && r <= '9' { 96 | return false 97 | } 98 | return isRuneInvalidForFirstCharacter(r) 99 | } 100 | 101 | func checkName(s string) error { 102 | if len(s) == 0 { 103 | return &ErrInvalidName{"not be empty"} 104 | } 105 | if strings.IndexFunc(s[:1], isRuneInvalidForFirstCharacter) != -1 { 106 | return &ErrInvalidName{"start with [A-Za-z_]"} 107 | } 108 | if strings.IndexFunc(s[1:], isRuneInvalidForOtherCharacters) != -1 { 109 | return &ErrInvalidName{"have second and remaining characters contain only [A-Za-z0-9_]"} 110 | } 111 | return nil 112 | } 113 | 114 | func nameName(someName string) nameSetter { 115 | return func(n *name) (err error) { 116 | if err = checkName(someName); err == nil { 117 | n.n = someName 118 | } 119 | return 120 | } 121 | } 122 | 123 | func nameEnclosingNamespace(someNamespace string) nameSetter { 124 | return func(n *name) error { 125 | n.ens = someNamespace 126 | return nil 127 | } 128 | } 129 | 130 | func nameNamespace(someNamespace string) nameSetter { 131 | return func(n *name) error { 132 | n.ns = someNamespace 133 | return nil 134 | } 135 | } 136 | 137 | func (n *name) equals(b *name) bool { 138 | if n.n == b.n { 139 | return true 140 | } 141 | return false 142 | } 143 | 144 | func (n name) namespace() string { 145 | li := strings.LastIndex(n.n, ".") 146 | if li == -1 { 147 | return "" 148 | } 149 | return n.n[:li] 150 | } 151 | 152 | func (n name) GoString() string { 153 | return n.n 154 | } 155 | 156 | func (n name) String() string { 157 | return n.n 158 | } 159 | -------------------------------------------------------------------------------- /examples/nestedRecords/nestedRecords.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/linkedin/goavro" 25 | "log" 26 | ) 27 | 28 | const innerSchema = ` 29 | { 30 | "type": "record", 31 | "name": "user", 32 | "namespace": "com.example", 33 | "doc": "User information", 34 | "fields": [ 35 | { 36 | "type": "string", 37 | "name": "account", 38 | "doc": "The user's account name" 39 | }, 40 | { 41 | "type": "long", 42 | "name": "creationDate", 43 | "doc": "Unix epoch time in milliseconds" 44 | } 45 | ] 46 | } 47 | ` 48 | 49 | var ( 50 | outerSchema string 51 | codec goavro.Codec 52 | ) 53 | 54 | func init() { 55 | outerSchema = fmt.Sprintf(` 56 | { 57 | "type": "record", 58 | "name": "comments", 59 | "doc:": "A basic schema for storing blog comments", 60 | "namespace": "com.example", 61 | "fields": [ 62 | { 63 | "name": "user", 64 | "type": %s 65 | }, 66 | { 67 | "doc": "The content of the user's message", 68 | "type": "string", 69 | "name": "comment" 70 | }, 71 | { 72 | "doc": "Unix epoch time in milliseconds", 73 | "type": "long", 74 | "name": "timestamp" 75 | } 76 | ] 77 | } 78 | `, innerSchema) 79 | 80 | var err error 81 | // If you want speed, create the codec one time for each 82 | // schema and reuse it to create multiple Writer instances. 83 | codec, err = goavro.NewCodec(outerSchema) 84 | if err != nil { 85 | log.Fatal(err) 86 | } 87 | } 88 | 89 | func main() { 90 | // If we want to encode data, we need to put it in an actual 91 | // goavro.Record instance corresponding to the schema we wish 92 | // to encode against. 93 | // 94 | // NewRecord will create a goavro.Record instance 95 | // corresponding to the specified schema. 96 | innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema)) 97 | if err != nil { 98 | log.Fatal(err) 99 | } 100 | innerRecord.Set("account", "Aquaman") 101 | innerRecord.Set("creationDate", int64(1082196484)) 102 | 103 | // We create both an innerRecord and an outerRecord. 104 | outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema)) 105 | if err != nil { 106 | log.Fatal(err) 107 | } 108 | // innerRecord is a completely seperate record instance from 109 | // outerRecord. Once we have an innerRecord instance it can be 110 | // assigned to the appropriate Datum item of the outerRecord. 111 | outerRecord.Set("user", innerRecord) 112 | // Other fields are set on the outerRecord. 113 | outerRecord.Set("comment", "The Atlantic is oddly cold this morning!") 114 | outerRecord.Set("timestamp", int64(1427255074)) 115 | 116 | // Encode the outerRecord into a bytes.Buffer 117 | bb := new(bytes.Buffer) 118 | if err = codec.Encode(bb, outerRecord); err != nil { 119 | log.Fatal(err) 120 | } 121 | // Compare encoded bytes against the expected bytes. 122 | actual := bb.Bytes() 123 | expected := []byte( 124 | "\x0eAquaman" + // account 125 | "\x88\x88\x88\x88\x08" + // creationDate 126 | "\x50" + // 50 hex == 80 dec variable length integer encoded == 40 -> string is 40 characters long 127 | "The Atlantic is oddly cold this morning!" + // comment 128 | "\xc4\xbc\x91\xd1\x0a") // timestamp 129 | if bytes.Compare(actual, expected) != 0 { 130 | log.Printf("Actual: %#v; Expected: %#v", actual, expected) 131 | } 132 | // Let's decode the blob and print the output in JSON format 133 | // using goavro.Record's String() method. 134 | decoded, err := codec.Decode(bytes.NewReader(actual)) 135 | fmt.Println(decoded) 136 | // we only need to perform type assertion if we want to access inside 137 | record := decoded.(*goavro.Record) 138 | fmt.Println("Record Name:", record.Name) 139 | fmt.Println("Record Fields:") 140 | for i, field := range record.Fields { 141 | fmt.Println(" field", i, field.Name, ":", field.Datum) 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /examples/nestedRecordsMultipleDefinitions/nestedRecordsMultipleDefinitions.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "bytes" 23 | "fmt" 24 | "github.com/fede1024/goavro" 25 | "log" 26 | ) 27 | 28 | const innerSchema = ` 29 | { 30 | "type": "record", 31 | "name": "user", 32 | "namespace": "com.example", 33 | "doc": "User information", 34 | "fields": [ 35 | { 36 | "type": "string", 37 | "name": "account", 38 | "doc": "The user's account name" 39 | }, 40 | { 41 | "type": "long", 42 | "name": "creationDate", 43 | "doc": "Unix epoch time in milliseconds" 44 | } 45 | ] 46 | } 47 | ` 48 | 49 | const outerSchema = ` 50 | { 51 | "type": "record", 52 | "name": "comments", 53 | "doc:": "A basic schema for storing blog comments", 54 | "namespace": "com.example", 55 | "fields": [ 56 | { 57 | "name": "user", 58 | "type": "user" 59 | }, 60 | { 61 | "doc": "The content of the user's message", 62 | "type": "string", 63 | "name": "comment" 64 | }, 65 | { 66 | "doc": "Unix epoch time in milliseconds", 67 | "type": "long", 68 | "name": "timestamp" 69 | } 70 | ] 71 | } 72 | ` 73 | 74 | var codec goavro.Codec 75 | 76 | func init() { 77 | var err error 78 | var st goavro.Symtab 79 | 80 | st = goavro.NewSymtab() 81 | _, err = st.NewCodec(innerSchema) 82 | if err != nil { 83 | log.Fatal(err) 84 | } 85 | // If you want speed, create the codec one time for each 86 | // schema and reuse it to create multiple Writer instances. 87 | codec, err = st.NewCodec(outerSchema) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | } 92 | 93 | func main() { 94 | // If we want to encode data, we need to put it in an actual 95 | // goavro.Record instance corresponding to the schema we wish 96 | // to encode against. 97 | // 98 | // NewRecord will create a goavro.Record instance 99 | // corresponding to the specified schema. 100 | innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema)) 101 | if err != nil { 102 | log.Fatal(err) 103 | } 104 | innerRecord.Set("account", "Aquaman") 105 | innerRecord.Set("creationDate", int64(1082196484)) 106 | 107 | // We create both an innerRecord and an outerRecord. 108 | outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema)) 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | // innerRecord is a completely seperate record instance from 113 | // outerRecord. Once we have an innerRecord instance it can be 114 | // assigned to the appropriate Datum item of the outerRecord. 115 | outerRecord.Set("user", innerRecord) 116 | // Other fields are set on the outerRecord. 117 | outerRecord.Set("comment", "The Atlantic is oddly cold this morning!") 118 | outerRecord.Set("timestamp", int64(1427255074)) 119 | 120 | // Encode the outerRecord into a bytes.Buffer 121 | bb := new(bytes.Buffer) 122 | if err = codec.Encode(bb, outerRecord); err != nil { 123 | log.Fatal(err) 124 | } 125 | // Compare encoded bytes against the expected bytes. 126 | actual := bb.Bytes() 127 | expected := []byte( 128 | "\x0eAquaman" + // account 129 | "\x88\x88\x88\x88\x08" + // creationDate 130 | "\x50" + // 50 hex == 80 dec variable length integer encoded == 40 -> string is 40 characters long 131 | "The Atlantic is oddly cold this morning!" + // comment 132 | "\xc4\xbc\x91\xd1\x0a") // timestamp 133 | if bytes.Compare(actual, expected) != 0 { 134 | log.Printf("Actual: %#v; Expected: %#v", actual, expected) 135 | } 136 | // Let's decode the blob and print the output in JSON format 137 | // using goavro.Record's String() method. 138 | decoded, err := codec.Decode(bytes.NewReader(actual)) 139 | fmt.Println(decoded) 140 | // we only need to perform type assertion if we want to access inside 141 | record := decoded.(*goavro.Record) 142 | fmt.Println("Record Name:", record.Name) 143 | fmt.Println("Record Fields:") 144 | for i, field := range record.Fields { 145 | fmt.Println(" field", i, field.Name, ":", field.Datum) 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /examples/blocktick/pipe.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package main 20 | 21 | import ( 22 | "fmt" 23 | "github.com/linkedin/goavro" 24 | "io" 25 | "log" 26 | "math/rand" 27 | "os" 28 | "os/signal" 29 | "time" 30 | ) 31 | 32 | const innerSchema = ` 33 | { 34 | "type": "record", 35 | "name": "user", 36 | "namespace": "com.example", 37 | "doc": "User information", 38 | "fields": [ 39 | { 40 | "type": "string", 41 | "name": "account", 42 | "doc": "The user's account name" 43 | }, 44 | { 45 | "type": "long", 46 | "name": "creationDate", 47 | "doc": "Unix epoch time" 48 | } 49 | ] 50 | } 51 | ` 52 | 53 | var ( 54 | outerSchema string 55 | codec goavro.Codec 56 | ) 57 | 58 | func init() { 59 | outerSchema = fmt.Sprintf(` 60 | { 61 | "type": "record", 62 | "name": "comments", 63 | "doc:": "A basic schema for storing blog comments", 64 | "namespace": "com.example", 65 | "fields": [ 66 | { 67 | "name": "user", 68 | "type": %s 69 | }, 70 | { 71 | "doc": "The content of the user's message", 72 | "type": "string", 73 | "name": "comment" 74 | }, 75 | { 76 | "doc": "Unix epoch time in nanoseconds", 77 | "type": "long", 78 | "name": "timestamp" 79 | } 80 | ] 81 | } 82 | `, innerSchema) 83 | 84 | var err error 85 | // If you want speed, create the codec one time for each 86 | // schema and reuse it to create multiple Writer instances. 87 | codec, err = goavro.NewCodec(outerSchema) 88 | if err != nil { 89 | log.Fatal(err) 90 | } 91 | } 92 | 93 | func main() { 94 | pr, pw, err := os.Pipe() 95 | if err != nil { 96 | log.Fatal(err) 97 | } 98 | 99 | go dumpWriter(pw, codec) 100 | dumpReader(pr) 101 | } 102 | 103 | func dumpWriter(w io.Writer, codec goavro.Codec) { 104 | fw, err := codec.NewWriter( 105 | goavro.BlockSize(5), // queue up no more than 5 items 106 | goavro.BlockTick(3*time.Second), // but flush at least every 3 seconds 107 | goavro.Compression(goavro.CompressionDeflate), 108 | goavro.ToWriter(w)) 109 | if err != nil { 110 | log.Fatal(err) 111 | } 112 | defer fw.Close() 113 | 114 | sigs := make(chan os.Signal) 115 | signal.Notify(sigs) 116 | defer func() { 117 | signal.Stop(sigs) 118 | }() 119 | 120 | writeLoop: 121 | for { 122 | select { 123 | case <-time.After(time.Duration(rand.Intn(500)) * time.Millisecond): 124 | sendRecord(fw) 125 | case <-sigs: 126 | break writeLoop 127 | } 128 | } 129 | } 130 | 131 | func sendRecord(fw *goavro.Writer) { 132 | // If we want to encode data, we need to put it in an actual 133 | // goavro.Record instance corresponding to the schema we wish 134 | // to encode against. 135 | // 136 | // NewRecord will create a goavro.Record instance 137 | // corresponding to the specified schema. 138 | innerRecord, err := goavro.NewRecord(goavro.RecordSchema(innerSchema)) 139 | if err != nil { 140 | log.Fatal(err) 141 | } 142 | innerRecord.Set("account", "Aquaman") 143 | innerRecord.Set("creationDate", int64(1082196484)) 144 | 145 | // We create both an innerRecord and an outerRecord. 146 | outerRecord, err := goavro.NewRecord(goavro.RecordSchema(outerSchema)) 147 | if err != nil { 148 | log.Fatal(err) 149 | } 150 | // innerRecord is a completely seperate record instance from 151 | // outerRecord. Once we have an innerRecord instance it can be 152 | // assigned to the appropriate Datum item of the outerRecord. 153 | outerRecord.Set("user", innerRecord) 154 | // Other fields are set on the outerRecord. 155 | outerRecord.Set("comment", "The Atlantic is oddly cold this morning!") 156 | outerRecord.Set("timestamp", int64(time.Now().UnixNano())) 157 | fw.Write(outerRecord) 158 | } 159 | 160 | func dumpReader(r io.Reader) { 161 | fr, err := goavro.NewReader(goavro.FromReader(r)) 162 | if err != nil { 163 | log.Fatal(err) 164 | } 165 | defer func() { 166 | if err := fr.Close(); err != nil { 167 | log.Fatal(err) 168 | } 169 | }() 170 | 171 | for fr.Scan() { 172 | datum, err := fr.Read() 173 | if err != nil { 174 | log.Println(err) 175 | continue 176 | } 177 | fmt.Println(datum) 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /helpers_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bytes" 23 | "encoding/json" 24 | "fmt" 25 | "strings" 26 | "testing" 27 | ) 28 | 29 | func checkError(t *testing.T, actualError error, expectedError interface{}) { 30 | if expectedError == nil { 31 | if actualError != nil { 32 | t.Errorf("Actual: %#v; Expected: %#v", actualError.Error(), expectedError) 33 | } 34 | } else { 35 | if actualError == nil { 36 | t.Errorf("Actual: %#v; Expected: %#v", actualError, expectedError) 37 | } else { 38 | var expected error 39 | switch expectedError.(type) { 40 | case string: 41 | expected = fmt.Errorf(expectedError.(string)) 42 | case error: 43 | expected = expectedError.(error) 44 | } 45 | if !strings.Contains(actualError.Error(), expected.Error()) { 46 | t.Errorf("Actual: %#v; Expected to contain: %#v", 47 | actualError.Error(), expected.Error()) 48 | } 49 | } 50 | } 51 | } 52 | 53 | func checkErrorFatal(t *testing.T, actualError error, expectedError interface{}) { 54 | if expectedError == nil { 55 | if actualError != nil { 56 | t.Fatalf("Actual: %#v; Expected: %#v", actualError.Error(), expectedError) 57 | } 58 | } else { 59 | if actualError == nil { 60 | t.Fatalf("Actual: %#v; Expected: %#v", actualError, expectedError) 61 | } else { 62 | var expected error 63 | switch expectedError.(type) { 64 | case string: 65 | expected = fmt.Errorf(expectedError.(string)) 66 | case error: 67 | expected = expectedError.(error) 68 | } 69 | if !strings.Contains(actualError.Error(), expected.Error()) { 70 | t.Fatalf("Actual: %#v; Expected to contain: %#v", 71 | actualError.Error(), expected.Error()) 72 | } 73 | } 74 | } 75 | } 76 | 77 | func checkResponse(t *testing.T, bb *bytes.Buffer, n int, expectedBytes []byte) { 78 | expectedCount := len(expectedBytes) 79 | if n != expectedCount { 80 | t.Errorf("Actual: %#v; Expected: %#v", n, expectedCount) 81 | } 82 | if bytes.Compare(bb.Bytes(), expectedBytes) != 0 { 83 | t.Errorf("Actual: %#v; Expected: %#v", bb.Bytes(), expectedBytes) 84 | } 85 | } 86 | 87 | func schemaType(t *testing.T, someJSONSchema string) string { 88 | var schema interface{} 89 | err := json.Unmarshal([]byte(someJSONSchema), &schema) 90 | if err != nil { 91 | t.Fatal(err) 92 | } 93 | switch schema.(type) { 94 | case map[string]interface{}: 95 | someMap := schema.(map[string]interface{}) 96 | someValue, ok := someMap["type"] 97 | if !ok { 98 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 99 | } 100 | someString, ok := someValue.(string) 101 | if !ok { 102 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 103 | } 104 | return someString 105 | case []interface{}: 106 | return "union" 107 | default: 108 | t.Errorf("Actual: %T; Expected: map[string]interface{}", schema) 109 | return "" 110 | } 111 | } 112 | 113 | func schemaTypeCodec(t *testing.T, someJSONSchema string) string { 114 | var schema interface{} 115 | err := json.Unmarshal([]byte(someJSONSchema), &schema) 116 | if err != nil { 117 | t.Fatal(err) 118 | } 119 | switch schema.(type) { 120 | case map[string]interface{}: 121 | someMap := schema.(map[string]interface{}) 122 | someValue, ok := someMap["type"] 123 | if !ok { 124 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 125 | } 126 | someString, ok := someValue.(string) 127 | if !ok { 128 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 129 | } 130 | return someString 131 | case []interface{}: 132 | return "union" 133 | default: 134 | t.Errorf("Actual: %T; Expected: map[string]interface{}", schema) 135 | return "" 136 | } 137 | } 138 | 139 | func schemaName(t *testing.T, someJSONSchema string) string { 140 | var schema interface{} 141 | err := json.Unmarshal([]byte(someJSONSchema), &schema) 142 | if err != nil { 143 | t.Fatal(err) 144 | } 145 | someMap, ok := schema.(map[string]interface{}) 146 | if !ok { 147 | t.Errorf("Actual: %T; Expected: map[string]interface{}", schema) 148 | } 149 | someValue, ok := someMap["name"] 150 | if !ok { 151 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 152 | } 153 | someString, ok := someValue.(string) 154 | if !ok { 155 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 156 | } 157 | return someString 158 | } 159 | -------------------------------------------------------------------------------- /name_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "testing" 23 | ) 24 | 25 | func TestNameEnforcesNameRequirements(t *testing.T) { 26 | n := &name{} 27 | err := nameName("")(n) 28 | checkError(t, err, "not be empty") 29 | 30 | err = nameName("0")(n) 31 | checkError(t, err, "start with [A-Za-z_]") 32 | 33 | err = nameName("_.")(n) 34 | checkError(t, err, "remaining characters contain only [A-Za-z0-9_]") 35 | 36 | err = nameName("_0aZ")(n) 37 | checkError(t, err, nil) 38 | } 39 | 40 | func TestNameAndNamespaceBothSpecified(t *testing.T) { 41 | a, err := newName( 42 | nameName("X"), 43 | nameNamespace("org.foo"), 44 | nameEnclosingNamespace("enclosing.namespace")) 45 | if err != nil { 46 | t.Fatalf("%v", err) 47 | } 48 | expected := &name{n: "org.foo.X"} 49 | if !a.equals(expected) { 50 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 51 | } 52 | } 53 | 54 | func TestNameWithDots(t *testing.T) { 55 | a, err := newName( 56 | nameName("org.foo.X"), 57 | nameNamespace("namespace"), 58 | nameEnclosingNamespace("enclosing.namespace")) 59 | if err != nil { 60 | t.Fatalf("%v", err) 61 | } 62 | expected := &name{n: "org.foo.X"} 63 | if !a.equals(expected) { 64 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 65 | } 66 | } 67 | 68 | func TestNameWithoutDots(t *testing.T) { 69 | a, err := newName(nameName("X"), nameEnclosingNamespace("enclosing.namespace")) 70 | if err != nil { 71 | t.Fatalf("%v", err) 72 | } 73 | expected := &name{n: "enclosing.namespace.X"} 74 | if !a.equals(expected) { 75 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 76 | } 77 | 78 | a, err = newName(nameName("X")) 79 | if err != nil { 80 | t.Fatalf("%v", err) 81 | } 82 | expected = &name{n: "X"} 83 | if !a.equals(expected) { 84 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 85 | } 86 | } 87 | 88 | func TestNamePrimitiveTypesHaveNoDotPrefix(t *testing.T) { 89 | // null 90 | a, err := newName(nameName("null")) 91 | if err != nil { 92 | t.Fatalf("%v", err) 93 | } 94 | expected := &name{n: "null"} 95 | if !a.equals(expected) { 96 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 97 | } 98 | // bool 99 | a, err = newName(nameName("bool")) 100 | if err != nil { 101 | t.Fatalf("%v", err) 102 | } 103 | expected = &name{n: "bool"} 104 | if !a.equals(expected) { 105 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 106 | } 107 | // int 108 | a, err = newName(nameName("int")) 109 | if err != nil { 110 | t.Fatalf("%v", err) 111 | } 112 | expected = &name{n: "int"} 113 | if !a.equals(expected) { 114 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 115 | } 116 | // long 117 | a, err = newName(nameName("long")) 118 | if err != nil { 119 | t.Fatalf("%v", err) 120 | } 121 | expected = &name{n: "long"} 122 | if !a.equals(expected) { 123 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 124 | } 125 | // float 126 | a, err = newName(nameName("float")) 127 | if err != nil { 128 | t.Fatalf("%v", err) 129 | } 130 | expected = &name{n: "float"} 131 | if !a.equals(expected) { 132 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 133 | } 134 | // double 135 | a, err = newName(nameName("double")) 136 | if err != nil { 137 | t.Fatalf("%v", err) 138 | } 139 | expected = &name{n: "double"} 140 | if !a.equals(expected) { 141 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 142 | } 143 | // bytes 144 | a, err = newName(nameName("bytes")) 145 | if err != nil { 146 | t.Fatalf("%v", err) 147 | } 148 | expected = &name{n: "bytes"} 149 | if !a.equals(expected) { 150 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 151 | } 152 | // string 153 | a, err = newName(nameName("string")) 154 | if err != nil { 155 | t.Fatalf("%v", err) 156 | } 157 | expected = &name{n: "string"} 158 | if !a.equals(expected) { 159 | t.Errorf("Actual: %#v; Expected: %#v", a, expected) 160 | } 161 | } 162 | 163 | func TestNameNamespaceWithNamespace(t *testing.T) { 164 | someName := &name{n: "org.foo.X"} 165 | someNamespace := someName.namespace() 166 | if someNamespace != "org.foo" { 167 | t.Errorf("Actual: %#v; Expected: %#v", someNamespace, "org.foo") 168 | } 169 | } 170 | 171 | func TestNameNamespaceWithoutNamespace(t *testing.T) { 172 | someName := &name{n: "X"} 173 | someNamespace := someName.namespace() 174 | if someNamespace != "" { 175 | t.Errorf("Actual: %#v; Expected: %#v", someNamespace, "") 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /encoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "fmt" 23 | "io" 24 | "math" 25 | ) 26 | 27 | // ErrEncoder is returned when the encoder encounters an error. 28 | type ErrEncoder struct { 29 | Message string 30 | Err error 31 | } 32 | 33 | func (e ErrEncoder) Error() string { 34 | if e.Err == nil { 35 | return "cannot encode " + e.Message 36 | } 37 | return "cannot encode " + e.Message + ": " + e.Err.Error() 38 | } 39 | 40 | func newEncoderError(dataType string, a ...interface{}) *ErrEncoder { 41 | var err error 42 | var format, message string 43 | var ok bool 44 | if len(a) == 0 { 45 | return &ErrEncoder{dataType + ": no reason given", nil} 46 | } 47 | // if last item is error: save it 48 | if err, ok = a[len(a)-1].(error); ok { 49 | a = a[:len(a)-1] // pop it 50 | } 51 | // if items left, first ought to be format string 52 | if len(a) > 0 { 53 | if format, ok = a[0].(string); ok { 54 | a = a[1:] // unshift 55 | message = fmt.Sprintf(format, a...) 56 | } 57 | } 58 | if message != "" { 59 | message = ": " + message 60 | } 61 | return &ErrEncoder{dataType + message, err} 62 | } 63 | 64 | func nullEncoder(_ io.Writer, _ interface{}) error { 65 | return nil 66 | } 67 | 68 | func booleanEncoder(w io.Writer, datum interface{}) error { 69 | someBoolean, ok := datum.(bool) 70 | if !ok { 71 | return newEncoderError("boolean", "expected: bool; received: %T", datum) 72 | } 73 | bb := make([]byte, 1) 74 | if someBoolean { 75 | bb[0] = byte(1) 76 | } 77 | if _, err := w.Write(bb); err != nil { 78 | return newEncoderError("boolean", err) 79 | } 80 | return nil 81 | } 82 | 83 | func intEncoder(w io.Writer, datum interface{}) error { 84 | downShift := uint32(31) 85 | someInt, ok := datum.(int32) 86 | if !ok { 87 | return newEncoderError("int", "expected: int32; received: %T", datum) 88 | } 89 | encoded := int64((someInt << 1) ^ (someInt >> downShift)) 90 | bb := make([]byte, 0) 91 | if encoded == 0 { 92 | bb = append(bb, byte(0)) 93 | } else { 94 | for encoded > 0 { 95 | b := byte(encoded & 127) 96 | encoded = encoded >> 7 97 | if !(encoded == 0) { 98 | b |= 128 99 | } 100 | bb = append(bb, b) 101 | } 102 | } 103 | _, err := w.Write(bb) 104 | return err 105 | } 106 | 107 | func longEncoder(w io.Writer, datum interface{}) error { 108 | downShift := uint32(63) 109 | someInt, ok := datum.(int64) 110 | if !ok { 111 | return newEncoderError("long", "expected: int64; received: %T", datum) 112 | } 113 | encoded := int64((someInt << 1) ^ (someInt >> downShift)) 114 | bb := make([]byte, 0) 115 | if encoded == 0 { 116 | bb = append(bb, byte(0)) 117 | } else { 118 | for encoded > 0 { 119 | b := byte(encoded & 127) 120 | encoded = encoded >> 7 121 | if !(encoded == 0) { 122 | b |= 128 123 | } 124 | bb = append(bb, b) 125 | } 126 | } 127 | _, err := w.Write(bb) 128 | return err 129 | } 130 | 131 | func floatEncoder(w io.Writer, datum interface{}) error { 132 | someFloat, ok := datum.(float32) 133 | if !ok { 134 | return newEncoderError("float", "expected: float32; received: %T", datum) 135 | } 136 | bits := uint64(math.Float32bits(someFloat)) 137 | const byteCount = 4 138 | buf := make([]byte, byteCount) 139 | for i := 0; i < byteCount; i++ { 140 | buf[i] = byte(bits & 255) 141 | bits = bits >> 8 142 | } 143 | _, err := w.Write(buf) 144 | return err 145 | } 146 | 147 | func doubleEncoder(w io.Writer, datum interface{}) error { 148 | someFloat, ok := datum.(float64) 149 | if !ok { 150 | return newEncoderError("double", "expected: float64; received: %T", datum) 151 | } 152 | bits := uint64(math.Float64bits(someFloat)) 153 | const byteCount = 8 154 | buf := make([]byte, byteCount) 155 | for i := 0; i < byteCount; i++ { 156 | buf[i] = byte(bits & 255) 157 | bits = bits >> 8 158 | } 159 | _, err := w.Write(buf) 160 | return err 161 | } 162 | 163 | func bytesEncoder(w io.Writer, datum interface{}) error { 164 | someBytes, ok := datum.([]byte) 165 | if !ok { 166 | return newEncoderError("bytes", "expected: []byte; received: %T", datum) 167 | } 168 | err := longEncoder(w, int64(len(someBytes))) 169 | if err != nil { 170 | return newEncoderError("bytes", err) 171 | } 172 | _, err = w.Write(someBytes) 173 | return err 174 | } 175 | 176 | func stringEncoder(w io.Writer, datum interface{}) error { 177 | someString, ok := datum.(string) 178 | if !ok { 179 | return newEncoderError("string", "expected: string; received: %T", datum) 180 | } 181 | err := longEncoder(w, int64(len(someString))) 182 | if err != nil { 183 | return newEncoderError("string", err) 184 | } 185 | _, err = w.Write([]byte(someString)) 186 | return err 187 | } 188 | -------------------------------------------------------------------------------- /decoder.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "encoding/binary" 23 | "fmt" 24 | "io" 25 | "math" 26 | ) 27 | 28 | // ErrDecoder is returned when the encoder encounters an error. 29 | type ErrDecoder struct { 30 | Message string 31 | Err error 32 | } 33 | 34 | func (e ErrDecoder) Error() string { 35 | if e.Err == nil { 36 | return "cannot decode " + e.Message 37 | } 38 | return "cannot decode " + e.Message + ": " + e.Err.Error() 39 | } 40 | 41 | func newDecoderError(dataType string, a ...interface{}) *ErrDecoder { 42 | var err error 43 | var format, message string 44 | var ok bool 45 | if len(a) == 0 { 46 | return &ErrDecoder{dataType + ": no reason given", nil} 47 | } 48 | // if last item is error: save it 49 | if err, ok = a[len(a)-1].(error); ok { 50 | a = a[:len(a)-1] // pop it 51 | } 52 | // if items left, first ought to be format string 53 | if len(a) > 0 { 54 | if format, ok = a[0].(string); ok { 55 | a = a[1:] // unshift 56 | message = fmt.Sprintf(format, a...) 57 | } 58 | } 59 | if message != "" { 60 | message = ": " + message 61 | } 62 | return &ErrDecoder{dataType + message, err} 63 | } 64 | 65 | func nullDecoder(_ io.Reader) (interface{}, error) { 66 | return nil, nil 67 | } 68 | 69 | func booleanDecoder(r io.Reader) (interface{}, error) { 70 | bb := make([]byte, 1) 71 | if _, err := r.Read(bb); err != nil { 72 | return nil, newDecoderError("boolean", err) 73 | } 74 | var datum bool 75 | switch bb[0] { 76 | case byte(0): 77 | // zero value of boolean is false 78 | case byte(1): 79 | datum = true 80 | default: 81 | return nil, newDecoderError("boolean", "expected 1 or 0; received: %d", bb[0]) 82 | } 83 | return datum, nil 84 | } 85 | 86 | func intDecoder(r io.Reader) (interface{}, error) { 87 | var v int 88 | var err error 89 | bb := make([]byte, 1) 90 | for shift := uint(0); ; shift += 7 { 91 | if _, err = r.Read(bb); err != nil { 92 | return nil, newDecoderError("int", err) 93 | } 94 | b := bb[0] 95 | v |= int(b&mask) << shift 96 | if b&flag == 0 { 97 | break 98 | } 99 | } 100 | datum := (int32(v>>1) ^ -int32(v&1)) 101 | return datum, nil 102 | } 103 | 104 | func longDecoder(r io.Reader) (interface{}, error) { 105 | var v int 106 | var err error 107 | bb := make([]byte, 1) 108 | for shift := uint(0); ; shift += 7 { 109 | if _, err = r.Read(bb); err != nil { 110 | return nil, newDecoderError("long", err) 111 | } 112 | b := bb[0] 113 | v |= int(b&mask) << shift 114 | if b&flag == 0 { 115 | break 116 | } 117 | } 118 | datum := (int64(v>>1) ^ -int64(v&1)) 119 | return datum, nil 120 | } 121 | 122 | func floatDecoder(r io.Reader) (interface{}, error) { 123 | buf := make([]byte, 4) 124 | if _, err := r.Read(buf); err != nil { 125 | return nil, newDecoderError("float", err) 126 | } 127 | bits := binary.LittleEndian.Uint32(buf) 128 | datum := math.Float32frombits(bits) 129 | return datum, nil 130 | } 131 | 132 | func doubleDecoder(r io.Reader) (interface{}, error) { 133 | buf := make([]byte, 8) 134 | if _, err := r.Read(buf); err != nil { 135 | return nil, newDecoderError("double", err) 136 | } 137 | datum := math.Float64frombits(binary.LittleEndian.Uint64(buf)) 138 | return datum, nil 139 | } 140 | 141 | func bytesDecoder(r io.Reader) (interface{}, error) { 142 | someValue, err := longDecoder(r) 143 | if err != nil { 144 | return nil, newDecoderError("bytes", err) 145 | } 146 | size, ok := someValue.(int64) 147 | if !ok { 148 | return nil, newDecoderError("bytes", "expected int64; received: %T", someValue) 149 | } 150 | if size < 0 { 151 | return nil, newDecoderError("bytes", "negative length: %d", size) 152 | } 153 | buf := make([]byte, size) 154 | bytesRead, err := r.Read(buf) 155 | if err != nil { 156 | return nil, newDecoderError("bytes", err) 157 | } 158 | if int64(bytesRead) < size { 159 | return nil, newDecoderError("bytes", "buffer underrun") 160 | } 161 | return buf, nil 162 | } 163 | 164 | func stringDecoder(r io.Reader) (interface{}, error) { 165 | // NOTE: could have implemented in terms of makeBytesDecoder, 166 | // but prefer to not have nested error messages 167 | someValue, err := longDecoder(r) 168 | if err != nil { 169 | return nil, newDecoderError("string", err) 170 | } 171 | size, ok := someValue.(int64) 172 | if !ok { 173 | return nil, newDecoderError("string", "expected int64; received: %T", someValue) 174 | } 175 | if size < 0 { 176 | return nil, newDecoderError("string", "negative length: %d", size) 177 | } 178 | buf := make([]byte, size) 179 | byteCount, err := r.Read(buf) 180 | if err != nil { 181 | return nil, newDecoderError("string", err) 182 | } 183 | if int64(byteCount) < size { 184 | return nil, newDecoderError("string", "buffer underrun") 185 | } 186 | return string(buf), nil 187 | } 188 | -------------------------------------------------------------------------------- /record_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "fmt" 23 | "testing" 24 | ) 25 | 26 | func TestRecordRequiresSchema(t *testing.T) { 27 | _, err := NewRecord() 28 | checkErrorFatal(t, err, "cannot build record: no schema defined") 29 | } 30 | 31 | func TestRecordFieldNames(t *testing.T) { 32 | someJSONSchema := `{"type":"record","name":"org.foo.Y","fields":[{"type":"int","name":"X"},{"type":"string","name":"W"}]}` 33 | someRecord, err := NewRecord(RecordSchema(someJSONSchema)) 34 | checkErrorFatal(t, err, nil) 35 | if someRecord.Name != "org.foo.Y" { 36 | t.Errorf("Actual: %#v; Expected: %#v", someRecord.Name, "org.foo.Y") 37 | } 38 | if someRecord.Fields[0].Name != "org.foo.X" { 39 | t.Errorf("Actual: %#v; Expected: %#v", someRecord.Fields[0].Name, "org.foo.X") 40 | } 41 | if someRecord.Fields[1].Name != "org.foo.W" { 42 | t.Errorf("Actual: %#v; Expected: %#v", someRecord.Fields[1].Name, "org.foo.W") 43 | } 44 | } 45 | 46 | func TestRecordFieldBailsWithoutName(t *testing.T) { 47 | schema := make(map[string]interface{}) 48 | 49 | schema["type"] = "int" 50 | _, err := newRecordField(schema) 51 | checkError(t, err, "ought to have name key") 52 | 53 | schema["name"] = 5 54 | _, err = newRecordField(schema) 55 | checkError(t, err, "name ought to be non-empty string") 56 | 57 | schema["name"] = "" 58 | _, err = newRecordField(schema) 59 | checkError(t, err, "name ought to be non-empty string") 60 | } 61 | 62 | func TestRecordFieldChecksSchema(t *testing.T) { 63 | var err error 64 | schema := make(map[string]interface{}) 65 | 66 | schema["name"] = "" 67 | _, err = newRecordField(schema) 68 | checkError(t, err, "name ought to be non-empty string") 69 | 70 | schema["name"] = "someRecordField" 71 | _, err = newRecordField(schema) 72 | checkError(t, err, fmt.Errorf("ought to have type key")) 73 | } 74 | 75 | func TestRecordField(t *testing.T) { 76 | schema := make(map[string]interface{}) 77 | schema["name"] = "someRecordField" 78 | schema["type"] = "int" 79 | schema["doc"] = "contans some integer" 80 | someRecordField, err := newRecordField(schema) 81 | checkError(t, err, nil) 82 | if someRecordField.Name != "someRecordField" { 83 | t.Errorf("Actual: %#v; Expected: %#v", someRecordField.Name, "someRecordField") 84 | } 85 | } 86 | 87 | func TestRecordBailsWithoutName(t *testing.T) { 88 | recordFields := make([]*recordField, 0) 89 | { 90 | schema := make(map[string]interface{}) 91 | schema["name"] = "someRecordField" 92 | schema["type"] = "int" 93 | schema["doc"] = "contans some integer" 94 | someRecordField, err := newRecordField(schema) 95 | checkErrorFatal(t, err, nil) 96 | recordFields = append(recordFields, someRecordField) 97 | } 98 | 99 | schema := make(map[string]interface{}) 100 | schema["fields"] = recordFields 101 | 102 | schema["name"] = 5 103 | _, err := NewRecord(recordSchemaRaw(schema)) 104 | checkErrorFatal(t, err, "ought to be non-empty string") 105 | 106 | schema["name"] = "" 107 | _, err = NewRecord(recordSchemaRaw(schema)) 108 | checkError(t, err, "ought to be non-empty string") 109 | } 110 | 111 | func TestRecordBailsWithoutFields(t *testing.T) { 112 | schema := make(map[string]interface{}) 113 | 114 | schema["name"] = "someRecord" 115 | _, err := NewRecord(recordSchemaRaw(schema)) 116 | checkError(t, err, fmt.Errorf("record requires one or more fields")) 117 | 118 | schema["fields"] = 5 119 | _, err = NewRecord(recordSchemaRaw(schema)) 120 | checkError(t, err, fmt.Errorf("record fields ought to be non-empty array")) 121 | 122 | schema["fields"] = make([]interface{}, 0) 123 | _, err = NewRecord(recordSchemaRaw(schema)) 124 | checkError(t, err, fmt.Errorf("record fields ought to be non-empty array")) 125 | 126 | fields := make([]interface{}, 0) 127 | fields = append(fields, "int") 128 | schema["fields"] = fields 129 | _, err = NewRecord(recordSchemaRaw(schema)) 130 | checkError(t, err, fmt.Errorf("expected: map[string]interface{}; received: string")) 131 | } 132 | 133 | func TestRecordFieldUnion(t *testing.T) { 134 | someJSONSchema := `{"type":"record","name":"Foo","fields":[{"type":["null","string"],"name":"field1"}]}` 135 | _, err := NewRecord(RecordSchema(someJSONSchema)) 136 | checkError(t, err, nil) 137 | } 138 | 139 | func TestRecordGetFieldSchema(t *testing.T) { 140 | outerSchema := ` 141 | { 142 | "type": "record", 143 | "name": "TestRecord", 144 | "fields": [ 145 | { 146 | "name": "value", 147 | "type": "int" 148 | }, 149 | { 150 | "name": "rec", 151 | "type": { 152 | "type": "array", 153 | "items": { 154 | "type": "record", 155 | "name": "TestRecord2", 156 | "fields": [ 157 | { 158 | "name": "stringValue", 159 | "type": "string" 160 | }, 161 | { 162 | "name": "intValue", 163 | "type": "int" 164 | } 165 | ] 166 | } 167 | } 168 | } 169 | ] 170 | } 171 | ` 172 | outerRecord, err := NewRecord(RecordSchema(outerSchema)) 173 | checkErrorFatal(t, err, nil) 174 | // make sure it bails when no such schema 175 | _, err = outerRecord.GetFieldSchema("no-such-field") 176 | checkError(t, err, "no such field: no-such-field") 177 | // get the inner schema 178 | schema, err := outerRecord.GetFieldSchema("rec") 179 | checkErrorFatal(t, err, nil) 180 | _, ok := schema.(map[string]interface{}) 181 | if !ok { 182 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /ocf_reader_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bytes" 23 | "testing" 24 | ) 25 | 26 | func TestNewReaderBailsBadHeader(t *testing.T) { 27 | _, err := NewReader(FromReader(new(bytes.Reader))) 28 | checkError(t, err, "cannot read magic number") 29 | 30 | bits := []byte("BAD\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 31 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 32 | checkError(t, err, "invalid magic number") 33 | 34 | bits = []byte("Obj\x01\x02") 35 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 36 | checkError(t, err, "cannot read header metadata") 37 | 38 | // missing codec 39 | bits = []byte("Obj\x01\x02\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 40 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 41 | checkError(t, err, "header ought to have avro.codec key") 42 | 43 | // unsupported codec 44 | bits = []byte("Obj\x01\x04\x14avro.codec\x16UNSUPPORTED\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 45 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 46 | checkError(t, err, "unsupported codec") 47 | 48 | // missing schema 49 | bits = []byte("Obj\x01\x02\x14avro.codec\x08null\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 50 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 51 | checkError(t, err, "header ought to have avro.schema key") 52 | 53 | // schema that doesn't compile 54 | bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22???\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 55 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 56 | checkError(t, err, "cannot compile schema") 57 | 58 | // missing sync marker 59 | bits = []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00") 60 | _, err = NewReader(FromReader(bytes.NewReader(bits))) 61 | checkError(t, err, "cannot read sync marker") 62 | } 63 | 64 | func TestFileRead(t *testing.T) { 65 | bits := []byte("\x4f\x62\x6a\x01\x04\x16\x61\x76\x72\x6f\x2e\x73\x63\x68\x65\x6d\x61\x96\x05\x7b\x22\x64\x6f\x63\x3a\x22\x3a\x22\x41\x20\x62\x61\x73\x69\x63\x20\x73\x63\x68\x65\x6d\x61\x20\x66\x6f\x72\x20\x73\x74\x6f\x72\x69\x6e\x67\x20\x62\x6c\x6f\x67\x20\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x66\x69\x65\x6c\x64\x73\x22\x3a\x5b\x7b\x22\x64\x6f\x63\x22\x3a\x22\x4e\x61\x6d\x65\x20\x6f\x66\x20\x75\x73\x65\x72\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x75\x73\x65\x72\x6e\x61\x6d\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x54\x68\x65\x20\x63\x6f\x6e\x74\x65\x6e\x74\x20\x6f\x66\x20\x74\x68\x65\x20\x75\x73\x65\x72\x27\x73\x20\x6d\x65\x73\x73\x61\x67\x65\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x73\x74\x72\x69\x6e\x67\x22\x7d\x2c\x7b\x22\x64\x6f\x63\x22\x3a\x22\x55\x6e\x69\x78\x20\x65\x70\x6f\x63\x68\x20\x74\x69\x6d\x65\x20\x69\x6e\x20\x6d\x69\x6c\x6c\x69\x73\x65\x63\x6f\x6e\x64\x73\x22\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x74\x69\x6d\x65\x73\x74\x61\x6d\x70\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x6c\x6f\x6e\x67\x22\x7d\x5d\x2c\x22\x6e\x61\x6d\x65\x22\x3a\x22\x63\x6f\x6d\x6d\x65\x6e\x74\x73\x22\x2c\x22\x6e\x61\x6d\x65\x73\x70\x61\x63\x65\x22\x3a\x22\x63\x6f\x6d\x2e\x65\x78\x61\x6d\x70\x6c\x65\x22\x2c\x22\x74\x79\x70\x65\x22\x3a\x22\x72\x65\x63\x6f\x72\x64\x22\x7d\x14\x61\x76\x72\x6f\x2e\x63\x6f\x64\x65\x63\x08\x6e\x75\x6c\x6c\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x04\xc0\x01\x0e\x41\x71\x75\x61\x6d\x61\x6e\x50\x54\x68\x65\x20\x41\x74\x6c\x61\x6e\x74\x69\x63\x20\x69\x73\x20\x6f\x64\x64\x6c\x79\x20\x63\x6f\x6c\x64\x20\x74\x68\x69\x73\x20\x6d\x6f\x72\x6e\x69\x6e\x67\x21\x88\x88\x88\x88\x08\x0c\x42\x61\x74\x6d\x61\x6e\x3a\x57\x68\x6f\x20\x61\x72\x65\x20\x61\x6c\x6c\x20\x6f\x66\x20\x74\x68\x65\x73\x65\x20\x63\x72\x61\x7a\x69\x65\x73\x3f\x8c\x92\xa1\xd1\x0a\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 66 | bb := bytes.NewBuffer(bits) 67 | fr, err := NewReader(BufferFromReader(bb)) 68 | if err != nil { 69 | panic(err) 70 | } 71 | defer func() { 72 | if err := fr.Close(); err != nil { 73 | panic(err) 74 | } 75 | }() 76 | var count int 77 | for fr.Scan() { 78 | datum, err := fr.Read() 79 | if err != nil { 80 | t.Errorf("Actual: %#v; Expected: %#v", err, nil) 81 | } 82 | _, ok := datum.(*Record) 83 | if !ok { 84 | t.Errorf("Actual: %T; Expected: *Record", datum) 85 | } 86 | count++ 87 | } 88 | if count != 2 { 89 | t.Errorf("Actual: %#v; Expected: %#v", count, 2) 90 | } 91 | } 92 | 93 | func TestReaderScanShouldNotBlock(t *testing.T) { 94 | bits := []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 95 | fr, err := NewReader(FromReader(bytes.NewReader(bits))) 96 | checkErrorFatal(t, err, nil) 97 | if available := fr.Scan(); available { 98 | t.Errorf("Actual: %#v; Expected: %#v", available, false) 99 | } 100 | if err = fr.Close(); err != nil { 101 | t.Errorf("Actual: %#v; Expected: %#v", err, nil) 102 | } 103 | } 104 | 105 | func TestReadBlockCountAndSizeWithNothing(t *testing.T) { 106 | bits := []byte("") 107 | bc, bs, err := readBlockCountAndSize(bytes.NewReader(bits)) 108 | if bc != 0 { 109 | t.Errorf("Actual: %#v; Expected: %#v", bc, 0) 110 | } 111 | if bs != 0 { 112 | t.Errorf("Actual: %#v; Expected: %#v", bs, 0) 113 | } 114 | if err != nil { 115 | t.Errorf("Actual: %#v; Expected: %#v", err, nil) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /ocf_writer_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bytes" 23 | "io" 24 | "testing" 25 | ) 26 | 27 | var defaultSync []byte 28 | 29 | func init() { 30 | defaultSync = []byte("\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a") 31 | } 32 | 33 | func TestNewWriterBailsUnsupportedCodec(t *testing.T) { 34 | var err error 35 | _, err = NewWriter(ToWriter(new(bytes.Buffer)), Compression("")) 36 | checkError(t, err, "unsupported codec") 37 | 38 | _, err = NewWriter(ToWriter(new(bytes.Buffer)), Compression("ficticious test codec name")) 39 | checkError(t, err, "unsupported codec") 40 | } 41 | 42 | func TestNewWriterBailsMissingWriterSchema(t *testing.T) { 43 | var err error 44 | _, err = NewWriter(ToWriter(new(bytes.Buffer))) 45 | checkError(t, err, "missing schema") 46 | 47 | _, err = NewWriter(ToWriter(new(bytes.Buffer)), Compression(CompressionNull)) 48 | checkError(t, err, "missing schema") 49 | 50 | _, err = NewWriter(ToWriter(new(bytes.Buffer)), Compression(CompressionDeflate)) 51 | checkError(t, err, "missing schema") 52 | 53 | _, err = NewWriter(ToWriter(new(bytes.Buffer)), Compression(CompressionSnappy)) 54 | checkError(t, err, "missing schema") 55 | } 56 | 57 | func TestNewWriterBailsInvalidWriterSchema(t *testing.T) { 58 | _, err := NewWriter(WriterSchema("this should not compile")) 59 | checkError(t, err, "cannot parse schema") 60 | } 61 | 62 | func TestNewWriterBailsBadSync(t *testing.T) { 63 | _, err := NewWriter(WriterSchema(`"int"`), Sync(make([]byte, 0))) 64 | checkError(t, err, "sync marker ought to be 16 bytes long") 65 | 66 | _, err = NewWriter(WriterSchema(`"int"`), Sync(make([]byte, syncLength-1))) 67 | checkError(t, err, "sync marker ought to be 16 bytes long") 68 | 69 | _, err = NewWriter(WriterSchema(`"int"`), Sync(make([]byte, syncLength+1))) 70 | checkError(t, err, "sync marker ought to be 16 bytes long") 71 | } 72 | 73 | func TestNewWriterCreatesRandomSync(t *testing.T) { 74 | bb := new(bytes.Buffer) 75 | func(w io.Writer) { 76 | fw, err := NewWriter(ToWriter(w), WriterSchema(`"int"`)) 77 | if err != nil { 78 | t.Fatalf("Actual: %#v; Expected: %#v", err, nil) 79 | } 80 | defer fw.Close() 81 | }(bb) 82 | 83 | notExpected := []byte("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00") 84 | actual := bb.Bytes() 85 | actual = actual[len(actual)-syncLength:] 86 | if bytes.Compare(actual, notExpected) == 0 { 87 | t.Errorf("Actual: %#v; Expected: some non-zero value bits", actual) 88 | } 89 | } 90 | 91 | func TestWriteHeaderCustomSync(t *testing.T) { 92 | bb := new(bytes.Buffer) 93 | func(w io.Writer) { 94 | fw, err := NewWriter(ToWriter(w), WriterSchema(`"int"`), Sync(defaultSync)) 95 | if err != nil { 96 | t.Fatalf("Actual: %#v; Expected: %#v", err, nil) 97 | } 98 | fw.Close() 99 | }(bb) 100 | 101 | // NOTE: because key value pair ordering is indeterminate, 102 | // there are two valid possibilities for the encoded map: 103 | option1 := []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 104 | option2 := []byte("Obj\x01\x04\x16avro.schema\x0a\x22int\x22\x14avro.codec\x08null\x00\x21\x0f\xc7\xbb\x81\x86\x39\xac\x48\xa4\xc6\xaf\xa2\xf1\x58\x1a\x00\x00") 105 | 106 | actual := bb.Bytes() 107 | if (bytes.Compare(actual, option1) != 0) && (bytes.Compare(actual, option2) != 0) { 108 | t.Errorf("Actual: %#v; Expected: %#v", actual, option1) 109 | } 110 | } 111 | 112 | func TestWriteWithNullCodec(t *testing.T) { 113 | bb := new(bytes.Buffer) 114 | func(w io.Writer) { 115 | fw, err := NewWriter(BufferToWriter(w), WriterSchema(`"int"`), Sync(defaultSync)) 116 | if err != nil { 117 | t.Fatalf("Actual: %#v; Expected: %#v", err, nil) 118 | } 119 | defer fw.Close() 120 | fw.Write(int32(13)) 121 | fw.Write(int32(42)) 122 | fw.Write(int32(54)) 123 | fw.Write(int32(99)) 124 | }(bb) 125 | t.Logf("bb: %+v", bb.Bytes()) 126 | 127 | // NOTE: because key value pair ordering is indeterminate, 128 | // there are two valid possibilities for the encoded map: 129 | option1 := []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00" + string(defaultSync) + "\x08\x0a\x1a\x54\x6c\xc6\x01" + string(defaultSync) + "\x00\x00") 130 | option2 := []byte("Obj\x01\x04\x16avro.schema\x0a\x22int\x22\x14avro.codec\x08null\x00" + string(defaultSync) + "\x08\x0a\x1a\x54\x6c\xc6\x01" + string(defaultSync) + "\x00\x00") 131 | 132 | actual := bb.Bytes() 133 | if (bytes.Compare(actual, option1) != 0) && (bytes.Compare(actual, option2) != 0) { 134 | t.Errorf("Actual: %#v; Expected: %#v", actual, option1) 135 | } 136 | } 137 | 138 | func _TestWriteWithDeflateCodec(t *testing.T) { 139 | bb := new(bytes.Buffer) 140 | func(w io.Writer) { 141 | fw, err := NewWriter( 142 | BlockSize(2), 143 | Compression(CompressionDeflate), 144 | WriterSchema(`"int"`), 145 | Sync(defaultSync), 146 | ToWriter(w)) 147 | if err != nil { 148 | t.Fatalf("Actual: %#v; Expected: %#v", err, nil) 149 | } 150 | defer fw.Close() 151 | fw.Write(int32(13)) 152 | fw.Write(int32(42)) 153 | fw.Write(int32(54)) 154 | fw.Write(int32(99)) 155 | }(bb) 156 | 157 | // NOTE: because key value pair ordering is indeterminate, 158 | // there are two valid possibilities for the encoded map: 159 | option1 := []byte("Obj\x01\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00" + string(defaultSync) + "\x08\x0a\x1a\x54\x6c\xc6\x01" + string(defaultSync) + "\x00\x00") 160 | option2 := []byte("Obj\x01\x04\x16avro.schema\x0a\x22int\x22\x14avro.codec\x08null\x00" + string(defaultSync) + "\x08\x0a\x1a\x54\x6c\xc6\x01" + string(defaultSync) + "\x00\x00") 161 | 162 | actual := bb.Bytes() 163 | if (bytes.Compare(actual, option1) != 0) && (bytes.Compare(actual, option2) != 0) { 164 | t.Errorf("Actual: %#v; Expected: %#v", actual, option1) 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /record.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "encoding/json" 23 | "fmt" 24 | "strings" 25 | ) 26 | 27 | // Record is an abstract data type used to hold data corresponding to 28 | // an Avro record. Wherever an Avro schema specifies a record, this 29 | // library's Decode method will return a Record initialized to the 30 | // record's values read from the io.Reader. Likewise, when using 31 | // Encode to convert data to an Avro record, it is necessary to create 32 | // and send a Record instance to the Encoder method. 33 | type Record struct { 34 | Name string 35 | Fields []*recordField 36 | aliases []string 37 | doc string 38 | n *name 39 | ens string 40 | schemaMap map[string]interface{} 41 | } 42 | 43 | // Get returns the datum of the specified Record field. 44 | func (r Record) Get(fieldName string) (interface{}, error) { 45 | // qualify fieldName searches based on record namespace 46 | fn, _ := newName(nameName(fieldName), nameNamespace(r.n.ns)) 47 | 48 | for _, field := range r.Fields { 49 | if field.Name == fn.n { 50 | return field.Datum, nil 51 | } 52 | } 53 | return nil, fmt.Errorf("no such field: %s", fieldName) 54 | } 55 | 56 | // GetFieldSchema returns the schema of the specified Record field. 57 | func (r Record) GetFieldSchema(fieldName string) (interface{}, error) { 58 | // qualify fieldName searches based on record namespace 59 | fn, _ := newName(nameName(fieldName), nameNamespace(r.n.ns)) 60 | 61 | for _, field := range r.Fields { 62 | if field.Name == fn.n { 63 | return field.schema, nil 64 | } 65 | } 66 | return nil, fmt.Errorf("no such field: %s", fieldName) 67 | } 68 | 69 | // Set updates the datum of the specified Record field. 70 | func (r Record) Set(fieldName string, value interface{}) error { 71 | // qualify fieldName searches based on record namespace 72 | fn, _ := newName(nameName(fieldName), nameNamespace(r.n.ns)) 73 | 74 | for _, field := range r.Fields { 75 | if field.Name == fn.n { 76 | field.Datum = value 77 | return nil 78 | } 79 | } 80 | return fmt.Errorf("no such field: %s", fieldName) 81 | } 82 | 83 | // String returns a string representation of the Record. 84 | func (r Record) String() string { 85 | fields := make([]string, len(r.Fields)) 86 | for idx, f := range r.Fields { 87 | fields[idx] = fmt.Sprintf("%v", f) 88 | } 89 | return fmt.Sprintf("{%s: [%v]}", r.Name, strings.Join(fields, ", ")) 90 | } 91 | 92 | // NewRecord will create a Record instance corresponding to the 93 | // specified schema. 94 | // 95 | // func recordExample(codec goavro.Codec, w io.Writer, recordSchema string) error { 96 | // // To encode a Record, you need to instantiate a Record instance 97 | // // that adheres to the schema the Encoder expect. 98 | // someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema)) 99 | // if err != nil { 100 | // return err 101 | // } 102 | // // Once you have a Record, you can set the values of the various fields. 103 | // someRecord.Set("username", "Aquaman") 104 | // someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 105 | // // Feel free to fully qualify the field name if you'd like 106 | // someRecord.Set("com.example.timestamp", int64(1082196484)) 107 | // 108 | // // Once the fields of the Record have the correct data, you can encode it 109 | // err = codec.Encode(w, someRecord) 110 | // return err 111 | // } 112 | func NewRecord(setters ...RecordSetter) (*Record, error) { 113 | record := &Record{n: &name{}} 114 | for _, setter := range setters { 115 | err := setter(record) 116 | if err != nil { 117 | return nil, err 118 | } 119 | } 120 | if record.schemaMap == nil { 121 | return nil, newCodecBuildError("record", "no schema defined") 122 | } 123 | var err error 124 | record.n, err = newName(nameSchema(record.schemaMap), nameEnclosingNamespace(record.ens)) 125 | if err != nil { 126 | return nil, newCodecBuildError("record", err) 127 | } 128 | record.Name = record.n.n 129 | ns := record.n.namespace() 130 | 131 | val, ok := record.schemaMap["fields"] 132 | if !ok { 133 | return nil, newCodecBuildError("record", "record requires one or more fields") 134 | } 135 | fields, ok := val.([]interface{}) 136 | if !ok || len(fields) == 0 { 137 | return nil, newCodecBuildError("record", "record fields ought to be non-empty array") 138 | } 139 | 140 | record.Fields = make([]*recordField, len(fields)) 141 | for i, field := range fields { 142 | rf, err := newRecordField(field, recordFieldEnclosingNamespace(ns)) 143 | if err != nil { 144 | return nil, newCodecBuildError("record", err) 145 | } 146 | record.Fields[i] = rf 147 | } 148 | 149 | // fields optional to the avro spec 150 | 151 | if val, ok = record.schemaMap["doc"]; ok { 152 | record.doc, ok = val.(string) 153 | if !ok { 154 | return nil, newCodecBuildError("record", "doc ought to be string") 155 | } 156 | } 157 | if val, ok = record.schemaMap["aliases"]; ok { 158 | record.aliases, ok = val.([]string) 159 | if !ok { 160 | return nil, newCodecBuildError("record", "aliases ought to be array of strings") 161 | } 162 | } 163 | record.schemaMap = nil 164 | return record, nil 165 | } 166 | 167 | // RecordSetter functions are those those which are used to 168 | // instantiate a new Record. 169 | type RecordSetter func(*Record) error 170 | 171 | // recordSchemaRaw specifies the schema of the record to create. Schema 172 | // must be `map[string]interface{}`. 173 | func recordSchemaRaw(schema interface{}) RecordSetter { 174 | return func(r *Record) error { 175 | var ok bool 176 | r.schemaMap, ok = schema.(map[string]interface{}) 177 | if !ok { 178 | return newCodecBuildError("record", "expected: map[string]interface{}; received: %T", schema) 179 | } 180 | return nil 181 | } 182 | } 183 | 184 | // RecordSchema specifies the schema of the record to 185 | // create. Schema must be a JSON string. 186 | func RecordSchema(recordSchemaJSON string) RecordSetter { 187 | return func(r *Record) error { 188 | var schema interface{} 189 | err := json.Unmarshal([]byte(recordSchemaJSON), &schema) 190 | if err != nil { 191 | return newCodecBuildError("record", err) 192 | } 193 | var ok bool 194 | r.schemaMap, ok = schema.(map[string]interface{}) 195 | if !ok { 196 | return newCodecBuildError("record", "expected: map[string]interface{}; received: %T", schema) 197 | } 198 | return nil 199 | } 200 | } 201 | 202 | // RecordEnclosingNamespace specifies the enclosing namespace of the 203 | // record to create. For instance, if the enclosing namespace is 204 | // `com.example`, and the record name is `Foo`, then the full record 205 | // name will be `com.example.Foo`. 206 | func RecordEnclosingNamespace(someNamespace string) RecordSetter { 207 | return func(r *Record) error { 208 | r.ens = someNamespace 209 | return nil 210 | } 211 | } 212 | 213 | //////////////////////////////////////// 214 | 215 | type recordField struct { 216 | Name string 217 | Datum interface{} 218 | doc string 219 | defval interface{} 220 | hasDefault bool 221 | order string 222 | aliases []string 223 | schema interface{} 224 | ens string 225 | } 226 | 227 | func (rf recordField) String() string { 228 | return fmt.Sprintf("%s: %v", rf.Name, rf.Datum) 229 | } 230 | 231 | type recordFieldSetter func(*recordField) error 232 | 233 | func recordFieldEnclosingNamespace(someNamespace string) recordFieldSetter { 234 | return func(rf *recordField) error { 235 | rf.ens = someNamespace 236 | return nil 237 | } 238 | } 239 | 240 | func newRecordField(schema interface{}, setters ...recordFieldSetter) (*recordField, error) { 241 | schemaMap, ok := schema.(map[string]interface{}) 242 | if !ok { 243 | return nil, newCodecBuildError("record field", "schema expected: map[string]interface{}; received: %T", schema) 244 | } 245 | 246 | rf := &recordField{} 247 | for _, setter := range setters { 248 | err := setter(rf) 249 | if err != nil { 250 | return nil, newCodecBuildError("record field", err) 251 | } 252 | } 253 | 254 | n, err := newName(nameSchema(schemaMap), nameEnclosingNamespace(rf.ens)) 255 | if err != nil { 256 | return nil, newCodecBuildError("record field", err) 257 | } 258 | rf.Name = n.n 259 | 260 | typeName, ok := schemaMap["type"] 261 | if !ok { 262 | return nil, newCodecBuildError("record field", "ought to have type key") 263 | } 264 | rf.schema = schema 265 | 266 | // fields optional to the avro spec 267 | 268 | val, ok := schemaMap["default"] 269 | if ok { 270 | rf.hasDefault = true 271 | switch typeName.(type) { 272 | case string: 273 | switch typeName { 274 | case "int": 275 | dv, ok := val.(float64) 276 | if !ok { 277 | return nil, newCodecBuildError("record field", "default value type mismatch: %s; expected: %s; received: %T", rf.Name, "int32", val) 278 | } 279 | rf.defval = int32(dv) 280 | case "long": 281 | dv, ok := val.(float64) 282 | if !ok { 283 | return nil, newCodecBuildError("record field", "default value type mismatch: %s; expected: %s; received: %T", rf.Name, "int64", val) 284 | } 285 | rf.defval = int64(dv) 286 | case "float": 287 | dv, ok := val.(float64) 288 | if !ok { 289 | return nil, newCodecBuildError("record field", "default value type mismatch: %s; expected: %s; received: %T", rf.Name, "float32", val) 290 | } 291 | rf.defval = float32(dv) 292 | case "bytes": 293 | dv, ok := val.(string) 294 | if !ok { 295 | return nil, newCodecBuildError("record field", "default value type mismatch: %s; expected: %s; received: %T", rf.Name, "string", val) 296 | } 297 | rf.defval = []byte(dv) 298 | default: 299 | rf.defval = val 300 | } 301 | default: 302 | rf.defval = val 303 | } 304 | } 305 | 306 | if val, ok = schemaMap["doc"]; ok { 307 | rf.doc, ok = val.(string) 308 | if !ok { 309 | return nil, newCodecBuildError("record field", "record field doc ought to be string") 310 | } 311 | } 312 | 313 | if val, ok = schemaMap["order"]; ok { 314 | rf.order, ok = val.(string) 315 | if !ok { 316 | return nil, newCodecBuildError("record field", "record field order ought to be string") 317 | } 318 | switch rf.order { 319 | case "ascending", "descending", "ignore": 320 | // ok 321 | default: 322 | return nil, newCodecBuildError("record field", "record field order ought to bescending, descending, or ignore") 323 | } 324 | } 325 | 326 | if val, ok = schemaMap["aliases"]; ok { 327 | rf.aliases, ok = val.([]string) 328 | if !ok { 329 | return nil, newCodecBuildError("record field", "record field aliases ought to be array of strings") 330 | } 331 | } 332 | 333 | return rf, nil 334 | } 335 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /ocf_reader.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "code.google.com/p/snappy-go/snappy" 25 | "compress/flate" 26 | "fmt" 27 | "io" 28 | "io/ioutil" 29 | ) 30 | 31 | // ErrReaderInit is returned when the encoder encounters an error. 32 | type ErrReaderInit struct { 33 | Message string 34 | Err error 35 | } 36 | 37 | func (e ErrReaderInit) Error() string { 38 | if e.Err == nil { 39 | return "cannot build " + e.Message 40 | } 41 | return "cannot build " + e.Message + ": " + e.Err.Error() 42 | } 43 | 44 | func newReaderInitError(a ...interface{}) *ErrReaderInit { 45 | var err error 46 | var format, message string 47 | var ok bool 48 | if len(a) == 0 { 49 | return &ErrReaderInit{"cannot create reader: no reason given", nil} 50 | } 51 | // if last item is error: save it 52 | if err, ok = a[len(a)-1].(error); ok { 53 | a = a[:len(a)-1] // pop it 54 | } 55 | // if items left, first ought to be format string 56 | if len(a) > 0 { 57 | if format, ok = a[0].(string); ok { 58 | a = a[1:] // unshift 59 | message = fmt.Sprintf(format, a...) 60 | } 61 | } 62 | return &ErrReaderInit{message, err} 63 | } 64 | 65 | // ErrReaderBlockCount is returned when a reader detects an error 66 | // while attempting to read the block count and block size. 67 | type ErrReaderBlockCount struct { 68 | Err error 69 | } 70 | 71 | func (e *ErrReaderBlockCount) Error() string { 72 | return "cannot read block count and size: " + e.Err.Error() 73 | } 74 | 75 | // ReaderSetter functions are those those which are used to instantiate 76 | // a new Reader. 77 | type ReaderSetter func(*Reader) error 78 | 79 | // BufferFromReader wraps the specified `io.Reader` using a 80 | // `bufio.Reader` to read from a file. 81 | func BufferFromReader(r io.Reader) ReaderSetter { 82 | return func(fr *Reader) error { 83 | fr.r = bufio.NewReader(r) 84 | return nil 85 | } 86 | } 87 | 88 | // FromReader specifies the `io.Reader` to use when reading a file. 89 | func FromReader(r io.Reader) ReaderSetter { 90 | return func(fr *Reader) error { 91 | fr.r = r 92 | return nil 93 | } 94 | } 95 | 96 | // Reader structure contains data necessary to read Avro files. 97 | type Reader struct { 98 | CompressionCodec string 99 | DataSchema string 100 | Sync []byte 101 | dataCodec Codec 102 | datum Datum 103 | deblocked chan Datum 104 | done bool 105 | err error 106 | r io.Reader 107 | } 108 | 109 | // NewReader returns a object to read data from an io.Reader using the 110 | // Avro Object Container Files format. 111 | // 112 | // func main() { 113 | // conn, err := net.Dial("tcp", "127.0.0.1:8080") 114 | // if err != nil { 115 | // log.Fatal(err) 116 | // } 117 | // fr, err := goavro.NewReader(goavro.FromReader(conn)) 118 | // if err != nil { 119 | // log.Fatal("cannot create Reader: ", err) 120 | // } 121 | // defer func() { 122 | // if err := fr.Close(); err != nil { 123 | // log.Fatal(err) 124 | // } 125 | // }() 126 | // 127 | // for fr.Scan() { 128 | // datum, err := fr.Read() 129 | // if err != nil { 130 | // log.Println("cannot read datum: ", err) 131 | // continue 132 | // } 133 | // fmt.Println("RECORD: ", datum) 134 | // } 135 | // } 136 | func NewReader(setters ...ReaderSetter) (*Reader, error) { 137 | var err error 138 | fr := &Reader{} 139 | for _, setter := range setters { 140 | err = setter(fr) 141 | if err != nil { 142 | return nil, newReaderInitError(err) 143 | } 144 | } 145 | if fr.r == nil { 146 | return nil, newReaderInitError("must specify io.Reader") 147 | } 148 | // read in header information and use it to initialize Reader 149 | magic := make([]byte, 4) 150 | _, err = fr.r.Read(magic) 151 | if err != nil { 152 | return nil, newReaderInitError("cannot read magic number", err) 153 | } 154 | if bytes.Compare(magic, []byte(magicBytes)) != 0 { 155 | return nil, &ErrReaderInit{Message: "invalid magic number: " + string(magic)} 156 | } 157 | meta, err := decodeHeaderMetadata(fr.r) 158 | if err != nil { 159 | return nil, newReaderInitError("cannot read header metadata", err) 160 | } 161 | fr.CompressionCodec, err = getHeaderString("avro.codec", meta) 162 | if err != nil { 163 | return nil, newReaderInitError("cannot read header metadata", err) 164 | } 165 | if !IsCompressionCodecSupported(fr.CompressionCodec) { 166 | return nil, newReaderInitError("unsupported codec: %s", fr.CompressionCodec) 167 | } 168 | fr.DataSchema, err = getHeaderString("avro.schema", meta) 169 | if err != nil { 170 | return nil, newReaderInitError("cannot read header metadata", err) 171 | } 172 | if fr.dataCodec, err = NewCodec(fr.DataSchema); err != nil { 173 | return nil, newReaderInitError("cannot compile schema", err) 174 | } 175 | fr.Sync = make([]byte, syncLength) 176 | if _, err = fr.r.Read(fr.Sync); err != nil { 177 | return nil, newReaderInitError("cannot read sync marker", err) 178 | } 179 | // setup reading pipeline 180 | toDecompress := make(chan *readerBlock) 181 | toDecode := make(chan *readerBlock) 182 | fr.deblocked = make(chan Datum) 183 | go read(fr, toDecompress) 184 | go decompress(fr, toDecompress, toDecode) 185 | go decode(fr, toDecode) 186 | return fr, nil 187 | } 188 | 189 | // Close releases resources and returns any Reader errors. 190 | func (fr *Reader) Close() error { 191 | return fr.err 192 | } 193 | 194 | // Scan returns true if more data is ready to be read. 195 | func (fr *Reader) Scan() bool { 196 | fr.datum = <-fr.deblocked 197 | return !fr.done 198 | } 199 | 200 | // Read returns the next element from the Reader. 201 | func (fr *Reader) Read() (interface{}, error) { 202 | return fr.datum.Value, fr.datum.Err 203 | } 204 | 205 | func decodeHeaderMetadata(r io.Reader) (map[string]interface{}, error) { 206 | md, err := metadataCodec.Decode(r) 207 | if err != nil { 208 | return nil, err 209 | } 210 | return md.(map[string]interface{}), nil 211 | } 212 | 213 | func getHeaderString(someKey string, header map[string]interface{}) (string, error) { 214 | v, ok := header[someKey] 215 | if !ok { 216 | return "", fmt.Errorf("header ought to have %v key", someKey) 217 | } 218 | return string(v.([]byte)), nil 219 | } 220 | 221 | type readerBlock struct { 222 | datumCount int 223 | err error 224 | r io.Reader 225 | } 226 | 227 | // ErrReader is returned when the reader encounters an error. 228 | type ErrReader struct { 229 | Message string 230 | Err error 231 | } 232 | 233 | func (e ErrReader) Error() string { 234 | if e.Err == nil { 235 | return "cannot read from reader: " + e.Message 236 | } 237 | return "cannot read from reader: " + e.Message + ": " + e.Err.Error() 238 | } 239 | 240 | func newReaderError(a ...interface{}) *ErrReader { 241 | var err error 242 | var format, message string 243 | var ok bool 244 | if len(a) == 0 { 245 | return &ErrReader{"no reason given", nil} 246 | } 247 | // if last item is error: save it 248 | if err, ok = a[len(a)-1].(error); ok { 249 | a = a[:len(a)-1] // pop it 250 | } 251 | // if items left, first ought to be format string 252 | if len(a) > 0 { 253 | if format, ok = a[0].(string); ok { 254 | a = a[1:] // unshift 255 | message = fmt.Sprintf(format, a...) 256 | } 257 | } 258 | return &ErrReader{message, err} 259 | } 260 | 261 | func read(fr *Reader, toDecompress chan<- *readerBlock) { 262 | // NOTE: these variables created outside loop to reduce churn 263 | var lr io.Reader 264 | var bits []byte 265 | sync := make([]byte, syncLength) 266 | 267 | blockCount, blockSize, err := readBlockCountAndSize(fr.r) 268 | if err != nil { 269 | blockCount = 0 270 | } 271 | for blockCount != 0 { 272 | lr = io.LimitReader(fr.r, int64(blockSize)) 273 | if bits, err = ioutil.ReadAll(lr); err != nil { 274 | err = newReaderError("cannot read block", err) 275 | break 276 | } 277 | toDecompress <- &readerBlock{datumCount: blockCount, r: bytes.NewReader(bits)} 278 | if _, err = fr.r.Read(sync); err != nil { 279 | err = newReaderError("cannot read sync marker", err) 280 | break 281 | } 282 | if bytes.Compare(fr.Sync, sync) != 0 { 283 | err = newReaderError(fmt.Sprintf("sync marker mismatch: %#v != %#v", sync, fr.Sync)) 284 | break 285 | } 286 | if blockCount, blockSize, err = readBlockCountAndSize(fr.r); err != nil { 287 | break 288 | } 289 | } 290 | if err != nil { 291 | fr.err = err 292 | } 293 | close(toDecompress) 294 | } 295 | 296 | func readBlockCountAndSize(r io.Reader) (blockCount, blockSize int, err error) { 297 | bc, err := longCodec.Decode(r) 298 | if err != nil { 299 | if ed, ok := err.(*ErrDecoder); ok && ed.Err.Error() == "EOF" { 300 | return 0, 0, nil // we're done 301 | } 302 | return 0, 0, &ErrReaderBlockCount{err} 303 | } 304 | bs, err := longCodec.Decode(r) 305 | if err != nil { 306 | return 0, 0, &ErrReaderBlockCount{err} 307 | } 308 | return int(bc.(int64)), int(bs.(int64)), nil 309 | } 310 | 311 | func decompress(fr *Reader, toDecompress <-chan *readerBlock, toDecode chan<- *readerBlock) { 312 | switch fr.CompressionCodec { 313 | case CompressionDeflate: 314 | var rc io.ReadCloser 315 | var bits []byte 316 | for block := range toDecompress { 317 | rc = flate.NewReader(block.r) 318 | bits, block.err = ioutil.ReadAll(rc) 319 | if block.err != nil { 320 | block.err = newReaderError("cannot read from deflate", block.err) 321 | toDecode <- block 322 | _ = rc.Close() // already have the read error; ignore the close error 323 | continue 324 | } 325 | block.err = rc.Close() 326 | if block.err != nil { 327 | block.err = newReaderError("cannot close deflate", block.err) 328 | toDecode <- block 329 | continue 330 | } 331 | block.r = bytes.NewReader(bits) 332 | toDecode <- block 333 | } 334 | case CompressionNull: 335 | for block := range toDecompress { 336 | toDecode <- block 337 | } 338 | case CompressionSnappy: 339 | var src, dst []byte 340 | for block := range toDecompress { 341 | src, block.err = ioutil.ReadAll(block.r) 342 | if block.err != nil { 343 | block.err = newReaderError("cannot read", block.err) 344 | toDecode <- block 345 | continue 346 | } 347 | dst, block.err = snappy.Decode(dst, src) 348 | if block.err != nil { 349 | block.err = newReaderError("cannot decompress", block.err) 350 | toDecode <- block 351 | continue 352 | } 353 | block.r = bytes.NewReader(dst) 354 | toDecode <- block 355 | } 356 | } 357 | close(toDecode) 358 | } 359 | 360 | func decode(fr *Reader, toDecode <-chan *readerBlock) { 361 | decodeLoop: 362 | for block := range toDecode { 363 | for i := 0; i < block.datumCount; i++ { 364 | var datum Datum 365 | datum.Value, datum.Err = fr.dataCodec.Decode(block.r) 366 | if datum.Value == nil && datum.Err == nil { 367 | break decodeLoop 368 | } 369 | fr.deblocked <- datum 370 | } 371 | } 372 | fr.done = true 373 | close(fr.deblocked) 374 | } 375 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # goavro 2 | 3 | ## Description 4 | 5 | Goavro is a golang library that implements encoding and decoding of 6 | Avro data. It provides an interface to encode data directly to 7 | `io.Writer` streams, and decoding data from `io.Reader` 8 | streams. Goavro fully adheres to 9 | [version 1.7.7 of the Avro specification](http://avro.apache.org/docs/1.7.7/spec.html). 10 | 11 | ## Resources 12 | 13 | * [Avro CLI Examples](https://github.com/miguno/avro-cli-examples) 14 | * [Avro](http://avro.apache.org/) 15 | * [Google Snappy](https://code.google.com/p/snappy/) 16 | * [JavaScript Object Notation, JSON](http://www.json.org/) 17 | * [Kafka](http://kafka.apache.org) 18 | 19 | ## Usage 20 | 21 | Documentation is available via 22 | [![GoDoc](https://godoc.org/github.com/linkedin/goavro?status.svg)](https://godoc.org/github.com/linkedin/goavro). 23 | 24 | Please see the example programs in the `examples` directory for 25 | reference. 26 | 27 | Although the Avro specification defines the terms reader and writer as 28 | library components which read and write Avro data, Go has particular 29 | strong emphasis on what a Reader and Writer are. Namely, it is bad 30 | form to define an interface which shares the same name but uses a 31 | different method signature. In other words, all Reader interfaces 32 | should effectively mirror an `io.Reader`, and all Writer interfaces 33 | should mirror an `io.Writer`. Adherance to this standard is essential 34 | to keep libraries easy to use. 35 | 36 | An `io.Reader` reads data from the stream specified at object creation 37 | time into the parameterized slice of bytes and returns both the number 38 | of bytes read and an error. An Avro reader also reads from a stream, 39 | but it is possible to create an Avro reader that can read from one 40 | stream, then read from another, using the same compiled schema. In 41 | other words, an Avro reader puts the schema first, whereas an 42 | `io.Reader` puts the stream first. 43 | 44 | To support an Avro reader being able to read from multiple streams, 45 | it's API must be different and incompatible with `io.Reader` interface 46 | from the Go standard. Instead, an Avro reader looks more like the 47 | Unmarshal functionality provided by the Go `encoding/json` library. 48 | 49 | ### Codec interface 50 | 51 | Creating a `goavro.Codec` is fast, but ought to be performed exactly 52 | once per Avro schema to process. Once a `Codec` is created, it may be 53 | used multiple times to either decode or encode data. 54 | 55 | The `Codec` interface exposes two methods, one to encode data and one 56 | to decode data. They encode directly into an `io.Writer`, and decode 57 | directly from an `io.Reader`. 58 | 59 | A particular `Codec` can work with only one Avro schema. However, 60 | there is no practical limit to how many `Codec`s may be created and 61 | used in a program. Internally a `goavro.codec` is merely a namespace 62 | and two function pointers to decode and encode data. Because `codec`s 63 | maintain no state, the same `Codec` can be concurrently used on 64 | different `io` streams as desired. 65 | 66 | ```Go 67 | func (c *codec) Decode(r io.Reader) (interface{}, error) 68 | func (c *codec) Encode(w io.Writer, datum interface{}) error 69 | ``` 70 | 71 | #### Creating a `Codec` 72 | 73 | The below is an example of creating a `Codec` from a provided JSON 74 | schema. `Codec`s do not maintain any internal state, and may be used 75 | multiple times on multiple `io.Reader`s, `io.Writer`s, concurrently if 76 | desired. 77 | 78 | ```Go 79 | someRecordSchemaJson := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string","default":"happy"}]}` 80 | codec, err := goavro.NewCodec(someRecordSchemaJson) 81 | if err != nil { 82 | return nil, err 83 | } 84 | ``` 85 | 86 | #### Decoding data 87 | 88 | The below is a simplified example of decoding binary data to be read 89 | from an `io.Reader` into a single datum using a previously compiled 90 | `Codec`. The `Decode` method of the `Codec` interface may be called 91 | multiple times, each time on the same or on different `io.Reader` 92 | objects. 93 | 94 | ```Go 95 | // uses codec created above, and an io.Reader, definition not shown 96 | datum, err := codec.Decode(r) 97 | if err != nil { 98 | return nil, err 99 | } 100 | ``` 101 | 102 | #### Encoding data 103 | 104 | The below is a simplified example of encoding a single datum into the 105 | Avro binary format using a previously compiled `Codec`. The `Encode` 106 | method of the `Codec` interface may be called multiple times, each 107 | time on the same or on different `io.Writer` objects. 108 | 109 | ```Go 110 | // uses codec created above, an io.Writer, definition not shown, 111 | // and some data 112 | err := codec.Encode(w, datum) 113 | if err != nil { 114 | return nil, err 115 | } 116 | ``` 117 | 118 | Another example, this time leveraging `bufio.Writer`: 119 | 120 | ```Go 121 | // Encoding data using bufio.Writer to buffer the writes 122 | // during data encoding: 123 | 124 | func encodeWithBufferedWriter(c Codec, w io.Writer, datum interface{}) error { 125 | bw := bufio.NewWriter(w) 126 | err := c.Encode(bw, datum) 127 | if err != nil { 128 | return err 129 | } 130 | return bw.Flush() 131 | } 132 | 133 | err := encodeWithBufferedWriter(codec, w, datum) 134 | if err != nil { 135 | return nil, err 136 | } 137 | ``` 138 | 139 | ### Reader and Writer helper types 140 | 141 | The `Codec` interface provides means to encode and decode any Avro 142 | data, but a number of additional helper types are provided to handle 143 | streaming of Avro data. 144 | 145 | See the example programs `examples/file/reader.go` and 146 | `examples/file/writer.go` for more context: 147 | 148 | This example wraps the provided `io.Reader` in a `bufio.Reader` and 149 | dumps the data to standard output. 150 | 151 | ```Go 152 | func dumpReader(r io.Reader) { 153 | fr, err := goavro.NewReader(goavro.BufferFromReader(r)) 154 | if err != nil { 155 | log.Fatal("cannot create Reader: ", err) 156 | } 157 | defer func() { 158 | if err := fr.Close(); err != nil { 159 | log.Fatal(err) 160 | } 161 | }() 162 | 163 | for fr.Scan() { 164 | datum, err := fr.Read() 165 | if err != nil { 166 | log.Println("cannot read datum: ", err) 167 | continue 168 | } 169 | fmt.Println(datum) 170 | } 171 | } 172 | ``` 173 | 174 | This example buffers the provided `io.Writer` in a `bufio.Writer`, and 175 | writes some data to the stream. 176 | 177 | ```Go 178 | func makeSomeData(w io.Writer) error { 179 | recordSchema := ` 180 | { 181 | "type": "record", 182 | "name": "example", 183 | "fields": [ 184 | { 185 | "type": "string", 186 | "name": "username" 187 | }, 188 | { 189 | "type": "string", 190 | "name": "comment" 191 | }, 192 | { 193 | "type": "long", 194 | "name": "timestamp" 195 | } 196 | ] 197 | } 198 | ` 199 | fw, err := goavro.NewWriter( 200 | goavro.BlockSize(13), // example; default is 10 201 | goavro.Compression(goavro.CompressionSnappy), // default is CompressionNull 202 | goavro.WriterSchema(recordSchema), 203 | goavro.ToWriter(w)) 204 | if err != nil { 205 | log.Fatal("cannot create Writer: ", err) 206 | } 207 | defer fw.Close() 208 | 209 | // make a record instance using the same schema 210 | someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema)) 211 | if err != nil { 212 | log.Fatal(err) 213 | } 214 | // identify field name to set datum for 215 | someRecord.Set("username", "Aquaman") 216 | someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 217 | // you can fully qualify the field name 218 | someRecord.Set("com.example.timestamp", int64(1082196484)) 219 | fw.Write(someRecord) 220 | 221 | // make another record 222 | someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)) 223 | if err != nil { 224 | log.Fatal(err) 225 | } 226 | someRecord.Set("username", "Batman") 227 | someRecord.Set("comment", "Who are all of these crazies?") 228 | someRecord.Set("com.example.timestamp", int64(1427383430)) 229 | fw.Write(someRecord) 230 | } 231 | ``` 232 | 233 | ## Limitations 234 | 235 | Goavro is a fully featured encoder and decoder of binary Avro data. It 236 | fully supports recursive data structures, unions, and namespacing. It 237 | does have a few limitations that have yet to be implemented. 238 | 239 | ### Aliases 240 | 241 | The Avro specification allows an implementation to optionally map a 242 | writer's schema to a reader's schema using aliases. Although goavro 243 | can complile schemas with aliases, it does not yet implement this 244 | feature. 245 | 246 | ### JSON Encoding 247 | 248 | The Avro Data Serialization format describes two encodings: binary and 249 | JSON. Goavro only implements binary encoding of data streams, because 250 | that is what most applications need. 251 | 252 | > Most applications will use the binary encoding, as it is smaller and 253 | > faster. But, for debugging and web-based applications, the JSON 254 | > encoding may sometimes be appropriate. 255 | 256 | Note that data schemas are always encoded using JSON, as per the 257 | specification. 258 | 259 | ### Kafka Streams 260 | 261 | [Kakfa](http://kafka.apache.org) is the reason goavro was 262 | written. Similar to Avro Object Container Files being a layer of 263 | abstraction above Avro Data Serialization format, Kafka's use of Avro 264 | is a layer of abstraction that also sits above Avro Data Serialization 265 | format, but has its own schema. Like Avro Object Container Files, this 266 | has been implemented but removed until the API can be improved. 267 | 268 | ## License 269 | 270 | ### Goavro license 271 | 272 | Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 273 | Version 2.0 (the "License"); you may not use this file except in 274 | compliance with the License.
 You may obtain a copy of the License at 275 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). 276 | 277 | Unless required by applicable law or agreed to in writing, software 278 | 
distributed under the License is distributed on an "AS IS" BASIS, 279 | 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 280 | implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 281 | License, Version 2.0 (the "License"); you may not use this file except 282 | in compliance with the License.
 You may obtain a copy of the License 283 | at 284 | [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0). 285 | 286 | Unless required by applicable law or agreed to in writing, software 287 | 
distributed under the License is distributed on an "AS IS" BASIS, 288 | 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 289 | implied. 290 | 291 | ### Google Snappy license 292 | 293 | Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. 294 | 295 | Redistribution and use in source and binary forms, with or without 296 | modification, are permitted provided that the following conditions are 297 | met: 298 | 299 | * Redistributions of source code must retain the above copyright 300 | notice, this list of conditions and the following disclaimer. 301 | * Redistributions in binary form must reproduce the above 302 | copyright notice, this list of conditions and the following disclaimer 303 | in the documentation and/or other materials provided with the 304 | distribution. 305 | * Neither the name of Google Inc. nor the names of its 306 | contributors may be used to endorse or promote products derived from 307 | this software without specific prior written permission. 308 | 309 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 310 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 311 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 312 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 313 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 314 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 315 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 316 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 317 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 318 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 319 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 320 | 321 | ## Third Party Dependencies 322 | 323 | ### Google Snappy 324 | 325 | Goavro links with [Google Snappy](https://code.google.com/p/snappy/) 326 | to provide Snappy compression and decompression support. 327 | -------------------------------------------------------------------------------- /ocf_writer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "code.google.com/p/snappy-go/snappy" 25 | "compress/flate" 26 | "fmt" 27 | "io" 28 | "log" 29 | "math/rand" 30 | "time" 31 | ) 32 | 33 | // DefaultWriterBlockSizeo specifies the default number of datum items 34 | // in a block when writing. 35 | const DefaultWriterBlockSize = 10 36 | 37 | // ErrWriterInit is returned when an error is created during Writer 38 | // initialization. 39 | type ErrWriterInit struct { 40 | Message string 41 | Err error 42 | } 43 | 44 | // Error converts the error instance to a string. 45 | func (e *ErrWriterInit) Error() string { 46 | if e.Err == nil { 47 | return "cannot create Writer: " + e.Message 48 | } else if e.Message == "" { 49 | return "cannot create Writer: " + e.Err.Error() 50 | } else { 51 | return "cannot create Writer: " + e.Message + "; " + e.Err.Error() 52 | } 53 | } 54 | 55 | // WriterSetter functions are those those which are used to 56 | // instantiate a new Writer. 57 | type WriterSetter func(*Writer) error 58 | 59 | // BlockSize specifies the default number of data items to be grouped 60 | // in a block, compressed, and written to the stream. 61 | // 62 | // It is a valid use case to set both BlockTick and BlockSize. For 63 | // example, if BlockTick is set to time.Minute and BlockSize is set to 64 | // 20, but only 13 items are written to the Writer in a minute, those 65 | // 13 items will be grouped in a block, compressed, and written to the 66 | // stream without waiting for the addition 7 items to complete the 67 | // BlockSize. 68 | // 69 | // By default, BlockSize is set to DefaultWriterBlockSize. 70 | func BlockSize(blockSize int64) WriterSetter { 71 | return func(fw *Writer) error { 72 | if blockSize <= 0 { 73 | return fmt.Errorf("BlockSize must be larger than 0: %d", blockSize) 74 | } 75 | fw.blockSize = blockSize 76 | return nil 77 | } 78 | } 79 | 80 | // BlockTick specifies the duration of time between when the Writer 81 | // will flush the blocks to the stream. 82 | // 83 | // It is a valid use case to set both BlockTick and BlockSize. For 84 | // example, if BlockTick is set to time.Minute and BlockSize is set to 85 | // 20, but only 13 items are written to the Writer in a minute, those 86 | // 13 items will be grouped in a block, compressed, and written to the 87 | // stream without waiting for the addition 7 items to complete the 88 | // BlockSize. 89 | // 90 | // By default, BlockTick is set to 0 and is ignored. This causes the 91 | // blocker to fill up its internal queue of data to BlockSize items 92 | // before flushing them to the stream. 93 | func BlockTick(blockTick time.Duration) WriterSetter { 94 | return func(fw *Writer) error { 95 | if blockTick < 0 { 96 | return fmt.Errorf("BlockTick must be non-negative time duration: %v", blockTick) 97 | } 98 | fw.blockTick = blockTick 99 | return nil 100 | } 101 | } 102 | 103 | // BufferToWriter specifies which io.Writer is the target of the 104 | // Writer stream, and creates a bufio.Writer around that io.Writer. It 105 | // is invalid to specify both BufferToWriter and ToWriter. Exactly one 106 | // of these must be called for a given Writer initialization. 107 | func BufferToWriter(w io.Writer) WriterSetter { 108 | return func(fw *Writer) error { 109 | fw.w = bufio.NewWriter(w) 110 | fw.buffered = true 111 | return nil 112 | } 113 | } 114 | 115 | // Compression is used to set the compression codec of 116 | // a new Writer instance. 117 | func Compression(someCompressionCodec string) WriterSetter { 118 | return func(fw *Writer) error { 119 | fw.CompressionCodec = someCompressionCodec 120 | return nil 121 | } 122 | } 123 | 124 | // Sync is used to set the sync marker bytes of a new 125 | // instance. It checks to ensure the byte slice is 16 bytes long, but 126 | // does not check that it has been set to something other than the 127 | // zero value. Usually you can elide the `Sync` call and allow it 128 | // to create a random byte sequence. 129 | func Sync(someSync []byte) WriterSetter { 130 | return func(fw *Writer) error { 131 | if syncLength != len(someSync) { 132 | return fmt.Errorf("sync marker ought to be %d bytes long: %d", syncLength, len(someSync)) 133 | } 134 | fw.Sync = make([]byte, syncLength) 135 | copy(fw.Sync, someSync) 136 | return nil 137 | } 138 | } 139 | 140 | // ToWriter specifies which io.Writer is the target of the Writer 141 | // stream. It is invalid to specify both BufferToWriter and 142 | // ToWriter. Exactly one of these must be called for a given Writer 143 | // initialization. 144 | func ToWriter(w io.Writer) WriterSetter { 145 | return func(fw *Writer) error { 146 | fw.w = w 147 | return nil 148 | } 149 | } 150 | 151 | // UseCodec specifies that a Writer should reuse an existing Codec 152 | // rather than creating a new one, needlessly recompling the same 153 | // schema. 154 | func UseCodec(codec Codec) WriterSetter { 155 | return func(fw *Writer) error { 156 | if codec != nil { 157 | fw.dataCodec = codec 158 | return nil 159 | } 160 | return fmt.Errorf("invalid Codec") 161 | } 162 | } 163 | 164 | // WriterSchema is used to set the Avro schema of a new instance. If a 165 | // codec has already been compiled for the schema, it is faster to use 166 | // the UseCodec method instead of WriterSchema. 167 | func WriterSchema(someSchema string) WriterSetter { 168 | return func(fw *Writer) (err error) { 169 | if fw.dataCodec, err = NewCodec(someSchema); err != nil { 170 | return 171 | } 172 | return 173 | } 174 | } 175 | 176 | // Writer structure contains data necessary to write Avro files. 177 | type Writer struct { 178 | CompressionCodec string 179 | Sync []byte 180 | blockSize int64 181 | buffered bool 182 | dataCodec Codec 183 | err error 184 | toBlock chan interface{} 185 | w io.Writer 186 | writerDone chan struct{} 187 | blockTick time.Duration 188 | } 189 | 190 | // NewWriter returns a object to write data to an io.Writer using the 191 | // Avro Object Container Files format. 192 | // 193 | // func serveClient(conn net.Conn, codec goavro.Codec) { 194 | // fw, err := goavro.NewWriter( 195 | // goavro.BlockSize(100), // flush data every 100 items 196 | // goavro.BlockTick(10 * time.Second), // but at least every 10 seconds 197 | // goavro.Compression(goavro.CompressionSnappy), 198 | // goavro.ToWriter(conn), 199 | // goavro.UseCodec(codec)) 200 | // if err != nil { 201 | // log.Fatal("cannot create Writer: ", err) 202 | // } 203 | // defer fw.Close() 204 | // 205 | // // create a record that matches the schema we want to encode 206 | // someRecord, err := goavro.NewRecord(goavro.RecordSchema(recordSchema)) 207 | // if err != nil { 208 | // log.Fatal(err) 209 | // } 210 | // // identify field name to set datum for 211 | // someRecord.Set("username", "Aquaman") 212 | // someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 213 | // // you can fully qualify the field name 214 | // someRecord.Set("com.example.timestamp", int64(1082196484)) 215 | // fw.Write(someRecord) 216 | // 217 | // // create another record 218 | // someRecord, err = goavro.NewRecord(goavro.RecordSchema(recordSchema)) 219 | // if err != nil { 220 | // log.Fatal(err) 221 | // } 222 | // someRecord.Set("username", "Batman") 223 | // someRecord.Set("comment", "Who are all of these crazies?") 224 | // someRecord.Set("com.example.timestamp", int64(1427383430)) 225 | // fw.Write(someRecord) 226 | // } 227 | func NewWriter(setters ...WriterSetter) (*Writer, error) { 228 | var err error 229 | fw := &Writer{CompressionCodec: CompressionNull, blockSize: DefaultWriterBlockSize} 230 | for _, setter := range setters { 231 | err = setter(fw) 232 | if err != nil { 233 | return nil, &ErrWriterInit{Err: err} 234 | } 235 | } 236 | if fw.w == nil { 237 | return nil, &ErrWriterInit{Message: "must specify io.Writer"} 238 | } 239 | // writer: stuff should already be initialized 240 | if !IsCompressionCodecSupported(fw.CompressionCodec) { 241 | return nil, &ErrWriterInit{Message: fmt.Sprintf("unsupported codec: %s", fw.CompressionCodec)} 242 | } 243 | if fw.dataCodec == nil { 244 | return nil, &ErrWriterInit{Message: "missing schema"} 245 | } 246 | if fw.Sync == nil { 247 | r := rand.New(rand.NewSource(time.Now().Unix())) 248 | 249 | // create random sequence of bytes for file sync marker 250 | fw.Sync = make([]byte, syncLength) 251 | for i := range fw.Sync { 252 | fw.Sync[i] = byte(r.Intn(256)) 253 | } 254 | } 255 | if err = fw.writeHeader(); err != nil { 256 | return nil, &ErrWriterInit{Err: err} 257 | } 258 | // setup writing pipeline 259 | fw.toBlock = make(chan interface{}) 260 | toEncode := make(chan *writerBlock) 261 | toCompress := make(chan *writerBlock) 262 | toWrite := make(chan *writerBlock) 263 | fw.writerDone = make(chan struct{}) 264 | go blocker(fw, fw.toBlock, toEncode) 265 | go encoder(fw, toEncode, toCompress) 266 | go compressor(fw, toCompress, toWrite) 267 | go writer(fw, toWrite) 268 | return fw, nil 269 | } 270 | 271 | // Close is called when the open file is no longer needed. It flushes 272 | // the bytes to the io.Writer if the file is being writtern. 273 | func (fw *Writer) Close() error { 274 | close(fw.toBlock) 275 | <-fw.writerDone 276 | if fw.buffered { 277 | // NOTE: error that happened before Close has 278 | // precedence of buffer flush error 279 | err := fw.w.(*bufio.Writer).Flush() 280 | if fw.err == nil { 281 | return err 282 | } 283 | } 284 | return fw.err 285 | } 286 | 287 | // Write places a datum into the pipeline to be written to the Writer. 288 | func (fw *Writer) Write(datum interface{}) { 289 | fw.toBlock <- datum 290 | } 291 | 292 | func (fw *Writer) writeHeader() (err error) { 293 | if _, err = fw.w.Write([]byte(magicBytes)); err != nil { 294 | return 295 | } 296 | // header metadata 297 | hm := make(map[string]interface{}) 298 | hm["avro.schema"] = []byte(fw.dataCodec.Schema()) 299 | hm["avro.codec"] = []byte(fw.CompressionCodec) 300 | if err = metadataCodec.Encode(fw.w, hm); err != nil { 301 | return 302 | } 303 | _, err = fw.w.Write(fw.Sync) 304 | return 305 | } 306 | 307 | type writerBlock struct { 308 | items []interface{} 309 | encoded *bytes.Buffer 310 | compressed []byte 311 | err error 312 | } 313 | 314 | func blocker(fw *Writer, toBlock <-chan interface{}, toEncode chan<- *writerBlock) { 315 | items := make([]interface{}, 0, fw.blockSize) 316 | 317 | if fw.blockTick > 0 { 318 | blockerLoop: 319 | for { 320 | select { 321 | case item, more := <-toBlock: 322 | if !more { 323 | break blockerLoop 324 | } 325 | items = append(items, item) 326 | if int64(len(items)) >= fw.blockSize { 327 | toEncode <- &writerBlock{items: items} 328 | items = make([]interface{}, 0, fw.blockSize) 329 | } 330 | case <-time.After(fw.blockTick): 331 | if len(items) > 0 { 332 | toEncode <- &writerBlock{items: items} 333 | items = make([]interface{}, 0, fw.blockSize) 334 | } 335 | } 336 | } 337 | } else { 338 | for item := range toBlock { 339 | items = append(items, item) 340 | if int64(len(items)) >= fw.blockSize { 341 | toEncode <- &writerBlock{items: items} 342 | items = make([]interface{}, 0, fw.blockSize) 343 | } 344 | } 345 | } 346 | if len(items) > 0 { 347 | toEncode <- &writerBlock{items: items} 348 | } 349 | close(toEncode) 350 | } 351 | 352 | func encoder(fw *Writer, toEncode <-chan *writerBlock, toCompress chan<- *writerBlock) { 353 | for block := range toEncode { 354 | if block.err == nil { 355 | block.encoded = new(bytes.Buffer) 356 | for _, item := range block.items { 357 | block.err = fw.dataCodec.Encode(block.encoded, item) 358 | if block.err != nil { 359 | break // ??? drops remainder of items on the floor 360 | } 361 | } 362 | } 363 | toCompress <- block 364 | } 365 | close(toCompress) 366 | } 367 | 368 | func compressor(fw *Writer, toCompress <-chan *writerBlock, toWrite chan<- *writerBlock) { 369 | switch fw.CompressionCodec { 370 | case CompressionDeflate: 371 | bb := new(bytes.Buffer) 372 | comp, _ := flate.NewWriter(bb, flate.DefaultCompression) 373 | for block := range toCompress { 374 | _, block.err = comp.Write(block.encoded.Bytes()) 375 | block.err = comp.Close() 376 | if block.err == nil { 377 | block.compressed = bb.Bytes() 378 | toWrite <- block 379 | } 380 | bb = new(bytes.Buffer) 381 | comp.Reset(bb) 382 | } 383 | case CompressionNull: 384 | for block := range toCompress { 385 | block.compressed = block.encoded.Bytes() 386 | toWrite <- block 387 | } 388 | case CompressionSnappy: 389 | for block := range toCompress { 390 | block.compressed, block.err = snappy.Encode(block.compressed, block.encoded.Bytes()) 391 | if block.err != nil { 392 | block.err = fmt.Errorf("cannot compress: %v", block.err) 393 | } 394 | toWrite <- block 395 | } 396 | } 397 | close(toWrite) 398 | } 399 | 400 | func writer(fw *Writer, toWrite <-chan *writerBlock) { 401 | for block := range toWrite { 402 | if block.err == nil { 403 | block.err = longCodec.Encode(fw.w, int64(len(block.items))) 404 | } 405 | if block.err == nil { 406 | block.err = longCodec.Encode(fw.w, int64(len(block.compressed))) 407 | } 408 | if block.err == nil { 409 | _, block.err = fw.w.Write(block.compressed) 410 | } 411 | if block.err == nil { 412 | _, block.err = fw.w.Write(fw.Sync) 413 | } 414 | if block.err != nil { 415 | log.Printf("[WARNING] cannot write block: %v", block.err) 416 | fw.err = block.err // ??? 417 | break 418 | // } else { 419 | // log.Printf("[DEBUG] block written: %d, %d, %v", len(block.items), len(block.compressed), block.compressed) 420 | } 421 | } 422 | if fw.err = longCodec.Encode(fw.w, int64(0)); fw.err == nil { 423 | fw.err = longCodec.Encode(fw.w, int64(0)) 424 | } 425 | fw.writerDone <- struct{}{} 426 | } 427 | -------------------------------------------------------------------------------- /codec.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | // Package goavro is a library that encodes and decodes of Avro 20 | // data. It provides an interface to encode data directly to io.Writer 21 | // streams, and to decode data from io.Reader streams. Goavro fully 22 | // adheres to version 1.7.7 of the Avro specification and data 23 | // encoding. 24 | package goavro 25 | 26 | import ( 27 | "encoding/json" 28 | "fmt" 29 | "io" 30 | "reflect" 31 | "strings" 32 | ) 33 | 34 | const ( 35 | mask = byte(127) 36 | flag = byte(128) 37 | ) 38 | 39 | // ErrSchemaParse is returned when a Codec cannot be created due to an 40 | // error while reading or parsing the schema. 41 | type ErrSchemaParse struct { 42 | Message string 43 | Err error 44 | } 45 | 46 | func (e ErrSchemaParse) Error() string { 47 | if e.Err == nil { 48 | return "cannot parse schema: " + e.Message 49 | } 50 | return "cannot parse schema: " + e.Message + ": " + e.Err.Error() 51 | } 52 | 53 | // ErrCodecBuild is returned when the encoder encounters an error. 54 | type ErrCodecBuild struct { 55 | Message string 56 | Err error 57 | } 58 | 59 | func (e ErrCodecBuild) Error() string { 60 | if e.Err == nil { 61 | return "cannot build " + e.Message 62 | } 63 | return "cannot build " + e.Message + ": " + e.Err.Error() 64 | } 65 | 66 | func newCodecBuildError(dataType string, a ...interface{}) *ErrCodecBuild { 67 | var err error 68 | var format, message string 69 | var ok bool 70 | if len(a) == 0 { 71 | return &ErrCodecBuild{dataType + ": no reason given", nil} 72 | } 73 | // if last item is error: save it 74 | if err, ok = a[len(a)-1].(error); ok { 75 | a = a[:len(a)-1] // pop it 76 | } 77 | // if items left, first ought to be format string 78 | if len(a) > 0 { 79 | if format, ok = a[0].(string); ok { 80 | a = a[1:] // unshift 81 | message = fmt.Sprintf(format, a...) 82 | } 83 | } 84 | if message != "" { 85 | message = ": " + message 86 | } 87 | return &ErrCodecBuild{dataType + message, err} 88 | } 89 | 90 | // Decoder interface specifies structures that may be decoded. 91 | type Decoder interface { 92 | Decode(io.Reader) (interface{}, error) 93 | } 94 | 95 | // Encoder interface specifies structures that may be encoded. 96 | type Encoder interface { 97 | Encode(io.Writer, interface{}) error 98 | } 99 | 100 | // The Codec interface supports both Decode and Encode operations. 101 | type Codec interface { 102 | Decoder 103 | Encoder 104 | Schema() string 105 | NewWriter(...WriterSetter) (*Writer, error) 106 | } 107 | 108 | // CodecSetter functions are those those which are used to modify a 109 | // newly instantiated Codec. 110 | type CodecSetter func(Codec) error 111 | 112 | type decoderFunction func(io.Reader) (interface{}, error) 113 | type encoderFunction func(io.Writer, interface{}) error 114 | 115 | type codec struct { 116 | nm *name 117 | df decoderFunction 118 | ef encoderFunction 119 | schema string 120 | } 121 | 122 | // String returns a string representation of the codec. 123 | func (c codec) String() string { 124 | return fmt.Sprintf("nm: %v, df: %v, ef: %v", c.nm, c.df, c.ef) 125 | } 126 | 127 | type Symtab map[string]*codec // map full name to codec 128 | 129 | func NewSymtab() Symtab { 130 | return make(Symtab) 131 | } 132 | 133 | // NewCodec creates a new object that supports both the Decode and 134 | // Encode methods. It requires an Avro schema, expressed as a JSON 135 | // string. The codec is created in a new Symtab. 136 | // 137 | // codec, err := goavro.NewCodec(someJSONSchema) 138 | // if err != nil { 139 | // return nil, err 140 | // } 141 | // 142 | // // Decoding data uses codec created above, and an io.Reader, 143 | // // definition not shown: 144 | // datum, err := codec.Decode(r) 145 | // if err != nil { 146 | // return nil, err 147 | // } 148 | // 149 | // // Encoding data uses codec created above, an io.Writer, 150 | // // definition not shown, and some data: 151 | // err := codec.Encode(w, datum) 152 | // if err != nil { 153 | // return nil, err 154 | // } 155 | // 156 | // // Encoding data using bufio.Writer to buffer the writes 157 | // // during data encoding: 158 | // 159 | // func encodeWithBufferedWriter(c Codec, w io.Writer, datum interface{}) error { 160 | // bw := bufio.NewWriter(w) 161 | // err := c.Encode(bw, datum) 162 | // if err != nil { 163 | // return err 164 | // } 165 | // return bw.Flush() 166 | // } 167 | // 168 | // err := encodeWithBufferedWriter(codec, w, datum) 169 | // if err != nil { 170 | // return nil, err 171 | // } 172 | func NewCodec(someJSONSchema string, setters ...CodecSetter) (Codec, error) { 173 | st := NewSymtab() 174 | return st.NewCodec(someJSONSchema, setters...) 175 | } 176 | 177 | // Create the codec in the specified Symtab 178 | func (st Symtab) NewCodec(someJSONSchema string, setters ...CodecSetter) (Codec, error) { 179 | // unmarshal into schema blob 180 | var schema interface{} 181 | if err := json.Unmarshal([]byte(someJSONSchema), &schema); err != nil { 182 | return nil, &ErrSchemaParse{"cannot unmarshal JSON", err} 183 | } 184 | // remarshal back into compressed json 185 | compressedSchema, err := json.Marshal(schema) 186 | if err != nil { 187 | return nil, fmt.Errorf("cannot marshal schema: %v", err) 188 | } 189 | 190 | newCodec, err := st.buildCodec(nullNamespace, schema) 191 | if err != nil { 192 | return nil, err 193 | } 194 | 195 | for _, setter := range setters { 196 | err = setter(newCodec) 197 | if err != nil { 198 | return nil, err 199 | } 200 | } 201 | newCodec.schema = string(compressedSchema) 202 | return newCodec, nil 203 | } 204 | 205 | // Decode will read from the specified io.Reader, and return the next 206 | // datum from the stream, or an error explaining why the stream cannot 207 | // be converted into the Codec's schema. 208 | func (c codec) Decode(r io.Reader) (interface{}, error) { 209 | return c.df(r) 210 | } 211 | 212 | // Encode will write the specified datum to the specified io.Writer, 213 | // or return an error explaining why the datum cannot be converted 214 | // into the Codec's schema. 215 | func (c codec) Encode(w io.Writer, datum interface{}) error { 216 | return c.ef(w, datum) 217 | } 218 | 219 | func (c codec) Schema() string { 220 | return c.schema 221 | } 222 | 223 | // NewWriter creates a new Writer that encodes using the given Codec. 224 | // 225 | // The following two code examples produce identical results: 226 | // 227 | // // method 1: 228 | // fw, err := codec.NewWriter(goavro.ToWriter(w)) 229 | // if err != nil { 230 | // log.Fatal(err) 231 | // } 232 | // defer fw.Close() 233 | // 234 | // // method 2: 235 | // fw, err := goavro.NewWriter(goavro.ToWriter(w), goavro.UseCodec(codec)) 236 | // if err != nil { 237 | // log.Fatal(err) 238 | // } 239 | // defer fw.Close() 240 | func (c codec) NewWriter(setters ...WriterSetter) (*Writer, error) { 241 | setters = append(setters, UseCodec(c)) 242 | return NewWriter(setters...) 243 | } 244 | 245 | var ( 246 | nullCodec, booleanCodec, intCodec, longCodec, floatCodec, doubleCodec, bytesCodec, stringCodec *codec 247 | ) 248 | 249 | func init() { 250 | // NOTE: use Go type names because for runtime resolution of 251 | // union member, it gets the Go type name of the datum sent to 252 | // the union encoder, and uses that string as a key into the 253 | // encoders map 254 | nullCodec = &codec{nm: &name{n: "null"}, df: nullDecoder, ef: nullEncoder} 255 | booleanCodec = &codec{nm: &name{n: "bool"}, df: booleanDecoder, ef: booleanEncoder} 256 | intCodec = &codec{nm: &name{n: "int32"}, df: intDecoder, ef: intEncoder} 257 | longCodec = &codec{nm: &name{n: "int64"}, df: longDecoder, ef: longEncoder} 258 | floatCodec = &codec{nm: &name{n: "float32"}, df: floatDecoder, ef: floatEncoder} 259 | doubleCodec = &codec{nm: &name{n: "float64"}, df: doubleDecoder, ef: doubleEncoder} 260 | bytesCodec = &codec{nm: &name{n: "[]uint8"}, df: bytesDecoder, ef: bytesEncoder} 261 | stringCodec = &codec{nm: &name{n: "string"}, df: stringDecoder, ef: stringEncoder} 262 | } 263 | 264 | func (st Symtab) buildCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 265 | switch schemaType := schema.(type) { 266 | case string: 267 | return st.buildString(enclosingNamespace, schemaType, schema) 268 | case []interface{}: 269 | return st.makeUnionCodec(enclosingNamespace, schema) 270 | case map[string]interface{}: 271 | return st.buildMap(enclosingNamespace, schema.(map[string]interface{})) 272 | default: 273 | return nil, newCodecBuildError("unknown", "schema type: %T", schema) 274 | } 275 | } 276 | 277 | func (st Symtab) buildMap(enclosingNamespace string, schema map[string]interface{}) (*codec, error) { 278 | t, ok := schema["type"] 279 | if !ok { 280 | return nil, newCodecBuildError("map", "ought have type: %v", schema) 281 | } 282 | switch t.(type) { 283 | case string: 284 | // EXAMPLE: "type":"int" 285 | // EXAMPLE: "type":"enum" 286 | return st.buildString(enclosingNamespace, t.(string), schema) 287 | case map[string]interface{}, []interface{}: 288 | // EXAMPLE: "type":{"type":fixed","name":"fixed_16","size":16} 289 | // EXAMPLE: "type":["null","int"] 290 | return st.buildCodec(enclosingNamespace, t) 291 | default: 292 | return nil, newCodecBuildError("map", "type ought to be either string, map[string]interface{}, or []interface{}; received: %T", t) 293 | } 294 | } 295 | 296 | func (st Symtab) buildString(enclosingNamespace, typeName string, schema interface{}) (*codec, error) { 297 | switch typeName { 298 | case "null": 299 | return nullCodec, nil 300 | case "boolean": 301 | return booleanCodec, nil 302 | case "int": 303 | return intCodec, nil 304 | case "long": 305 | return longCodec, nil 306 | case "float": 307 | return floatCodec, nil 308 | case "double": 309 | return doubleCodec, nil 310 | case "bytes": 311 | return bytesCodec, nil 312 | case "string": 313 | return stringCodec, nil 314 | case "record": 315 | return st.makeRecordCodec(enclosingNamespace, schema) 316 | case "enum": 317 | return st.makeEnumCodec(enclosingNamespace, schema) 318 | case "fixed": 319 | return st.makeFixedCodec(enclosingNamespace, schema) 320 | case "map": 321 | return st.makeMapCodec(enclosingNamespace, schema) 322 | case "array": 323 | return st.makeArrayCodec(enclosingNamespace, schema) 324 | default: 325 | t, err := newName(nameName(typeName), nameEnclosingNamespace(enclosingNamespace)) 326 | if err != nil { 327 | return nil, newCodecBuildError(typeName, "could not normalize name: %s", enclosingNamespace, typeName) 328 | } 329 | c, ok := st[t.n] 330 | if !ok { 331 | return nil, newCodecBuildError("unknown", "unknown type name: %s", t.n) 332 | } 333 | return c, nil 334 | } 335 | } 336 | 337 | type unionEncoder struct { 338 | ef encoderFunction 339 | index int32 340 | } 341 | 342 | func (st Symtab) makeUnionCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 343 | errorNamespace := "null namespace" 344 | if enclosingNamespace != nullNamespace { 345 | errorNamespace = enclosingNamespace 346 | } 347 | friendlyName := fmt.Sprintf("union (%s)", errorNamespace) 348 | 349 | // schema checks 350 | schemaArray, ok := schema.([]interface{}) 351 | if !ok { 352 | return nil, newCodecBuildError(friendlyName, "ought to be array: %T", schema) 353 | } 354 | if len(schemaArray) == 0 { 355 | return nil, newCodecBuildError(friendlyName, " ought have at least one member") 356 | } 357 | 358 | // setup 359 | nameToUnionEncoder := make(map[string]unionEncoder) 360 | indexToDecoder := make([]decoderFunction, len(schemaArray)) 361 | allowedNames := make([]string, len(schemaArray)) 362 | 363 | for idx, unionMemberSchema := range schemaArray { 364 | c, err := st.buildCodec(enclosingNamespace, unionMemberSchema) 365 | if err != nil { 366 | return nil, newCodecBuildError(friendlyName, "member ought to be decodable: %v", err) 367 | } 368 | allowedNames[idx] = c.nm.n 369 | indexToDecoder[idx] = c.df 370 | nameToUnionEncoder[c.nm.n] = unionEncoder{ef: c.ef, index: int32(idx)} 371 | } 372 | 373 | invalidType := "datum ought match schema: expected: " 374 | invalidType += strings.Join(allowedNames, ", ") 375 | invalidType += "; received: " 376 | 377 | nm, _ := newName(nameName("union")) 378 | friendlyName = fmt.Sprintf("union (%s)", nm.n) 379 | 380 | return &codec{ 381 | nm: nm, 382 | df: func(r io.Reader) (interface{}, error) { 383 | i, err := intDecoder(r) 384 | if err != nil { 385 | return nil, newEncoderError(friendlyName, err) 386 | } 387 | idx, ok := i.(int32) 388 | if !ok { 389 | return nil, newEncoderError(friendlyName, "expected: int; received: %T", i) 390 | } 391 | index := int(idx) 392 | if index < 0 || index >= len(indexToDecoder) { 393 | return nil, newEncoderError(friendlyName, "index must be between 0 and %d; read index: %d", len(indexToDecoder)-1, index) 394 | } 395 | return indexToDecoder[index](r) 396 | }, 397 | ef: func(w io.Writer, datum interface{}) error { 398 | var err error 399 | var name string 400 | switch datum.(type) { 401 | default: 402 | name = reflect.TypeOf(datum).String() 403 | case map[string]interface{}: 404 | name = "map" 405 | case []interface{}: 406 | name = "array" 407 | case nil: 408 | name = "null" 409 | case *Record: 410 | name = datum.(*Record).Name 411 | } 412 | ue, ok := nameToUnionEncoder[name] 413 | if !ok { 414 | return newEncoderError(friendlyName, invalidType+name) 415 | } 416 | if err = intEncoder(w, ue.index); err != nil { 417 | return newEncoderError(friendlyName, err) 418 | } 419 | if err = ue.ef(w, datum); err != nil { 420 | return newEncoderError(friendlyName, err) 421 | } 422 | return nil 423 | }, 424 | }, nil 425 | } 426 | 427 | func (st Symtab) makeEnumCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 428 | errorNamespace := "null namespace" 429 | if enclosingNamespace != nullNamespace { 430 | errorNamespace = enclosingNamespace 431 | } 432 | friendlyName := fmt.Sprintf("enum (%s)", errorNamespace) 433 | 434 | // schema checks 435 | schemaMap, ok := schema.(map[string]interface{}) 436 | if !ok { 437 | return nil, newCodecBuildError(friendlyName, "expected: map[string]interface{}; received: %T", schema) 438 | } 439 | nm, err := newName(nameEnclosingNamespace(enclosingNamespace), nameSchema(schemaMap)) 440 | if err != nil { 441 | return nil, err 442 | } 443 | friendlyName = fmt.Sprintf("enum (%s)", nm.n) 444 | 445 | s, ok := schemaMap["symbols"] 446 | if !ok { 447 | return nil, newCodecBuildError(friendlyName, "ought to have symbols key") 448 | } 449 | symtab, ok := s.([]interface{}) 450 | if !ok || len(symtab) == 0 { 451 | return nil, newCodecBuildError(friendlyName, "symbols ought to be non-empty array") 452 | } 453 | for _, v := range symtab { 454 | _, ok := v.(string) 455 | if !ok { 456 | return nil, newCodecBuildError(friendlyName, "symbols array member ought to be string") 457 | } 458 | } 459 | c := &codec{ 460 | nm: nm, 461 | df: func(r io.Reader) (interface{}, error) { 462 | someValue, err := longDecoder(r) 463 | if err != nil { 464 | return nil, newDecoderError(friendlyName, err) 465 | } 466 | index, ok := someValue.(int64) 467 | if !ok { 468 | return nil, newDecoderError(friendlyName, "expected long; received: %T", someValue) 469 | } 470 | if index < 0 || index >= int64(len(symtab)) { 471 | return nil, newDecoderError(friendlyName, "index must be between 0 and %d", len(symtab)-1) 472 | } 473 | return symtab[index], nil 474 | }, 475 | ef: func(w io.Writer, datum interface{}) error { 476 | someString, ok := datum.(string) 477 | if !ok { 478 | return newEncoderError(friendlyName, "expected: string; received: %T", datum) 479 | } 480 | for idx, symbol := range symtab { 481 | if symbol == someString { 482 | if err := longEncoder(w, int64(idx)); err != nil { 483 | return newEncoderError(friendlyName, err) 484 | } 485 | return nil 486 | } 487 | } 488 | return newEncoderError(friendlyName, "symbol not defined: %s", someString) 489 | }, 490 | } 491 | st[nm.n] = c 492 | return c, nil 493 | } 494 | 495 | func (st Symtab) makeFixedCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 496 | errorNamespace := "null namespace" 497 | if enclosingNamespace != nullNamespace { 498 | errorNamespace = enclosingNamespace 499 | } 500 | friendlyName := fmt.Sprintf("fixed (%s)", errorNamespace) 501 | 502 | // schema checks 503 | schemaMap, ok := schema.(map[string]interface{}) 504 | if !ok { 505 | return nil, newCodecBuildError(friendlyName, "expected: map[string]interface{}; received: %T", schema) 506 | } 507 | nm, err := newName(nameSchema(schemaMap), nameEnclosingNamespace(enclosingNamespace)) 508 | if err != nil { 509 | return nil, err 510 | } 511 | friendlyName = fmt.Sprintf("fixed (%s)", nm.n) 512 | s, ok := schemaMap["size"] 513 | if !ok { 514 | return nil, newCodecBuildError(friendlyName, "ought to have size key") 515 | } 516 | fs, ok := s.(float64) 517 | if !ok { 518 | return nil, newCodecBuildError(friendlyName, "size ought to be number: %T", s) 519 | } 520 | size := int32(fs) 521 | c := &codec{ 522 | nm: nm, 523 | df: func(r io.Reader) (interface{}, error) { 524 | buf := make([]byte, size) 525 | n, err := r.Read(buf) 526 | if err != nil { 527 | return nil, newDecoderError(friendlyName, err) 528 | } 529 | if n < int(size) { 530 | return nil, newDecoderError(friendlyName, "buffer underrun") 531 | } 532 | return buf, nil 533 | }, 534 | ef: func(w io.Writer, datum interface{}) error { 535 | someBytes, ok := datum.([]byte) 536 | if !ok { 537 | return newEncoderError(friendlyName, "expected: []byte; received: %T", datum) 538 | } 539 | if len(someBytes) != int(size) { 540 | return newEncoderError(friendlyName, "expected: %d bytes; received: %d", size, len(someBytes)) 541 | } 542 | n, err := w.Write(someBytes) 543 | if err != nil { 544 | return newEncoderError(friendlyName, err) 545 | } 546 | if n != int(size) { 547 | return newEncoderError(friendlyName, "buffer underrun") 548 | } 549 | return nil 550 | }, 551 | } 552 | st[nm.n] = c 553 | return c, nil 554 | } 555 | 556 | func (st Symtab) makeRecordCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 557 | errorNamespace := "null namespace" 558 | if enclosingNamespace != nullNamespace { 559 | errorNamespace = enclosingNamespace 560 | } 561 | friendlyName := fmt.Sprintf("record (%s)", errorNamespace) 562 | 563 | // delegate schema checks to NewRecord() 564 | recordTemplate, err := NewRecord(recordSchemaRaw(schema), RecordEnclosingNamespace(enclosingNamespace)) 565 | if err != nil { 566 | return nil, err 567 | } 568 | 569 | fieldCodecs := make([]*codec, len(recordTemplate.Fields)) 570 | for idx, field := range recordTemplate.Fields { 571 | var err error 572 | fieldCodecs[idx], err = st.buildCodec(recordTemplate.n.namespace(), field.schema) 573 | if err != nil { 574 | return nil, newCodecBuildError(friendlyName, "record field ought to be codec: %+v", st, err) 575 | } 576 | } 577 | 578 | friendlyName = fmt.Sprintf("record (%s)", recordTemplate.Name) 579 | 580 | c := &codec{ 581 | nm: recordTemplate.n, 582 | df: func(r io.Reader) (interface{}, error) { 583 | someRecord, _ := NewRecord(recordSchemaRaw(schema), RecordEnclosingNamespace(enclosingNamespace)) 584 | for idx, codec := range fieldCodecs { 585 | value, err := codec.Decode(r) 586 | if err != nil { 587 | return nil, newDecoderError(friendlyName, err) 588 | } 589 | someRecord.Fields[idx].Datum = value 590 | } 591 | return someRecord, nil 592 | }, 593 | ef: func(w io.Writer, datum interface{}) error { 594 | someRecord, ok := datum.(*Record) 595 | if !ok { 596 | return newEncoderError(friendlyName, "expected: Record; received: %T", datum) 597 | } 598 | if someRecord.Name != recordTemplate.Name { 599 | return newEncoderError(friendlyName, "expected: %v; received: %v", recordTemplate.Name, someRecord.Name) 600 | } 601 | for idx, field := range someRecord.Fields { 602 | var value interface{} 603 | // check whether field datum is valid 604 | if reflect.ValueOf(field.Datum).IsValid() { 605 | value = field.Datum 606 | } else if field.hasDefault { 607 | value = field.defval 608 | } else { 609 | return newEncoderError(friendlyName, "field has no data and no default set: %v", field.Name) 610 | } 611 | err = fieldCodecs[idx].Encode(w, value) 612 | if err != nil { 613 | return newEncoderError(friendlyName, err) 614 | } 615 | } 616 | return nil 617 | }, 618 | } 619 | st[recordTemplate.Name] = c 620 | return c, nil 621 | } 622 | 623 | func (st Symtab) makeMapCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 624 | errorNamespace := "null namespace" 625 | if enclosingNamespace != nullNamespace { 626 | errorNamespace = enclosingNamespace 627 | } 628 | friendlyName := fmt.Sprintf("map (%s)", errorNamespace) 629 | 630 | // schema checks 631 | schemaMap, ok := schema.(map[string]interface{}) 632 | if !ok { 633 | return nil, newCodecBuildError(friendlyName, "expected: map[string]interface{}; received: %T", schema) 634 | } 635 | v, ok := schemaMap["values"] 636 | if !ok { 637 | return nil, newCodecBuildError(friendlyName, "ought to have values key") 638 | } 639 | valuesCodec, err := st.buildCodec(enclosingNamespace, v) 640 | if err != nil { 641 | return nil, newCodecBuildError(friendlyName, err) 642 | } 643 | 644 | nm := &name{n: "map"} 645 | friendlyName = fmt.Sprintf("map (%s)", nm.n) 646 | 647 | return &codec{ 648 | nm: nm, 649 | df: func(r io.Reader) (interface{}, error) { 650 | data := make(map[string]interface{}) 651 | someValue, err := longDecoder(r) 652 | if err != nil { 653 | return nil, newDecoderError(friendlyName, err) 654 | } 655 | blockCount := someValue.(int64) 656 | 657 | for blockCount != 0 { 658 | if blockCount < 0 { 659 | blockCount = -blockCount 660 | // read and discard number of bytes in block 661 | _, err := longDecoder(r) 662 | if err != nil { 663 | return nil, newDecoderError(friendlyName, err) 664 | } 665 | } 666 | for i := int64(0); i < blockCount; i++ { 667 | someValue, err := stringDecoder(r) 668 | if err != nil { 669 | return nil, newDecoderError(friendlyName, err) 670 | } 671 | mapKey, ok := someValue.(string) 672 | if !ok { 673 | return nil, newDecoderError(friendlyName, "key ought to be string") 674 | } 675 | datum, err := valuesCodec.df(r) 676 | if err != nil { 677 | return nil, err 678 | } 679 | data[mapKey] = datum 680 | } 681 | someValue, err = longDecoder(r) 682 | if err != nil { 683 | return nil, newDecoderError(friendlyName, err) 684 | } 685 | blockCount = someValue.(int64) 686 | } 687 | return data, nil 688 | }, 689 | ef: func(w io.Writer, datum interface{}) error { 690 | dict, ok := datum.(map[string]interface{}) 691 | if !ok { 692 | return newEncoderError(friendlyName, "expected: map[string]interface{}; received: %T", datum) 693 | } 694 | if err = longEncoder(w, int64(len(dict))); err != nil { 695 | return newEncoderError(friendlyName, err) 696 | } 697 | for k, v := range dict { 698 | if err = stringEncoder(w, k); err != nil { 699 | return newEncoderError(friendlyName, err) 700 | } 701 | if err = valuesCodec.ef(w, v); err != nil { 702 | return newEncoderError(friendlyName, err) 703 | } 704 | } 705 | if err = longEncoder(w, int64(0)); err != nil { 706 | return newEncoderError(friendlyName, err) 707 | } 708 | return nil 709 | }, 710 | }, nil 711 | } 712 | 713 | func (st Symtab) makeArrayCodec(enclosingNamespace string, schema interface{}) (*codec, error) { 714 | errorNamespace := "null namespace" 715 | if enclosingNamespace != nullNamespace { 716 | errorNamespace = enclosingNamespace 717 | } 718 | friendlyName := fmt.Sprintf("array (%s)", errorNamespace) 719 | 720 | // schema checks 721 | schemaMap, ok := schema.(map[string]interface{}) 722 | if !ok { 723 | return nil, newCodecBuildError(friendlyName, "expected: map[string]interface{}; received: %T", schema) 724 | } 725 | v, ok := schemaMap["items"] 726 | if !ok { 727 | return nil, newCodecBuildError(friendlyName, "ought to have items key") 728 | } 729 | valuesCodec, err := st.buildCodec(enclosingNamespace, v) 730 | if err != nil { 731 | return nil, newCodecBuildError(friendlyName, err) 732 | } 733 | 734 | const itemsPerArrayBlock = 10 735 | nm := &name{n: "array"} 736 | friendlyName = fmt.Sprintf("array (%s)", nm.n) 737 | 738 | return &codec{ 739 | nm: nm, 740 | df: func(r io.Reader) (interface{}, error) { 741 | data := make([]interface{}, 0) 742 | 743 | someValue, err := longDecoder(r) 744 | if err != nil { 745 | return nil, newDecoderError(friendlyName, err) 746 | } 747 | blockCount := someValue.(int64) 748 | 749 | for blockCount != 0 { 750 | if blockCount < 0 { 751 | blockCount = -blockCount 752 | // read and discard number of bytes in block 753 | _, err = longDecoder(r) 754 | if err != nil { 755 | return nil, newDecoderError(friendlyName, err) 756 | } 757 | } 758 | for i := int64(0); i < blockCount; i++ { 759 | datum, err := valuesCodec.df(r) 760 | if err != nil { 761 | return nil, newDecoderError(friendlyName, err) 762 | } 763 | data = append(data, datum) 764 | } 765 | someValue, err = longDecoder(r) 766 | if err != nil { 767 | return nil, newDecoderError(friendlyName, err) 768 | } 769 | blockCount = someValue.(int64) 770 | } 771 | return data, nil 772 | }, 773 | ef: func(w io.Writer, datum interface{}) error { 774 | someArray, ok := datum.([]interface{}) 775 | if !ok { 776 | return newEncoderError(friendlyName, "expected: []interface{}; received: %T", datum) 777 | } 778 | for leftIndex := 0; leftIndex < len(someArray); leftIndex += itemsPerArrayBlock { 779 | rightIndex := leftIndex + itemsPerArrayBlock 780 | if rightIndex > len(someArray) { 781 | rightIndex = len(someArray) 782 | } 783 | items := someArray[leftIndex:rightIndex] 784 | err = longEncoder(w, int64(len(items))) 785 | if err != nil { 786 | return newEncoderError(friendlyName, err) 787 | } 788 | for _, item := range items { 789 | err = valuesCodec.ef(w, item) 790 | if err != nil { 791 | return newEncoderError(friendlyName, err) 792 | } 793 | } 794 | } 795 | return longEncoder(w, int64(0)) 796 | }, 797 | }, nil 798 | } 799 | -------------------------------------------------------------------------------- /codec_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2015 LinkedIn Corp. Licensed under the Apache License, 2 | // Version 2.0 (the "License"); you may not use this file except in 3 | // compliance with the License.
 You may obtain a copy of the License 4 | // at http://www.apache.org/licenses/LICENSE-2.0 5 | // 6 | // Unless required by applicable law or agreed to in writing, software 7 | // 
distributed under the License is distributed on an "AS IS" BASIS, 8 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 9 | // implied.Copyright [201X] LinkedIn Corp. Licensed under the Apache 10 | // License, Version 2.0 (the "License"); you may not use this file 11 | // except in compliance with the License.
 You may obtain a copy of 12 | // the License at http://www.apache.org/licenses/LICENSE-2.0 13 | // 14 | // Unless required by applicable law or agreed to in writing, software 15 | // 
distributed under the License is distributed on an "AS IS" BASIS, 16 | // 
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 17 | // implied. 18 | 19 | package goavro 20 | 21 | import ( 22 | "bufio" 23 | "bytes" 24 | "encoding/json" 25 | "io" 26 | "math" 27 | "reflect" 28 | "testing" 29 | ) 30 | 31 | //////////////////////////////////////// 32 | // helpers 33 | //////////////////////////////////////// 34 | 35 | func checkCodecDecoderError(t *testing.T, schema string, bits []byte, expectedError interface{}) { 36 | codec, err := NewCodec(schema) 37 | checkErrorFatal(t, err, nil) 38 | bb := bytes.NewBuffer(bits) 39 | _, err = codec.Decode(bb) 40 | checkError(t, err, expectedError) 41 | } 42 | 43 | func checkCodecDecoderResult(t *testing.T, schema string, bits []byte, datum interface{}) { 44 | codec, err := NewCodec(schema) 45 | checkErrorFatal(t, err, nil) 46 | bb := bytes.NewBuffer(bits) 47 | decoded, err := codec.Decode(bb) 48 | checkErrorFatal(t, err, nil) 49 | 50 | if reflect.TypeOf(decoded) == reflect.TypeOf(datum) { 51 | switch datum.(type) { 52 | case []byte: 53 | if bytes.Compare(decoded.([]byte), datum.([]byte)) != 0 { 54 | t.Errorf("Actual: %#v; Expected: %#v", decoded, datum) 55 | } 56 | default: 57 | if decoded != datum { 58 | t.Errorf("Actual: %v; Expected: %v", decoded, datum) 59 | } 60 | } 61 | } else { 62 | t.Errorf("Actual: %T; Expected: %T", decoded, datum) 63 | } 64 | } 65 | 66 | func checkCodecEncoderError(t *testing.T, schema string, datum interface{}, expectedError interface{}) { 67 | codec, err := NewCodec(schema) 68 | checkErrorFatal(t, err, nil) 69 | 70 | bb := new(bytes.Buffer) 71 | err = codec.Encode(bb, datum) 72 | checkErrorFatal(t, err, expectedError) 73 | } 74 | 75 | func checkCodecEncoderResult(t *testing.T, schema string, datum interface{}, bits []byte) { 76 | codec, err := NewCodec(schema) 77 | checkErrorFatal(t, err, nil) 78 | 79 | bb := new(bytes.Buffer) 80 | err = codec.Encode(bb, datum) 81 | if err != nil { 82 | t.Errorf("Actual: %v; Expected: %v", err, nil) 83 | } 84 | if bytes.Compare(bb.Bytes(), bits) != 0 { 85 | t.Errorf("Actual: %#v; Expected: %#v", bb.Bytes(), bits) 86 | } 87 | } 88 | 89 | func checkCodecRoundTrip(t *testing.T, schema string, datum interface{}) { 90 | codec, err := NewCodec(schema) 91 | if err != nil { 92 | t.Errorf("%v", err) 93 | return 94 | } 95 | bb := new(bytes.Buffer) 96 | err = codec.Encode(bb, datum) 97 | if err != nil { 98 | t.Errorf("%v", err) 99 | return 100 | } 101 | actual, err := codec.Decode(bb) 102 | if err != nil { 103 | t.Errorf("%v", err) 104 | return 105 | } 106 | actualJSON, err := json.Marshal(actual) 107 | if err != nil { 108 | t.Errorf("%v", err) 109 | return 110 | } 111 | expectedJSON, err := json.Marshal(datum) 112 | if err != nil { 113 | t.Errorf("%v", err) 114 | return 115 | } 116 | if string(actualJSON) != string(expectedJSON) { 117 | t.Errorf("Actual: %#v; Expected: %#v", string(actualJSON), string(expectedJSON)) 118 | } 119 | } 120 | 121 | //////////////////////////////////////// 122 | 123 | func TestCodecRoundTrip(t *testing.T) { 124 | // null 125 | checkCodecRoundTrip(t, `"null"`, nil) 126 | checkCodecRoundTrip(t, `{"type":"null"}`, nil) 127 | // boolean 128 | checkCodecRoundTrip(t, `"boolean"`, false) 129 | checkCodecRoundTrip(t, `"boolean"`, true) 130 | // int 131 | checkCodecRoundTrip(t, `"int"`, int32(-3)) 132 | checkCodecRoundTrip(t, `"int"`, int32(-65)) 133 | checkCodecRoundTrip(t, `"int"`, int32(0)) 134 | checkCodecRoundTrip(t, `"int"`, int32(1016)) 135 | checkCodecRoundTrip(t, `"int"`, int32(3)) 136 | checkCodecRoundTrip(t, `"int"`, int32(42)) 137 | checkCodecRoundTrip(t, `"int"`, int32(64)) 138 | checkCodecRoundTrip(t, `"int"`, int32(66052)) 139 | checkCodecRoundTrip(t, `"int"`, int32(8454660)) 140 | // long 141 | checkCodecRoundTrip(t, `"long"`, int64(-2147483648)) 142 | checkCodecRoundTrip(t, `"long"`, int64(-3)) 143 | checkCodecRoundTrip(t, `"long"`, int64(-65)) 144 | checkCodecRoundTrip(t, `"long"`, int64(0)) 145 | checkCodecRoundTrip(t, `"long"`, int64(1082196484)) 146 | checkCodecRoundTrip(t, `"long"`, int64(138521149956)) 147 | checkCodecRoundTrip(t, `"long"`, int64(17730707194372)) 148 | checkCodecRoundTrip(t, `"long"`, int64(2147483647)) 149 | checkCodecRoundTrip(t, `"long"`, int64(2269530520879620)) 150 | checkCodecRoundTrip(t, `"long"`, int64(3)) 151 | checkCodecRoundTrip(t, `"long"`, int64(64)) 152 | // float 153 | checkCodecRoundTrip(t, `"float"`, float32(3.5)) 154 | // checkCodecRoundTrip(t, `"float"`, float32(math.Inf(-1))) 155 | // checkCodecRoundTrip(t, `"float"`, float32(math.Inf(1))) 156 | // checkCodecRoundTrip(t, `"float"`, float32(math.NaN())) 157 | // double 158 | checkCodecRoundTrip(t, `"double"`, float64(3.5)) 159 | // checkCodecRoundTrip(t, `"double"`, float64(math.Inf(-1))) 160 | // checkCodecRoundTrip(t, `"double"`, float64(math.Inf(1))) 161 | // checkCodecRoundTrip(t, `"double"`, float64(math.NaN())) 162 | // bytes 163 | checkCodecRoundTrip(t, `"bytes"`, []byte("")) 164 | checkCodecRoundTrip(t, `"bytes"`, []byte("some bytes")) 165 | // string 166 | checkCodecRoundTrip(t, `"string"`, "") 167 | checkCodecRoundTrip(t, `"string"`, "filibuster") 168 | } 169 | 170 | func TestCodecDecoderPrimitives(t *testing.T) { 171 | // null 172 | checkCodecDecoderResult(t, `"null"`, []byte("\x01"), nil) 173 | // boolean 174 | checkCodecDecoderError(t, `"boolean"`, []byte("\x02"), "cannot decode boolean") 175 | checkCodecDecoderError(t, `"boolean"`, []byte(""), "cannot decode boolean: EOF") 176 | checkCodecDecoderResult(t, `"boolean"`, []byte("\x00"), false) 177 | checkCodecDecoderResult(t, `"boolean"`, []byte("\x01"), true) 178 | // int 179 | checkCodecDecoderError(t, `"int"`, []byte(""), "cannot decode int: EOF") 180 | checkCodecDecoderResult(t, `"int"`, []byte("\x00"), int32(0)) 181 | checkCodecDecoderResult(t, `"int"`, []byte("\x05"), int32(-3)) 182 | checkCodecDecoderResult(t, `"int"`, []byte("\x06"), int32(3)) 183 | checkCodecDecoderResult(t, `"int"`, []byte("\x80\x01"), int32(64)) 184 | checkCodecDecoderResult(t, `"int"`, []byte("\x81\x01"), int32(-65)) 185 | checkCodecDecoderResult(t, `"int"`, []byte("\xf0\x0f"), int32(1016)) 186 | checkCodecDecoderResult(t, `"int"`, []byte("\x88\x88\x08"), int32(66052)) 187 | checkCodecDecoderResult(t, `"int"`, []byte("\x88\x88\x88\x08"), int32(8454660)) 188 | // long 189 | checkCodecDecoderError(t, `"long"`, []byte(""), "cannot decode long: EOF") 190 | checkCodecDecoderResult(t, `"long"`, []byte("\x00"), int64(0)) 191 | checkCodecDecoderResult(t, `"long"`, []byte("\x05"), int64(-3)) 192 | checkCodecDecoderResult(t, `"long"`, []byte("\x06"), int64(3)) 193 | checkCodecDecoderResult(t, `"long"`, []byte("\x80\x01"), int64(64)) 194 | checkCodecDecoderResult(t, `"long"`, []byte("\x81\x01"), int64(-65)) 195 | checkCodecDecoderResult(t, `"long"`, []byte("\xfe\xff\xff\xff\x0f"), int64(2147483647)) 196 | checkCodecDecoderResult(t, `"long"`, []byte("\xff\xff\xff\xff\x0f"), int64(-2147483648)) 197 | checkCodecDecoderResult(t, `"long"`, []byte("\x88\x88\x88\x88\x08"), int64(1082196484)) 198 | checkCodecDecoderResult(t, `"long"`, []byte("\x88\x88\x88\x88\x88\x08"), int64(138521149956)) 199 | checkCodecDecoderResult(t, `"long"`, []byte("\x88\x88\x88\x88\x88\x88\x08"), int64(17730707194372)) 200 | checkCodecDecoderResult(t, `"long"`, []byte("\x88\x88\x88\x88\x88\x88\x88\x08"), int64(2269530520879620)) 201 | 202 | // float 203 | checkCodecDecoderError(t, `"float"`, []byte(""), "cannot decode float: EOF") 204 | checkCodecDecoderResult(t, `"float"`, []byte("\x00\x00\x60\x40"), float32(3.5)) 205 | checkCodecDecoderResult(t, `"float"`, []byte("\x00\x00\x80\u007f"), float32(math.Inf(1))) 206 | checkCodecDecoderResult(t, `"float"`, []byte("\x00\x00\x80\xff"), float32(math.Inf(-1))) 207 | // double 208 | checkCodecDecoderError(t, `"double"`, []byte(""), "cannot decode double: EOF") 209 | checkCodecDecoderResult(t, `"double"`, []byte("\x00\x00\x00\x00\x00\x00\f@"), float64(3.5)) 210 | checkCodecDecoderResult(t, `"double"`, []byte("\x00\x00\x00\x00\x00\x00\xf0\u007f"), float64(math.Inf(1))) 211 | checkCodecDecoderResult(t, `"double"`, []byte("\x00\x00\x00\x00\x00\x00\xf0\xff"), float64(math.Inf(-1))) 212 | // bytes 213 | checkCodecDecoderError(t, `"bytes"`, []byte(""), "cannot decode bytes: cannot decode long: EOF") 214 | checkCodecDecoderError(t, `"bytes"`, []byte("\x01"), "cannot decode bytes: negative length: -1") 215 | checkCodecDecoderError(t, `"bytes"`, []byte("\x02"), "cannot decode bytes: EOF") 216 | checkCodecDecoderResult(t, `"bytes"`, []byte("\x00"), []byte("")) 217 | checkCodecDecoderResult(t, `"bytes"`, []byte("\x14some bytes"), []byte("some bytes")) 218 | // string 219 | checkCodecDecoderError(t, `"string"`, []byte(""), "cannot decode string: cannot decode long: EOF") 220 | checkCodecDecoderError(t, `"string"`, []byte("\x01"), "cannot decode string: negative length: -1") 221 | checkCodecDecoderError(t, `"string"`, []byte("\x02"), "cannot decode string: EOF") 222 | checkCodecDecoderResult(t, `"string"`, []byte("\x00"), "") 223 | checkCodecDecoderResult(t, `"string"`, []byte("\x16some string"), "some string") 224 | } 225 | 226 | func TestCodecDecoderFloatNaN(t *testing.T) { 227 | decoder, err := NewCodec(`"float"`) 228 | checkErrorFatal(t, err, nil) 229 | 230 | // NOTE: NaN never equals NaN (math is fun) 231 | bits := []byte("\x00\x00\xc0\u007f") 232 | bb := bytes.NewBuffer(bits) 233 | actual, err := decoder.Decode(bb) 234 | checkErrorFatal(t, err, nil) 235 | 236 | someFloat, ok := actual.(float32) 237 | if !ok { 238 | t.Fatalf("Actual: %#v; Expected: %#v", ok, true) 239 | } 240 | if !math.IsNaN(float64(someFloat)) { 241 | expected := math.NaN() 242 | t.Errorf("Actual: %T(%#v); Expected: %T(%#v)", actual, actual, expected, expected) 243 | } 244 | } 245 | 246 | func TestCodecDecoderDoubleNaN(t *testing.T) { 247 | decoder, err := NewCodec(`"double"`) 248 | checkErrorFatal(t, err, nil) 249 | 250 | // NOTE: NaN never equals NaN (math is fun) 251 | bits := []byte("\x01\x00\x00\x00\x00\x00\xf8\u007f") 252 | bb := bytes.NewBuffer(bits) 253 | actual, err := decoder.Decode(bb) 254 | checkErrorFatal(t, err, nil) 255 | 256 | someFloat, ok := actual.(float64) 257 | if !ok { 258 | t.Fatalf("Actual: %#v; Expected: %#v", ok, true) 259 | } 260 | if !math.IsNaN(float64(someFloat)) { 261 | expected := math.NaN() 262 | t.Errorf("Actual: %T(%#v); Expected: %T(%#v)", actual, actual, expected, expected) 263 | } 264 | } 265 | 266 | func TestCodecEncoderPrimitives(t *testing.T) { 267 | // null 268 | checkCodecEncoderResult(t, `"null"`, nil, []byte("")) 269 | checkCodecEncoderResult(t, `{"type":"null"}`, nil, []byte("")) 270 | // boolean 271 | checkCodecEncoderResult(t, `"boolean"`, false, []byte("\x00")) 272 | checkCodecEncoderResult(t, `"boolean"`, true, []byte("\x01")) 273 | // int 274 | checkCodecEncoderResult(t, `"int"`, int32(-53), []byte("\x69")) 275 | checkCodecEncoderResult(t, `"int"`, int32(-33), []byte("\x41")) 276 | checkCodecEncoderResult(t, `"int"`, int32(-3), []byte("\x05")) 277 | checkCodecEncoderResult(t, `"int"`, int32(-65), []byte("\x81\x01")) 278 | checkCodecEncoderResult(t, `"int"`, int32(0), []byte("\x00")) 279 | checkCodecEncoderResult(t, `"int"`, int32(1016), []byte("\xf0\x0f")) 280 | checkCodecEncoderResult(t, `"int"`, int32(3), []byte("\x06")) 281 | checkCodecEncoderResult(t, `"int"`, int32(42), []byte("\x54")) 282 | checkCodecEncoderResult(t, `"int"`, int32(64), []byte("\x80\x01")) 283 | checkCodecEncoderResult(t, `"int"`, int32(66052), []byte("\x88\x88\x08")) 284 | checkCodecEncoderResult(t, `"int"`, int32(8454660), []byte("\x88\x88\x88\x08")) 285 | // long 286 | checkCodecEncoderResult(t, `"long"`, int64(-2147483648), []byte("\xff\xff\xff\xff\x0f")) 287 | checkCodecEncoderResult(t, `"long"`, int64(-3), []byte("\x05")) 288 | checkCodecEncoderResult(t, `"long"`, int64(-65), []byte("\x81\x01")) 289 | checkCodecEncoderResult(t, `"long"`, int64(0), []byte("\x00")) 290 | checkCodecEncoderResult(t, `"long"`, int64(1082196484), []byte("\x88\x88\x88\x88\x08")) 291 | checkCodecEncoderResult(t, `"long"`, int64(138521149956), []byte("\x88\x88\x88\x88\x88\x08")) 292 | checkCodecEncoderResult(t, `"long"`, int64(17730707194372), []byte("\x88\x88\x88\x88\x88\x88\x08")) 293 | checkCodecEncoderResult(t, `"long"`, int64(2147483647), []byte("\xfe\xff\xff\xff\x0f")) 294 | checkCodecEncoderResult(t, `"long"`, int64(2269530520879620), []byte("\x88\x88\x88\x88\x88\x88\x88\x08")) 295 | checkCodecEncoderResult(t, `"long"`, int64(3), []byte("\x06")) 296 | checkCodecEncoderResult(t, `"long"`, int64(64), []byte("\x80\x01")) 297 | // float 298 | checkCodecEncoderResult(t, `"float"`, float32(3.5), []byte("\x00\x00\x60\x40")) 299 | checkCodecEncoderResult(t, `"float"`, float32(math.Inf(-1)), []byte("\x00\x00\x80\xff")) 300 | checkCodecEncoderResult(t, `"float"`, float32(math.Inf(1)), []byte("\x00\x00\x80\u007f")) 301 | checkCodecEncoderResult(t, `"float"`, float32(math.NaN()), []byte("\x00\x00\xc0\u007f")) 302 | // double 303 | checkCodecEncoderResult(t, `"double"`, float64(3.5), []byte("\x00\x00\x00\x00\x00\x00\f@")) 304 | checkCodecEncoderResult(t, `"double"`, float64(math.Inf(-1)), []byte("\x00\x00\x00\x00\x00\x00\xf0\xff")) 305 | checkCodecEncoderResult(t, `"double"`, float64(math.Inf(1)), []byte("\x00\x00\x00\x00\x00\x00\xf0\u007f")) 306 | checkCodecEncoderResult(t, `"double"`, float64(math.NaN()), []byte("\x01\x00\x00\x00\x00\x00\xf8\u007f")) 307 | // bytes 308 | checkCodecEncoderResult(t, `"bytes"`, []byte(""), []byte("\x00")) 309 | checkCodecEncoderResult(t, `"bytes"`, []byte("some bytes"), []byte("\x14some bytes")) 310 | // string 311 | checkCodecEncoderResult(t, `"string"`, "", []byte("\x00")) 312 | checkCodecEncoderResult(t, `"string"`, "filibuster", []byte("\x14filibuster")) 313 | } 314 | 315 | func TestCodecUnionChecksSchema(t *testing.T) { 316 | var err error 317 | _, err = NewCodec(`[]`) 318 | checkErrorFatal(t, err, "ought have at least one member") 319 | _, err = NewCodec(`["null","flubber"]`) 320 | checkErrorFatal(t, err, "member ought to be decodable") 321 | } 322 | 323 | func TestCodecUnionPrimitives(t *testing.T) { 324 | // null 325 | checkCodecEncoderResult(t, `["null"]`, nil, []byte("\x00")) 326 | checkCodecEncoderResult(t, `[{"type":"null"}]`, nil, []byte("\x00")) 327 | // boolean 328 | checkCodecEncoderResult(t, `["null","boolean"]`, nil, []byte("\x00")) 329 | checkCodecEncoderResult(t, `["null","boolean"]`, false, []byte("\x02\x00")) 330 | checkCodecEncoderResult(t, `["null","boolean"]`, true, []byte("\x02\x01")) 331 | checkCodecEncoderResult(t, `["boolean","null"]`, true, []byte("\x00\x01")) 332 | // int 333 | checkCodecEncoderResult(t, `["null","int"]`, nil, []byte("\x00")) 334 | checkCodecEncoderResult(t, `["boolean","int"]`, true, []byte("\x00\x01")) 335 | checkCodecEncoderResult(t, `["boolean","int"]`, int32(3), []byte("\x02\x06")) 336 | checkCodecEncoderResult(t, `["int",{"type":"boolean"}]`, int32(42), []byte("\x00\x54")) 337 | // long 338 | checkCodecEncoderResult(t, `["boolean","long"]`, int64(3), []byte("\x02\x06")) 339 | // float 340 | checkCodecEncoderResult(t, `["int","float"]`, float32(3.5), []byte("\x02\x00\x00\x60\x40")) 341 | // double 342 | checkCodecEncoderResult(t, `["float","double"]`, float64(3.5), []byte("\x02\x00\x00\x00\x00\x00\x00\f@")) 343 | // bytes 344 | checkCodecEncoderResult(t, `["int","bytes"]`, []byte("foobar"), []byte("\x02\x0cfoobar")) 345 | // string 346 | checkCodecEncoderResult(t, `["string","float"]`, "filibuster", []byte("\x00\x14filibuster")) 347 | } 348 | 349 | func TestCodecDecoderUnion(t *testing.T) { 350 | checkCodecDecoderResult(t, `["string","float"]`, []byte("\x00\x14filibuster"), "filibuster") 351 | checkCodecDecoderResult(t, `["string","int"]`, []byte("\x02\x1a"), int32(13)) 352 | } 353 | 354 | func TestCodecEncoderUnionArray(t *testing.T) { 355 | checkCodecEncoderResult(t, `[{"type":"array","items":"int"},"string"]`, "filibuster", []byte("\x02\x14filibuster")) 356 | 357 | someArray := make([]interface{}, 0) 358 | someArray = append(someArray, int32(3)) 359 | someArray = append(someArray, int32(13)) 360 | checkCodecEncoderResult(t, `[{"type":"array","items":"int"},"string"]`, someArray, []byte("\x00\x04\x06\x1a\x00")) 361 | } 362 | 363 | func TestCodecEncoderUnionMap(t *testing.T) { 364 | someMap := make(map[string]interface{}) 365 | someMap["superhero"] = "Batman" 366 | checkCodecEncoderResult(t, `["null",{"type":"map","values":"string"}]`, someMap, []byte("\x02\x02\x12superhero\x0cBatman\x00")) 367 | } 368 | 369 | func TestCodecEncoderUnionRecord(t *testing.T) { 370 | recordSchemaJSON := `{"type":"record","name":"record1","fields":[{"type":"int","name":"field1"},{"type":"string","name":"field2"}]}` 371 | 372 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 373 | checkErrorFatal(t, err, nil) 374 | 375 | someRecord.Set("field1", int32(13)) 376 | someRecord.Set("field2", "Superman") 377 | 378 | bits := []byte("\x02\x1a\x10Superman") 379 | checkCodecEncoderResult(t, `["null",`+recordSchemaJSON+`]`, someRecord, bits) 380 | } 381 | 382 | func TestCodecEncoderEnumChecksSchema(t *testing.T) { 383 | var err error 384 | 385 | _, err = NewCodec(`{"type":"enum"}`) 386 | checkError(t, err, "ought to have name key") 387 | 388 | _, err = NewCodec(`{"type":"enum","name":5}`) 389 | checkError(t, err, "name ought to be non-empty string") 390 | 391 | _, err = NewCodec(`{"type":"enum","name":"enum1"}`) 392 | checkError(t, err, "ought to have symbols key") 393 | 394 | _, err = NewCodec(`{"type":"enum","name":"enum1","symbols":5}`) 395 | checkError(t, err, "symbols ought to be non-empty array") 396 | 397 | _, err = NewCodec(`{"type":"enum","name":"enum1","symbols":[]}`) 398 | checkError(t, err, "symbols ought to be non-empty array") 399 | 400 | _, err = NewCodec(`{"type":"enum","name":"enum1","symbols":[5]}`) 401 | checkError(t, err, "symbols array member ought to be string") 402 | } 403 | 404 | func TestCodecDecoderEnum(t *testing.T) { 405 | schema := `{"type":"enum","name":"cards","symbols":["HEARTS","DIAMONDS","SPADES","CLUBS"]}` 406 | checkCodecDecoderError(t, schema, []byte("\x01"), "index must be between 0 and 3") 407 | checkCodecDecoderError(t, schema, []byte("\x08"), "index must be between 0 and 3") 408 | checkCodecDecoderResult(t, schema, []byte("\x04"), "SPADES") 409 | } 410 | 411 | func TestCodecEncoderEnum(t *testing.T) { 412 | schema := `{"type":"enum","name":"cards","symbols":["HEARTS","DIAMONDS","SPADES","CLUBS"]}` 413 | checkCodecEncoderError(t, schema, []byte("\x01"), "expected: string; received: []uint8") 414 | checkCodecEncoderError(t, schema, "some symbol not in schema", "symbol not defined") 415 | checkCodecEncoderResult(t, schema, "SPADES", []byte("\x04")) 416 | } 417 | 418 | func TestCodecFixedChecksSchema(t *testing.T) { 419 | var err error 420 | 421 | _, err = NewCodec(`{"type":"fixed","size":5}`) 422 | checkError(t, err, "ought to have name key") 423 | 424 | _, err = NewCodec(`{"type":"fixed","name":5,"size":5}`) 425 | checkError(t, err, "name ought to be non-empty string") 426 | 427 | _, err = NewCodec(`{"type":"fixed","name":"fixed1"}`) 428 | checkError(t, err, "ought to have size key") 429 | 430 | _, err = NewCodec(`{"type":"fixed","name":"fixed1","size":"5"}`) 431 | checkError(t, err, "size ought to be number") 432 | } 433 | 434 | func TestCodecFixed(t *testing.T) { 435 | schema := `{"type":"fixed","name":"fixed1","size":5}` 436 | checkCodecDecoderError(t, schema, []byte(""), "EOF") 437 | checkCodecDecoderError(t, schema, []byte("hap"), "buffer underrun") 438 | checkCodecEncoderError(t, schema, "happy day", "expected: []byte; received: string") 439 | checkCodecEncoderError(t, schema, []byte("day"), "expected: 5 bytes; received: 3") 440 | checkCodecEncoderError(t, schema, []byte("happy day"), "expected: 5 bytes; received: 9") 441 | checkCodecEncoderResult(t, schema, []byte("happy"), []byte("happy")) 442 | } 443 | 444 | func TestCodecNamedTypes(t *testing.T) { 445 | schema := `{"name":"guid","type":{"type":"fixed","name":"fixed_16","size":16},"doc":"event unique id"}` 446 | var err error 447 | _, err = NewCodec(schema) 448 | checkError(t, err, nil) 449 | } 450 | 451 | func TestCodecReferToNamedTypes(t *testing.T) { 452 | schema := `{"type":"record","name":"record1","fields":[{"name":"guid","type":{"type":"fixed","name":"fixed_16","size":16},"doc":"event unique id"},{"name":"treeId","type":"fixed_16","doc":"call tree uuid"}]}` 453 | _, err := NewCodec(schema) 454 | checkError(t, err, nil) 455 | } 456 | 457 | func TestCodecRecordFieldDefaultValueNamedType(t *testing.T) { 458 | schemaJSON := `{"type":"record","name":"record1","fields":[{"type":"fixed","name":"fixed_16","size":16},{"type":"fixed_16","name":"another","default":3}]}` 459 | _, err := NewCodec(schemaJSON) 460 | checkError(t, err, nil) 461 | } 462 | 463 | func TestCodecRecordFieldChecksDefaultType(t *testing.T) { 464 | recordSchemaJSON := `{"type":"record","name":"record1","fields":[{"type":"int","name":"field1","default":true},{"type":"string","name":"field2"}]}` 465 | _, err := NewCodec(recordSchemaJSON) 466 | checkError(t, err, "expected: int32; received: bool") 467 | } 468 | 469 | func TestCodecEncoderArrayChecksSchema(t *testing.T) { 470 | _, err := NewCodec(`{"type":"array"}`) 471 | checkErrorFatal(t, err, "ought to have items key") 472 | 473 | _, err = NewCodec(`{"type":"array","items":"flubber"}`) 474 | checkErrorFatal(t, err, "unknown type name") 475 | 476 | checkCodecEncoderError(t, `{"type":"array","items":"long"}`, int64(5), "expected: []interface{}; received: int64") 477 | } 478 | 479 | func TestCodecDecoderArrayEOF(t *testing.T) { 480 | schema := `{"type":"array","items":"string"}` 481 | checkCodecDecoderError(t, schema, []byte(""), "cannot decode long: EOF") 482 | } 483 | 484 | func TestCodecDecoderArrayEmpty(t *testing.T) { 485 | schema := `{"type":"array","items":"string"}` 486 | decoder, err := NewCodec(schema) 487 | checkErrorFatal(t, err, nil) 488 | 489 | bb := bytes.NewBuffer([]byte{0}) 490 | actual, err := decoder.Decode(bb) 491 | checkError(t, err, nil) 492 | 493 | someArray, ok := actual.([]interface{}) 494 | if !ok { 495 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 496 | } 497 | if len(someArray) != 0 { 498 | t.Errorf("Actual: %#v; Expected: %#v", len(someArray), 0) 499 | } 500 | } 501 | 502 | func TestCodecDecoderArray(t *testing.T) { 503 | schema := `{"type":"array","items":"int"}` 504 | decoder, err := NewCodec(schema) 505 | checkErrorFatal(t, err, nil) 506 | 507 | bb := bytes.NewBuffer([]byte("\x04\x06\x36\x00")) 508 | actual, err := decoder.Decode(bb) 509 | checkError(t, err, nil) 510 | 511 | someArray, ok := actual.([]interface{}) 512 | if !ok { 513 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 514 | } 515 | expected := []int32{3, 27} 516 | if len(someArray) != len(expected) { 517 | t.Errorf("Actual: %#v; Expected: %#v", len(someArray), len(expected)) 518 | } 519 | if len(someArray) != len(expected) { 520 | t.Errorf("Actual: %#v; Expected: %#v", len(someArray), len(expected)) 521 | } 522 | for i, v := range someArray { 523 | val, ok := v.(int32) 524 | if !ok { 525 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 526 | } 527 | if val != expected[i] { 528 | t.Errorf("Actual: %#v; Expected: %#v", val, expected[i]) 529 | } 530 | } 531 | } 532 | 533 | func TestCodecDecoderArrayOfRecords(t *testing.T) { 534 | schema := ` 535 | { 536 | "type": "array", 537 | "items": { 538 | "type": "record", 539 | "name": "someRecord", 540 | "fields": [ 541 | { 542 | "name": "someString", 543 | "type": "string" 544 | }, 545 | { 546 | "name": "someInt", 547 | "type": "int" 548 | } 549 | ] 550 | } 551 | } 552 | ` 553 | decoder, err := NewCodec(schema) 554 | checkErrorFatal(t, err, nil) 555 | 556 | encoded := []byte("\x04\x0aHello\x1a\x0aWorld\x54\x00") 557 | bb := bytes.NewBuffer(encoded) 558 | actual, err := decoder.Decode(bb) 559 | checkError(t, err, nil) 560 | 561 | someArray, ok := actual.([]interface{}) 562 | if !ok { 563 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 564 | } 565 | if len(someArray) != 2 { 566 | t.Errorf("Actual: %#v; Expected: %#v", len(someArray), 2) 567 | } 568 | // first element 569 | actualString, err := someArray[0].(*Record).Get("someString") 570 | checkError(t, err, nil) 571 | expectedString := "Hello" 572 | if actualString != expectedString { 573 | t.Errorf("Actual: %#v; Expected: %#v", actualString, expectedString) 574 | } 575 | actualInt, err := someArray[0].(*Record).Get("someInt") 576 | checkError(t, err, nil) 577 | expectedInt := int32(13) 578 | if actualInt != expectedInt { 579 | t.Errorf("Actual: %#v; Expected: %#v", actualInt, expectedInt) 580 | } 581 | // second element 582 | actualString, err = someArray[1].(*Record).Get("someString") 583 | checkError(t, err, nil) 584 | expectedString = "World" 585 | if actualString != expectedString { 586 | t.Errorf("Actual: %#v; Expected: %#v", actualString, expectedString) 587 | } 588 | actualInt, err = someArray[1].(*Record).Get("someInt") 589 | checkError(t, err, nil) 590 | expectedInt = int32(42) 591 | if actualInt != expectedInt { 592 | t.Errorf("Actual: %#v; Expected: %#v", actualInt, expectedInt) 593 | } 594 | } 595 | 596 | func TestCodecDecoderArrayMultipleBlocks(t *testing.T) { 597 | schema := `{"type":"array","items":"int"}` 598 | decoder, err := NewCodec(schema) 599 | checkErrorFatal(t, err, nil) 600 | 601 | bb := bytes.NewBuffer([]byte("\x06\x06\x08\x0a\x03\x04\x36\x0c\x00")) 602 | actual, err := decoder.Decode(bb) 603 | checkError(t, err, nil) 604 | 605 | someArray, ok := actual.([]interface{}) 606 | if !ok { 607 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 608 | } 609 | expected := []int32{3, 4, 5, 27, 6} 610 | if len(someArray) != len(expected) { 611 | t.Errorf("Actual: %#v; Expected: %#v", len(someArray), len(expected)) 612 | } 613 | for i, v := range someArray { 614 | val, ok := v.(int32) 615 | if !ok { 616 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 617 | } 618 | if val != expected[i] { 619 | t.Errorf("Actual: %#v; Expected: %#v", val, expected[i]) 620 | } 621 | } 622 | } 623 | 624 | func TestCodecEncoderArray(t *testing.T) { 625 | schema := `{"type":"array","items":{"type":"long"}}` 626 | 627 | datum := make([]interface{}, 0) 628 | datum = append(datum, int64(-1)) 629 | datum = append(datum, int64(-2)) 630 | datum = append(datum, int64(-3)) 631 | datum = append(datum, int64(-4)) 632 | datum = append(datum, int64(-5)) 633 | datum = append(datum, int64(-6)) 634 | datum = append(datum, int64(0)) 635 | datum = append(datum, int64(1)) 636 | datum = append(datum, int64(2)) 637 | datum = append(datum, int64(3)) 638 | datum = append(datum, int64(4)) 639 | datum = append(datum, int64(5)) 640 | datum = append(datum, int64(6)) 641 | 642 | bits := []byte{ 643 | 20, 644 | 1, 3, 5, 7, 9, 11, 0, 2, 4, 6, 645 | 6, 646 | 8, 10, 12, 647 | 0, 648 | } 649 | 650 | checkCodecEncoderResult(t, schema, datum, bits) 651 | } 652 | 653 | func TestCodecMapChecksSchema(t *testing.T) { 654 | _, err := NewCodec(`{"type":"map"}`) 655 | checkErrorFatal(t, err, "ought to have values key") 656 | 657 | _, err = NewCodec(`{"type":"map","values":"flubber"}`) 658 | checkErrorFatal(t, err, "unknown type name") 659 | 660 | checkCodecEncoderError(t, `{"type":"map","values":"long"}`, int64(5), "expected: map[string]interface{}; received: int64") 661 | checkCodecEncoderError(t, `{"type":"map","values":"string"}`, 3, "expected: map[string]interface{}; received: int") 662 | } 663 | 664 | func TestCodecDecoderMapEOF(t *testing.T) { 665 | schema := `{"type":"map","values":"string"}` 666 | checkCodecDecoderError(t, schema, []byte(""), "cannot decode long: EOF") 667 | } 668 | 669 | func TestCodecDecoderMapZeroBlocks(t *testing.T) { 670 | decoder, err := NewCodec(`{"type":"map","values":"string"}`) 671 | checkErrorFatal(t, err, nil) 672 | 673 | bb := bytes.NewBuffer([]byte("\x00")) 674 | actual, err := decoder.Decode(bb) 675 | checkErrorFatal(t, err, nil) 676 | 677 | someMap, ok := actual.(map[string]interface{}) 678 | if !ok { 679 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 680 | } 681 | if len(someMap) != 0 { 682 | t.Errorf(`received: %v; Expected: %v`, len(someMap), 0) 683 | } 684 | } 685 | 686 | func TestCodecDecoderMapReturnsExpectedMap(t *testing.T) { 687 | decoder, err := NewCodec(`{"type":"map","values":"string"}`) 688 | checkErrorFatal(t, err, nil) 689 | 690 | bb := bytes.NewBuffer([]byte("\x01\x04\x06\x66\x6f\x6f\x06\x42\x41\x52\x00")) 691 | actual, err := decoder.Decode(bb) 692 | checkErrorFatal(t, err, nil) 693 | 694 | someMap, ok := actual.(map[string]interface{}) 695 | if !ok { 696 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 697 | } 698 | if len(someMap) != 1 { 699 | t.Errorf(`received: %v; Expected: %v`, len(someMap), 1) 700 | } 701 | datum, ok := someMap["foo"] 702 | if !ok { 703 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 704 | } 705 | someString, ok := datum.(string) 706 | if !ok { 707 | t.Errorf("Actual: %#v; Expected: %#v", ok, true) 708 | } 709 | if someString != "BAR" { 710 | t.Errorf("Actual: %#v; Expected: %#v", someString, "BAR") 711 | } 712 | } 713 | 714 | func TestCodecEncoderMapChecksValueTypeDuringWrite(t *testing.T) { 715 | schema := `{"type":"map","values":"string"}` 716 | datum := make(map[string]interface{}) 717 | datum["name"] = 13 718 | checkCodecEncoderError(t, schema, datum, "expected: string; received: int") 719 | } 720 | 721 | func TestCodecEncoderMapMetadataSchema(t *testing.T) { 722 | md := make(map[string]interface{}) 723 | md["avro.codec"] = []byte("null") 724 | md["avro.schema"] = []byte(`"int"`) 725 | 726 | // NOTE: because key value pair ordering is indeterminate, 727 | // there are two valid possibilities for the encoded map: 728 | option1 := []byte("\x04\x14avro.codec\x08null\x16avro.schema\x0a\x22int\x22\x00") 729 | option2 := []byte("\x04\x16avro.schema\x0a\x22int\x22\x14avro.codec\x08null\x00") 730 | 731 | bb := new(bytes.Buffer) 732 | err := metadataCodec.Encode(bb, md) 733 | checkErrorFatal(t, err, nil) 734 | actual := bb.Bytes() 735 | if (bytes.Compare(actual, option1) != 0) && (bytes.Compare(actual, option2) != 0) { 736 | t.Errorf("Actual: %#v; Expected: %#v", actual, option1) 737 | } 738 | } 739 | 740 | func TestCodecRecordChecksSchema(t *testing.T) { 741 | var err error 742 | 743 | _, err = NewCodec(`{"type":"record","fields":[{"name":"age","type":"int"},{"name":"status","type":"string"}]}`) 744 | checkError(t, err, "ought to have name key") 745 | 746 | _, err = NewCodec(`{"type":"record","name":5,"fields":[{"name":"age","type":"int"},{"name":"status","type":"string"}]}`) 747 | checkError(t, err, "name ought to be non-empty string") 748 | 749 | _, err = NewCodec(`{"type":"record","name":"Foo"}`) 750 | checkError(t, err, "record requires one or more fields") 751 | 752 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":5}`) 753 | checkError(t, err, "fields ought to be non-empty array") 754 | 755 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":[]}`) 756 | checkError(t, err, "fields ought to be non-empty array") 757 | 758 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":["foo"]}`) 759 | checkError(t, err, "schema expected") 760 | 761 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":[{"type":"int"}]}`) 762 | checkError(t, err, "ought to have name key") 763 | 764 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":[{"name":"field1","type":5}]}`) 765 | checkError(t, err, "type ought to be") 766 | 767 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":[{"type":"int"}]}`) 768 | checkError(t, err, "ought to have name key") 769 | 770 | _, err = NewCodec(`{"type":"record","name":"Foo","fields":[{"type":"int","name":5}]}`) 771 | checkError(t, err, "name ought to be non-empty string") 772 | } 773 | 774 | func TestCodecDecoderRecord(t *testing.T) { 775 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"age","type":"int"},{"name":"status","type":"string"}]}` 776 | 777 | decoder, err := NewCodec(recordSchemaJSON) 778 | checkErrorFatal(t, err, nil) 779 | 780 | bits := []byte("\x80\x01\x0ahappy") 781 | bb := bytes.NewBuffer(bits) 782 | 783 | actual, err := decoder.Decode(bb) 784 | checkErrorFatal(t, err, nil) 785 | 786 | decoded, ok := actual.(*Record) 787 | if !ok { 788 | t.Fatalf("Actual: %T; Expected: Record", actual) 789 | } 790 | 791 | if decoded.Name != "Foo" { 792 | t.Errorf("Actual: %#v; Expected: %#v", decoded.Name, "Foo") 793 | } 794 | if decoded.Fields[0].Datum != int32(64) { 795 | t.Errorf("Actual: %#v; Expected: %#v", decoded.Fields[0].Datum, int32(64)) 796 | } 797 | if decoded.Fields[1].Datum != "happy" { 798 | t.Errorf("Actual: %#v; Expected: %#v", decoded.Fields[1].Datum, "happy") 799 | } 800 | } 801 | 802 | func TestCodecEncoderRecord(t *testing.T) { 803 | recordSchemaJSON := `{"type":"record","name":"comments","namespace":"com.example","fields":[{"name":"username","type":"string","doc":"Name of user"},{"name":"comment","type":"string","doc":"The content of the user's message"},{"name":"timestamp","type":"long","doc":"Unix epoch time in milliseconds"}],"doc:":"A basic schema for storing blog comments"}` 804 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 805 | checkErrorFatal(t, err, nil) 806 | 807 | someRecord.Set("username", "Aquaman") 808 | someRecord.Set("comment", "The Atlantic is oddly cold this morning!") 809 | someRecord.Set("timestamp", int64(1082196484)) 810 | 811 | bits := []byte("\x0eAquamanPThe Atlantic is oddly cold this morning!\x88\x88\x88\x88\x08") 812 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 813 | } 814 | 815 | func TestCodecEncoderRecordWithFieldDefaultNull(t *testing.T) { 816 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int"},{"name":"field2","type":["null","string"],"default":null}]}` 817 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 818 | checkErrorFatal(t, err, nil) 819 | 820 | someRecord.Set("field1", int32(42)) 821 | bits := []byte("\x54\x00") 822 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 823 | } 824 | 825 | func TestCodecEncoderRecordWithFieldDefaultBoolean(t *testing.T) { 826 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"boolean","default":true}]}` 827 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 828 | checkErrorFatal(t, err, nil) 829 | 830 | someRecord.Set("field1", int32(64)) 831 | 832 | bits := []byte("\x80\x01\x01") 833 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 834 | } 835 | 836 | func TestCodecEncoderRecordWithFieldDefaultInt(t *testing.T) { 837 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int","default":3}]}` 838 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 839 | checkErrorFatal(t, err, nil) 840 | 841 | bits := []byte("\x06") 842 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 843 | } 844 | 845 | func TestCodecEncoderRecordWithFieldDefaultLong(t *testing.T) { 846 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"long","default":3}]}` 847 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 848 | checkErrorFatal(t, err, nil) 849 | 850 | bits := []byte("\x06") 851 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 852 | } 853 | 854 | func TestCodecEncoderRecordWithFieldDefaultFloat(t *testing.T) { 855 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"float","default":3.5}]}` 856 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 857 | checkErrorFatal(t, err, nil) 858 | 859 | bits := []byte("\x00\x00\x60\x40") 860 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 861 | } 862 | 863 | func TestCodecEncoderRecordWithFieldDefaultDouble(t *testing.T) { 864 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"double","default":3.5}]}` 865 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 866 | checkErrorFatal(t, err, nil) 867 | 868 | bits := []byte("\x00\x00\x00\x00\x00\x00\f@") 869 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 870 | } 871 | 872 | func TestCodecEncoderRecordWithFieldDefaultBytes(t *testing.T) { 873 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"bytes","default":"happy"}]}` 874 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 875 | checkErrorFatal(t, err, nil) 876 | 877 | someRecord.Set("field1", int32(64)) 878 | 879 | bits := []byte("\x80\x01\x0ahappy") 880 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 881 | } 882 | 883 | func TestCodecEncoderRecordWithFieldDefaultString(t *testing.T) { 884 | recordSchemaJSON := `{"type":"record","name":"Foo","fields":[{"name":"field1","type":"int"},{"name":"field2","type":"string","default":"happy"}]}` 885 | someRecord, err := NewRecord(RecordSchema(recordSchemaJSON)) 886 | checkErrorFatal(t, err, nil) 887 | 888 | someRecord.Set("field1", int32(64)) 889 | 890 | bits := []byte("\x80\x01\x0ahappy") 891 | checkCodecEncoderResult(t, recordSchemaJSON, someRecord, bits) 892 | } 893 | 894 | //////////////////////////////////////// 895 | 896 | func TestBufferedEncoder(t *testing.T) { 897 | bits, err := bufferedEncoder(`"string"`, "filibuster") 898 | if err != nil { 899 | t.Fatal(err) 900 | } 901 | expected := []byte("\x14filibuster") 902 | if bytes.Compare(bits, expected) != 0 { 903 | t.Errorf("Actual: %#v; Expected: %#v", bits, expected) 904 | } 905 | } 906 | 907 | func bufferedEncoder(someSchemaJSON string, datum interface{}) (bits []byte, err error) { 908 | bb := new(bytes.Buffer) 909 | defer func() { 910 | bits = bb.Bytes() 911 | }() 912 | 913 | var c Codec 914 | c, err = NewCodec(someSchemaJSON) 915 | if err != nil { 916 | return 917 | } 918 | err = encodeWithBufferedWriter(c, bb, datum) 919 | return 920 | } 921 | 922 | func encodeWithBufferedWriter(c Codec, w io.Writer, datum interface{}) error { 923 | bw := bufio.NewWriter(w) 924 | err := c.Encode(bw, datum) 925 | if err != nil { 926 | return err 927 | } 928 | return bw.Flush() 929 | } 930 | --------------------------------------------------------------------------------