├── .gitignore ├── .travis.yml ├── Makefile ├── README.md ├── decoders.go ├── examples ├── u2bench.go ├── u2extract.go └── u2json.go ├── recordreader.go ├── spoolrecordreader.go ├── spoolrecordreader_example_test.go ├── spoolrecordreader_test.go ├── test ├── multi-record-event-x2.log ├── multi-record-event.log ├── short-read-on-body.log └── short-read-on-header.log ├── unified2.go ├── unified2_example_test.go └── unified2_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /.idea 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - 1.2 5 | - 1.7 6 | - 1.8 7 | 8 | script: 9 | - go test -v 10 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: test 2 | 3 | all: 4 | go build 5 | cd examples && go build u2bench.go 6 | cd examples && go build u2extract.go 7 | 8 | test: 9 | go test 10 | 11 | # Test with coverage. 12 | test-coverage: 13 | go test -coverprofile cover.out 14 | go tool cover -func=cover.out 15 | 16 | clean: 17 | go clean 18 | find . -name \*~ -exec rm -f {} \; 19 | rm -f examples/u2bench 20 | rm -f examples/u2extract 21 | rm -f cover.out 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-unified2 [![GoDoc](https://godoc.org/github.com/jasonish/go-unified2?status.png)](https://godoc.org/github.com/jasonish/go-unified2) 2 | 3 | A Go(lang) Library for decoding unified2 log files as generated by IDS 4 | applications such as Snort and Suricata. 5 | 6 | ## Installation 7 | 8 | ``` 9 | go get github.com/jasonish/go-unified2 10 | ``` 11 | 12 | ## Documentation 13 | 14 | See https://godoc.org/github.com/jasonish/go-unified2 15 | 16 | For more information on the unified2 file format see the 17 | [Snort Manual](http://manual.snort.org/node44.html). 18 | -------------------------------------------------------------------------------- /decoders.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2013 Jason Ish 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 22 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 23 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | package unified2 28 | 29 | import ( 30 | "bytes" 31 | "encoding/binary" 32 | "errors" 33 | "io" 34 | "log" 35 | ) 36 | 37 | // DecodingError is the error returned if an error is encountered 38 | // while decoding a record buffer. 39 | // 40 | // We use this error to differentiate between file level reading 41 | // errors. 42 | var DecodingError = errors.New("DecodingError") 43 | 44 | // Helper function for reading binary data as all reads are big 45 | // endian. 46 | func read(reader io.Reader, data interface{}) error { 47 | return binary.Read(reader, binary.BigEndian, data) 48 | } 49 | 50 | // DecodeEventRecord decodes a raw record into an EventRecord. 51 | // 52 | // This function will decode any of the event record types. 53 | func DecodeEventRecord(eventType uint32, data []byte) (*EventRecord, error) { 54 | 55 | event := &EventRecord{} 56 | 57 | reader := bytes.NewBuffer(data) 58 | 59 | // SensorId 60 | if err := read(reader, &event.SensorId); err != nil { 61 | return nil, err 62 | } 63 | if err := read(reader, &event.EventId); err != nil { 64 | return nil, err 65 | } 66 | if err := read(reader, &event.EventSecond); err != nil { 67 | return nil, err 68 | } 69 | if err := read(reader, &event.EventMicrosecond); err != nil { 70 | return nil, err 71 | } 72 | 73 | /* SignatureId */ 74 | if err := read(reader, &event.SignatureId); err != nil { 75 | return nil, err 76 | } 77 | 78 | /* GeneratorId */ 79 | if err := read(reader, &event.GeneratorId); err != nil { 80 | return nil, err 81 | } 82 | 83 | /* SignatureRevision */ 84 | if err := read(reader, &event.SignatureRevision); err != nil { 85 | return nil, err 86 | } 87 | 88 | /* ClassificationId */ 89 | if err := read(reader, &event.ClassificationId); err != nil { 90 | return nil, err 91 | } 92 | 93 | /* Priority */ 94 | if err := read(reader, &event.Priority); err != nil { 95 | return nil, err 96 | } 97 | 98 | /* Source and destination IP addresses. */ 99 | switch eventType { 100 | 101 | case UNIFIED2_EVENT, UNIFIED2_EVENT_V2, UNIFIED2_EVENT_APPID: 102 | event.IpSource = make([]byte, 4) 103 | if err := read(reader, &event.IpSource); err != nil { 104 | log.Fatal(err) 105 | return nil, err 106 | } 107 | event.IpDestination = make([]byte, 4) 108 | if err := read(reader, &event.IpDestination); err != nil { 109 | return nil, err 110 | } 111 | 112 | case UNIFIED2_EVENT_IP6, UNIFIED2_EVENT_V2_IP6, UNIFIED2_EVENT_APPID_IP6: 113 | event.IpSource = make([]byte, 16) 114 | if err := read(reader, &event.IpSource); err != nil { 115 | return nil, err 116 | } 117 | event.IpDestination = make([]byte, 16) 118 | if err := read(reader, &event.IpDestination); err != nil { 119 | return nil, err 120 | } 121 | } 122 | 123 | /* Source port/ICMP type. */ 124 | if err := read(reader, &event.SportItype); err != nil { 125 | return nil, err 126 | } 127 | 128 | /* Destination port/ICMP code. */ 129 | if err := read(reader, &event.DportIcode); err != nil { 130 | return nil, err 131 | } 132 | 133 | /* Protocol. */ 134 | if err := read(reader, &event.Protocol); err != nil { 135 | return nil, err 136 | } 137 | 138 | /* Impact flag. */ 139 | if err := read(reader, &event.ImpactFlag); err != nil { 140 | return nil, err 141 | } 142 | 143 | /* Impact. */ 144 | if err := read(reader, &event.Impact); err != nil { 145 | return nil, err 146 | } 147 | 148 | /* Blocked. */ 149 | if err := read(reader, &event.Blocked); err != nil { 150 | return nil, err 151 | } 152 | 153 | switch eventType { 154 | case UNIFIED2_EVENT_V2, 155 | UNIFIED2_EVENT_V2_IP6, 156 | UNIFIED2_EVENT_APPID, 157 | UNIFIED2_EVENT_APPID_IP6: 158 | 159 | /* MplsLabel. */ 160 | if err := read(reader, &event.MplsLabel); err != nil { 161 | return nil, err 162 | } 163 | 164 | /* VlanId. */ 165 | if err := read(reader, &event.VlanId); err != nil { 166 | return nil, err 167 | } 168 | 169 | /* Pad2. */ 170 | if err := read(reader, &event.Pad2); err != nil { 171 | return nil, err 172 | } 173 | } 174 | 175 | // Any remaining data is the appid. 176 | appid := make([]byte, 64) 177 | n, err := reader.Read(appid) 178 | if err == nil { 179 | end := bytes.IndexByte(appid, 0) 180 | if end < 0 { 181 | end = n 182 | } 183 | event.AppId = string(appid[0:end]) 184 | } 185 | 186 | return event, nil 187 | } 188 | 189 | // DecodePacketRecord decodes a raw unified2 record into a 190 | // PacketRecord. 191 | func DecodePacketRecord(data []byte) (packet *PacketRecord, err error) { 192 | 193 | packet = &PacketRecord{} 194 | 195 | reader := bytes.NewBuffer(data) 196 | 197 | if err = read(reader, &packet.SensorId); err != nil { 198 | goto error 199 | } 200 | 201 | if err = read(reader, &packet.EventId); err != nil { 202 | goto error 203 | } 204 | 205 | if err = read(reader, &packet.EventSecond); err != nil { 206 | goto error 207 | } 208 | 209 | if err = read(reader, &packet.PacketSecond); err != nil { 210 | goto error 211 | } 212 | 213 | if err = read(reader, &packet.PacketMicrosecond); err != nil { 214 | goto error 215 | } 216 | 217 | if err = read(reader, &packet.LinkType); err != nil { 218 | goto error 219 | } 220 | 221 | if err = read(reader, &packet.Length); err != nil { 222 | goto error 223 | } 224 | 225 | packet.Data = data[PACKET_RECORD_HDR_LEN:] 226 | 227 | return packet, nil 228 | 229 | error: 230 | return nil, DecodingError 231 | } 232 | 233 | // DecodeExtraDataRecord decodes a raw extra data record into an 234 | // ExtraDataRecord. 235 | func DecodeExtraDataRecord(data []byte) (extra *ExtraDataRecord, err error) { 236 | 237 | extra = &ExtraDataRecord{} 238 | 239 | reader := bytes.NewBuffer(data) 240 | 241 | if err = read(reader, &extra.EventType); err != nil { 242 | goto error 243 | } 244 | 245 | if err = read(reader, &extra.EventLength); err != nil { 246 | goto error 247 | } 248 | 249 | if err = read(reader, &extra.SensorId); err != nil { 250 | goto error 251 | } 252 | 253 | if err = read(reader, &extra.EventId); err != nil { 254 | goto error 255 | } 256 | 257 | if err = read(reader, &extra.EventSecond); err != nil { 258 | goto error 259 | } 260 | 261 | if err = read(reader, &extra.Type); err != nil { 262 | goto error 263 | } 264 | 265 | if err = read(reader, &extra.DataType); err != nil { 266 | goto error 267 | } 268 | 269 | if err = read(reader, &extra.DataLength); err != nil { 270 | goto error 271 | } 272 | 273 | extra.Data = data[EXTRA_DATA_RECORD_HDR_LEN:] 274 | 275 | return extra, nil 276 | 277 | error: 278 | return nil, DecodingError 279 | } 280 | -------------------------------------------------------------------------------- /examples/u2bench.go: -------------------------------------------------------------------------------- 1 | // Benchmark reading and decoding unified2 records. 2 | package main 3 | 4 | import "os" 5 | import "fmt" 6 | import "io" 7 | import "flag" 8 | import "time" 9 | import "github.com/jasonish/go-unified2" 10 | 11 | type stats struct { 12 | Events int 13 | Packets int 14 | ExtraData int 15 | } 16 | 17 | func main() { 18 | 19 | flag.Parse() 20 | args := flag.Args() 21 | 22 | startTime := time.Now() 23 | var recordCount int 24 | var stats stats 25 | for _, arg := range args { 26 | 27 | fmt.Println("Opening", arg) 28 | file, err := os.Open(arg) 29 | if err != nil { 30 | fmt.Println("error opening ", arg, ":", err) 31 | os.Exit(1) 32 | } 33 | 34 | for { 35 | record, err := unified2.ReadRecord(file) 36 | if err != nil { 37 | if err != io.EOF { 38 | fmt.Println("failed to read record:", err) 39 | } 40 | break 41 | } 42 | recordCount++ 43 | 44 | switch record.(type) { 45 | case *unified2.EventRecord: 46 | stats.Events++ 47 | case *unified2.PacketRecord: 48 | stats.Packets++ 49 | case *unified2.ExtraDataRecord: 50 | stats.ExtraData++ 51 | } 52 | } 53 | 54 | file.Close() 55 | } 56 | 57 | elapsedTime := time.Now().Sub(startTime) 58 | perSecond := float64(recordCount) / elapsedTime.Seconds() 59 | 60 | fmt.Printf("Records: %d; Time: %s; Records/sec: %d\n", 61 | recordCount, elapsedTime, int(perSecond)) 62 | fmt.Printf(" Events: %d; Packets: %d; ExtraData: %d\n", 63 | stats.Events, stats.Packets, stats.ExtraData) 64 | } 65 | -------------------------------------------------------------------------------- /examples/u2extract.go: -------------------------------------------------------------------------------- 1 | // Extract events from a unified2 log file with the specified event-id 2 | // and event-second. 3 | package main 4 | 5 | import "os" 6 | import "flag" 7 | import "log" 8 | import "io" 9 | import "github.com/jasonish/go-unified2" 10 | import "encoding/binary" 11 | 12 | func writeRecord(out *os.File, record *unified2.RawRecord) (err error) { 13 | 14 | recordType := record.Type 15 | recordLen := uint32(len(record.Data)) 16 | 17 | err = binary.Write(out, binary.BigEndian, &recordType) 18 | if err != nil { 19 | return err 20 | } 21 | 22 | err = binary.Write(out, binary.BigEndian, &recordLen) 23 | if err != nil { 24 | return err 25 | } 26 | 27 | n, err := out.Write(record.Data) 28 | if err != nil { 29 | return err 30 | } else if n != len(record.Data) { 31 | return io.ErrShortWrite 32 | } 33 | 34 | return nil 35 | } 36 | 37 | func main() { 38 | 39 | var filterEventId uint 40 | var eventSecond uint 41 | 42 | flag.UintVar(&filterEventId, "event-id", 0, "filter on event-id") 43 | flag.UintVar(&eventSecond, "event-second", 0, "filter on event-secon") 44 | flag.Parse() 45 | 46 | if filterEventId == 0 || eventSecond == 0 { 47 | log.Fatalf("error: both -event-id and -event-second must be specified") 48 | } 49 | 50 | args := flag.Args() 51 | 52 | var written uint 53 | 54 | for _, arg := range args { 55 | 56 | file, err := os.Open(arg) 57 | if err != nil { 58 | log.Fatal(err) 59 | } 60 | 61 | var currentEvent *unified2.EventRecord 62 | 63 | for { 64 | /* Want to read the raw record and decode it separately, 65 | /* so we can write out the raw records. */ 66 | 67 | raw, err := unified2.ReadRawRecord(file) 68 | if err != nil { 69 | if err == io.EOF { 70 | break 71 | } 72 | log.Fatal(err) 73 | } 74 | 75 | switch raw.Type { 76 | case unified2.UNIFIED2_EVENT, 77 | unified2.UNIFIED2_EVENT_IP6, 78 | unified2.UNIFIED2_EVENT_V2, 79 | unified2.UNIFIED2_EVENT_V2_IP6: 80 | event, err := unified2.DecodeEventRecord(raw.Type, raw.Data) 81 | if err != nil { 82 | log.Fatalf("failed to decode event") 83 | } 84 | 85 | /* Filter. */ 86 | if uint32(filterEventId) == event.EventId && 87 | uint32(eventSecond) == event.EventSecond { 88 | currentEvent = event 89 | } else { 90 | currentEvent = nil 91 | } 92 | } 93 | 94 | if currentEvent != nil { 95 | writeRecord(os.Stdout, raw) 96 | written++ 97 | } 98 | 99 | } 100 | 101 | file.Close() 102 | 103 | } 104 | 105 | log.Printf("Records written: %d\n", written) 106 | 107 | } 108 | -------------------------------------------------------------------------------- /examples/u2json.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2013 Jason Ish 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 22 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 23 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | package main 28 | 29 | import ( 30 | "encoding/json" 31 | "github.com/jasonish/go-unified2" 32 | "log" 33 | "os" 34 | ) 35 | 36 | func main() { 37 | 38 | file, err := os.Open(os.Args[1]) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | encoder := json.NewEncoder(os.Stdout) 44 | 45 | for { 46 | record, err := unified2.ReadRecord(file) 47 | if err != nil { 48 | log.Fatal(err) 49 | } 50 | if record == nil { 51 | log.Fatal("Record is nil.") 52 | } 53 | encoder.Encode(record) 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /recordreader.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2013 Jason Ish 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 22 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 23 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | package unified2 28 | 29 | import ( 30 | "log" 31 | "os" 32 | ) 33 | 34 | // RecordReader reads and decodes unified2 records from a file. 35 | // 36 | // RecordReaders should be created with NewRecordReader(). 37 | type RecordReader struct { 38 | File *os.File 39 | } 40 | 41 | // NewRecordReader creates a new RecordReader using the provided 42 | // filename and starting at the provided offset. 43 | func NewRecordReader(filename string, offset int64) (*RecordReader, error) { 44 | 45 | file, err := os.Open(filename) 46 | if err != nil { 47 | return nil, err 48 | } 49 | 50 | if offset > 0 { 51 | ret, err := file.Seek(offset, 0) 52 | if err != nil { 53 | log.Printf("Failed to seek to offset %d: %s", offset, err) 54 | file.Close() 55 | return nil, err 56 | } else if ret != offset { 57 | log.Printf("Failed to seek to offset %d: current offset: %s", 58 | offset, ret) 59 | file.Close() 60 | return nil, err 61 | } 62 | } 63 | 64 | return &RecordReader{file}, nil 65 | } 66 | 67 | // Next reads and returns the next unified2 record. The record is 68 | // returned as an interface{} which will be one of the types 69 | // EventRecord, PacketRecord or ExtraDataRecord. 70 | func (r *RecordReader) Next() (interface{}, error) { 71 | return ReadRecord(r.File) 72 | } 73 | 74 | // Close closes this reader and the underlying file. 75 | func (r *RecordReader) Close() { 76 | r.File.Close() 77 | } 78 | 79 | // Offset returns the current offset of this reader. 80 | func (r *RecordReader) Offset() int64 { 81 | offset, err := r.File.Seek(0, 1) 82 | if err != nil { 83 | return 0 84 | } 85 | return offset 86 | } 87 | 88 | // Name returns the name of the file being read. 89 | func (r *RecordReader) Name() string { 90 | return r.File.Name() 91 | } 92 | 93 | // Exists returns whether the file exists 94 | func (r *RecordReader) Exists() bool { 95 | _, err := os.Stat(r.File.Name()) 96 | return err == nil 97 | } 98 | -------------------------------------------------------------------------------- /spoolrecordreader.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2013 Jason Ish 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 22 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 23 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | package unified2 28 | 29 | import ( 30 | "io" 31 | "io/ioutil" 32 | "log" 33 | "os" 34 | "path" 35 | "strings" 36 | ) 37 | 38 | // SpoolRecordReader is a unified2 record reader that reads from a 39 | // directory containing unified2 "spool" files. 40 | // 41 | // Unified2 spool files are files that have a common prefix and are 42 | // suffixed by a timestamp. This is the typical format used by Snort 43 | // and Suricata as new unified2 files are closed and a new one is 44 | // created when they reach a certain size. 45 | type SpoolRecordReader struct { 46 | 47 | // CloseHook will be called when a file is closed. It can be used 48 | // to delete or archive the file. 49 | CloseHook func(string) 50 | 51 | directory string 52 | prefix string 53 | logger *log.Logger 54 | reader *RecordReader 55 | } 56 | 57 | // NewSpoolRecordReader creates a new RecordSpoolReader reading files 58 | // prefixed with the provided prefix in the passed in directory. 59 | func NewSpoolRecordReader(directory string, prefix string) *SpoolRecordReader { 60 | reader := new(SpoolRecordReader) 61 | reader.directory = directory 62 | reader.prefix = prefix 63 | return reader 64 | } 65 | 66 | func (r *SpoolRecordReader) log(format string, v ...interface{}) { 67 | if r.logger != nil { 68 | r.logger.Printf(format, v...) 69 | } 70 | } 71 | 72 | // Logger sets a logger. Useful while testing/debugging. 73 | func (r *SpoolRecordReader) Logger(logger *log.Logger) { 74 | r.logger = logger 75 | } 76 | 77 | // getFiles returns a sorted list of filename in the spool 78 | // directory with the specified prefix. 79 | func (r *SpoolRecordReader) getFiles() ([]os.FileInfo, error) { 80 | files, err := ioutil.ReadDir(r.directory) 81 | if err != nil { 82 | return nil, err 83 | } 84 | 85 | filtered := make([]os.FileInfo, len(files)) 86 | filtered_idx := 0 87 | 88 | for _, file := range files { 89 | if strings.HasPrefix(file.Name(), r.prefix) { 90 | filtered[filtered_idx] = file 91 | filtered_idx++ 92 | } 93 | } 94 | 95 | return filtered[0:filtered_idx], nil 96 | } 97 | 98 | // openNext opens the next available file if it exists. If a new file 99 | // is opened its filename will be returned. 100 | func (r *SpoolRecordReader) openNext() bool { 101 | files, err := r.getFiles() 102 | if err != nil { 103 | r.log("Failed to get filenames: %s", err) 104 | return false 105 | } 106 | 107 | if len(files) == 0 { 108 | // Nothing to do. 109 | return false 110 | } 111 | 112 | if r.reader != nil { 113 | r.log("Currently open file: %s", r.reader.Name()) 114 | } 115 | 116 | var nextFilename string 117 | var foundCurrentFile bool 118 | 119 | for _, file := range files { 120 | if r.reader == nil || !r.reader.Exists() { 121 | nextFilename = path.Join(r.directory, file.Name()) 122 | break 123 | } else { 124 | if path.Base(r.reader.Name()) == file.Name() { 125 | foundCurrentFile = true 126 | } else if foundCurrentFile { 127 | nextFilename = path.Join(r.directory, file.Name()) 128 | break 129 | } 130 | } 131 | } 132 | 133 | if nextFilename == "" { 134 | r.log("No new files found.") 135 | return false 136 | } 137 | 138 | if r.reader != nil { 139 | r.log("Closing %s.", r.reader.Name()) 140 | r.reader.Close() 141 | 142 | // Call the close hook if set. 143 | if r.CloseHook != nil { 144 | r.CloseHook(r.reader.Name()) 145 | } 146 | } 147 | 148 | r.log("Opening file %s", nextFilename) 149 | r.reader, err = NewRecordReader(nextFilename, 0) 150 | if err != nil { 151 | r.log("Failed to open %s: %s", nextFilename, err) 152 | return false 153 | } 154 | return true 155 | } 156 | 157 | // Next returns the next record read from the spool. 158 | func (r *SpoolRecordReader) Next() (interface{}, error) { 159 | 160 | for { 161 | 162 | // If we have no current file, try to open one. 163 | if r.reader == nil { 164 | r.openNext() 165 | } 166 | 167 | // If we still don't have a current file, return. 168 | if r.reader == nil { 169 | return nil, nil 170 | } 171 | 172 | record, err := r.reader.Next() 173 | 174 | if err == io.EOF { 175 | if r.openNext() { 176 | continue 177 | } 178 | } 179 | 180 | return record, err 181 | 182 | } 183 | 184 | } 185 | 186 | // Offset returns the current filename that is being processed and its 187 | // read position (the offset). 188 | func (r *SpoolRecordReader) Offset() (string, int64) { 189 | if r.reader != nil { 190 | return path.Base(r.reader.Name()), r.reader.Offset() 191 | } else { 192 | return "", 0 193 | } 194 | } 195 | -------------------------------------------------------------------------------- /spoolrecordreader_example_test.go: -------------------------------------------------------------------------------- 1 | package unified2_test 2 | 3 | import ( 4 | "io" 5 | "log" 6 | "time" 7 | 8 | "github.com/jasonish/go-unified2" 9 | ) 10 | 11 | func ExampleSpoolRecordReader() { 12 | 13 | // Create a SpoolRecordReader. 14 | reader := unified2.NewSpoolRecordReader("/var/log/snort", "unified2.log") 15 | 16 | for { 17 | record, err := reader.Next() 18 | if err != nil { 19 | if err == io.EOF { 20 | // EOF is returned when the end of the last spool file 21 | // is reached and there is nothing else to read. For 22 | // the purposes of the example, just sleep for a 23 | // moment and try again. 24 | time.Sleep(time.Millisecond) 25 | } else { 26 | // Unexpected error. 27 | log.Fatal(err) 28 | } 29 | } 30 | 31 | if record == nil { 32 | // The record and err are nil when there are no files at 33 | // all to be read. This will happen if the Next() is 34 | // called before any files exist in the spool 35 | // directory. For now, sleep. 36 | time.Sleep(time.Millisecond) 37 | continue 38 | } 39 | 40 | switch record := record.(type) { 41 | case *unified2.EventRecord: 42 | log.Printf("Event: EventId=%d\n", record.EventId) 43 | case *unified2.ExtraDataRecord: 44 | log.Printf("- Extra Data: EventId=%d\n", record.EventId) 45 | case *unified2.PacketRecord: 46 | log.Printf("- Packet: EventId=%d\n", record.EventId) 47 | } 48 | 49 | filename, offset := reader.Offset() 50 | log.Printf("Current position: filename=%s; offset=%d", filename, offset) 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /spoolrecordreader_test.go: -------------------------------------------------------------------------------- 1 | package unified2 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | // Utility function to copy a file. 14 | func copyFile(source string, dest string) error { 15 | src, err := os.Open(source) 16 | if err != nil { 17 | return err 18 | } 19 | defer src.Close() 20 | 21 | dst, err := os.Create(dest) 22 | if err != nil { 23 | return err 24 | } 25 | defer dst.Close() 26 | 27 | _, err = io.Copy(dst, src) 28 | if err != nil { 29 | return err 30 | } 31 | 32 | return nil 33 | } 34 | 35 | // Test that only files with the provided prefix are returned. 36 | func TestRecordSpoolReader_getFiles(t *testing.T) { 37 | 38 | test_filename := "test/multi-record-event.log" 39 | 40 | tmpdir, err := ioutil.TempDir("", "unified2-test-") 41 | if err != nil { 42 | t.Fatal(err) 43 | } 44 | defer os.RemoveAll(tmpdir) 45 | 46 | copyFile(test_filename, fmt.Sprintf("%s/merged.log.005", tmpdir)) 47 | copyFile(test_filename, fmt.Sprintf("%s/merged.log.004", tmpdir)) 48 | copyFile(test_filename, fmt.Sprintf("%s/merged.log.003", tmpdir)) 49 | copyFile(test_filename, fmt.Sprintf("%s/asdf.log.002", tmpdir)) 50 | 51 | reader := NewSpoolRecordReader(tmpdir, "merged.log") 52 | if reader == nil { 53 | t.Fatal("reader should not be nil") 54 | } 55 | 56 | files, err := reader.getFiles() 57 | for _, file := range files { 58 | if !strings.HasPrefix(file.Name(), "merged.log") { 59 | t.Fatalf("unexpected filename: %s", file.Name()) 60 | } 61 | } 62 | } 63 | 64 | // Basic test for RecordSpoolReader. 65 | func TestRecordSpoolReader(t *testing.T) { 66 | 67 | test_filename := "test/multi-record-event.log" 68 | 69 | tmpdir, err := ioutil.TempDir("", "unified2-test-") 70 | if err != nil { 71 | t.Fatal(err) 72 | } 73 | defer os.RemoveAll(tmpdir) 74 | 75 | closeHookCount := 0 76 | 77 | reader := NewSpoolRecordReader(tmpdir, "merged.log") 78 | if reader == nil { 79 | t.Fatal("reader should not be nil") 80 | } 81 | reader.Logger(log.New(os.Stderr, "RecordSpoolReader: ", 0)) 82 | reader.CloseHook = func(filename string) { 83 | closeHookCount++ 84 | } 85 | 86 | // Offset should return an empty string and 0. 87 | filename, offset := reader.Offset() 88 | if filename != "" { 89 | t.Fatal(filename) 90 | } 91 | if offset != 0 { 92 | t.Fatal(offset) 93 | } 94 | 95 | copyFile(test_filename, fmt.Sprintf("%s/merged.log.1382627900", tmpdir)) 96 | 97 | files, err := reader.getFiles() 98 | if err != nil { 99 | t.Fatal(err) 100 | } 101 | for _, file := range files { 102 | log.Println(file.Name()) 103 | } 104 | 105 | // Read the first record and check the offset. 106 | record, err := reader.Next() 107 | if err != nil { 108 | t.Fatal(err) 109 | } 110 | if record == nil { 111 | t.Fatal("record is nil") 112 | } 113 | filename, offset = reader.Offset() 114 | if filename != "merged.log.1382627900" { 115 | t.Fatalf("got %s, expected %s", filename, "merged.log.1382627900") 116 | } 117 | 118 | // Offset known from previous testing. 119 | if offset != 68 { 120 | t.Fatal("bad offset") 121 | } 122 | 123 | // We know the input file has 17 records, so read 16 and make sure 124 | // we get back a record for each call. 125 | for i := 0; i < 16; i++ { 126 | record, err := reader.Next() 127 | if err != nil { 128 | t.Fatalf("unexpected error: %s", err) 129 | } 130 | if record == nil { 131 | t.Fatalf("unexpected nil record") 132 | } 133 | } 134 | 135 | // On the next call, record should be nul and we should have an 136 | // error of EOF. 137 | record, err = reader.Next() 138 | if record != nil || err != io.EOF { 139 | t.Fatalf("unexpected results: record not nil, err not EOF") 140 | } 141 | 142 | // Copy in another file that should be picked up by the spool 143 | // reader. 144 | copyFile(test_filename, fmt.Sprintf("%s/merged.log.1382627901", tmpdir)) 145 | 146 | // We should now read records again. 147 | record, err = reader.Next() 148 | if record == nil { 149 | t.Fatalf("expected non-nil record: err=%s", err) 150 | } 151 | 152 | if closeHookCount != 1 { 153 | t.Fatalf("bad closeHookCount: expected 1, got %d", closeHookCount) 154 | } 155 | 156 | // Finish reading the rest of the records. 157 | for i := 0; i < 16; i++ { 158 | next, err := reader.Next() 159 | if err != nil { 160 | t.Fatal(err) 161 | } 162 | if next == nil { 163 | t.Fatal(err) 164 | } 165 | } 166 | 167 | // Read more, we should get a nil event instead of re-opening the 168 | // first file. 169 | record, err = reader.Next() 170 | if record != nil { 171 | t.Fatal("expected nil record") 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /test/multi-record-event-x2.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jasonish/go-unified2/f12db2c9efe93186ee129cc000a4e0b71cfcc533/test/multi-record-event-x2.log -------------------------------------------------------------------------------- /test/multi-record-event.log: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/jasonish/go-unified2/f12db2c9efe93186ee129cc000a4e0b71cfcc533/test/multi-record-event.log -------------------------------------------------------------------------------- /test/short-read-on-body.log: -------------------------------------------------------------------------------- 1 | h< -------------------------------------------------------------------------------- /test/short-read-on-header.log: -------------------------------------------------------------------------------- 1 | h -------------------------------------------------------------------------------- /unified2.go: -------------------------------------------------------------------------------- 1 | /* Copyright (c) 2013 Jason Ish 2 | * All rights reserved. 3 | * 4 | * Redistribution and use in source and binary forms, with or without 5 | * modification, are permitted provided that the following conditions 6 | * are met: 7 | * 8 | * 1. Redistributions of source code must retain the above copyright 9 | * notice, this list of conditions and the following disclaimer. 10 | * 2. Redistributions in binary form must reproduce the above copyright 11 | * notice, this list of conditions and the following disclaimer in the 12 | * documentation and/or other materials provided with the distribution. 13 | * 14 | * THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESS OR IMPLIED 15 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 16 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 18 | * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 19 | * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 | * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 22 | * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 23 | * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 24 | * POSSIBILITY OF SUCH DAMAGE. 25 | */ 26 | 27 | /* 28 | 29 | Package unified2 provides a decoder for unified v2 log files 30 | produced by Snort and Suricata. 31 | 32 | */ 33 | package unified2 34 | 35 | import ( 36 | "encoding/binary" 37 | "io" 38 | "net" 39 | ) 40 | 41 | // Unified2 record types. 42 | const ( 43 | UNIFIED2_PACKET = 2 44 | UNIFIED2_EVENT = 7 45 | UNIFIED2_EVENT_IP6 = 72 46 | UNIFIED2_EVENT_V2 = 104 47 | UNIFIED2_EVENT_V2_IP6 = 105 48 | UNIFIED2_EXTRA_DATA = 110 49 | UNIFIED2_EVENT_APPID = 111 50 | UNIFIED2_EVENT_APPID_IP6 = 112 51 | ) 52 | 53 | // RawHeader is the raw unified2 record header. 54 | type RawHeader struct { 55 | Type uint32 56 | Len uint32 57 | } 58 | 59 | // RawRecord is a holder type for a raw un-decoded record. 60 | type RawRecord struct { 61 | Type uint32 62 | Data []byte 63 | } 64 | 65 | // EventRecord is a struct representing a decoded event record. 66 | // 67 | // This struct is used to represent the decoded form of all the event 68 | // types. The difference between an IPv4 and IPv6 event will be the 69 | // length of the IP address IpSource and IpDestination. 70 | type EventRecord struct { 71 | SensorId uint32 72 | EventId uint32 73 | EventSecond uint32 74 | EventMicrosecond uint32 75 | SignatureId uint32 76 | GeneratorId uint32 77 | SignatureRevision uint32 78 | ClassificationId uint32 79 | Priority uint32 80 | IpSource net.IP 81 | IpDestination net.IP 82 | SportItype uint16 83 | DportIcode uint16 84 | Protocol uint8 85 | ImpactFlag uint8 86 | Impact uint8 87 | Blocked uint8 88 | MplsLabel uint32 89 | VlanId uint16 90 | Pad2 uint16 91 | AppId string 92 | } 93 | 94 | // PacketRecord is a struct representing a decoded packet record. 95 | type PacketRecord struct { 96 | SensorId uint32 97 | EventId uint32 98 | EventSecond uint32 99 | PacketSecond uint32 100 | PacketMicrosecond uint32 101 | LinkType uint32 102 | Length uint32 103 | Data []byte 104 | } 105 | 106 | // The length of a PacketRecord before variable length data. 107 | const PACKET_RECORD_HDR_LEN = 28 108 | 109 | // ExtraDataRecord is a struct representing a decoded extra data record. 110 | type ExtraDataRecord struct { 111 | EventType uint32 112 | EventLength uint32 113 | SensorId uint32 114 | EventId uint32 115 | EventSecond uint32 116 | Type uint32 117 | DataType uint32 118 | DataLength uint32 119 | Data []byte 120 | } 121 | 122 | // The length of an ExtraDataRecord before variable length data. 123 | const EXTRA_DATA_RECORD_HDR_LEN = 32 124 | 125 | // ReadRawRecord reads a raw record from the provided file. 126 | // 127 | // On error, err will no non-nil. Expected error values are io.EOF 128 | // when the end of the file has been reached or io.ErrUnexpectedEOF if 129 | // a complete record was unable to be read. 130 | // 131 | // In the case of io.ErrUnexpectedEOF the file offset will be reset 132 | // back to where it was upon entering this function so it is ready to 133 | // be read from again if it is expected more data will be written to 134 | // the file. 135 | func ReadRawRecord(file io.ReadWriteSeeker) (*RawRecord, error) { 136 | var header RawHeader 137 | 138 | /* Get the current offset so we can seek back to it. */ 139 | offset, _ := file.Seek(0, 1) 140 | 141 | /* Now read in the header. */ 142 | err := binary.Read(file, binary.BigEndian, &header) 143 | if err != nil { 144 | file.Seek(offset, 0) 145 | return nil, err 146 | } 147 | 148 | /* Create a buffer to hold the raw record data and read the 149 | /* record data into it */ 150 | data := make([]byte, header.Len) 151 | n, err := file.Read(data) 152 | if err != nil { 153 | file.Seek(offset, 0) 154 | return nil, err 155 | } 156 | if uint32(n) != header.Len { 157 | file.Seek(offset, 0) 158 | return nil, io.ErrUnexpectedEOF 159 | } 160 | 161 | return &RawRecord{header.Type, data}, nil 162 | } 163 | 164 | // ReadRecord reads a record from the provided file and returns a 165 | // decoded record. 166 | // 167 | // On error, err will be non-nil. Expected error values are io.EOF 168 | // when the end of the file has been reached or io.ErrUnexpectedEOF if 169 | // a complete record was unable to be read. 170 | // 171 | // In the case of io.ErrUnexpectedEOF the file offset will be reset 172 | // back to where it was upon entering this function so it is ready to 173 | // be read from again if it is expected that more data will be written to 174 | // the file. 175 | // 176 | // If an error occurred during decoding of the read data a 177 | // DecodingError will be returned. This likely means the input is 178 | // corrupt. 179 | func ReadRecord(file io.ReadWriteSeeker) (interface{}, error) { 180 | 181 | record, err := ReadRawRecord(file) 182 | if err != nil { 183 | return nil, err 184 | } 185 | 186 | var decoded interface{} 187 | 188 | switch record.Type { 189 | case UNIFIED2_EVENT, 190 | UNIFIED2_EVENT_IP6, 191 | UNIFIED2_EVENT_V2, 192 | UNIFIED2_EVENT_V2_IP6, 193 | UNIFIED2_EVENT_APPID, 194 | UNIFIED2_EVENT_APPID_IP6: 195 | decoded, err = DecodeEventRecord(record.Type, record.Data) 196 | case UNIFIED2_PACKET: 197 | decoded, err = DecodePacketRecord(record.Data) 198 | case UNIFIED2_EXTRA_DATA: 199 | decoded, err = DecodeExtraDataRecord(record.Data) 200 | } 201 | 202 | if err != nil { 203 | return nil, err 204 | } else if decoded != nil { 205 | return decoded, nil 206 | } else { 207 | // Unknown record type. 208 | return nil, nil 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /unified2_example_test.go: -------------------------------------------------------------------------------- 1 | package unified2_test 2 | 3 | import ( 4 | "github.com/jasonish/go-unified2" 5 | "io" 6 | "log" 7 | "os" 8 | ) 9 | 10 | func ExampleReadRecord() { 11 | 12 | // Open a file. 13 | file, err := os.Open(os.Args[1]) 14 | if err != nil { 15 | log.Fatal(err) 16 | } 17 | 18 | // Read records. 19 | for { 20 | record, err := unified2.ReadRecord(file) 21 | if err != nil { 22 | if err == io.EOF || err == io.ErrUnexpectedEOF { 23 | // End of file is reached. You may want to break here 24 | // or sleep and try again if you are expected more 25 | // data to be written to the input file. 26 | // 27 | // Lets break for the purpose of this example. 28 | break 29 | } else if err == unified2.DecodingError { 30 | // Error decoding a record, probably corrupt. 31 | log.Fatal(err) 32 | } 33 | // Some other error. 34 | log.Fatal(err) 35 | } 36 | 37 | switch record := record.(type) { 38 | case *unified2.EventRecord: 39 | log.Printf("Event: EventId=%d\n", record.EventId) 40 | case *unified2.ExtraDataRecord: 41 | log.Printf("- Extra Data: EventId=%d\n", record.EventId) 42 | case *unified2.PacketRecord: 43 | log.Printf("- Packet: EventId=%d\n", record.EventId) 44 | } 45 | } 46 | 47 | file.Close() 48 | 49 | } 50 | 51 | // RecordReader example. 52 | func ExampleRecordReader() { 53 | 54 | // Create a reader starting at offset 0 of the provided file. 55 | reader, err := unified2.NewRecordReader("test/multi-record-event.log", 0) 56 | if err != nil { 57 | log.Fatal(err) 58 | } 59 | 60 | for { 61 | record, err := reader.Next() 62 | if err != nil { 63 | if err == io.EOF || err == io.ErrUnexpectedEOF { 64 | // End of file is reached. You may want to break here 65 | // or sleep and try again if you are expected more 66 | // data to be written to the input file. 67 | // 68 | // Lets break for the purpose of this example. 69 | break 70 | } else if err == unified2.DecodingError { 71 | // Error decoding a record, probably corrupt. 72 | log.Fatal(err) 73 | } 74 | // Some other error. 75 | log.Fatal(err) 76 | } 77 | 78 | switch record := record.(type) { 79 | case *unified2.EventRecord: 80 | log.Printf("Event: EventId=%d\n", record.EventId) 81 | case *unified2.ExtraDataRecord: 82 | log.Printf("- Extra Data: EventId=%d\n", record.EventId) 83 | case *unified2.PacketRecord: 84 | log.Printf("- Packet: EventId=%d\n", record.EventId) 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /unified2_test.go: -------------------------------------------------------------------------------- 1 | package unified2 2 | 3 | import ( 4 | "io" 5 | "os" 6 | "testing" 7 | ) 8 | 9 | // Check that we get EOF at the end of a file. 10 | func TestReadRecordEOF(t *testing.T) { 11 | 12 | // Use test/multi-record-event.log, its a complete file and should 13 | // finish up with an EOF. 14 | input, err := os.Open("test/multi-record-event.log") 15 | if err != nil { 16 | t.Fatal(err) 17 | } 18 | 19 | for { 20 | _, err := ReadRecord(input) 21 | if err != nil { 22 | if err != io.EOF { 23 | t.Fatalf("expected err of io.EOF, got %s", err) 24 | } 25 | break 26 | } 27 | } 28 | 29 | } 30 | 31 | func TestShortReadOnHeader(t *testing.T) { 32 | 33 | input, err := os.Open("test/short-read-on-header.log") 34 | if err != nil { 35 | t.Fatal(err) 36 | } 37 | 38 | _, err = ReadRecord(input) 39 | if err == nil { 40 | t.Fatalf("expected non-nil err") 41 | } 42 | if err != io.ErrUnexpectedEOF { 43 | t.Fatalf("expected err == io.ErrUnexpectedEOF, got %s", err) 44 | } 45 | offset, err := input.Seek(0, 1) 46 | if err != nil { 47 | t.Fatal(err) 48 | } 49 | if offset != 0 { 50 | t.Fatalf("expected file offset to be at 0, was at %d", offset) 51 | } 52 | 53 | input.Close() 54 | } 55 | 56 | func TestShortReadOnBody(t *testing.T) { 57 | 58 | input, err := os.Open("test/short-read-on-body.log") 59 | if err != nil { 60 | t.Fatal(err) 61 | } 62 | 63 | _, err = ReadRecord(input) 64 | if err == nil { 65 | t.Fatalf("expected non-nil err") 66 | } 67 | if err != io.ErrUnexpectedEOF { 68 | t.Fatalf("expected err == io.ErrUnexpectedEOF, got %s", err) 69 | } 70 | offset, err := input.Seek(0, 1) 71 | if err != nil { 72 | t.Fatal(err) 73 | } 74 | if offset != 0 { 75 | t.Fatalf("expected file offset to be at 0, was at %d", offset) 76 | } 77 | 78 | input.Close() 79 | 80 | } 81 | 82 | func TestDecodeError(t *testing.T) { 83 | 84 | data := []byte("this should fail") 85 | 86 | _, err := DecodeEventRecord(UNIFIED2_EVENT_V2, data) 87 | if err == nil { 88 | t.Fatal("expected non-nil error") 89 | } 90 | } 91 | 92 | func TestRecordReaderWithOffset(t *testing.T) { 93 | test_filename := "test/multi-record-event.log" 94 | 95 | // First open a known file at offset 0. 96 | reader, err := NewRecordReader(test_filename, 0) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | 101 | // Read one record. 102 | record, err := reader.Next() 103 | if err != nil { 104 | t.Fatal(err) 105 | } 106 | if record == nil { 107 | t.Fatalf("unexpected nil record") 108 | } 109 | 110 | offset := reader.Offset() 111 | if offset == 0 { 112 | t.Fatal("unpexpected offset %d", offset) 113 | } 114 | 115 | // Close and reopen with offset, check offset and make sure the 116 | // first record returned is not an event record. 117 | reader.Close() 118 | reader, err = NewRecordReader(test_filename, offset) 119 | if err != nil { 120 | t.Fatal(err) 121 | } 122 | if offset != reader.Offset() { 123 | t.Fatalf("unexpected reader offset: expected %d; got %d", offset, 124 | reader.Offset()) 125 | } 126 | record, err = reader.Next() 127 | if err != nil { 128 | t.Fatal(err) 129 | } 130 | if _, ok := record.(*EventRecord); ok { 131 | t.Fatal("did not expect Next() to return *EventRecord") 132 | } 133 | 134 | } 135 | --------------------------------------------------------------------------------