├── .gitignore ├── LICENSE ├── README.md ├── build.sh ├── dictionary.go ├── doc.go ├── main.go ├── msgstats.go ├── procera-fields.ini └── trafficstats.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.tar.gz 2 | *.zip 3 | *.exe 4 | ipfixcat 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (C) 2013 Jakob Borg 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | - The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ipfixcat 2 | 3 | [![Build Status](https://img.shields.io/circleci/project/calmh/ipfixcat.svg?style=flat-square)](https://circleci.com/gh/calmh/ipfixcat) 4 | 5 | `ipfixcat` is a utility to parse and print an IPFIX stream, as defined by RFC 6 | 5101. It's also the minimal demo of how to use the github.com/calmh/ipfix 7 | package. 8 | 9 | 10 | ### Installation 11 | 12 | Grab a binary release from https://github.com/calmh/ipfixcat/releases. 13 | 14 | You can also build from source. Make sure you have Go 1.1 installed. See 15 | http://golang.org/doc/install. 16 | 17 | $ go install github.com/calmh/ipfixcat 18 | 19 | 20 | ### Output 21 | 22 | The output format is JSON with one object per line. Each object has fields 23 | `exportTime` (UNIX epoch seconds), `templateId` and `elements`. The latter is an 24 | array containing the information elements in the same order as received by the 25 | exporter. 26 | 27 | Each information element has the fields `name`, `enterprise`, `field`, `value` 28 | and `rawvalue`. For vendor fields that are not described by a user dictionary, 29 | `name` and `value` will be empty and `rawvalue` contains a byte array. For fully 30 | understood fields, `value` contains the parsed value and `rawvalue` is empty. 31 | 32 | There are some statistics that can be enabled as well, see `ipfixcat -help` for 33 | more information. 34 | 35 | 36 | ### Examples 37 | 38 | Parse a UDP IPFIX stream, using a custom dictionary to interpret vendor fields. 39 | Note that it might take a while to start displaying datasets, because we need to 40 | receive the periodically sent template sets first in order to be able to parse 41 | them. 42 | 43 | $ socat udp-recv:4739 stdout | ipfixcat -dict procera-fields.ini 44 | {"exportTime":1374745620,"templateId":49836,"fields":[{"name":"destinationIPv4Address","field":12,"value":"194.153.... 45 | {"exportTime":1374745620,"templateId":10299,"fields":[{"name":"destinationIPv6Address","field":28,"value":"2001:470... 46 | {"exportTime":1374745620,"templateId":10299,"fields":[{"name":"destinationIPv6Address","field":28,"value":"2001:470... 47 | ... 48 | 49 | Don't attempt to use netcat (`nc`) for reading UDP streams. Almost all 50 | distributed versions are broken and truncate UDP packets at 1024 bytes. 51 | 52 | 53 | ### License 54 | 55 | The MIT License. 56 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -euo pipefail 3 | 4 | BIN=ipfixcat 5 | VER=$(git describe --always) 6 | 7 | case "$1" in 8 | pkg) 9 | rm -f *.tar.gz *.zip 10 | for GOARCH in amd64 ; do 11 | for GOOS in darwin linux freebsd solaris ; do 12 | export GOOS 13 | export GOARCH 14 | 15 | NAME="$BIN-${VER}_$GOOS-$GOARCH" 16 | rm -rf "$NAME" "$NAME.tar.gz" "$BIN" "$BIN.exe" 17 | go build -ldflags "-X main.ipfixcatVersion=${VER}" 18 | 19 | mkdir "$NAME" 20 | cp *.ini ipfixcat "$NAME" && tar zcf "$NAME.tar.gz" "$NAME" 21 | rm -r "$NAME" 22 | done 23 | 24 | for GOOS in windows ; do 25 | export GOOS 26 | export GOARCH 27 | 28 | NAME="$BIN-${VER}_$GOOS-$GOARCH" 29 | rm -rf "$NAME" "$NAME.tar.gz" "$BIN" "$BIN.exe" 30 | go build -ldflags "-X main.ipfixcatVersion=${VER}" 31 | 32 | mkdir "$NAME" 33 | cp *.ini ipfixcat.exe "$NAME" && zip -r "$NAME.zip" "$NAME" 34 | rm -r "$NAME" 35 | done 36 | done 37 | ;; 38 | 39 | *) 40 | go install 41 | esac 42 | -------------------------------------------------------------------------------- /dictionary.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "gopkg.in/gcfg.v1" 5 | "github.com/calmh/ipfix" 6 | ) 7 | 8 | type Field struct { 9 | ID uint16 10 | Enterprise uint32 11 | Type string 12 | } 13 | 14 | func (f Field) DictionaryEntry(name string) ipfix.DictionaryEntry { 15 | return ipfix.DictionaryEntry{ 16 | Name: name, 17 | EnterpriseID: f.Enterprise, 18 | FieldID: f.ID, 19 | Type: ipfix.FieldTypes[f.Type], 20 | } 21 | } 22 | 23 | type UserDictionary struct { 24 | Field map[string]*Field 25 | } 26 | 27 | func loadUserDictionary(fname string, i *ipfix.Interpreter) error { 28 | dict := UserDictionary{} 29 | err := gcfg.ReadFileInto(&dict, fname) 30 | if err != nil { 31 | return err 32 | } 33 | 34 | for name, entry := range dict.Field { 35 | i.AddDictionaryEntry(entry.DictionaryEntry(name)) 36 | } 37 | 38 | return nil 39 | } 40 | -------------------------------------------------------------------------------- /doc.go: -------------------------------------------------------------------------------- 1 | /* 2 | `ipfixcat` is a utility to parse and print an IPFIX stream, as defined 3 | by RFC 5101. It's also the minimal demo of how to use the 4 | github.com/calmh/ipfix package. 5 | 6 | Installation 7 | 8 | Grab a binary release from https://github.com/calmh/ipfixcat/releases. 9 | 10 | You can also build from source. Make sure you have Go 1.1 installed. See 11 | http://golang.org/doc/install. 12 | 13 | $ go install github.com/calmh/ipfixcat 14 | 15 | Output 16 | 17 | The output format is JSON with one object per line. Each object has 18 | fields `exportTime` (UNIX epoch seconds), `templateId` and `elements`. 19 | The latter is an array containing the information elements in the same 20 | order as received by the exporter. 21 | 22 | Each information element has the fields `name`, `enterprise`, `field`, 23 | `value` and `rawvalue`. For vendor fields that are not described by a 24 | user dictionary, `name` and `value` will be empty and `rawvalue` 25 | contains a byte array. For fully understood fields, `value` contains the 26 | parsed value and `rawvalue` is empty. 27 | 28 | There are some statistics that can be enabled as well, see 29 | `ipfixcat -help` for more information. 30 | 31 | Examples 32 | 33 | Parse a UDP IPFIX stream, using a custom dictionary to interpret vendor 34 | fields. Note that it might take a while to start displaying datasets, 35 | because we need to receive the periodically sent template sets first in 36 | order to be able to parse them. 37 | 38 | $ socat udp-recv:4739 stdout | ipfixcat -dict procera-fields.ini 39 | {"exportTime":1374745620,"templateId":49836,"fields":[{"name":"destinationIPv4Address","field":12,"value":"194.153.... 40 | {"exportTime":1374745620,"templateId":10299,"fields":[{"name":"destinationIPv6Address","field":28,"value":"2001:470... 41 | {"exportTime":1374745620,"templateId":10299,"fields":[{"name":"destinationIPv6Address","field":28,"value":"2001:470... 42 | ... 43 | 44 | Don't attempt to use netcat (`nc`) for reading UDP streams. Almost all 45 | distributed versions are broken and truncate UDP packets at 1024 bytes. 46 | 47 | License 48 | 49 | The MIT License. 50 | */ 51 | package main 52 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "io" 7 | "log" 8 | "os" 9 | "time" 10 | 11 | "github.com/calmh/ipfix" 12 | ) 13 | 14 | var ipfixcatVersion string 15 | 16 | type InterpretedRecord struct { 17 | ExportTime uint32 `json:"exportTime"` 18 | TemplateId uint16 `json:"templateId"` 19 | Fields []myInterpretedField `json:"fields"` 20 | } 21 | 22 | // Because we want to control JSON serialization 23 | type myInterpretedField struct { 24 | Name string `json:"name"` 25 | EnterpriseId uint32 `json:"enterprise,omitempty"` 26 | FieldId uint16 `json:"field"` 27 | Value interface{} `json:"value,omitempty"` 28 | RawValue []int `json:"raw,omitempty"` 29 | } 30 | 31 | func messagesGenerator(r io.Reader, s *ipfix.Session, i *ipfix.Interpreter) <-chan []InterpretedRecord { 32 | c := make(chan []InterpretedRecord) 33 | 34 | errors := 0 35 | go func() { 36 | for { 37 | msg, err := s.ParseReader(r) 38 | if err == io.EOF { 39 | close(c) 40 | return 41 | } 42 | if err != nil { 43 | errors++ 44 | if errors > 3 { 45 | panic(err) 46 | } else { 47 | log.Println(err) 48 | } 49 | continue 50 | } else { 51 | errors = 0 52 | } 53 | 54 | irecs := make([]InterpretedRecord, len(msg.DataRecords)) 55 | for j, record := range msg.DataRecords { 56 | ifs := i.Interpret(record) 57 | mfs := make([]myInterpretedField, len(ifs)) 58 | for k, iif := range ifs { 59 | mfs[k] = myInterpretedField{iif.Name, iif.EnterpriseID, iif.FieldID, iif.Value, integers(iif.RawValue)} 60 | } 61 | ir := InterpretedRecord{msg.Header.ExportTime, record.TemplateID, mfs} 62 | irecs[j] = ir 63 | } 64 | 65 | c <- irecs 66 | } 67 | }() 68 | 69 | return c 70 | } 71 | 72 | func main() { 73 | log.Println("ipfixcat", ipfixcatVersion) 74 | dictFile := flag.String("dict", "", "User dictionary file") 75 | messageStats := flag.Bool("mstats", false, "Log IPFIX message statistics") 76 | trafficStats := flag.Bool("acc", false, "Log traffic rates (Procera)") 77 | output := flag.Bool("output", true, "Display received flow records in JSON format") 78 | statsIntv := flag.Int("statsintv", 60, "Statistics log interval (s)") 79 | flag.Parse() 80 | 81 | if *messageStats { 82 | log.Printf("Logging message statistics every %d seconds", *statsIntv) 83 | } 84 | 85 | if *trafficStats { 86 | log.Printf("Logging traffic rates every %d seconds", *statsIntv) 87 | } 88 | 89 | if !*messageStats && !*trafficStats && !*output { 90 | log.Fatal("If you don't want me to do anything, don't run me at all.") 91 | } 92 | 93 | s := ipfix.NewSession() 94 | i := ipfix.NewInterpreter(s) 95 | 96 | if *dictFile != "" { 97 | if err := loadUserDictionary(*dictFile, i); err != nil { 98 | log.Fatal(err) 99 | } 100 | } 101 | 102 | msgs := messagesGenerator(os.Stdin, s, i) 103 | tick := time.Tick(time.Duration(*statsIntv) * time.Second) 104 | enc := json.NewEncoder(os.Stdout) 105 | for { 106 | select { 107 | case irecs, ok := <-msgs: 108 | if !ok { 109 | return 110 | } 111 | if *messageStats { 112 | accountMsgStats(irecs) 113 | } 114 | 115 | for _, rec := range irecs { 116 | if *trafficStats { 117 | accountTraffic(rec) 118 | } 119 | 120 | if *output { 121 | for i := range rec.Fields { 122 | f := &rec.Fields[i] 123 | switch v := f.Value.(type) { 124 | case []byte: 125 | f.RawValue = integers(v) 126 | f.Value = nil 127 | } 128 | } 129 | enc.Encode(rec) 130 | } 131 | } 132 | case <-tick: 133 | if *messageStats { 134 | logMsgStats() 135 | } 136 | 137 | if *trafficStats { 138 | logAccountedTraffic() 139 | } 140 | } 141 | } 142 | } 143 | 144 | func integers(s []byte) []int { 145 | if s == nil { 146 | return nil 147 | } 148 | 149 | r := make([]int, len(s)) 150 | for i := range s { 151 | r[i] = int(s[i]) 152 | } 153 | return r 154 | } 155 | -------------------------------------------------------------------------------- /msgstats.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | ) 7 | 8 | var ( 9 | msgT0 time.Time = time.Now() 10 | records int 11 | msgs int 12 | ) 13 | 14 | func accountMsgStats(recs []InterpretedRecord) { 15 | msgs++ 16 | records += len(recs) 17 | } 18 | 19 | type MsgStats struct { 20 | Msgs int 21 | Msgsps int 22 | Records int 23 | Recordsps int 24 | AvgMsgRecs int 25 | } 26 | 27 | func logMsgStats() { 28 | now := time.Now() 29 | diff := now.Sub(msgT0).Seconds() 30 | 31 | ms := MsgStats{} 32 | ms.Msgs = msgs 33 | ms.Msgsps = int(float64(msgs) / diff) 34 | ms.Records = records 35 | ms.Recordsps = int(float64(records) / diff) 36 | if records > 0 { 37 | ms.AvgMsgRecs = records / msgs 38 | } 39 | 40 | log.Printf("%#v\n", ms) 41 | msgT0 = now 42 | records = 0 43 | msgs = 0 44 | } 45 | -------------------------------------------------------------------------------- /procera-fields.ini: -------------------------------------------------------------------------------- 1 | [field "proceraService"] 2 | type = string 3 | enterprise = 15397 4 | id = 1 5 | 6 | [field "proceraBaseService"] 7 | type = string 8 | enterprise = 15397 9 | id = 2 10 | 11 | [field "proceraIncomingOctets"] 12 | type = unsigned64 13 | enterprise = 15397 14 | id = 3 15 | 16 | [field "proceraOutgoingOctets"] 17 | type = unsigned64 18 | enterprise = 15397 19 | id = 4 20 | 21 | [field "proceraIncomingPackets"] 22 | type = unsigned64 23 | enterprise = 15397 24 | id = 5 25 | 26 | [field "proceraOutgoingPackets"] 27 | type = unsigned64 28 | enterprise = 15397 29 | id = 6 30 | 31 | [field "proceraIncomingShapingLatency"] 32 | type = unsigned16 33 | enterprise = 15397 34 | id = 7 35 | 36 | [field "proceraOutgoingShapingLatency"] 37 | type = unsigned16 38 | enterprise = 15397 39 | id = 8 40 | 41 | [field "proceraIncomingShapingDrops"] 42 | type = unsigned32 43 | enterprise = 15397 44 | id = 9 45 | 46 | [field "proceraOutgoingShapingDrops"] 47 | type = unsigned32 48 | enterprise = 15397 49 | id = 10 50 | 51 | [field "proceraInternalRtt"] 52 | type = unsigned32 53 | enterprise = 15397 54 | id = 11 55 | 56 | [field "proceraExternalRtt"] 57 | type = unsigned32 58 | enterprise = 15397 59 | id = 12 60 | 61 | [field "proceraFlowBehavior"] 62 | type = string 63 | enterprise = 15397 64 | id = 15 65 | 66 | [field "proceraContentCategories"] 67 | type = string 68 | enterprise = 15397 69 | id = 16 70 | 71 | [field "proceraProperty"] 72 | type = string 73 | enterprise = 15397 74 | id = 17 75 | 76 | [field "proceraServerHostname"] 77 | type = string 78 | enterprise = 15397 79 | id = 18 80 | 81 | [field "proceraHttpRequestMethod"] 82 | type = string 83 | enterprise = 15397 84 | id = 19 85 | 86 | [field "proceraHttpUserAgent"] 87 | type = string 88 | enterprise = 15397 89 | id = 20 90 | 91 | [field "proceraHttpContentType"] 92 | type = string 93 | enterprise = 15397 94 | id = 21 95 | 96 | [field "proceraHttpUrl"] 97 | type = string 98 | enterprise = 15397 99 | id = 22 100 | 101 | [field "proceraHttpReferer"] 102 | type = string 103 | enterprise = 15397 104 | id = 23 105 | 106 | [field "proceraHttpResponseStatus"] 107 | type = unsigned16 108 | enterprise = 15397 109 | id = 24 110 | 111 | [field "proceraHttpFileLength"] 112 | type = unsigned32 113 | enterprise = 15397 114 | id = 25 115 | 116 | [field "proceraHttpLocation"] 117 | type = string 118 | enterprise = 15397 119 | id = 26 120 | 121 | [field "proceraHttpLanguage"] 122 | type = string 123 | enterprise = 15397 124 | id = 27 125 | 126 | [field "proceraSubscriberId"] 127 | type = string 128 | enterprise = 15397 129 | id = 28 130 | 131 | [field "proceraMsisdn"] 132 | type = unsigned64 133 | enterprise = 15397 134 | id = 29 135 | 136 | [field "proceraImsi"] 137 | type = unsigned64 138 | enterprise = 15397 139 | id = 30 140 | 141 | [field "proceraRat"] 142 | type = string 143 | enterprise = 15397 144 | id = 31 145 | 146 | [field "proceraDeviceId"] 147 | type = unsigned64 148 | enterprise = 15397 149 | id = 32 150 | 151 | [field "proceraSgsn"] 152 | type = string 153 | enterprise = 15397 154 | id = 33 155 | 156 | [field "proceraRnc"] 157 | type = unsigned16 158 | enterprise = 15397 159 | id = 34 160 | 161 | [field "proceraApn"] 162 | type = string 163 | enterprise = 15397 164 | id = 35 165 | 166 | [field "proceraUserLocationInformation"] 167 | type = string 168 | enterprise = 15397 169 | id = 36 170 | 171 | [field "proceraGgsn"] 172 | type = string 173 | enterprise = 15397 174 | id = 37 175 | 176 | [field "proceraQoeIncomingInternal"] 177 | type = float32 178 | enterprise = 15397 179 | id = 38 180 | 181 | [field "proceraQoeIncomingExternal"] 182 | type = float32 183 | enterprise = 15397 184 | id = 39 185 | 186 | [field "proceraQoeOutgoingInternal"] 187 | type = float32 188 | enterprise = 15397 189 | id = 40 190 | 191 | [field "proceraQoeOutgoingExternal"] 192 | type = float32 193 | enterprise = 15397 194 | id = 41 195 | 196 | [field "proceraLocalIPv4Host"] 197 | type = ipv4Address 198 | enterprise = 15397 199 | id = 42 200 | 201 | [field "proceraLocalIPv6Host"] 202 | type = ipv6Address 203 | enterprise = 15397 204 | id = 43 205 | 206 | [field "proceraRemoteIPv4Host"] 207 | type = ipv4Address 208 | enterprise = 15397 209 | id = 44 210 | 211 | [field "proceraRemoteIPv6Host"] 212 | type = ipv6Address 213 | enterprise = 15397 214 | id = 45 215 | 216 | [field "proceraHttpRequestVersion"] 217 | type = string 218 | enterprise = 15397 219 | id = 46 220 | 221 | [field "proceraTemplateName"] 222 | type = string 223 | enterprise = 15397 224 | id = 47 225 | -------------------------------------------------------------------------------- /trafficstats.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "time" 6 | ) 7 | 8 | var ( 9 | outOctets uint64 10 | inOctets uint64 11 | accountTime time.Time = time.Now() 12 | ) 13 | 14 | type TrafStats struct { 15 | InKbps int 16 | OutKbps int 17 | } 18 | 19 | func accountTraffic(rec InterpretedRecord) { 20 | for _, f := range rec.Fields { 21 | if f.Name == "proceraIncomingOctets" { 22 | inOctets += f.Value.(uint64) 23 | } else if f.Name == "proceraOutgoingOctets" { 24 | outOctets += f.Value.(uint64) 25 | } 26 | } 27 | } 28 | 29 | func logAccountedTraffic() { 30 | now := time.Now() 31 | diff := now.Sub(accountTime).Seconds() 32 | 33 | ts := TrafStats{} 34 | ts.InKbps = int(float64(inOctets*8/1000) / diff) 35 | ts.OutKbps = int(float64(outOctets*8/1000) / diff) 36 | 37 | log.Printf("%#v\n", ts) 38 | 39 | outOctets = 0 40 | inOctets = 0 41 | accountTime = now 42 | } 43 | --------------------------------------------------------------------------------