├── .gitignore ├── .travis.yml ├── LICENSE ├── Makefile ├── README.md ├── aggregate.go ├── aggregate_test.go ├── dns-ans-query.bro ├── go.mod ├── go.sum ├── index.go ├── local_test.txt ├── main.go ├── read.go ├── read_ascii.go ├── read_json.go ├── read_test.go ├── store.go ├── store_clickhouse.go ├── store_pg.go ├── store_sql_common.go ├── store_sqlite.go ├── store_test.go ├── template └── index.html ├── test_data ├── bad_ttl.log ├── dns_json.log ├── dns_json_iso8601.json ├── garbage.log ├── nbtstat.log ├── reddit_1.txt ├── reddit_2.txt └── reddit_dns_2016-04-01.log ├── version.go └── web.go /.gitignore: -------------------------------------------------------------------------------- 1 | *.py[co] 2 | 3 | # Packages 4 | *.egg 5 | *.egg-info 6 | dist 7 | build 8 | eggs 9 | parts 10 | bin 11 | develop-eggs 12 | .installed.cfg 13 | 14 | # Installer logs 15 | pip-log.txt 16 | 17 | # Unit test / coverage reports 18 | .coverage 19 | .tox 20 | bro-pdns 21 | bro-pdns_linux_amd64 22 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | go: 4 | - "1.16" 5 | 6 | services: 7 | - postgresql 8 | - docker 9 | 10 | env: 11 | - PG_TEST_URL=postgres://postgres:password@localhost/pdns_test?sslmode=disable 12 | - CH_TEST_URL=tcp://localhost:9000/default 13 | 14 | before_script: 15 | - psql -c 'create database pdns_test;' -U postgres 16 | - docker run -d -p 127.0.0.1:9000:9000 -p 127.0.0.1:8123:8123 --name test-clickhouse-server --ulimit nofile=262144:262144 yandex/clickhouse-server 17 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Justin Azoff 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | all: build test 2 | build: 3 | go get -t -v ./... 4 | go build 5 | test: 6 | go test -v ./... 7 | static: 8 | go get -t -v ./... 9 | go build --ldflags '-extldflags "-static"' 10 | 11 | .PHONY: rpm 12 | rpm: build 13 | rpm: VERSION=$(shell ./zeek-pdns version) 14 | rpm: 15 | fpm -f -s dir -t rpm -n zeek-pdns -v $(VERSION) \ 16 | --iteration=1 \ 17 | --architecture native \ 18 | --description "Zeek Passive DNS" \ 19 | ./zeek-pdns=/usr/bin/ 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Passive DNS for Zeek 2 | =================== 3 | 4 | This is an extremely simple implementation of a passive DNS collection system 5 | that utilizes Zeek for DNS log collection. 6 | 7 | Passive DNS collection can be used for various security or troubleshooting 8 | purposes. Many queries to raw DNS logs can be done faster by using 9 | the aggregated data in the passive DNS database, which is more compact. 10 | 11 | This tool uses the Zeek DNS logs to build a database of unique query+type+answer 12 | tuples. 13 | 14 | It produces a table like this: 15 | 16 | pdns=# select * from dns where answer='74.125.225.18' order by last desc limit 4; 17 | query | type | answer | count | ttl | first | last 18 | ------------------+------+---------------+-------+-----+---------------+------------ 19 | www.google.com | A | 74.125.225.18 | 7517 | 198 | 2014-09-03 .. | 2014-10-30 .. 20 | t0.gstatic.com | A | 74.125.225.18 | 266 | 300 | 2014-09-03 .. | 2014-10-30 .. 21 | googlegroups.com | A | 74.125.225.18 | 266 | 300 | 2014-09-03 .. | 2014-10-30 .. 22 | t3.gstatic.com | A | 74.125.225.18 | 291 | 300 | 2014-09-03 .. | 2014-10-30 .. 23 | 24 | This is helpful because the PTR record itself for 74.125.225.18 is ord08s12-in-f18.1e100.net. 25 | 26 | Examples of questions this database can answer faster than the raw logs: 27 | 28 | * Did anything ever resolve example.com, and if so, when was the first time? 29 | * What IPs has example.com resolved to? 30 | * What other names resolve to this IP? 31 | 32 | Requirements 33 | ------------ 34 | 35 | * go compiler ( to build ) 36 | * postgresql ( optional ) 37 | * clickhouse ( optional ) 38 | 39 | Build 40 | ----- 41 | 42 | $ go build 43 | 44 | Index logs 45 | ---------- 46 | 47 | # for postgresql 48 | export PDNS_STORE_TYPE="postgresql" 49 | export PDNS_STORE_URI="postgres://pdns:foo@localhost/pdns?sslmode=disable" 50 | 51 | # for clickhouse 52 | export PDNS_STORE_TYPE="clickhouse" 53 | export PDNS_STORE_URI="tcp://localhost:9000/?database=pdns" 54 | 55 | # for built in sqlite 56 | export PDNS_STORE_TYPE="sqlite" 57 | export PDNS_STORE_URI="/path/to/passivedns.sqlite" 58 | 59 | # then finally index logs 60 | find /usr/local/zeek/logs -name 'dns*' | sort -n | xargs -n 50 zeek-pdns index 61 | 62 | Query Database 63 | -------------- 64 | 65 | # suffix search: 66 | $ zeek-pdns like tuples google.com 67 | $ zeek-pdns like individual google.com 68 | 69 | # exact match 70 | $ zeek-pdns find tuples google.com 71 | $ zeek-pdns find individual google.com 72 | 73 | Start HTTP server 74 | ----------------- 75 | 76 | $ zeek-pdns web 77 | 78 | Query HTTP API 79 | -------------- 80 | 81 | $ curl localhost:8080/dns/like/tuples/google.com 82 | $ curl localhost:8080/dns/like/individual/google.com 83 | $ curl localhost:8080/dns/find/tuples/google.com 84 | $ curl localhost:8080/dns/find/individual/google.com 85 | -------------------------------------------------------------------------------- /aggregate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "io" 7 | "log" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | opendecompress "github.com/JustinAzoff/go-opendecompress" 13 | ) 14 | 15 | var MAX_SANE_VALUE_LEN = 1000 16 | 17 | func stripDecimal(value string) string { 18 | if value == "-" { 19 | return "0" 20 | } 21 | idx := strings.Index(value, ".") 22 | if idx == -1 { 23 | return value 24 | } 25 | return value[:idx] 26 | } 27 | 28 | type DNSRecord struct { 29 | ts string 30 | query string 31 | qtype string 32 | answers []string 33 | ttls []string 34 | } 35 | 36 | type uniqueTuple struct { 37 | query string 38 | answer string 39 | qtype string 40 | } 41 | type uniqueIndividual struct { 42 | value string 43 | which string // "Q" or "A" 44 | } 45 | 46 | type queryStat struct { 47 | count uint 48 | first string 49 | last string 50 | ttl string 51 | } 52 | 53 | type aggregationResult struct { 54 | Duration time.Duration 55 | TotalRecords uint 56 | SkippedRecords uint 57 | Tuples []aggregatedTuple 58 | TuplesLen int 59 | Individual []aggregatedIndividual 60 | IndividualLen int 61 | } 62 | 63 | type aggregatedTuple struct { 64 | uniqueTuple 65 | queryStat 66 | } 67 | type aggregatedIndividual struct { 68 | uniqueIndividual 69 | queryStat 70 | } 71 | 72 | type DNSAggregator struct { 73 | queries map[uniqueTuple]*queryStat 74 | values map[uniqueIndividual]*queryStat 75 | totalRecords uint 76 | skippedRecords uint 77 | start time.Time 78 | } 79 | 80 | func NewDNSAggregator() *DNSAggregator { 81 | queries := make(map[uniqueTuple]*queryStat) 82 | values := make(map[uniqueIndividual]*queryStat) 83 | return &DNSAggregator{ 84 | queries: queries, 85 | values: values, 86 | start: time.Now(), 87 | } 88 | } 89 | func (d *DNSAggregator) SkipRecord() { 90 | d.skippedRecords++ 91 | } 92 | 93 | func (d *DNSAggregator) AddRecord(r DNSRecord) { 94 | if len(r.query) > MAX_SANE_VALUE_LEN { 95 | log.Printf("Skipping record with insane query length: %#v\n", r) 96 | d.skippedRecords++ 97 | return 98 | } 99 | r.query = strings.TrimRight(r.query, "\u0000") 100 | if strings.ContainsRune(r.query, '\u0000') { 101 | log.Printf("Skipping record with null byte in query: %#v\n", r) 102 | d.skippedRecords++ 103 | return 104 | } 105 | d.totalRecords++ 106 | query_value := uniqueIndividual{value: r.query, which: "Q"} 107 | 108 | arec := d.values[query_value] 109 | if arec == nil { 110 | arec = &queryStat{ 111 | first: r.ts, 112 | last: r.ts, 113 | count: 1, 114 | } 115 | d.values[query_value] = arec 116 | } else { 117 | arec.count++ 118 | arec.last = r.ts 119 | } 120 | 121 | for idx, answer := range r.answers { 122 | if len(answer) > MAX_SANE_VALUE_LEN { 123 | log.Printf("Skipping record with insane answer length: %#v\n", r) 124 | d.skippedRecords++ 125 | return 126 | } 127 | if answer == "-" { 128 | continue 129 | } 130 | ttl := stripDecimal(r.ttls[idx]) 131 | //Validate that a ttl fits in a 32bit int 132 | _, err := strconv.ParseInt(ttl, 10, 32) 133 | if err != nil { 134 | log.Printf("Skipping record with insane ttl: %#v\n", r) 135 | d.skippedRecords++ 136 | return 137 | } 138 | if len(ttl) > 0 && ttl[0] == '-' { 139 | ttl = "0" 140 | } 141 | uquery := uniqueTuple{ 142 | query: r.query, 143 | answer: answer, 144 | qtype: r.qtype, 145 | } 146 | rec := d.queries[uquery] 147 | if rec == nil { 148 | rec = &queryStat{ 149 | first: r.ts, 150 | last: r.ts, 151 | ttl: ttl, 152 | count: 1, 153 | } 154 | d.queries[uquery] = rec 155 | } else { 156 | rec.count++ 157 | rec.last = r.ts 158 | rec.ttl = ttl 159 | } 160 | 161 | answer_value := uniqueIndividual{value: answer, which: "A"} 162 | arec := d.values[answer_value] 163 | if arec == nil { 164 | arec = &queryStat{ 165 | first: r.ts, 166 | last: r.ts, 167 | ttl: ttl, 168 | count: 1, 169 | } 170 | d.values[answer_value] = arec 171 | } else { 172 | arec.count++ 173 | arec.last = r.ts 174 | arec.ttl = ttl 175 | } 176 | } 177 | } 178 | 179 | func (d *DNSAggregator) GetResult() aggregationResult { 180 | var result aggregationResult 181 | for q, stat := range d.queries { 182 | agg := aggregatedTuple{ 183 | uniqueTuple: q, 184 | queryStat: *stat, 185 | } 186 | result.Tuples = append(result.Tuples, agg) 187 | } 188 | for value, stat := range d.values { 189 | agg := aggregatedIndividual{ 190 | uniqueIndividual: value, 191 | queryStat: *stat, 192 | } 193 | result.Individual = append(result.Individual, agg) 194 | } 195 | result.TotalRecords = d.totalRecords 196 | result.SkippedRecords = d.skippedRecords 197 | result.Duration = time.Since(d.start) 198 | result.TuplesLen = len(result.Tuples) 199 | result.IndividualLen = len(result.Individual) 200 | return result 201 | 202 | } 203 | 204 | //timeCompare compares timestamps, doesn't care about subsecond 205 | func timeCompare(a, b string) int { 206 | a = stripDecimal(a) 207 | b = stripDecimal(b) 208 | 209 | if strings.Contains(a, "-") { 210 | //Formatted timestamps are the same length, and can just be 211 | //Compared as is 212 | if a < b { 213 | return -1 214 | } else if a > b { 215 | return 1 216 | } else { 217 | return 0 218 | } 219 | } else { 220 | ai, err := strconv.ParseInt(a, 10, 64) 221 | if err != nil { 222 | log.Printf("Invalid timestamp: %v", a) 223 | return 0 224 | } 225 | bi, err := strconv.ParseInt(b, 10, 64) 226 | if err != nil { 227 | log.Printf("Invalid timestamp: %v", b) 228 | return 0 229 | } 230 | if ai < bi { 231 | return -1 232 | } else if ai > bi { 233 | return 1 234 | } else { 235 | return 0 236 | } 237 | } 238 | } 239 | 240 | func (d *DNSAggregator) Merge(other *DNSAggregator) { 241 | for q, stat := range other.queries { 242 | rec := d.queries[q] 243 | if rec == nil { 244 | d.queries[q] = stat 245 | } else { 246 | rec.count += stat.count 247 | if timeCompare(stat.first, rec.first) < 0 { 248 | rec.first = stat.first 249 | } 250 | if timeCompare(stat.last, rec.last) > 0 { 251 | rec.last = stat.last 252 | } 253 | rec.ttl = stat.ttl 254 | } 255 | } 256 | for q, stat := range other.values { 257 | rec := d.values[q] 258 | if rec == nil { 259 | d.values[q] = stat 260 | } else { 261 | rec.count += stat.count 262 | if timeCompare(stat.first, rec.first) < 0 { 263 | rec.first = stat.first 264 | } 265 | if timeCompare(stat.last, rec.last) > 0 { 266 | rec.last = stat.last 267 | } 268 | rec.ttl = stat.ttl 269 | } 270 | } 271 | return 272 | } 273 | 274 | func aggregate(aggregator *DNSAggregator, fn string) error { 275 | f, err := opendecompress.Open(fn) 276 | if err != nil { 277 | return err 278 | } 279 | defer f.Close() 280 | br, err := NewBroReader(f) 281 | if err != nil { 282 | return err 283 | } 284 | 285 | for { 286 | rec, err := br.Next() 287 | if errors.Is(err, io.ErrUnexpectedEOF) { 288 | log.Printf("Possible truncated file %s: %v", fn, err) 289 | break 290 | } 291 | if err != nil { 292 | return err 293 | } 294 | if rec == nil { 295 | break 296 | } 297 | ts := rec.GetTimestamp("ts") 298 | query := rec.GetString("query") 299 | qtype_name := rec.GetString("qtype_name") 300 | answers := rec.GetStringList("answers") 301 | ttls := rec.GetStringList("TTLs") 302 | if rec.Error() != nil { 303 | if rec.IsMissingFieldError() { 304 | log.Printf("Skipping record with missing fields: %s", rec) 305 | aggregator.SkipRecord() 306 | continue 307 | } else { 308 | return rec.Error() 309 | } 310 | } 311 | dns_record := DNSRecord{ 312 | ts: ts, 313 | query: query, 314 | qtype: qtype_name, 315 | answers: answers, 316 | ttls: ttls, 317 | } 318 | aggregator.AddRecord(dns_record) 319 | } 320 | 321 | return nil 322 | } 323 | 324 | func (ar *aggregationResult) ShallowCopy() aggregationResult { 325 | return aggregationResult{ 326 | Duration: ar.Duration, 327 | TotalRecords: ar.TotalRecords, 328 | SkippedRecords: ar.SkippedRecords, 329 | TuplesLen: ar.TuplesLen, 330 | IndividualLen: ar.IndividualLen, 331 | } 332 | } 333 | 334 | type JSONTuple struct { 335 | Query string `json:"query"` 336 | Type string `json:"type"` 337 | Answer string `json:"answer"` 338 | TTL string `json:"ttl"` 339 | Count uint `json:"count"` 340 | First string `json:"first"` 341 | Last string `json:"last"` 342 | } 343 | 344 | func (ar *aggregationResult) TupleJSONReader(reverseQuery bool) io.ReadCloser { 345 | pr, pw := io.Pipe() 346 | 347 | encoder := json.NewEncoder(pw) 348 | go func() { 349 | defer pw.Close() 350 | var q string 351 | for _, t := range ar.Tuples { 352 | if reverseQuery { 353 | q = Reverse(t.query) 354 | } else { 355 | q = t.query 356 | } 357 | v := JSONTuple{ 358 | Query: q, 359 | Type: t.qtype, 360 | Answer: t.answer, 361 | TTL: t.ttl, 362 | Count: t.count, 363 | First: t.first, 364 | Last: t.last, 365 | } 366 | err := encoder.Encode(v) 367 | if err != nil { 368 | pr.CloseWithError(err) 369 | return 370 | } 371 | } 372 | }() 373 | return pr 374 | } 375 | 376 | type JSONIndividual struct { 377 | Value string `json:"value"` 378 | Which string `json:"which"` 379 | Count uint `json:"count"` 380 | First string `json:"first"` 381 | Last string `json:"last"` 382 | } 383 | 384 | func (ar *aggregationResult) IndividualJSONReader(reverseQuery bool) io.ReadCloser { 385 | pr, pw := io.Pipe() 386 | 387 | encoder := json.NewEncoder(pw) 388 | go func() { 389 | defer pw.Close() 390 | var q string 391 | for _, t := range ar.Individual { 392 | if t.which == "Q" && reverseQuery { 393 | q = Reverse(t.value) 394 | } else { 395 | q = t.value 396 | } 397 | v := JSONIndividual{ 398 | Value: q, 399 | Which: t.which, 400 | Count: t.count, 401 | First: t.first, 402 | Last: t.last, 403 | } 404 | err := encoder.Encode(v) 405 | if err != nil { 406 | pr.CloseWithError(err) 407 | return 408 | } 409 | } 410 | }() 411 | return pr 412 | } 413 | -------------------------------------------------------------------------------- /aggregate_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "sort" 8 | "testing" 9 | ) 10 | 11 | type ByValue []aggregatedIndividual 12 | 13 | func (a ByValue) Len() int { return len(a) } 14 | func (a ByValue) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 15 | func (a ByValue) Less(i, j int) bool { return a[i].value < a[j].value } 16 | 17 | type ByTuple []aggregatedTuple 18 | 19 | func (a ByTuple) Len() int { return len(a) } 20 | func (a ByTuple) Swap(i, j int) { a[i], a[j] = a[j], a[i] } 21 | func (a ByTuple) Less(i, j int) bool { return a[i].query+a[i].answer < a[j].query+a[j].answer } 22 | 23 | func ExampleAggregate() { 24 | ag := NewDNSAggregator() 25 | 26 | ag.AddRecord(DNSRecord{ 27 | ts: "10", 28 | query: "www.example.com", 29 | qtype: "A", 30 | answers: []string{"1.2.3.4"}, 31 | ttls: []string{"300"}, 32 | }) 33 | ag.AddRecord(DNSRecord{ 34 | ts: "20", 35 | query: "www.example.com", 36 | qtype: "A", 37 | answers: []string{"1.2.3.4"}, 38 | ttls: []string{"300"}, 39 | }) 40 | 41 | res := ag.GetResult() 42 | sort.Sort(ByTuple(res.Tuples)) 43 | sort.Sort(ByValue(res.Individual)) 44 | 45 | fmt.Printf("Tuples:\n") 46 | for _, r := range res.Tuples { 47 | fmt.Printf("%#v\n", r) 48 | } 49 | fmt.Printf("\nIndividual:\n") 50 | for _, r := range res.Individual { 51 | fmt.Printf("%#v\n", r) 52 | } 53 | // Output: 54 | //Tuples: 55 | //main.aggregatedTuple{uniqueTuple:main.uniqueTuple{query:"www.example.com", answer:"1.2.3.4", qtype:"A"}, queryStat:main.queryStat{count:0x2, first:"10", last:"20", ttl:"300"}} 56 | // 57 | //Individual: 58 | //main.aggregatedIndividual{uniqueIndividual:main.uniqueIndividual{value:"1.2.3.4", which:"A"}, queryStat:main.queryStat{count:0x2, first:"10", last:"20", ttl:"300"}} 59 | //main.aggregatedIndividual{uniqueIndividual:main.uniqueIndividual{value:"www.example.com", which:"Q"}, queryStat:main.queryStat{count:0x2, first:"10", last:"20", ttl:""}} 60 | } 61 | 62 | func ExampleAggregateMerge() { 63 | ag := NewDNSAggregator() 64 | 65 | ag.AddRecord(DNSRecord{ 66 | ts: "10", 67 | query: "www.example.com", 68 | qtype: "A", 69 | answers: []string{"1.2.3.4"}, 70 | ttls: []string{"300"}, 71 | }) 72 | ag.AddRecord(DNSRecord{ 73 | ts: "200", 74 | query: "www.example.com", 75 | qtype: "A", 76 | answers: []string{"1.2.3.4"}, 77 | ttls: []string{"300"}, 78 | }) 79 | ag2 := NewDNSAggregator() 80 | ag2.AddRecord(DNSRecord{ 81 | ts: "30", 82 | query: "www.example.com", 83 | qtype: "A", 84 | answers: []string{"1.2.3.4"}, 85 | ttls: []string{"300"}, 86 | }) 87 | ag2.AddRecord(DNSRecord{ 88 | ts: "30", 89 | query: "www.example.com", 90 | qtype: "A", 91 | answers: []string{"1.2.3.5"}, 92 | ttls: []string{"300"}, 93 | }) 94 | ag2.AddRecord(DNSRecord{ 95 | ts: "40", 96 | query: "www.example.com", 97 | qtype: "A", 98 | answers: []string{"1.2.3.5"}, 99 | ttls: []string{"300"}, 100 | }) 101 | 102 | ag.Merge(ag2) 103 | 104 | res := ag.GetResult() 105 | sort.Sort(ByTuple(res.Tuples)) 106 | sort.Sort(ByValue(res.Individual)) 107 | 108 | fmt.Printf("Tuples:\n") 109 | for _, r := range res.Tuples { 110 | fmt.Printf("%#v\n", r) 111 | } 112 | fmt.Printf("\nIndividual:\n") 113 | for _, r := range res.Individual { 114 | fmt.Printf("%#v\n", r) 115 | } 116 | // Output: 117 | //Tuples: 118 | //main.aggregatedTuple{uniqueTuple:main.uniqueTuple{query:"www.example.com", answer:"1.2.3.4", qtype:"A"}, queryStat:main.queryStat{count:0x3, first:"10", last:"200", ttl:"300"}} 119 | //main.aggregatedTuple{uniqueTuple:main.uniqueTuple{query:"www.example.com", answer:"1.2.3.5", qtype:"A"}, queryStat:main.queryStat{count:0x2, first:"30", last:"40", ttl:"300"}} 120 | // 121 | //Individual: 122 | //main.aggregatedIndividual{uniqueIndividual:main.uniqueIndividual{value:"1.2.3.4", which:"A"}, queryStat:main.queryStat{count:0x3, first:"10", last:"200", ttl:"300"}} 123 | //main.aggregatedIndividual{uniqueIndividual:main.uniqueIndividual{value:"1.2.3.5", which:"A"}, queryStat:main.queryStat{count:0x2, first:"30", last:"40", ttl:"300"}} 124 | //main.aggregatedIndividual{uniqueIndividual:main.uniqueIndividual{value:"www.example.com", which:"Q"}, queryStat:main.queryStat{count:0x5, first:"10", last:"200", ttl:""}} 125 | 126 | } 127 | 128 | func BenchmarkAggregate(b *testing.B) { 129 | aggregator := NewDNSAggregator() 130 | var total uint 131 | for i := 0; i < b.N; i++ { 132 | err := aggregate(aggregator, "test_data/dns_json.log") 133 | if err != nil { 134 | b.Fatal(err) 135 | } 136 | aggregated := aggregator.GetResult() 137 | total += aggregated.TotalRecords 138 | } 139 | } 140 | 141 | func ExampleResultTupleJSONReader() { 142 | ag := NewDNSAggregator() 143 | 144 | ag.AddRecord(DNSRecord{ 145 | ts: "10", 146 | query: "www.example.com", 147 | qtype: "A", 148 | answers: []string{"1.2.3.4"}, 149 | ttls: []string{"300"}, 150 | }) 151 | ag.AddRecord(DNSRecord{ 152 | ts: "20", 153 | query: "www.example.com", 154 | qtype: "A", 155 | answers: []string{"1.2.3.5"}, 156 | ttls: []string{"300"}, 157 | }) 158 | 159 | res := ag.GetResult() 160 | sort.Sort(ByTuple(res.Tuples)) 161 | reader := res.TupleJSONReader(false) 162 | 163 | body, err := ioutil.ReadAll(reader) 164 | if err != nil { 165 | fmt.Fprintf(os.Stderr, "%v", err) 166 | } 167 | fmt.Printf("%s", body) 168 | // Output: 169 | //{"query":"www.example.com","type":"A","answer":"1.2.3.4","ttl":"300","count":1,"first":"10","last":"10"} 170 | //{"query":"www.example.com","type":"A","answer":"1.2.3.5","ttl":"300","count":1,"first":"20","last":"20"} 171 | } 172 | 173 | func ExampleResultIndividualJSONReader() { 174 | ag := NewDNSAggregator() 175 | 176 | ag.AddRecord(DNSRecord{ 177 | ts: "10", 178 | query: "www.example.com", 179 | qtype: "A", 180 | answers: []string{"1.2.3.4"}, 181 | ttls: []string{"300"}, 182 | }) 183 | ag.AddRecord(DNSRecord{ 184 | ts: "20", 185 | query: "www.example.com", 186 | qtype: "A", 187 | answers: []string{"1.2.3.5"}, 188 | ttls: []string{"300"}, 189 | }) 190 | 191 | res := ag.GetResult() 192 | sort.Sort(ByValue(res.Individual)) 193 | reader := res.IndividualJSONReader(false) 194 | 195 | body, err := ioutil.ReadAll(reader) 196 | if err != nil { 197 | fmt.Fprintf(os.Stderr, "%v", err) 198 | } 199 | fmt.Printf("%s", body) 200 | // Output: 201 | //{"value":"1.2.3.4","which":"A","count":1,"first":"10","last":"10"} 202 | //{"value":"1.2.3.5","which":"A","count":1,"first":"20","last":"20"} 203 | //{"value":"www.example.com","which":"Q","count":2,"first":"10","last":"20"} 204 | } 205 | -------------------------------------------------------------------------------- /dns-ans-query.bro: -------------------------------------------------------------------------------- 1 | ##! Add the peer to the connection logs. 2 | 3 | module DNS; 4 | 5 | export { 6 | redef record DNS::Info += { 7 | ans_query: vector of string &optional &log; 8 | }; 9 | } 10 | 11 | event dns_query_reply(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) 12 | { 13 | if(!c?$dns) 14 | return; 15 | 16 | if(!c$dns?$ans_query) 17 | c$dns$ans_query = vector(); 18 | 19 | c$dns$ans_query[|c$dns$ans_query|] = query; 20 | } 21 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/JustinAzoff/zeek-pdns 2 | 3 | go 1.16 4 | 5 | require ( 6 | github.com/ClickHouse/clickhouse-go v1.4.3 7 | github.com/JustinAzoff/go-opendecompress v0.0.0-20210404020920-ed787af23844 8 | github.com/buger/jsonparser v1.1.1 9 | github.com/gorilla/mux v1.7.3 10 | github.com/jmoiron/sqlx v1.2.0 11 | github.com/lib/pq v1.3.0 12 | github.com/mattn/go-sqlite3 v2.0.2+incompatible 13 | github.com/pkg/errors v0.9.1 14 | github.com/spf13/cobra v0.0.5 15 | github.com/spf13/viper v1.6.2 16 | github.com/stretchr/testify v1.7.0 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 3 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 4 | github.com/ClickHouse/clickhouse-go v1.4.3 h1:iAFMa2UrQdR5bHJ2/yaSLffZkxpcOYQMCUuKeNXGdqc= 5 | github.com/ClickHouse/clickhouse-go v1.4.3/go.mod h1:EaI/sW7Azgz9UATzd5ZdZHRUhHgv5+JMS9NSr2smCJI= 6 | github.com/JustinAzoff/go-opendecompress v0.0.0-20210404020920-ed787af23844 h1:dozqygFyq+gPF1qHj5BXtUwap2ZWNYkXv5SOBn2re7o= 7 | github.com/JustinAzoff/go-opendecompress v0.0.0-20210404020920-ed787af23844/go.mod h1:HhAjXnUSLm0cIlX5FnaucNpSOkh/u1CFjRpZsy5TLTA= 8 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= 9 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 10 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 11 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 12 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 13 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 14 | github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAKk= 15 | github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= 16 | github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= 17 | github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= 18 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= 19 | github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= 20 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58 h1:F1EaeKL/ta07PY/k9Os/UFtwERei2/XzGemhpGnBKNg= 21 | github.com/cloudflare/golz4 v0.0.0-20150217214814-ef862a3cdc58/go.mod h1:EOBUe0h4xcZ5GoxqC5SDxFQ8gwyZPKQoEzownBlhI80= 22 | github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= 23 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 24 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 25 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 26 | github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= 27 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= 28 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 29 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 30 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 31 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 32 | github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= 33 | github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= 34 | github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= 35 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 36 | github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= 37 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 38 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 39 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 40 | github.com/go-sql-driver/mysql v1.4.0 h1:7LxgVwFb2hIQtMm87NdgAVfXjnt4OePseqT1tKx+opk= 41 | github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 42 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 43 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 44 | github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= 45 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= 46 | github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 47 | github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= 48 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 49 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 50 | github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= 51 | github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= 52 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= 53 | github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= 54 | github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= 55 | github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= 56 | github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= 57 | github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= 58 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= 59 | github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= 60 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 61 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 62 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 63 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 64 | github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= 65 | github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= 66 | github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= 67 | github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= 68 | github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= 69 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 70 | github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= 71 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 72 | github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= 73 | github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= 74 | github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= 75 | github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= 76 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 77 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 78 | github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= 79 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 80 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 81 | github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= 82 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 83 | github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 84 | github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= 85 | github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 86 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 87 | github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= 88 | github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 89 | github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 90 | github.com/mattn/go-sqlite3 v2.0.2+incompatible h1:qzw9c2GNT8UFrgWNDhCTqRqYUSmu/Dav/9Z58LGpk7U= 91 | github.com/mattn/go-sqlite3 v2.0.2+incompatible/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= 92 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 93 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 94 | github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= 95 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 96 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 97 | github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= 98 | github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= 99 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 100 | github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= 101 | github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= 102 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 103 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 104 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 105 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 106 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 107 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 108 | github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= 109 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 110 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 111 | github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= 112 | github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 113 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 114 | github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 115 | github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= 116 | github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= 117 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 118 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 119 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= 120 | github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= 121 | github.com/smartystreets/goconvey v1.6.4 h1:fv0U8FUIMPNf1L9lnHLvLhgicrIVChEkdzIKYqbNC9s= 122 | github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= 123 | github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= 124 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= 125 | github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI= 126 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 127 | github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8= 128 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 129 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 130 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 131 | github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk= 132 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 133 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 134 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 135 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 136 | github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= 137 | github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= 138 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 139 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 140 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 141 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 142 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 143 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 144 | github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= 145 | github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= 146 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= 147 | github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= 148 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 149 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= 150 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 151 | go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= 152 | go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= 153 | go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= 154 | go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= 155 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 156 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 157 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 158 | golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= 159 | golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 160 | golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 161 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 162 | golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 163 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 164 | golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 165 | golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= 166 | golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 167 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 168 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 169 | golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 170 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 171 | golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 172 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 173 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 174 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= 175 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 176 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 177 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 178 | golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= 179 | golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 180 | golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 181 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 182 | golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 183 | google.golang.org/appengine v1.1.0 h1:igQkv0AAhEIvTEpD5LIpAfav2eeVO9HBTjvKHVJPRSs= 184 | google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= 185 | google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= 186 | google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= 187 | google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= 188 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 189 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 190 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= 191 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 192 | gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= 193 | gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= 194 | gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= 195 | gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= 196 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 197 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 198 | gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= 199 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 200 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 201 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 202 | honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= 203 | -------------------------------------------------------------------------------- /index.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | func index(store Store, filenames []string) error { 9 | var didWork bool 10 | store.Begin() 11 | aggregator := NewDNSAggregator() 12 | var emptyStoreResult UpdateResult 13 | aggMap := make(map[string]aggregationResult) 14 | for _, fn := range filenames { 15 | indexed, err := store.IsLogIndexed(fn) 16 | if err != nil { 17 | return fmt.Errorf("store.IsLogIndexed: %w", err) 18 | } 19 | if indexed { 20 | log.Printf("%s: Already indexed", fn) 21 | continue 22 | } 23 | 24 | fileAgg := NewDNSAggregator() 25 | err = aggregate(fileAgg, fn) 26 | if err != nil { 27 | return fmt.Errorf("Error Aggregating %s: %w", fn, err) 28 | } 29 | aggregator.Merge(fileAgg) 30 | aggregated := fileAgg.GetResult() 31 | log.Printf("%s: Aggregation: Duration=%0.1f TotalRecords=%d SkippedRecords=%d Tuples=%d Individual=%d", 32 | fn, 33 | aggregated.Duration.Seconds(), 34 | aggregated.TotalRecords, 35 | aggregated.SkippedRecords, 36 | aggregated.TuplesLen, 37 | aggregated.IndividualLen, 38 | ) 39 | aggMap[fn] = aggregated.ShallowCopy() 40 | didWork = true 41 | } 42 | if !didWork { 43 | return nil 44 | //TODO: rollback transaction 45 | } 46 | aggregated := aggregator.GetResult() 47 | result, err := store.Update(aggregated) 48 | if err != nil { 49 | return fmt.Errorf("store.Update: %w", err) 50 | } 51 | log.Printf("batch: Store: Duration=%0.1f Inserted=%d Updated=%d", result.Duration.Seconds(), result.Inserted, result.Updated) 52 | for fn, aggregated := range aggMap { 53 | err = store.SetLogIndexed(fn, aggregated, emptyStoreResult) 54 | if err != nil { 55 | return fmt.Errorf("store.SetLogIndexed: %w", err) 56 | } 57 | } 58 | err = store.Commit() 59 | if err != nil { 60 | return fmt.Errorf("store.Commit: %w", err) 61 | } 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /local_test.txt: -------------------------------------------------------------------------------- 1 | docker run --rm -t -i -p 5432:5432 -e POSTGRES_DB=pdns_test -e POSTGRES_PASSWORD=pdns postgres 2 | PG_TEST_URL="postgres://postgres:password@$(docker-machine ip)/pdns_test?sslmode=disable" go test -v 3 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os" 7 | 8 | "github.com/spf13/cobra" 9 | "github.com/spf13/viper" 10 | ) 11 | 12 | func getStore() Store { 13 | storeType := viper.GetString("store.type") 14 | storeUri := viper.GetString("store.uri") 15 | mystore, err := NewStore(storeType, storeUri) 16 | if err != nil { 17 | log.Fatal(err) 18 | } 19 | return mystore 20 | } 21 | 22 | var RootCmd = &cobra.Command{ 23 | Use: "zeek-pdns", 24 | Short: "Passive DNS Collection for BRO", 25 | Run: nil, 26 | } 27 | 28 | var IndexCmd = &cobra.Command{ 29 | Use: "index", 30 | Short: "Index one or more dns log files", 31 | Run: func(cmd *cobra.Command, args []string) { 32 | mystore := getStore() 33 | err := index(mystore, args) 34 | if err != nil { 35 | log.Fatal(err) 36 | } 37 | }, 38 | } 39 | 40 | var FindCmd = &cobra.Command{ 41 | Use: "find", 42 | Short: "find records", 43 | Run: nil, 44 | } 45 | var FindTupleCmd = &cobra.Command{ 46 | Use: "tuples", 47 | Short: "find dns tuples", 48 | Run: func(cmd *cobra.Command, args []string) { 49 | mystore := getStore() 50 | 51 | for _, value := range args { 52 | recs, err := mystore.FindTuples(value) 53 | if err != nil { 54 | log.Fatal(err) 55 | } 56 | recs.Display() 57 | } 58 | }, 59 | } 60 | 61 | var FindIndividualCmd = &cobra.Command{ 62 | Use: "individual", 63 | Short: "find an individual dns value", 64 | Run: func(cmd *cobra.Command, args []string) { 65 | mystore := getStore() 66 | 67 | for _, value := range args { 68 | recs, err := mystore.FindIndividual(value) 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | recs.Display() 73 | } 74 | }, 75 | } 76 | var LikeCmd = &cobra.Command{ 77 | Use: "like", 78 | Short: "find records like something", 79 | Run: nil, 80 | } 81 | var LikeTupleCmd = &cobra.Command{ 82 | Use: "tuples", 83 | Short: "find like dns tuples", 84 | Run: func(cmd *cobra.Command, args []string) { 85 | mystore := getStore() 86 | 87 | for _, value := range args { 88 | recs, err := mystore.LikeTuples(value) 89 | if err != nil { 90 | log.Fatal(err) 91 | } 92 | recs.Display() 93 | } 94 | }, 95 | } 96 | 97 | var LikeIndividualCmd = &cobra.Command{ 98 | Use: "individual", 99 | Short: "find like individual dns values", 100 | Run: func(cmd *cobra.Command, args []string) { 101 | mystore := getStore() 102 | 103 | for _, value := range args { 104 | recs, err := mystore.LikeIndividual(value) 105 | if err != nil { 106 | log.Fatal(err) 107 | } 108 | recs.Display() 109 | } 110 | }, 111 | } 112 | 113 | var DeleteOldCmd = &cobra.Command{ 114 | Use: "delete-old", 115 | Short: "delete old records", 116 | Run: func(cmd *cobra.Command, args []string) { 117 | mystore := getStore() 118 | days := viper.GetInt("deleteold.days") 119 | rows, err := mystore.DeleteOld(int64(days)) 120 | if err != nil { 121 | log.Fatal(err) 122 | } 123 | log.Printf("Deleted %d records", rows) 124 | }, 125 | } 126 | 127 | var WebCmd = &cobra.Command{ 128 | Use: "web", 129 | Short: "start http API", 130 | Run: func(cmd *cobra.Command, args []string) { 131 | mystore := getStore() 132 | bind := viper.GetString("http.listen") 133 | startWeb(mystore, bind) 134 | }, 135 | } 136 | 137 | var VersionCmd = &cobra.Command{ 138 | Use: "version", 139 | Short: "Output version number", 140 | Run: func(cmd *cobra.Command, args []string) { 141 | fmt.Println(VERSION) 142 | }, 143 | } 144 | 145 | func init() { 146 | RootCmd.AddCommand(IndexCmd) 147 | 148 | RootCmd.AddCommand(FindCmd) 149 | FindCmd.AddCommand(FindIndividualCmd) 150 | FindCmd.AddCommand(FindTupleCmd) 151 | 152 | RootCmd.AddCommand(LikeCmd) 153 | LikeCmd.AddCommand(LikeIndividualCmd) 154 | LikeCmd.AddCommand(LikeTupleCmd) 155 | 156 | DeleteOldCmd.Flags().Int64("days", 365, "Age in days of records to be deleted") 157 | viper.BindPFlag("deleteold.days", DeleteOldCmd.Flags().Lookup("days")) 158 | viper.BindEnv("deleteold.days", "PDNS_DELETE_OLD_DAYS") 159 | RootCmd.AddCommand(DeleteOldCmd) 160 | 161 | WebCmd.Flags().String("listen", ":8080", "Address to listen on") 162 | viper.BindPFlag("http.listen", WebCmd.Flags().Lookup("listen")) 163 | viper.BindEnv("http.listen", "PDNS_HTTP_LISTEN") 164 | 165 | RootCmd.AddCommand(WebCmd) 166 | RootCmd.AddCommand(VersionCmd) 167 | 168 | RootCmd.PersistentFlags().String("store", "sqlite", "Backend data store") 169 | viper.BindPFlag("store.type", RootCmd.PersistentFlags().Lookup("store")) 170 | viper.BindEnv("store.type", "PDNS_STORE_TYPE") 171 | 172 | RootCmd.PersistentFlags().String("uri", "db.sqlite", "Backend data store URI") 173 | viper.BindPFlag("store.uri", RootCmd.PersistentFlags().Lookup("uri")) 174 | viper.BindEnv("store.uri", "PDNS_STORE_URI") 175 | 176 | viper.AutomaticEnv() 177 | } 178 | 179 | func main() { 180 | 181 | if err := RootCmd.Execute(); err != nil { 182 | fmt.Println(err) 183 | os.Exit(-1) 184 | } 185 | 186 | } 187 | -------------------------------------------------------------------------------- /read.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | ) 8 | 9 | type Reader interface { 10 | Next() (Record, error) 11 | } 12 | 13 | type Record interface { 14 | String() string 15 | GetString(string) string 16 | GetTimestamp(string) string 17 | GetStringList(string) []string 18 | GetFloat(string) float64 19 | Error() error 20 | IsMissingFieldError() bool 21 | } 22 | 23 | func NewBroReader(r io.Reader) (Reader, error) { 24 | wrapped := bufio.NewReader(r) 25 | first_byte, err := wrapped.Peek(1) 26 | if err != nil { 27 | return nil, err 28 | } 29 | switch first_byte[0] { 30 | case '#': 31 | return NewBroAsciiReader(wrapped), nil 32 | case '{': 33 | return NewBroJSONReader(wrapped), nil 34 | default: 35 | return nil, fmt.Errorf("Unable to determine file type, first byte was %q", first_byte) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /read_ascii.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "encoding/hex" 6 | "fmt" 7 | "io" 8 | "log" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | func grab_value(line string) string { 14 | val := strings.Split(line, " ")[1] 15 | return val 16 | } 17 | 18 | func extract_sep(line string) string { 19 | sep := grab_value(line) 20 | sepchar, err := hex.DecodeString(sep[2:]) 21 | if err != nil { 22 | log.Panic(err) 23 | } 24 | return string(sepchar) 25 | } 26 | 27 | type BroAsciiReader struct { 28 | r io.Reader 29 | br *bufio.Reader 30 | sep string 31 | fields []string 32 | fieldsMap map[string]int 33 | types []string 34 | timeFields map[int]bool 35 | 36 | newHeaders bool 37 | } 38 | 39 | type ASCIIRecord struct { 40 | line *string 41 | cols *[]string 42 | fields *map[string]int 43 | err error 44 | } 45 | 46 | func (r *ASCIIRecord) String() string { 47 | return *r.line 48 | } 49 | 50 | func (r *ASCIIRecord) GetString(field string) string { 51 | idx, ok := (*r.fields)[field] 52 | if !ok { 53 | r.err = fmt.Errorf("Invalid field %s", field) 54 | return "" 55 | } 56 | return (*r.cols)[idx] 57 | } 58 | func (r *ASCIIRecord) GetTimestamp(field string) string { 59 | return r.GetString(field) 60 | } 61 | func (r *ASCIIRecord) GetStringList(field string) []string { 62 | raw := r.GetString(field) 63 | spl := strings.Split(raw, ",") 64 | return spl 65 | } 66 | func (r *ASCIIRecord) GetStringByIndex(index int) string { 67 | return (*r.cols)[index] 68 | } 69 | func (r *ASCIIRecord) GetFloat(field string) float64 { 70 | idx, ok := (*r.fields)[field] 71 | if !ok { 72 | r.err = fmt.Errorf("Invalid field %s", field) 73 | return 0.0 74 | } 75 | val := (*r.cols)[idx] 76 | fl, err := strconv.ParseFloat(val, 64) 77 | if err != nil { 78 | panic(err) 79 | } 80 | return fl 81 | } 82 | func (r *ASCIIRecord) GetFloatByIndex(index int) float64 { 83 | val := (*r.cols)[index] 84 | fl, err := strconv.ParseFloat(val, 64) 85 | if err != nil { 86 | panic(err) 87 | } 88 | return fl 89 | } 90 | func (r *ASCIIRecord) IsMissingFieldError() bool { 91 | //TODO: handle here or jsut skip in Next? 92 | return false 93 | } 94 | func (r *ASCIIRecord) Error() error { 95 | if r.err != nil { 96 | return fmt.Errorf("Error parsing %s: %w", r, r.err) 97 | } 98 | return nil 99 | } 100 | 101 | func (r *ASCIIRecord) GetFieldIndex(field string) int { 102 | idx, ok := (*r.fields)[field] 103 | if ok { 104 | return idx 105 | } 106 | r.err = fmt.Errorf("Invalid field %s", field) 107 | return -1 108 | } 109 | 110 | func NewBroAsciiReader(r io.Reader) *BroAsciiReader { 111 | br := bufio.NewReader(r) 112 | tf := make(map[int]bool) 113 | return &BroAsciiReader{r: r, br: br, timeFields: tf} 114 | } 115 | 116 | func (b *BroAsciiReader) Next() (Record, error) { 117 | line, err := b.br.ReadString('\n') 118 | if err == io.EOF { 119 | return nil, nil 120 | } 121 | if err != nil { 122 | return nil, err 123 | } 124 | line = strings.Trim(line, "\n") 125 | if strings.HasPrefix(line, "#") { 126 | b.handleHeader(line) 127 | return b.Next() 128 | } 129 | parts := strings.Split(line, "\t") 130 | rec := ASCIIRecord{ 131 | line: &line, 132 | cols: &parts, 133 | fields: &b.fieldsMap, 134 | } 135 | return &rec, nil 136 | } 137 | 138 | func (b *BroAsciiReader) handleHeader(line string) error { 139 | b.newHeaders = true 140 | if strings.HasPrefix(line, "#separator") { 141 | b.sep = extract_sep(line) 142 | } else if strings.HasPrefix(line, "#fields") { 143 | b.fields = strings.Split(line, "\t")[1:] 144 | b.fieldsMap = make(map[string]int) 145 | for idx, f := range b.fields { 146 | b.fieldsMap[f] = idx 147 | } 148 | } else if strings.HasPrefix(line, "#types") { 149 | b.types = strings.Split(line, "\t")[1:] 150 | for idx, typ := range b.types { 151 | if typ == "time" { 152 | b.timeFields[idx] = true 153 | } 154 | } 155 | } 156 | return nil 157 | } 158 | func (b *BroAsciiReader) HeadersChanged() bool { 159 | return b.newHeaders 160 | } 161 | func (b *BroAsciiReader) HandledHeaders() { 162 | b.newHeaders = false 163 | } 164 | -------------------------------------------------------------------------------- /read_json.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "strings" 8 | 9 | "github.com/buger/jsonparser" 10 | ) 11 | 12 | type BroJSONReader struct { 13 | r *bufio.Reader 14 | } 15 | 16 | type JSONRecord struct { 17 | line []byte 18 | err error 19 | } 20 | 21 | func (r *JSONRecord) String() string { 22 | return strings.Trim(string(r.line), "\n") 23 | } 24 | 25 | func (r *JSONRecord) GetString(field string) string { 26 | val, err := jsonparser.GetString(r.line, field) 27 | r.err = err 28 | return val 29 | } 30 | func (r *JSONRecord) GetStringList(field string) []string { 31 | var strings []string 32 | jsonparser.ArrayEach(r.line, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { 33 | strings = append(strings, string(value)) 34 | r.err = err 35 | }, field) 36 | return strings 37 | } 38 | func (r *JSONRecord) GetFloat(field string) float64 { 39 | val, err := jsonparser.GetFloat(r.line, field) 40 | r.err = err 41 | return val 42 | } 43 | 44 | func (r *JSONRecord) IsMissingFieldError() bool { 45 | return r.err == jsonparser.KeyPathNotFoundError 46 | } 47 | 48 | func (r *JSONRecord) Error() error { 49 | if r.err != nil { 50 | return fmt.Errorf("Error parsing %s: %w", r, r.err) 51 | } 52 | return nil 53 | } 54 | 55 | func NewBroJSONReader(r *bufio.Reader) *BroJSONReader { 56 | return &BroJSONReader{r: r} 57 | } 58 | 59 | func (b *BroJSONReader) Next() (Record, error) { 60 | line, err := b.r.ReadBytes('\n') 61 | if err == io.EOF { 62 | return nil, nil 63 | } 64 | if err != nil { 65 | return nil, err 66 | } 67 | rec := JSONRecord{ 68 | line: line, 69 | } 70 | return &rec, nil 71 | } 72 | 73 | //GetTimestamp tries to get a field that may be a String or a float64 74 | //Yes, this is terrible, but supporting both timestamps and iso8601 at the same 75 | //time is tricky 76 | //TODO: use a struct type that is a string or float and can be lazy evaluated 77 | //for sorting and for final representation when being upserted 78 | func (r *JSONRecord) GetTimestamp(field string) string { 79 | val, err := jsonparser.GetString(r.line, field) 80 | if err == nil { 81 | return val 82 | } 83 | fval, err := jsonparser.GetFloat(r.line, field) 84 | r.err = err 85 | return fmt.Sprintf("%f", fval) 86 | } 87 | -------------------------------------------------------------------------------- /read_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "testing" 5 | ) 6 | 7 | func BenchmarkReadASCII(b *testing.B) { 8 | for i := 0; i < b.N; i++ { 9 | fn := "test_data/reddit_dns_2016-04-01.log" 10 | aggregator := NewDNSAggregator() 11 | aggregate(aggregator, fn) 12 | } 13 | } 14 | func BenchmarkReadJSON(b *testing.B) { 15 | for i := 0; i < b.N; i++ { 16 | fn := "test_data/dns_json.log" 17 | aggregator := NewDNSAggregator() 18 | aggregate(aggregator, fn) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /store.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | "time" 8 | "log" 9 | ) 10 | 11 | type Store interface { 12 | Init() error 13 | Clear() error 14 | Begin() error 15 | Commit() error 16 | IsLogIndexed(filename string) (bool, error) 17 | SetLogIndexed(filename string, ar aggregationResult, ur UpdateResult) error 18 | Update(aggregationResult) (UpdateResult, error) 19 | FindQueryTuples(query string) (tupleResults, error) 20 | FindTuples(query string) (tupleResults, error) 21 | FindIndividual(value string) (individualResults, error) 22 | LikeTuples(query string) (tupleResults, error) 23 | LikeIndividual(value string) (individualResults, error) 24 | DeleteOld(days int64) (int64, error) 25 | Close() error 26 | } 27 | 28 | type tupleResult struct { 29 | Query string 30 | Type string 31 | Answer string 32 | Count uint 33 | TTL uint 34 | First string 35 | Last string 36 | } 37 | 38 | type tupleResults []tupleResult 39 | 40 | func (tr tupleResults) Display() { 41 | if len(tr) == 0 { 42 | return 43 | } 44 | header := []string{"Query", "Type", "Answer", "Count", "TTL", "First", "Last"} 45 | fmt.Println(strings.Join(header, "\t")) 46 | for _, rec := range tr { 47 | fmt.Println(rec) 48 | } 49 | } 50 | func (tr tupleResult) String() string { 51 | count := fmt.Sprintf("%d", tr.Count) 52 | ttl := fmt.Sprintf("%d", tr.TTL) 53 | s := []string{tr.Query, tr.Type, tr.Answer, count, ttl, tr.First, tr.Last} 54 | return strings.Join(s, "\t") 55 | } 56 | 57 | type individualResult struct { 58 | Value string 59 | Which string 60 | Count uint 61 | First string 62 | Last string 63 | } 64 | type individualResults []individualResult 65 | 66 | func (ir individualResults) Display() { 67 | if len(ir) == 0 { 68 | return 69 | } 70 | header := []string{"Value", "Which", "Count", "First", "Last"} 71 | fmt.Println(strings.Join(header, "\t")) 72 | for _, rec := range ir { 73 | fmt.Println(rec) 74 | } 75 | } 76 | func (ir individualResult) String() string { 77 | count := fmt.Sprintf("%d", ir.Count) 78 | s := []string{ir.Value, ir.Which, count, ir.First, ir.Last} 79 | return strings.Join(s, "\t") 80 | } 81 | 82 | type UpdateResult struct { 83 | Inserted uint 84 | Updated uint 85 | Duration time.Duration 86 | } 87 | 88 | var storeFactories = map[string]func(string) (Store, error){ 89 | "clickhouse": NewCHStore, 90 | "sqlite": NewSQLiteStore, 91 | "postgresql": NewPGStore, 92 | } 93 | 94 | func NewStore(storeType string, filename string) (Store, error) { 95 | storeFactory, ok := storeFactories[storeType] 96 | if !ok { 97 | return nil, errors.New("Invalid store type") 98 | } 99 | s, err := storeFactory(filename) 100 | if err != nil { 101 | return nil, err 102 | } 103 | err = s.Init() 104 | if err != nil { 105 | return nil, err 106 | } 107 | return s, err 108 | } 109 | 110 | //ToTS ensures a string is a unix timestamp 111 | //Yes, this is terrible, but supporting both timestamps and iso8601 at the same 112 | //time is tricky 113 | func ToTS(t string) string { 114 | //If it doesn't have a dash, it should be a unix timestamp already 115 | if !strings.Contains(t, "-") { 116 | return t 117 | } 118 | parsed, err := time.Parse(time.RFC3339, t) 119 | if err != nil { 120 | log.Fatalf("Unparsable timestamp, don't know what to do here: %v", t) 121 | } 122 | return fmt.Sprintf("%d", parsed.Unix()) 123 | } 124 | -------------------------------------------------------------------------------- /store_clickhouse.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "net/url" 7 | "time" 8 | 9 | _ "github.com/ClickHouse/clickhouse-go" 10 | "github.com/jmoiron/sqlx" 11 | ) 12 | 13 | var chschema = []string{ 14 | ` 15 | CREATE TABLE IF NOT EXISTS tuples ( 16 | whatever Date DEFAULT '2000-01-01', 17 | query String, 18 | type String, 19 | answer String, 20 | ttl AggregateFunction(anyLast, UInt16), 21 | first AggregateFunction(min, DateTime), 22 | last AggregateFunction(max, DateTime), 23 | count AggregateFunction(sum, UInt64) 24 | ) ENGINE = AggregatingMergeTree(whatever, (query, type, answer), 8192); 25 | `, 26 | 27 | ` 28 | CREATE TABLE IF NOT EXISTS individual ( 29 | whatever Date DEFAULT '2000-01-01', 30 | which Enum8('Q'=0, 'A'=1), 31 | value String, 32 | first AggregateFunction(min, DateTime), 33 | last AggregateFunction(max, DateTime), 34 | count AggregateFunction(sum, UInt64) 35 | ) ENGINE = AggregatingMergeTree(whatever, (which, value), 8192); 36 | `, 37 | ` 38 | CREATE TABLE IF NOT EXISTS filenames ( 39 | day Date DEFAULT toDate(ts), 40 | ts DateTime DEFAULT now(), 41 | filename String, 42 | aggregation_time Float64, 43 | total_records UInt64, 44 | skipped_records UInt64, 45 | tuples UInt64, 46 | individual UInt64, 47 | store_time Float64, 48 | inserted UInt64, 49 | updated UInt64 50 | ) ENGINE = MergeTree(day, (filename), 8192); 51 | `} 52 | 53 | const tuples_temp_stmt = ` 54 | CREATE TEMPORARY TABLE tuples_temp ( 55 | query String, 56 | type String, 57 | answer String, 58 | ttl String, 59 | first String, 60 | last String, 61 | count UInt64 62 | ) ENGINE = Memory` 63 | 64 | const individual_temp_stmt = ` 65 | CREATE TEMPORARY TABLE individual_temp ( 66 | which Enum8('Q'=0, 'A'=1), 67 | value String, 68 | first String, 69 | last String, 70 | count UInt64 71 | ) ENGINE = Memory` 72 | 73 | type CHStore struct { 74 | conn *sqlx.DB 75 | } 76 | 77 | func NewCHStore(uri string) (Store, error) { 78 | _, err := url.Parse(uri) 79 | if err != nil { 80 | return nil, err 81 | } 82 | 83 | conn, err := sqlx.Open("clickhouse", uri) 84 | if err != nil { 85 | return nil, err 86 | } 87 | err = conn.Ping() 88 | if err != nil { 89 | return nil, err 90 | } 91 | 92 | return &CHStore{ 93 | conn: conn, 94 | }, nil 95 | } 96 | 97 | func (s *CHStore) Close() error { 98 | return s.Close() 99 | } 100 | func (s *CHStore) Exec(stmt string) error { 101 | _, err := s.conn.Exec(stmt) 102 | return err 103 | } 104 | 105 | func (s *CHStore) Init() error { 106 | for _, stmt := range chschema { 107 | err := s.Exec(stmt) 108 | if err != nil { 109 | return err 110 | } 111 | } 112 | return nil 113 | } 114 | func (s *CHStore) Clear() error { 115 | stmts := []string{ 116 | "drop table filenames", 117 | "drop table individual", 118 | "drop table tuples", 119 | } 120 | for _, stmt := range stmts { 121 | err := s.Exec(stmt) 122 | if err != nil { 123 | return err 124 | } 125 | } 126 | return nil 127 | } 128 | 129 | func (s *CHStore) Begin() error { 130 | return fmt.Errorf("clickhouse doesn't support transactions") 131 | } 132 | func (s *CHStore) Commit() error { 133 | //log.Printf("clickhouse doesn't support transactions") 134 | return nil 135 | } 136 | 137 | //DeleteOld Deletes records that haven't been seen in DAYS, returns the total records deleted 138 | func (s *CHStore) DeleteOld(days int64) (int64, error) { 139 | return 0, fmt.Errorf("clickhouse doesn't support delete") 140 | } 141 | 142 | func (s *CHStore) Update(ar aggregationResult) (UpdateResult, error) { 143 | var result UpdateResult 144 | var err error 145 | start := time.Now() 146 | 147 | s.Exec("DROP TABLE tuples_temp") 148 | s.Exec("DROP TABLE individual_temp") 149 | 150 | err = s.Exec(tuples_temp_stmt) 151 | if err != nil { 152 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 153 | } 154 | err = s.Exec(individual_temp_stmt) 155 | if err != nil { 156 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 157 | } 158 | 159 | tx, err := s.conn.Begin() 160 | if err != nil { 161 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 162 | } 163 | stmt, err := tx.Prepare(`INSERT INTO tuples_temp 164 | (query, type, answer, ttl, first, last, count) 165 | values (?,?,?,?,?,?,?)`, 166 | ) 167 | if err != nil { 168 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 169 | } 170 | // Ok, now let's update stuff 171 | // tuples 172 | for _, q := range ar.Tuples { 173 | //Update the tuples table 174 | query := Reverse(q.query) 175 | _, err := stmt.Exec(query, q.qtype, q.answer, q.ttl, ToTS(q.first), ToTS(q.last), uint64(q.count)) 176 | if err != nil { 177 | return result, fmt.Errorf("CHStore.Update failed to run query: %w", err) 178 | } 179 | } 180 | err = tx.Commit() 181 | if err != nil { 182 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 183 | } 184 | err = s.Exec(`INSERT INTO tuples (query, type, answer, ttl, first, last, count) SELECT 185 | query, type, answer, 186 | anyLastState(toUInt16(ttl)), 187 | minState(toDateTime(toFloat64(first))), 188 | maxState(toDateTime(toFloat64(last))), 189 | sumState(count) from tuples_temp group by query, type, answer`, 190 | ) 191 | if err != nil { 192 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 193 | } 194 | 195 | tx, err = s.conn.Begin() 196 | if err != nil { 197 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 198 | } 199 | // Individuals 200 | stmt, err = tx.Prepare(`INSERT INTO individual_temp 201 | (value, which, first, last, count) 202 | values (?,?,?,?,?)`, 203 | ) 204 | if err != nil { 205 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 206 | } 207 | for _, q := range ar.Individual { 208 | //Update the tuples table 209 | value := q.value 210 | if q.which == "Q" { 211 | value = Reverse(value) 212 | } 213 | _, err := stmt.Exec(value, q.which, ToTS(q.first), ToTS(q.last), uint64(q.count)) 214 | if err != nil { 215 | return result, fmt.Errorf("CHStore.Update failed to run query: %w", err) 216 | } 217 | } 218 | err = tx.Commit() 219 | if err != nil { 220 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 221 | } 222 | err = s.Exec(`INSERT INTO individual (which, value, first, last, count) SELECT which, value, 223 | minState(toDateTime(toFloat64(first))), 224 | maxState(toDateTime(toFloat64(last))), 225 | sumState(count) from individual_temp group by which, value`) 226 | if err != nil { 227 | return result, fmt.Errorf("CHStore.Update failed: %w", err) 228 | } 229 | 230 | result.Updated = uint(ar.TuplesLen + ar.IndividualLen) 231 | result.Duration = time.Since(start) 232 | return result, nil 233 | } 234 | 235 | func (s *CHStore) IsLogIndexed(filename string) (bool, error) { 236 | var fn string 237 | err := s.conn.QueryRow("SELECT filename FROM filenames WHERE filename=?", filename).Scan(&fn) 238 | switch { 239 | case err == sql.ErrNoRows: 240 | return false, nil 241 | case err != nil: 242 | return false, err 243 | default: 244 | return true, nil 245 | } 246 | } 247 | func (s *CHStore) SetLogIndexed(filename string, ar aggregationResult, ur UpdateResult) error { 248 | tx, _ := s.conn.Begin() 249 | q := `INSERT INTO filenames (filename, 250 | aggregation_time, total_records, skipped_records, tuples, individual, 251 | store_time, inserted, updated) 252 | VALUES (?,?,?,?,?,?,?,?,?)` 253 | _, err := tx.Exec(q, filename, 254 | ar.Duration.Seconds(), uint64(ar.TotalRecords), uint64(ar.SkippedRecords), len(ar.Tuples), len(ar.Individual), 255 | ur.Duration.Seconds(), uint64(ur.Inserted), uint64(ur.Updated)) 256 | if err != nil { 257 | return err 258 | } 259 | return tx.Commit() 260 | } 261 | 262 | func (s *CHStore) FindQueryTuples(query string) (tupleResults, error) { 263 | tr := []tupleResult{} 264 | query = Reverse(query) 265 | err := s.conn.Select(&tr, "SELECT * FROM tuples WHERE query = ?", query) 266 | reverseQuery(tr) 267 | return tr, err 268 | } 269 | func (s *CHStore) FindTuples(query string) (tupleResults, error) { 270 | tr := []tupleResult{} 271 | rquery := Reverse(query) 272 | err := s.conn.Select(&tr, "SELECT query, type, answer, anyLastMerge(ttl) as ttl, minMerge(first) as first, maxMerge(last) as last, sumMerge(count) as count from tuples WHERE query = ? OR answer = ? group by query, type, answer ORDER BY query, answer", rquery, query) 273 | reverseQuery(tr) 274 | 275 | return tr, err 276 | } 277 | func (s *CHStore) LikeTuples(query string) (tupleResults, error) { 278 | tr := []tupleResult{} 279 | rquery := Reverse(query) 280 | err := s.conn.Select(&tr, "SELECT query, type, answer, anyLastMerge(ttl) as ttl, minMerge(first) as first, maxMerge(last) as last, sumMerge(count) as count from tuples WHERE query like ? OR answer like ? group by query, type, answer ORDER BY query, answer", rquery+"%", query+"%") 281 | reverseQuery(tr) 282 | return tr, err 283 | } 284 | func (s *CHStore) FindIndividual(value string) (individualResults, error) { 285 | rvalue := Reverse(value) 286 | tr := []individualResult{} 287 | err := s.conn.Select(&tr, `SELECT which, value, minMerge(first) as first, maxMerge(last) as last, sumMerge(count) as count from individual WHERE (which='A' AND value = ?) OR (which='Q' AND value = ?) group by which, value ORDER BY value`, value, rvalue) 288 | reverseValue(tr) 289 | return tr, err 290 | } 291 | 292 | func (s *CHStore) LikeIndividual(value string) (individualResults, error) { 293 | rvalue := Reverse(value) 294 | tr := []individualResult{} 295 | err := s.conn.Select(&tr, `SELECT which, value, minMerge(first) as first, maxMerge(last) as last, sumMerge(count) as count from individual WHERE (which='A' AND value like ?) OR (which='Q' AND value like ?) group by which, value ORDER BY value`, value+"%", rvalue+"%") 296 | reverseValue(tr) 297 | return tr, err 298 | } 299 | -------------------------------------------------------------------------------- /store_pg.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | "strings" 8 | "time" 9 | 10 | "github.com/jmoiron/sqlx" 11 | 12 | "github.com/lib/pq" 13 | ) 14 | 15 | const pgschema = ` 16 | set synchronous_commit to off; 17 | CREATE TABLE IF NOT EXISTS tuples ( 18 | query text, 19 | type text, 20 | answer text, 21 | count bigint, 22 | ttl integer, 23 | first timestamp, 24 | last timestamp, 25 | PRIMARY KEY (query, type, answer) 26 | ) ; 27 | CREATE INDEX tuples_query ON tuples(query varchar_pattern_ops); 28 | CREATE INDEX tuples_answer ON tuples(answer varchar_pattern_ops); 29 | -- CREATE INDEX tuples_first ON tuples(first); 30 | -- CREATE INDEX tuples_last ON tuples(last); 31 | 32 | CREATE TABLE IF NOT EXISTS individual ( 33 | which char(1), 34 | value text, 35 | count bigint, 36 | first timestamp, 37 | last timestamp, 38 | PRIMARY KEY (which, value) 39 | ); 40 | CREATE INDEX individual_value ON individual(value varchar_pattern_ops); 41 | -- CREATE INDEX individual_first ON individual(first); 42 | -- CREATE INDEX individual_last ON individual(last); 43 | 44 | CREATE TABLE IF NOT EXISTS filenames ( 45 | filename text PRIMARY KEY UNIQUE NOT NULL, 46 | time timestamp DEFAULT now(), 47 | aggregation_time real, 48 | total_records int, 49 | skipped_records int, 50 | tuples int, 51 | individual int, 52 | store_time real, 53 | inserted int, 54 | updated int 55 | ); 56 | CREATE OR REPLACE FUNCTION update_individual(w char(1), v text, c integer,f timestamp,l timestamp) RETURNS CHAR(1) AS 57 | $$ 58 | BEGIN 59 | LOOP 60 | -- first try to update the key 61 | UPDATE individual SET count=count+c, 62 | first=least(f, first), 63 | last =greatest(l, last) 64 | WHERE value=v AND which=w; 65 | IF found THEN 66 | RETURN 'U'; 67 | END IF; 68 | -- not there, so try to insert the key 69 | -- if someone else inserts the same key concurrently, 70 | -- we could get a unique-key failure 71 | BEGIN 72 | INSERT INTO individual (value, which, count, first, last) VALUES (v,w,c,f,l); 73 | RETURN 'I'; 74 | EXCEPTION WHEN unique_violation THEN 75 | -- do nothing, and loop to try the UPDATE again 76 | END; 77 | END LOOP; 78 | END; 79 | $$ 80 | LANGUAGE plpgsql; 81 | 82 | 83 | CREATE OR REPLACE FUNCTION update_tuples(q text, ty text, a text, tt integer, c integer ,f timestamp,l timestamp) RETURNS CHAR(1) AS 84 | $$ 85 | BEGIN 86 | LOOP 87 | -- first try to update the key 88 | UPDATE tuples SET count=count+c, 89 | ttl=tt, 90 | first=least(f, first), 91 | last =greatest(l, last) 92 | WHERE query=q AND type=ty AND answer=a; 93 | IF found THEN 94 | RETURN 'U'; 95 | END IF; 96 | -- not there, so try to insert the key 97 | -- if someone else inserts the same key concurrently, 98 | -- we could get a unique-key failure 99 | BEGIN 100 | INSERT INTO tuples (query, type, answer, ttl, count, first, last) VALUES (q, ty, a, tt, c, f, l); 101 | RETURN 'I'; 102 | EXCEPTION WHEN unique_violation THEN 103 | -- do nothing, and loop to try the UPDATE again 104 | END; 105 | END LOOP; 106 | END; 107 | $$ 108 | LANGUAGE plpgsql; 109 | ` 110 | 111 | type PGStore struct { 112 | conn *sqlx.DB 113 | *SQLCommonStore 114 | } 115 | 116 | func NewPGStore(uri string) (Store, error) { 117 | conn, err := sqlx.Open("postgres", uri) 118 | if err != nil { 119 | return nil, err 120 | } 121 | common := &SQLCommonStore{conn: conn} 122 | return &PGStore{conn: conn, SQLCommonStore: common}, nil 123 | } 124 | 125 | func (s *PGStore) Close() error { 126 | return s.Close() 127 | } 128 | 129 | func (s *PGStore) Init() error { 130 | _, err := s.conn.Exec(pgschema) 131 | // Ignore a duplicte table error message 132 | if pqerr, ok := err.(*pq.Error); ok { 133 | if pqerr.Code == "42P07" { 134 | return nil 135 | } 136 | } 137 | 138 | return err 139 | } 140 | 141 | func genFullBatchSelect(tmpl string, batchSize int) string { 142 | var queries []string 143 | numParams := strings.Count(tmpl, "$") 144 | arg := 1 145 | for i := 0; i < batchSize; i++ { 146 | var args []interface{} 147 | for p := 0; p < numParams; p++ { 148 | args = append(args, arg) 149 | arg++ 150 | } 151 | queries = append(queries, fmt.Sprintf(tmpl, args...)) 152 | } 153 | fullq := fmt.Sprintf("SELECT %s", strings.Join(queries, " || ")) 154 | return fullq 155 | } 156 | 157 | var BATCHSIZE = 200 158 | 159 | func (s *PGStore) Update(ar aggregationResult) (UpdateResult, error) { 160 | var result UpdateResult 161 | start := time.Now() 162 | 163 | tx, err := s.BeginTx() 164 | if err != nil { 165 | return result, err 166 | } 167 | //Setup the 2 different prepared statements 168 | updateTupleTmpl := "update_tuples($%d, $%d, $%d, $%d, $%d, to_timestamp($%d)::timestamp, to_timestamp($%d)::timestamp)" 169 | updateTupleBatch, err := tx.Prepare(genFullBatchSelect(updateTupleTmpl, BATCHSIZE)) 170 | if err != nil { 171 | return result, err 172 | } 173 | defer updateTupleBatch.Close() 174 | 175 | updateIndividualTmpl := "update_individual($%d, $%d, $%d, to_timestamp($%d)::timestamp, to_timestamp($%d)::timestamp)" 176 | updateIndividualeBatch, err := tx.Prepare(genFullBatchSelect(updateIndividualTmpl, BATCHSIZE)) 177 | if err != nil { 178 | return result, err 179 | } 180 | defer updateIndividualeBatch.Close() 181 | 182 | var arguments []interface{} 183 | batchCounter := 0 184 | 185 | runBatch := func(tmpl string, preparedBatch *sql.Stmt, arguments []interface{}, batchSize int) { 186 | if batchSize == 0 { 187 | return 188 | } 189 | var stmt *sql.Stmt 190 | if batchSize == BATCHSIZE { 191 | stmt = preparedBatch 192 | } else { 193 | stmt, err = tx.Prepare(genFullBatchSelect(tmpl, batchSize)) 194 | defer stmt.Close() 195 | } 196 | res, err := stmt.Query(arguments...) 197 | //log.Printf("Fullq is: %s", fullq) 198 | //log.Printf("Arguments is: %#v", arguments) 199 | if err != nil { 200 | log.Fatal(err) 201 | } 202 | res.Next() 203 | var update_result string 204 | res.Scan(&update_result) 205 | res.Close() 206 | for _, ch := range update_result { 207 | if ch == 'I' { 208 | result.Inserted++ 209 | } else { 210 | result.Updated++ 211 | } 212 | } 213 | } 214 | 215 | // Ok, now let's update stuff 216 | for _, q := range ar.Tuples { 217 | //Update the tuples table 218 | query := Reverse(q.query) 219 | arguments = append(arguments, query, q.qtype, q.answer, q.ttl, q.count, ToTS(q.first), ToTS(q.last)) 220 | batchCounter++ 221 | if batchCounter == BATCHSIZE { 222 | runBatch(updateTupleTmpl, updateTupleBatch, arguments, batchCounter) 223 | arguments = arguments[:0] 224 | batchCounter = 0 225 | } 226 | } 227 | runBatch(updateTupleTmpl, updateTupleBatch, arguments, batchCounter) 228 | arguments = arguments[:0] 229 | batchCounter = 0 230 | for _, q := range ar.Individual { 231 | value := q.value 232 | if q.which == "Q" { 233 | value = Reverse(value) 234 | } 235 | arguments = append(arguments, q.which, value, q.count, ToTS(q.first), ToTS(q.last)) 236 | batchCounter++ 237 | if batchCounter == BATCHSIZE { 238 | runBatch(updateIndividualTmpl, updateIndividualeBatch, arguments, batchCounter) 239 | arguments = arguments[:0] 240 | batchCounter = 0 241 | } 242 | } 243 | runBatch(updateIndividualTmpl, updateIndividualeBatch, arguments, batchCounter) 244 | result.Duration = time.Since(start) 245 | return result, s.Commit() 246 | } 247 | -------------------------------------------------------------------------------- /store_sql_common.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "database/sql" 5 | "errors" 6 | "time" 7 | 8 | "github.com/jmoiron/sqlx" 9 | ) 10 | 11 | func Reverse(s string) string { 12 | runes := []rune(s) 13 | for i, j := 0, len(runes)-1; i < j; i, j = i+1, j-1 { 14 | runes[i], runes[j] = runes[j], runes[i] 15 | } 16 | return string(runes) 17 | } 18 | 19 | type SQLCommonStore struct { 20 | conn *sqlx.DB 21 | tx *sql.Tx 22 | txDepth int 23 | } 24 | 25 | func (s *SQLCommonStore) Clear() error { 26 | _, err := s.conn.Exec("DELETE FROM filenames;DELETE FROM individual;DELETE FROM tuples;") 27 | return err 28 | } 29 | func (s *SQLCommonStore) Begin() error { 30 | _, err := s.BeginTx() 31 | return err 32 | } 33 | 34 | func (s *SQLCommonStore) BeginTx() (*sql.Tx, error) { 35 | if s.tx != nil { 36 | s.txDepth += 1 37 | //log.Printf("Returning existing transaction: depth=%d\n", s.txDepth) 38 | return s.tx, nil 39 | } 40 | //log.Printf("new transaction\n") 41 | tx, err := s.conn.Begin() 42 | if err != nil { 43 | return tx, err 44 | } 45 | s.tx = tx 46 | s.txDepth += 1 47 | return s.tx, nil 48 | } 49 | func (s *SQLCommonStore) Commit() error { 50 | if s.tx == nil { 51 | return errors.New("Commit outside of transaction") 52 | } 53 | s.txDepth -= 1 54 | if s.txDepth > 0 { 55 | //log.Printf("Not commiting stacked transaction: depth=%d\n", s.txDepth) 56 | return nil // No OP 57 | } 58 | //log.Printf("Commiting transaction: depth=%d\n", s.txDepth) 59 | err := s.tx.Commit() 60 | s.tx = nil 61 | return err 62 | } 63 | 64 | func (s *SQLCommonStore) IsLogIndexed(filename string) (bool, error) { 65 | tx, err := s.BeginTx() 66 | if err != nil { 67 | return false, err 68 | } 69 | defer s.Commit() 70 | var fn string 71 | err = tx.QueryRow("SELECT filename FROM filenames WHERE filename=$1", filename).Scan(&fn) 72 | switch { 73 | case err == sql.ErrNoRows: 74 | return false, nil 75 | case err != nil: 76 | return false, err 77 | default: 78 | return true, nil 79 | } 80 | } 81 | 82 | func (s *SQLCommonStore) SetLogIndexed(filename string, ar aggregationResult, ur UpdateResult) error { 83 | tx, err := s.BeginTx() 84 | defer s.Commit() 85 | if err != nil { 86 | return err 87 | } 88 | q := `INSERT INTO filenames (filename, 89 | aggregation_time, total_records, skipped_records, tuples, individual, 90 | store_time, inserted, updated) 91 | VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9)` 92 | _, err = tx.Exec(q, filename, 93 | ar.Duration.Seconds(), ar.TotalRecords, ar.SkippedRecords, ar.TuplesLen, ar.IndividualLen, 94 | ur.Duration.Seconds(), ur.Inserted, ur.Updated) 95 | return err 96 | } 97 | 98 | func reverseQuery(tr tupleResults) { 99 | for idx, rec := range tr { 100 | rec.Query = Reverse(rec.Query) 101 | tr[idx] = rec 102 | } 103 | } 104 | func reverseValue(tr individualResults) { 105 | for idx, rec := range tr { 106 | if rec.Which == "Q" { 107 | rec.Value = Reverse(rec.Value) 108 | tr[idx] = rec 109 | } 110 | } 111 | } 112 | 113 | func (s *SQLCommonStore) FindQueryTuples(query string) (tupleResults, error) { 114 | tr := []tupleResult{} 115 | query = Reverse(query) 116 | err := s.conn.Select(&tr, "SELECT * FROM tuples WHERE query = $1", query) 117 | reverseQuery(tr) 118 | return tr, err 119 | } 120 | func (s *SQLCommonStore) FindTuples(query string) (tupleResults, error) { 121 | tr := []tupleResult{} 122 | rquery := Reverse(query) 123 | err := s.conn.Select(&tr, "SELECT * FROM tuples WHERE query = $1 OR answer = $2 ORDER BY query, answer", rquery, query) 124 | reverseQuery(tr) 125 | 126 | return tr, err 127 | } 128 | func (s *SQLCommonStore) LikeTuples(query string) (tupleResults, error) { 129 | tr := []tupleResult{} 130 | rquery := Reverse(query) 131 | err := s.conn.Select(&tr, "SELECT * FROM tuples WHERE query like $1 OR answer like $2 ORDER BY query, answer", rquery+"%", query+"%") 132 | reverseQuery(tr) 133 | return tr, err 134 | } 135 | func (s *SQLCommonStore) FindIndividual(value string) (individualResults, error) { 136 | rvalue := Reverse(value) 137 | tr := []individualResult{} 138 | err := s.conn.Select(&tr, "SELECT * FROM individual WHERE (which='A' AND value = $1) OR (which='Q' AND value = $2) ORDER BY value", value, rvalue) 139 | reverseValue(tr) 140 | return tr, err 141 | } 142 | 143 | func (s *SQLCommonStore) LikeIndividual(value string) (individualResults, error) { 144 | rvalue := Reverse(value) 145 | tr := []individualResult{} 146 | err := s.conn.Select(&tr, "SELECT * FROM individual WHERE (which='A' AND value like $1) OR (which='Q' AND value like $2) ORDER BY value", value+"%", rvalue+"%") 147 | reverseValue(tr) 148 | return tr, err 149 | } 150 | 151 | //DeleteOld Deletes records that haven't been seen in DAYS, returns the total records deleted 152 | func (s *SQLCommonStore) DeleteOld(days int64) (int64, error) { 153 | var deletedRows int64 154 | cutoff := time.Now().Add(time.Duration(-1*days) * time.Hour * 24) 155 | res, err := s.conn.Exec("DELETE FROM individual WHERE last < $1", cutoff) 156 | if err != nil { 157 | return deletedRows, err 158 | } 159 | rows, err := res.RowsAffected() 160 | deletedRows += rows 161 | if err != nil { 162 | return deletedRows, err 163 | } 164 | 165 | res, err = s.conn.Exec("DELETE FROM tuples WHERE last < $1", cutoff) 166 | if err != nil { 167 | return deletedRows, err 168 | } 169 | rows, err = res.RowsAffected() 170 | deletedRows += rows 171 | return deletedRows, err 172 | } 173 | -------------------------------------------------------------------------------- /store_sqlite.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "time" 5 | 6 | "github.com/jmoiron/sqlx" 7 | 8 | _ "github.com/mattn/go-sqlite3" 9 | ) 10 | 11 | const schema = ` 12 | CREATE TABLE IF NOT EXISTS tuples ( 13 | query character varying, 14 | type character varying, 15 | answer character varying, 16 | count integer, 17 | ttl integer, 18 | first REAL, 19 | last REAL, 20 | PRIMARY KEY (query, type, answer) 21 | ) ; 22 | CREATE INDEX IF NOT EXISTS tuples_query ON tuples(query); 23 | CREATE INDEX IF NOT EXISTS tuples_answer ON tuples(answer); 24 | CREATE INDEX IF NOT EXISTS tuples_first ON tuples(first); 25 | CREATE INDEX IF NOT EXISTS tuples_last ON tuples(last); 26 | 27 | CREATE TABLE IF NOT EXISTS individual ( 28 | which char(1), 29 | value character varying, 30 | count integer, 31 | first REAL, 32 | last REAL, 33 | PRIMARY KEY (which, value) 34 | ); 35 | CREATE INDEX IF NOT EXISTS individual_first ON individual(first); 36 | CREATE INDEX IF NOT EXISTS individual_last ON individual(last); 37 | 38 | CREATE TABLE IF NOT EXISTS filenames ( 39 | filename character varying PRIMARY KEY UNIQUE NOT NULL, 40 | time REAL DEFAULT (datetime('now', 'localtime')), 41 | aggregation_time real, 42 | total_records int, 43 | skipped_records int, 44 | tuples int, 45 | individual int, 46 | store_time real, 47 | inserted int, 48 | updated int 49 | ); 50 | PRAGMA case_sensitive_like=ON; 51 | -- PRAGMA journal_mode=WAL; 52 | -- PRAGMA synchronous=off; 53 | PRAGMA temp_store = MEMORY; 54 | PRAGMA cache_size = 5000; 55 | ` 56 | 57 | type SQLiteStore struct { 58 | conn *sqlx.DB 59 | *SQLCommonStore 60 | } 61 | 62 | func NewSQLiteStore(uri string) (Store, error) { 63 | conn, err := sqlx.Open("sqlite3", uri) 64 | if err != nil { 65 | return nil, err 66 | } 67 | common := &SQLCommonStore{conn: conn} 68 | return &SQLiteStore{conn: conn, SQLCommonStore: common}, nil 69 | } 70 | 71 | func (s *SQLiteStore) Close() error { 72 | return s.Close() 73 | } 74 | 75 | func (s *SQLiteStore) Init() error { 76 | _, err := s.conn.Exec(schema) 77 | return err 78 | } 79 | 80 | func (s *SQLiteStore) Update(ar aggregationResult) (UpdateResult, error) { 81 | var result UpdateResult 82 | start := time.Now() 83 | 84 | tx, err := s.BeginTx() 85 | if err != nil { 86 | return result, err 87 | } 88 | //Setup the 4 different prepared statements 89 | update_tuples, err := tx.Prepare(`UPDATE tuples SET 90 | count=count+$1, 91 | ttl=$2, 92 | first=min(datetime($3, 'unixepoch'), first), 93 | last =max(datetime($4, 'unixepoch'), last) 94 | WHERE query=$5 AND type=$6 AND answer=$7`) 95 | if err != nil { 96 | return result, err 97 | } 98 | defer update_tuples.Close() 99 | insert_tuples, err := tx.Prepare(`INSERT INTO tuples (query, type, answer, ttl, count, first, last) 100 | VALUES ($1, $2, $3, $4, $5, datetime($6, 'unixepoch'), datetime($7,'unixepoch'))`) 101 | if err != nil { 102 | return result, err 103 | } 104 | defer insert_tuples.Close() 105 | 106 | update_individual, err := tx.Prepare(`UPDATE individual SET 107 | count=count+$1, 108 | first=min(datetime($2, 'unixepoch'), first), 109 | last =max(datetime($3, 'unixepoch'), last) 110 | WHERE value=$4 AND which=$5`) 111 | if err != nil { 112 | return result, err 113 | } 114 | defer update_individual.Close() 115 | insert_individual, err := tx.Prepare(`INSERT INTO individual (value, which, count, first, last) 116 | VALUES ($1, $2, $3, datetime($4, 'unixepoch'), datetime($5,'unixepoch'))`) 117 | if err != nil { 118 | return result, err 119 | } 120 | defer insert_individual.Close() 121 | 122 | // Ok, now let's update stuff 123 | for _, q := range ar.Tuples { 124 | //Update the tuples table 125 | query := Reverse(q.query) 126 | res, err := update_tuples.Exec(q.count, q.ttl, ToTS(q.first), ToTS(q.last), query, q.qtype, q.answer) 127 | if err != nil { 128 | return result, err 129 | } 130 | rows, err := res.RowsAffected() 131 | if err != nil { 132 | return result, err 133 | } 134 | if rows == 0 { 135 | _, err := insert_tuples.Exec(query, q.qtype, q.answer, q.ttl, q.count, ToTS(q.first), ToTS(q.last)) 136 | if err != nil { 137 | return result, err 138 | } 139 | result.Inserted++ 140 | } else { 141 | result.Updated++ 142 | } 143 | } 144 | for _, q := range ar.Individual { 145 | value := q.value 146 | if q.which == "Q" { 147 | value = Reverse(value) 148 | } 149 | res, err := update_individual.Exec(q.count, ToTS(q.first), ToTS(q.last), value, q.which) 150 | if err != nil { 151 | return result, err 152 | } 153 | rows, err := res.RowsAffected() 154 | if err != nil { 155 | return result, err 156 | } 157 | if rows == 0 { 158 | _, err := insert_individual.Exec(value, q.which, q.count, ToTS(q.first), ToTS(q.last)) 159 | if err != nil { 160 | return result, err 161 | } 162 | result.Inserted++ 163 | } else { 164 | result.Updated++ 165 | } 166 | } 167 | result.Duration = time.Since(start) 168 | return result, s.Commit() 169 | } 170 | -------------------------------------------------------------------------------- /store_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type storeTest struct { 12 | storetype string 13 | uri string 14 | } 15 | 16 | var testStores = []storeTest{ 17 | {"sqlite", ":memory:"}, 18 | } 19 | 20 | func init() { 21 | pgTestUrl := os.Getenv("PG_TEST_URL") 22 | if pgTestUrl != "" { 23 | testStores = append(testStores, storeTest{"postgresql", pgTestUrl}) 24 | } 25 | chTestUrl := os.Getenv("CH_TEST_URL") 26 | if chTestUrl != "" { 27 | testStores = append(testStores, storeTest{"clickhouse", chTestUrl}) 28 | } 29 | } 30 | 31 | func doTestLogIndexed(t *testing.T, s Store) { 32 | s.Clear() 33 | s.Init() 34 | testFilename := "test.log" 35 | indexed, err := s.IsLogIndexed(testFilename) 36 | if err != nil { 37 | t.Fatal(err) 38 | } 39 | if indexed != false { 40 | t.Errorf("IsLogIndexed(%q) == %t, want false", testFilename, indexed) 41 | } 42 | 43 | var ar aggregationResult 44 | var ur UpdateResult 45 | 46 | err = s.SetLogIndexed(testFilename, ar, ur) 47 | if err != nil { 48 | t.Fatal(err) 49 | } 50 | indexed, err = s.IsLogIndexed(testFilename) 51 | if err != nil { 52 | t.Fatal(err) 53 | } 54 | if indexed != true { 55 | t.Errorf("IsLogIndexed(%q) == %t, want true", testFilename, indexed) 56 | } 57 | } 58 | 59 | func LoadFile(t *testing.T, s Store, fn string) UpdateResult { 60 | aggregator := NewDNSAggregator() 61 | err := aggregate(aggregator, fn) 62 | if err != nil { 63 | t.Fatal(err) 64 | } 65 | aggregated := aggregator.GetResult() 66 | result, err := s.Update(aggregated) 67 | if err != nil { 68 | t.Fatal(err) 69 | } 70 | return result 71 | } 72 | 73 | func doTestUpdating(t *testing.T, s Store, forward bool) { 74 | s.Clear() 75 | s.Init() 76 | 77 | var files []string 78 | 79 | if forward { 80 | files = []string{"test_data/reddit_1.txt", "test_data/reddit_2.txt"} 81 | } else { 82 | files = []string{"test_data/reddit_2.txt", "test_data/reddit_1.txt"} 83 | } 84 | 85 | result_a := LoadFile(t, s, files[0]) 86 | result_b := LoadFile(t, s, files[1]) 87 | 88 | // Hack for now. Clickhouse store doesn't report inserted vs updated 89 | //TODO: add a method to Store interface to return a bool for this 90 | expected_inserted := 31 91 | expected_updated := 0 92 | if _, ok := s.(*CHStore); ok { 93 | expected_inserted = 0 94 | expected_updated = 31 95 | } 96 | assert.EqualValues(t, result_a.Inserted, expected_inserted) 97 | assert.EqualValues(t, result_a.Updated, expected_updated) 98 | 99 | assert.EqualValues(t, result_b.Inserted, 0) 100 | assert.EqualValues(t, result_b.Updated, 31) 101 | 102 | recs, err := s.FindIndividual("www.reddit.com") 103 | if err != nil { 104 | t.Fatalf("Failed to find: %v", err) 105 | return 106 | } 107 | //www.reddit.com Q 2 2016-04-01 00:03:03 2016-04-01 21:55:04 108 | if assert.Equal(t, len(recs), 1) { 109 | rec := recs[0] 110 | assert.Equal(t, rec.Value, "www.reddit.com") 111 | assert.Equal(t, rec.Which, "Q") 112 | assert.EqualValues(t, rec.Count, 2) 113 | //This is stupid, but I need to fix things so that they return actual dates 114 | //and get a handle on the timezone BS. 115 | //So for now, ignore the ' ' vs 'T' difference, and the hour 116 | assert.Regexp(t, "2016-04-01...:03:03", rec.First) 117 | assert.Regexp(t, "2016-04-01...:55:04", rec.Last) 118 | } 119 | //www.reddit.com A 198.41.208.138 2 300 2016-04-01 00:03:03 2016-04-01 21:55:04 120 | 121 | trecs, err := s.FindTuples("198.41.208.138") 122 | if err != nil { 123 | t.Fatalf("Failed to find: %v", err) 124 | return 125 | } 126 | if assert.Equal(t, len(trecs), 1) { 127 | rec := trecs[0] 128 | assert.Equal(t, rec.Query, "www.reddit.com") 129 | assert.Equal(t, rec.Type, "A") 130 | assert.Equal(t, rec.Answer, "198.41.208.138") 131 | assert.EqualValues(t, rec.Count, 2) 132 | //This is stupid, but I need to fix things so that they return actual dates 133 | //and get a handle on the timezone BS. 134 | //So for now, ignore the ' ' vs 'T' difference, and the hour 135 | assert.Regexp(t, "2016-04-01...:03:03", rec.First) 136 | assert.Regexp(t, "2016-04-01...:55:04", rec.Last) 137 | } 138 | 139 | } 140 | 141 | // Output: 142 | //A: Inserted=31 Updated=0 143 | //B: Inserted=0 Updated=31 144 | //Individual records: 1 145 | //www.reddit.com Q 2 2016-04-01 00:03:03 2016-04-01 21:55:04 146 | //Tuple records: 1 147 | //www.reddit.com A 198.41.208.138 2 300 2016-04-01 00:03:03 2016-04-01 21:55:04 148 | 149 | func BenchmarkUpdateStores(b *testing.B) { 150 | for _, ts := range testStores { 151 | b.Run(fmt.Sprintf("Indexing/%s", ts.storetype), func(b *testing.B) { 152 | store, err := NewStore(ts.storetype, ts.uri) 153 | if err != nil { 154 | b.Fatalf("NewStore failed: %s", err) 155 | } 156 | 157 | aggregator := NewDNSAggregator() 158 | err = aggregate(aggregator, "big.log.gz") 159 | if err != nil { 160 | b.Fatal(err) 161 | } 162 | aggregated := aggregator.GetResult() 163 | b.ResetTimer() 164 | for i := 0; i < b.N; i++ { 165 | _, err := store.Update(aggregated) 166 | if err != nil { 167 | b.Fatalf("store.Update failed: %s", err) 168 | } 169 | } 170 | }) 171 | } 172 | } 173 | 174 | func testFile(t *testing.T, s Store, fn string) error { 175 | aggregator := NewDNSAggregator() 176 | err := aggregate(aggregator, fn) 177 | if err != nil { 178 | return err 179 | } 180 | aggregated := aggregator.GetResult() 181 | _, err = s.Update(aggregated) 182 | if err != nil { 183 | return err 184 | } 185 | return nil 186 | } 187 | 188 | func TestIndexingFiles(t *testing.T) { 189 | allFiles := []string{ 190 | "./test_data/nbtstat.log", 191 | "./test_data/garbage.log", 192 | "./test_data/bad_ttl.log", 193 | "./test_data/dns_json_iso8601.json", 194 | } 195 | for _, ts := range testStores { 196 | t.Run(fmt.Sprintf("Indexing/%s", ts.storetype), func(t *testing.T) { 197 | store, err := NewStore(ts.storetype, ts.uri) 198 | if err != nil { 199 | t.Fatalf("can't create store at %s: %v", ts.uri, err) 200 | } 201 | for _, fn := range allFiles { 202 | t.Run(fn, func(t *testing.T) { 203 | testFile(t, store, fn) 204 | }) 205 | } 206 | }) 207 | } 208 | } 209 | 210 | func doTestStore(t *testing.T, store Store) { 211 | t.Run("testLogIndexed", func(t *testing.T) { 212 | doTestLogIndexed(t, store) 213 | }) 214 | t.Run("forward", func(t *testing.T) { 215 | doTestUpdating(t, store, true) 216 | }) 217 | t.Run("reverse", func(t *testing.T) { 218 | doTestUpdating(t, store, false) 219 | }) 220 | } 221 | 222 | func TestStoreIndexing(t *testing.T) { 223 | for _, ts := range testStores { 224 | t.Run(ts.storetype, func(t *testing.T) { 225 | store, err := NewStore(ts.storetype, ts.uri) 226 | if err != nil { 227 | t.Fatalf("can't create store at %s: %v", ts.uri, err) 228 | } 229 | doTestStore(t, store) 230 | }) 231 | } 232 | } 233 | -------------------------------------------------------------------------------- /template/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | pdns{{if .Query}} - {{.Query }}{{end}} 5 | 6 | 15 | 16 | 17 | 18 | 19 |
20 |
21 | pdns search 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 |
30 |
31 | 32 | {{ if .Error }} 33 | Error searching: {{.Error }} 34 | {{ end }} 35 | 36 | {{ if .Individual }} 37 |

Individual values

38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | {{range $val := .Individual}} 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | {{end}} 58 | 59 |
ValueWhichCountFirstLast
{{$val.Value}} {{$val.Which}} {{$val.Count}} {{$val.First}} {{$val.Last}}
60 | {{ end }} 61 | 62 | {{ if .Tuples }} 63 |

Tuples

64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | {{range $val := .Tuples}} 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | {{end}} 88 | 89 |
QueryTypeAnswerTTLCountFirstLast
{{$val.Query}} {{$val.Type}} {{$val.Answer}} {{$val.TTL}} {{$val.Count}} {{$val.First}} {{$val.Last}}
90 | {{ end }} 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /test_data/bad_ttl.log: -------------------------------------------------------------------------------- 1 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[3286499344.0,14.0],"rejected":false} 2 | -------------------------------------------------------------------------------- /test_data/dns_json.log: -------------------------------------------------------------------------------- 1 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 2 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 3 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 4 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 5 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 6 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 7 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 8 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 9 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 10 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 11 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 12 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 13 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 14 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 15 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 16 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 17 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 18 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 19 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 20 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 21 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 22 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 23 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 24 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 25 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 26 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 27 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 28 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 29 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 30 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 31 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 32 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 33 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 34 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 35 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 36 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 37 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 38 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 39 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 40 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 41 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 42 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 43 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 44 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 45 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 46 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 47 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 48 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 49 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 50 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 51 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 52 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 53 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 54 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 55 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 56 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 57 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 58 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 59 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 60 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 61 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 62 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 63 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 64 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 65 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 66 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 67 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 68 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 69 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 70 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 71 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 72 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 73 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 74 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 75 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 76 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 77 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 78 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 79 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 80 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 81 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 82 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 83 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 84 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 85 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 86 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 87 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 88 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 89 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 90 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 91 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 92 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 93 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 94 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 95 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 96 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 97 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 98 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 99 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 100 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 101 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 102 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 103 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 104 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 105 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 106 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 107 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 108 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 109 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 110 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 111 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 112 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 113 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 114 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 115 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 116 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 117 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 118 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 119 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 120 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 121 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 122 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 123 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 124 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 125 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 126 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 127 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 128 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 129 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 130 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 131 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 132 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 133 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 134 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 135 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 136 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 137 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 138 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 139 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 140 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 141 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 142 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 143 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 144 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 145 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 146 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 147 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 148 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 149 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 150 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 151 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 152 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 153 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 154 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 155 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 156 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 157 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 158 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 159 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 160 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 161 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 162 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 163 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 164 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 165 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 166 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 167 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 168 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 169 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 170 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 171 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 172 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 173 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 174 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 175 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 176 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 177 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 178 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 179 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 180 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 181 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 182 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 183 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 184 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 185 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 186 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 187 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 188 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 189 | {"ts":1504226064.37572,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 190 | {"ts":1504226073.518031,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 191 | {"ts":1504226060.051722,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 192 | {"ts":1504226061.201758,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 193 | {"ts":1504226064.375771,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 194 | {"ts":1504226073.518083,"uid":"CWsRPl2Qx6gNhHARM","id.orig_h":"fe80::5ef9:38ff:fe95:536c","id.orig_p":5353,"id.resp_h":"ff02::fb","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 195 | {"ts":1504226093.757659,"uid":"CyCXT53AEW082wEJuk","id.orig_h":"192.168.2.161","id.orig_p":33275,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15917,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":true} 196 | {"ts":1504226093.772416,"uid":"CMbrzzpSPxeegCR4","id.orig_h":"192.168.2.161","id.orig_p":5841,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":15037,"rtt":0.001594,"query":"ssl.google-analytics.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":true,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["0.0.0.0"],"TTLs":[0.0],"rejected":false} 197 | {"ts":1504226100.889636,"uid":"CeRmRa4wChxaXX5bPl","id.orig_h":"192.168.2.144","id.orig_p":52291,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":38092,"rtt":0.013108,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.190","74.125.136.91","74.125.136.136","74.125.136.93"],"TTLs":[75.0,75.0,75.0,75.0],"rejected":false} 198 | {"ts":1504226100.892947,"uid":"CeRJhn3RZX4EGTnUqe","id.orig_h":"192.168.2.144","id.orig_p":51541,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":62630,"rtt":0.014942,"query":"sb.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:80f::200e"],"TTLs":[171.0],"rejected":false} 199 | {"ts":1504226100.986226,"uid":"CUGNDW1T74UvdK9Mf9","id.orig_h":"192.168.2.144","id.orig_p":63234,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":4160,"rtt":0.019922,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.177.101","64.233.177.102","64.233.177.138","64.233.177.100","64.233.177.113","64.233.177.139"],"TTLs":[196.0,196.0,196.0,196.0,196.0,196.0],"rejected":false} 200 | {"ts":1504226100.990028,"uid":"CWDOPh2uTQkhU7WH04","id.orig_h":"192.168.2.144","id.orig_p":61705,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9459,"rtt":0.017509,"query":"safebrowsing.cache.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:811::200e"],"TTLs":[95.0],"rejected":false} 201 | {"ts":1504226037.090323,"uid":"CC3BJqLx0zgN89c35","id.orig_h":"192.168.2.157","id.orig_p":53477,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":36570,"rtt":0.012916,"query":"www.reddit.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["reddit.map.fastly.net","151.101.57.140"],"TTLs":[100.0,14.0],"rejected":false} 202 | {"ts":1504226037.107365,"uid":"ConzS63HBcUhAmsxj1","id.orig_h":"192.168.2.157","id.orig_p":37538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":51137,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 203 | {"ts":1504226037.121395,"uid":"CjjfFwN6sIpL3CNX6","id.orig_h":"192.168.2.157","id.orig_p":57316,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":46466,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 204 | {"ts":1504226061.300376,"uid":"CC5fjfToo9iIu1S6d","id.orig_h":"192.168.2.157","id.orig_p":60315,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":26918,"rtt":0.012398,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["173.194.175.106","173.194.175.99","173.194.175.147","173.194.175.103","173.194.175.105","173.194.175.104"],"TTLs":[153.0,153.0,153.0,153.0,153.0,153.0],"rejected":false} 205 | {"ts":1504226061.31575,"uid":"Cwb8jt3ylC0gNoPhP2","id.orig_h":"192.168.2.157","id.orig_p":43538,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":57700,"rtt":0.009332,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:400d:c08::6a"],"TTLs":[59.0],"rejected":false} 206 | {"ts":1504226061.326262,"uid":"CbHWTgnel1m6H88hg","id.orig_h":"192.168.2.157","id.orig_p":57906,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":23801,"query":"www.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 207 | {"ts":1504226077.114845,"uid":"Cseinn4ur1mzgm7Kb1","id.orig_h":"192.168.2.157","id.orig_p":34365,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41784,"rtt":0.012139,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["151.101.57.140"],"TTLs":[13.0],"rejected":false} 208 | {"ts":1504226077.129055,"uid":"CjMZsD3QI6tfxwInEf","id.orig_h":"192.168.2.157","id.orig_p":35859,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":14171,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 209 | {"ts":1504226077.151007,"uid":"CYkgGr4Pvo594QvbEl","id.orig_h":"192.168.2.157","id.orig_p":59605,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":32601,"query":"reddit.map.fastly.net","qclass":1,"qclass_name":"C_INTERNET","qtype":15,"qtype_name":"MX","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 210 | {"ts":1504226083.730927,"uid":"Cc4BXO1HuOstqlUuZe","id.orig_h":"192.168.2.161","id.orig_p":28208,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":9969,"rtt":0.01341,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::5e"],"TTLs":[161.0],"rejected":false} 211 | {"ts":1504226083.749612,"uid":"CPx0AD3djrVpt5vcdf","id.orig_h":"192.168.2.161","id.orig_p":12786,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":13082,"rtt":0.012164,"query":"update.googleapis.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.136.94"],"TTLs":[116.0],"rejected":false} 212 | {"ts":1504226060.051483,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":32769,"qclass_name":"qclass-32769","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 213 | {"ts":1504226061.201694,"uid":"CYmiFV2KquvFq4lac4","id.orig_h":"192.168.2.144","id.orig_p":5353,"id.resp_h":"224.0.0.251","id.resp_p":5353,"proto":"udp","trans_id":0,"query":"_googlecast._tcp.local","qclass":1,"qclass_name":"C_INTERNET","qtype":12,"qtype_name":"PTR","AA":false,"TC":false,"RD":false,"RA":false,"Z":0,"rejected":false} 214 | -------------------------------------------------------------------------------- /test_data/dns_json_iso8601.json: -------------------------------------------------------------------------------- 1 | {"ts":"2019-08-27T21:00:00.058798Z","uid":"CERjsr4nnMSFf6Q617","id.orig_h":"192.168.2.116","id.orig_p":53678,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":30824,"rtt":0.007465,"query":"googlemail.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c06::11"],"TTLs":[299.0],"rejected":false} 2 | {"ts":"2019-08-27T21:00:00.050787Z","uid":"Cxx0no4PbWXq8e8aha","id.orig_h":"192.168.2.116","id.orig_p":56470,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":41183,"rtt":0.007586,"query":"googlemail.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["74.125.196.17","74.125.196.18","74.125.196.83","74.125.196.19"],"TTLs":[299.0,299.0,299.0,299.0],"rejected":false} 3 | {"ts":"2019-08-27T21:05:00.455676Z","uid":"CacroF3sLUaPW1Tpw7","id.orig_h":"192.168.2.116","id.orig_p":40955,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":1304,"rtt":0.009959,"query":"googlemail.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.185.19","64.233.185.18","64.233.185.17","64.233.185.83"],"TTLs":[299.0,299.0,299.0,299.0],"rejected":false} 4 | {"ts":"2019-08-27T21:05:00.466232Z","uid":"Cs1FUC32JFma4IAahi","id.orig_h":"192.168.2.116","id.orig_p":53514,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":21792,"rtt":0.008193,"query":"googlemail.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":28,"qtype_name":"AAAA","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["2607:f8b0:4002:c0c::13"],"TTLs":[299.0],"rejected":false} 5 | {"ts":"2019-08-27T21:05:25.005297Z","uid":"CfPP9AblRknTTsXY8","id.orig_h":"192.168.2.133","id.orig_p":64423,"id.resp_h":"192.168.2.1","id.resp_p":53,"proto":"udp","trans_id":52998,"rtt":0.000628,"query":"googlemail.l.google.com","qclass":1,"qclass_name":"C_INTERNET","qtype":1,"qtype_name":"A","rcode":0,"rcode_name":"NOERROR","AA":false,"TC":false,"RD":true,"RA":true,"Z":0,"answers":["64.233.185.83","64.233.185.17","64.233.185.18","64.233.185.19"],"TTLs":[274.0,274.0,274.0,274.0],"rejected":false} 6 | -------------------------------------------------------------------------------- /test_data/garbage.log: -------------------------------------------------------------------------------- 1 | {"ts":1546319860.813048,"uid":"CrkLPv18FirmddtvQ","id.orig_h":"1.2.3.4","id.orig_p":58517,"id.resp_h":"4.3.2.1","id.resp_p":53,"proto":"udp","trans_id":3083,"query":"\u001e\u00d1$ppyt\u00a38+\u0093\u0016\u0036\u00dc\u00f5\u00a5\u00ac[5\u00e9*y\u00a2\u00b3\u00876\u00be#\u00bd;}\u000a\u001a\u00f1@\u0026\u0000\u000fz\u0018\u0009.no\u0026\u00e4\u00bele\u00fc\u008c","qclass":29467,"qclass_name":"qclass-29467","qtype":35599,"qtype_name":"query-35599","AA":false,"TC":false,"RD":true,"RA":false,"Z":0,"rejected":false} 2 | -------------------------------------------------------------------------------- /test_data/nbtstat.log: -------------------------------------------------------------------------------- 1 | {"ts":1538959195.632702,"uid":"CDC4zv1yqQAZJ4dDXe","id.orig_h":"10.1.2.3","id.orig_p":137,"id.resp_h":"10.4.5.6","id.resp_p":137,"proto":"udp","trans_id":38607,"query":"*\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000\u0000","qclass":1,"qclass_name":"C_INTERNET","qtype":33,"qtype_name":"NBSTAT","AA":false,"TC":false,"RD":false,"RA":false,"Z":1,"rejected":false} 2 | -------------------------------------------------------------------------------- /test_data/reddit_1.txt: -------------------------------------------------------------------------------- 1 | #separator \x09 2 | #set_separator , 3 | #empty_field (empty) 4 | #unset_field - 5 | #path dns 6 | #open 2016-04-01-00-00-40 7 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected cc 8 | #types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval] bool string 9 | 1459468983.750000 COKlryLloBE6EB7oe 192.168.1.1 62834 198.41.222.24 53 udp 22543 www.reddit.com 1 C_INTERNET 1 A 0 NOERROR T F F F 0 198.41.209.142,198.41.208.142,198.41.208.143,198.41.208.140,198.41.209.139,198.41.209.140,198.41.209.141,198.41.208.138,198.41.209.136,198.41.208.137,198.41.209.137,198.41.208.139,198.41.209.143,198.41.208.141,198.41.209.138 300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000 F - 10 | -------------------------------------------------------------------------------- /test_data/reddit_2.txt: -------------------------------------------------------------------------------- 1 | #separator \x09 2 | #set_separator , 3 | #empty_field (empty) 4 | #unset_field - 5 | #path dns 6 | #open 2016-04-01-00-00-40 7 | #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto trans_id query qclass qclass_name qtype qtype_name rcode rcode_name AA TC RD RA Z answers TTLs rejected cc 8 | #types time string addr port addr port enum count string count string count string count string bool bool bool bool count vector[string] vector[interval] bool string 9 | 1459547704.500000 CSJeQh1Uu4zoeDAkTa 192.168.1.1 54496 173.245.58.24 53 udp 45516 www.reddit.com 1 C_INTERNET 1 A 0 NOERROR T F F F 0 198.41.208.138,198.41.208.142,198.41.208.143,198.41.209.136,198.41.209.143,198.41.209.140,198.41.208.137,198.41.208.141,198.41.209.137,198.41.209.142,198.41.208.139,198.41.208.140,198.41.209.141,198.41.209.139,198.41.209.138 300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000,300.000000 F - 10 | -------------------------------------------------------------------------------- /version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const ( 4 | VERSION = "0.7.0" 5 | ) 6 | -------------------------------------------------------------------------------- /web.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "embed" 5 | "encoding/json" 6 | "log" 7 | "net/http" 8 | "text/template" 9 | 10 | "github.com/gorilla/mux" 11 | ) 12 | 13 | //go:embed template 14 | var content embed.FS 15 | 16 | type pdnsHandler struct { 17 | s Store 18 | } 19 | 20 | func (h *pdnsHandler) handleSearchTuples(w http.ResponseWriter, req *http.Request) { 21 | vars := mux.Vars(req) 22 | searchType := vars["searchType"] 23 | query := vars["query"] 24 | 25 | if query == "" { 26 | http.Error(w, "Missing parameter: q", http.StatusBadRequest) 27 | return 28 | } 29 | var err error 30 | var recs tupleResults 31 | if searchType == "like" { 32 | recs, err = h.s.LikeTuples(query) 33 | } else { 34 | recs, err = h.s.FindTuples(query) 35 | } 36 | 37 | if err != nil { 38 | http.Error(w, err.Error(), http.StatusInternalServerError) 39 | return 40 | } 41 | 42 | json.NewEncoder(w).Encode(recs) 43 | } 44 | func (h *pdnsHandler) handleSearchIndividual(w http.ResponseWriter, req *http.Request) { 45 | vars := mux.Vars(req) 46 | searchType := vars["searchType"] 47 | query := vars["query"] 48 | 49 | if query == "" { 50 | http.Error(w, "Missing parameter: q", http.StatusBadRequest) 51 | return 52 | } 53 | var err error 54 | var recs individualResults 55 | if searchType == "like" { 56 | recs, err = h.s.LikeIndividual(query) 57 | } else { 58 | recs, err = h.s.FindIndividual(query) 59 | } 60 | 61 | if err != nil { 62 | http.Error(w, err.Error(), http.StatusInternalServerError) 63 | return 64 | } 65 | 66 | json.NewEncoder(w).Encode(recs) 67 | } 68 | 69 | type Results struct { 70 | Query string 71 | Exact bool 72 | Individual individualResults 73 | Tuples tupleResults 74 | Error error 75 | } 76 | 77 | func (h *pdnsHandler) handleUI(w http.ResponseWriter, req *http.Request) { 78 | var err error 79 | var res Results 80 | res.Query = req.FormValue("query") 81 | res.Exact = req.FormValue("exact") == "on" 82 | if res.Query != "" { 83 | if res.Exact { 84 | res.Individual, err = h.s.FindIndividual(res.Query) 85 | if err != nil { 86 | res.Error = err 87 | } 88 | res.Tuples, err = h.s.FindTuples(res.Query) 89 | if err != nil { 90 | res.Error = err 91 | } 92 | } else { 93 | res.Individual, err = h.s.LikeIndividual(res.Query) 94 | if err != nil { 95 | res.Error = err 96 | } 97 | res.Tuples, err = h.s.LikeTuples(res.Query) 98 | if err != nil { 99 | res.Error = err 100 | } 101 | } 102 | } 103 | 104 | var t = template.Must(template.New("index.html").ParseFS(content, "template/index.html")) 105 | err = t.Execute(w, res) 106 | if err != nil { 107 | log.Printf("Error rendering template: %v", err) 108 | } 109 | } 110 | 111 | func startWeb(s Store, bind string) { 112 | h := &pdnsHandler{s: s} 113 | r := mux.NewRouter() 114 | 115 | r.HandleFunc("/dns/{searchType}/tuples/{query}", h.handleSearchTuples) 116 | r.HandleFunc("/dns/{searchType}/individual/{query}", h.handleSearchIndividual) 117 | 118 | r.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 119 | http.Redirect(w, r, "/ui/", http.StatusSeeOther) 120 | }) 121 | r.HandleFunc("/ui/", h.handleUI) 122 | 123 | http.Handle("/", r) 124 | 125 | log.Printf("Listening on %q\n", bind) 126 | log.Fatal(http.ListenAndServe(bind, nil)) 127 | } 128 | --------------------------------------------------------------------------------