├── LICENSE ├── README.md ├── logeater-dnssec.go ├── logeater-queries.go ├── logeater-resolvers.go └── logeater-update.go /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2017, Men & Mice Services 4 | All rights reserved. 5 | 6 | Redistribution and use in source and binary forms, with or without 7 | modification, are permitted provided that the following conditions are met: 8 | 9 | * Redistributions of source code must retain the above copyright notice, this 10 | list of conditions and the following disclaimer. 11 | 12 | * Redistributions in binary form must reproduce the above copyright notice, 13 | this list of conditions and the following disclaimer in the documentation 14 | and/or other materials provided with the distribution. 15 | 16 | * Neither the name of the copyright holder nor the names of its 17 | contributors may be used to endorse or promote products derived from 18 | this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 21 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # logeater 2 | Log-Aggregation Tools for BIND 9 logs 3 | 4 | 5 | Example use of ```logeater-queries```: 6 | ===================================== 7 | 8 | Statistics: 9 | 10 | ``` 11 | cat query.n | ./logeater-queries -s | column -t -s ":" 12 | Query-Statistics 13 | 21385 total queries ( 100 % ) 14 | 20471 iterative queries ( 95 % ) 15 | 914 recursive queries ( 4 % ) 16 | 863 queries over TCP ( 4 % ) 17 | 16987 queries with EDNS support ( 79 % ) 18 | 15197 queries indicate DNSSEC support ( 71 % ) 19 | 8804 queries with DNSSEC validation disabled (CD-flag) ( 41 % ) 20 | 1571 queries TSIG signed ( 7 % ) 21 | ``` 22 | 23 | Example - printing network classes and query types of queries: 24 | 25 | ``` 26 | $ cat query.log | ./logeater-queries -c -t | column -t 27 | Query-Network-Classes 28 | 21379 : IN 29 | 6 : CH 30 | 31 | Query-Network-Types 32 | 8514 : A 33 | 4060 : AAAA 34 | 3079 : SOA 35 | 2372 : DNSKEY 36 | 927 : PTR 37 | 658 : MX 38 | 543 : NS 39 | 312 : DS 40 | 286 : TXT 41 | 186 : NSEC 42 | 129 : ANY 43 | 115 : CNAME 44 | ``` 45 | 46 | Example - printing the top ten query IP addresses with reverse name 47 | resolution (can be slow): 48 | 49 | ``` 50 | $ cat query.log | ./logeater-queries -i | head | column -t 51 | 52 | Query-IP-Addresses 53 | 1571 : 212.114.206.217 [muc.example.de.] 54 | 821 : 2620:74:13:4400::41 [dnsviz-db.verisignlabs.com.] 55 | 794 : 72.13.58.112 [dnsviz-db.verisignlabs.com.] 56 | 704 : 54.234.42.241 [241.compute-1.amazonaws.com.] 57 | 682 : 2001:19f0:5001:df:76d7:5703:ba0a:e220 [] 58 | 565 : 185.92.221.212 [185.92.221.212.vultr.com.] 59 | 467 : 185.22.143.29 [b9168f1d.cgn.dg-w.de.] 60 | 314 : 91.51.184.46 [3b82e.dip0.t-ipconnect.de.] 61 | ``` 62 | 63 | ```logeater-dnssec``` analyses the a log file with messages from the 64 | "DNSSEC" category and groups the error messages : 65 | 66 | ``` 67 | $ cat dnssec.log | ./logeater-dnssec | head 68 | 8727 : 0C9F6LGOE6NADAS8KG1CLIK9UO9G7EIG.ad/NSEC3: no valid signature found 69 | 6953 : ad/SOA: no valid signature found 70 | 3976 : sat-tv.com/A: got insecure response; parent indicates it should be secure 71 | 1730 : mozilla.com/SOA: no valid signature found 72 | 1586 : stream.bestvideostreaming.is/A: no valid signature found 73 | 1577 : 8FC1DQ3C2Q3ERFD4UO40ENDBTSFME5JO5.ad/NSEC3: no valid signature found 74 | 1576 : sat-tv.com/SOA: got insecure response; parent indicates it should be secure 75 | 1576 : cdws.eu-west-1.amazonaws.com.Cisco/AAAA: bad cache hit (amazonaws.com.Cisco/DS) 76 | 1483 : 0c9f6lgoe6n13ad9iu1clik9uo9g7eig.ad/NSEC3: no valid signature found 77 | 968 : cbr.de/NSEC: no valid signature found 78 | ``` 79 | 80 | ```logeater-resolver``` analyses the a log file with messages from the 81 | "resolver" category and groups the error messages: 82 | 83 | ``` 84 | $ cat resolvers.log | ./logeater-resolvers | head 85 | 42908 : s-cnc1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-cnc1.qq.com -- invalid response 86 | 42713 : s-tel1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-tel1.qq.com -- invalid response 87 | 42484 : s-os1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-os1.qq.com -- invalid response 88 | 42297 : s-cmn1.qq.com/AAAA: Name qq.com (SOA) not subdomain of zone ns-cmn1.qq.com -- invalid response 89 | 20346 : mails.sonymusicfans.com/DS: invalid response 90 | 10920 : tp1.glb.nist.gov/DS: invalid response 91 | 9693 : media.netd.com.tr/AAAA for client 192.0.2.165#3347: Name netd.com.tr (SOA) not subdomain of zone media.netd.com.tr -- invalid response 92 | 7932 : service.superc.net/AAAA for client 192.0.2.11#3073: Name superc.net (SOA) not subdomain of zone service.superc.net — invalid response 93 | 04597 : brickleonavon.com/NS for client 192.0.2.46#3073: Name . (SOA) not subdomain of zone brickleonavon.com -- invalid response 94 | 4474 : promo.mobile.de/AAAA for client 2001:db8:1800:88:78f9:ba4:45fe:d438#48296: Name mobile.de (SOA) not subdomain of zone promo.mobile.de -- invalid response 95 | ``` 96 | 97 | Compiling from source 98 | ===================== 99 | 100 | Install ```git``` (https://git-scm.com/) and the ```go``` 101 | (https://golang.org) programming language. Both are available in the 102 | repositories of most Linux and BSD distributions. 103 | 104 | Download the source 105 | 106 | ``` 107 | git clone https://github.com/menandmice-services/logeater.git 108 | ``` 109 | 110 | Compile the source 111 | 112 | ``` 113 | cd logeater 114 | go build logeater-queries.go 115 | go build logeater-resolvers.go 116 | go build logeater-dnssec.go 117 | ``` 118 | 119 | BIND 9 versions 120 | =============== 121 | 122 | Tested with BIND 9.10 and BIND 9.11. 123 | 124 | Binary download 125 | =============== 126 | 127 | Binary versions of the ```logeater``` tool can be found at http://packages.menandmice.com/logeater/ 128 | 129 | More Information 130 | ================ 131 | 132 | The ```logeater``` tools have been discussed in the "BIND 9 logging 133 | best practices" Webinar by Men & Mice: 134 | 135 | https://www.menandmice.com/resources/educational-resources/webinars/bind-9-logging-best-practices/ 136 | -------------------------------------------------------------------------------- /logeater-dnssec.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | type sortedMap struct { 12 | m map[string]int 13 | s []string 14 | } 15 | 16 | func (sm *sortedMap) Len() int { 17 | return len(sm.m) 18 | } 19 | 20 | func (sm *sortedMap) Less(i, j int) bool { 21 | return sm.m[sm.s[i]] > sm.m[sm.s[j]] 22 | } 23 | 24 | func (sm *sortedMap) Swap(i, j int) { 25 | sm.s[i], sm.s[j] = sm.s[j], sm.s[i] 26 | } 27 | 28 | func sortedKeys(m map[string]int) []string { 29 | sm := new(sortedMap) 30 | sm.m = m 31 | sm.s = make([]string, len(m)) 32 | i := 0 33 | for key, _ := range m { 34 | sm.s[i] = key 35 | i++ 36 | } 37 | sort.Sort(sm) 38 | return sm.s 39 | } 40 | 41 | func main() { 42 | errormap := make(map[string]int) 43 | scanner := bufio.NewScanner(os.Stdin) 44 | for scanner.Scan() { 45 | txt := scanner.Text() 46 | i := strings.Index(txt, "validating") 47 | if (i > 0) { 48 | msg := txt[i+11:] 49 | errormap[msg]++ 50 | } 51 | } 52 | for _, res := range sortedKeys(errormap) { 53 | fmt.Println(errormap[res], ":", res) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /logeater-queries.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "flag" 6 | "fmt" 7 | "net" 8 | "os" 9 | "sort" 10 | "strings" 11 | ) 12 | 13 | type sortedMap struct { 14 | m map[string]int 15 | s []string 16 | } 17 | 18 | func (sm *sortedMap) Len() int { 19 | return len(sm.m) 20 | } 21 | 22 | func (sm *sortedMap) Less(i, j int) bool { 23 | return sm.m[sm.s[i]] > sm.m[sm.s[j]] 24 | } 25 | 26 | func (sm *sortedMap) Swap(i, j int) { 27 | sm.s[i], sm.s[j] = sm.s[j], sm.s[i] 28 | } 29 | 30 | func sortedKeys(m map[string]int) []string { 31 | sm := new(sortedMap) 32 | sm.m = m 33 | sm.s = make([]string, len(m)) 34 | i := 0 35 | for key, _ := range m { 36 | sm.s[i] = key 37 | i++ 38 | } 39 | sort.Sort(sm) 40 | return sm.s 41 | } 42 | 43 | func main() { 44 | listdomains := flag.Bool("d", false, "list domain names") 45 | listqueryip := flag.Bool("i", false, "list query IP addresses") 46 | listqueryclass := flag.Bool("c", false, "list query network classes") 47 | listquerytype := flag.Bool("t", false, "list query type") 48 | liststats := flag.Bool("s", false, "list statistics") 49 | nolookup := flag.Bool("n", false, "no reverse IP lookup") 50 | flag.Parse() 51 | 52 | querynames := make(map[string]int) 53 | queryips := make(map[string]int) 54 | queryclasses := make(map[string]int) 55 | querytypes := make(map[string]int) 56 | 57 | qnum := 0 58 | qrecursive := 0 59 | qiterative := 0 60 | qedns := 0 61 | qdnssecok := 0 62 | qcd := 0 63 | qsigned := 0 64 | qtcp := 0 65 | 66 | scanner := bufio.NewScanner(os.Stdin) 67 | for scanner.Scan() { 68 | txt := scanner.Text() 69 | qnum++ 70 | 71 | // collect query names 72 | i := strings.Index(txt, "query: ") 73 | qname := strings.SplitN(txt[i+7:], " ", 5) 74 | querynames[strings.ToLower(qname[0])]++ 75 | 76 | // collect DNS query classes 77 | queryclasses[qname[1]]++ 78 | 79 | // collect DNS query types 80 | querytypes[qname[2]]++ 81 | 82 | // collect DNS query flags 83 | qflags := qname[3] 84 | if qflags[:1] == "-" { 85 | qiterative++ 86 | } else { 87 | qrecursive++ 88 | } 89 | 90 | if strings.ContainsRune(qflags,'E') { 91 | qedns++ 92 | } 93 | if strings.ContainsRune(qflags,'D') { 94 | qdnssecok++ 95 | } 96 | if strings.ContainsRune(qflags,'C') { 97 | qcd++ 98 | } 99 | if strings.ContainsRune(qflags,'S') { 100 | qsigned++ 101 | } 102 | if strings.ContainsRune(qflags,'T') { 103 | qtcp++ 104 | } 105 | 106 | // collect IP address of clients 107 | i = strings.Index(txt, "client @") 108 | ip := strings.SplitN(txt[i+19:], "#", 2) 109 | queryips[strings.ToLower(ip[0])]++ 110 | 111 | } 112 | 113 | if *listdomains { 114 | fmt.Println("Query-Domain-Names\n") 115 | 116 | for _, res := range sortedKeys(querynames) { 117 | fmt.Println(querynames[res], ":", res) 118 | } 119 | } 120 | 121 | if *listqueryclass { 122 | fmt.Println("Query-Network-Classes\n") 123 | 124 | for _, class := range sortedKeys(queryclasses) { 125 | fmt.Println(queryclasses[class], ":", class) 126 | } 127 | } 128 | 129 | if *listquerytype { 130 | fmt.Println("Query-Network-Types\n") 131 | 132 | for _, qtype := range sortedKeys(querytypes) { 133 | fmt.Println(querytypes[qtype], ":", qtype) 134 | } 135 | } 136 | 137 | if *listqueryip { 138 | fmt.Println("Query-IP-Addresses\n") 139 | 140 | for _, ip := range sortedKeys(queryips) { 141 | num := queryips[ip] 142 | if *nolookup == false { 143 | name, _ := net.LookupAddr(ip) 144 | fmt.Println(num, ":", ip, name) 145 | } else { 146 | fmt.Println(num, ":", ip) 147 | } 148 | } 149 | } 150 | 151 | if *liststats { 152 | fmt.Println("Query-Statistics\n") 153 | 154 | fmt.Println(qnum,": total queries ( 100 % )") 155 | fmt.Println(qiterative,": iterative queries (", qiterative*100/qnum,"% )") 156 | fmt.Println(qrecursive,": recursive queries (", qrecursive*100/qnum," % )") 157 | fmt.Println(qtcp,": queries over TCP (", qtcp*100/qnum," % )") 158 | fmt.Println(qedns,": queries with EDNS support (", qedns*100/qnum," % )") 159 | fmt.Println(qdnssecok,": queries indicate DNSSEC support (", qdnssecok*100/qnum," % )") 160 | fmt.Println(qcd,": queries with DNSSEC validation disabled (CD-flag) (", qcd*100/qnum," % )") 161 | fmt.Println(qsigned,": queries TSIG signed (", qsigned*100/qnum," % )") 162 | } 163 | 164 | } 165 | -------------------------------------------------------------------------------- /logeater-resolvers.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "strings" 8 | "sort" 9 | ) 10 | 11 | type sortedMap struct { 12 | m map[string]int 13 | s []string 14 | } 15 | 16 | func (sm *sortedMap) Len() int { 17 | return len(sm.m) 18 | } 19 | 20 | func (sm *sortedMap) Less(i, j int) bool { 21 | return sm.m[sm.s[i]] > sm.m[sm.s[j]] 22 | } 23 | 24 | func (sm *sortedMap) Swap(i, j int) { 25 | sm.s[i], sm.s[j] = sm.s[j], sm.s[i] 26 | } 27 | 28 | func sortedKeys(m map[string]int) []string { 29 | sm := new(sortedMap) 30 | sm.m = m 31 | sm.s = make([]string, len(m)) 32 | i := 0 33 | for key, _ := range m { 34 | sm.s[i] = key 35 | i++ 36 | } 37 | sort.Sort(sm) 38 | return sm.s 39 | } 40 | 41 | func main() { 42 | errormap := make(map[string]int) 43 | scanner := bufio.NewScanner(os.Stdin) 44 | for scanner.Scan() { 45 | txt := scanner.Text() 46 | i := strings.Index(txt, "resolving") 47 | if (i > 0) { 48 | msg := txt[i+11:] 49 | errormap[msg]++ 50 | } 51 | } 52 | for _, res := range sortedKeys(errormap) { 53 | fmt.Println(errormap[res],":",res) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /logeater-update.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | type sortedMap struct { 12 | m map[string]int 13 | s []string 14 | } 15 | 16 | func (sm *sortedMap) Len() int { 17 | return len(sm.m) 18 | } 19 | 20 | func (sm *sortedMap) Less(i, j int) bool { 21 | return sm.m[sm.s[i]] > sm.m[sm.s[j]] 22 | } 23 | 24 | func (sm *sortedMap) Swap(i, j int) { 25 | sm.s[i], sm.s[j] = sm.s[j], sm.s[i] 26 | } 27 | 28 | func sortedKeys(m map[string]int) []string { 29 | sm := new(sortedMap) 30 | sm.m = m 31 | sm.s = make([]string, len(m)) 32 | i := 0 33 | for key, _ := range m { 34 | sm.s[i] = key 35 | i++ 36 | } 37 | sort.Sort(sm) 38 | return sm.s 39 | } 40 | 41 | func main() { 42 | errormap := make(map[string]int) 43 | scanner := bufio.NewScanner(os.Stdin) 44 | for scanner.Scan() { 45 | txt := scanner.Text() 46 | i := strings.Index(txt, "updating") 47 | if (i > 0) { 48 | msg := txt[i+13:] 49 | errormap[msg]++ 50 | } 51 | } 52 | for _, res := range sortedKeys(errormap) { 53 | fmt.Println(errormap[res], ":", res) 54 | } 55 | } 56 | --------------------------------------------------------------------------------