├── .gitignore ├── .travis.yml ├── LICENSE.txt ├── Makefile ├── README.md ├── api.go ├── bgp.go ├── cmd └── udig │ └── main.go ├── ct.go ├── dns.go ├── dns_test.go ├── doc └── res │ └── udig_demo.gif ├── geo.go ├── go.mod ├── go.sum ├── http.go ├── log.go ├── tls.go ├── udig.go ├── utils.go ├── utils_test.go └── whois.go /.gitignore: -------------------------------------------------------------------------------- 1 | ### JetBrains ### 2 | .idea/ -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.10.x 4 | - 1.11.x 5 | - 1.12.x 6 | - tip -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 nx1.cz, LLC 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Vars and params 2 | GOCMD=go 3 | BINARY_NAME=udig 4 | PACKAGE=github.com/netrixone/udig/cmd/udig 5 | GEODB_NAME=IP2LOCATION-LITE-DB1.IPV6.BIN 6 | INSTALL_DIR=$(shell dirname "`which $(BINARY_NAME)`") 7 | 8 | all: build test 9 | 10 | clean: 11 | $(GOCMD) clean -i $(PACKAGE) 12 | rm -f $(BINARY_NAME) 13 | rm -f $(BINARY_NAME)_min 14 | rm -f $(GEODB_NAME) 15 | 16 | build: deps 17 | $(GOCMD) build -v -o $(BINARY_NAME) $(PACKAGE) 18 | 19 | install: deps test 20 | $(GOCMD) install $(PACKAGE) 21 | ifneq (,$(wildcard $(GEODB_NAME))) 22 | cp $(GEODB_NAME) "$(INSTALL_DIR)" 23 | endif 24 | 25 | release: deps test 26 | $(GOCMD) build -ldflags="-s -w" -v -o $(BINARY_NAME)_min $(PACKAGE) 27 | upx --brute $(BINARY_NAME)_min 28 | 29 | deps: 30 | $(GOCMD) get -v -t ./... 31 | ifeq (,$(wildcard $(GEODB_NAME))) 32 | wget -q -O tmp.zip "https://download.ip2location.com/lite/$(GEODB_NAME).ZIP" && unzip -p tmp.zip $(GEODB_NAME) > $(GEODB_NAME) && rm tmp.zip 33 | endif 34 | 35 | test: build 36 | $(GOCMD) test 37 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Build Status](https://travis-ci.com/netrixone/udig.svg?branch=master)](https://travis-ci.com/netrixone/udig) 2 | [![Go Report Card](https://goreportcard.com/badge/github.com/netrixone/udig)](https://goreportcard.com/report/github.com/netrixone/udig) 3 | [![Go Doc](https://godoc.org/github.com/netrixone/udig?status.svg)](https://godoc.org/github.com/netrixone/udig) 4 | [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnetrixone%2Fudig.svg?type=shield)](https://app.fossa.io/projects/git%2Bgithub.com%2Fnetrixone%2Fudig?ref=badge_shield) 5 | 6 | # ÜberDig - dig on steroids 7 | 8 | **Simple GoLang tool for domain recon.** 9 | 10 | The purpose of this tool is to provide fast overview of a target domain setup. Several active scanning techniques 11 | are employed for this purpose like DNS ping-pong, TLS certificate scraping, WHOIS banner parsing and more. 12 | Some tools on the other hand are not - intentionally (e.g. nmap, brute-force, search engines etc.). This is not 13 | a full-blown DNS enumerator, but rather something more unobtrusive and fast which can be deployed in long-term 14 | experiments with lots of targets. 15 | 16 | Feature set: 17 | 18 | - [x] Resolves a given domain to all DNS records of interest 19 | - [x] Resolves a given domain to a set of WHOIS contacts (selected properties only) 20 | - [x] Resolves a given domain to a TLS certificate chain 21 | - [x] Supports automatic NS discovery with custom override 22 | - [x] Dissects domains from resolutions and resolves them recursively 23 | - [x] Unobtrusive human-readable CLI output as well as machine readable JSON 24 | - [x] Supports multiple domains on the input 25 | - [x] Colorized output 26 | - [x] Parses domains in HTTP headers 27 | - [x] Parses domains in Certificate Transparency logs 28 | - [x] Parses IPs found in SPF record 29 | - [x] Looks up BGP AS for each discovered IP 30 | - [x] Looks up GeoIP record for each discovered IP 31 | - [ ] Attempts to detect DNS wildcards 32 | - [ ] Supports graph output 33 | 34 | ## Download as dependency 35 | 36 | `go get github.com/netrixone/udig` 37 | 38 | ## Basic usage 39 | 40 | ```go 41 | dig := udig.NewUdig() 42 | resolutions := dig.Resolve("example.com") 43 | for _, res := range resolutions { 44 | ... 45 | } 46 | ``` 47 | 48 | ## API 49 | 50 | ``` 51 | +------------+ 52 | | | 53 | +------+ Udig +-----------------------------------+ 54 | Delegates: | | | | 55 | | +------------+ | 56 | |* |* 57 | +------------------+ +------------+ 58 | | DomainResolver | | IPResolver | 59 | +----------------------> +------------------+ <------------------+ +------------+ 60 | | ^ ^ ^ | ^ ^ 61 | Implements: | +-----+ | | | | +-------+ 62 | | | | | | | | 63 | +-------------+ +-------------+ +--------------+ +---------------+ +------------+ +-------------+ +---------------+ 64 | | DNSResolver | | TLSResolver | | HTTPResolver | | WhoisResolver | | CTResolver | | BGPResolver | | GeoipResolver | 65 | +-------------+ +-------------+ +--------------+ +---------------+ +------------+ +-------------+ +---------------+ 66 | | | | | | | | 67 | | | | | | | | 68 | Produces: | | | | | | | 69 | | | | | | | | 70 | |* |* |* |* |* |* |* 71 | +-----------+ +----------------+ +------------+ +--------------+ +-------+ +----------+ +-------------+ 72 | | DNSRecord | | TLSCertificate | | HTTPHeader | | WhoisContact | | CTLog | | ASRecord | | GeoipRecord | 73 | +-----------+ +----------------+ +------------+ +--------------+ +-------+ +----------+ +-------------+ 74 | 75 | ``` 76 | 77 | ## CLI app 78 | 79 | ### Download app 80 | 81 | `go get github.com/netrixone/udig/cmd/udig` 82 | 83 | ### Build from the sources 84 | 85 | `make` or `make install` 86 | 87 | This will also download the latest GeoIP database (IPLocation-lite). 88 | 89 | ### Usage 90 | 91 | ```bash 92 | udig [-h|--help] [-v|--version] [-V|--verbose] [-s|--strict] 93 | [-d|--domain ""] [--ct:expired] [--ct:from ""] 94 | [--json] 95 | 96 | ÜberDig - dig on steroids v1.5 by stuchl4n3k 97 | 98 | Arguments: 99 | 100 | -h --help Print help information 101 | -v --version Print version and exit 102 | -V --verbose Be more verbose 103 | -s --strict Strict domain relation (TLD match) 104 | -d --domain Domain to resolve 105 | --ct:expired Collect expired CT logs 106 | --ct:from Date to collect logs from. Default: 1 year ago (2022-11-10) 107 | --json Output payloads as JSON objects 108 | ``` 109 | 110 | ### Demo 111 | 112 | ![udig demo](doc/res/udig_demo.gif) 113 | 114 | ## Dependencies and attributions 115 | 116 | * https://github.com/akamensky/argparse - Argparse for golang 117 | * https://github.com/miekg/dns - DNS library in Go 118 | * https://github.com/domainr/whois - Whois client for Go 119 | * https://github.com/ip2location/ip2location-go - GeoIP localization package. This product uses IP2Location LITE data available from [https://lite.ip2location.com](https://lite.ip2location.com). 120 | * https://www.team-cymru.com/IP-ASN-mapping.html - IP to ASN mapping service by Team Cymru 121 | 122 | ## License 123 | [![FOSSA Status](https://app.fossa.io/api/projects/git%2Bgithub.com%2Fnetrixone%2Fudig.svg?type=large)](https://app.fossa.io/projects/git%2Bgithub.com%2Fnetrixone%2Fudig?ref=badge_large) -------------------------------------------------------------------------------- /api.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "crypto/x509" 5 | "net/http" 6 | "time" 7 | 8 | "github.com/domainr/whois" 9 | "github.com/miekg/dns" 10 | ) 11 | 12 | ///////////////////////////////////////// 13 | // COMMON 14 | ///////////////////////////////////////// 15 | 16 | const ( 17 | // DefaultTimeout is a default timeout used in all network clients. 18 | DefaultTimeout = 3 * time.Second 19 | ) 20 | 21 | // ResolutionType is an enumeration type for resolutions types. 22 | type ResolutionType string 23 | 24 | const ( 25 | // TypeDNS is a type of all DNS resolutions. 26 | TypeDNS ResolutionType = "DNS" 27 | 28 | // TypeWHOIS is a type of all WHOIS resolutions. 29 | TypeWHOIS ResolutionType = "WHOIS" 30 | 31 | // TypeTLS is a type of all TLS resolutions. 32 | TypeTLS ResolutionType = "TLS" 33 | 34 | // TypeHTTP is a type of all HTTP resolutions. 35 | TypeHTTP ResolutionType = "HTTP" 36 | 37 | // TypeCT is a type of all CT resolutions. 38 | TypeCT ResolutionType = "CT" 39 | 40 | // TypeBGP is a type of all BGP resolutions. 41 | TypeBGP ResolutionType = "BGP" 42 | 43 | // TypeGEO is a type of all GeoIP resolutions. 44 | TypeGEO ResolutionType = "GEO" 45 | ) 46 | 47 | // Udig is a high-level facade for domain resolution which: 48 | // 1. delegates work to specific resolvers 49 | // 2. deals with domain crawling 50 | // 3. caches intermediate results and summarizes the outputs 51 | type Udig interface { 52 | Resolve(domain string) []Resolution 53 | AddDomainResolver(resolver DomainResolver) 54 | AddIPResolver(resolver IPResolver) 55 | } 56 | 57 | // DomainResolver is an API contract for all Resolver modules that resolve domains. 58 | // Discovered domains that relate to the original query are recursively resolved. 59 | type DomainResolver interface { 60 | ResolveDomain(domain string) Resolution // Resolves a given domain. 61 | } 62 | 63 | // IPResolver is an API contract for all Resolver modules that resolve IPs. 64 | type IPResolver interface { 65 | ResolveIP(ip string) Resolution // Resolves a given IP. 66 | } 67 | 68 | // Resolution is an API contract for all Resolutions (i.e. results). 69 | type Resolution interface { 70 | Type() ResolutionType // Returns a type of this resolution. 71 | Query() string // Returns the queried domain or IP. 72 | Domains() []string // Returns a list of domains discovered in this resolution. 73 | IPs() []string // Returns a list of IP addresses discovered in this resolution. 74 | } 75 | 76 | // ResolutionBase is a shared implementation for all Resolutions (i.e. results). 77 | type ResolutionBase struct { 78 | Resolution `json:"-"` 79 | query string 80 | } 81 | 82 | // Query getter. 83 | func (res *ResolutionBase) Query() string { 84 | return res.query 85 | } 86 | 87 | // Domains returns a list of domains discovered in this resolution. 88 | func (res *ResolutionBase) Domains() (domains []string) { 89 | // Not supported by default. 90 | return domains 91 | } 92 | 93 | // IPs returns a list of IP addresses discovered in this resolution. 94 | func (res *ResolutionBase) IPs() (ips []string) { 95 | // Not supported by default. 96 | return ips 97 | } 98 | 99 | ///////////////////////////////////////// 100 | // DNS 101 | ///////////////////////////////////////// 102 | 103 | // DNSResolver is a Resolver which is able to resolve a domain 104 | // to a bunch of the most interesting DNS records. 105 | // 106 | // You can configure which query types are actually used 107 | // and you can also supply a custom name server. 108 | // If you don't a name server for each domain is discovered 109 | // using NS record query, falling back to a local NS 110 | // (e.g. the one in /etc/resolv.conf). 111 | type DNSResolver struct { 112 | DomainResolver 113 | QueryTypes []uint16 114 | NameServer string 115 | Client *dns.Client 116 | nameServerCache map[string]string 117 | resolvedDomains map[string]bool 118 | } 119 | 120 | // DNSResolution is a DNS multi-query resolution yielding many DNS records 121 | // in a form of query-answer pairs. 122 | type DNSResolution struct { 123 | *ResolutionBase 124 | Records []DNSRecordPair 125 | nameServer string 126 | } 127 | 128 | // DNSRecordPair is a pair of DNS record type used in the query 129 | // and a corresponding record found in the answer. 130 | type DNSRecordPair struct { 131 | QueryType uint16 132 | Record *DNSRecord 133 | } 134 | 135 | // DNSRecord is a wrapper for the actual DNS resource record. 136 | type DNSRecord struct { 137 | dns.RR 138 | } 139 | 140 | ///////////////////////////////////////// 141 | // WHOIS 142 | ///////////////////////////////////////// 143 | 144 | // WhoisResolver is a Resolver responsible for resolution of a given 145 | // domain to a list of WHOIS contacts. 146 | type WhoisResolver struct { 147 | DomainResolver 148 | Client *whois.Client 149 | } 150 | 151 | // WhoisResolution is a WHOIS query resolution yielding many contacts. 152 | type WhoisResolution struct { 153 | *ResolutionBase 154 | Contacts []WhoisContact 155 | } 156 | 157 | // WhoisContact is a wrapper for any item of interest from a WHOIS banner. 158 | type WhoisContact struct { 159 | RegistryDomainId string 160 | Registrant string 161 | RegistrantOrganization string 162 | RegistrantStateProvince string 163 | RegistrantCountry string 164 | Registrar string 165 | RegistrarIanaId string 166 | RegistrarWhoisServer string 167 | RegistrarUrl string 168 | CreationDate string 169 | UpdatedDate string 170 | Registered string 171 | Changed string 172 | Expire string 173 | NSSet string 174 | Contact string 175 | Name string 176 | Address string 177 | } 178 | 179 | ///////////////////////////////////////// 180 | // TLS 181 | ///////////////////////////////////////// 182 | 183 | // TLSResolver is a Resolver responsible for resolution of a given domain 184 | // to a list of TLS certificates. 185 | type TLSResolver struct { 186 | DomainResolver 187 | Client *http.Client 188 | } 189 | 190 | // TLSResolution is a TLS handshake resolution, which yields a certificate chain. 191 | type TLSResolution struct { 192 | *ResolutionBase 193 | Certificates []TLSCertificate 194 | } 195 | 196 | // TLSCertificate is a wrapper for the actual x509.Certificate. 197 | type TLSCertificate struct { 198 | x509.Certificate 199 | } 200 | 201 | ///////////////////////////////////////// 202 | // HTTP 203 | ///////////////////////////////////////// 204 | 205 | // HTTPResolver is a Resolver responsible for resolution of a given domain 206 | // to a list of corresponding HTTP headers. 207 | type HTTPResolver struct { 208 | DomainResolver 209 | Headers []string 210 | Client *http.Client 211 | } 212 | 213 | // HTTPResolution is a HTTP header resolution yielding many HTTP protocol headers. 214 | type HTTPResolution struct { 215 | *ResolutionBase 216 | Headers []HTTPHeader 217 | } 218 | 219 | // HTTPHeader is a pair of HTTP header name and corresponding value(s). 220 | type HTTPHeader struct { 221 | Name string 222 | Value []string 223 | } 224 | 225 | ///////////////////////////////////////// 226 | // CT 227 | ///////////////////////////////////////// 228 | 229 | // CTResolver is a Resolver responsible for resolution of a given domain 230 | // to a list of CT logs. 231 | type CTResolver struct { 232 | DomainResolver 233 | Client *http.Client 234 | cachedResults map[string]*CTResolution 235 | } 236 | 237 | // CTResolution is a certificate transparency project resolution, which yields a CT log. 238 | type CTResolution struct { 239 | *ResolutionBase 240 | Logs []CTAggregatedLog 241 | } 242 | 243 | // CTAggregatedLog is a wrapper of a CT log that is aggregated over all logs 244 | // with the same CN in time. 245 | type CTAggregatedLog struct { 246 | CTLog 247 | FirstSeen string 248 | LastSeen string 249 | } 250 | 251 | // CTLog is a wrapper for attributes of interest that appear in the CT log. 252 | // The json mapping comes from crt.sh API schema. 253 | type CTLog struct { 254 | Id int64 `json:"id"` 255 | IssuerName string `json:"issuer_name"` 256 | NameValue string `json:"name_value"` 257 | LoggedAt string `json:"entry_timestamp"` 258 | NotBefore string `json:"not_before"` 259 | NotAfter string `json:"not_after"` 260 | } 261 | 262 | ///////////////////////////////////////// 263 | // BGP 264 | ///////////////////////////////////////// 265 | 266 | // BGPResolver is a Resolver which is able to resolve an IP 267 | // to AS name and ASN. 268 | // 269 | // Internally this resolver is leveraging a DNS interface of 270 | // IP-to-ASN lookup service by Team Cymru. 271 | type BGPResolver struct { 272 | IPResolver 273 | Client *dns.Client 274 | cachedResults map[string]*BGPResolution 275 | } 276 | 277 | // BGPResolution is a BGP resolution of a given IP yielding AS records. 278 | type BGPResolution struct { 279 | *ResolutionBase 280 | Records []ASRecord 281 | } 282 | 283 | // ASRecord contains information about an Autonomous System (AS). 284 | type ASRecord struct { 285 | Name string 286 | ASN uint32 287 | BGPPrefix string 288 | Registry string 289 | Allocated string 290 | } 291 | 292 | ///////////////////////////////////////// 293 | // GEO 294 | ///////////////////////////////////////// 295 | 296 | // GeoResolver is a Resolver which is able to resolve an IP to a geographical location. 297 | type GeoResolver struct { 298 | IPResolver 299 | enabled bool 300 | cachedResults map[string]*GeoResolution 301 | } 302 | 303 | // GeoResolution is a GeoIP resolution of a given IP yielding geographical records. 304 | type GeoResolution struct { 305 | *ResolutionBase 306 | Record *GeoRecord 307 | } 308 | 309 | // GeoRecord contains information about a geographical location. 310 | type GeoRecord struct { 311 | CountryCode string 312 | } 313 | -------------------------------------------------------------------------------- /bgp.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "fmt" 5 | "github.com/miekg/dns" 6 | "net" 7 | "regexp" 8 | "strconv" 9 | ) 10 | 11 | var ( 12 | // For parsing ASN records, eg. "13335 | 104.28.16.0/20 | US | arin | 2014-03-28" 13 | asnRecordPattern = regexp.MustCompile(`([0-9]+) \| (.+) \| ([A-Z]+) \| (.+) \| (.+)`) 14 | // For parsing AS records e.g. "13335 | US | arin | 2010-07-14 | CLOUDFLARENET, US" 15 | asRecordPattern = regexp.MustCompile(`([0-9]+) \| ([A-Z]+) \| (.+) \| (.+) \| (.+)`) 16 | ) 17 | 18 | // lookupASN uses Team Cymru's IP->ASN lookup via DNS, returns matching ASN records. 19 | func lookupASN(ip string, client *dns.Client) (asnRecords []string) { 20 | ipAddr := net.ParseIP(ip) 21 | if ipAddr == nil { 22 | LogErr("%s: IP %s is invalid.", TypeBGP, ip) 23 | return asnRecords 24 | } 25 | 26 | var query string 27 | if ipAddr.To4() != nil { 28 | query = fmt.Sprintf("%s.origin.asn.cymru.com", reverseIPv4(ipAddr)) 29 | } else { 30 | query = fmt.Sprintf("%s.origin6.asn.cymru.com", reverseIPv6(ipAddr)) 31 | } 32 | 33 | msg, err := queryOneCallback(query, dns.TypeTXT, localNameServer, client) 34 | if err != nil { 35 | if err.Error() == "NXDOMAIN" { 36 | LogDebug("%s: No ASN record found for IP %s (query %s).", TypeBGP, ip, query) 37 | } else { 38 | LogErr("%s: Could not query BGP endpoint (TXT %s). The cause was: %s", TypeBGP, query, err.Error()) 39 | } 40 | return asnRecords 41 | } 42 | 43 | for _, record := range msg.Answer { 44 | if record.Header().Rrtype != dns.TypeTXT { 45 | LogDebug("%s: TXT query %s returned non-TXT record %s. Skipping.", TypeBGP, query, dns.TypeToString[record.Header().Rrtype]) 46 | continue 47 | } 48 | 49 | txt := (record).(*dns.TXT).Txt 50 | for _, val := range txt { 51 | asnRecords = append(asnRecords, val) 52 | } 53 | } 54 | 55 | return asnRecords 56 | } 57 | 58 | // lookupAS uses Team Cymru's ASN->AS lookup via DNS, returns a matching ASN record or "". 59 | func lookupAS(asn uint32, client *dns.Client) string { 60 | query := fmt.Sprintf("AS%d.asn.cymru.com", asn) 61 | 62 | msg, err := queryOneCallback(query, dns.TypeTXT, localNameServer, client) 63 | if err != nil { 64 | if err.Error() == "NXDOMAIN" { 65 | LogDebug("%s: No AS record found for AS%d (query %s).", TypeBGP, asn, query) 66 | } else { 67 | LogErr("%s: Could not query BGP endpoint (TXT %s). The cause was: %s", TypeBGP, query, err.Error()) 68 | } 69 | return "" 70 | } 71 | 72 | var asRecord string 73 | for _, record := range msg.Answer { 74 | if record.Header().Rrtype != dns.TypeTXT { 75 | LogDebug("%s: TXT query %s returned non-TXT record %s. Skipping.", TypeBGP, query, dns.TypeToString[record.Header().Rrtype]) 76 | continue 77 | } 78 | 79 | txt := (record).(*dns.TXT).Txt 80 | for _, val := range txt { 81 | asRecord = val 82 | break 83 | } 84 | break 85 | } 86 | 87 | return asRecord 88 | } 89 | 90 | // parseASNRecord parses a given ASN record string to ASRecord structure. 91 | // The string is expected to match following form: 92 | // "13335 | 104.28.16.0/20 | US | arin | 2014-03-28" 93 | func parseASNRecord(asnRecord string) *ASRecord { 94 | groups := asnRecordPattern.FindStringSubmatch(asnRecord) 95 | if groups == nil { 96 | LogErr("%s: Invalid ASN record '%s'.", TypeBGP, asnRecord) 97 | return nil 98 | } 99 | 100 | asn, err := strconv.ParseInt(groups[1], 10, 32) 101 | if err != nil { 102 | LogErr("%s: Invalid ASN '%s'.", TypeBGP, groups[1]) 103 | return nil 104 | } 105 | 106 | return &ASRecord{ 107 | ASN: uint32(asn), 108 | BGPPrefix: groups[2], 109 | Registry: groups[4], 110 | Allocated: groups[5], 111 | } 112 | } 113 | 114 | // parseASNRecord parses a given AS record string and returns AS name. 115 | // The string is expected to match following form: 116 | // "13335 | US | arin | 2010-07-14 | CLOUDFLARENET, US" 117 | func parseASName(asRecord string) string { 118 | groups := asRecordPattern.FindStringSubmatch(asRecord) 119 | if groups == nil { 120 | LogErr("%s: Invalid AS record '%s'.", TypeBGP, asRecord) 121 | return "" 122 | } 123 | 124 | return groups[5] 125 | } 126 | 127 | ///////////////////////////////////////// 128 | // BGP RESOLVER 129 | ///////////////////////////////////////// 130 | 131 | // NewBGPResolver creates a new BGPResolver with sensible defaults. 132 | func NewBGPResolver() *BGPResolver { 133 | return &BGPResolver{ 134 | Client: &dns.Client{ReadTimeout: DefaultTimeout}, 135 | cachedResults: map[string]*BGPResolution{}, 136 | } 137 | } 138 | 139 | // ResolveIP resolves a given IP address to a list of corresponding AS records. 140 | func (resolver *BGPResolver) ResolveIP(ip string) Resolution { 141 | resolution := resolver.cachedResults[ip] 142 | if resolution != nil { 143 | return resolution 144 | } 145 | resolution = &BGPResolution{ResolutionBase: &ResolutionBase{query: ip}} 146 | resolver.cachedResults[ip] = resolution 147 | 148 | results := lookupASN(ip, resolver.Client) 149 | for _, result := range results { 150 | asRecord := parseASNRecord(result) 151 | if asRecord == nil { 152 | continue 153 | } 154 | 155 | asRecord.Name = parseASName(lookupAS(asRecord.ASN, resolver.Client)) 156 | resolution.Records = append(resolution.Records, *asRecord) 157 | } 158 | 159 | return resolution 160 | } 161 | 162 | // Type returns "BGP". 163 | func (resolver *BGPResolver) Type() ResolutionType { 164 | return TypeBGP 165 | } 166 | 167 | ///////////////////////////////////////// 168 | // BGP RESOLUTION 169 | ///////////////////////////////////////// 170 | 171 | // Type returns "BGP". 172 | func (res *BGPResolution) Type() ResolutionType { 173 | return TypeBGP 174 | } 175 | 176 | ///////////////////////////////////////// 177 | // AS RECORD 178 | ///////////////////////////////////////// 179 | 180 | func (record *ASRecord) String() string { 181 | return fmt.Sprintf( 182 | "ASN: %d, AS: %s, prefix: %s, registry: %s, allocated: %s", 183 | record.ASN, record.Name, record.BGPPrefix, record.Registry, record.Allocated, 184 | ) 185 | } 186 | -------------------------------------------------------------------------------- /cmd/udig/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/url" 7 | "os" 8 | "time" 9 | 10 | "github.com/akamensky/argparse" 11 | "github.com/miekg/dns" 12 | "github.com/netrixone/udig" 13 | ) 14 | 15 | const ( 16 | prog = "udig" 17 | version = "1.5" 18 | author = "stuchl4n3k" 19 | description = "ÜberDig - dig on steroids v" + version + " by " + author 20 | ) 21 | 22 | var ( 23 | banner = ` 24 | _ _ ____ ___ ____ 25 | (_) (_| _ |_ _/ ___| 26 | | | | | | | | | | _ 27 | | |_| | |_| | | |_| | 28 | \__,_|____|___\____| v`[1:] + version + ` 29 | ` 30 | ) 31 | var outputJson = false 32 | 33 | func resolve(domain string) { 34 | // Some input checks. 35 | if !isValidDomain(domain) { 36 | udig.LogErr("'%s' does not appear like a valid domain to me -> skipping.", domain) 37 | return 38 | } 39 | 40 | dig := udig.NewUdig() 41 | resolutions := dig.Resolve(domain) 42 | 43 | for _, res := range resolutions { 44 | switch res.Type() { 45 | case udig.TypeDNS: 46 | for _, rr := range (res).(*udig.DNSResolution).Records { 47 | udig.LogInfo("%s: %s %s -> %s", res.Type(), dns.TypeToString[rr.QueryType], res.Query(), formatPayload(rr.Record)) 48 | } 49 | break 50 | 51 | case udig.TypeTLS: 52 | for _, cert := range (res).(*udig.TLSResolution).Certificates { 53 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&cert)) 54 | } 55 | break 56 | 57 | case udig.TypeWHOIS: 58 | for _, contact := range (res).(*udig.WhoisResolution).Contacts { 59 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&contact)) 60 | } 61 | break 62 | 63 | case udig.TypeHTTP: 64 | for _, header := range (res).(*udig.HTTPResolution).Headers { 65 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&header)) 66 | } 67 | break 68 | 69 | case udig.TypeCT: 70 | for _, ctLog := range (res).(*udig.CTResolution).Logs { 71 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&ctLog)) 72 | } 73 | break 74 | 75 | case udig.TypeBGP: 76 | for _, as := range (res).(*udig.BGPResolution).Records { 77 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload(&as)) 78 | } 79 | break 80 | 81 | case udig.TypeGEO: 82 | if (res).(*udig.GeoResolution).Record != nil { 83 | udig.LogInfo("%s: %s -> %s", res.Type(), res.Query(), formatPayload((res).(*udig.GeoResolution).Record)) 84 | } 85 | break 86 | } 87 | } 88 | } 89 | 90 | func isValidDomain(domain string) bool { 91 | if len(domain) == 0 { 92 | return false 93 | } 94 | 95 | if _, err := url.Parse("https://" + domain); err != nil { 96 | return false 97 | } 98 | 99 | return true 100 | } 101 | 102 | func formatPayload(resolution fmt.Stringer) string { 103 | if outputJson { 104 | result, _ := json.Marshal(resolution) 105 | return string(result) 106 | } 107 | return resolution.String() 108 | } 109 | 110 | func main() { 111 | parser := argparse.NewParser(prog, description) 112 | printVersion := parser.Flag("v", "version", &argparse.Options{Required: false, Help: "Print version and exit"}) 113 | beVerbose := parser.Flag("V", "verbose", &argparse.Options{Required: false, Help: "Be more verbose"}) 114 | beStrict := parser.Flag("s", "strict", &argparse.Options{Required: false, Help: "Strict domain relation (TLD match)"}) 115 | domain := parser.String("d", "domain", &argparse.Options{Required: false, Help: "Domain to resolve"}) 116 | ctExpired := parser.Flag("", "ct:expired", &argparse.Options{Required: false, Help: "Collect expired CT logs"}) 117 | ctFrom := parser.String("", "ct:from", &argparse.Options{ 118 | Required: false, 119 | Help: "Date to collect logs from", 120 | Default: fmt.Sprintf("1 year ago (%s)", udig.CTLogFrom), 121 | Validate: func(args []string) error { 122 | _, err := time.Parse("2006-01-02", args[0]) 123 | return err 124 | }, 125 | }) 126 | jsonOutput := parser.Flag("", "json", &argparse.Options{Required: false, Help: "Output payloads as JSON objects"}) 127 | 128 | err := parser.Parse(os.Args) 129 | if err != nil { 130 | fmt.Fprint(os.Stderr, parser.Usage(err)) 131 | os.Exit(1) 132 | } 133 | 134 | if *printVersion { 135 | fmt.Println(version) 136 | os.Exit(0) 137 | } else if *domain == "" { 138 | fmt.Fprint(os.Stderr, parser.Usage(err)) 139 | os.Exit(1) 140 | } 141 | 142 | if *beVerbose { 143 | udig.LogLevel = udig.LogLevelDebug 144 | } else { 145 | udig.LogLevel = udig.LogLevelInfo 146 | } 147 | 148 | if *beStrict { 149 | udig.IsDomainRelated = udig.StrictDomainRelation 150 | } 151 | 152 | if *ctExpired { 153 | udig.CTExclude = "" 154 | } 155 | 156 | if *ctFrom != "" { 157 | udig.CTLogFrom = *ctFrom 158 | } 159 | 160 | outputJson = *jsonOutput 161 | 162 | fmt.Println(banner) 163 | resolve(*domain) 164 | } 165 | -------------------------------------------------------------------------------- /ct.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net" 9 | "net/http" 10 | "time" 11 | ) 12 | 13 | ///////////////////////////////////////// 14 | // CT RESOLVER 15 | ///////////////////////////////////////// 16 | 17 | const DefaultCTApiUrl = "https://crt.sh" 18 | 19 | var CTApiUrl = DefaultCTApiUrl 20 | var CTLogFrom = time.Now().AddDate(-1, 0, 0).Format("2006-01-02") 21 | var CTExclude = "expired" 22 | 23 | // NewCTResolver creates a new CTResolver with sensible defaults. 24 | func NewCTResolver() *CTResolver { 25 | transport := http.DefaultTransport.(*http.Transport) 26 | 27 | transport.DialContext = (&net.Dialer{ 28 | Timeout: DefaultTimeout, 29 | KeepAlive: DefaultTimeout, 30 | }).DialContext 31 | 32 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 33 | transport.TLSHandshakeTimeout = DefaultTimeout 34 | 35 | client := &http.Client{ 36 | Transport: transport, 37 | Timeout: DefaultTimeout, 38 | } 39 | 40 | return &CTResolver{ 41 | Client: client, 42 | cachedResults: make(map[string]*CTResolution), 43 | } 44 | } 45 | 46 | // Type returns "CT". 47 | func (resolver *CTResolver) Type() ResolutionType { 48 | return TypeCT 49 | } 50 | 51 | // ResolveDomain resolves a given domain to a list of TLS certificates. 52 | func (resolver *CTResolver) ResolveDomain(domain string) Resolution { 53 | resolution := &CTResolution{ 54 | ResolutionBase: &ResolutionBase{query: domain}, 55 | } 56 | 57 | if cached := resolver.cacheLookup(domain); cached != nil { 58 | // Ignore, otherwise the output would burn without adding no/little value. 59 | return resolution 60 | } 61 | 62 | resolution.Logs = resolver.fetchLogs(domain) 63 | resolver.cachedResults[domain] = resolution 64 | 65 | return resolution 66 | } 67 | 68 | func (resolver *CTResolver) cacheLookup(domain string) *CTResolution { 69 | resolution := resolver.cachedResults[domain] 70 | if resolution != nil { 71 | return resolution 72 | } 73 | 74 | // Try parent domain as well (unless it is a 2nd order domain). 75 | for ; domain != ""; domain = ParentDomainOf(domain) { 76 | resolution = resolver.cachedResults[domain] 77 | if resolution != nil { 78 | return resolution 79 | } 80 | } 81 | 82 | return nil 83 | } 84 | 85 | func (resolver *CTResolver) fetchLogs(domain string) (logs []CTAggregatedLog) { 86 | url := fmt.Sprintf("%s/?match=LIKE&exclude=%s&CN=%s&output=json", CTApiUrl, CTExclude, domain) 87 | res, err := resolver.Client.Get(url) 88 | if err != nil { 89 | LogErr("%s: %s -> %s", TypeCT, domain, err.Error()) 90 | return logs 91 | } 92 | 93 | var rawBody []byte 94 | if rawBody, err = io.ReadAll(res.Body); err != nil { 95 | LogErr("%s: %s -> %s", TypeCT, domain, err.Error()) 96 | return logs 97 | } 98 | 99 | rawLogs := make([]CTLog, 0) 100 | if err = json.Unmarshal(rawBody, &rawLogs); err != nil { 101 | LogErr("%s: %s -> %s", TypeCT, domain, err.Error()) 102 | return logs 103 | } 104 | 105 | // Aggregate the Logs by CN (domain), while keeping min/max log time. 106 | aggregatedLogs := make(map[string]*CTAggregatedLog) 107 | for _, log := range rawLogs { 108 | 109 | // Skip logs outside of our time scope. 110 | // @todo: maybe use a DB to query CRT.sh and filter the logs directly 111 | if log.LoggedAt < CTLogFrom { 112 | continue 113 | } 114 | 115 | // Save every unique name record and keep the last known record. 116 | if aggregatedLogs[log.NameValue] == nil { 117 | aggregatedLogs[log.NameValue] = &CTAggregatedLog{ 118 | CTLog: log, 119 | FirstSeen: log.LoggedAt, 120 | LastSeen: log.LoggedAt, 121 | } 122 | } else { 123 | // Update log. 124 | if aggregatedLogs[log.NameValue].FirstSeen > log.LoggedAt { 125 | aggregatedLogs[log.NameValue].FirstSeen = log.LoggedAt 126 | } 127 | if aggregatedLogs[log.NameValue].LastSeen < log.LoggedAt { 128 | aggregatedLogs[log.NameValue].LastSeen = log.LoggedAt 129 | aggregatedLogs[log.NameValue].CTLog = log 130 | } 131 | } 132 | } 133 | 134 | for _, log := range aggregatedLogs { 135 | logs = append(logs, *log) 136 | } 137 | 138 | return logs 139 | } 140 | 141 | ///////////////////////////////////////// 142 | // CT RESOLUTION 143 | ///////////////////////////////////////// 144 | 145 | // Type returns "CT". 146 | func (res *CTResolution) Type() ResolutionType { 147 | return TypeCT 148 | } 149 | 150 | // Domains returns a list of domains discovered in records within this Resolution. 151 | func (res *CTResolution) Domains() (domains []string) { 152 | seen := make(map[string]bool, 0) 153 | 154 | for _, log := range res.Logs { 155 | logDomains := log.ExtractDomains() 156 | for _, domain := range logDomains { 157 | if !seen[domain] { 158 | domains = append(domains, domain) 159 | seen[domain] = true 160 | } 161 | } 162 | } 163 | 164 | return domains 165 | } 166 | 167 | ///////////////////////////////////////// 168 | // CT AGGREGATED LOG 169 | ///////////////////////////////////////// 170 | 171 | func (log *CTAggregatedLog) String() string { 172 | return fmt.Sprintf( 173 | "name: %s, first_seen: %s, last_seen: %s, not_before: %s, not_after: %s, issuer: %s", 174 | log.NameValue, log.FirstSeen, log.LastSeen, log.NotBefore, log.NotAfter, log.IssuerName, 175 | ) 176 | } 177 | 178 | ///////////////////////////////////////// 179 | // CT LOG 180 | ///////////////////////////////////////// 181 | 182 | func (log *CTLog) ExtractDomains() (domains []string) { 183 | domains = append(domains, DissectDomainsFromString(log.NameValue)...) 184 | return domains 185 | } 186 | 187 | func (log *CTLog) String() string { 188 | return fmt.Sprintf( 189 | "name: %s, logged_at: %s, not_before: %s, not_after: %s, issuer: %s", 190 | log.NameValue, log.LoggedAt, log.NotBefore, log.NotAfter, log.IssuerName, 191 | ) 192 | } 193 | -------------------------------------------------------------------------------- /dns.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/miekg/dns" 11 | ) 12 | 13 | var ( 14 | // DefaultDNSQueryTypes is a list of default DNS RR types that we query. 15 | DefaultDNSQueryTypes = [...]uint16{ 16 | dns.TypeA, 17 | dns.TypeNS, 18 | dns.TypeSOA, 19 | dns.TypeMX, 20 | dns.TypeTXT, 21 | dns.TypeSIG, 22 | dns.TypeKEY, 23 | dns.TypeAAAA, 24 | dns.TypeSRV, 25 | dns.TypeCERT, 26 | dns.TypeDNAME, 27 | dns.TypeOPT, 28 | dns.TypeKX, 29 | dns.TypeDS, 30 | dns.TypeRRSIG, 31 | dns.TypeNSEC, 32 | dns.TypeDNSKEY, 33 | dns.TypeNSEC3, 34 | dns.TypeNSEC3PARAM, 35 | dns.TypeTKEY, 36 | dns.TypeTSIG, 37 | dns.TypeIXFR, 38 | dns.TypeAXFR, 39 | dns.TypeMAILB, 40 | dns.TypeANY, 41 | } 42 | 43 | localNameServer string // A name server resolved using resolv.conf. 44 | queryOneCallback = queryOne // Callback reference which performs the actual DNS query (monkey patch). 45 | ) 46 | 47 | func init() { 48 | localNameServer = findLocalNameServer() 49 | } 50 | 51 | func findLocalNameServer() string { 52 | config, err := dns.ClientConfigFromFile("/etc/resolv.conf") 53 | if err != nil || config == nil { 54 | LogPanic("Cannot initialize the local resolver: %s", err) 55 | } else if len(config.Servers) == 0 { 56 | LogPanic("No local name server found") 57 | } 58 | return config.Servers[0] + ":53" 59 | } 60 | 61 | func queryOne(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 62 | msg := &dns.Msg{} 63 | msg.SetQuestion(dns.Fqdn(domain), qType) 64 | 65 | res, _, err := client.Exchange(msg, nameServer) 66 | if err != nil { 67 | if ne, ok := err.(*net.OpError); ok && ne.Timeout() { 68 | return nil, fmt.Errorf("timeout") 69 | } else if _, ok := err.(*net.OpError); ok { 70 | return nil, fmt.Errorf("network error") 71 | } 72 | return nil, err 73 | } else if res.Rcode != dns.RcodeSuccess { 74 | // If the rCode wasn't successful, return an error with the rCode as the string. 75 | return nil, errors.New(dns.RcodeToString[res.Rcode]) 76 | } 77 | 78 | return res, nil 79 | } 80 | 81 | func dissectDomainsFromRecord(record dns.RR) (domains []string) { 82 | switch record.Header().Rrtype { 83 | case dns.TypeNS: 84 | domains = append(domains, (record).(*dns.NS).Ns) 85 | break 86 | 87 | case dns.TypeCNAME: 88 | domains = append(domains, (record).(*dns.CNAME).Target) 89 | break 90 | 91 | case dns.TypeSOA: 92 | domains = append(domains, (record).(*dns.SOA).Mbox) 93 | break 94 | 95 | case dns.TypeMX: 96 | domains = append(domains, (record).(*dns.MX).Mx) 97 | break 98 | 99 | case dns.TypeTXT: 100 | domains = DissectDomainsFromStrings((record).(*dns.TXT).Txt) 101 | break 102 | 103 | case dns.TypeRRSIG: 104 | domains = append(domains, (record).(*dns.RRSIG).SignerName) 105 | break 106 | 107 | case dns.TypeNSEC: 108 | domains = append(domains, (record).(*dns.NSEC).NextDomain) 109 | break 110 | 111 | case dns.TypeKX: 112 | domains = append(domains, (record).(*dns.KX).Exchanger) 113 | break 114 | } 115 | 116 | for i := range domains { 117 | domains[i] = CleanDomain(domains[i]) 118 | } 119 | 120 | return domains 121 | } 122 | 123 | func dissectIPsFromRecord(record dns.RR) (ips []string) { 124 | switch record.Header().Rrtype { 125 | case dns.TypeA: 126 | ips = append(ips, (record).(*dns.A).A.String()) 127 | break 128 | 129 | case dns.TypeAAAA: 130 | ips = append(ips, (record).(*dns.AAAA).AAAA.String()) 131 | break 132 | 133 | case dns.TypeTXT: 134 | // For SPF typically. 135 | ips = DissectIpsFromStrings((record).(*dns.TXT).Txt) 136 | break 137 | } 138 | 139 | return ips 140 | } 141 | 142 | ///////////////////////////////////////// 143 | // DNS RESOLVER 144 | ///////////////////////////////////////// 145 | 146 | // NewDNSResolver creates a new DNS resolver instance pre-populated 147 | // with sensible defaults. 148 | func NewDNSResolver() *DNSResolver { 149 | return &DNSResolver{ 150 | QueryTypes: DefaultDNSQueryTypes[:], 151 | Client: &dns.Client{ReadTimeout: DefaultTimeout}, 152 | nameServerCache: map[string]string{}, 153 | resolvedDomains: map[string]bool{}, 154 | } 155 | } 156 | 157 | // Type returns "DNS". 158 | func (resolver *DNSResolver) Type() ResolutionType { 159 | return TypeDNS 160 | } 161 | 162 | // ResolveDomain attempts to resolve a given domain for every DNS record 163 | // type defined in resolver.QueryTypes using either a user-supplied 164 | // name-server or dynamically resolved one for this domain. 165 | func (resolver *DNSResolver) ResolveDomain(domain string) Resolution { 166 | // First find a name server for this domain (if not pre-defined). 167 | nameServer := resolver.findNameServerFor(domain) 168 | LogDebug("%s: Using NS %s for domain %s.", TypeDNS, nameServer, domain) 169 | 170 | resolution := &DNSResolution{ 171 | ResolutionBase: &ResolutionBase{query: domain}, 172 | nameServer: nameServer, 173 | } 174 | 175 | // Now do a DNS query for each record type (in parallel). 176 | recordChannel := make(chan []DNSRecordPair, 128) 177 | var wg sync.WaitGroup 178 | wg.Add(len(resolver.QueryTypes)) 179 | 180 | for _, qType := range resolver.QueryTypes { 181 | go func(qType uint16) { 182 | recordChannel <- resolver.resolveOne(domain, qType, nameServer) 183 | wg.Done() 184 | }(qType) 185 | } 186 | wg.Wait() 187 | 188 | // Collect the records. 189 | for len(recordChannel) > 0 { 190 | resolution.Records = append(resolution.Records, <-recordChannel...) 191 | } 192 | 193 | return resolution 194 | } 195 | 196 | func (resolver *DNSResolver) resolveOne(domain string, qType uint16, nameServer string) (answers []DNSRecordPair) { 197 | msg, err := queryOneCallback(domain, qType, nameServer, resolver.Client) 198 | if err != nil { 199 | LogErr("%s: %s %s -> %s", TypeDNS, dns.TypeToString[qType], domain, err.Error()) 200 | return answers 201 | } 202 | 203 | for _, rr := range msg.Answer { 204 | answers = append(answers, DNSRecordPair{ 205 | QueryType: qType, 206 | Record: &DNSRecord{rr}, 207 | }) 208 | } 209 | 210 | return answers 211 | } 212 | 213 | func (resolver *DNSResolver) findNameServerFor(domain string) string { 214 | // Use user-supplied NS if available. 215 | if resolver.NameServer != "" { 216 | return resolver.NameServer 217 | } 218 | 219 | // Check NS cache. 220 | if resolver.nameServerCache[domain] != "" { 221 | return resolver.nameServerCache[domain] 222 | } 223 | 224 | // Use DNS NS lookup. 225 | nameServer := resolver.getNameServerFor(domain) 226 | 227 | if nameServer != "" { 228 | // OK, NS found. 229 | } else if IsSubdomain(domain) { 230 | // This is a subdomain -> try the parent. 231 | LogDebug("%s: No NS found for subdomain %s -> trying parent domain.", TypeDNS, domain) 232 | nameServer = resolver.findNameServerFor(ParentDomainOf(domain)) 233 | } else { 234 | // Fallback to local NS. 235 | LogErr("%s: Could not resolve NS for domain %s -> falling back to local.", TypeDNS, domain) 236 | nameServer = localNameServer 237 | } 238 | 239 | // Cache the result. 240 | resolver.nameServerCache[domain] = nameServer 241 | 242 | return nameServer 243 | } 244 | 245 | func (resolver *DNSResolver) getNameServerFor(domain string) string { 246 | var nsRecord *dns.NS 247 | 248 | // Do a NS query. 249 | msg, err := queryOneCallback(domain, dns.TypeNS, localNameServer, resolver.Client) 250 | if err != nil { 251 | LogErr("%s: %s %s -> %s", TypeDNS, "NS", domain, err.Error()) 252 | } else { 253 | // Try to find a NS record. 254 | for _, record := range msg.Answer { 255 | if record.Header().Rrtype == dns.TypeNS { 256 | nsRecord = record.(*dns.NS) 257 | break 258 | } 259 | } 260 | } 261 | 262 | if nsRecord != nil { 263 | // NS record found -> take the NS name. 264 | nameServerFqdn := nsRecord.Ns 265 | return nameServerFqdn[:len(nameServerFqdn)-1] + ":53" 266 | } 267 | 268 | // No record found. 269 | return "" 270 | } 271 | 272 | ///////////////////////////////////////// 273 | // DNS RESOLUTION 274 | ///////////////////////////////////////// 275 | 276 | // Type returns "DNS". 277 | func (res *DNSResolution) Type() ResolutionType { 278 | return TypeDNS 279 | } 280 | 281 | // Domains returns a list of domains discovered in records within this Resolution. 282 | func (res *DNSResolution) Domains() (domains []string) { 283 | for _, answer := range res.Records { 284 | domains = append(domains, dissectDomainsFromRecord(answer.Record.RR)...) 285 | } 286 | return domains 287 | } 288 | 289 | // IPs returns a list of IP addresses discovered in this resolution. 290 | func (res *DNSResolution) IPs() (ips []string) { 291 | for _, answer := range res.Records { 292 | ips = append(ips, dissectIPsFromRecord(answer.Record.RR)...) 293 | } 294 | return ips 295 | } 296 | 297 | ///////////////////////////////////////// 298 | // DNS RECORD 299 | ///////////////////////////////////////// 300 | 301 | func (record *DNSRecord) String() string { 302 | return fmt.Sprintf("%s %s", 303 | dns.TypeToString[record.RR.Header().Rrtype], 304 | strings.Replace(record.RR.String(), record.RR.Header().String(), "", 1), 305 | ) 306 | } 307 | -------------------------------------------------------------------------------- /dns_test.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "errors" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/miekg/dns" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func Test_When_DnsResolver_Resolve_completes_Then_all_records_are_picked(t *testing.T) { 13 | // Mock. 14 | const recordsAvailable = 5 15 | 16 | counterMux := sync.Mutex{} 17 | invocationCount := 0 18 | queryOneCallback = func(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 19 | count := recordsAvailable - invocationCount 20 | 21 | // We need to count with a mutex, because DNS queries are run concurrently. 22 | counterMux.Lock() 23 | invocationCount++ 24 | counterMux.Unlock() 25 | 26 | if count > 0 { 27 | count = 1 28 | } else { 29 | count = 0 30 | } 31 | 32 | msg := mockDNSResponse(dns.TypeA, count) 33 | return msg, nil 34 | } 35 | 36 | // Setup. 37 | resolver := NewDNSResolver() 38 | 39 | // Execute. 40 | resolution := resolver.ResolveDomain("all.tens.ten").(*DNSResolution) 41 | 42 | // Assert. 43 | 44 | // There should have been 1 invocation per DNS query type and additional 2 spent on NS queries for all.tens.ten + tens.ten. 45 | assert.Equal(t, len(DefaultDNSQueryTypes)+2, invocationCount) 46 | 47 | // There should be a record for each mocked response. 48 | assert.Len(t, resolution.Records, recordsAvailable-2) 49 | } 50 | 51 | func Test_When_DnsResolver_Resolve_completes_Then_custom_NameServer_was_used(t *testing.T) { 52 | // Mock. 53 | var usedNameServer string 54 | queryOneCallback = func(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 55 | usedNameServer = nameServer 56 | return &dns.Msg{}, nil 57 | } 58 | 59 | // Setup. 60 | resolver := NewDNSResolver() 61 | resolver.NameServer = "1.1.1.1" 62 | 63 | // Execute. 64 | resolver.ResolveDomain("example.com") 65 | 66 | // Assert. 67 | assert.Equal(t, resolver.NameServer, usedNameServer) 68 | } 69 | 70 | func Test_When_queryOne_returns_error_Then_empty_response(t *testing.T) { 71 | // Mock. 72 | queryOneCallback = func(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 73 | var msg *dns.Msg 74 | return msg, errors.New("something silly happened") 75 | } 76 | 77 | // Setup. 78 | resolver := NewDNSResolver() 79 | resolver.QueryTypes = []uint16{dns.TypeA} 80 | 81 | // Execute. 82 | resolution := resolver.ResolveDomain("example.com") 83 | 84 | // Assert. 85 | assert.Len(t, resolution.Domains(), 0) 86 | } 87 | 88 | func Test_That_findNameServerFor_dissects_NS_records(t *testing.T) { 89 | // Mock. 90 | queryOneCallback = func(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 91 | msg := mockDNSResponse(dns.TypeNS, 1) 92 | rr := &msg.Answer[0] 93 | (*rr).(*dns.NS).Ns = "ns.example.com." 94 | 95 | return msg, nil 96 | } 97 | 98 | // Setup. 99 | resolver := NewDNSResolver() 100 | 101 | // Execute. 102 | nameServer := resolver.findNameServerFor("example.com") 103 | 104 | // Assert. 105 | assert.Equal(t, "ns.example.com:53", nameServer) 106 | } 107 | 108 | func Test_That_findNameServerFor_caches_results(t *testing.T) { 109 | // Mock. 110 | counterMux := sync.Mutex{} 111 | var invocationCount int 112 | queryOneCallback = func(domain string, qType uint16, nameServer string, client *dns.Client) (*dns.Msg, error) { 113 | // We need to count with a mutex, because DNS queries are run concurrently. 114 | counterMux.Lock() 115 | invocationCount++ 116 | counterMux.Unlock() 117 | 118 | msg := mockDNSResponse(dns.TypeNS, 1) 119 | rr := &msg.Answer[0] 120 | (*rr).(*dns.NS).Ns = "ns.example.com." 121 | 122 | return msg, nil 123 | } 124 | 125 | // Setup. 126 | resolver := NewDNSResolver() 127 | 128 | // Execute. 129 | _ = resolver.findNameServerFor("example.com") 130 | _ = resolver.findNameServerFor("example.com") 131 | 132 | // Assert. 133 | assert.Equal(t, 1, invocationCount) 134 | } 135 | 136 | func Test_dissectDomain_By_NS_record(t *testing.T) { 137 | // Setup. 138 | record := &dns.NS{ 139 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeNS}, 140 | Ns: "ns1.example.com.", 141 | } 142 | 143 | // Execute. 144 | domains := dissectDomainsFromRecord(record) 145 | 146 | // Assert. 147 | assert.Equal(t, "ns1.example.com", domains[0]) 148 | } 149 | 150 | func Test_dissectDomain_By_TXT_record(t *testing.T) { 151 | // Setup. 152 | record := &dns.TXT{ 153 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeTXT}, 154 | Txt: []string{ 155 | "foo; bar; baz=1029umadmcald;1205+%!$ 0", 156 | "foo; bar; baz=related.example.com;afasf=asd123 1", 157 | }, 158 | } 159 | 160 | // Execute. 161 | domains := dissectDomainsFromRecord(record) 162 | 163 | // Assert. 164 | assert.Equal(t, "related.example.com", domains[0]) 165 | } 166 | 167 | func Test_dissectDomain_By_RRSIG_record(t *testing.T) { 168 | // Setup. 169 | record := &dns.RRSIG{ 170 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeRRSIG}, 171 | SignerName: "related.example.com.", 172 | } 173 | 174 | // Execute. 175 | domains := dissectDomainsFromRecord(record) 176 | 177 | // Assert. 178 | assert.Equal(t, "related.example.com", domains[0]) 179 | } 180 | 181 | func Test_dissectDomain_By_CNAME_record(t *testing.T) { 182 | // Setup. 183 | record := &dns.CNAME{ 184 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeCNAME}, 185 | Target: "related.example.com.", 186 | } 187 | 188 | // Execute. 189 | domains := dissectDomainsFromRecord(record) 190 | 191 | // Assert. 192 | assert.Equal(t, "related.example.com", domains[0]) 193 | } 194 | 195 | func Test_dissectDomain_By_SOA_record(t *testing.T) { 196 | // Setup. 197 | record := &dns.SOA{ 198 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeSOA}, 199 | Mbox: "related.example.com.", 200 | } 201 | 202 | // Execute. 203 | domains := dissectDomainsFromRecord(record) 204 | 205 | // Assert. 206 | assert.Equal(t, "related.example.com", domains[0]) 207 | } 208 | 209 | func Test_dissectDomain_By_MX_record(t *testing.T) { 210 | // Setup. 211 | record := &dns.MX{ 212 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeMX}, 213 | Mx: "related.example.com.", 214 | } 215 | 216 | // Execute. 217 | domains := dissectDomainsFromRecord(record) 218 | 219 | // Assert. 220 | assert.Equal(t, "related.example.com", domains[0]) 221 | } 222 | 223 | func Test_dissectDomain_By_NSEC_record(t *testing.T) { 224 | // Setup. 225 | record := &dns.NSEC{ 226 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeNSEC}, 227 | NextDomain: "*.related.example.com.", 228 | } 229 | 230 | // Execute. 231 | domains := dissectDomainsFromRecord(record) 232 | 233 | // Assert. 234 | assert.Equal(t, "related.example.com", domains[0]) 235 | } 236 | 237 | func Test_dissectDomain_By_KX_record(t *testing.T) { 238 | // Setup. 239 | record := &dns.KX{ 240 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeKX}, 241 | Exchanger: "related.example.com.", 242 | } 243 | 244 | // Execute. 245 | domains := dissectDomainsFromRecord(record) 246 | 247 | // Assert. 248 | assert.Equal(t, "related.example.com", domains[0]) 249 | } 250 | 251 | func Test_dissectDomain_By_unsupported_record(t *testing.T) { 252 | // Setup. 253 | record := &dns.MB{ 254 | Hdr: dns.RR_Header{Name: "example.com", Rrtype: dns.TypeMB}, 255 | Mb: "related.example.com.", 256 | } 257 | 258 | // Execute. 259 | domains := dissectDomainsFromRecord(record) 260 | 261 | // Assert. 262 | assert.Empty(t, domains) 263 | } 264 | 265 | func Test_parentDomainOf_By_subdomain(t *testing.T) { 266 | // Setup. 267 | domain := "sub.example.com" 268 | 269 | // Execute. 270 | parent := ParentDomainOf(domain) 271 | 272 | // Assert. 273 | assert.Equal(t, "example.com", parent) 274 | } 275 | 276 | func Test_parentDomainOf_By_domain(t *testing.T) { 277 | // Setup. 278 | domain := "example.com" 279 | 280 | // Execute. 281 | parent := ParentDomainOf(domain) 282 | 283 | // Assert. 284 | assert.Empty(t, parent) 285 | } 286 | 287 | func Test_parentDomainOf_By_TLD(t *testing.T) { 288 | // Setup. 289 | domain := "com" 290 | 291 | // Execute. 292 | parent := ParentDomainOf(domain) 293 | 294 | // Assert. 295 | assert.Empty(t, parent) 296 | } 297 | 298 | func mockDNSResponse(qType uint16, numRecords int) *dns.Msg { 299 | msg := &dns.Msg{} 300 | 301 | for i := 0; i < numRecords; i++ { 302 | rrNewFun := dns.TypeToRR[qType] 303 | rr := rrNewFun() 304 | rr.Header().Rrtype = qType 305 | msg.Answer = append(msg.Answer, rr) 306 | } 307 | 308 | return msg 309 | } 310 | -------------------------------------------------------------------------------- /doc/res/udig_demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/netrixone/udig/86a2941585e54593104e86d7741715296cdae149/doc/res/udig_demo.gif -------------------------------------------------------------------------------- /geo.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ip2location/ip2location-go" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | var ( 11 | // GeoDBPath is a path to IP2Location DB file. 12 | GeoDBPath = findGeoipDatabase("IP2LOCATION-LITE-DB1.IPV6.BIN") 13 | ) 14 | 15 | // CheckGeoipDatabase returns true if a given path points to a valid GeoIP DB file. 16 | func checkGeoipDatabase(geoipPath string) bool { 17 | if info, err := os.Stat(geoipPath); err != nil || info.IsDir() { 18 | LogErr("%s: Cannot use IP2Location DB at '%s' (file exists: %t).", TypeGEO, geoipPath, os.IsExist(err)) 19 | return false 20 | } 21 | _, err := ip2location.OpenDB(geoipPath) 22 | return err == nil 23 | } 24 | 25 | // FindGeoipDatabase attempts to locate a GeoIP database file at a given path. 26 | // 27 | // If the given path is absolute, it is used as it is. 28 | // If the path is relative, then it is first checked against CWD and then against 29 | // the dir where the executable resides in. 30 | func findGeoipDatabase(geoipPath string) string { 31 | // If the path is absolute, leave it as it is. 32 | if filepath.IsAbs(geoipPath) { 33 | return geoipPath 34 | } 35 | 36 | // Otherwise check CWD first. 37 | cwd, err := os.Getwd() 38 | if err != nil { 39 | panic(err) 40 | } 41 | 42 | relPath := filepath.Join(cwd, geoipPath) 43 | if _, err := os.Stat(relPath); !os.IsNotExist(err) { 44 | return relPath 45 | } 46 | 47 | // Finally, try a path relative to the binary. 48 | executable, err := os.Executable() 49 | if err != nil { 50 | panic(err) 51 | } 52 | 53 | return filepath.Join(filepath.Dir(executable), geoipPath) 54 | } 55 | 56 | func queryIP(ip string) *ip2location.IP2Locationrecord { 57 | db, err := ip2location.OpenDB(GeoDBPath) 58 | if err != nil { 59 | LogErr("%s: Could not open DB. The cause was: %s", TypeGEO, err.Error()) 60 | return nil 61 | } 62 | 63 | record, err := db.Get_country_short(ip) 64 | if err != nil { 65 | LogErr("%s: Could not query DB for IP %s. The cause was: %s", TypeGEO, ip, err.Error()) 66 | return nil 67 | } 68 | 69 | db.Close() 70 | 71 | return &record 72 | } 73 | 74 | ///////////////////////////////////////// 75 | // GEO RESOLVER 76 | ///////////////////////////////////////// 77 | 78 | // NewGeoResolver creates a new GeoResolver with sensible defaults. 79 | func NewGeoResolver() *GeoResolver { 80 | return &GeoResolver{ 81 | enabled: checkGeoipDatabase(GeoDBPath), 82 | cachedResults: map[string]*GeoResolution{}, 83 | } 84 | } 85 | 86 | // ResolveIP resolves a given IP address to a corresponding GeoIP record. 87 | func (resolver *GeoResolver) ResolveIP(ip string) Resolution { 88 | resolution := resolver.cachedResults[ip] 89 | if resolution != nil { 90 | return resolution 91 | } 92 | resolution = &GeoResolution{ResolutionBase: &ResolutionBase{query: ip}} 93 | resolver.cachedResults[ip] = resolution 94 | 95 | if !resolver.enabled { 96 | return resolution 97 | } 98 | 99 | geoRecord := queryIP(ip) 100 | if geoRecord == nil { 101 | return resolution 102 | } 103 | resolution.Record = &GeoRecord{CountryCode: geoRecord.Country_short} 104 | 105 | return resolution 106 | } 107 | 108 | // Type returns "GEO". 109 | func (resolver *GeoResolver) Type() ResolutionType { 110 | return TypeGEO 111 | } 112 | 113 | ///////////////////////////////////////// 114 | // GEO RESOLUTION 115 | ///////////////////////////////////////// 116 | 117 | // Type returns "BGP". 118 | func (res *GeoResolution) Type() ResolutionType { 119 | return TypeGEO 120 | } 121 | 122 | ///////////////////////////////////////// 123 | // GEO RECORD 124 | ///////////////////////////////////////// 125 | 126 | func (record *GeoRecord) String() string { 127 | return fmt.Sprintf("country code: %s", record.CountryCode) 128 | } 129 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/netrixone/udig 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/akamensky/argparse v1.2.1 7 | github.com/domainr/whois v0.0.0-20180714175948-975c7833b02e 8 | github.com/domainr/whoistest v0.0.0-20180714175718-26cad4b7c941 // indirect 9 | github.com/ip2location/ip2location-go v8.3.0+incompatible 10 | github.com/miekg/dns v1.1.27 11 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 // indirect 12 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca // indirect 13 | github.com/stretchr/testify v1.5.1 14 | github.com/zonedb/zonedb v1.0.2611 // indirect 15 | ) 16 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= 2 | github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= 3 | github.com/akamensky/argparse v1.2.1 h1:YMYF1VMku+dnz7TVTJpYhsCXHSYCVMAIcKaBbjwbvZo= 4 | github.com/akamensky/argparse v1.2.1/go.mod h1:S5kwC7IuDcEr5VeXtGPRVZ5o/FdhcMlQz4IZQuw64xA= 5 | github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= 6 | github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= 7 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 8 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 9 | github.com/domainr/whois v0.0.0-20180714175948-975c7833b02e h1:iZ+fhG8+rFlxdqv88o5tMyH89H6AWx0WJgUXhiSxkhA= 10 | github.com/domainr/whois v0.0.0-20180714175948-975c7833b02e/go.mod h1:gtNPHEJSqhDlO2zVW0ai2b4voo5NWzFyNFPBMB0fDs0= 11 | github.com/domainr/whoistest v0.0.0-20180714175718-26cad4b7c941 h1:E7ehdIemEeScp8nVs0JXNXEbzb2IsHCk13ijvwKqRWI= 12 | github.com/domainr/whoistest v0.0.0-20180714175718-26cad4b7c941/go.mod h1:iuCHv1qZDoHJNQs56ZzzoKRSKttGgTr2yByGpSlKsII= 13 | github.com/ip2location/ip2location-go v8.3.0+incompatible h1:QwUE+FlSbo6bjOWZpv2Grb57vJhWYFNPyBj2KCvfWaM= 14 | github.com/ip2location/ip2location-go v8.3.0+incompatible/go.mod h1:3JUY1TBjTx1GdA7oRT7Zeqfc0bg3lMMuU5lXmzdpuME= 15 | github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= 16 | github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= 17 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4= 18 | github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= 19 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 20 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 21 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca h1:NugYot0LIVPxTvN8n+Kvkn6TrbMyxQiuvKdEwFdR9vI= 22 | github.com/saintfish/chardet v0.0.0-20120816061221-3af4cd4741ca/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 23 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 24 | github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= 25 | github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= 26 | github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0/go.mod h1:IXCdmsXIht47RaVFLEdVnh1t+pgYtTAhQGj73kz+2DM= 27 | github.com/zonedb/zonedb v1.0.2611 h1:rCWGirR+bj9uWJj0mPPT+QMMXqkDUHtmod+7eHs4Puk= 28 | github.com/zonedb/zonedb v1.0.2611/go.mod h1:qAnQYVzv7gm1szAc7u5LNjUCienVQwXsRz4CoM7dzXE= 29 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 30 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= 31 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 32 | golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= 33 | golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 34 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 35 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 36 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7MvYXnkV9/iQNo1lX6g= 37 | golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 38 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= 39 | golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 40 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= 41 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 42 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 43 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 44 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= 45 | golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 46 | golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= 47 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 48 | golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= 49 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 50 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 51 | golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= 52 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 53 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 54 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= 56 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 57 | -------------------------------------------------------------------------------- /http.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "crypto/tls" 5 | "fmt" 6 | "net" 7 | "net/http" 8 | ) 9 | 10 | var ( 11 | // DefaultHTTPHeaders is a list of default HTTP header names that we look for. 12 | DefaultHTTPHeaders = [...]string{ 13 | "access-control-allow-origin", 14 | "alt-svc", 15 | "content-security-policy", 16 | "content-security-policy-report-only", 17 | } 18 | ) 19 | 20 | // fetchHeaders connects to a given URL and on successful connection returns 21 | // a map of HTTP headers in the response. 22 | func fetchHeaders(url string) http.Header { 23 | transport := http.DefaultTransport.(*http.Transport) 24 | 25 | transport.DialContext = (&net.Dialer{ 26 | Timeout: DefaultTimeout, 27 | KeepAlive: DefaultTimeout, 28 | }).DialContext 29 | 30 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 31 | transport.TLSHandshakeTimeout = DefaultTimeout 32 | 33 | client := &http.Client{ 34 | Transport: transport, 35 | Timeout: DefaultTimeout, 36 | } 37 | 38 | response, err := client.Get(url) 39 | if err != nil { 40 | // Don't bother trying to find CSP on non-TLS sites. 41 | LogErr("HTTP: Could not GET %s - the cause was: %s.", url, err.Error()) 42 | return map[string][]string{} 43 | } 44 | 45 | return response.Header 46 | } 47 | 48 | ///////////////////////////////////////// 49 | // HTTP RESOLVER 50 | ///////////////////////////////////////// 51 | 52 | // NewHTTPResolver creates a new HTTPResolver with sensible defaults. 53 | func NewHTTPResolver() *HTTPResolver { 54 | transport := http.DefaultTransport.(*http.Transport) 55 | 56 | transport.DialContext = (&net.Dialer{ 57 | Timeout: DefaultTimeout, 58 | KeepAlive: DefaultTimeout, 59 | }).DialContext 60 | 61 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 62 | transport.TLSHandshakeTimeout = DefaultTimeout 63 | 64 | client := &http.Client{ 65 | Transport: transport, 66 | Timeout: DefaultTimeout, 67 | } 68 | 69 | return &HTTPResolver{ 70 | Headers: DefaultHTTPHeaders[:], 71 | Client: client, 72 | } 73 | } 74 | 75 | // Type returns "HTTP". 76 | func (resolver *HTTPResolver) Type() ResolutionType { 77 | return TypeHTTP 78 | } 79 | 80 | // ResolveDomain resolves a given domain to a list of corresponding HTTP headers. 81 | func (resolver *HTTPResolver) ResolveDomain(domain string) Resolution { 82 | resolution := &HTTPResolution{ 83 | ResolutionBase: &ResolutionBase{query: domain}, 84 | } 85 | 86 | headers := fetchHeaders("https://" + domain) 87 | for _, name := range resolver.Headers { 88 | value := headers[http.CanonicalHeaderKey(name)] 89 | if len(DissectDomainsFromStrings(value)) > 0 { 90 | resolution.Headers = append(resolution.Headers, HTTPHeader{name, value}) 91 | } 92 | } 93 | 94 | return resolution 95 | } 96 | 97 | ///////////////////////////////////////// 98 | // HTTP RESOLUTION 99 | ///////////////////////////////////////// 100 | 101 | // Type returns "HTTP". 102 | func (res *HTTPResolution) Type() ResolutionType { 103 | return TypeHTTP 104 | } 105 | 106 | // Domains returns a list of domains discovered in records within this Resolution. 107 | func (res *HTTPResolution) Domains() (domains []string) { 108 | for _, header := range res.Headers { 109 | domains = append(domains, DissectDomainsFromStrings(header.Value)...) 110 | } 111 | return domains 112 | } 113 | 114 | ///////////////////////////////////////// 115 | // HTTP HEADER 116 | ///////////////////////////////////////// 117 | 118 | func (header *HTTPHeader) String() string { 119 | return fmt.Sprintf("%s: %v", header.Name, header.Value) 120 | } 121 | -------------------------------------------------------------------------------- /log.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | // Logging levels: the smaller value the more verbose the output will be. 9 | const ( 10 | LogLevelDebug = 0 11 | LogLevelInfo = 10 12 | LogLevelErr = 100 13 | LogLevelNone = 1000 14 | ) 15 | 16 | const ( 17 | errColor = "\033[1;91m" 18 | infoColor = "\033[1;92m" 19 | debugColor = "" 20 | noColor = "\033[0m" 21 | ) 22 | 23 | // LogLevel contains the actual log level setting. 24 | var LogLevel = LogLevelDebug 25 | 26 | // LogPanic formats and prints a given log on STDERR and panics. 27 | func LogPanic(format string, a ...interface{}) { 28 | LogErr(format, a) 29 | panic(nil) 30 | } 31 | 32 | // LogErr formats and prints a given log on STDERR. 33 | func LogErr(format string, a ...interface{}) { 34 | if LogLevel <= LogLevelErr { 35 | fmt.Fprintf(os.Stderr, errColor+"[!] "+format+"\n"+noColor, a...) 36 | } 37 | } 38 | 39 | // LogInfo formats and prints a given log on STDOUT. 40 | func LogInfo(format string, a ...interface{}) { 41 | if LogLevel <= LogLevelInfo { 42 | fmt.Printf(infoColor+"[+] "+format+"\n"+noColor, a...) 43 | } 44 | } 45 | 46 | // LogDebug formats and prints a given log on STDOUT. 47 | func LogDebug(format string, a ...interface{}) { 48 | if LogLevel <= LogLevelDebug { 49 | fmt.Printf(debugColor+"[~] "+format+"\n"+noColor, a...) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /tls.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "net" 8 | "net/http" 9 | ) 10 | 11 | ///////////////////////////////////////// 12 | // TLS RESOLVER 13 | ///////////////////////////////////////// 14 | 15 | // NewTLSResolver creates a new TLSResolver with sensible defaults. 16 | func NewTLSResolver() *TLSResolver { 17 | transport := http.DefaultTransport.(*http.Transport) 18 | 19 | transport.DialContext = (&net.Dialer{ 20 | Timeout: DefaultTimeout, 21 | KeepAlive: DefaultTimeout, 22 | DualStack: true, 23 | }).DialContext 24 | 25 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 26 | transport.TLSHandshakeTimeout = DefaultTimeout 27 | 28 | client := &http.Client{ 29 | Transport: transport, 30 | Timeout: DefaultTimeout, 31 | } 32 | 33 | return &TLSResolver{ 34 | Client: client, 35 | } 36 | } 37 | 38 | // Type returns "TLS". 39 | func (resolver *TLSResolver) Type() ResolutionType { 40 | return TypeTLS 41 | } 42 | 43 | // ResolveDomain resolves a given domain to a list of TLS certificates. 44 | func (resolver *TLSResolver) ResolveDomain(domain string) Resolution { 45 | resolution := &TLSResolution{ 46 | ResolutionBase: &ResolutionBase{query: domain}, 47 | } 48 | 49 | certificates := resolver.fetchTLSCertChain(domain) 50 | for _, cert := range certificates { 51 | resolution.Certificates = append(resolution.Certificates, TLSCertificate{*cert}) 52 | } 53 | 54 | return resolution 55 | } 56 | 57 | func (resolver *TLSResolver) fetchTLSCertChain(domain string) (chain []*x509.Certificate) { 58 | res, err := resolver.Client.Get("https://" + domain) 59 | if err != nil { 60 | LogErr("%s: %s -> %s", TypeTLS, domain, err.Error()) 61 | return chain 62 | } 63 | 64 | if res.TLS == nil { 65 | // No cert available. 66 | return chain 67 | } 68 | 69 | return res.TLS.PeerCertificates 70 | } 71 | 72 | ///////////////////////////////////////// 73 | // TLS RESOLUTION 74 | ///////////////////////////////////////// 75 | 76 | // Type returns "TLS". 77 | func (res *TLSResolution) Type() ResolutionType { 78 | return TypeTLS 79 | } 80 | 81 | // Domains returns a list of domains discovered in records within this Resolution. 82 | func (res *TLSResolution) Domains() (domains []string) { 83 | for _, cert := range res.Certificates { 84 | domains = append(domains, dissectDomainsFromCert(&cert)...) 85 | } 86 | return domains 87 | } 88 | 89 | ///////////////////////////////////////// 90 | // TLS CERTIFICATE 91 | ///////////////////////////////////////// 92 | 93 | func (cert *TLSCertificate) String() string { 94 | subject := cert.Subject.CommonName 95 | if subject == "" { 96 | subject = cert.Subject.String() 97 | } 98 | issuer := cert.Issuer.CommonName 99 | if issuer == "" { 100 | issuer = cert.Issuer.String() 101 | } 102 | return fmt.Sprintf("subject: %s, issuer: %s, domains: %v", subject, issuer, cert.DNSNames) 103 | } 104 | 105 | func dissectDomainsFromCert(cert *TLSCertificate) (domains []string) { 106 | var haystack []string 107 | haystack = append(haystack, cert.CRLDistributionPoints...) 108 | haystack = append(haystack, cert.DNSNames...) 109 | haystack = append(haystack, cert.EmailAddresses...) 110 | haystack = append(haystack, cert.ExcludedDNSDomains...) 111 | haystack = append(haystack, cert.ExcludedEmailAddresses...) 112 | haystack = append(haystack, cert.ExcludedURIDomains...) 113 | haystack = append(haystack, cert.Issuer.String()) 114 | haystack = append(haystack, cert.PermittedDNSDomains...) 115 | haystack = append(haystack, cert.PermittedEmailAddresses...) 116 | haystack = append(haystack, cert.PermittedURIDomains...) 117 | haystack = append(haystack, cert.Subject.String()) 118 | for _, uri := range cert.URIs { 119 | haystack = append(haystack, uri.Host) 120 | } 121 | 122 | return DissectDomainsFromStrings(haystack) 123 | } 124 | -------------------------------------------------------------------------------- /udig.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "sync" 5 | 6 | "github.com/miekg/dns" 7 | ) 8 | 9 | type udigImpl struct { 10 | Udig 11 | domainResolvers []DomainResolver 12 | ipResolvers []IPResolver 13 | domainQueue chan string 14 | ipQueue chan string 15 | processed map[string]bool 16 | seen map[string]bool 17 | } 18 | 19 | // NewUdig creates a new Udig instances provisioned with 20 | // all supported resolvers. You can also supply your own 21 | // resolvers to the returned instance. 22 | func NewUdig() Udig { 23 | udig := &udigImpl{ 24 | domainResolvers: []DomainResolver{}, 25 | ipResolvers: []IPResolver{}, 26 | domainQueue: make(chan string, 1024), 27 | ipQueue: make(chan string, 1024), 28 | processed: map[string]bool{}, 29 | seen: map[string]bool{}, 30 | } 31 | 32 | udig.AddDomainResolver(NewDNSResolver()) 33 | udig.AddDomainResolver(NewWhoisResolver()) 34 | udig.AddDomainResolver(NewTLSResolver()) 35 | udig.AddDomainResolver(NewHTTPResolver()) 36 | udig.AddDomainResolver(NewCTResolver()) 37 | 38 | udig.AddIPResolver(NewBGPResolver()) 39 | udig.AddIPResolver(NewGeoResolver()) 40 | 41 | return udig 42 | } 43 | 44 | func (udig *udigImpl) Resolve(domain string) []Resolution { 45 | udig.domainQueue <- domain 46 | return udig.resolveDomains() 47 | } 48 | 49 | func (udig *udigImpl) AddDomainResolver(resolver DomainResolver) { 50 | udig.domainResolvers = append(udig.domainResolvers, resolver) 51 | } 52 | 53 | func (udig *udigImpl) AddIPResolver(resolver IPResolver) { 54 | udig.ipResolvers = append(udig.ipResolvers, resolver) 55 | } 56 | 57 | func (udig *udigImpl) resolveDomains() (resolutions []Resolution) { 58 | for len(udig.domainQueue) > 0 { 59 | // Poll a domain. 60 | domain := <-udig.domainQueue 61 | 62 | // Resolve it. 63 | newResolutions := udig.resolveOneDomain(domain) 64 | 65 | // Store the results. 66 | resolutions = append(resolutions, newResolutions...) 67 | 68 | // Enqueue all related domains from the result. 69 | udig.enqueueDomains(udig.getRelatedDomains(newResolutions)...) 70 | 71 | // Resolve all the discovered IPs. 72 | resolutions = append(resolutions, udig.resolveIPs()...) 73 | } 74 | 75 | return resolutions 76 | } 77 | 78 | func (udig *udigImpl) resolveIPs() (resolutions []Resolution) { 79 | for len(udig.ipQueue) > 0 { 80 | // Poll an IP. 81 | ip := <-udig.ipQueue 82 | 83 | // Resolve it. 84 | newResolutions := udig.resolveOneIP(ip) 85 | 86 | resolutions = append(resolutions, newResolutions...) 87 | } 88 | 89 | return resolutions 90 | } 91 | 92 | func (udig *udigImpl) resolveOneDomain(domain string) (resolutions []Resolution) { 93 | // Make sure we don't repeat ourselves. 94 | if udig.isProcessed(domain) { 95 | return resolutions 96 | } 97 | defer udig.addProcessed(domain) 98 | 99 | resolutionChannel := make(chan Resolution, 1024) 100 | 101 | var wg sync.WaitGroup 102 | wg.Add(len(udig.domainResolvers)) 103 | 104 | for _, resolver := range udig.domainResolvers { 105 | go func(resolver DomainResolver) { 106 | resolution := resolver.ResolveDomain(domain) 107 | resolutionChannel <- resolution 108 | 109 | // Enqueue all discovered IPs. 110 | udig.enqueueIps(resolution.IPs()...) 111 | 112 | wg.Done() 113 | }(resolver) 114 | } 115 | 116 | wg.Wait() 117 | 118 | for len(resolutionChannel) > 0 { 119 | resolutions = append(resolutions, <-resolutionChannel) 120 | } 121 | 122 | return resolutions 123 | } 124 | 125 | func (udig *udigImpl) resolveOneIP(ip string) (resolutions []Resolution) { 126 | // Make sure we don't repeat ourselves. 127 | if udig.isProcessed(ip) { 128 | return resolutions 129 | } 130 | defer udig.addProcessed(ip) 131 | 132 | resolutionChannel := make(chan Resolution, 1024) 133 | 134 | var wg sync.WaitGroup 135 | wg.Add(len(udig.ipResolvers)) 136 | 137 | for _, resolver := range udig.ipResolvers { 138 | go func(resolver IPResolver) { 139 | resolutionChannel <- resolver.ResolveIP(ip) 140 | wg.Done() 141 | }(resolver) 142 | } 143 | 144 | wg.Wait() 145 | 146 | for len(resolutionChannel) > 0 { 147 | resolutions = append(resolutions, <-resolutionChannel) 148 | } 149 | 150 | return resolutions 151 | } 152 | 153 | func (udig *udigImpl) isCnameOrRelated(nextDomain string, resolution Resolution) bool { 154 | switch resolution.Type() { 155 | case TypeDNS: 156 | for _, rr := range resolution.(*DNSResolution).Records { 157 | if rr.Record.Header().Rrtype == dns.TypeCNAME && rr.Record.RR.(*dns.CNAME).Target == nextDomain { 158 | // Follow DNS CNAME pointers. 159 | return true 160 | } 161 | } 162 | break 163 | } 164 | 165 | // Otherwise try heuristics. 166 | return IsDomainRelated(nextDomain, resolution.Query()) 167 | } 168 | 169 | func (udig *udigImpl) getRelatedDomains(resolutions []Resolution) (domains []string) { 170 | for _, resolution := range resolutions { 171 | for _, nextDomain := range resolution.Domains() { 172 | // Crawl new and related domains only. 173 | if udig.isProcessed(nextDomain) || udig.isSeen(nextDomain) { 174 | continue 175 | } 176 | 177 | udig.addSeen(nextDomain) 178 | 179 | if !udig.isCnameOrRelated(nextDomain, resolution) { 180 | LogDebug("%s: Domain %s is not related to %s -> skipping.", resolution.Type(), nextDomain, resolution.Query()) 181 | continue 182 | } 183 | 184 | LogDebug("%s: Discovered a related domain %s via %s.", resolution.Type(), nextDomain, resolution.Query()) 185 | 186 | domains = append(domains, nextDomain) 187 | } 188 | } 189 | return domains 190 | } 191 | 192 | func (udig *udigImpl) enqueueDomains(domains ...string) { 193 | for _, domain := range domains { 194 | udig.domainQueue <- domain 195 | } 196 | } 197 | 198 | func (udig *udigImpl) enqueueIps(ips ...string) { 199 | for _, ip := range ips { 200 | udig.ipQueue <- ip 201 | } 202 | } 203 | 204 | func (udig *udigImpl) isProcessed(query string) bool { 205 | return udig.processed[query] 206 | } 207 | 208 | func (udig *udigImpl) addProcessed(query string) { 209 | udig.processed[query] = true 210 | } 211 | 212 | func (udig *udigImpl) isSeen(query string) bool { 213 | return udig.seen[query] 214 | } 215 | 216 | func (udig *udigImpl) addSeen(query string) { 217 | udig.seen[query] = true 218 | } 219 | -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "net" 5 | "regexp" 6 | "strings" 7 | 8 | "github.com/miekg/dns" 9 | ) 10 | 11 | const ( 12 | // Note: there is no support for unicode domains. 13 | _char = `[a-z0-9]` 14 | _charOrSymbol = `[a-z0-9-_]` 15 | _domainWord = _char + `(?:` + _charOrSymbol + `*` + _char + `)?` 16 | 17 | // TLDs by IANA, sorted by longest to shortest. Source: https://data.iana.org/TLD/tlds-alpha-by-domain.txt 18 | _tlds = "northwesternmutual|travelersinsurance|americanexpress|kerryproperties|sandvikcoromant|afamilycompany|americanfamily|bananarepublic|cancerresearch|cookingchannel|kerrylogistics|weatherchannel|international|lifeinsurance|spreadbetting|travelchannel|wolterskluwer|construction|lplfinancial|scholarships|versicherung|accountants|barclaycard|blackfriday|blockbuster|bridgestone|calvinklein|contractors|creditunion|engineering|enterprises|foodnetwork|investments|kerryhotels|lamborghini|motorcycles|olayangroup|photography|playstation|productions|progressive|redumbrella|rightathome|williamhill|accountant|apartments|associates|basketball|bnpparibas|boehringer|capitalone|consulting|creditcard|cuisinella|eurovision|extraspace|foundation|healthcare|immobilien|industries|management|mitsubishi|nationwide|newholland|nextdirect|onyourside|properties|protection|prudential|realestate|republican|restaurant|schaeffler|swiftcover|tatamotors|technology|telefonica|university|vistaprint|vlaanderen|volkswagen|accenture|alfaromeo|allfinanz|amsterdam|analytics|aquarelle|barcelona|bloomberg|christmas|community|directory|education|equipment|fairwinds|financial|firestone|fresenius|frontdoor|fujixerox|furniture|goldpoint|hisamitsu|homedepot|homegoods|homesense|honeywell|institute|insurance|kuokgroup|ladbrokes|lancaster|landrover|lifestyle|marketing|marshalls|melbourne|microsoft|panasonic|passagens|pramerica|richardli|scjohnson|shangrila|solutions|statebank|statefarm|stockholm|travelers|vacations|yodobashi|abudhabi|airforce|allstate|attorney|barclays|barefoot|bargains|baseball|boutique|bradesco|broadway|brussels|budapest|builders|business|capetown|catering|catholic|chrysler|cipriani|cityeats|cleaning|clinique|clothing|commbank|computer|delivery|deloitte|democrat|diamonds|discount|discover|download|engineer|ericsson|esurance|etisalat|everbank|exchange|feedback|fidelity|firmdale|football|frontier|goodyear|grainger|graphics|guardian|hdfcbank|helsinki|holdings|hospital|infiniti|ipiranga|istanbul|jpmorgan|lighting|lundbeck|marriott|maserati|mckinsey|memorial|merckmsd|mortgage|movistar|observer|partners|pharmacy|pictures|plumbing|property|redstone|reliance|saarland|samsclub|security|services|shopping|showtime|softbank|software|stcgroup|supplies|symantec|training|uconnect|vanguard|ventures|verisign|woodside|yokohama|abogado|academy|agakhan|alibaba|android|athleta|auction|audible|auspost|avianca|banamex|bauhaus|bentley|bestbuy|booking|brother|bugatti|capital|caravan|careers|cartier|channel|charity|chintai|citadel|clubmed|college|cologne|comcast|company|compare|contact|cooking|corsica|country|coupons|courses|cricket|cruises|dentist|digital|domains|exposed|express|farmers|fashion|ferrari|ferrero|finance|fishing|fitness|flights|florist|flowers|forsale|frogans|fujitsu|gallery|genting|godaddy|grocery|guitars|hamburg|hangout|hitachi|holiday|hosting|hoteles|hotmail|hyundai|iselect|ismaili|jewelry|juniper|kitchen|komatsu|lacaixa|lancome|lanxess|lasalle|latrobe|leclerc|liaison|limited|lincoln|markets|metlife|monster|netbank|netflix|network|neustar|okinawa|oldnavy|organic|origins|philips|pioneer|politie|realtor|recipes|rentals|reviews|rexroth|samsung|sandvik|schmidt|schwarz|science|shiksha|shriram|singles|staples|starhub|storage|support|surgery|systems|temasek|theater|theatre|tickets|tiffany|toshiba|trading|walmart|wanggou|watches|weather|website|wedding|whoswho|windows|winners|xfinity|yamaxun|youtube|zuerich|abarth|abbott|abbvie|africa|agency|airbus|airtel|alipay|alsace|alstom|anquan|aramco|author|bayern|beauty|berlin|bharti|bostik|boston|broker|camera|career|caseih|casino|center|chanel|chrome|church|circle|claims|clinic|coffee|comsec|condos|coupon|credit|cruise|dating|datsun|dealer|degree|dental|design|direct|doctor|dunlop|dupont|durban|emerck|energy|estate|events|expert|family|flickr|futbol|gallup|garden|george|giving|global|google|gratis|health|hermes|hiphop|hockey|hotels|hughes|imamat|insure|intuit|jaguar|joburg|juegos|kaufen|kinder|kindle|kosher|lancia|latino|lawyer|lefrak|living|locker|london|luxury|madrid|maison|makeup|market|mattel|mobile|mobily|monash|mormon|moscow|museum|mutual|nagoya|natura|nissan|nissay|norton|nowruz|office|olayan|online|oracle|orange|otsuka|pfizer|photos|physio|piaget|pictet|quebec|racing|realty|reisen|repair|report|review|rocher|rogers|ryukyu|safety|sakura|sanofi|school|schule|search|secure|select|shouji|soccer|social|stream|studio|supply|suzuki|swatch|sydney|taipei|taobao|target|tattoo|tennis|tienda|tjmaxx|tkmaxx|toyota|travel|unicom|viajes|viking|villas|virgin|vision|voting|voyage|vuelos|walter|warman|webcam|xihuan|yachts|yandex|zappos|actor|adult|aetna|amfam|amica|apple|archi|audio|autos|azure|baidu|beats|bible|bingo|black|boats|bosch|build|canon|cards|chase|cheap|cisco|citic|click|cloud|coach|codes|crown|cymru|dabur|dance|deals|delta|dodge|drive|dubai|earth|edeka|email|epson|faith|fedex|final|forex|forum|gallo|games|gifts|gives|glade|glass|globo|gmail|green|gripe|group|gucci|guide|homes|honda|horse|house|hyatt|ikano|intel|irish|iveco|jetzt|koeln|kyoto|lamer|lease|legal|lexus|lilly|linde|lipsy|lixil|loans|locus|lotte|lotto|lupin|macys|mango|media|miami|money|mopar|movie|nadex|nexus|nikon|ninja|nokia|nowtv|omega|onion|osaka|paris|parts|party|phone|photo|pizza|place|poker|praxi|press|prime|promo|quest|radio|rehab|reise|ricoh|rocks|rodeo|rugby|salon|sener|seven|sharp|shell|shoes|skype|sling|smart|smile|solar|space|sport|stada|store|study|style|sucks|swiss|tatar|tires|tirol|tmall|today|tokyo|tools|toray|total|tours|trade|trust|tunes|tushu|ubank|vegas|video|vodka|volvo|wales|watch|weber|weibo|works|world|xerox|yahoo|aarp|able|adac|aero|aigo|akdn|ally|amex|arab|army|arpa|arte|asda|asia|audi|auto|baby|band|bank|bbva|beer|best|bike|bing|blog|blue|bofa|bond|book|buzz|cafe|call|camp|care|cars|casa|case|cash|cbre|cern|chat|citi|city|club|cool|coop|cyou|data|date|dclk|deal|dell|desi|diet|dish|docs|doha|duck|duns|dvag|erni|fage|fail|fans|farm|fast|fiat|fido|film|fire|fish|flir|food|ford|free|fund|game|gbiz|gent|ggee|gift|gmbh|gold|golf|goog|guge|guru|hair|haus|hdfc|help|here|hgtv|host|hsbc|icbc|ieee|imdb|immo|info|itau|java|jeep|jobs|jprs|kddi|kiwi|kpmg|kred|land|lego|lgbt|lidl|life|like|limo|link|live|loan|loft|love|ltda|luxe|maif|meet|meme|menu|mini|mint|mobi|moda|moto|name|navy|news|next|nico|nike|ollo|open|page|pars|pccw|pics|ping|pink|play|plus|pohl|porn|post|prod|prof|qpon|raid|read|reit|rent|rest|rich|rmit|room|rsvp|ruhr|safe|sale|sarl|save|saxo|scor|scot|seat|seek|sexy|shaw|shia|shop|show|silk|sina|site|skin|sncf|sohu|song|sony|spot|star|surf|talk|taxi|team|tech|teva|tiaa|tips|town|toys|tube|vana|visa|viva|vivo|vote|voto|wang|weir|wien|wiki|wine|work|xbox|yoga|zara|zero|zone|aaa|abb|abc|aco|ads|aeg|afl|aig|anz|aol|app|art|aws|axa|bar|bbc|bbt|bcg|bcn|bet|bid|bio|biz|bms|bmw|bnl|bom|boo|bot|box|buy|bzh|cab|cal|cam|car|cat|cba|cbn|cbs|ceb|ceo|cfa|cfd|com|crs|csc|dad|day|dds|dev|dhl|diy|dnp|dog|dot|dtv|dvr|eat|eco|edu|esq|eus|fan|fit|fly|foo|fox|frl|ftr|fun|fyi|gal|gap|gdn|gea|gle|gmo|gmx|goo|gop|got|gov|hbo|hiv|hkt|hot|how|ibm|ice|icu|ifm|inc|ing|ink|int|ist|itv|jcb|jcp|jio|jll|jmp|jnj|jot|joy|kfh|kia|kim|kpn|krd|lat|law|lds|llc|lol|lpl|ltd|man|map|mba|med|men|mil|mit|mlb|mls|mma|moe|moi|mom|mov|msd|mtn|mtr|nab|nba|nec|net|new|nfl|ngo|nhk|now|nra|nrw|ntt|nyc|obi|off|one|ong|onl|ooo|org|ott|ovh|pay|pet|phd|pid|pin|pnc|pro|pru|pub|pwc|qvc|red|ren|ril|rio|rip|run|rwe|sap|sas|sbi|sbs|sca|scb|ses|sew|sex|sfr|ski|sky|soy|srl|srt|stc|tab|tax|tci|tdk|tel|thd|tjx|top|trv|tui|tvs|ubs|uno|uol|ups|vet|vig|vin|vip|wed|win|wme|wow|wtc|wtf|xin|xxx|xyz|you|yun|zip|ac|ad|ae|af|ag|ai|al|am|ao|aq|ar|as|at|au|aw|ax|az|ba|bb|bd|be|bf|bg|bh|bi|bj|bm|bn|bo|br|bs|bt|bv|bw|by|bz|ca|cc|cd|cf|cg|ch|ci|ck|cl|cm|cn|co|cr|cu|cv|cw|cx|cy|cz|de|dj|dk|dm|do|dz|ec|ee|eg|er|es|et|eu|fi|fj|fk|fm|fo|fr|ga|gb|gd|ge|gf|gg|gh|gi|gl|gm|gn|gp|gq|gr|gs|gt|gu|gw|gy|hk|hm|hn|hr|ht|hu|id|ie|il|im|in|io|iq|ir|is|it|je|jm|jo|jp|ke|kg|kh|ki|km|kn|kp|kr|kw|ky|kz|la|lb|lc|li|lk|lr|ls|lt|lu|lv|ly|ma|mc|md|me|mg|mh|mk|ml|mm|mn|mo|mp|mq|mr|ms|mt|mu|mv|mw|mx|my|mz|na|nc|ne|nf|ng|ni|nl|no|np|nr|nu|nz|om|pa|pe|pf|pg|ph|pk|pl|pm|pn|pr|ps|pt|pw|py|qa|re|ro|rs|ru|rw|sa|sb|sc|sd|se|sg|sh|si|sj|sk|sl|sm|sn|so|sr|ss|st|su|sv|sx|sy|sz|tc|td|tf|tg|th|tj|tk|tl|tm|tn|to|tr|tt|tv|tw|tz|ua|ug|uk|us|uy|uz|va|vc|ve|vg|vi|vn|vu|wf|ws|ye|yt|za|zm|zw" 19 | _domain = `\b(?i)(?:` + _domainWord + `\.)+(?:` + _tlds + `)\b` 20 | 21 | // IPv4 and IPv6. 22 | _octet = `(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9][0-9]|[0-9])` 23 | _ipv4 = `\b` + _octet + `\.` + _octet + `\.` + _octet + `\.` + _octet + `\b` 24 | _ipv6 = `([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:([0-9a-fA-F]{1,4}:[0-9a-fA-F]{0,4}|:[0-9a-fA-F]{1,4})?|(:[0-9a-fA-F]{1,4}){0,2})|(:[0-9a-fA-F]{1,4}){0,3})|(:[0-9a-fA-F]{1,4}){0,4})|:(:[0-9a-fA-F]{1,4}){0,5})((:[0-9a-fA-F]{1,4}){2}|:(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])(\.(25[0-5]|(2[0-4]|1[0-9]|[1-9])?[0-9])){3})|(([0-9a-fA-F]{1,4}:){1,6}|:):[0-9a-fA-F]{1,4}|([0-9a-fA-F]{1,4}:){7}:` 25 | _ip = `(` + _ipv4 + `|` + _ipv6 + `)` 26 | 27 | hexDigit = "0123456789abcdef" 28 | ) 29 | 30 | var ( 31 | DefaultDomainRelation DomainRelationFn = func(domainA string, domainB string) bool { 32 | return isDomainRelated(domainA, domainB, false) 33 | } 34 | StrictDomainRelation DomainRelationFn = func(domainA string, domainB string) bool { 35 | return isDomainRelated(domainA, domainB, true) 36 | } 37 | IsDomainRelated = DefaultDomainRelation 38 | domainPattern = regexp.MustCompile(_domain) 39 | ipPattern = regexp.MustCompile(_ip) 40 | ) 41 | 42 | type DomainRelationFn func(domainA string, domainB string) bool 43 | 44 | func init() { 45 | domainPattern.Longest() 46 | } 47 | 48 | func DissectDomainsFromStrings(haystacks []string) (domains []string) { 49 | for _, haystack := range haystacks { 50 | domains = append(domains, DissectDomainsFromString(haystack)...) 51 | } 52 | return domains 53 | } 54 | 55 | func DissectDomainsFromString(haystack string) []string { 56 | domains := domainPattern.FindAllString(haystack, -1) 57 | for i := range domains { 58 | domains[i] = CleanDomain(domains[i]) 59 | } 60 | return domains 61 | } 62 | 63 | func DissectIpsFromStrings(haystacks []string) (ips []string) { 64 | for _, haystack := range haystacks { 65 | ips = append(ips, DissectIpsFromString(haystack)...) 66 | } 67 | return ips 68 | } 69 | 70 | func DissectIpsFromString(haystack string) []string { 71 | return ipPattern.FindAllString(haystack, -1) 72 | } 73 | 74 | func IsSubdomain(domain string) bool { 75 | return dns.CountLabel(domain) >= 3 76 | } 77 | 78 | func ParentDomainOf(domain string) string { 79 | labels := strings.Split(domain, ".") 80 | if len(labels) <= 2 { 81 | // We don't want a TLD. 82 | return "" 83 | } 84 | return strings.Join(labels[1:], ".") 85 | } 86 | 87 | func CleanDomain(domain string) string { 88 | domain = strings.TrimSuffix(domain, ".") 89 | domain = strings.TrimPrefix(domain, "*.") 90 | domain = strings.TrimPrefix(domain, "www.") 91 | return strings.ToLower(domain) 92 | } 93 | 94 | func isDomainRelated(domainA string, domainB string, strict bool) bool { 95 | labelsA := dns.SplitDomainName(domainA) 96 | labelsB := dns.SplitDomainName(domainB) 97 | 98 | if labelsA == nil || labelsB == nil { 99 | // Cut invalid domains. 100 | return false 101 | } 102 | 103 | labelsALen := len(labelsA) 104 | labelsBLen := len(labelsB) 105 | 106 | if labelsALen < 2 || labelsBLen < 2 { 107 | // Ignore TLDs. 108 | return false 109 | } 110 | 111 | // Heuristics: 112 | // 113 | // 1) Subdomain and its parent are related: sub.example.com <--> example.com 114 | // 2) Different subdomains are related: foo.example.com <--> bar.example.com 115 | // 3) Same 2nd order domains with different TLD are related: sub.example.com <--> example.cz 116 | // 117 | // => Therefor we say the domains are related iff at least 2nd order domains are the same. 118 | 119 | related := labelsA[labelsALen-2] == labelsB[labelsBLen-2] 120 | if related && strict { 121 | // In strict mode we also require TLD match. 122 | related = labelsA[labelsALen-1] == labelsB[labelsBLen-1] 123 | } 124 | return related 125 | } 126 | 127 | // reverseIPv4 returns a given IPv4 address in ARPA-like rDNS form. 128 | func reverseIPv4(ip net.IP) string { 129 | return uitoa(uint(ip[15])) + "." + uitoa(uint(ip[14])) + "." + uitoa(uint(ip[13])) + "." + uitoa(uint(ip[12])) 130 | } 131 | 132 | // reverseIPv6 returns a given IPv6 address in ARPA-like rDNS form. 133 | func reverseIPv6(ip net.IP) string { 134 | buf := make([]byte, 0, len(ip)*4) 135 | // Add it, in reverse, to the buffer 136 | for i := len(ip) - 1; i >= 0; i-- { 137 | buf = append(buf, hexDigit[ip[i]&0xF], '.', hexDigit[ip[i]>>4]) 138 | if i > 0 { 139 | buf = append(buf, '.') 140 | } 141 | } 142 | 143 | return string(buf) 144 | } 145 | 146 | // uitoa converts an unsigned integer to decimal string. 147 | // Carved from net.dnsclient. 148 | func uitoa(val uint) string { 149 | if val == 0 { // avoid string allocation 150 | return "0" 151 | } 152 | var buf [20]byte // big enough for 64bit value base 10 153 | i := len(buf) - 1 154 | for val >= 10 { 155 | q := val / 10 156 | buf[i] = byte('0' + val - q*10) 157 | i-- 158 | val = q 159 | } 160 | // val < 10 161 | buf[i] = byte('0' + val) 162 | return string(buf[i:]) 163 | } 164 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func Test_DissectDomainsFrom_By_simple_domain(t *testing.T) { 10 | // Execute. 11 | domains := DissectDomainsFromString("example.com") 12 | 13 | // Assert. 14 | assert.Len(t, domains, 1) 15 | assert.Equal(t, "example.com", domains[0]) 16 | } 17 | 18 | func Test_DissectDomainsFrom_By_subdomain(t *testing.T) { 19 | // Execute. 20 | domains := DissectDomainsFromString("example.domain-hyphen.com") 21 | 22 | // Assert. 23 | assert.Len(t, domains, 1) 24 | assert.Equal(t, "example.domain-hyphen.com", domains[0]) 25 | } 26 | 27 | func Test_DissectDomainsFrom_By_www_subdomain(t *testing.T) { 28 | // Execute. 29 | domains := DissectDomainsFromString("www.example.domain-hyphen.com") 30 | 31 | // Assert. 32 | assert.Len(t, domains, 1) 33 | assert.Equal(t, "example.domain-hyphen.com", domains[0]) 34 | } 35 | 36 | func Test_DissectDomainsFrom_By_exotic_tld(t *testing.T) { 37 | // Execute. 38 | domains := DissectDomainsFromString("www.example.domain-hyphen.museum") 39 | 40 | // Assert. 41 | assert.Len(t, domains, 1) 42 | assert.Equal(t, "example.domain-hyphen.museum", domains[0]) 43 | } 44 | 45 | func Test_DissectDomainsFrom_By_complex_domain(t *testing.T) { 46 | // Execute. 47 | domains := DissectDomainsFromString("external.asd1230-123.asd_internal.asd.gm-_ail.aero") 48 | 49 | // Assert. 50 | assert.Len(t, domains, 1) 51 | assert.Equal(t, "external.asd1230-123.asd_internal.asd.gm-_ail.aero", domains[0]) 52 | } 53 | 54 | func Test_DissectDomainsFrom_By_complex_url_in_text(t *testing.T) { 55 | // Execute. 56 | domains := DissectDomainsFromString("Hello world: https://user:password@external.asd1230-123.asd_internal.asd.gm-_ail.aero:8080/foo/bar.html is really cool\nURL") 57 | 58 | // Assert. 59 | assert.Len(t, domains, 1) 60 | assert.Equal(t, "external.asd1230-123.asd_internal.asd.gm-_ail.aero", domains[0]) 61 | } 62 | 63 | func Test_DissectDomainsFrom_By_multiple_urls(t *testing.T) { 64 | // Execute. 65 | domains := DissectDomainsFromString("Hello world: https://user:password@external.asd1230-123.asd_internal.asd.gm-_ail.aero:8080/foo/bar.html is really cool\nURL and this is another one http://www.foo-bar_baz.co") 66 | 67 | // Assert. 68 | assert.Len(t, domains, 2) 69 | assert.Equal(t, "external.asd1230-123.asd_internal.asd.gm-_ail.aero", domains[0]) 70 | assert.Equal(t, "foo-bar_baz.co", domains[1]) 71 | } 72 | 73 | func Test_DissectDomainsFrom_By_invalid_domain(t *testing.T) { 74 | // Execute. 75 | domains := DissectDomainsFromString("bad.-example.com") 76 | 77 | // Assert. 78 | assert.Len(t, domains, 1) 79 | assert.Equal(t, "example.com", domains[0]) 80 | } 81 | 82 | func Test_isDomainRelated_By_same_domain(t *testing.T) { 83 | // Setup. 84 | domainA := "example.com" 85 | domainB := domainA 86 | 87 | // Execute. 88 | res1 := isDomainRelated(domainA, domainB, false) 89 | res2 := isDomainRelated(domainB, domainA, false) 90 | 91 | // Assert. 92 | assert.Equal(t, true, res1) 93 | assert.Equal(t, true, res2) 94 | } 95 | 96 | func Test_isDomainRelated_By_subdomain(t *testing.T) { 97 | // Setup. 98 | domainA := "example.com" 99 | domainB := "sub.example.com" 100 | 101 | // Execute. 102 | res1 := isDomainRelated(domainA, domainB, false) 103 | res2 := isDomainRelated(domainB, domainA, false) 104 | 105 | // Assert. 106 | assert.Equal(t, true, res1) 107 | assert.Equal(t, true, res2) 108 | } 109 | 110 | func Test_isDomainRelated_By_domain_with_different_TLD(t *testing.T) { 111 | // Setup. 112 | domainA := "example.com" 113 | domainB := "sub.example.net" 114 | 115 | // Execute. 116 | res1 := isDomainRelated(domainA, domainB, false) 117 | res2 := isDomainRelated(domainB, domainA, false) 118 | 119 | // Assert. 120 | assert.Equal(t, true, res1) 121 | assert.Equal(t, true, res2) 122 | } 123 | 124 | func Test_isDomainRelated_By_domain_with_different_TLD_strict(t *testing.T) { 125 | // Setup. 126 | domainA := "example.com" 127 | domainB := "sub.example.net" 128 | 129 | // Execute. 130 | res1 := isDomainRelated(domainA, domainB, true) 131 | res2 := isDomainRelated(domainB, domainA, true) 132 | 133 | // Assert. 134 | assert.Equal(t, false, res1) 135 | assert.Equal(t, false, res2) 136 | } 137 | 138 | func Test_isDomainRelated_By_TLDs(t *testing.T) { 139 | // Setup. 140 | domainA := "com" 141 | domainB := "com" 142 | 143 | // Execute. 144 | res := isDomainRelated(domainA, domainB, false) 145 | 146 | // Assert. 147 | assert.Equal(t, false, res) 148 | } 149 | 150 | func Test_isDomainRelated_By_invalid_domain(t *testing.T) { 151 | // Setup. 152 | domainA := "." 153 | domainB := "example.com" 154 | 155 | // Execute. 156 | res1 := isDomainRelated(domainA, domainB, false) 157 | res2 := isDomainRelated(domainB, domainA, false) 158 | 159 | // Assert. 160 | assert.Equal(t, false, res1) 161 | assert.Equal(t, false, res2) 162 | } 163 | -------------------------------------------------------------------------------- /whois.go: -------------------------------------------------------------------------------- 1 | package udig 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "io" 7 | "strings" 8 | 9 | "github.com/domainr/whois" 10 | ) 11 | 12 | // Expect to receive a reader to text with 3 parts: 13 | // 1. Key-value pairs separated by colon (":") 14 | // 2. A line `>>> Last update of WHOIS database: [date]<<<` 15 | // 3. Follow by an empty line, then free text of the legal disclaimers. 16 | func parseWhoisResponse(reader io.Reader) (contacts []WhoisContact) { 17 | scanner := bufio.NewScanner(reader) 18 | contact := WhoisContact{} 19 | 20 | var lineNumber int 21 | for lineNumber = 1; scanner.Scan(); lineNumber++ { 22 | // Grab the line and clean it. 23 | line := strings.Trim(scanner.Text(), " \n\r\t") 24 | line = strings.ToLower(line) 25 | 26 | if line == "" { 27 | // Empty line usually separates contacts -> create a new one. 28 | if !contact.IsEmpty() { 29 | contacts = append(contacts, contact) 30 | contact = WhoisContact{} 31 | } 32 | continue 33 | } else if line[0] == '%' { 34 | // Comment/disclaimer -> skip. 35 | continue 36 | } else if strings.Index(line, ">>> last update of whois database") == 0 { 37 | // Last line -> break. 38 | if !contact.IsEmpty() { 39 | contacts = append(contacts, contact) 40 | } 41 | break 42 | } 43 | 44 | // Parse the individual parts. 45 | parts := strings.SplitN(line, ":", 2) 46 | if len(parts) != 2 { 47 | // Invalid line -> skip. 48 | continue 49 | } 50 | 51 | key := strings.Trim(parts[0], " \n\r\t") 52 | value := strings.Trim(parts[1], " \n\r\t") 53 | if key == "" || value == "" { 54 | // No key/value -> skip. 55 | continue 56 | } 57 | 58 | switch key { 59 | case "registry domain id": 60 | setOrAppendString(&contact.RegistryDomainId, value) 61 | break 62 | case "registrant": 63 | setOrAppendString(&(contact.Registrant), value) 64 | break 65 | case "registrant organization": 66 | setOrAppendString(&contact.RegistrantOrganization, value) 67 | break 68 | case "registrant state/province": 69 | setOrAppendString(&contact.RegistrantStateProvince, value) 70 | break 71 | case "registrant country": 72 | setOrAppendString(&contact.RegistrantCountry, value) 73 | break 74 | case "registrar": 75 | setOrAppendString(&contact.Registrar, value) 76 | break 77 | case "registrar iana id": 78 | setOrAppendString(&contact.RegistrarIanaId, value) 79 | break 80 | case "registrar whois server": 81 | setOrAppendString(&contact.RegistrarWhoisServer, value) 82 | break 83 | case "registrar url": 84 | setOrAppendString(&contact.RegistrarUrl, value) 85 | break 86 | case "creation date": 87 | setOrAppendString(&contact.CreationDate, value) 88 | break 89 | case "updated date": 90 | setOrAppendString(&contact.UpdatedDate, value) 91 | break 92 | case "registered": 93 | setOrAppendString(&contact.Registered, value) 94 | break 95 | case "changed": 96 | setOrAppendString(&contact.Changed, value) 97 | break 98 | case "expire": 99 | setOrAppendString(&contact.Expire, value) 100 | break 101 | case "nsset": 102 | setOrAppendString(&contact.NSSet, value) 103 | break 104 | case "contact": 105 | setOrAppendString(&contact.Contact, value) 106 | break 107 | case "name": 108 | setOrAppendString(&contact.Name, value) 109 | break 110 | case "address": 111 | setOrAppendString(&contact.Address, value) 112 | break 113 | } 114 | } 115 | 116 | return contacts 117 | } 118 | 119 | func setOrAppendString(target *string, value string) { 120 | if *target != "" { 121 | value = *target + ", " + value 122 | } 123 | *target = value 124 | } 125 | 126 | ///////////////////////////////////////// 127 | // WHOIS RESOLVER 128 | ///////////////////////////////////////// 129 | 130 | // NewWhoisResolver creates a new WhoisResolver instance provisioned 131 | // with sensible defaults. 132 | func NewWhoisResolver() *WhoisResolver { 133 | return &WhoisResolver{ 134 | Client: whois.NewClient(DefaultTimeout), 135 | } 136 | } 137 | 138 | // Type returns "WHOIS". 139 | func (resolver *WhoisResolver) Type() ResolutionType { 140 | return TypeWHOIS 141 | } 142 | 143 | // ResolveDomain attempts to resolve a given domain using WHOIS query 144 | // yielding a list of WHOIS contacts. 145 | func (resolver *WhoisResolver) ResolveDomain(domain string) Resolution { 146 | resolution := &WhoisResolution{ 147 | ResolutionBase: &ResolutionBase{query: domain}, 148 | } 149 | 150 | // Prepare a request. 151 | request, err := whois.NewRequest(domain) 152 | if err != nil { 153 | LogErr("%s: %s -> %s", TypeWHOIS, domain, err.Error()) 154 | return resolution 155 | } 156 | 157 | response, err := resolver.Client.Fetch(request) 158 | if err != nil { 159 | LogErr("%s: %s -> %s", TypeWHOIS, domain, err.Error()) 160 | return resolution 161 | } 162 | 163 | contacts := parseWhoisResponse(bytes.NewReader(response.Body)) 164 | for _, contact := range contacts { 165 | resolution.Contacts = append(resolution.Contacts, contact) 166 | } 167 | 168 | return resolution 169 | } 170 | 171 | ///////////////////////////////////////// 172 | // WHOIS RESOLUTION 173 | ///////////////////////////////////////// 174 | 175 | // Type returns "WHOIS". 176 | func (res *WhoisResolution) Type() ResolutionType { 177 | return TypeWHOIS 178 | } 179 | 180 | // Domains returns a list of domains discovered in records within this Resolution. 181 | func (res *WhoisResolution) Domains() (domains []string) { 182 | for _, contact := range res.Contacts { 183 | domains = append(domains, DissectDomainsFromString(contact.RegistryDomainId)...) 184 | domains = append(domains, DissectDomainsFromString(contact.Registrant)...) 185 | domains = append(domains, DissectDomainsFromString(contact.RegistrantOrganization)...) 186 | domains = append(domains, DissectDomainsFromString(contact.RegistrantStateProvince)...) 187 | domains = append(domains, DissectDomainsFromString(contact.RegistrantCountry)...) 188 | domains = append(domains, DissectDomainsFromString(contact.Registrar)...) 189 | domains = append(domains, DissectDomainsFromString(contact.RegistrarIanaId)...) 190 | domains = append(domains, DissectDomainsFromString(contact.RegistrarWhoisServer)...) 191 | domains = append(domains, DissectDomainsFromString(contact.RegistrarUrl)...) 192 | domains = append(domains, DissectDomainsFromString(contact.CreationDate)...) 193 | domains = append(domains, DissectDomainsFromString(contact.UpdatedDate)...) 194 | domains = append(domains, DissectDomainsFromString(contact.Registered)...) 195 | domains = append(domains, DissectDomainsFromString(contact.Changed)...) 196 | domains = append(domains, DissectDomainsFromString(contact.Expire)...) 197 | domains = append(domains, DissectDomainsFromString(contact.NSSet)...) 198 | domains = append(domains, DissectDomainsFromString(contact.Contact)...) 199 | domains = append(domains, DissectDomainsFromString(contact.Name)...) 200 | domains = append(domains, DissectDomainsFromString(contact.Address)...) 201 | } 202 | return domains 203 | } 204 | 205 | ///////////////////////////////////////// 206 | // WHOIS CONTACT 207 | ///////////////////////////////////////// 208 | 209 | func (contact *WhoisContact) IsEmpty() bool { 210 | return contact.RegistryDomainId == "" && 211 | contact.Registrant == "" && 212 | contact.RegistrantOrganization == "" && 213 | contact.RegistrantStateProvince == "" && 214 | contact.RegistrantCountry == "" && 215 | contact.Registrar == "" && 216 | contact.RegistrarIanaId == "" && 217 | contact.RegistrarWhoisServer == "" && 218 | contact.RegistrarUrl == "" && 219 | contact.CreationDate == "" && 220 | contact.UpdatedDate == "" && 221 | contact.Registered == "" && 222 | contact.Changed == "" && 223 | contact.Expire == "" && 224 | contact.NSSet == "" && 225 | contact.Contact == "" && 226 | contact.Name == "" && 227 | contact.Address == "" 228 | } 229 | 230 | func (contact *WhoisContact) String() string { 231 | var entries []string 232 | 233 | if contact.RegistryDomainId != "" { 234 | entries = append(entries, "registry domain id: "+contact.RegistryDomainId) 235 | } 236 | if contact.Registrant != "" { 237 | entries = append(entries, "registrant: "+contact.Registrant) 238 | } 239 | if contact.RegistrantOrganization != "" { 240 | entries = append(entries, "registrant organization: "+contact.RegistrantOrganization) 241 | } 242 | if contact.RegistrantStateProvince != "" { 243 | entries = append(entries, "registrant state/province: "+contact.RegistrantStateProvince) 244 | } 245 | if contact.RegistrantCountry != "" { 246 | entries = append(entries, "registrant country: "+contact.RegistrantCountry) 247 | } 248 | if contact.Registrar != "" { 249 | entries = append(entries, "registrar: "+contact.Registrar) 250 | } 251 | if contact.RegistrarIanaId != "" { 252 | entries = append(entries, "registrar iana id: "+contact.RegistrarIanaId) 253 | } 254 | if contact.RegistrarWhoisServer != "" { 255 | entries = append(entries, "registrar whois server: "+contact.RegistrarWhoisServer) 256 | } 257 | if contact.RegistrarUrl != "" { 258 | entries = append(entries, "registrar url: "+contact.RegistrarUrl) 259 | } 260 | if contact.CreationDate != "" { 261 | entries = append(entries, "creation date: "+contact.CreationDate) 262 | } 263 | if contact.UpdatedDate != "" { 264 | entries = append(entries, "updated date: "+contact.UpdatedDate) 265 | } 266 | if contact.Registered != "" { 267 | entries = append(entries, "registered: "+contact.Registered) 268 | } 269 | if contact.Changed != "" { 270 | entries = append(entries, "changed: "+contact.Changed) 271 | } 272 | if contact.Expire != "" { 273 | entries = append(entries, "expire: "+contact.Expire) 274 | } 275 | if contact.NSSet != "" { 276 | entries = append(entries, "nsset: "+contact.NSSet) 277 | } 278 | if contact.Contact != "" { 279 | entries = append(entries, "contact: "+contact.Contact) 280 | } 281 | if contact.Name != "" { 282 | entries = append(entries, "name: "+contact.Name) 283 | } 284 | if contact.Address != "" { 285 | entries = append(entries, "address: "+contact.Address) 286 | } 287 | 288 | return strings.Join(entries, ", ") 289 | } 290 | --------------------------------------------------------------------------------