├── .gitignore
├── LICENSE
├── README.md
├── bsw
    ├── axfr.go
    ├── axfr_test.go
    ├── bing.go
    ├── bing_test.go
    ├── bsw.go
    ├── censys.go
    ├── common_crawl.go
    ├── config.go
    ├── ct.go
    ├── dictionary.go
    ├── dictionary_test.go
    ├── exfiltrated.go
    ├── generic.go
    ├── header.go
    ├── logontube.go
    ├── logontube_test.go
    ├── mx.go
    ├── mx_test.go
    ├── ns.go
    ├── ns_test.go
    ├── reverse.go
    ├── shodan.go
    ├── shodan_test.go
    ├── srv.go
    ├── tls.go
    ├── version.go
    ├── viewdns.go
    ├── viewdns_test.go
    ├── vt.go
    └── yandex.go
├── helpers
    └── helpers.go
└── main.go


/.gitignore:
--------------------------------------------------------------------------------
 1 | *.tar.gz
 2 | *.sh
 3 | *.sqp
 4 | blacksheepwall
 5 | *.swp
 6 | .DS_Store
 7 | .idea/
 8 | *.yaml
 9 | blacksheepwall.exe
10 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2012, 2013 Tom Steele, Jason Doyle
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is
 8 | furnished to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in
11 | all copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
19 | THE SOFTWARE.
20 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | blacksheepwall
  2 | ===
  3 | 
  4 | ## Archive Notice
  5 | I am no longer maintaing this tool. BSW has served me well, and was written in Go in 2013. I would suggest using [amass](https://github.com/OWASP/Amass) as it is actively supported.
  6 | 
  7 | [![](https://godoc.org/github.com/tomsteele/blacksheepwall/bsw?status.svg)](http://godoc.org/github.com/tomsteele/blacksheepwall/bsw)
  8 | 
  9 | blacksheepwall is a hostname reconnaissance tool written in Go. It can also be used as a stand-alone package in your tools.
 10 | 
 11 | ## Download
 12 | 
 13 | Binary packages for every supported operating system are available [here](https://github.com/tomsteele/blacksheepwall/releases/latest).
 14 | 
 15 | ## Install
 16 | 
 17 | You can download a compiled binary and just run it. Alternatively, if you have Go installed and configured with a workspace, you can run:
 18 | ```
 19 | $ go get github.com/tomsteele/blacksheepwall
 20 | ```
 21 | 
 22 | ## Usage
 23 | 
 24 | ```
 25 |  Usage: blacksheepwall [options] <ip address or CIDR>
 26 | 
 27 |  Options:
 28 |   -h, --help            Show Usage and exit.
 29 | 
 30 |   -version              Show version and exit.
 31 | 
 32 |   -debug                Enable debugging and show errors returned from tasks.
 33 | 
 34 |   -config               Location of a YAML file containing any of the options below.
 35 |                         Hypens should be replaced with underscores (e.g. bing-html, bing_html).
 36 |                         Options that do not take an argument are booleans and should be represented
 37 |                         using true/false (e.g. bing_html: true).
 38 | 
 39 |   -timeout              Maximum timeout in seconds for SOCKET connections.  [default .5 seconds]
 40 | 
 41 |   -concurrency <int>    Max amount of concurrent tasks.  [default: 100]
 42 | 
 43 |   -server <string>      DNS server address.  [default: "8.8.8.8"]
 44 | 
 45 |   -input <string>       Line separated file of networks (CIDR) or IP Addresses.
 46 | 
 47 |   -ipv6                 Look for additional AAAA records where applicable.
 48 | 
 49 |   -domain <string>      Target domain to use for certain tasks, can be a single
 50 |                         domain or a file of line separated domains.
 51 | 
 52 |   -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
 53 |                         each result previously identified hostname.
 54 | 
 55 |   -parse <string>       Generate output by parsing JSON from a file from a previous scan.
 56 | 
 57 |   -validate             Validate hostnames using a RFC compliant regex.
 58 | 
 59 |  Passive:
 60 |   -dictionary <string>  Attempt to retrieve the CNAME and A record for
 61 |                         each subdomain in the line separated file.
 62 | 
 63 |   -ns                   Lookup the ip and hostname of any nameservers for the domain.
 64 | 
 65 |   -mx                   Lookup the ip and hostmame of any mx records for the domain.
 66 | 
 67 |   -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
 68 |                         search 'rhost:' operator to find subdomains of a
 69 |                         provided domain.
 70 | 
 71 |   -bing <string>        Provided a base64 encoded API key. Use the Bing search
 72 |                         API's 'ip:' operator to lookup hostnames for each ip, and the
 73 |                         'domain:' operator to find ips/hostnames for a domain.
 74 | 
 75 |   -bing-html            Use Bing search 'ip:' operator to lookup hostname for each ip, and the
 76 |                         'domain:' operator to find ips/hostnames for a domain. Only
 77 |                         the first page is scraped. This does not use the API.
 78 | 
 79 |   -shodan <string>      Provided a Shodan API key. Use Shodan's API '/dns/reverse' to lookup hostnames for
 80 |                         each ip, and '/shodan/host/search' to lookup ips/hostnames for a domain.
 81 |                         A single call is made for all ips.
 82 | 
 83 |   -reverse              Retrieve the PTR for each host.
 84 | 
 85 |   -viewdns-html         Lookup each host using viewdns.info's Reverse IP
 86 |                         Lookup function. Use sparingly as they will block you.
 87 | 
 88 |   -viewdns <string>     Lookup each host using viewdns.info's API and Reverse IP Lookup function.
 89 | 
 90 |   -logontube            Lookup each host and/or domain using logontube.com's API. As of this release
 91 |                         the site is down.
 92 | 
 93 |   -exfiltrated          Lookup hostnames returned from exfiltrated.com's hostname search.
 94 | 
 95 |   -censys <string>      Searches censys.io for a domain. Names are gathered from TLS certificates for each host
 96 |                         returned from this search. The provided string should be your API ID and Secret separated
 97 | 						by a colon.
 98 | 
 99 |   -crtsh                Searches crt.sh for certificates related to the provided domain.
100 |   
101 |   -vt                   Searches VirusTotal for subdomains for the provided domain.
102 | 
103 |   -srv                  Find DNS SRV record and retrieve associated hostname/IP info.
104 | 
105 |   -cmn-crawl <string>   Search commoncrawl.org for subdomains of a domain. The provided argument should be the index
106 |                         to be used. For example: "CC-MAIN-2017-04-index"
107 | 
108 |  Active:
109 |   -axfr                 Attempt a zone transfer on the domain.
110 | 
111 |   -headers              Perform HTTP(s) requests to each host and look for
112 |                         hostnames in a possible Location header.
113 | 
114 |   -tls                  Attempt to retrieve names from TLS certificates
115 |                         (CommonName and Subject Alternative Name).
116 | 
117 |  Output Options:
118 |   -clean                Print results as unique hostnames for each host.
119 |   -csv                  Print results in csv format.
120 |   -json                 Print results as JSON.
121 | ```
122 | 


--------------------------------------------------------------------------------
/bsw/axfr.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"strings"
 5 | 
 6 | 	"github.com/miekg/dns"
 7 | )
 8 | 
 9 | // AXFR attempts a zone transfer for the domain.
10 | func AXFR(domain, serverAddr string) *Tsk {
11 | 	t := newTsk("axfr")
12 | 	servers, err := LookupNS(domain, serverAddr)
13 | 	if err != nil {
14 | 		t.SetErr(err)
15 | 		return t
16 | 	}
17 | 
18 | 	for _, s := range servers {
19 | 		tr := dns.Transfer{}
20 | 		m := &dns.Msg{}
21 | 		m.SetAxfr(dns.Fqdn(domain))
22 | 		in, err := tr.In(m, s+":53")
23 | 		if err != nil {
24 | 			t.SetErr(err)
25 | 			return t
26 | 		}
27 | 		for ex := range in {
28 | 			for _, a := range ex.RR {
29 | 				var ip, hostname string
30 | 				switch v := a.(type) {
31 | 				case *dns.A:
32 | 					ip = v.A.String()
33 | 					hostname = v.Hdr.Name
34 | 				case *dns.AAAA:
35 | 					ip = v.AAAA.String()
36 | 					hostname = v.Hdr.Name
37 | 				case *dns.PTR:
38 | 					ip = v.Hdr.Name
39 | 					hostname = v.Ptr
40 | 				case *dns.NS:
41 | 					cip, err := LookupName(v.Ns, serverAddr)
42 | 					if err != nil || len(cip) == 0 {
43 | 						continue
44 | 					}
45 | 					ip = cip[0]
46 | 					hostname = v.Ns
47 | 				case *dns.CNAME:
48 | 					cip, err := LookupName(v.Target, serverAddr)
49 | 					if err != nil || len(cip) == 0 {
50 | 						continue
51 | 					}
52 | 					hostname = v.Hdr.Name
53 | 					ip = cip[0]
54 | 				case *dns.SRV:
55 | 					cip, err := LookupName(v.Target, serverAddr)
56 | 					if err != nil || len(cip) == 0 {
57 | 						continue
58 | 					}
59 | 					ip = cip[0]
60 | 					hostname = v.Target
61 | 				default:
62 | 					continue
63 | 				}
64 | 				t.AddResult(ip, strings.TrimRight(hostname, "."))
65 | 			}
66 | 		}
67 | 	}
68 | 	return t
69 | }
70 | 


--------------------------------------------------------------------------------
/bsw/axfr_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestAXFR(t *testing.T) {
 8 | 	tsk := AXFR("zonetransfer.me", "8.8.8.8")
 9 | 	if tsk.Err() != nil {
10 | 		t.Error("error returned from AXFR")
11 | 		t.Log(tsk.Err())
12 | 	}
13 | 	if len(tsk.Results()) < 10 {
14 | 		t.Error("expected more results from AXFR")
15 | 	}
16 | }
17 | 


--------------------------------------------------------------------------------
/bsw/bing.go:
--------------------------------------------------------------------------------
  1 | package bsw
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"errors"
  6 | 	"io/ioutil"
  7 | 	"net/http"
  8 | 	"net/url"
  9 | 
 10 | 	"github.com/PuerkitoBio/goquery"
 11 | )
 12 | 
 13 | type bingMessage struct {
 14 | 	D bingResults `json:"D"`
 15 | }
 16 | 
 17 | type bingResults struct {
 18 | 	Results []bingResult `json:"Results"`
 19 | }
 20 | 
 21 | type bingResult struct {
 22 | 	Metadata    bingMetadata `json:"__Metadata"`
 23 | 	ID          string       `json:"id"`
 24 | 	Title       string       `json:"Title"`
 25 | 	Description string       `json:"Description"`
 26 | 	DisplayURL  string       `json:"DisplayUrl"`
 27 | 	URL         string       `json:"Url"`
 28 | }
 29 | 
 30 | type bingMetadata struct {
 31 | 	URI  string `json:"Uri"`
 32 | 	Type string `json:"Type"`
 33 | }
 34 | 
 35 | const azureURL = "https://api.datamarket.azure.com"
 36 | 
 37 | // FindBingSearchPath attempts an authenticated search request to two different Bing API paths. If and when a
 38 | // search is successfull, that path will be returned. If no path is valid this function
 39 | // returns an error.
 40 | func FindBingSearchPath(key string) (string, error) {
 41 | 	paths := []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
 42 | 	query := "?Query=%27I<3BSW%27"
 43 | 	for _, path := range paths {
 44 | 		fullURL := azureURL + path + query
 45 | 		client := &http.Client{}
 46 | 		req, err := http.NewRequest("GET", fullURL, nil)
 47 | 		if err != nil {
 48 | 			return "", err
 49 | 		}
 50 | 		req.SetBasicAuth(key, key)
 51 | 		resp, err := client.Do(req)
 52 | 		if err != nil {
 53 | 			return "", err
 54 | 		}
 55 | 		if resp.StatusCode == 200 {
 56 | 			return path, nil
 57 | 		}
 58 | 	}
 59 | 	return "", errors.New("invalid Bing API key")
 60 | }
 61 | 
 62 | // BingAPIIP uses the bing search API and 'ip' search operator to find alternate hostnames for
 63 | // a single IP.
 64 | func BingAPIIP(ip, key, path string) *Tsk {
 65 | 	t := newTsk("bing API")
 66 | 	client := &http.Client{}
 67 | 	req, err := http.NewRequest("GET", azureURL+path+"?Query=%27ip:"+ip+"%27&$top=50&Adult=%27off%27&$format=json", nil)
 68 | 	if err != nil {
 69 | 		t.SetErr(err)
 70 | 		return t
 71 | 	}
 72 | 	req.SetBasicAuth(key, key)
 73 | 	resp, err := client.Do(req)
 74 | 	if err != nil {
 75 | 		t.SetErr(err)
 76 | 		return t
 77 | 	}
 78 | 	defer resp.Body.Close()
 79 | 	body, err := ioutil.ReadAll(resp.Body)
 80 | 	if err != nil {
 81 | 		t.SetErr(err)
 82 | 		return t
 83 | 	}
 84 | 	m := &bingMessage{}
 85 | 	if err = json.Unmarshal(body, &m); err != nil {
 86 | 		t.SetErr(err)
 87 | 		return t
 88 | 	}
 89 | 	for _, res := range m.D.Results {
 90 | 		if u, err := url.Parse(res.URL); err == nil && u.Host != "" {
 91 | 			t.AddResult(ip, u.Host)
 92 | 		}
 93 | 	}
 94 | 	return t
 95 | }
 96 | 
 97 | // BingAPIDomain uses the bing search API and 'domain' search operator to find hostnames for
 98 | // a single domain.
 99 | func BingAPIDomain(domain, key, path, server string) *Tsk {
100 | 	t := newTsk("bing API")
101 | 	client := &http.Client{}
102 | 	req, err := http.NewRequest("GET", azureURL+path+"?Query=%27domain:"+domain+"%27&$top=50&Adult=%27off%27&$format=json", nil)
103 | 	if err != nil {
104 | 		t.SetErr(err)
105 | 		return t
106 | 	}
107 | 	req.SetBasicAuth(key, key)
108 | 	resp, err := client.Do(req)
109 | 	if err != nil {
110 | 		t.SetErr(err)
111 | 		return t
112 | 	}
113 | 	defer resp.Body.Close()
114 | 	body, err := ioutil.ReadAll(resp.Body)
115 | 	if err != nil {
116 | 		t.SetErr(err)
117 | 		return t
118 | 	}
119 | 	m := &bingMessage{}
120 | 	if err = json.Unmarshal(body, &m); err != nil {
121 | 		t.SetErr(err)
122 | 		return t
123 | 	}
124 | 	for _, res := range m.D.Results {
125 | 		u, err := url.Parse(res.URL)
126 | 		if err != nil || u.Host == "" {
127 | 			continue
128 | 		}
129 | 		ips, err := LookupName(u.Host, server)
130 | 		if err != nil || len(ips) == 0 {
131 | 			cfqdn, err := LookupCname(u.Host, server)
132 | 			if err != nil || cfqdn == "" {
133 | 				continue
134 | 			}
135 | 			ips, err = LookupName(cfqdn, server)
136 | 			if err != nil || len(ips) == 0 {
137 | 				continue
138 | 			}
139 | 		}
140 | 		for _, ip := range ips {
141 | 			t.AddResult(ip, u.Host)
142 | 		}
143 | 	}
144 | 	return t
145 | }
146 | 
147 | // BingIP uses bing's 'ip:' search operator and scrapes the HTML to find hostnames for an ip.
148 | func BingIP(ip string) *Tsk {
149 | 	t := newTsk("bing ip")
150 | 	resp, err := http.Get("http://www.bing.com/search?q=ip:" + ip)
151 | 	if err != nil {
152 | 		t.SetErr(err)
153 | 		return t
154 | 	}
155 | 	doc, err := goquery.NewDocumentFromResponse(resp)
156 | 	if err != nil {
157 | 		t.SetErr(err)
158 | 		return t
159 | 	}
160 | 	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
161 | 		u, err := url.Parse(s.Text())
162 | 		if err != nil || u.Host == "" {
163 | 			return
164 | 		}
165 | 		t.AddResult(ip, u.Host)
166 | 	})
167 | 	return t
168 | }
169 | 
170 | // BingDomain uses bing's 'domain:' search operator and scrapes the HTML to find ips and hostnames for a domain.
171 | func BingDomain(domain, server string) *Tsk {
172 | 	t := newTsk("bing domain")
173 | 	resp, err := http.Get("http://www.bing.com/search?q=domain:" + domain)
174 | 	if err != nil {
175 | 		t.SetErr(err)
176 | 		return t
177 | 	}
178 | 	doc, err := goquery.NewDocumentFromResponse(resp)
179 | 	if err != nil {
180 | 		t.SetErr(err)
181 | 		return t
182 | 	}
183 | 	doc.Selection.Find("cite").Each(func(_ int, s *goquery.Selection) {
184 | 		u, err := url.Parse(s.Text())
185 | 		if err != nil || u.Host == "" {
186 | 			return
187 | 		}
188 | 		ips, err := LookupName(u.Host, server)
189 | 		if err != nil || len(ips) == 0 {
190 | 			cfqdn, err := LookupCname(u.Host, server)
191 | 			if err != nil || cfqdn == "" {
192 | 				return
193 | 			}
194 | 			ips, err = LookupName(cfqdn, server)
195 | 			if err != nil || len(ips) == 0 {
196 | 				return
197 | 			}
198 | 
199 | 		}
200 | 		for _, ip := range ips {
201 | 			t.AddResult(ip, u.Host)
202 | 		}
203 | 	})
204 | 	return t
205 | }
206 | 


--------------------------------------------------------------------------------
/bsw/bing_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"strings"
 5 | 	"testing"
 6 | )
 7 | 
 8 | func TestInvalidKeyToPath(t *testing.T) {
 9 | 	_, err := FindBingSearchPath("notavalidkey")
10 | 	if err == nil {
11 | 		t.Error("FindBingSearchPath did not return error for bad key")
12 | 	}
13 | }
14 | 
15 | func TestInvalidBingKey(t *testing.T) {
16 | 	tsk := BingAPIIP("4.2.2.2", "notavalidkey", "/Data.ashx/Bing/Search/v1/Web")
17 | 	if tsk.Err() == nil {
18 | 		t.Error("BingAPI did not return error for bad key and path")
19 | 	}
20 | }
21 | 
22 | func TestBingIP(t *testing.T) {
23 | 	tsk := BingIP("198.41.208.143")
24 | 	if err := tsk.Err(); err != nil {
25 | 		t.Error("bing returned an error")
26 | 		t.Log(err)
27 | 	}
28 | 	if tsk.Task() != "bing ip" {
29 | 		t.Error("task from Bing was not bing")
30 | 	}
31 | 	found := false
32 | 	for _, r := range tsk.Results() {
33 | 		if strings.Contains(r.Hostname, "reddit.com") {
34 | 			found = true
35 | 		}
36 | 	}
37 | 	if !found {
38 | 		t.Error("Bing did not find the correct domain")
39 | 	}
40 | }
41 | 


--------------------------------------------------------------------------------
/bsw/bsw.go:
--------------------------------------------------------------------------------
  1 | package bsw
  2 | 
  3 | import (
  4 | 	"encoding/binary"
  5 | 	"net"
  6 | )
  7 | 
  8 | // DomainRegex is used to validate a hostname to ensure it is legitimate.
  9 | var DomainRegex = `^\.?[a-z\d]+(?:(?:[a-z\d]*)|(?:[a-z\d\-]*[a-z\d]))(?:\.[a-z\d]+(?:(?:[a-z\d]*)|(?:[a-z\d\-]*[a-z\d])))*$`
 10 | 
 11 | // Tsk is used to return the results of a task to the caller.
 12 | type Tsk struct {
 13 | 	task    string
 14 | 	results []Result
 15 | 	errs    []error
 16 | }
 17 | 
 18 | func newTsk(task string) *Tsk {
 19 | 	return &Tsk{task: task}
 20 | }
 21 | 
 22 | // Task returns the descriptive name of a task.
 23 | func (t *Tsk) Task() string {
 24 | 	return t.task
 25 | }
 26 | 
 27 | // SetTask will set the task.
 28 | func (t *Tsk) SetTask(task string) {
 29 | 	t.task = task
 30 | }
 31 | 
 32 | // AddResult adds a result to results.
 33 | func (t *Tsk) AddResult(ip, hostname string) {
 34 | 	t.results = append(t.results, Result{
 35 | 		Source:   t.task,
 36 | 		IP:       ip,
 37 | 		Hostname: hostname,
 38 | 	})
 39 | }
 40 | 
 41 | // HasResults return true if len of results is greater than 0.
 42 | func (t *Tsk) HasResults() bool {
 43 | 	return len(t.results) > 0
 44 | }
 45 | 
 46 | // Err returns the value of err.
 47 | func (t *Tsk) Err() []error {
 48 | 	return t.errs
 49 | }
 50 | 
 51 | // SetErr sets the value of err
 52 | func (t *Tsk) SetErr(err error) {
 53 | 	t.errs = append(t.errs, err)
 54 | }
 55 | 
 56 | // Results returns the results.
 57 | func (t *Tsk) Results() []Result {
 58 | 	return t.results
 59 | }
 60 | 
 61 | // Result is used to store a single IP and Hostname record.
 62 | type Result struct {
 63 | 	Source   string `json:"src"`
 64 | 	IP       string `json:"ip"`
 65 | 	Hostname string `json:"hostname"`
 66 | }
 67 | 
 68 | // Results is a slice of Result.
 69 | type Results []Result
 70 | 
 71 | func (r Results) Len() int      { return len(r) }
 72 | func (r Results) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
 73 | 
 74 | // Sorts by IPv4 address, IPv6 addresses will be show first and will be unsorted.
 75 | func (r Results) Less(i, j int) bool {
 76 | 	first := net.ParseIP(r[i].IP).To4()
 77 | 	second := net.ParseIP(r[j].IP).To4()
 78 | 	if first == nil {
 79 | 		return true
 80 | 	}
 81 | 	if second == nil {
 82 | 		return false
 83 | 	}
 84 | 	return binary.BigEndian.Uint32(first) < binary.BigEndian.Uint32(second)
 85 | }
 86 | 
 87 | func removeDuplicates(in []string) []string {
 88 | 	m := map[string]bool{}
 89 | 	out := []string{}
 90 | 	for _, i := range in {
 91 | 		if i == "" {
 92 | 			continue
 93 | 		}
 94 | 		if _, ok := m[i]; ok {
 95 | 			continue
 96 | 		}
 97 | 		m[i] = true
 98 | 		out = append(out, i)
 99 | 	}
100 | 	return out
101 | }
102 | 


--------------------------------------------------------------------------------
/bsw/censys.go:
--------------------------------------------------------------------------------
  1 | package bsw
  2 | 
  3 | import (
  4 | 	"bytes"
  5 | 	"encoding/json"
  6 | 	"errors"
  7 | 	"fmt"
  8 | 	"io/ioutil"
  9 | 	"net/http"
 10 | 	"regexp"
 11 | 	"strings"
 12 | )
 13 | 
 14 | const censysURL = "https://www.censys.io/api/v1"
 15 | 
 16 | type censysSearchResponse struct {
 17 | 	Status  string `json:"status"`
 18 | 	Results []struct {
 19 | 		IP string `json:"ip"`
 20 | 	} `json:"results"`
 21 | 	Metadata struct {
 22 | 		Pages int `json:"pages"`
 23 | 	} `json:"metadata"`
 24 | }
 25 | 
 26 | type censysViewResponse struct {
 27 | 	Num443 struct {
 28 | 		HTTPS struct {
 29 | 			TLS struct {
 30 | 				Certificate struct {
 31 | 					Parsed struct {
 32 | 						Extensions struct {
 33 | 							SubjectAltName struct {
 34 | 								DNSNames []string `json:"dns_names"`
 35 | 							} `json:"subject_alt_name"`
 36 | 						} `json:"extensions"`
 37 | 						Subject struct {
 38 | 							CommonName []string `json:"common_name"`
 39 | 						} `json:"subject"`
 40 | 					} `json:"parsed"`
 41 | 				} `json:"certificate"`
 42 | 			} `json:"tls"`
 43 | 		} `json:"https"`
 44 | 	} `json:"443"`
 45 | }
 46 | 
 47 | // CensysDomain search censys.io for a particular domain.
 48 | // After a list of IP addresses are found to be matching the domain,
 49 | // each ip in the list is looked up using the 'view' search.
 50 | // This TLS certificates for each IP, hostnames are gathers from these
 51 | // TLS certificates.
 52 | func CensysDomain(domain, auth string) *Tsk {
 53 | 	t := newTsk("censys.io Domain")
 54 | 	p := 1
 55 | 	ips, pages, err := censysSearch(domain, auth, p)
 56 | 	if err != nil {
 57 | 		t.SetErr(err)
 58 | 		return t
 59 | 	}
 60 | 	p++
 61 | 	for p <= pages {
 62 | 		i, _, err := censysSearch(domain, auth, p)
 63 | 		if err != nil {
 64 | 			t.SetErr(err)
 65 | 			return t
 66 | 		}
 67 | 		p++
 68 | 		ips = append(ips, i...)
 69 | 	}
 70 | 	for _, ip := range ips {
 71 | 		names, err := censysView(ip, auth)
 72 | 		if err != nil {
 73 | 			t.SetErr(err)
 74 | 			return t
 75 | 		}
 76 | 		for _, n := range removeDuplicates(names) {
 77 | 			if ok, err := regexp.Match(DomainRegex, []byte(n)); !ok || err != nil {
 78 | 				continue
 79 | 			}
 80 | 			if strings.Contains(n, domain) && n != domain {
 81 | 				t.AddResult(ip, n)
 82 | 			}
 83 | 		}
 84 | 	}
 85 | 	return t
 86 | }
 87 | 
 88 | // CensysIP search an ip using censys.io's ipv4 view.
 89 | // Hostnames are extracted from previously gathered TLS certificates.
 90 | func CensysIP(ip, auth string) *Tsk {
 91 | 	t := newTsk("censys.io IP search")
 92 | 	names, err := censysView(ip, auth)
 93 | 	if err != nil {
 94 | 		t.SetErr(err)
 95 | 		return t
 96 | 	}
 97 | 	for _, n := range removeDuplicates(names) {
 98 | 		if ok, err := regexp.Match(DomainRegex, []byte(n)); ok && err == nil {
 99 | 			t.AddResult(ip, n)
100 | 		}
101 | 	}
102 | 	return t
103 | }
104 | 
105 | func censysSearch(domain, auth string, page int) ([]string, int, error) {
106 | 	buf := bytes.NewBuffer([]byte(fmt.Sprintf("{\"query\": \"%s\", \"page\": %d}", domain, page)))
107 | 	req, err := http.NewRequest("POST", fmt.Sprintf("%s/search/ipv4", censysURL), buf)
108 | 	ips := []string{}
109 | 	if err != nil {
110 | 		return ips, 0, err
111 | 	}
112 | 	parts := strings.Split(auth, ":")
113 | 	if len(parts) != 2 {
114 | 		return ips, 0, errors.New("Invalid auth string for censys.io")
115 | 	}
116 | 	req.SetBasicAuth(parts[0], parts[1])
117 | 	req.Header.Set("Content-Type", "application/json")
118 | 
119 | 	client := &http.Client{}
120 | 	resp, err := client.Do(req)
121 | 	if err != nil {
122 | 		return ips, 0, err
123 | 	}
124 | 	if resp.StatusCode != 200 {
125 | 		return ips, 0, errors.New("Request returned non 200 status code")
126 | 	}
127 | 	defer resp.Body.Close()
128 | 	body, err := ioutil.ReadAll(resp.Body)
129 | 	if err != nil {
130 | 		return ips, 0, err
131 | 	}
132 | 	m := &censysSearchResponse{}
133 | 	if err = json.Unmarshal(body, &m); err != nil {
134 | 		return ips, 0, err
135 | 	}
136 | 	for _, r := range m.Results {
137 | 		ips = append(ips, r.IP)
138 | 	}
139 | 	return ips, m.Metadata.Pages, nil
140 | }
141 | 
142 | func censysView(ip, auth string) ([]string, error) {
143 | 	names := []string{}
144 | 	client := &http.Client{}
145 | 	req, err := http.NewRequest("GET", fmt.Sprintf("%s/view/ipv4/%s", censysURL, ip), nil)
146 | 	if err != nil {
147 | 		return names, err
148 | 	}
149 | 	parts := strings.Split(auth, ":")
150 | 	if len(parts) != 2 {
151 | 		return names, errors.New("Invalid auth string for censys.io")
152 | 	}
153 | 	req.SetBasicAuth(parts[0], parts[1])
154 | 	resp, err := client.Do(req)
155 | 	if err != nil {
156 | 		return names, err
157 | 	}
158 | 	if resp.StatusCode != 200 {
159 | 		return names, errors.New("Request returned non 200 status code")
160 | 	}
161 | 	defer resp.Body.Close()
162 | 	body, err := ioutil.ReadAll(resp.Body)
163 | 	if err != nil {
164 | 		return names, err
165 | 	}
166 | 	m := &censysViewResponse{}
167 | 	if err = json.Unmarshal(body, &m); err != nil {
168 | 		return names, err
169 | 	}
170 | 	names = append(names, m.Num443.HTTPS.TLS.Certificate.Parsed.Extensions.SubjectAltName.DNSNames...)
171 | 	names = append(names, m.Num443.HTTPS.TLS.Certificate.Parsed.Subject.CommonName...)
172 | 	return names, nil
173 | }
174 | 


--------------------------------------------------------------------------------
/bsw/common_crawl.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"encoding/json"
 6 | 	"fmt"
 7 | 	"net/http"
 8 | 	"net/url"
 9 | 	"strings"
10 | 	"sync"
11 | )
12 | 
13 | type commonCrawlMessage struct {
14 | 	URL string `json:"url"`
15 | }
16 | 
17 | // CommonCrawl search commoncrawl.org for subdomains of the provided domain.
18 | func CommonCrawl(domain, path, serverAddr string) *Tsk {
19 | 	t := newTsk("commoncrawl.org")
20 | 	client := &http.Client{}
21 | 	u := fmt.Sprintf("http://index.commoncrawl.org/%s?url=*.%s&output=json", path, domain)
22 | 	req, err := http.NewRequest("GET", u, nil)
23 | 	if err != nil {
24 | 		t.SetErr(err)
25 | 		return t
26 | 	}
27 | 	resp, err := client.Do(req)
28 | 	if err != nil {
29 | 		t.SetErr(err)
30 | 		return t
31 | 	}
32 | 	subSet := map[string]bool{}
33 | 	scanner := bufio.NewScanner(resp.Body)
34 | 	for scanner.Scan() {
35 | 		var msg commonCrawlMessage
36 | 		if err := json.Unmarshal(scanner.Bytes(), &msg); err != nil {
37 | 			continue
38 | 		}
39 | 		xurl, err := url.Parse(msg.URL)
40 | 		if err != nil {
41 | 			continue
42 | 		}
43 | 		subdomain := strings.SplitN(xurl.Host, domain, 2)[0]
44 | 		subdomain = strings.TrimRight(subdomain, ".")
45 | 		subSet[subdomain] = true
46 | 	}
47 | 	var wg sync.WaitGroup
48 | 	var mutex sync.Mutex
49 | 
50 | 	for k := range subSet {
51 | 		wg.Add(1)
52 | 		go func(sub string) {
53 | 			defer wg.Done()
54 | 			xtsk := Dictionary(domain, sub, nil, serverAddr)
55 | 			if len(xtsk.Err()) > 0 {
56 | 				return
57 | 			}
58 | 			for _, r := range xtsk.Results() {
59 | 				mutex.Lock()
60 | 				t.AddResult(r.IP, r.Hostname)
61 | 				mutex.Unlock()
62 | 			}
63 | 		}(k)
64 | 	}
65 | 	wg.Wait()
66 | 	return t
67 | }
68 | 


--------------------------------------------------------------------------------
/bsw/config.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"io/ioutil"
 5 | 
 6 | 	"gopkg.in/yaml.v2"
 7 | )
 8 | 
 9 | // C is used to parse a YAML config file.
10 | type C struct {
11 | 	Timeout        int64  `yaml:"timeout"`
12 | 	Concurrency    int    `yaml:"concurrency"`
13 | 	Validate       bool   `yaml:"validate"`
14 | 	IPv6           bool   `yaml:"ipv6"`
15 | 	Server         string `yaml:"server"`
16 | 	Reverse        bool   `yaml:"reverse"`
17 | 	Headers        bool   `yaml:"headers"`
18 | 	TLS            bool   `yaml:"tls"`
19 | 	AXFR           bool   `yaml:"axfr"`
20 | 	MX             bool   `yaml:"mx"`
21 | 	NS             bool   `yaml:"ns"`
22 | 	ViewDNSInfo    bool   `yaml:"viewdns_html"`
23 | 	ViewDNSInfoAPI string `yaml:"viewdns"`
24 | 	Robtex         bool   `yaml:"robtex"`
25 | 	LogonTube      bool   `yaml:"logontube"`
26 | 	SRV            bool   `yaml:"srv"`
27 | 	Bing           string `yaml:"bing"`
28 | 	BingHTML       bool   `yaml:"bing_html"`
29 | 	Shodan         string `yaml:"shodan"`
30 | 	Censys         string `yaml:"censys"`
31 | 	Yandex         string `yaml:"yandex"`
32 | 	Exfil          bool   `yaml:"exfiltrated"`
33 | 	DictFile       string `yaml:"dictionary"`
34 | 	FCRDNS         bool   `yaml:"fcrdns"`
35 | 	CommonCrawl    string `yaml:"cmn_crawl"`
36 | 	CRTSH          bool   `yaml:"crtsh"`
37 | 	VT             bool   `yaml:"vt"`
38 | }
39 | 
40 | // ReadConfig parses a yaml file and returns a pointer to a new config.
41 | func ReadConfig(location string) (*C, error) {
42 | 	c := &C{}
43 | 	data, err := ioutil.ReadFile(location)
44 | 	if err != nil {
45 | 		return c, err
46 | 	}
47 | 	err = yaml.Unmarshal(data, c)
48 | 	return c, err
49 | }
50 | 


--------------------------------------------------------------------------------
/bsw/ct.go:
--------------------------------------------------------------------------------
  1 | package bsw
  2 | 
  3 | import (
  4 | 	"crypto/x509"
  5 | 	"encoding/pem"
  6 | 	"errors"
  7 | 	"fmt"
  8 | 	"io/ioutil"
  9 | 	"net/http"
 10 | 	"sync"
 11 | 
 12 | 	"github.com/PuerkitoBio/goquery"
 13 | )
 14 | 
 15 | // GoogleCT searches https://transparencyreport.google.com
 16 | // for a list of certificates.
 17 | func GoogleCT(domain string) *Tsk {
 18 | 	t := newTsk("Google CT")
 19 | 	t.SetErr(errors.New("not implemented"))
 20 | 	return t
 21 | }
 22 | 
 23 | const crtshURL = "https://crt.sh"
 24 | 
 25 | // CRTSHCT searches https://crt.sh for a list of
 26 | // certificates
 27 | func CRTSHCT(domain, serverAddr string) *Tsk {
 28 | 	t := newTsk("CRT.SH CT")
 29 | 	resp, err := http.Get(fmt.Sprintf("%s/?q=%%.%s", crtshURL, domain))
 30 | 	if err != nil {
 31 | 		t.SetErr(err)
 32 | 		return t
 33 | 	}
 34 | 	doc, err := goquery.NewDocumentFromResponse(resp)
 35 | 	if err != nil {
 36 | 		t.SetErr(err)
 37 | 		return t
 38 | 	}
 39 | 	doc.Selection.Find("td:nth-child(1) a").Each(func(_ int, s *goquery.Selection) {
 40 | 		id := s.Text()
 41 | 		certresp, err := http.Get(fmt.Sprintf("%s/?d=%s", crtshURL, id))
 42 | 		if err != nil {
 43 | 			return
 44 | 		}
 45 | 		data, err := ioutil.ReadAll(certresp.Body)
 46 | 		if err != nil {
 47 | 			return
 48 | 		}
 49 | 		certresp.Body.Close()
 50 | 		block, _ := pem.Decode(data)
 51 | 		if block == nil {
 52 | 			return
 53 | 		}
 54 | 		cert, err := x509.ParseCertificate(block.Bytes)
 55 | 		if err != nil {
 56 | 			return
 57 | 		}
 58 | 
 59 | 		names := append(cert.DNSNames, cert.Subject.CommonName)
 60 | 
 61 | 		var wg sync.WaitGroup
 62 | 		var mutex sync.Mutex
 63 | 
 64 | 		for _, n := range names {
 65 | 			wg.Add(1)
 66 | 
 67 | 			go func(name string) {
 68 | 				defer wg.Done()
 69 | 				ips, err := LookupName(name, serverAddr)
 70 | 				if err == nil {
 71 | 					mutex.Lock()
 72 | 					for _, ip := range ips {
 73 | 						t.AddResult(ip, name)
 74 | 					}
 75 | 					mutex.Unlock()
 76 | 					return
 77 | 				}
 78 | 
 79 | 				ecount := 0
 80 | 				cfqdn := ""
 81 | 				tfqdn := name
 82 | 				cfqdns := []string{}
 83 | 
 84 | 				for {
 85 | 					cfqdn, err = LookupCname(tfqdn, serverAddr)
 86 | 					if err != nil {
 87 | 						break
 88 | 					}
 89 | 					cfqdns = append(cfqdns, cfqdn)
 90 | 					ips, err = LookupName(cfqdn, serverAddr)
 91 | 					if err != nil {
 92 | 						ecount++
 93 | 						if ecount > 10 {
 94 | 							break
 95 | 						}
 96 | 						tfqdn = cfqdn
 97 | 						continue
 98 | 					}
 99 | 					break
100 | 				}
101 | 
102 | 				mutex.Lock()
103 | 				for _, ip := range ips {
104 | 					t.AddResult(ip, name)
105 | 					for _, c := range cfqdns {
106 | 						t.AddResult(ip, c)
107 | 					}
108 | 				}
109 | 				mutex.Unlock()
110 | 			}(n)
111 | 		}
112 | 		wg.Wait()
113 | 	})
114 | 	return t
115 | }
116 | 


--------------------------------------------------------------------------------
/bsw/dictionary.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import "fmt"
 4 | import "reflect"
 5 | 
 6 | const wildcardsub = "youmustcontstuctmoreplyons."
 7 | 
 8 | // GetWildCard searches for a possible wild card host by attempting to
 9 | // get A records for wildcardsub + domain.
10 | func GetWildCards(domain, serverAddr string) []string {
11 | 	fqdn := wildcardsub + domain
12 | 	ips, _ := LookupName(fqdn, serverAddr)
13 | 	return ips
14 | }
15 | 
16 | // GetWildCard6 searches for a possible wild card host by attempting to
17 | // get AAAA records wildcardsub + domain.
18 | func GetWildCards6(domain, serverAddr string) []string {
19 | 	fqdn := wildcardsub + domain
20 | 	ips, _ := LookupName6(fqdn, serverAddr)
21 | 	return ips
22 | }
23 | 
24 | // Dictionary attempts to get an A and CNAME record for a sub domain of domain.
25 | func Dictionary(domain string, subname string, blacklist []string, serverAddr string) *Tsk {
26 | 	t := newTsk("Dictionary IPv4")
27 | 	fqdn := subname + "." + domain
28 | 	ips, err := LookupName(fqdn, serverAddr)
29 | 	if err == nil {
30 | 		if reflect.DeepEqual(ips, blacklist) {
31 | 			t.SetErr(fmt.Errorf("%v: returned IPs in blackslist", ips))
32 | 			return t
33 | 		}
34 | 		for _, ip := range ips {
35 | 			t.AddResult(ip, fqdn)
36 | 		}
37 | 		return t
38 | 	}
39 | 
40 | 	ecount := 0
41 | 	cfqdn := ""
42 | 	tfqdn := fqdn
43 | 	cfqdns := []string{}
44 | 
45 | 	for {
46 | 		cfqdn, err = LookupCname(tfqdn, serverAddr)
47 | 		if err != nil {
48 | 			t.SetErr(err)
49 | 			return t
50 | 		}
51 | 		cfqdns = append(cfqdns, cfqdn)
52 | 		ips, err = LookupName(cfqdn, serverAddr)
53 | 		if err != nil {
54 | 			ecount++
55 | 			if ecount > 10 {
56 | 				t.SetErr(err)
57 | 				return t
58 | 			}
59 | 			tfqdn = cfqdn
60 | 			continue
61 | 		}
62 | 		break
63 | 	}
64 | 
65 | 	if reflect.DeepEqual(ips, blacklist) {
66 | 		t.SetErr(fmt.Errorf("%v: returned IPs in blackslist", ips))
67 | 		return t
68 | 	}
69 | 	t.SetTask("Dictionary-CNAME")
70 | 	for _, ip := range ips {
71 | 		t.AddResult(ip, fqdn)
72 | 		for _, c := range cfqdns {
73 | 			t.AddResult(ip, c)
74 | 		}
75 | 	}
76 | 	return t
77 | }
78 | 
79 | // Dictionary6 attempts to get an AAAA record for a sub domain of a domain.
80 | func Dictionary6(domain string, subname string, blacklist []string, serverAddr string) *Tsk {
81 | 	t := newTsk("Dictionary IPv6")
82 | 	fqdn := subname + "." + domain
83 | 	ips, err := LookupName6(fqdn, serverAddr)
84 | 	if err != nil {
85 | 		t.SetErr(err)
86 | 		return t
87 | 	}
88 | 	if reflect.DeepEqual(ips, blacklist) {
89 | 		t.SetErr(fmt.Errorf("%v: returned IPs in blacklist", ips))
90 | 		return t
91 | 	}
92 | 	for _, ip := range ips {
93 | 		t.AddResult(ip, fqdn)
94 | 	}
95 | 	return t
96 | }
97 | 


--------------------------------------------------------------------------------
/bsw/dictionary_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestWildCard(t *testing.T) {
 8 | 	ip := GetWildCard("stacktitan.com", "8.8.8.8")
 9 | 	if ip == "" {
10 | 		t.Error("Failed to get A record for wildcard")
11 | 	}
12 | }
13 | 
14 | func TestDictionary(t *testing.T) {
15 | 	tsk := Dictionary("stacktitan.com", "foo", "", "8.8.8.8")
16 | 	if !tsk.HasResults() {
17 | 		t.Fatal("Dictionary did not return any results")
18 | 	}
19 | 	results := tsk.Results()
20 | 	if results[0].IP != "104.131.56.170" {
21 | 		t.Error("Dictionary returned incorrect or non-existent IP Address")
22 | 	}
23 | 	if results[0].Hostname != "foo.stacktitan.com" {
24 | 		t.Error("Dictionary returned incorrect hostname")
25 | 	}
26 | 	if results[0].Source != "Dictionary IPv4" {
27 | 		t.Error("Dictionary returned incorrect source")
28 | 	}
29 | 
30 | 	tsk = Dictionary("stacktitan.com", "autodiscover", "", "8.8.8.8")
31 | 	if !tsk.HasResults() {
32 | 		t.Fatal("Dictionary did not return any results")
33 | 	}
34 | 	results = tsk.Results()
35 | 	if results[0].IP != "184.106.31.93" {
36 | 		t.Error("Dictionary returned incorrect or non-existent IP Address")
37 | 	}
38 | 	if results[0].Hostname != "autodiscover.stacktitan.com" {
39 | 		t.Error("Dictionary returned incorrect hostname")
40 | 	}
41 | 	if results[0].Source != "Dictionary-CNAME" {
42 | 		t.Error("Dictionary returned incorrect source")
43 | 	}
44 | }
45 | 


--------------------------------------------------------------------------------
/bsw/exfiltrated.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"net/http"
 6 | 	"sync"
 7 | 
 8 | 	"github.com/PuerkitoBio/goquery"
 9 | )
10 | 
11 | // ExfiltratedHostname uses exfiltrated.com's hostname search to identify
12 | // possible hostnames for a domain. Each returned hostname is then resolved to the current IP.
13 | func ExfiltratedHostname(domain, server string) *Tsk {
14 | 	t := newTsk("exfiltrated.com")
15 | 	resp, err := http.Get(fmt.Sprintf("http://exfiltrated.com/queryhostname.php?hostname=%s", domain))
16 | 	if err != nil {
17 | 		t.SetErr(err)
18 | 		return t
19 | 	}
20 | 	doc, err := goquery.NewDocumentFromResponse(resp)
21 | 	if err != nil {
22 | 		t.SetErr(err)
23 | 		return t
24 | 	}
25 | 
26 | 	wg := sync.WaitGroup{}
27 | 	mutex := sync.Mutex{}
28 | 
29 | 	doc.Selection.Find("td:nth-child(1)").Each(func(_ int, s *goquery.Selection) {
30 | 		wg.Add(1)
31 | 		go func(hostname string) {
32 | 			defer wg.Done()
33 | 			ips, err := LookupName(hostname, server)
34 | 			if err != nil || len(ips) == 0 {
35 | 				cfqdn, err := LookupCname(hostname, server)
36 | 				if err != nil || cfqdn == "" {
37 | 					return
38 | 				}
39 | 				ips, err = LookupName(cfqdn, server)
40 | 				if err != nil || len(ips) == 0 {
41 | 					return
42 | 				}
43 | 			}
44 | 			mutex.Lock()
45 | 			for _, ip := range ips {
46 | 				t.AddResult(ip, hostname)
47 | 			}
48 | 			mutex.Unlock()
49 | 		}(s.Text())
50 | 	})
51 | 	wg.Wait()
52 | 	return t
53 | }
54 | 


--------------------------------------------------------------------------------
/bsw/generic.go:
--------------------------------------------------------------------------------
  1 | package bsw
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"sort"
  6 | 	"strings"
  7 | 
  8 | 	"github.com/miekg/dns"
  9 | )
 10 | 
 11 | // LookupMX returns all the mx servers for a domain.
 12 | func LookupMX(domain, serverAddr string) ([]string, error) {
 13 | 	servers := []string{}
 14 | 	m := &dns.Msg{}
 15 | 	m.SetQuestion(dns.Fqdn(domain), dns.TypeMX)
 16 | 	in, err := dns.Exchange(m, serverAddr+":53")
 17 | 	if err != nil {
 18 | 		return servers, err
 19 | 	}
 20 | 	if len(in.Answer) < 1 {
 21 | 		return servers, errors.New("no Answer")
 22 | 	}
 23 | 	for _, a := range in.Answer {
 24 | 		if mx, ok := a.(*dns.MX); ok {
 25 | 			parts := strings.Split(mx.Mx, " ")
 26 | 			if len(parts) < 1 {
 27 | 				continue
 28 | 			}
 29 | 			servers = append(servers, parts[len(parts)-1])
 30 | 		}
 31 | 	}
 32 | 	return servers, nil
 33 | }
 34 | 
 35 | // LookupNS returns the names servers for a domain.
 36 | func LookupNS(domain, serverAddr string) ([]string, error) {
 37 | 	servers := []string{}
 38 | 	m := &dns.Msg{}
 39 | 	m.SetQuestion(dns.Fqdn(domain), dns.TypeNS)
 40 | 	in, err := dns.Exchange(m, serverAddr+":53")
 41 | 	if err != nil {
 42 | 		return servers, err
 43 | 	}
 44 | 	if len(in.Answer) < 1 {
 45 | 		return servers, errors.New("no Answer")
 46 | 	}
 47 | 	for _, a := range in.Answer {
 48 | 		if ns, ok := a.(*dns.NS); ok {
 49 | 			servers = append(servers, ns.Ns)
 50 | 		}
 51 | 	}
 52 | 	return servers, nil
 53 | }
 54 | 
 55 | // LookupIP returns hostname from PTR record or error.
 56 | func LookupIP(ip, serverAddr string) ([]string, error) {
 57 | 	names := []string{}
 58 | 	m := &dns.Msg{}
 59 | 	ipArpa, err := dns.ReverseAddr(ip)
 60 | 	if err != nil {
 61 | 		return names, err
 62 | 	}
 63 | 	m.SetQuestion(ipArpa, dns.TypePTR)
 64 | 	in, err := dns.Exchange(m, serverAddr+":53")
 65 | 	if err != nil {
 66 | 		return names, err
 67 | 	}
 68 | 	if len(in.Answer) < 1 {
 69 | 		return names, errors.New("no Answer")
 70 | 	}
 71 | 
 72 | 	for _, a := range in.Answer {
 73 | 		if ptr, ok := a.(*dns.PTR); ok {
 74 | 			if strings.Contains(ptr.Ptr, strings.Join(strings.Split(ip, "."), "-")) {
 75 | 				continue
 76 | 			}
 77 | 			names = append(names, strings.TrimRight(ptr.Ptr, "."))
 78 | 		}
 79 | 	}
 80 | 
 81 | 	if len(names) < 1 {
 82 | 		return names, errors.New("no PTR")
 83 | 	}
 84 | 
 85 | 	return names, nil
 86 | }
 87 | 
 88 | // LookupName returns IPv4 addresses from A records or error.
 89 | func LookupName(fqdn, serverAddr string) ([]string, error) {
 90 | 	ips := []string{}
 91 | 	m := &dns.Msg{}
 92 | 	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA)
 93 | 	in, err := dns.Exchange(m, serverAddr+":53")
 94 | 	if err != nil {
 95 | 		return ips, err
 96 | 	}
 97 | 	if len(in.Answer) < 1 {
 98 | 		return ips, errors.New("no Answer")
 99 | 	}
100 | 	for _, answer := range in.Answer {
101 | 		if a, ok := answer.(*dns.A); ok {
102 | 			ip := a.A.String()
103 | 			ips = append(ips, ip)
104 | 		}
105 | 	}
106 | 
107 | 	if len(ips) == 0 {
108 | 		err = errors.New("no A record returned")
109 | 	}
110 | 	sort.Sort(sort.StringSlice(ips))
111 | 	return ips, err
112 | }
113 | 
114 | // LookupCname returns a fqdn address from CNAME record or error.
115 | func LookupCname(fqdn, serverAddr string) (string, error) {
116 | 	m := &dns.Msg{}
117 | 	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeCNAME)
118 | 	in, err := dns.Exchange(m, serverAddr+":53")
119 | 	if err != nil {
120 | 		return "", err
121 | 	}
122 | 	if len(in.Answer) < 1 {
123 | 		return "", errors.New("no Answer")
124 | 	}
125 | 	if a, ok := in.Answer[0].(*dns.CNAME); ok {
126 | 		name := a.Target
127 | 		return strings.TrimRight(name, "."), nil
128 | 	}
129 | 	return "", errors.New("no CNAME record returned")
130 | }
131 | 
132 | // LookupName6 returns IPv6 addresses from AAAA records or error.
133 | func LookupName6(fqdn, serverAddr string) ([]string, error) {
134 | 	ips := []string{}
135 | 	m := &dns.Msg{}
136 | 	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeAAAA)
137 | 	in, err := dns.Exchange(m, serverAddr+":53")
138 | 	if err != nil {
139 | 		return ips, err
140 | 	}
141 | 	if len(in.Answer) < 1 {
142 | 		return ips, errors.New("no Answer")
143 | 	}
144 | 	for _, answer := range in.Answer {
145 | 		if a, ok := answer.(*dns.AAAA); ok {
146 | 			ip := a.AAAA.String()
147 | 			ips = append(ips, ip)
148 | 		}
149 | 	}
150 | 
151 | 	if len(ips) == 0 {
152 | 		err = errors.New("no A record returned")
153 | 	}
154 | 	return ips, err
155 | }
156 | 
157 | // LookupSRV returns a hostname from SRV record or error.
158 | func LookupSRV(fqdn, dnsServer string) (string, error) {
159 | 	m := &dns.Msg{}
160 | 	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeSRV)
161 | 	in, err := dns.Exchange(m, dnsServer+":53")
162 | 	if err != nil {
163 | 		return "", err
164 | 	}
165 | 	if len(in.Answer) < 1 {
166 | 		return "", errors.New("no Answer")
167 | 	}
168 | 	if a, ok := in.Answer[0].(*dns.SRV); ok {
169 | 		return strings.TrimRight(a.Target, "."), nil
170 | 	}
171 | 	return "", errors.New("no SRV record returned")
172 | }
173 | 


--------------------------------------------------------------------------------
/bsw/header.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"crypto/tls"
 5 | 	"fmt"
 6 | 	"net"
 7 | 	"net/http"
 8 | 	"net/url"
 9 | 	"regexp"
10 | 	"time"
11 | )
12 | 
13 | // Headers uses attempts to connect to IP over http(s).
14 | // If connection is successfull return any hostnames from the possible 'Location' headers.
15 | func Headers(ip string, timeout int64) *Tsk {
16 | 	t := newTsk("Headers")
17 | 	for _, proto := range []string{"http", "https"} {
18 | 		host, err := hostnameFromHTTPLocationHeader(ip, proto, timeout)
19 | 		if err != nil {
20 | 			t.SetErr(err)
21 | 		} else if host != "" {
22 | 			t.AddResult(ip, host)
23 | 		}
24 | 	}
25 | 	return t
26 | }
27 | 
28 | // Performs http(s) request and parses possible 'Location' headers.
29 | func hostnameFromHTTPLocationHeader(ip, protocol string, timeout int64) (string, error) {
30 | 	req, err := http.NewRequest("GET", protocol+"://"+ip, nil)
31 | 	if err != nil {
32 | 		return "", err
33 | 	}
34 | 	tr := &http.Transport{
35 | 		Dial: func(network, addr string) (net.Conn, error) {
36 | 			conn, err := net.DialTimeout(network, addr, time.Duration(timeout)*time.Millisecond)
37 | 			if err != nil {
38 | 				return nil, err
39 | 			}
40 | 			conn.SetDeadline(time.Now().Add(time.Duration(timeout) * time.Millisecond))
41 | 			return conn, nil
42 | 		},
43 | 		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
44 | 	}
45 | 	res, err := tr.RoundTrip(req)
46 | 	if err != nil {
47 | 		return "", err
48 | 	}
49 | 	location := res.Header["Location"]
50 | 	if location != nil {
51 | 		u, err := url.Parse(location[0])
52 | 		if err != nil {
53 | 			return "", err
54 | 		}
55 | 		host := u.Host
56 | 		if m, _ := regexp.Match("[a-zA-Z]+", []byte(host)); m == true {
57 | 			return host, nil
58 | 		}
59 | 		return "", fmt.Errorf("%v: unsuccessful header match", ip)
60 | 	}
61 | 	return "", fmt.Errorf("%v: unsuccessful header match", ip)
62 | }
63 | 


--------------------------------------------------------------------------------
/bsw/logontube.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"net/http"
 8 | )
 9 | 
10 | type logontubeMessage struct {
11 | 	Hostip   string `json:"hostip"`
12 | 	Hostname string `json:"hostname"`
13 | 	Response struct {
14 | 		DomainCount string   `json:"domain_count"`
15 | 		Domains     []string `json:"domains"`
16 | 	} `json:"response"`
17 | }
18 | 
19 | // LogonTubeAPI sends either a domain or IP to logontube.com's API.
20 | func LogonTubeAPI(search string) *Tsk {
21 | 	t := newTsk("logontube.com API")
22 | 	resp, err := http.Get(fmt.Sprintf("http://reverseip.logontube.com/?url=%s&output=json", search))
23 | 	if err != nil {
24 | 		t.SetErr(err)
25 | 		return t
26 | 	}
27 | 	defer resp.Body.Close()
28 | 	body, err := ioutil.ReadAll(resp.Body)
29 | 	if err != nil {
30 | 		t.SetErr(err)
31 | 		return t
32 | 	}
33 | 	m := &logontubeMessage{}
34 | 	if err := json.Unmarshal(body, &m); err != nil {
35 | 		t.SetErr(err)
36 | 		return t
37 | 	}
38 | 	for _, r := range m.Response.Domains {
39 | 		t.AddResult(m.Hostip, r)
40 | 	}
41 | 	return t
42 | }
43 | 


--------------------------------------------------------------------------------
/bsw/logontube_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import "testing"
 4 | 
 5 | func TestLogontubeAPI(t *testing.T) {
 6 | 	tsk := LogonTubeAPI("stacktitan.com")
 7 | 	if err := tsk.Err(); err != nil {
 8 | 		t.Log(err)
 9 | 		t.Fatal("Error returned from logontube.com")
10 | 	}
11 | 	if !tsk.HasResults() {
12 | 		t.Fatal("No results")
13 | 	}
14 | 	results := tsk.Results()
15 | 	if results[0].Hostname != "stacktitan.com" {
16 | 		t.Error("LogonTubeAPI returned incorrect Hostname")
17 | 	}
18 | 	if results[0].Source != "logontube.com API" {
19 | 		t.Error("LogonTubeAPI returned incorrect Source")
20 | 	}
21 | 	if results[0].IP != "104.131.56.170" {
22 | 		t.Error("LogonTubeAPI returned incorrect IP")
23 | 	}
24 | }
25 | 


--------------------------------------------------------------------------------
/bsw/mx.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"strings"
 5 | )
 6 | 
 7 | // MX returns the A record for any MX records for a domain.
 8 | func MX(domain, serverAddr string) *Tsk {
 9 | 	t := newTsk("mx")
10 | 	servers, err := LookupMX(domain, serverAddr)
11 | 	if err != nil {
12 | 		t.SetErr(err)
13 | 		return t
14 | 	}
15 | 	for _, s := range servers {
16 | 		ips, err := LookupName(s, serverAddr)
17 | 		if err != nil || len(ips) == 0 {
18 | 			continue
19 | 		}
20 | 		for _, ip := range ips {
21 | 			t.AddResult(ip, strings.TrimRight(s, "."))
22 | 		}
23 | 	}
24 | 	return t
25 | }
26 | 


--------------------------------------------------------------------------------
/bsw/mx_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestMX(t *testing.T) {
 8 | 	tsk := MX("stacktitan.com", "8.8.8.8")
 9 | 	if err := tsk.Err(); err != nil {
10 | 		t.Error("error returned from MX")
11 | 		t.Log(err)
12 | 	}
13 | 	found := false
14 | 	for _, r := range tsk.Results() {
15 | 		if r.Hostname == "mx1.emailsrvr.com" {
16 | 			found = true
17 | 		}
18 | 	}
19 | 	if !found {
20 | 		t.Error("MX did not find correct mx server")
21 | 	}
22 | }
23 | 


--------------------------------------------------------------------------------
/bsw/ns.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"strings"
 5 | )
 6 | 
 7 | // NS returns the A record for any NS records for a domain.
 8 | func NS(domain, serverAddr string) *Tsk {
 9 | 	t := newTsk("ns")
10 | 	servers, err := LookupNS(domain, serverAddr)
11 | 	if err != nil {
12 | 		t.SetErr(err)
13 | 		return t
14 | 	}
15 | 	for _, s := range servers {
16 | 		ips, err := LookupName(s, serverAddr)
17 | 		if err != nil || len(ips) == 0 {
18 | 			continue
19 | 		}
20 | 		for _, ip := range ips {
21 | 			t.AddResult(ip, strings.TrimRight(s, "."))
22 | 		}
23 | 	}
24 | 	return t
25 | }
26 | 


--------------------------------------------------------------------------------
/bsw/ns_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"testing"
 5 | )
 6 | 
 7 | func TestNS(t *testing.T) {
 8 | 	tsk := NS("stacktitan.com", "8.8.8.8")
 9 | 	if err := tsk.Err(); err != nil {
10 | 		t.Error("error returned from NS")
11 | 		t.Log(err)
12 | 	}
13 | 	found := false
14 | 	for _, r := range tsk.Results() {
15 | 		if r.Hostname == "ns1.digitalocean.com" {
16 | 			found = true
17 | 		}
18 | 	}
19 | 	if !found {
20 | 		t.Error("NS did not find correct ns server")
21 | 	}
22 | }
23 | 


--------------------------------------------------------------------------------
/bsw/reverse.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | // Reverse uses LookupIP to get PTR record for an IP.
 4 | func Reverse(ip, serverAddr string) *Tsk {
 5 | 	t := newTsk("Reverse")
 6 | 	hostname, err := LookupIP(ip, serverAddr)
 7 | 	if err != nil {
 8 | 		t.SetErr(err)
 9 | 		return t
10 | 	}
11 | 	for _, host := range hostname {
12 | 		t.AddResult(ip, host)
13 | 	}
14 | 	return t
15 | }
16 | 


--------------------------------------------------------------------------------
/bsw/shodan.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"net/url"
 5 | 	"strconv"
 6 | 
 7 | 	"github.com/tomsteele/go-shodan"
 8 | )
 9 | 
10 | // Append Shodan results to BSW Task
11 | func appendlist(ips []string, t *Tsk, c *shodan.Client) error {
12 | 	d, err := c.DNSReverse(ips)
13 | 	if err != nil {
14 | 		return err
15 | 	}
16 | 	for _, i := range d {
17 | 		for _, h := range i.Hostnames {
18 | 			t.AddResult(i.IP, h)
19 | 		}
20 | 	}
21 | 	return nil
22 | }
23 | 
24 | // ShodanAPIReverse uses Shodan's '/dns/reverse' REST API to get hostnames for
25 | // a list of ips.
26 | func ShodanAPIReverse(ips []string, key string) *Tsk {
27 | 	t := newTsk("shodan API reverse")
28 | 	c := shodan.New(key)
29 | 	for i := 0; i <= len(ips)/100; i++ {
30 | 		start := i * 100
31 | 		end := start + 100
32 | 		if end >= len(ips) {
33 | 			err := appendlist(ips[start:], t, c)
34 | 			if err != nil {
35 | 				t.SetErr(err)
36 | 				return t
37 | 			}
38 | 		} else {
39 | 			err := appendlist(ips[start:end], t, c)
40 | 			if err != nil {
41 | 				t.SetErr(err)
42 | 				return t
43 | 			}
44 | 		}
45 | 	}
46 | 	return t
47 | }
48 | 
49 | // ShodanAPIHostSearch uses Shodan's '/shodan/host/search' REST API endpoint
50 | // to find hostnames and ip addresses for a domain.
51 | func ShodanAPIHostSearch(domain string, key string) *Tsk {
52 | 	t := newTsk("shodan API host search")
53 | 	if domain[0] != 46 {
54 | 		domain = "." + domain
55 | 	}
56 | 	c := shodan.New(key)
57 | 	count, err := c.HostCount("hostname:"+domain, []string{})
58 | 	if err != nil {
59 | 		t.SetErr(err)
60 | 		return t
61 | 	}
62 | 	pages := count.Total / 100
63 | 	if pages < 1 {
64 | 		pages = 1
65 | 	}
66 | 	for i := 1; i <= pages; i++ {
67 | 		opts := url.Values{}
68 | 		opts.Set("page", strconv.Itoa(i))
69 | 		hs, err := c.HostSearch("hostname:"+domain, []string{}, opts)
70 | 		if err != nil {
71 | 			t.SetErr(err)
72 | 			return t
73 | 		}
74 | 		for _, m := range hs.Matches {
75 | 			for _, h := range m.Hostnames {
76 | 				if v, ok := h.(string); ok {
77 | 					t.AddResult(m.IPStr, v)
78 | 				}
79 | 			}
80 | 		}
81 | 	}
82 | 	return t
83 | }
84 | 


--------------------------------------------------------------------------------
/bsw/shodan_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"strings"
 6 | 	"testing"
 7 | )
 8 | 
 9 | func TestShodanAPIReverse(t *testing.T) {
10 | 	key := os.Getenv("SHODAN_API_KEY")
11 | 	if key == "" {
12 | 		t.Fatal("SHODAN_API_KEY environment variable not set")
13 | 	}
14 | 	tsk := ShodanAPIReverse([]string{"104.131.56.170"}, key)
15 | 	if err := tsk.Err(); err != nil {
16 | 		t.Error("ShodanAPIReverse returned an error")
17 | 		t.Log(err)
18 | 	}
19 | 	if tsk.Task() != "shodan API reverse" {
20 | 		t.Error("task from ShodanAPIReverse not shodan API reverse")
21 | 	}
22 | 	found := false
23 | 	for _, r := range tsk.Results() {
24 | 		if strings.Contains(r.Hostname, "stacktitan.com") {
25 | 			found = true
26 | 		}
27 | 	}
28 | 	if !found {
29 | 		t.Error("ShodanAPIReverse did not find the correct domain")
30 | 	}
31 | }
32 | 
33 | func TestShodanAPIHostSerach(t *testing.T) {
34 | 	key := os.Getenv("SHODAN_API_KEY")
35 | 	if key == "" {
36 | 		t.Fatal("SHODAN_API_KEY environment variable not set")
37 | 	}
38 | 	tsk := ShodanAPIHostSearch("stacktitan.com", key)
39 | 	if err := tsk.Err(); err != nil {
40 | 		t.Error("ShodanAPIHostSearch returned an error")
41 | 		t.Log(err)
42 | 	}
43 | 	if tsk.Task() != "shodan API host search" {
44 | 		t.Error("task from ShodanAPIHostSearch not shodan API host search")
45 | 	}
46 | 	found := false
47 | 	for _, r := range tsk.Results() {
48 | 		if strings.Contains(r.Hostname, "stacktitan.com") {
49 | 			found = true
50 | 		}
51 | 	}
52 | 	if !found {
53 | 		t.Error("ShodanAPIHostSearch did not find the correct domain")
54 | 	}
55 | }
56 | 


--------------------------------------------------------------------------------
/bsw/srv.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | // SRV iterates over a list of common SRV records, returning hostname and IP results for each.
 4 | func SRV(domain, dnsServer string) *Tsk {
 5 | 	t := newTsk("SRV")
 6 | 	srvrcdarr := [...]string{"_gc._tcp.", "_kerberos._tcp.", "_kerberos._udp.", "_ldap._tcp.",
 7 | 		"_test._tcp.", "_sips._tcp.", "_sip._udp.", "_sip._tcp.", "_aix._tcp.",
 8 | 		"_aix._tcp.", "_finger._tcp.", "_ftp._tcp.", "_http._tcp.", "_nntp._tcp.",
 9 | 		"_telnet._tcp.", "_whois._tcp.", "_h323cs._tcp.", "_h323cs._udp.",
10 | 		"_h323be._tcp.", "_h323be._udp.", "_h323ls._tcp.", "_https._tcp.",
11 | 		"_h323ls._udp.", "_sipinternal._tcp.", "_sipinternaltls._tcp.",
12 | 		"_sip._tls.", "_sipfederationtls._tcp.", "_jabber._tcp.",
13 | 		"_xmpp-server._tcp.", "_xmpp-client._tcp.", "_imap.tcp.",
14 | 		"_certificates._tcp.", "_crls._tcp.", "_pgpkeys._tcp.",
15 | 		"_pgprevokations._tcp.", "_cmp._tcp.", "_svcp._tcp.", "_crl._tcp.",
16 | 		"_ocsp._tcp.", "_PKIXREP._tcp.", "_smtp._tcp.", "_hkp._tcp.",
17 | 		"_hkps._tcp.", "_jabber._udp.", "_xmpp-server._udp.", "_xmpp-client._udp.",
18 | 		"_jabber-client._tcp.", "_jabber-client._udp.", "_kpasswd._tcp.", "_kpasswd._udp.",
19 | 		"_imap._tcp."}
20 | 
21 | 	for _, value := range srvrcdarr {
22 | 		fqdn := value + domain
23 | 		srvTarget, err := LookupSRV(fqdn, dnsServer)
24 | 		if err != nil {
25 | 			continue
26 | 		}
27 | 		ips, err := LookupName(srvTarget, dnsServer)
28 | 		if err != nil {
29 | 			continue
30 | 		}
31 | 		for _, ip := range ips {
32 | 			t.AddResult(ip, srvTarget)
33 | 		}
34 | 	}
35 | 	return t
36 | }
37 | 


--------------------------------------------------------------------------------
/bsw/tls.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"crypto/tls"
 5 | 	"net"
 6 | 	"time"
 7 | )
 8 | 
 9 | // TLS attempts connection to an IP using TLS on port 443, and if successfull, will parse the server
10 | // certificate for CommonName and SubjectAlt names.
11 | func TLS(ip string, timeout int64) *Tsk {
12 | 	t := newTsk("TLS Certificate")
13 | 	tconn, err := net.DialTimeout("tcp", ip+":443", time.Duration(timeout)*time.Millisecond)
14 | 	if err != nil {
15 | 		t.SetErr(err)
16 | 		return t
17 | 	}
18 | 	if err := tconn.SetDeadline(time.Now().Add(time.Duration(timeout) * time.Millisecond)); err != nil {
19 | 		t.SetErr(err)
20 | 		return t
21 | 	}
22 | 	conn := tls.Client(tconn, &tls.Config{InsecureSkipVerify: true})
23 | 	defer conn.Close()
24 | 	if err := conn.Handshake(); err != nil {
25 | 		t.SetErr(err)
26 | 		return t
27 | 	}
28 | 	state := conn.ConnectionState()
29 | 	cert := state.PeerCertificates[0]
30 | 	t.AddResult(ip, cert.Subject.CommonName)
31 | 	for _, name := range cert.DNSNames {
32 | 		t.AddResult(ip, name)
33 | 	}
34 | 	return t
35 | }
36 | 


--------------------------------------------------------------------------------
/bsw/version.go:
--------------------------------------------------------------------------------
1 | package bsw
2 | 
3 | // VERSION is the version of blacksheepwall.
4 | const VERSION = "3.3.1"
5 | 


--------------------------------------------------------------------------------
/bsw/viewdns.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"net/http"
 8 | 
 9 | 	"github.com/PuerkitoBio/goquery"
10 | )
11 | 
12 | // Very long selector...
13 | const viewDNSSelector = "#null > tbody:nth-child(1) > tr:nth-child(3) > td:nth-child(1) > font:nth-child(1) > i:nth-child(7) > table:nth-child(4) > tbody:nth-child(1) > tr:nth-child(n+1) > td:nth-child(1)"
14 | 
15 | // ViewDNSInfo uses viewdns.info's reverseip functionality, parsing
16 | // the HTML table for hostnames.
17 | func ViewDNSInfo(ip string) *Tsk {
18 | 	t := newTsk("viewdns.info")
19 | 	resp, err := http.Get(fmt.Sprintf("http://viewdns.info/reverseip/?host=%s&t=1", ip))
20 | 	if err != nil {
21 | 		t.SetErr(err)
22 | 		return t
23 | 	}
24 | 	defer resp.Body.Close()
25 | 	doc, err := goquery.NewDocumentFromReader(resp.Body)
26 | 	if err != nil {
27 | 		t.SetErr(err)
28 | 		return t
29 | 	}
30 | 	doc.Selection.Find(viewDNSSelector).Each(func(_ int, s *goquery.Selection) {
31 | 		t.AddResult(ip, s.Text())
32 | 	})
33 | 	return t
34 | }
35 | 
36 | type viewDNSInfoMessage struct {
37 | 	Query struct {
38 | 		Tool string `json:"tool"`
39 | 		Host string `json:"host"`
40 | 	} `json:"query"`
41 | 	Response struct {
42 | 		DomainCount string `json:"domain_count"`
43 | 		Domains     []struct {
44 | 			Name         string `json:"name"`
45 | 			LastResovled string `json:"last_resolved"`
46 | 		} `json:"domains"`
47 | 	} `json:"response"`
48 | }
49 | 
50 | // ViewDNSInfoAPI uses viewdns.iinfo's API and reverseip function to find hostnames for an ip.
51 | func ViewDNSInfoAPI(ip, key string) *Tsk {
52 | 	t := newTsk("viewdns.info API")
53 | 	resp, err := http.Get(fmt.Sprintf("http://pro.viewdns.info/reverseip/?host=%s&apikey=%s&output=json", ip, key))
54 | 	if err != nil {
55 | 		t.SetErr(err)
56 | 		return t
57 | 	}
58 | 	defer resp.Body.Close()
59 | 	body, err := ioutil.ReadAll(resp.Body)
60 | 	if err != nil {
61 | 		t.SetErr(err)
62 | 		return t
63 | 	}
64 | 	m := &viewDNSInfoMessage{}
65 | 	if err := json.Unmarshal(body, &m); err != nil {
66 | 		t.SetErr(err)
67 | 		return t
68 | 	}
69 | 
70 | 	for _, domain := range m.Response.Domains {
71 | 		t.AddResult(ip, domain.Name)
72 | 	}
73 | 	return t
74 | }
75 | 


--------------------------------------------------------------------------------
/bsw/viewdns_test.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"os"
 5 | 	"strings"
 6 | 	"testing"
 7 | )
 8 | 
 9 | func TestViewDNSInfoAPI(t *testing.T) {
10 | 	key := os.Getenv("VIEWDNS_API_KEY")
11 | 	if key == "" {
12 | 		t.Fatal("Can not test ViewDNSInfoAPI with out api key in evironment variable VIEWDNS_API_KEY")
13 | 	}
14 | 	tsk := ViewDNSInfoAPI("104.131.56.170", key)
15 | 	if tsk.Task() != "viewdns.info API" {
16 | 		t.Error("task for ViewDNSInfoAPI not viewdns.info API")
17 | 	}
18 | 	if err := tsk.Err(); err != nil {
19 | 		t.Error("error returned from ViewDNSInfoAPI")
20 | 		t.Log(err)
21 | 	}
22 | 	if !tsk.HasResults() {
23 | 		t.Error("no results returned from ViewDNSInfoAPI")
24 | 	}
25 | 
26 | 	found := false
27 | 	for _, r := range tsk.Results() {
28 | 		if strings.Contains(r.Hostname, "stacktitan.com") {
29 | 			found = true
30 | 		}
31 | 	}
32 | 	if !found {
33 | 		t.Error("no results were correct for ViewDNSInfoAPI")
34 | 	}
35 | }
36 | 


--------------------------------------------------------------------------------
/bsw/vt.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"net/http"
 6 | 	"strings"
 7 | 
 8 | 	"github.com/PuerkitoBio/goquery"
 9 | )
10 | 
11 | const virusTotalURL = "https://www.virustotal.com"
12 | 
13 | // VirusTotal searches VirusTotal for sudbomains related to a domain.
14 | func VirusTotal(domain, serverAddr string) *Tsk {
15 | 	t := newTsk("VirusTotal")
16 | 
17 | 	resp, err := http.Get(fmt.Sprintf("%s/en/domain/%s/information/", virusTotalURL, domain))
18 | 	if err != nil {
19 | 		t.SetErr(err)
20 | 		return t
21 | 	}
22 | 	doc, err := goquery.NewDocumentFromResponse(resp)
23 | 	if err != nil {
24 | 		t.SetErr(err)
25 | 		return t
26 | 	}
27 | 
28 | 	doc.Selection.Find("#observed-subdomains a").Each(func(_ int, s *goquery.Selection) {
29 | 		name := strings.TrimSpace(s.Text())
30 | 
31 | 		ips, err := LookupName(name, serverAddr)
32 | 		if err == nil {
33 | 			for _, ip := range ips {
34 | 				t.AddResult(ip, name)
35 | 			}
36 | 			return
37 | 		}
38 | 
39 | 		ecount := 0
40 | 		cfqdn := ""
41 | 		tfqdn := name
42 | 		cfqdns := []string{}
43 | 
44 | 		for {
45 | 			cfqdn, err = LookupCname(tfqdn, serverAddr)
46 | 			if err != nil {
47 | 				break
48 | 			}
49 | 			cfqdns = append(cfqdns, cfqdn)
50 | 			ips, err = LookupName(cfqdn, serverAddr)
51 | 			if err != nil {
52 | 				ecount++
53 | 				if ecount > 10 {
54 | 					break
55 | 				}
56 | 				tfqdn = cfqdn
57 | 				continue
58 | 			}
59 | 			break
60 | 		}
61 | 
62 | 		for _, ip := range ips {
63 | 			t.AddResult(ip, name)
64 | 			for _, c := range cfqdns {
65 | 				t.AddResult(ip, c)
66 | 			}
67 | 		}
68 | 
69 | 	})
70 | 	return t
71 | }
72 | 


--------------------------------------------------------------------------------
/bsw/yandex.go:
--------------------------------------------------------------------------------
 1 | package bsw
 2 | 
 3 | import (
 4 | 	"errors"
 5 | 	"fmt"
 6 | 	"net/http"
 7 | 	"strings"
 8 | 
 9 | 	"github.com/PuerkitoBio/goquery"
10 | )
11 | 
12 | // YandexAPI uses Yandex XML API and the 'rhost' search operator to find
13 | // subdomains of a given domain.
14 | func YandexAPI(domain, apiURL, serverAddr string) *Tsk {
15 | 	t := newTsk("yandex API")
16 | 	xmlTemplate := "<?xml version='1.0' encoding='UTF-8'?><request><query>%s</query><sortby>rlv</sortby><maxpassages>1</maxpassages><page>0</page><groupings><groupby attr=\" \" mode=\"flat\" groups-on-page=\"100\" docs-in-group=\"1\" /></groupings></request>"
17 | 
18 | 	// Split the domain and reverse the order, then rejoin it for the query.
19 | 	parts := strings.Split(domain, ".")
20 | 	if len(parts) < 2 {
21 | 		t.SetErr(errors.New("Invalid domain"))
22 | 		return t
23 | 	}
24 | 	for i, j := 0, len(parts)-1; i < j; i, j = i+1, j-1 {
25 | 		parts[i], parts[j] = parts[j], parts[i]
26 | 	}
27 | 	query := "rhost:" + strings.Join(parts, ".") + ".*"
28 | 
29 | 	postBody := fmt.Sprintf(xmlTemplate, query)
30 | 	resp, err := http.Post(apiURL, "text/xml", strings.NewReader(postBody))
31 | 	if err != nil {
32 | 		t.SetErr(err)
33 | 		return t
34 | 	}
35 | 	defer resp.Body.Close()
36 | 	doc, err := goquery.NewDocumentFromReader(resp.Body)
37 | 	if err != nil {
38 | 		t.SetErr(err)
39 | 		return t
40 | 	}
41 | 	domainSet := make(map[string]bool)
42 | 	doc.Find("domain").Each(func(_ int, s *goquery.Selection) {
43 | 		domain := s.Text()
44 | 		if domainSet[domain] {
45 | 			return
46 | 		}
47 | 		ips, err := LookupName(domain, serverAddr)
48 | 		if err != nil || len(ips) == 0 {
49 | 			cfqdn, err := LookupCname(domain, serverAddr)
50 | 			if err != nil || cfqdn == "" {
51 | 				return
52 | 			}
53 | 			ips, err = LookupName(cfqdn, serverAddr)
54 | 			if err != nil || len(ips) == 0 {
55 | 				return
56 | 			}
57 | 		}
58 | 		for _, ip := range ips {
59 | 			t.AddResult(ip, domain)
60 | 		}
61 | 	})
62 | 	return t
63 | }
64 | 


--------------------------------------------------------------------------------
/helpers/helpers.go:
--------------------------------------------------------------------------------
 1 | package helpers
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"fmt"
 6 | 	"net"
 7 | 	"os"
 8 | 	"strings"
 9 | )
10 | 
11 | // LinesToIPList processes a list of IP addresses or networks in CIDR format.
12 | // Returning a list of all possible IP addresses.
13 | func LinesToIPList(lines []string) ([]string, error) {
14 | 	ipList := []string{}
15 | 	for _, line := range lines {
16 | 		if net.ParseIP(line) != nil {
17 | 			ipList = append(ipList, line)
18 | 		} else if ip, network, err := net.ParseCIDR(line); err == nil {
19 | 			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIP(ip) {
20 | 				ipList = append(ipList, ip.String())
21 | 			}
22 | 		} else if strings.Contains(line, "-") {
23 | 			splitIP := strings.SplitN(line, "-", 2)
24 | 			ip := net.ParseIP(splitIP[0])
25 | 			endIP := net.ParseIP(splitIP[1])
26 | 			if endIP != nil {
27 | 				if !isStartingIPLower(ip, endIP) {
28 | 					return ipList, fmt.Errorf("%s is greater than %s", ip.String(), endIP.String())
29 | 				}
30 | 				ipList = append(ipList, ip.String())
31 | 				for !ip.Equal(endIP) {
32 | 					increaseIP(ip)
33 | 					ipList = append(ipList, ip.String())
34 | 				}
35 | 			} else {
36 | 				ipOct := strings.SplitN(ip.String(), ".", 4)
37 | 				endIP := net.ParseIP(ipOct[0] + "." + ipOct[1] + "." + ipOct[2] + "." + splitIP[1])
38 | 				if endIP != nil {
39 | 					if !isStartingIPLower(ip, endIP) {
40 | 						return ipList, fmt.Errorf("%s is greater than %s", ip.String(), endIP.String())
41 | 					}
42 | 					ipList = append(ipList, ip.String())
43 | 					for !ip.Equal(endIP) {
44 | 						increaseIP(ip)
45 | 						ipList = append(ipList, ip.String())
46 | 					}
47 | 				} else {
48 | 					return ipList, fmt.Errorf("%s is not an IP Address or CIDR Network", line)
49 | 				}
50 | 			}
51 | 		} else {
52 | 			return ipList, fmt.Errorf("%s is not an IP Address or CIDR Network", line)
53 | 		}
54 | 	}
55 | 	return ipList, nil
56 | }
57 | 
58 | // increases an IP by a single address.
59 | func increaseIP(ip net.IP) {
60 | 	for j := len(ip) - 1; j >= 0; j-- {
61 | 		ip[j]++
62 | 		if ip[j] > 0 {
63 | 			break
64 | 		}
65 | 	}
66 | }
67 | 
68 | func isStartingIPLower(start, end net.IP) bool {
69 | 	if len(start) != len(end) {
70 | 		return false
71 | 	}
72 | 	for i := range start {
73 | 		if start[i] > end[i] {
74 | 			return false
75 | 		}
76 | 	}
77 | 	return true
78 | }
79 | 
80 | // ReadFileLines returns all the lines in a file.
81 | func ReadFileLines(path string) ([]string, error) {
82 | 	file, err := os.Open(path)
83 | 	if err != nil {
84 | 		return nil, err
85 | 	}
86 | 	defer file.Close()
87 | 	lines := []string{}
88 | 	scanner := bufio.NewScanner(file)
89 | 	for scanner.Scan() {
90 | 		if scanner.Text() == "" {
91 | 			continue
92 | 		}
93 | 		lines = append(lines, scanner.Text())
94 | 	}
95 | 	return lines, scanner.Err()
96 | }
97 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | /*blacksheepwall is a hostname reconnaissance tool, it is similar to other
  2 | tools, but has a focus on speed.*/
  3 | 
  4 | package main
  5 | 
  6 | import (
  7 | 	"encoding/json"
  8 | 	"flag"
  9 | 	"fmt"
 10 | 	"io/ioutil"
 11 | 	"log"
 12 | 	"os"
 13 | 	"regexp"
 14 | 	"sort"
 15 | 	"strings"
 16 | 	"text/tabwriter"
 17 | 
 18 | 	"github.com/tomsteele/blacksheepwall/bsw"
 19 | 	"github.com/tomsteele/blacksheepwall/helpers"
 20 | )
 21 | 
 22 | const usage = `
 23 |  Usage: blacksheepwall [options] <ip address or CIDR>
 24 | 
 25 |  Options:
 26 |   -h, --help            Show Usage and exit.
 27 | 
 28 |   -version              Show version and exit.
 29 | 
 30 |   -debug                Enable debugging and show errors returned from tasks.
 31 | 
 32 |   -config               Location of a YAML file containing any of the options below.
 33 |                         Hypens should be replaced with underscores (e.g. bing-html, bing_html).
 34 |                         Options that do not take an argument are booleans and should be represented
 35 |                         using true/false (e.g. bing_html: true).
 36 | 
 37 |   -timeout              Maximum timeout in seconds for SOCKET connections.  [default .5 seconds]
 38 | 
 39 |   -concurrency <int>    Max amount of concurrent tasks.  [default: 100]
 40 | 
 41 |   -server <string>      DNS server address.  [default: "8.8.8.8"]
 42 | 
 43 |   -input <string>       Line separated file of networks (CIDR) or IP Addresses.
 44 | 
 45 |   -ipv6                 Look for additional AAAA records where applicable.
 46 | 
 47 |   -domain <string>      Target domain to use for certain tasks, can be a single
 48 |                         domain or a file of line separated domains.
 49 | 
 50 |   -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
 51 |                         each result previously identified hostname.
 52 | 
 53 |   -parse <string>       Generate output by parsing JSON from a file from a previous scan.
 54 | 
 55 |   -validate             Validate hostnames using a RFC compliant regex.
 56 | 
 57 |  Passive:
 58 |   -dictionary <string>  Attempt to retrieve the CNAME and A record for
 59 |                         each subdomain in the line separated file.
 60 | 
 61 |   -ns                   Lookup the ip and hostname of any nameservers for the domain.
 62 | 
 63 |   -mx                   Lookup the ip and hostmame of any mx records for the domain.
 64 | 
 65 |   -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
 66 |                         search 'rhost:' operator to find subdomains of a
 67 |                         provided domain.
 68 | 
 69 |   -bing <string>        Provided a base64 encoded API key. Use the Bing search
 70 |                         API's 'ip:' operator to lookup hostnames for each ip, and the
 71 |                         'domain:' operator to find ips/hostnames for a domain.
 72 | 
 73 |   -bing-html            Use Bing search 'ip:' operator to lookup hostname for each ip, and the
 74 |                         'domain:' operator to find ips/hostnames for a domain. Only
 75 |                         the first page is scraped. This does not use the API.
 76 | 
 77 |   -shodan <string>      Provided a Shodan API key. Use Shodan's API '/dns/reverse' to lookup hostnames for
 78 |                         each ip, and '/shodan/host/search' to lookup ips/hostnames for a domain.
 79 |                         A single call is made for all ips.
 80 | 
 81 |   -reverse              Retrieve the PTR for each host.
 82 | 
 83 |   -viewdns-html         Lookup each host using viewdns.info's Reverse IP
 84 |                         Lookup function. Use sparingly as they will block you.
 85 | 
 86 |   -viewdns <string>     Lookup each host using viewdns.info's API and Reverse IP Lookup function.
 87 | 
 88 |   -logontube            Lookup each host and/or domain using logontube.com's API. As of this release
 89 |                         the site is down.
 90 | 
 91 |   -exfiltrated          Lookup hostnames returned from exfiltrated.com's hostname search.
 92 | 
 93 |   -censys <string>      Searches censys.io for a domain. Names are gathered from TLS certificates for each host
 94 |                         returned from this search. The provided string should be your API ID and Secret separated
 95 | 						by a colon.
 96 | 
 97 |   -crtsh                Searches crt.sh for certificates related to the provided domain.
 98 |   
 99 |   -vt                   Searches VirusTotal for subdomains for the provided domain.
100 | 
101 |   -srv                  Find DNS SRV record and retrieve associated hostname/IP info.
102 | 
103 |   -cmn-crawl <string>   Search commoncrawl.org for subdomains of a domain. The provided argument should be the index
104 |                         to be used. For example: "CC-MAIN-2017-04-index"
105 | 
106 |  Active:
107 |   -axfr                 Attempt a zone transfer on the domain.
108 | 
109 |   -headers              Perform HTTP(s) requests to each host and look for
110 |                         hostnames in a possible Location header.
111 | 
112 |   -tls                  Attempt to retrieve names from TLS certificates
113 |                         (CommonName and Subject Alternative Name).
114 | 
115 |  Output Options:
116 |   -clean                Print results as unique hostnames for each host.
117 |   -csv                  Print results in csv format.
118 |   -json                 Print results as JSON.
119 | 
120 | `
121 | 
122 | func readDataAndOutput(path string, ojson, ocsv, oclean bool) {
123 | 	data, err := ioutil.ReadFile(path)
124 | 	if err != nil {
125 | 		log.Fatal("Error reading file provided to -parse")
126 | 	}
127 | 	r := bsw.Results{}
128 | 	if err := json.Unmarshal(data, &r); err != nil {
129 | 		log.Fatal("Error parsing JSON from file provided to -parse")
130 | 	}
131 | 	output(r, ojson, ocsv, oclean)
132 | }
133 | 
134 | func output(results bsw.Results, ojson, ocsv, oclean bool) {
135 | 	switch {
136 | 	case ojson:
137 | 		j, _ := json.MarshalIndent(results, "", "    ")
138 | 		fmt.Println(string(j))
139 | 	case ocsv:
140 | 		for _, r := range results {
141 | 			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
142 | 		}
143 | 	case oclean:
144 | 		cleanSet := make(map[string][]string)
145 | 		for _, r := range results {
146 | 			cleanSet[r.IP] = append(cleanSet[r.IP], r.Hostname)
147 | 		}
148 | 		for k, v := range cleanSet {
149 | 			fmt.Printf("%s:\n", k)
150 | 			for _, h := range v {
151 | 				fmt.Printf("\t%s\n", h)
152 | 			}
153 | 		}
154 | 	default:
155 | 		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
156 | 		fmt.Fprintln(w, "IP\tHostname\tSource")
157 | 		for _, r := range results {
158 | 			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
159 | 		}
160 | 		w.Flush()
161 | 	}
162 | }
163 | 
164 | type task func() *bsw.Tsk
165 | type empty struct{}
166 | 
167 | func main() {
168 | 	// Command line options. For usage information see the
169 | 	// usage variable above.
170 | 	var (
171 | 		flVersion        = flag.Bool("version", false, "")
172 | 		flTimeout        = flag.Int64("timeout", 600, "")
173 | 		flConcurrency    = flag.Int("concurrency", 100, "")
174 | 		flDebug          = flag.Bool("debug", false, "")
175 | 		flValidate       = flag.Bool("validate", false, "")
176 | 		flConfig         = flag.String("config", "", "")
177 | 		flipv6           = flag.Bool("ipv6", false, "")
178 | 		flServerAddr     = flag.String("server", "8.8.8.8", "")
179 | 		flIPFile         = flag.String("input", "", "")
180 | 		flParse          = flag.String("parse", "", "")
181 | 		flReverse        = flag.Bool("reverse", false, "")
182 | 		flHeader         = flag.Bool("headers", false, "")
183 | 		flTLS            = flag.Bool("tls", false, "")
184 | 		flAXFR           = flag.Bool("axfr", false, "")
185 | 		flMX             = flag.Bool("mx", false, "")
186 | 		flNS             = flag.Bool("ns", false, "")
187 | 		flViewDNSInfo    = flag.Bool("viewdns-html", false, "")
188 | 		flViewDNSInfoAPI = flag.String("viewdns", "", "")
189 | 		flLogonTube      = flag.Bool("logontube", false, "")
190 | 		flCommonCrawl    = flag.String("cmn-crawl", "", "")
191 | 		flSRV            = flag.Bool("srv", false, "")
192 | 		flCRTSH          = flag.Bool("crtsh", false, "")
193 | 		flVT             = flag.Bool("vt", false, "")
194 | 		flBing           = flag.String("bing", "", "")
195 | 		flShodan         = flag.String("shodan", "", "")
196 | 		flCensys         = flag.String("censys", "", "")
197 | 		flBingHTML       = flag.Bool("bing-html", false, "")
198 | 		flYandex         = flag.String("yandex", "", "")
199 | 		flExfil          = flag.Bool("exfiltrated", false, "")
200 | 		flDomain         = flag.String("domain", "", "")
201 | 		flDictFile       = flag.String("dictionary", "", "")
202 | 		flFcrdns         = flag.Bool("fcrdns", false, "")
203 | 		flClean          = flag.Bool("clean", false, "")
204 | 		flCsv            = flag.Bool("csv", false, "")
205 | 		flJSON           = flag.Bool("json", false, "")
206 | 	)
207 | 	flag.Usage = func() { fmt.Print(usage) }
208 | 	flag.Parse()
209 | 
210 | 	if *flVersion {
211 | 		fmt.Println("blacksheepwall version ", bsw.VERSION)
212 | 		os.Exit(0)
213 | 	}
214 | 
215 | 	if *flParse != "" {
216 | 		readDataAndOutput(*flParse, *flJSON, *flCsv, *flClean)
217 | 		os.Exit(0)
218 | 	}
219 | 
220 | 	config := &bsw.C{}
221 | 	if *flConfig != "" {
222 | 		var err error
223 | 		config, err = bsw.ReadConfig(*flConfig)
224 | 		if err != nil {
225 | 			log.Fatalf("Error reading config file. Error: %s", err.Error())
226 | 		}
227 | 	}
228 | 
229 | 	// Modify timeout to Milliseconds for function calls.
230 | 	// Adjust some options for the config file.
231 | 	if config.Timeout != 0 && *flTimeout == 600 {
232 | 		*flTimeout = config.Timeout * 1000
233 | 	}
234 | 	if *flTimeout != 600 {
235 | 		*flTimeout = *flTimeout * 1000
236 | 	}
237 | 	if config.Concurrency != 0 && *flConcurrency == 100 {
238 | 		*flConcurrency = config.Concurrency
239 | 	}
240 | 	if config.Server != "" && *flServerAddr == "8.8.8.8" {
241 | 		*flServerAddr = config.Server
242 | 	}
243 | 
244 | 	// Ingest options from config.
245 | 	if !*flValidate {
246 | 		*flValidate = config.Validate
247 | 	}
248 | 	if !*flipv6 {
249 | 		*flipv6 = config.IPv6
250 | 	}
251 | 	if !*flReverse {
252 | 		*flReverse = config.Reverse
253 | 	}
254 | 	if !*flHeader {
255 | 		*flHeader = config.Headers
256 | 	}
257 | 	if !*flTLS {
258 | 		*flTLS = config.TLS
259 | 	}
260 | 	if !*flAXFR {
261 | 		*flAXFR = config.AXFR
262 | 	}
263 | 	if !*flMX {
264 | 		*flMX = config.MX
265 | 	}
266 | 	if !*flNS {
267 | 		*flNS = config.NS
268 | 	}
269 | 	if !*flCRTSH {
270 | 		*flCRTSH = config.CRTSH
271 | 	}
272 | 	if !*flVT {
273 | 		*flVT = config.VT
274 | 	}
275 | 	if !*flViewDNSInfo {
276 | 		*flViewDNSInfo = config.ViewDNSInfo
277 | 	}
278 | 	if *flViewDNSInfoAPI == "" {
279 | 		*flViewDNSInfoAPI = config.ViewDNSInfoAPI
280 | 	}
281 | 	if !*flLogonTube {
282 | 		*flLogonTube = config.LogonTube
283 | 	}
284 | 	if !*flSRV {
285 | 		*flSRV = config.SRV
286 | 	}
287 | 	if *flBing == "" {
288 | 		*flBing = config.Bing
289 | 	}
290 | 	if *flCommonCrawl == "" {
291 | 		*flCommonCrawl = config.CommonCrawl
292 | 	}
293 | 	if *flShodan == "" {
294 | 		*flShodan = config.Shodan
295 | 	}
296 | 	if *flCensys == "" {
297 | 		*flCensys = config.Censys
298 | 	}
299 | 	if !*flBingHTML {
300 | 		*flBingHTML = config.BingHTML
301 | 	}
302 | 	if *flYandex == "" {
303 | 		*flYandex = config.Yandex
304 | 	}
305 | 	if !*flExfil {
306 | 		*flExfil = config.Exfil
307 | 	}
308 | 	if *flDictFile == "" {
309 | 		*flDictFile = config.DictFile
310 | 	}
311 | 	if !*flFcrdns {
312 | 		*flFcrdns = config.FCRDNS
313 | 	}
314 | 
315 | 	// Holds all IP addresses for testing.
316 | 	ipAddrList := []string{}
317 | 
318 | 	stat, err := os.Stdin.Stat()
319 | 	var isStdIn bool
320 | 	if err == nil {
321 | 		isStdIn = (stat.Mode() & os.ModeCharDevice) == 0
322 | 	}
323 | 	// Verify that some sort of work load was given in commands.
324 | 	if !isStdIn && *flIPFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
325 | 		log.Fatal("You didn't provide any work for me to do")
326 | 	}
327 | 	if *flYandex != "" && *flDomain == "" {
328 | 		log.Fatal("Yandex API requires domain set with -domain")
329 | 	}
330 | 	if *flDictFile != "" && *flDomain == "" {
331 | 		log.Fatal("Dictionary lookup requires domain set with -domain")
332 | 	}
333 | 	if *flDomain == "" && *flSRV == true {
334 | 		log.Fatal("SRV lookup requires domain set with -domain")
335 | 	}
336 | 	if *flExfil && *flDomain == "" {
337 | 		log.Fatal("Exfiltrated requires domain set with -domain")
338 | 	}
339 | 	if *flNS && *flDomain == "" {
340 | 		log.Fatal("NS lookup requires domain set with -domain")
341 | 	}
342 | 	if *flMX && *flDomain == "" {
343 | 		log.Fatal("MX lookup requires domain set with -domain")
344 | 	}
345 | 	if *flCRTSH && *flDomain == "" {
346 | 		log.Fatal("CRTSH requires a domain set with -domain")
347 | 	}
348 | 	if *flVT && *flDomain == "" {
349 | 		log.Fatal("VirusTotal requires a domain set with -domain")
350 | 	}
351 | 	if *flAXFR && *flDomain == "" {
352 | 		log.Fatal("Zone transfer requires domain set with -domain")
353 | 	}
354 | 	if *flCommonCrawl != "" && *flDomain == "" {
355 | 		log.Fatal("Common Crawl requires domain set with -domain")
356 | 	}
357 | 	if *flDomain != "" && *flYandex == "" && *flDictFile == "" && !*flSRV && !*flLogonTube && *flShodan == "" && *flBing == "" && !*flBingHTML && !*flAXFR && !*flNS && !*flMX && !*flVT && !*flCRTSH && !*flExfil && *flCensys == "" && *flCommonCrawl == "" {
358 | 		log.Fatal("-domain provided but no methods provided that use it")
359 | 	}
360 | 
361 | 	// Build list of domains.
362 | 	domains := []string{}
363 | 	if *flDomain != "" {
364 | 		if _, err := os.Stat(*flDomain); os.IsNotExist(err) {
365 | 			domains = append(domains, *flDomain)
366 | 		} else {
367 | 			lines, err := helpers.ReadFileLines(*flDomain)
368 | 			if err != nil {
369 | 				log.Fatal("Error reading " + *flDomain + " " + err.Error())
370 | 			}
371 | 			domains = append(domains, lines...)
372 | 		}
373 | 	}
374 | 
375 | 	// Get first argument that is not an option and turn it into a list of IPs.
376 | 	if len(flag.Args()) > 0 {
377 | 		flNetwork := flag.Arg(0)
378 | 		list, err := helpers.LinesToIPList([]string{flNetwork})
379 | 		if err != nil {
380 | 			log.Fatal(err.Error())
381 | 		}
382 | 		ipAddrList = append(ipAddrList, list...)
383 | 	}
384 | 
385 | 	// If file given as -input, read lines and turn each possible IP or network into
386 | 	// a list of IPs. Appends list to ipAddrList. Will fail fatally if line in file
387 | 	// is not a valid IP or CIDR range.
388 | 	if *flIPFile != "" {
389 | 		lines, err := helpers.ReadFileLines(*flIPFile)
390 | 		if err != nil {
391 | 			log.Fatal("Error reading " + *flIPFile + " " + err.Error())
392 | 		}
393 | 		list, err := helpers.LinesToIPList(lines)
394 | 		if err != nil {
395 | 			log.Fatal(err.Error())
396 | 		}
397 | 		ipAddrList = append(ipAddrList, list...)
398 | 	}
399 | 
400 | 	// Use a map that acts like a set to store only unique results.
401 | 	resMap := make(map[bsw.Result]bool)
402 | 
403 | 	if isStdIn {
404 | 		stdin, err := ioutil.ReadAll(os.Stdin)
405 | 		if err == nil {
406 | 			pipedResults := bsw.Results{}
407 | 			if err := json.Unmarshal(stdin, &pipedResults); err != nil {
408 | 				log.Fatal("Error parsing JSON from stdin")
409 | 			}
410 | 			for _, r := range pipedResults {
411 | 				ipAddrList = append(ipAddrList, r.IP)
412 | 				resMap[r] = true
413 | 			}
414 | 		}
415 | 	}
416 | 
417 | 	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
418 | 	//          have completed as well as a single call from the gatherer.
419 | 	//
420 | 	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
421 | 	//          a function wrapper that returns a slice of results and a possible error.
422 | 	//
423 | 	// res:     When each task is called in the pool, it will send valid results to
424 | 	//          the res channel.
425 | 	tracker := make(chan empty)
426 | 	tasks := make(chan task, *flConcurrency)
427 | 	res := make(chan *bsw.Tsk, *flConcurrency)
428 | 
429 | 	// Start up *flConcurrency amount of goroutines.
430 | 	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
431 | 	for i := 0; i < *flConcurrency; i++ {
432 | 		go func() {
433 | 			for def := range tasks {
434 | 				res <- def()
435 | 			}
436 | 			tracker <- empty{}
437 | 		}()
438 | 	}
439 | 
440 | 	// Ingest incoming results.
441 | 	go func() {
442 | 		c := 0
443 | 		for t := range res {
444 | 			if !*flDebug {
445 | 				if m := c % 2; m == 0 {
446 | 					c = 3
447 | 					os.Stderr.WriteString("\rWorking \\")
448 | 				} else {
449 | 					c = 2
450 | 					os.Stderr.WriteString("\rWorking /")
451 | 				}
452 | 			}
453 | 			if err := t.Err(); err != nil && *flDebug {
454 | 				log.Printf("%v: %v", t.Task(), err)
455 | 				continue
456 | 			}
457 | 			if t.Err() != nil {
458 | 				continue
459 | 			}
460 | 			if !t.HasResults() {
461 | 				continue
462 | 			}
463 | 			result := t.Results()
464 | 			if *flDebug {
465 | 				log.Printf("%v: %v %v: task completed successfully\n", t.Task(), result[0].Hostname, result[0].IP)
466 | 			}
467 | 			if *flFcrdns {
468 | 				for _, r := range result {
469 | 					r.Hostname = strings.ToLower(r.Hostname)
470 | 					ips, err := bsw.LookupName(r.Hostname, *flServerAddr)
471 | 					if err == nil {
472 | 						for _, ip := range ips {
473 | 							resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
474 | 						}
475 | 						continue
476 | 					}
477 | 					var (
478 | 						ecount    int
479 | 						cfqdn     string
480 | 						cfqdns    []string
481 | 						isErrored bool
482 | 					)
483 | 					tfqdn := r.Hostname
484 | 					for {
485 | 						cfqdn, err = bsw.LookupCname(tfqdn, *flServerAddr)
486 | 						if err != nil {
487 | 							isErrored = true
488 | 							break
489 | 						}
490 | 						cfqdns = append(cfqdns, cfqdn)
491 | 						ips, err = bsw.LookupName(cfqdn, *flServerAddr)
492 | 						if err != nil {
493 | 							ecount++
494 | 							if ecount > 10 {
495 | 								isErrored = true
496 | 								break
497 | 							}
498 | 							tfqdn = cfqdn
499 | 							continue
500 | 						}
501 | 						break
502 | 					}
503 | 					if !isErrored {
504 | 						for _, ip := range ips {
505 | 							resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
506 | 							for _, c := range cfqdns {
507 | 								resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: c}] = true
508 | 							}
509 | 						}
510 | 					} else {
511 | 						ips, err = bsw.LookupName6(r.Hostname, *flServerAddr)
512 | 						if err == nil {
513 | 							for _, ip := range ips {
514 | 								resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
515 | 							}
516 | 						}
517 | 					}
518 | 				}
519 | 			} else {
520 | 				for _, r := range result {
521 | 					r.Hostname = strings.ToLower(r.Hostname)
522 | 					if *flValidate {
523 | 						if ok, err := regexp.Match(bsw.DomainRegex, []byte(r.Hostname)); err != nil || !ok {
524 | 							continue
525 | 						}
526 | 					}
527 | 					resMap[r] = true
528 | 				}
529 | 			}
530 | 		}
531 | 		tracker <- empty{}
532 | 	}()
533 | 
534 | 	// Bing has two possible search paths. We need to find which one is valid.
535 | 	var bingPath string
536 | 	if *flBing != "" {
537 | 		p, err := bsw.FindBingSearchPath(*flBing)
538 | 		if err != nil {
539 | 			log.Fatal(err.Error())
540 | 		}
541 | 		bingPath = p
542 | 	}
543 | 
544 | 	if *flShodan != "" && len(ipAddrList) > 0 {
545 | 		tasks <- func() *bsw.Tsk { return bsw.ShodanAPIReverse(ipAddrList, *flShodan) }
546 | 	}
547 | 
548 | 	// IP based functionality should be added to the pool here.
549 | 	for _, h := range ipAddrList {
550 | 		host := h
551 | 		if *flReverse {
552 | 			tasks <- func() *bsw.Tsk { return bsw.Reverse(host, *flServerAddr) }
553 | 		}
554 | 		if *flTLS {
555 | 			tasks <- func() *bsw.Tsk { return bsw.TLS(host, *flTimeout) }
556 | 		}
557 | 		if *flViewDNSInfo {
558 | 			tasks <- func() *bsw.Tsk { return bsw.ViewDNSInfo(host) }
559 | 		}
560 | 		if *flViewDNSInfoAPI != "" {
561 | 			tasks <- func() *bsw.Tsk { return bsw.ViewDNSInfoAPI(host, *flViewDNSInfoAPI) }
562 | 		}
563 | 		if *flLogonTube {
564 | 			tasks <- func() *bsw.Tsk { return bsw.LogonTubeAPI(host) }
565 | 		}
566 | 		if *flBingHTML {
567 | 			tasks <- func() *bsw.Tsk { return bsw.BingIP(host) }
568 | 		}
569 | 		if *flBing != "" && bingPath != "" {
570 | 			tasks <- func() *bsw.Tsk { return bsw.BingAPIIP(host, *flBing, bingPath) }
571 | 		}
572 | 		if *flHeader {
573 | 			tasks <- func() *bsw.Tsk { return bsw.Headers(host, *flTimeout) }
574 | 		}
575 | 	}
576 | 
577 | 	// Domain based functions will likely require separate blocks and should be added below.
578 | 
579 | 	// Subdomain dictionary guessing.
580 | 	for _, d := range domains {
581 | 		domain := d
582 | 		if *flDictFile != "" {
583 | 			nameList, err := helpers.ReadFileLines(*flDictFile)
584 | 			if err != nil {
585 | 				log.Fatal("Error reading " + *flDictFile + " " + err.Error())
586 | 			}
587 | 			// Get an IP for a possible wildcard domain and use it as a blacklist.
588 | 			blacklist := bsw.GetWildCards(domain, *flServerAddr)
589 | 			for _, wildcardIp := range blacklist {
590 | 				ip := wildcardIp
591 | 				tasks <- func() *bsw.Tsk {
592 | 					t := &bsw.Tsk{}
593 | 					t.SetTask("Wildcard IPv4")
594 | 					t.AddResult(ip, "*."+domain)
595 | 					return t
596 | 				}
597 | 			}
598 | 			var blacklist6 []string
599 | 			if *flipv6 {
600 | 				blacklist6 = bsw.GetWildCards6(domain, *flServerAddr)
601 | 				for _, wildcardIp := range blacklist6 {
602 | 					ip := wildcardIp
603 | 					tasks <- func() *bsw.Tsk {
604 | 						t := &bsw.Tsk{}
605 | 						t.SetTask("Wildcard IPv6")
606 | 						t.AddResult(ip, "*."+domain)
607 | 						return t
608 | 					}
609 | 				}
610 | 			}
611 | 			for _, n := range nameList {
612 | 				sub := n
613 | 				tasks <- func() *bsw.Tsk { return bsw.Dictionary(domain, sub, blacklist, *flServerAddr) }
614 | 				if *flipv6 {
615 | 					tasks <- func() *bsw.Tsk { return bsw.Dictionary6(domain, sub, blacklist6, *flServerAddr) }
616 | 				}
617 | 			}
618 | 		}
619 | 
620 | 		if *flExfil {
621 | 			tasks <- func() *bsw.Tsk { return bsw.ExfiltratedHostname(domain, *flServerAddr) }
622 | 		}
623 | 		if *flSRV {
624 | 			tasks <- func() *bsw.Tsk { return bsw.SRV(domain, *flServerAddr) }
625 | 		}
626 | 		if *flYandex != "" {
627 | 			tasks <- func() *bsw.Tsk { return bsw.YandexAPI(domain, *flYandex, *flServerAddr) }
628 | 		}
629 | 		if *flLogonTube {
630 | 			tasks <- func() *bsw.Tsk { return bsw.LogonTubeAPI(domain) }
631 | 		}
632 | 		if *flShodan != "" {
633 | 			tasks <- func() *bsw.Tsk { return bsw.ShodanAPIHostSearch(domain, *flShodan) }
634 | 		}
635 | 		if *flBing != "" && bingPath != "" {
636 | 			tasks <- func() *bsw.Tsk { return bsw.BingAPIDomain(domain, *flBing, bingPath, *flServerAddr) }
637 | 		}
638 | 		if *flBingHTML {
639 | 			tasks <- func() *bsw.Tsk { return bsw.BingDomain(domain, *flServerAddr) }
640 | 		}
641 | 		if *flAXFR {
642 | 			tasks <- func() *bsw.Tsk { return bsw.AXFR(domain, *flServerAddr) }
643 | 		}
644 | 		if *flNS {
645 | 			tasks <- func() *bsw.Tsk { return bsw.NS(domain, *flServerAddr) }
646 | 		}
647 | 		if *flMX {
648 | 			tasks <- func() *bsw.Tsk { return bsw.MX(domain, *flServerAddr) }
649 | 		}
650 | 		if *flCensys != "" {
651 | 			tasks <- func() *bsw.Tsk { return bsw.CensysDomain(domain, *flCensys) }
652 | 		}
653 | 		if *flCommonCrawl != "" {
654 | 			tasks <- func() *bsw.Tsk { return bsw.CommonCrawl(domain, *flCommonCrawl, *flServerAddr) }
655 | 		}
656 | 		if *flCRTSH {
657 | 			tasks <- func() *bsw.Tsk { return bsw.CRTSHCT(domain, *flServerAddr) }
658 | 		}
659 | 		if *flVT {
660 | 			tasks <- func() *bsw.Tsk { return bsw.VirusTotal(domain, *flServerAddr) }
661 | 		}
662 | 	}
663 | 
664 | 	// Close the tasks channel after all jobs have completed and for each
665 | 	// goroutine in the pool receive an empty message from  tracker.
666 | 	close(tasks)
667 | 	for i := 0; i < *flConcurrency; i++ {
668 | 		<-tracker
669 | 	}
670 | 	close(res)
671 | 	// Receive and empty message from the result gatherer.
672 | 	<-tracker
673 | 	os.Stderr.WriteString("\r")
674 | 	log.Println("All tasks completed")
675 | 
676 | 	// Create a results slice from the unique set in resMap. Allows for sorting.
677 | 	results := bsw.Results{}
678 | 	for k := range resMap {
679 | 		results = append(results, k)
680 | 	}
681 | 	sort.Sort(results)
682 | 	output(results, *flJSON, *flCsv, *flClean)
683 | }
684 | 


--------------------------------------------------------------------------------