├── .gitignore
├── LICENSE
├── README.md
├── bufferoverrun.go
├── certspotter.go
├── crtsh.go
├── facebook.go
├── findsubdomains.go
├── hackertarget.go
├── main.go
├── ratelimit.go
├── script
    └── release
├── threatcrowd.go
├── urlscan.go
├── virustotal.go
└── wayback.go


/.gitignore:
--------------------------------------------------------------------------------
1 | assetfinder
2 | *.sw*
3 | *.tgz
4 | *.zip
5 | *.exe
6 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Tom Hudson
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # assetfinder
 2 | 
 3 | Find domains and subdomains potentially related to a given domain.
 4 | 
 5 | 
 6 | ## Install
 7 | 
 8 | If you have Go installed and configured (i.e. with `$GOPATH/bin` in your `$PATH`):
 9 | 
10 | ```
11 | go get -u github.com/tomnomnom/assetfinder
12 | ```
13 | 
14 | Otherwise [download a release for your platform](https://github.com/tomnomnom/assetfinder/releases).
15 | To make it easier to execute you can put the binary in your `$PATH`.
16 | 
17 | ## Usage
18 | 
19 | ```
20 | assetfinder [--subs-only] <domain>
21 | ```
22 | 
23 | ## Sources
24 | 
25 | Please feel free to issue pull requests with new sources! :)
26 | 
27 | ### Implemented
28 | * crt.sh
29 | * certspotter
30 | * hackertarget
31 | * threatcrowd
32 | * wayback machine
33 | * dns.bufferover.run
34 | * facebook
35 |     * Needs `FB_APP_ID` and `FB_APP_SECRET` environment variables set (https://developers.facebook.com/)
36 |     * You need to be careful with your app's rate limits
37 | * virustotal
38 |     * Needs `VT_API_KEY` environment variable set (https://developers.virustotal.com/reference)
39 | * findsubdomains
40 |     * Needs `SPYSE_API_TOKEN` environment variable set (the free version always gives the first response page, and you also get "25 unlimited requests") — (https://spyse.com/apidocs)
41 | 
42 | ### Sources to be implemented
43 | * http://api.passivetotal.org/api/docs/
44 | * https://community.riskiq.com/ (?)
45 | * https://riddler.io/
46 | * http://www.dnsdb.org/
47 | * https://certdb.com/api-documentation
48 | 
49 | ## TODO
50 | * Flags to control which sources are used
51 |     * Likely to be all on by default and a flag to disable
52 | * Read domains from stdin
53 | 


--------------------------------------------------------------------------------
/bufferoverrun.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"strings"
 6 | )
 7 | 
 8 | func fetchBufferOverrun(domain string) ([]string, error) {
 9 | 	out := make([]string, 0)
10 | 
11 | 	fetchURL := fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain)
12 | 
13 | 	wrapper := struct {
14 | 		Records []string `json:"FDNS_A"`
15 | 	}{}
16 | 	err := fetchJSON(fetchURL, &wrapper)
17 | 	if err != nil {
18 | 		return out, err
19 | 	}
20 | 
21 | 	for _, r := range wrapper.Records {
22 | 		parts := strings.SplitN(r, ",", 2)
23 | 		if len(parts) != 2 {
24 | 			continue
25 | 		}
26 | 		out = append(out, parts[1])
27 | 	}
28 | 
29 | 	return out, nil
30 | }
31 | 


--------------------------------------------------------------------------------
/certspotter.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | )
 6 | 
 7 | func fetchCertSpotter(domain string) ([]string, error) {
 8 | 	out := make([]string, 0)
 9 | 
10 | 	fetchURL := fmt.Sprintf("https://certspotter.com/api/v0/certs?domain=%s", domain)
11 | 
12 | 	wrapper := []struct {
13 | 		DNSNames []string `json:"dns_names"`
14 | 	}{}
15 | 	err := fetchJSON(fetchURL, &wrapper)
16 | 	if err != nil {
17 | 		return out, err
18 | 	}
19 | 
20 | 	for _, w := range wrapper {
21 | 		out = append(out, w.DNSNames...)
22 | 	}
23 | 
24 | 	return out, nil
25 | }
26 | 


--------------------------------------------------------------------------------
/crtsh.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"io/ioutil"
 7 | 	"net/http"
 8 | )
 9 | 
10 | type CrtShResult struct {
11 | 	Name string `json:"name_value"`
12 | }
13 | 
14 | func fetchCrtSh(domain string) ([]string, error) {
15 | 	var results []CrtShResult
16 | 
17 | 	resp, err := http.Get(
18 | 		fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain),
19 | 	)
20 | 	if err != nil {
21 | 		return []string{}, err
22 | 	}
23 | 	defer resp.Body.Close()
24 | 
25 | 	output := make([]string, 0)
26 | 
27 | 	body, _ := ioutil.ReadAll(resp.Body)
28 | 
29 | 	if err := json.Unmarshal(body, &results); err != nil {
30 | 		return []string{}, err
31 | 	}
32 | 
33 | 	for _, res := range results {
34 | 		output = append(output, res.Name)
35 | 	}
36 | 	return output, nil
37 | }
38 | 


--------------------------------------------------------------------------------
/facebook.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"encoding/json"
  5 | 	"errors"
  6 | 	"fmt"
  7 | 	"net/http"
  8 | 	"os"
  9 | )
 10 | 
 11 | func fetchFacebook(domain string) ([]string, error) {
 12 | 
 13 | 	appId := os.Getenv("FB_APP_ID")
 14 | 	appSecret := os.Getenv("FB_APP_SECRET")
 15 | 	if appId == "" || appSecret == "" {
 16 | 		// fail silently because it's reasonable not to have
 17 | 		// the Facebook API creds
 18 | 		return []string{}, nil
 19 | 	}
 20 | 
 21 | 	accessToken, err := facebookAuth(appId, appSecret)
 22 | 	if err != nil {
 23 | 		return []string{}, err
 24 | 	}
 25 | 
 26 | 	domains, err := getFacebookCerts(accessToken, domain)
 27 | 	if err != nil {
 28 | 		return []string{}, err
 29 | 	}
 30 | 
 31 | 	return domains, nil
 32 | }
 33 | 
 34 | func getFacebookCerts(accessToken, query string) ([]string, error) {
 35 | 	out := make([]string, 0)
 36 | 	fetchURL := fmt.Sprintf(
 37 | 		"https://graph.facebook.com/certificates?fields=domains&access_token=%s&query=*.%s",
 38 | 		accessToken, query,
 39 | 	)
 40 | 
 41 | 	for {
 42 | 
 43 | 		wrapper := struct {
 44 | 			Data []struct {
 45 | 				Domains []string `json:"domains"`
 46 | 			} `json:"data"`
 47 | 
 48 | 			Paging struct {
 49 | 				Next string `json:"next"`
 50 | 			} `json:"paging"`
 51 | 		}{}
 52 | 
 53 | 		err := fetchJSON(fetchURL, &wrapper)
 54 | 		if err != nil {
 55 | 			return out, err
 56 | 		}
 57 | 
 58 | 		for _, data := range wrapper.Data {
 59 | 			for _, d := range data.Domains {
 60 | 				out = append(out, d)
 61 | 			}
 62 | 		}
 63 | 
 64 | 		fetchURL = wrapper.Paging.Next
 65 | 		if fetchURL == "" {
 66 | 			break
 67 | 		}
 68 | 	}
 69 | 	return out, nil
 70 | }
 71 | 
 72 | func facebookAuth(appId, appSecret string) (string, error) {
 73 | 	authUrl := fmt.Sprintf(
 74 | 		"https://graph.facebook.com/oauth/access_token?client_id=%s&client_secret=%s&grant_type=client_credentials",
 75 | 		appId, appSecret,
 76 | 	)
 77 | 
 78 | 	resp, err := http.Get(authUrl)
 79 | 	if err != nil {
 80 | 		return "", err
 81 | 	}
 82 | 
 83 | 	defer resp.Body.Close()
 84 | 
 85 | 	dec := json.NewDecoder(resp.Body)
 86 | 
 87 | 	auth := struct {
 88 | 		AccessToken string `json:"access_token"`
 89 | 	}{}
 90 | 	err = dec.Decode(&auth)
 91 | 	if err != nil {
 92 | 		return "", err
 93 | 	}
 94 | 
 95 | 	if auth.AccessToken == "" {
 96 | 		return "", errors.New("no access token in Facebook API response")
 97 | 	}
 98 | 
 99 | 	return auth.AccessToken, nil
100 | }
101 | 


--------------------------------------------------------------------------------
/findsubdomains.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"fmt"
  5 | 	"os"
  6 | )
  7 | 
  8 | var apiToken = os.Getenv("SPYSE_API_TOKEN")
  9 | 
 10 | func callSubdomainsAggregateEndpoint(domain string) []string {
 11 | 	out := make([]string, 0)
 12 | 
 13 | 	fetchURL := fmt.Sprintf(
 14 | 		"https://api.spyse.com/v1/subdomains-aggregate?api_token=%s&domain=%s",
 15 | 		apiToken, domain,
 16 | 	)
 17 | 
 18 | 	type Cidr struct {
 19 | 		Results []struct {
 20 | 			Data struct {
 21 | 				Domains []string `json:"domains"`
 22 | 			} `json:"data"`
 23 | 		} `json:"results"`
 24 | 	}
 25 | 
 26 | 	type Cidrs struct {
 27 | 		Cidr16, Cidr24 Cidr
 28 | 	}
 29 | 
 30 | 	wrapper := struct {
 31 | 		Cidrs Cidrs `json:"cidr"`
 32 | 	}{}
 33 | 
 34 | 	err := fetchJSON(fetchURL, &wrapper)
 35 | 
 36 | 	if err != nil {
 37 | 		// Fail silently
 38 | 		return []string{}
 39 | 	}
 40 | 
 41 | 	for _, result := range wrapper.Cidrs.Cidr16.Results {
 42 | 		for _, domain := range result.Data.Domains {
 43 | 			out = append(out, domain)
 44 | 		}
 45 | 	}
 46 | 	for _, result := range wrapper.Cidrs.Cidr24.Results {
 47 | 		for _, domain := range result.Data.Domains {
 48 | 			out = append(out, domain)
 49 | 		}
 50 | 	}
 51 | 
 52 | 	return out
 53 | }
 54 | 
 55 | /**
 56 | 
 57 |  */
 58 | func callSubdomainsEndpoint(domain string) []string {
 59 | 	out := make([]string, 0)
 60 | 
 61 | 	// Start querying the Spyse API from page 1
 62 | 	page := 1
 63 | 
 64 | 	for {
 65 | 		wrapper := struct {
 66 | 			Records []struct {
 67 | 				Domain string `json:"domain"`
 68 | 			} `json:"records"`
 69 | 		}{}
 70 | 
 71 | 		fetchURL := fmt.Sprintf(
 72 | 			"https://api.spyse.com/v1/subdomains?api_token=%s&domain=%s&page=%d",
 73 | 			apiToken, domain, page,
 74 | 		)
 75 | 
 76 | 		err := fetchJSON(fetchURL, &wrapper)
 77 | 		if err != nil {
 78 | 			// Fail silently, by returning what we got so far
 79 | 			return out
 80 | 		}
 81 | 
 82 | 		// The API does not respond with any paging, nor does it give us any idea of
 83 | 		// the total amount of domains, so we just have to keep asking for a new page until
 84 | 		// the returned `records` array is empty
 85 | 		// NOTE: The free tier always gives you the first page for free, and you get "25 unlimited search requests"
 86 | 		if len(wrapper.Records) == 0 {
 87 | 			break
 88 | 		}
 89 | 
 90 | 		for _, record := range wrapper.Records {
 91 | 			out = append(out, record.Domain)
 92 | 		}
 93 | 
 94 | 		page++
 95 | 	}
 96 | 
 97 | 	return out
 98 | }
 99 | 
100 | func fetchFindSubDomains(domain string) ([]string, error) {
101 | 
102 | 	out := make([]string, 0)
103 | 
104 | 	apiToken := os.Getenv("SPYSE_API_TOKEN")
105 | 	if apiToken == "" {
106 | 		// Must have an API token
107 | 		return []string{}, nil
108 | 	}
109 | 
110 | 	// The Subdomains-Aggregate endpoint returns some, but not all available domains
111 | 	out = append(out, callSubdomainsAggregateEndpoint(domain)...)
112 | 
113 | 	// The Subdomains endpoint only guarantees the first 30 domains, the rest needs credit at Spyze
114 | 	out = append(out, callSubdomainsEndpoint(domain)...)
115 | 
116 | 	return out, nil
117 | }
118 | 


--------------------------------------------------------------------------------
/hackertarget.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"bytes"
 6 | 	"fmt"
 7 | 	"strings"
 8 | )
 9 | 
10 | func fetchHackerTarget(domain string) ([]string, error) {
11 | 	out := make([]string, 0)
12 | 
13 | 	raw, err := httpGet(
14 | 		fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain),
15 | 	)
16 | 	if err != nil {
17 | 		return out, err
18 | 	}
19 | 
20 | 	sc := bufio.NewScanner(bytes.NewReader(raw))
21 | 	for sc.Scan() {
22 | 		parts := strings.SplitN(sc.Text(), ",", 2)
23 | 		if len(parts) != 2 {
24 | 			continue
25 | 		}
26 | 
27 | 		out = append(out, parts[0])
28 | 	}
29 | 
30 | 	return out, sc.Err()
31 | }
32 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"encoding/json"
  6 | 	"flag"
  7 | 	"fmt"
  8 | 	"io"
  9 | 	"io/ioutil"
 10 | 	"net/http"
 11 | 	"os"
 12 | 	"strings"
 13 | 	"sync"
 14 | 	"time"
 15 | )
 16 | 
 17 | func main() {
 18 | 	var subsOnly bool
 19 | 	flag.BoolVar(&subsOnly, "subs-only", false, "Only include subdomains of search domain")
 20 | 	flag.Parse()
 21 | 
 22 | 	var domains io.Reader
 23 | 	domains = os.Stdin
 24 | 
 25 | 	domain := flag.Arg(0)
 26 | 	if domain != "" {
 27 | 		domains = strings.NewReader(domain)
 28 | 	}
 29 | 
 30 | 	sources := []fetchFn{
 31 | 		fetchCertSpotter,
 32 | 		fetchHackerTarget,
 33 | 		fetchThreatCrowd,
 34 | 		fetchCrtSh,
 35 | 		fetchFacebook,
 36 | 		//fetchWayback, // A little too slow :(
 37 | 		fetchVirusTotal,
 38 | 		fetchFindSubDomains,
 39 | 		fetchUrlscan,
 40 | 		fetchBufferOverrun,
 41 | 	}
 42 | 
 43 | 	out := make(chan string)
 44 | 	var wg sync.WaitGroup
 45 | 
 46 | 	sc := bufio.NewScanner(domains)
 47 | 	rl := newRateLimiter(time.Second)
 48 | 
 49 | 	for sc.Scan() {
 50 | 		domain := strings.ToLower(sc.Text())
 51 | 
 52 | 		// call each of the source workers in a goroutine
 53 | 		for _, source := range sources {
 54 | 			wg.Add(1)
 55 | 			fn := source
 56 | 
 57 | 			go func() {
 58 | 				defer wg.Done()
 59 | 
 60 | 				rl.Block(fmt.Sprintf("%#v", fn))
 61 | 				names, err := fn(domain)
 62 | 
 63 | 				if err != nil {
 64 | 					//fmt.Fprintf(os.Stderr, "err: %s\n", err)
 65 | 					return
 66 | 				}
 67 | 
 68 | 				for _, n := range names {
 69 | 					n = cleanDomain(n)
 70 | 					if subsOnly && !strings.HasSuffix(n, domain) {
 71 | 						continue
 72 | 					}
 73 | 					out <- n
 74 | 				}
 75 | 			}()
 76 | 		}
 77 | 	}
 78 | 
 79 | 	// close the output channel when all the workers are done
 80 | 	go func() {
 81 | 		wg.Wait()
 82 | 		close(out)
 83 | 	}()
 84 | 
 85 | 	// track what we've already printed to avoid duplicates
 86 | 	printed := make(map[string]bool)
 87 | 
 88 | 	for n := range out {
 89 | 		if _, ok := printed[n]; ok {
 90 | 			continue
 91 | 		}
 92 | 		printed[n] = true
 93 | 
 94 | 		fmt.Println(n)
 95 | 	}
 96 | }
 97 | 
 98 | type fetchFn func(string) ([]string, error)
 99 | 
100 | func httpGet(url string) ([]byte, error) {
101 | 	res, err := http.Get(url)
102 | 	if err != nil {
103 | 		return []byte{}, err
104 | 	}
105 | 
106 | 	raw, err := ioutil.ReadAll(res.Body)
107 | 
108 | 	res.Body.Close()
109 | 	if err != nil {
110 | 		return []byte{}, err
111 | 	}
112 | 
113 | 	return raw, nil
114 | }
115 | 
116 | func cleanDomain(d string) string {
117 | 	d = strings.ToLower(d)
118 | 
119 | 	// no idea what this is, but we can't clean it ¯\_(ツ)_/¯
120 | 	if len(d) < 2 {
121 | 		return d
122 | 	}
123 | 
124 | 	if d[0] == '*' || d[0] == '%' {
125 | 		d = d[1:]
126 | 	}
127 | 
128 | 	if d[0] == '.' {
129 | 		d = d[1:]
130 | 	}
131 | 
132 | 	return d
133 | 
134 | }
135 | 
136 | func fetchJSON(url string, wrapper interface{}) error {
137 | 	resp, err := http.Get(url)
138 | 	if err != nil {
139 | 		return err
140 | 	}
141 | 	defer resp.Body.Close()
142 | 	dec := json.NewDecoder(resp.Body)
143 | 
144 | 	return dec.Decode(wrapper)
145 | }
146 | 


--------------------------------------------------------------------------------
/ratelimit.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"sync"
 5 | 	"time"
 6 | )
 7 | 
 8 | // a rateLimiter allows you to delay operations
 9 | // on a per-key basis. I.e. only one operation for
10 | // a given key can be done within the delay time
11 | type rateLimiter struct {
12 | 	sync.Mutex
13 | 	delay time.Duration
14 | 	ops   map[string]time.Time
15 | }
16 | 
17 | // newRateLimiter returns a new *rateLimiter for the
18 | // provided delay
19 | func newRateLimiter(delay time.Duration) *rateLimiter {
20 | 	return &rateLimiter{
21 | 		delay: delay,
22 | 		ops:   make(map[string]time.Time),
23 | 	}
24 | }
25 | 
26 | // Block blocks until an operation for key is
27 | // allowed to proceed
28 | func (r *rateLimiter) Block(key string) {
29 | 	now := time.Now()
30 | 
31 | 	r.Lock()
32 | 
33 | 	// if there's nothing in the map we can
34 | 	// return straight away
35 | 	if _, ok := r.ops[key]; !ok {
36 | 		r.ops[key] = now
37 | 		r.Unlock()
38 | 		return
39 | 	}
40 | 
41 | 	// if time is up we can return straight away
42 | 	t := r.ops[key]
43 | 	deadline := t.Add(r.delay)
44 | 	if now.After(deadline) {
45 | 		r.ops[key] = now
46 | 		r.Unlock()
47 | 		return
48 | 	}
49 | 
50 | 	remaining := deadline.Sub(now)
51 | 
52 | 	// Set the time of the operation
53 | 	r.ops[key] = now.Add(remaining)
54 | 	r.Unlock()
55 | 
56 | 	// Block for the remaining time
57 | 	<-time.After(remaining)
58 | }
59 | 


--------------------------------------------------------------------------------
/script/release:
--------------------------------------------------------------------------------
 1 | #!/bin/bash
 2 | PROJDIR=$(cd `dirname $0`/.. && pwd)
 3 | 
 4 | VERSION="${1}"
 5 | TAG="v${VERSION}"
 6 | USER="tomnomnom"
 7 | REPO="assetfinder"
 8 | BINARY="${REPO}"
 9 | 
10 | if [[ -z "${VERSION}" ]]; then
11 |     echo "Usage: ${0} <version>"
12 |     exit 1
13 | fi
14 | 
15 | if [[ -z "${GITHUB_TOKEN}" ]]; then
16 |     echo "You forgot to set your GITHUB_TOKEN"
17 |     exit 2
18 | fi
19 | 
20 | cd ${PROJDIR}
21 | 
22 | # Run the tests
23 | go test
24 | if [ $? -ne 0 ]; then
25 |     echo "Tests failed. Aborting."
26 |     exit 3
27 | fi
28 | 
29 | # Check if tag exists
30 | git fetch --tags
31 | git tag | grep "^${TAG}$"
32 | 
33 | if [ $? -ne 0 ]; then
34 |     github-release release \
35 |         --user ${USER} \
36 |         --repo ${REPO} \
37 |         --tag ${TAG} \
38 |         --name "${REPO} ${TAG}" \
39 |         --description "${TAG}" \
40 |         --pre-release
41 | fi
42 | 
43 | 
44 | for ARCH in "amd64" "386"; do
45 |     for OS in "darwin" "linux" "windows" "freebsd"; do
46 | 
47 |         BINFILE="${BINARY}"
48 | 
49 |         if [[ "${OS}" == "windows" ]]; then
50 |             BINFILE="${BINFILE}.exe"
51 |         fi
52 | 
53 |         rm -f ${BINFILE}
54 | 
55 |         GOOS=${OS} GOARCH=${ARCH} go build -ldflags "-X main.gronVersion=${VERSION}" github.com/${USER}/${REPO}
56 | 
57 |         if [[ "${OS}" == "windows" ]]; then
58 |             ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.zip"
59 |             zip ${ARCHIVE} ${BINFILE}
60 |         else
61 |             ARCHIVE="${BINARY}-${OS}-${ARCH}-${VERSION}.tgz"
62 |             tar --create --gzip --file=${ARCHIVE} ${BINFILE}
63 |         fi
64 | 
65 |         echo "Uploading ${ARCHIVE}..."
66 |         github-release upload \
67 |             --user ${USER} \
68 |             --repo ${REPO} \
69 |             --tag ${TAG} \
70 |             --name "${ARCHIVE}" \
71 |             --file ${PROJDIR}/${ARCHIVE}
72 |     done
73 | done
74 | 


--------------------------------------------------------------------------------
/threatcrowd.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | )
 6 | 
 7 | func fetchThreatCrowd(domain string) ([]string, error) {
 8 | 	out := make([]string, 0)
 9 | 
10 | 	fetchURL := fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)
11 | 
12 | 	wrapper := struct {
13 | 		Subdomains []string `json:"subdomains"`
14 | 	}{}
15 | 	err := fetchJSON(fetchURL, &wrapper)
16 | 	if err != nil {
17 | 		return out, err
18 | 	}
19 | 
20 | 	out = append(out, wrapper.Subdomains...)
21 | 
22 | 	return out, nil
23 | }
24 | 


--------------------------------------------------------------------------------
/urlscan.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"encoding/json"
 5 | 	"fmt"
 6 | 	"net/http"
 7 | 	"net/url"
 8 | )
 9 | 
10 | func fetchUrlscan(domain string) ([]string, error) {
11 | 	resp, err := http.Get(
12 | 		fmt.Sprintf("https://urlscan.io/api/v1/search/?q=domain:%s", domain),
13 | 	)
14 | 	if err != nil {
15 | 		return []string{}, err
16 | 	}
17 | 	defer resp.Body.Close()
18 | 
19 | 	output := make([]string, 0)
20 | 
21 | 	dec := json.NewDecoder(resp.Body)
22 | 
23 | 	wrapper := struct {
24 | 		Results []struct {
25 | 			Task struct {
26 | 				URL string `json:"url"`
27 | 			} `json:"task"`
28 | 
29 | 			Page struct {
30 | 				URL string `json:"url"`
31 | 			} `json:"page"`
32 | 		} `json:"results"`
33 | 	}{}
34 | 
35 | 	err = dec.Decode(&wrapper)
36 | 	if err != nil {
37 | 		return []string{}, err
38 | 	}
39 | 
40 | 	for _, r := range wrapper.Results {
41 | 		u, err := url.Parse(r.Task.URL)
42 | 		if err != nil {
43 | 			continue
44 | 		}
45 | 
46 | 		output = append(output, u.Hostname())
47 | 	}
48 | 
49 | 	for _, r := range wrapper.Results {
50 | 		u, err := url.Parse(r.Page.URL)
51 | 		if err != nil {
52 | 			continue
53 | 		}
54 | 
55 | 		output = append(output, u.Hostname())
56 | 	}
57 | 
58 | 	return output, nil
59 | }
60 | 


--------------------------------------------------------------------------------
/virustotal.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"os"
 6 | )
 7 | 
 8 | func fetchVirusTotal(domain string) ([]string, error) {
 9 | 
10 | 	apiKey := os.Getenv("VT_API_KEY")
11 | 	if apiKey == "" {
12 | 		// swallow not having an API key, just
13 | 		// don't fetch
14 | 		return []string{}, nil
15 | 	}
16 | 
17 | 	fetchURL := fmt.Sprintf(
18 | 		"https://www.virustotal.com/vtapi/v2/domain/report?domain=%s&apikey=%s",
19 | 		domain, apiKey,
20 | 	)
21 | 
22 | 	wrapper := struct {
23 | 		Subdomains []string `json:"subdomains"`
24 | 	}{}
25 | 
26 | 	err := fetchJSON(fetchURL, &wrapper)
27 | 	return wrapper.Subdomains, err
28 | }
29 | 


--------------------------------------------------------------------------------
/wayback.go:
--------------------------------------------------------------------------------
 1 | package main
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"net/url"
 6 | )
 7 | 
 8 | func fetchWayback(domain string) ([]string, error) {
 9 | 
10 | 	fetchURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=json&collapse=urlkey", domain)
11 | 
12 | 	var wrapper [][]string
13 | 	err := fetchJSON(fetchURL, &wrapper)
14 | 	if err != nil {
15 | 		return []string{}, err
16 | 	}
17 | 
18 | 	out := make([]string, 0)
19 | 
20 | 	skip := true
21 | 	for _, item := range wrapper {
22 | 		// The first item is always just the string "original",
23 | 		// so we should skip the first item
24 | 		if skip {
25 | 			skip = false
26 | 			continue
27 | 		}
28 | 
29 | 		if len(item) < 3 {
30 | 			continue
31 | 		}
32 | 
33 | 		u, err := url.Parse(item[2])
34 | 		if err != nil {
35 | 			continue
36 | 		}
37 | 
38 | 		out = append(out, u.Hostname())
39 | 	}
40 | 
41 | 	return out, nil
42 | }
43 | 


--------------------------------------------------------------------------------