├── README.md ├── lib ├── arin │ └── arin.go ├── httputils │ └── httputils.go └── spf │ └── spf.go └── main.go /README.md: -------------------------------------------------------------------------------- 1 | # SPFWALKER 2 | 3 | Tool to recursively walk SPF records given a target domain. This will walk all related domains in includes and query relevant WHOIS information from all the "includees". Can also save the information into a JSON file. 4 | 5 | ## Installation 6 | 7 | `go get -u github.com/djhohnstein/spfwalker` 8 | 9 | ## Usage 10 | 11 | ``` 12 | homedir\go> spfwalker.exe -h 13 | Usage of spfwalker.exe: 14 | -domain string 15 | Domain to walk and retrieve SPF records for. 16 | -outfile string 17 | File to save JSON results to. 18 | ``` 19 | 20 | ## Example 21 | 22 | ``` 23 | .\spfwalker.exe -domain tesla.com 24 | 25 | Domain: tesla.com (5 includes, 14 ip4) 26 | [Include] spf.protection.outlook.com 27 | [Include] mail.zendesk.com 28 | [Include] _spf.salesforce.com 29 | [Include] _spfsn.teslamotors.com 30 | [Include] _spf.qualtrics.com 31 | [IP4] 149.96.231.186 32 | [IP4] 149.96.247.186 33 | [IP4] 148.163.155.1 34 | [IP4] 148.163.151.57 35 | [IP4] 209.11.133.122 36 | [IP4] 13.111.88.1 37 | [IP4] 13.111.88.2 38 | [IP4] 13.111.88.52 39 | [IP4] 13.111.88.53 40 | [IP4] 13.111.62.118 41 | [IP4] 94.103.153.130 42 | [IP4] 82.199.68.176/28 43 | [IP4] 95.172.66.176/28 44 | [IP4] 51.163.163.128/25 45 | 46 | Whois Information 47 | [CN] *.service-now.com 48 | [IPv4] 149.96.247.186 49 | [Name] SERVICENOW, INC. 50 | [HostName] vip-149-96-247-186.cust.service-now.com. 51 | [SearchTerm] 149.96.247.186 52 | 53 | Whois Information 54 | [CN] *.service-now.com 55 | [IPv4] 149.96.231.186 56 | [Name] SERVICENOW, INC. 57 | [HostName] vip-149-96-231-186.cust.service-now.com. 58 | [SearchTerm] 149.96.231.186 59 | 60 | Whois Information 61 | [CN] *.calero.com 62 | [IPv4] 94.103.153.130 63 | [Name] RIPE Network Coordination Centre 64 | [HostName] mail.ab-groep.nl. 65 | [SearchTerm] 94.103.153.130 66 | 67 | ... snip ... 68 | ``` -------------------------------------------------------------------------------- /lib/arin/arin.go: -------------------------------------------------------------------------------- 1 | package arin 2 | 3 | import ( 4 | "context" 5 | "encoding/csv" 6 | "errors" 7 | "fmt" 8 | "io/ioutil" 9 | "log" 10 | "net" 11 | "net/http" 12 | "os" 13 | "spfwalker/lib/httputils" 14 | "strings" 15 | "time" 16 | 17 | "github.com/buger/jsonparser" // Progress bar 18 | "golang.org/x/sync/semaphore" 19 | ) 20 | 21 | // Structure for the information valuable from ARIN 22 | type WhoisResult struct { 23 | SearchTerm string `json:"SearchTerm"` 24 | IPv4 string `json:"IPv4"` 25 | Name string `json:"Name"` 26 | HostName string `json:"HostName"` 27 | CN string `json:"CN"` 28 | } 29 | 30 | // Structure to ensure we throttle everything A-OKAY 31 | type WhoisLookupObject struct { 32 | host string 33 | lock *semaphore.Weighted 34 | } 35 | 36 | // Retrieve the DNS name of an IP address via reverse lookup. 37 | func ReverseLookup(ip string) (string, error) { 38 | addr, err := net.LookupAddr(ip) 39 | if err != nil { 40 | return "", err 41 | } 42 | return addr[0], nil 43 | } 44 | 45 | // Run a query on the host/ip and return a WhoIsResult pointer 46 | func (obj *WhoisLookupObject) Query() (*WhoisResult, error) { 47 | // Throttle the request 48 | obj.lock.Acquire(context.TODO(), 1) 49 | defer obj.lock.Release(1) 50 | _, err := net.DialTimeout("tcp", "whois.arin.net:80", 10*time.Second) 51 | if err != nil { 52 | return nil, err 53 | } 54 | url := fmt.Sprintf("http://whois.arin.net/rest/ip/%s.json", obj.host) 55 | resp, err := http.Get(url) 56 | if err != nil { 57 | return nil, err 58 | } 59 | defer resp.Body.Close() 60 | if resp.StatusCode != 200 { 61 | // Could not open 62 | return nil, errors.New("Invalid IP address or hostname given.") 63 | } 64 | body, err := ioutil.ReadAll(resp.Body) 65 | if err != nil { 66 | return nil, err 67 | } 68 | // fmt.Println(string(body)) 69 | value, _, _, err := jsonparser.Get(body, "net", "orgRef", "@name") 70 | if err != nil { 71 | value = nil 72 | } 73 | name := string(value) 74 | hostname, err := ReverseLookup(obj.host) 75 | if err != nil { 76 | hostname = "" 77 | } 78 | cert := httputils.GetSSLCertificate(obj.host) 79 | cn := "" 80 | if cert != nil { 81 | cn = cert.Subject.CommonName 82 | } 83 | result := &WhoisResult{ 84 | IPv4: obj.host, 85 | Name: name, 86 | HostName: hostname, 87 | CN: cn, 88 | } 89 | return result, nil 90 | } 91 | 92 | // Perform en-masse whoislookups 93 | func WhoisLookup(ips []string) ([]*WhoisResult, error) { 94 | // NO work to do 95 | if len(ips) == 0 { 96 | return nil, nil 97 | } 98 | // bar := pb.StartNew(len(ips)) 99 | 100 | ch := make(chan *WhoisResult) 101 | for i := 0; i < len(ips); i++ { 102 | go func( /*bar *pb.ProgressBar, */ ip string, ch chan *WhoisResult) { 103 | // defer bar.Increment() 104 | lookup := &WhoisLookupObject{ 105 | host: ip, 106 | lock: semaphore.NewWeighted(100), 107 | } 108 | result, err := lookup.Query() 109 | if err != nil { 110 | // fmt.Println("err!") 111 | log.Fatalln(err) 112 | return 113 | } 114 | result.SearchTerm = ip 115 | // fmt.Println("Sending result to channel!") 116 | ch <- result 117 | // fmt.Println("done!") 118 | }( /*bar, */ ips[i], ch) 119 | } 120 | 121 | var results []*WhoisResult 122 | for i := 0; i < len(ips); i++ { 123 | // fmt.Println("Fetching from channel...") 124 | r := <-ch 125 | // fmt.Println("Done!") 126 | // fmt.Println("Got one from channel!") 127 | // fmt.Println(r) 128 | results = append(results, r) 129 | } 130 | return results, nil 131 | 132 | } 133 | 134 | // Convert the struct into a string slice 135 | func (obj *WhoisResult) ToStringSlice() []string { 136 | result := []string{obj.IPv4, obj.Name, obj.HostName, obj.CN} 137 | return result 138 | } 139 | 140 | // Write results to csv file 141 | func WriteWhoisResultCSV(filename string, data []*WhoisResult) error { 142 | if !strings.HasSuffix(filename, ".csv") { 143 | filename += ".csv" 144 | } 145 | file, err := os.Create(filename) 146 | if err != nil { 147 | return err 148 | } 149 | defer file.Close() 150 | 151 | writer := csv.NewWriter(file) 152 | defer writer.Flush() 153 | // Write the headers 154 | writer.Write([]string{"IPv4", "Name", "HostName", "CN"}) 155 | for _, result := range data { 156 | if err := writer.Write(result.ToStringSlice()); err != nil { 157 | log.Fatalln("error writing to csv:", err) 158 | } 159 | } 160 | return nil 161 | } 162 | 163 | func Test() { 164 | fmt.Println("Hello from ARIN!") 165 | } 166 | -------------------------------------------------------------------------------- /lib/httputils/httputils.go: -------------------------------------------------------------------------------- 1 | package httputils 2 | 3 | import ( 4 | "crypto/tls" 5 | "crypto/x509" 6 | "fmt" 7 | "net" 8 | "strings" 9 | "time" 10 | 11 | "golang.org/x/net/html" 12 | ) 13 | 14 | // Retrieve the x509 cert from the remote server; times out otherwise. 15 | func GetSSLCertificate(host string) *x509.Certificate { 16 | conn, err := net.DialTimeout("tcp", fmt.Sprintf("%s:443", host), 5*time.Second) 17 | if err != nil { 18 | return nil 19 | } 20 | config := &tls.Config{ 21 | InsecureSkipVerify: true, 22 | } 23 | client := tls.Client(conn, config) 24 | err = client.Handshake() 25 | if err != nil { 26 | return nil 27 | } 28 | certs := client.ConnectionState().PeerCertificates 29 | if len(certs) == 0 { 30 | return nil 31 | } 32 | 33 | return certs[0] 34 | } 35 | 36 | // Given an html node, recursively find all tags and stuff into the results 37 | func GetMatchingNodes(n *html.Node, tag string, results *[]*html.Node) { 38 | if n.Type == html.ElementNode && n.Data == tag { 39 | // fmt.Println("Found a table!") 40 | *results = append(*results, n) 41 | } 42 | for c := n.FirstChild; c != nil; c = c.NextSibling { 43 | GetMatchingNodes(c, tag, results) 44 | } 45 | } 46 | 47 | // Retrieve any text within a node, otherwise return nothing 48 | func GetNodeText(n *html.Node) string { 49 | for c := n.FirstChild; c != nil; c = c.NextSibling { 50 | // fmt.Println("Type:", c.Type) 51 | // fmt.Println("Data", c.Data) 52 | if c.Type == html.TextNode && strings.TrimSpace(c.Data) != "" { 53 | return strings.TrimSpace(c.Data) 54 | } 55 | } 56 | return "" 57 | } 58 | 59 | func GetAttrValue(n *html.Node, target string) string { 60 | for _, r := range n.Attr { 61 | if r.Key == target { 62 | return r.Val 63 | } 64 | } 65 | return "" 66 | } 67 | 68 | // func FilterSetCookieForCookie(resp *http.Response, target string) { 69 | // // Filters the cookies of the request for a specific key (target) 70 | // cookies := resp.Cookies() 71 | // for _, cookie := range cookies { 72 | // // split the strings 73 | // parts := strings.Split(cookie, ",") 74 | // } 75 | 76 | // } 77 | -------------------------------------------------------------------------------- /lib/spf/spf.go: -------------------------------------------------------------------------------- 1 | package spf 2 | 3 | import ( 4 | "log" 5 | "net" 6 | "spfwalker/lib/arin" 7 | "strings" 8 | "sync" 9 | 10 | "golang.org/x/sync/semaphore" 11 | ) 12 | 13 | type SPFRecord struct { 14 | Domain string `json:"Domain"` // Root domain you looked up 15 | IPv4 []string `json:"IPv4"` // IPv4's returned that are allowed to send. 16 | Include []string `json:"Include"` // List of domains that are allowed to send. 17 | WhoisRecords []*arin.WhoisResult `json:"WhoisRecords"` // Once SPF records are resolved, we do whois lookups 18 | } 19 | 20 | type SPFWorker struct { 21 | lock *semaphore.Weighted 22 | mtx sync.Mutex 23 | Results []*SPFRecord 24 | } 25 | 26 | // controls threads 27 | func NewSPFWorker() SPFWorker { 28 | return SPFWorker{ 29 | lock: semaphore.NewWeighted(20), 30 | mtx: sync.Mutex{}, 31 | } 32 | } 33 | 34 | func GetSPFRecord(domain string) *SPFRecord { 35 | records, err := net.LookupTXT(domain) 36 | if err != nil { 37 | return nil 38 | } 39 | for _, r := range records { 40 | if strings.HasPrefix(r, "v=spf") { 41 | spfRec := new(SPFRecord) 42 | spfRec.Domain = domain 43 | parseSPFString(r, spfRec) 44 | 45 | return spfRec 46 | } 47 | } 48 | return nil 49 | } 50 | 51 | func parseSPFString(spfString string, spfRec *SPFRecord) { 52 | // v=spf1 include:dispatch-us.ppe-hosted.com include:spf.protection.outlook.com -all 53 | parts := strings.Split(spfString, " ") 54 | for _, part := range parts { 55 | spfInfoParts := strings.SplitN(part, ":", 2) 56 | if len(spfInfoParts) != 2 { 57 | continue 58 | } 59 | switch spfInfoParts[0] { 60 | case "include": 61 | spfRec.Include = append(spfRec.Include, spfInfoParts[1]) 62 | case "ip4": 63 | spfRec.IPv4 = append(spfRec.IPv4, spfInfoParts[1]) 64 | default: 65 | continue 66 | } 67 | } 68 | } 69 | 70 | func (worker *SPFWorker) WalkAllSPFRecords(domain string) { 71 | worker.WalkSPFRecord(domain) 72 | worker.ResolveWhoisInfo() 73 | } 74 | 75 | func (worker *SPFWorker) WalkSPFRecord(domain string) { 76 | rec := GetSPFRecord(domain) 77 | if rec == nil { 78 | return 79 | } 80 | worker.mtx.Lock() 81 | worker.Results = append(worker.Results, rec) 82 | worker.mtx.Unlock() 83 | // records = append(records, rec) 84 | if len(rec.Include) > 0 { 85 | ch := make(chan int) 86 | for i := 0; i < len(rec.Include); i++ { 87 | go func(domain *string, ch chan int) { 88 | worker.WalkSPFRecord(*domain) 89 | ch <- 0 90 | }(&rec.Include[i], ch) 91 | // records = append(records, res...) 92 | } 93 | for i := 0; i < len(rec.Include); i++ { 94 | <-ch 95 | } 96 | } 97 | } 98 | 99 | func (worker *SPFWorker) ResolveWhoisInfo() error { 100 | 101 | // ip: domainName 102 | domainIpMap := make(map[string]string) 103 | // ch := make(chan int) 104 | mtx := sync.Mutex{} 105 | for _, record := range worker.Results { 106 | var iplist []string 107 | wg := sync.WaitGroup{} 108 | for _, d := range record.Include { 109 | wg.Add(1) 110 | go func(domain string, domainIpMap *map[string]string, iplist *[]string) { 111 | defer wg.Done() 112 | ips, err := net.LookupIP(domain) 113 | if err != nil { 114 | // fmt.Println("Couldn't resolve ip for", d) 115 | return 116 | } 117 | mtx.Lock() 118 | (*domainIpMap)[ips[0].String()] = domain 119 | // fmt.Println("Added IP to the list:", ips[0]) 120 | *iplist = append(*iplist, ips[0].String()) 121 | mtx.Unlock() 122 | }(d, &domainIpMap, &iplist) 123 | wg.Wait() 124 | // ips, err := net.LookupIP(d) 125 | // if err != nil { 126 | // fmt.Println("Couldn't resolve ip for", d) 127 | // continue 128 | // } 129 | // domainIpMap[ips[0].String()] = d 130 | // fmt.Println("Added IP to the list:", ips[0]) 131 | // iplist = append(iplist, ips[0].String()) 132 | } 133 | for _, ipStr := range record.IPv4 { 134 | ipStr = strings.SplitN(ipStr, "/", 2)[0] 135 | iplist = append(iplist, ipStr) 136 | } 137 | whoisRecords, err := arin.WhoisLookup(iplist) 138 | if err != nil { 139 | log.Fatalln(err) 140 | } 141 | for _, whoisRecord := range whoisRecords { 142 | if val, ok := domainIpMap[whoisRecord.SearchTerm]; ok { 143 | whoisRecord.SearchTerm = val 144 | } 145 | } 146 | record.WhoisRecords = whoisRecords 147 | } 148 | return nil 149 | } 150 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "spfwalker/lib/spf" 10 | ) 11 | 12 | func main() { 13 | outFilePtr := flag.String("outfile", "", "File to save JSON results to.") 14 | domainPtr := flag.String("domain", "", "Domain to walk and retrieve SPF records for.") 15 | flag.Parse() 16 | 17 | if *domainPtr == "" { 18 | fmt.Println("[-] Error: Not enough arguments given. Please pass a domain with the -domain flag.") 19 | os.Exit(1) 20 | } 21 | 22 | worker := spf.NewSPFWorker() 23 | worker.WalkAllSPFRecords(*domainPtr) 24 | // records := spf.WalkSPFRecord(*domainPtr) 25 | 26 | // err := spf.ResolveWhoisInfo(&records) 27 | // if err != nil { 28 | // log.Fatalln(err) 29 | // } 30 | for _, r := range worker.Results { 31 | fmt.Printf("Domain: %s (%d includes, %d ip4)\n", r.Domain, len(r.Include), len(r.IPv4)) 32 | for _, d := range r.Include { 33 | fmt.Printf("\t[Include] %s\n", d) 34 | } 35 | for _, ip := range r.IPv4 { 36 | fmt.Printf("\t[IP4] %s\n", ip) 37 | } 38 | fmt.Println() 39 | for _, rec := range r.WhoisRecords { 40 | fmt.Println("\tWhois Information") 41 | fmt.Printf("\t\t[CN] %s\n", rec.CN) 42 | fmt.Printf("\t\t[IPv4] %s\n", rec.IPv4) 43 | fmt.Printf("\t\t[Name] %s\n", rec.Name) 44 | fmt.Printf("\t\t[HostName] %s\n", rec.HostName) 45 | fmt.Printf("\t\t[SearchTerm] %s\n", rec.SearchTerm) 46 | fmt.Println() 47 | } 48 | fmt.Println() 49 | } 50 | 51 | if *outFilePtr != "" { 52 | file, _ := json.MarshalIndent(worker.Results, "", " ") 53 | _ = ioutil.WriteFile(*outFilePtr, file, 0644) 54 | fmt.Println("[+] Wrote results to", *outFilePtr) 55 | } 56 | } 57 | --------------------------------------------------------------------------------