├── main.go
└── readme.md


/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"fmt"
  6 | 	"github.com/projectdiscovery/cdncheck"
  7 | 	"log"
  8 | 	"net"
  9 | 	"net/url"
 10 | 	"os"
 11 | 	"strings"
 12 | 	"sync"
 13 | 	"github.com/projectdiscovery/dnsx/libs/dnsx"
 14 | )
 15 | 
 16 | func isURL(candidate string) bool {
 17 | 	return strings.Contains(candidate, "://")
 18 | }
 19 | 
 20 | func extractHost(rawurl string) string {
 21 | 	u, err := url.Parse(rawurl)
 22 | 	if err != nil {
 23 | 		log.Fatal(err)
 24 | 	}
 25 | 
 26 | 	host, _, err := net.SplitHostPort(u.Host)
 27 | 	if err != nil {
 28 | 		return u.Host
 29 | 	}
 30 | 	return host
 31 | }
 32 | 
 33 | func CDNFilter() func(string) bool {
 34 | 	client, err := cdncheck.NewWithCache()
 35 | 	if err != nil {
 36 | 		log.Fatal(err)
 37 | 	}
 38 | 	resolveName := Resolver()
 39 | 
 40 | 	return func(line string) bool {
 41 | 		host := line
 42 | 		if isURL(line) {
 43 | 			host = extractHost(line)
 44 | 		}
 45 | 
 46 | 		ip := net.ParseIP(host)
 47 | 		ips := []net.IP{}
 48 | 		if ip != nil {
 49 | 			ips = append(ips, ip)
 50 | 		} else {
 51 | 			ips = append(ips, resolveName(host)...)
 52 | 		}
 53 | 		for _, ip := range ips {
 54 | 			found, err := client.Check(ip)
 55 | 			if found && err == nil {
 56 | 				return true
 57 | 			}
 58 | 		}
 59 | 		return false
 60 | 	}
 61 | }
 62 | 
 63 | func Resolver() func(string) []net.IP {
 64 | 	resolver, err := dnsx.New(dnsx.DefaultOptions)
 65 | 	if err != nil {
 66 | 		log.Fatal(err)
 67 | 	}
 68 | 
 69 | 	return func (name string) []net.IP {
 70 | 		validIPs := []net.IP{}
 71 | 		ips, err := resolver.Lookup(name)
 72 | 		if err != nil {
 73 | 			return validIPs
 74 | 		}
 75 | 		for _, ip := range ips {
 76 | 			parsedIP := net.ParseIP(ip)
 77 | 			if parsedIP.To4() == nil {
 78 | 				continue
 79 | 			}
 80 | 			validIPs = append(validIPs, parsedIP)
 81 | 		}
 82 | 		return validIPs
 83 | 	}
 84 | }
 85 | 
 86 | func main() {
 87 | 	scanner := bufio.NewScanner(os.Stdin)
 88 | 	filter := CDNFilter()
 89 | 
 90 | 	var wg sync.WaitGroup
 91 | 	lines := make(chan string)
 92 | 
 93 | 	for i := 0; i < 100; i++ {
 94 | 		wg.Add(1)
 95 | 
 96 | 		go func() {
 97 | 			defer wg.Done()
 98 | 			for line := range lines {
 99 | 				if !filter(line) {
100 | 					fmt.Println(line)
101 | 				}
102 | 			}
103 | 		}()
104 | 	}
105 | 
106 | 	for scanner.Scan() {
107 | 		lines <- scanner.Text()
108 | 	}
109 | 
110 | 	close(lines)
111 | 	wg.Wait()
112 | }
113 | 


--------------------------------------------------------------------------------
/readme.md:
--------------------------------------------------------------------------------
 1 | #### Exclude CDN
 2 | 
 3 | This tool wraps ProjectDiscovery's CDNCheck library to allow users to filter out CDN hosts from a list consisting of IP's, URL's, and Domains passed via stdin. 
 4 | 
 5 | ```bash
 6 | λ cgboal [~] → crobat -s hackerone.com | httpx -silent | exclude-cdn
 7 | https://docs.hackerone.com
 8 | https://mta-sts.forwarding.hackerone.com
 9 | https://mta-sts.managed.hackerone.com
10 | https://mta-sts.hackerone.com
11 | 
12 | λ cgboal [~] → crobat -s hackerone.com | httpx -silent              
13 | https://mta-sts.hackerone.com
14 | https://mta-sts.forwarding.hackerone.com
15 | https://mta-sts.managed.hackerone.com
16 | https://docs.hackerone.com
17 | https://hackerone.com
18 | https://api.hackerone.com
19 | https://www.hackerone.com
20 | https://api.hackerone.com
21 | https://www.hackerone.com
22 | https://support.hackerone.com
23 | https://hackerone.com
24 | ```
25 | 


--------------------------------------------------------------------------------