├── .gitignore
├── README.md
├── cmd
    ├── main.go
    ├── model
    │   └── model.go
    ├── processor
    │   └── process.go
    └── utils
    │   ├── fileutils.go
    │   └── stringutils.go
├── go.mod
├── go.sum
└── pkg
    └── dnshandler
        └── dnshandler.go


/.gitignore:
--------------------------------------------------------------------------------
 1 | main
 2 | gowc
 3 | gowc_*
 4 | *.txt
 5 | *.prof
 6 | test
 7 | *.out
 8 | ./.vscode
 9 | .vscode
10 | *.test


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # GoWC
  2 | A fast & accurate tool to clean **wildcards** from **[Massdns](https://github.com/blechschmidt/massdns) output file**.  
  3 | This is **not** a wrapper. A wrapper will have massdns's parameters fixed inside, what is not my style. Massdns should be used in flexible way.  
  4 | Generally, algorithm is based on [puredns](https://github.com/d3mondev/puredns), but there are few changes to make the algorithm more accurate and faster. 
  5 | 
  6 | GoWC, first it will ask for NS of target domain (Ex. ns1.<target>.com, ns2.<target>.com). Then, belong with Google & CloudFlare DNS, these NS will be used to clean wildcards faster and more accurate. Why? Because sometimes, ns1 (of target) could accept wildcard subdomains, but ns2 doesn't that lead to **False Positive**. This tool will solve all these problems.
  7 | 
  8 | ## What's new
  9 | + With GoWC v1.3.5, wildcard subdomains will be cleared faster, and more accurate. You now can set some more options: 
 10 | 
 11 | ```
 12 |   -s, --timeout= Timeout in seconds (default: 5)
 13 |   -q, --qps=     Queries per second (default: 10000)
 14 |   -r, --retries= Max retries each failed query (default: 1)
 15 | ```
 16 | 
 17 | + All progress info just be printed to `stderr`. Only valid subdomains will be printed to `stdout`.
 18 | 
 19 | ## Build
 20 | 
 21 | ```
 22 | git clone https://github.com/sting8k/gowc
 23 | cd ./gowc/
 24 | go build ./cmd/
 25 | ```
 26 | 
 27 | Or use the [pre-built binary](https://github.com/sting8k/gowc/releases)
 28 | 
 29 | ## Usage
 30 | 
 31 | ```
 32 | ./gowc -h
 33 | 
 34 |  ██████╗  ██████╗ ██╗    ██╗ ██████╗
 35 | ██╔════╝ ██╔═══██╗██║    ██║██╔════╝
 36 | ██║  ███╗██║   ██║██║ █╗ ██║██║     
 37 | ██║   ██║██║   ██║██║███╗██║██║     
 38 | ╚██████╔╝╚██████╔╝╚███╔███╔╝╚██████╗
 39 |  ╚═════╝  ╚═════╝  ╚══╝╚══╝  ╚═════╝
 40 |                          GoWC v1.3.5
 41 | Usage:
 42 |   cmd [OPTIONS]
 43 | 
 44 | Application Options:
 45 |   -m=            Massdns output file
 46 |   -d, --domain=  Domain of target
 47 |   -t, --threads= Threads (default: 20)
 48 |   -s, --timeout= Timeout in seconds (default: 5)
 49 |   -q, --qps=     Queries per second (default: 10000)
 50 |   -r, --retries= Max retries each failed query (default: 1)
 51 |   -o, --output=  Output file
 52 |   -i, --ip       Output with ips from massdns
 53 | 
 54 | Help Options:
 55 |   -h, --help     Show this help message
 56 | 
 57 | ```
 58 | 
 59 | 
 60 | For normal output:
 61 | ```
 62 | ./gowc -d <target.com> -m <massdnsOutput> -o <output>
 63 | ```
 64 | 
 65 | For output with ips of domains:
 66 | ```
 67 | ./gowc -d <target.com> -m <massdnsOutput> -o <output> -i
 68 | ```
 69 | 
 70 | # Example
 71 | 
 72 | Cleaned `~190k` subdomains in just `~1.72s`
 73 | 
 74 | ```
 75 | ./gowc -d vk.com -m vk.com_massdns.txt -o output.txt -i
 76 | 
 77 |  ██████╗  ██████╗ ██╗    ██╗ ██████╗
 78 | ██╔════╝ ██╔═══██╗██║    ██║██╔════╝
 79 | ██║  ███╗██║   ██║██║ █╗ ██║██║     
 80 | ██║   ██║██║   ██║██║███╗██║██║     
 81 | ╚██████╔╝╚██████╔╝╚███╔███╔╝╚██████╗
 82 |  ╚═════╝  ╚═════╝  ╚══╝╚══╝  ╚═════╝
 83 |                          GoWC v1.3.5
 84 | [+] Nameserver list: 
 85 |         + ns3.vkontakte.ru
 86 |         + ns4.vkontakte.ru
 87 |         + ns1.vkontakte.ru
 88 |         + ns2.vkontakte.ru
 89 |         + 8.8.8.8
 90 |         + 8.8.4.4
 91 |         + 1.1.1.1
 92 |         + 1.0.0.1
 93 | [+] Loaded 190468 subdomains in MassDns cache file.
 94 | [i] Sent 1856 queries. 190450 subdomains remaining ...
 95 | [!] Sent 2625 queries. All subdomains resolved.
 96 | 
 97 | [+] Wildcard domains:
 98 |         + 93.186.225.208
 99 |         + 87.240.137.158
100 |         + 87.240.190.67
101 |         + 87.240.190.72
102 |         + 95.213.1.137
103 |         + 87.240.139.194
104 |         + 87.240.190.78
105 |         + 217.69.139.17
106 |         + 87.240.139.153
107 | [i] Crafting output ...
108 | [i] Saving output to file: output.txt
109 | 
110 | [!] Found 1053 valid subdomains in 1.720892373s
111 | 
112 | ```
113 | 
114 | Output:
115 | ```
116 | ...
117 | papi.vk.com [87.240.139.156]
118 | post.vk.com [87.240.182.130]
119 | ps.vk.com [pu.vk.com]
120 | pu.vk.com [87.240.129.180, 87.240.137.139, 87.240.190.85, 87.240.190.74, 87.240.129.188]
121 | queue.vk.com [87.240.129.131, 87.240.129.186, 93.186.225.201, 93.186.225.198, 87.240.129.129]
122 | queuev4.vk.com [87.240.129.186, 93.186.225.201, 93.186.225.198, 87.240.129.129, 87.240.129.131]
123 | reply.vk.com [95.142.194.149]
124 | rim.vk.com [87.240.129.186]
125 | security.vk.com [95.142.199.216]
126 | smtp.vk.com [87.240.169.121]
127 | storage2.vk.com [87.240.139.151]
128 | streaming.vk.com [87.240.129.187, 87.240.190.64]
129 | team.vk.com [185.29.130.131]
130 | ...
131 | ```
132 | 
133 | 
134 | 


--------------------------------------------------------------------------------
/cmd/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"fmt"
  6 | 	"log"
  7 | 	"os"
  8 | 	"strings"
  9 | 	"sync"
 10 | 	"time"
 11 | 
 12 | 	"github.com/jessevdk/go-flags"
 13 | 	"github.com/sirupsen/logrus"
 14 | 	"github.com/sting8k/gowc/cmd/model"
 15 | 	"github.com/sting8k/gowc/cmd/processor"
 16 | 	"github.com/sting8k/gowc/cmd/utils"
 17 | 	"github.com/sting8k/gowc/pkg/dnshandler"
 18 | )
 19 | 
 20 | func craftOutput(gWC *model.GoWCModel) map[string][]string {
 21 | 
 22 | 	output := make(map[string][]string)
 23 | 
 24 | 	rootDomains := gWC.GetRootDomains()
 25 | 	rootIPs := gWC.GetRootIPs()
 26 | 
 27 | 	for domain := range gWC.IpsMap.IterBuffered() {
 28 | 		for _, rD := range rootDomains {
 29 | 			if strings.Contains(domain.Key, rD) && domain.Key != rD {
 30 | 				newIPs := make([]string, 0)
 31 | 				for _, iV := range domain.Val.([]string) {
 32 | 					if !utils.StringInSlice(iV, rootIPs) {
 33 | 						newIPs = append(newIPs, iV)
 34 | 					}
 35 | 				}
 36 | 				gWC.IpsMap.Set(domain.Key, newIPs)
 37 | 				break
 38 | 			}
 39 | 		}
 40 | 
 41 | 		if v, _ := gWC.IpsMap.Get(domain.Key); len(v.([]string)) != 0 && !strings.HasPrefix(domain.Key, model.GeneratedMagicStr) {
 42 | 			output[domain.Key] = v.([]string)
 43 | 		}
 44 | 	}
 45 | 
 46 | 	return output
 47 | }
 48 | 
 49 | func getNSOfTarget(domain string) ([]string, error) {
 50 | 	var NSans []string
 51 | 	options := &dnshandler.Options{
 52 | 		BaseResolvers: dnshandler.DefaultOptions.BaseResolvers,
 53 | 		MaxRetries:    dnshandler.DefaultOptions.MaxRetries,
 54 | 		Qps:           1,
 55 | 	}
 56 | 	Resolvers := dnshandler.DefaultOptions.BaseResolvers
 57 | 
 58 | 	dnsMachineNormal, err := dnshandler.InitDNSFactory(options)
 59 | 
 60 | 	if err != nil {
 61 | 		log.Fatal(err)
 62 | 	}
 63 | 
 64 | 	NSans = dnsMachineNormal.GreedyQuery(domain, "NS")
 65 | 	if len(NSans) != 0 {
 66 | 		return NSans, nil
 67 | 	}
 68 | 
 69 | 	return Resolvers, errors.New("empty NS")
 70 | }
 71 | 
 72 | func CollectAnswer(gWC *model.GoWCModel, dnsMachine *dnshandler.DNSFactory) {
 73 | 	for {
 74 | 		select {
 75 | 		case AnsData, more := <-dnsMachine.AnsMap:
 76 | 			if !more {
 77 | 				return
 78 | 			}
 79 | 			if ds, ok := gWC.IpsMap.Get(AnsData[0]); ok {
 80 | 				tmp := append(ds.([]string), AnsData[1:]...)
 81 | 				gWC.IpsMap.Set(AnsData[0], utils.RemoveDuplicates(tmp))
 82 | 			} else {
 83 | 				gWC.IpsMap.Set(AnsData[0], utils.RemoveDuplicates(AnsData[1:]))
 84 | 			}
 85 | 
 86 | 			v, _ := gWC.IpsMap.Get(AnsData[0])
 87 | 			logrus.Debug("Added total new ", AnsData[0], v.([]string))
 88 | 		default:
 89 | 			continue
 90 | 		}
 91 | 	}
 92 | }
 93 | 
 94 | func ProcessDomain(domain string, gWC *model.GoWCModel, dnsMachine *dnshandler.DNSFactory) {
 95 | 	ips, err := gWC.Resolve(domain, dnsMachine)
 96 | 	if err != nil {
 97 | 		gWC.DomainsChan <- domain
 98 | 		return
 99 | 	}
100 | 	if len(ips) == 0 {
101 | 		return
102 | 	}
103 | 
104 | 	if gWC.IpIsWildcard(domain, ips[0]) {
105 | 		return
106 | 	}
107 | 
108 | 	parentDomain := model.GetParentDomain(domain)
109 | 	tmpDomain := model.GeneratedMagicStr + "." + parentDomain
110 | 	tmpDomainIps, err := gWC.Resolve(tmpDomain, dnsMachine)
111 | 	if err != nil {
112 | 		gWC.DomainsChan <- domain
113 | 		return
114 | 	}
115 | 
116 | 	if utils.StringInSlice(ips[0], tmpDomainIps) {
117 | 		rootDomainCheck, err := gWC.GetRootOfWildcard(domain, dnsMachine)
118 | 		if err != nil {
119 | 			gWC.DomainsChan <- domain
120 | 			return
121 | 		}
122 | 		for _, IP := range tmpDomainIps {
123 | 			model.AddQueue(&gWC.KnownWcResult, IP, []string{rootDomainCheck}, model.KnownWcMutex)
124 | 		}
125 | 	}
126 | 
127 | }
128 | 
129 | func Worker(gWC *model.GoWCModel, dnsMachine *dnshandler.DNSFactory, wg *sync.WaitGroup) {
130 | 	var domain string
131 | 	defer wg.Done()
132 | 
133 | 	for len(gWC.DomainsChan) > 0 {
134 | 		select {
135 | 		case domain = <-gWC.DomainsChan:
136 | 			if domain != "" {
137 | 				ProcessDomain(domain, gWC, dnsMachine)
138 | 			}
139 | 		default:
140 | 			continue
141 | 		}
142 | 	}
143 | }
144 | 
145 | type GoWcArgs struct {
146 | 	MassdnsCache string `short:"m" description:"Massdns output file" required:"true"`
147 | 	Domain       string `short:"d" long:"domain" description:"Domain of target" required:"true"`
148 | 	Threads      int    `short:"t" long:"threads" description:"Threads" default:"20"`
149 | 	Timeout      int    `short:"s" long:"timeout" description:"Timeout in seconds" default:"5"`
150 | 	Qps          int    `short:"q" long:"qps" description:"Queries per second" default:"10000"`
151 | 	MaxRetries   int    `short:"r" long:"retries" description:"Max retries each failed query" default:"1"`
152 | 	Output       string `short:"o" long:"output" description:"Output file"`
153 | 	WithIp       bool   `short:"i" long:"ip" description:"Output with ips from massdns"`
154 | }
155 | 
156 | var gowcArgs GoWcArgs
157 | 
158 | // *GoWcArgs
159 | func argsParse() *GoWcArgs {
160 | 	banner := `
161 |  ██████╗  ██████╗ ██╗    ██╗ ██████╗
162 | ██╔════╝ ██╔═══██╗██║    ██║██╔════╝
163 | ██║  ███╗██║   ██║██║ █╗ ██║██║     
164 | ██║   ██║██║   ██║██║███╗██║██║     
165 | ╚██████╔╝╚██████╔╝╚███╔███╔╝╚██████╗
166 |  ╚═════╝  ╚═════╝  ╚══╝╚══╝  ╚═════╝
167 |                          GoWC v1.3.5					
168 | `
169 | 	fmt.Fprint(os.Stderr, banner)
170 | 	_, err := flags.Parse(&gowcArgs)
171 | 
172 | 	if err != nil {
173 | 		flagError := err.(*flags.Error)
174 | 		if flagError.Type == flags.ErrHelp {
175 | 			// user asked for help on flags.
176 | 			// program can exit successfully
177 | 			os.Exit(0)
178 | 		}
179 | 		if flagError.Type == flags.ErrUnknownFlag {
180 | 			fmt.Println("Use --help to view all available options.")
181 | 			os.Exit(1)
182 | 		}
183 | 		// fmt.Printf("Error parsing flags: %s\n", err)
184 | 		os.Exit(1)
185 | 	}
186 | 	return &gowcArgs
187 | }
188 | 
189 | func main() {
190 | 	// log.SetFlags(log.Flags() &^ (log.Ldate | log.Ltime))
191 | 	logrus.SetFormatter(&logrus.TextFormatter{
192 | 		DisableColors: false,
193 | 		FullTimestamp: false,
194 | 	})
195 | 	logrus.SetOutput(os.Stderr)
196 | 	logrus.SetLevel(logrus.ErrorLevel)
197 | 
198 | 	args := argsParse()
199 | 
200 | 	//Get root NS of target
201 | 	NSans, _ := getNSOfTarget(args.Domain)
202 | 	fmt.Fprintf(os.Stderr, "[+] Nameserver list: \n\t+ %s\n", strings.Join(append(NSans, dnshandler.DefaultOptions.BaseResolvers...), "\n\t+ "))
203 | 	//Initialize gWC model
204 | 	dnsMachine, _ := dnshandler.InitDNSFactory(&dnshandler.Options{
205 | 		BaseResolvers: append(dnshandler.DefaultOptions.BaseResolvers, NSans...),
206 | 		MaxRetries:    args.MaxRetries,
207 | 		Qps:           args.Qps,
208 | 		Timeout:       args.Timeout},
209 | 	)
210 | 
211 | 	gWC := &model.GoWCModel{}
212 | 	gWC.Init()
213 | 	gWC.SetMainDomain(args.Domain)
214 | 
215 | 	processor.ProcessMassdnsCache(args.MassdnsCache, gWC)
216 | 
217 | 	fmt.Fprintf(os.Stderr, "[+] Loaded %d subdomains in MassDns cache file.\n", gWC.IpsMap.Count())
218 | 
219 | 	if !gWC.IpsMap.Has(gWC.MainDomain) {
220 | 		gWC.IpsMap.Set(gWC.MainDomain, dnsMachine.GreedyQuery(gWC.MainDomain, "A"))
221 | 	}
222 | 
223 | 	start := time.Now()
224 | 
225 | 	gWC.DomainsChan = make(chan string, gWC.IpsMap.Count()*2)
226 | 	for i := range gWC.SortedList {
227 | 		gWC.DomainsChan <- gWC.SortedList[i]
228 | 	}
229 | 
230 | 	var wg sync.WaitGroup
231 | 	concurrency := args.Threads
232 | 	for i := 0; i < concurrency; i++ {
233 | 		wg.Add(1)
234 | 		go Worker(gWC, dnsMachine, &wg)
235 | 	}
236 | 
237 | 	go CollectAnswer(gWC, dnsMachine)
238 | 	go dnsMachine.ProcessPool(args.Timeout)
239 | 
240 | 	go func() {
241 | 		time.Sleep(200 * time.Millisecond)
242 | 		progressLine := len(fmt.Sprintf("\r[i] Sent %d queries. %d subdomains remaining ...", dnsMachine.QueryCounter, len(gWC.DomainsChan)))
243 | 		// logrus.Debugf("On %d | %d | %d | %d \n", dnsMachine.QueryCounter, len(gWC.DomainsChan), len(gWC.KnownWcResult), runtime.NumGoroutine())
244 | 		for dnsMachine.QueryCounter != -1 {
245 | 			// logrus.Debugf("On %d | %d | %d | %d \n", dnsMachine.QueryCounter, len(gWC.DomainsChan), len(gWC.KnownWcResult), runtime.NumGoroutine())
246 | 			currentLine := fmt.Sprintf("\r[i] Sent %d queries. %d subdomains remaining ...", dnsMachine.QueryCounter, len(gWC.DomainsChan))
247 | 			padding := ""
248 | 			if progressLine-len(currentLine) > 0 {
249 | 				padding = strings.Repeat(" ", progressLine-len(currentLine))
250 | 			}
251 | 			fmt.Fprintf(os.Stderr, currentLine+padding)
252 | 			time.Sleep(5000 * time.Millisecond)
253 | 		}
254 | 	}()
255 | 
256 | 	wg.Wait()
257 | 
258 | 	fmt.Fprintf(os.Stderr, "\n[!] Sent %d queries. All subdomains resolved.\n", dnsMachine.QueryCounter)
259 | 
260 | 	// Clean
261 | 	dnsMachine.KillSwitch <- true
262 | 	dnsMachine.StopResolveEngine()
263 | 	close(gWC.DomainsChan)
264 | 
265 | 	fmt.Fprintln(os.Stderr, "\n[+] Wildcard domains:\n\t+", strings.Join(gWC.GetRootIPs(), "\n\t+ "))
266 | 
267 | 	fmt.Fprintln(os.Stderr, "[i] Crafting output ...")
268 | 	output := craftOutput(gWC)
269 | 	elapsed := time.Since(start)
270 | 	validDomains := processor.ExportOutput(output, args.Output, args.WithIp)
271 | 	fmt.Fprintf(os.Stderr, "\n[!] Found %d valid subdomains in %s\n", validDomains, elapsed)
272 | }
273 | 


--------------------------------------------------------------------------------
/cmd/model/model.go:
--------------------------------------------------------------------------------
  1 | package model
  2 | 
  3 | import (
  4 | 	"errors"
  5 | 	"sort"
  6 | 	"strings"
  7 | 	"sync"
  8 | 
  9 | 	cmap "github.com/orcaman/concurrent-map"
 10 | 	"github.com/rs/xid"
 11 | 	"github.com/sting8k/gowc/cmd/utils"
 12 | 	"github.com/sting8k/gowc/pkg/dnshandler"
 13 | )
 14 | 
 15 | var GeneratedMagicStr = xid.New().String()
 16 | var IpsMutex = &sync.RWMutex{}
 17 | var KnownWcMutex = &sync.RWMutex{}
 18 | var resolverMutex = &sync.Mutex{}
 19 | var DomainQueueMutex = &sync.Mutex{}
 20 | 
 21 | type GoWCModel struct {
 22 | 	MainDomain    string
 23 | 	KnownWcResult map[string][]string
 24 | 	NsRecord      []string
 25 | 	SortedList    []string
 26 | 	DomainsChan   chan string
 27 | 	IpsMap        cmap.ConcurrentMap
 28 | }
 29 | 
 30 | func (m *GoWCModel) Init() {
 31 | 	m.KnownWcResult = make(map[string][]string)
 32 | 	m.NsRecord = make([]string, 0)
 33 | 	m.SortedList = make([]string, 0)
 34 | 	m.IpsMap = cmap.New()
 35 | 
 36 | }
 37 | 
 38 | func (m *GoWCModel) SetMainDomain(mdm string) {
 39 | 	m.MainDomain = strings.ToLower(mdm)
 40 | }
 41 | 
 42 | func (m *GoWCModel) Resolve(domain string, dnsMachine *dnshandler.DNSFactory) ([]string, error) {
 43 | 	flagx := false
 44 | 	if !strings.HasSuffix(domain, m.MainDomain) {
 45 | 		return []string{}, nil
 46 | 	}
 47 | 
 48 | 	if !m.IpsMap.Has(domain) {
 49 | 		resolverMutex.Lock()
 50 | 		if _, flag2 := dnsMachine.QueryDict[domain]; !flag2 {
 51 | 			dnsMachine.QueryDict[domain] = struct{}{}
 52 | 			flagx = true
 53 | 		}
 54 | 		resolverMutex.Unlock()
 55 | 
 56 | 		if flagx {
 57 | 			dnsMachine.QueryPool <- domain
 58 | 		}
 59 | 		// logrus.Error(domain, " waiting for resolve")
 60 | 		return nil, errors.New("waiting for resolve")
 61 | 
 62 | 	} else {
 63 | 		v, _ := m.IpsMap.Get(domain)
 64 | 		return v.([]string), nil
 65 | 	}
 66 | 
 67 | }
 68 | 
 69 | func (m *GoWCModel) PushToResolvePool(domain string, dnsMachine *dnshandler.DNSFactory) {
 70 | 	if !strings.HasSuffix(domain, m.MainDomain) {
 71 | 		return
 72 | 	}
 73 | 	if _, ok := m.IpsMap.Get(domain); !ok {
 74 | 		if _, inDict := dnsMachine.QueryDict[domain]; !inDict {
 75 | 			dnsMachine.QueryDict[domain] = struct{}{}
 76 | 		}
 77 | 	}
 78 | }
 79 | 
 80 | func (m *GoWCModel) IpIsWildcard(domain, ip string) bool {
 81 | 	defer KnownWcMutex.RUnlock()
 82 | 	KnownWcMutex.RLock()
 83 | 	if _, ok := m.KnownWcResult[ip]; ok {
 84 | 		for wcIP := range m.KnownWcResult {
 85 | 			for _, rootDomainGot := range m.KnownWcResult[wcIP] {
 86 | 				if len(domain) > len(rootDomainGot)+1 && strings.HasSuffix(domain, "."+rootDomainGot) {
 87 | 					return true
 88 | 				}
 89 | 			}
 90 | 		}
 91 | 	}
 92 | 	return false
 93 | }
 94 | 
 95 | func (m *GoWCModel) GetRootOfWildcard(domain string, dnsMachine *dnshandler.DNSFactory) (string, error) {
 96 | 	tmpRoot := ""
 97 | 	domainPieces := strings.Split(domain, ".")
 98 | 
 99 | 	root := domain
100 | 
101 | 	for i := len(domainPieces) - 1; i > 0; i-- {
102 | 		tmpRoot = strings.ToLower(strings.Join(domainPieces[i-1:], "."))
103 | 		isWc, err := m.IsWildcardRoot(domain, tmpRoot, dnsMachine)
104 | 		if err != nil {
105 | 			return "", err
106 | 		}
107 | 		if isWc {
108 | 			break
109 | 		}
110 | 		root = tmpRoot
111 | 	}
112 | 	return root, nil
113 | }
114 | 
115 | func (m *GoWCModel) IsWildcardRoot(domain, tmpRoot string, dnsMachine *dnshandler.DNSFactory) (bool, error) {
116 | 	parentDomain := GetParentDomain(domain)
117 | 	tmpDomain := GeneratedMagicStr + "." + parentDomain
118 | 	tmpDomainIps, err := m.Resolve(tmpDomain, dnsMachine)
119 | 	if err != nil {
120 | 		return false, err
121 | 	}
122 | 
123 | 	tmpParent := GeneratedMagicStr + "." + GetParentDomain(tmpRoot)
124 | 	tmpParentIps, err := m.Resolve(tmpParent, dnsMachine)
125 | 	if err != nil {
126 | 		return false, err
127 | 	}
128 | 
129 | 	return utils.StringInSlice(tmpDomainIps[0], tmpParentIps), nil
130 | }
131 | 
132 | func (m *GoWCModel) ResolveRoot(domain, tmpRoot string, dnsMachine *dnshandler.DNSFactory) {
133 | 	tmpParent := GeneratedMagicStr + "." + GetParentDomain(tmpRoot)
134 | 	m.PushToResolvePool(tmpParent, dnsMachine)
135 | }
136 | 
137 | func (m *GoWCModel) GetRootDomains() []string {
138 | 	rootDomains := make([]string, 0)
139 | 	for ip := range m.KnownWcResult {
140 | 		rootDomains = append(rootDomains, m.KnownWcResult[ip]...)
141 | 	}
142 | 	return utils.RemoveDuplicates(rootDomains)
143 | }
144 | 
145 | func (m *GoWCModel) GetRootIPs() []string {
146 | 	rootIPs := make([]string, 0)
147 | 	for ip := range m.KnownWcResult {
148 | 		rootIPs = append(rootIPs, ip)
149 | 	}
150 | 	return utils.RemoveDuplicates(rootIPs)
151 | }
152 | 
153 | func GetParentDomain(s string) string {
154 | 	return strings.Join(strings.Split(s, ".")[1:], ".")
155 | }
156 | 
157 | func AddQueue(q *map[string][]string, key string, values []string, mutex *sync.RWMutex) {
158 | 	defer mutex.Unlock()
159 | 	mutex.Lock()
160 | 	if _, ok := (*q)[key]; !ok {
161 | 		(*q)[key] = []string{}
162 | 	}
163 | 
164 | 	(*q)[key] = append((*q)[key], values...)
165 | 	(*q)[key] = utils.RemoveDuplicates((*q)[key])
166 | 	if len((*q)[key])%10 == 0 {
167 | 		sort.Slice((*q)[key], func(i, j int) bool {
168 | 			return len((*q)[key][i]) < len((*q)[key][j])
169 | 		})
170 | 	}
171 | }
172 | 
173 | func RemoveQueue(q *map[string][]string, key string, mutex *sync.Mutex) {
174 | 	defer mutex.Unlock()
175 | 	mutex.Lock()
176 | 	delete(*q, key)
177 | }
178 | 


--------------------------------------------------------------------------------
/cmd/processor/process.go:
--------------------------------------------------------------------------------
 1 | package processor
 2 | 
 3 | import (
 4 | 	"fmt"
 5 | 	"os"
 6 | 	"sort"
 7 | 	"strings"
 8 | 
 9 | 	"github.com/sting8k/gowc/cmd/model"
10 | 	"github.com/sting8k/gowc/cmd/utils"
11 | )
12 | 
13 | func ProcessMassdnsCache(path string, gWC *model.GoWCModel) {
14 | 	var tmpDomain, tmpIP string
15 | 	lines, _ := utils.ReadLines(path) // x.y.z A 22.52.25.25
16 | 	tmpMap := make(map[string][]string, 0)
17 | 	for _, domain := range lines {
18 | 		pieces := strings.Split(domain, " ")
19 | 		if len(pieces) != 3 {
20 | 			continue
21 | 		}
22 | 		if pieces[1] != "CNAME" && pieces[1] != "A" && pieces[1] != "AAAA" {
23 | 			continue
24 | 		}
25 | 		tmpDomain = strings.ToLower(strings.TrimSuffix(pieces[0], "."))
26 | 		tmpIP = strings.TrimSuffix(pieces[2], ".")
27 | 		gWC.SortedList = append(gWC.SortedList, tmpDomain)
28 | 		tmpMap[tmpDomain] = append(tmpMap[tmpDomain], tmpIP)
29 | 	}
30 | 
31 | 	gWC.SortedList = utils.RemoveDuplicates(gWC.SortedList)
32 | 
33 | 	sort.Slice(gWC.SortedList, func(i, j int) bool {
34 | 		return len(strings.Split(gWC.SortedList[i], ".")) < len(strings.Split(gWC.SortedList[j], "."))
35 | 	})
36 | 
37 | 	for d := range tmpMap {
38 | 		gWC.IpsMap.Set(d, utils.RemoveDuplicates(tmpMap[d]))
39 | 	}
40 | }
41 | 
42 | func ExportOutput(data map[string][]string, path string, withip bool) int {
43 | 	var output []string
44 | 	for d := range data {
45 | 		if withip {
46 | 			output = append(output, d+" ["+strings.Join(data[d], ", ")+"]")
47 | 		} else {
48 | 			output = append(output, d)
49 | 		}
50 | 
51 | 	}
52 | 	sort.Strings(output)
53 | 
54 | 	if path != "" {
55 | 		fmt.Fprintln(os.Stderr, "[i] Saving output to file: "+path)
56 | 		utils.WriteLines(output, path)
57 | 	} else {
58 | 		for _, line := range output {
59 | 			fmt.Println(line)
60 | 		}
61 | 	}
62 | 	return len(output)
63 | }
64 | 


--------------------------------------------------------------------------------
/cmd/utils/fileutils.go:
--------------------------------------------------------------------------------
 1 | package utils
 2 | 
 3 | import (
 4 | 	"bufio"
 5 | 	"fmt"
 6 | 	"os"
 7 | )
 8 | 
 9 | func ReadLines(path string) ([]string, error) {
10 | 	file, err := os.Open(path)
11 | 	if err != nil {
12 | 		return nil, err
13 | 	}
14 | 	defer file.Close()
15 | 
16 | 	var lines []string
17 | 	scanner := bufio.NewScanner(file)
18 | 	for scanner.Scan() {
19 | 		lines = append(lines, scanner.Text())
20 | 	}
21 | 	return lines, scanner.Err()
22 | }
23 | 
24 | func WriteLines(lines []string, path string) error {
25 | 	file, err := os.Create(path)
26 | 	if err != nil {
27 | 		return err
28 | 	}
29 | 	defer file.Close()
30 | 
31 | 	w := bufio.NewWriter(file)
32 | 	for _, line := range lines {
33 | 		fmt.Fprintln(w, line)
34 | 	}
35 | 	return w.Flush()
36 | }
37 | 
38 | func Stdinput() {
39 | 	scanner := bufio.NewScanner(os.Stdin)
40 | 	for scanner.Scan() {
41 | 		fmt.Println("Next =======================>")
42 | 		break
43 | 	}
44 | }
45 | 


--------------------------------------------------------------------------------
/cmd/utils/stringutils.go:
--------------------------------------------------------------------------------
 1 | package utils
 2 | 
 3 | import (
 4 | 	"regexp"
 5 | 	"strings"
 6 | )
 7 | 
 8 | var rp = regexp.MustCompile("[^\\s]+")
 9 | 
10 | func IntInSlice(a uint16, list []uint16) bool {
11 | 	for _, b := range list {
12 | 		if b == a {
13 | 			return true
14 | 		}
15 | 	}
16 | 	return false
17 | }
18 | 
19 | func StringInSlice(a string, list []string) bool {
20 | 	for _, b := range list {
21 | 		if len(b) == len(a) && b == a { //list[i] == a
22 | 			return true
23 | 		}
24 | 	}
25 | 	return false
26 | }
27 | 
28 | func StringInSliceWithIndex(a string, list []string) (bool, int) {
29 | 	for i := range list {
30 | 		if list[i] == a {
31 | 			return true, i
32 | 		}
33 | 	}
34 | 	return false, -1
35 | }
36 | 
37 | func RemoveDuplicates(s []string) []string {
38 | 
39 | 	encountered := make(map[string]struct{})
40 | 	result := make([]string, 0)
41 | 	for _, v := range s {
42 | 		if _, ok := encountered[v]; ok {
43 | 			continue
44 | 		} else {
45 | 			encountered[v] = struct{}{}
46 | 			result = append(result, v)
47 | 		}
48 | 	}
49 | 	return result
50 | }
51 | 
52 | func RemoveIndex(slice []string, index int) []string {
53 | 	// return append(slice[:index], slice[index+1:]...)
54 | 	sliceLen := len(slice)
55 | 	sliceLastIndex := sliceLen - 1
56 | 
57 | 	if index != sliceLastIndex {
58 | 		slice[index] = slice[sliceLastIndex]
59 | 	}
60 | 
61 | 	return slice[:sliceLastIndex]
62 | }
63 | 
64 | func CNAMEparse(str string) string {
65 | 	pieces := rp.FindAllString(str, -1)
66 | 	result := pieces[len(pieces)-1]
67 | 	result = strings.TrimSpace(strings.TrimSuffix(result, "."))
68 | 	return result
69 | }
70 | 
71 | func NSparse(str string) string {
72 | 	pieces := strings.Split(str, "NS")
73 | 	result := pieces[len(pieces)-1]
74 | 	result = strings.TrimSpace(strings.TrimSuffix(result, "."))
75 | 	return result
76 | }
77 | 
78 | func ValidateNSFmt(str string) string {
79 | 	r := str
80 | 	if !strings.HasSuffix(str, ":53") {
81 | 		r = str + ":53"
82 | 	}
83 | 	return r
84 | }
85 | 


--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
 1 | module github.com/sting8k/gowc
 2 | 
 3 | go 1.18
 4 | 
 5 | require (
 6 | 	github.com/caffix/resolve v0.5.5-0.20220408161844-7560a499d325
 7 | 	github.com/jessevdk/go-flags v1.5.0
 8 | 	github.com/miekg/dns v1.1.49
 9 | 	github.com/rs/xid v1.2.1
10 | )
11 | 
12 | require (
13 | 	github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
14 | 	github.com/BobuSumisu/aho-corasick v1.0.3 // indirect
15 | 	github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect
16 | 	github.com/caffix/queue v0.1.3 // indirect
17 | 	github.com/caffix/stringset v0.1.0 // indirect
18 | 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
19 | 	github.com/dgraph-io/badger v1.6.2 // indirect
20 | 	github.com/dgraph-io/ristretto v0.1.0 // indirect
21 | 	github.com/dustin/go-humanize v1.0.0 // indirect
22 | 	github.com/gansidui/ahocorasick v0.0.0-20220115224400-45c8304295ef // indirect
23 | 	github.com/golang/glog v1.0.0 // indirect
24 | 	github.com/golang/protobuf v1.5.2 // indirect
25 | 	github.com/iohub/ahocorasick v0.0.0-20190713143823-b7bfd8ad9e27 // indirect
26 | 	github.com/orcaman/concurrent-map v1.0.0 // indirect
27 | 	github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect
28 | 	github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect
29 | 	github.com/pkg/errors v0.9.1 // indirect
30 | 	github.com/sirupsen/logrus v1.8.1 // indirect
31 | 	github.com/vcaesar/cedar v0.20.1 // indirect
32 | 	go.uber.org/ratelimit v0.2.0 // indirect
33 | 	golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 // indirect
34 | 	golang.org/x/net v0.0.0-20220526153639-5463443f8c37 // indirect
35 | 	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
36 | 	golang.org/x/tools v0.1.10 // indirect
37 | 	golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
38 | 	google.golang.org/protobuf v1.28.0 // indirect
39 | )
40 | 
41 | // replace github.com/google/glog => github.com/slok/noglog v0.2.0
42 | 


--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
  1 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
  2 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
  3 | github.com/BobuSumisu/aho-corasick v1.0.3 h1:uuf+JHwU9CHP2Vx+wAy6jcksJThhJS9ehR8a+4nPE9g=
  4 | github.com/BobuSumisu/aho-corasick v1.0.3/go.mod h1:hm4jLcvZKI2vRF2WDU1N4p/jpWtpOzp3nLmi9AzX/XE=
  5 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
  6 | github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
  7 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 h1:MzBOUgng9orim59UnfUTLRjMpd09C5uEVQ6RPGeCaVI=
  8 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129/go.mod h1:rFgpPQZYZ8vdbc+48xibu8ALc3yeyd64IhHS+PU6Yyg=
  9 | github.com/anknown/ahocorasick v0.0.0-20170415101647-0c5fc0283558/go.mod h1:4yg+jNTYlDEzBjhGS96v+zjyA3lfXlFd5CiTLIkPBLI=
 10 | github.com/anknown/darts v0.0.0-20151216065714-83ff685239e6/go.mod h1:pbiaLIeYLUbgMY1kwEAdwO6UKD5ZNwdPGQlwokS9fe8=
 11 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 12 | github.com/caffix/queue v0.1.3 h1:rAMgUqdcBi5bI2H0dB/sL/ky710KBKHrQqFyJZktn5E=
 13 | github.com/caffix/queue v0.1.3/go.mod h1:uZb+KY0S+jd6t9C6jgadV7CDSBMhLODCdwrxHKie2vI=
 14 | github.com/caffix/resolve v0.5.5-0.20220408161844-7560a499d325 h1:F78JjhYLHXkumh17r2kDszXg4SgKns0xE/DWw7inNx0=
 15 | github.com/caffix/resolve v0.5.5-0.20220408161844-7560a499d325/go.mod h1:2wI2OUFg36bFN4zNgDv1gtthwc7yBgQ03nXs4vs6yU0=
 16 | github.com/caffix/stringset v0.0.0-20210920202210-bde5591d523d/go.mod h1:LlF238uAHb6xk7lg+L+nKj3yWAlxin2POgV4avSjAFw=
 17 | github.com/caffix/stringset v0.1.0 h1:KVtRp2RW0lWPVSTPJ7MeTSiWWATpvK9vBEetMfEqZoc=
 18 | github.com/caffix/stringset v0.1.0/go.mod h1:yoHySP4aY8Q7y4Fd7A5JSbVHy1s/T9HdBp+lJY58v+8=
 19 | github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 20 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 21 | github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=
 22 | github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 23 | github.com/cloudflare/ahocorasick v0.0.0-20131126104932-1ce46e42b741/go.mod h1:tGWUZLZp9ajsxUOnHmFFLnqnlKXsCn6GReG4jAD59H0=
 24 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 25 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 26 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 27 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 28 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 29 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 30 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 31 | github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8=
 32 | github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE=
 33 | github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 34 | github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
 35 | github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
 36 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
 37 | github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 38 | github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 39 | github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 40 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 41 | github.com/gansidui/ahocorasick v0.0.0-20220115224400-45c8304295ef h1:MK0qlBnxHWgy8LocGPQHO2vD72vizIJRXc/X5XWYfAY=
 42 | github.com/gansidui/ahocorasick v0.0.0-20220115224400-45c8304295ef/go.mod h1:o4Ibd3nHyd9dcibHOgR7hmAOQqfsiSmNHWtoq4o6cvc=
 43 | github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 44 | github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
 45 | github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 46 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 47 | github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 48 | github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 49 | github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 50 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 51 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 52 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 53 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 54 | github.com/iohub/ahocorasick v0.0.0-20190713143823-b7bfd8ad9e27 h1:OqwPHTYpoLiWOiaxiqaT3lcJ1gRPOxLymnIweR9em2c=
 55 | github.com/iohub/ahocorasick v0.0.0-20190713143823-b7bfd8ad9e27/go.mod h1:HYvkm/DoDY4MzKrf1Iobik7c3a64giSEeWhrSdb3e8o=
 56 | github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc=
 57 | github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=
 58 | github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 59 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 60 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 61 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 62 | github.com/miekg/dns v1.1.46/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 63 | github.com/miekg/dns v1.1.49 h1:qe0mQU3Z/XpFeE+AEBo2rqaS1IPBJ3anmqZ4XiZJVG8=
 64 | github.com/miekg/dns v1.1.49/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME=
 65 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 66 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 67 | github.com/orcaman/concurrent-map v1.0.0 h1:I/2A2XPCb4IuQWcQhBhSwGfiuybl/J0ev9HDbW65HOY=
 68 | github.com/orcaman/concurrent-map v1.0.0/go.mod h1:Lu3tH6HLW3feq74c2GC+jIMS/K2CFcDWnWD9XkenwhI=
 69 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 70 | github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=
 71 | github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=
 72 | github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d h1:U+PMnTlV2tu7RuMK5etusZG3Cf+rpow5hqQByeCzJ2g=
 73 | github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d/go.mod h1:lXfE4PvvTW5xOjO6Mba8zDPyw8M93B6AQ7frTGnMlA8=
 74 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 75 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 76 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 77 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 78 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 79 | github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 80 | github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 81 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
 82 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 83 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 84 | github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 85 | github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 86 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 87 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 88 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 89 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
 90 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 91 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
 92 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 93 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 94 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 95 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 96 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 97 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 98 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 99 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
100 | github.com/vcaesar/cedar v0.20.1 h1:cDOmYWdprO7ZW8cngJrDi8Zivnscj9dA/y8Y+2SB1P0=
101 | github.com/vcaesar/cedar v0.20.1/go.mod h1:iMDweyuW76RvSrCkQeZeQk4iCbshiPzcCvcGCtpM7iI=
102 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
103 | github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
104 | github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
105 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
106 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
107 | go.uber.org/ratelimit v0.2.0 h1:UQE2Bgi7p2B85uP5dC2bbRtig0C+OeNRnNEafLjsLPA=
108 | go.uber.org/ratelimit v0.2.0/go.mod h1:YYBV4e4naJvhpitQrWJu1vCpgB7CboMe0qhltKt6mUg=
109 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
110 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
111 | golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
112 | golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
113 | golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
114 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3 h1:kQgndtyPBW/JIYERgdxfwMYh3AVStj88WQTlNDi2a+o=
115 | golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY=
116 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
117 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
118 | golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
119 | golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
120 | golang.org/x/net v0.0.0-20210924151903-3ad01bbaa167/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
121 | golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
122 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
123 | golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8=
124 | golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
125 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
126 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
127 | golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
128 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
129 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
130 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
131 | golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
132 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
133 | golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
134 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
135 | golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
136 | golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
137 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
138 | golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
139 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
140 | golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
141 | golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
142 | golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
143 | golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
144 | golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
145 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k=
146 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
147 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
148 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
149 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
150 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
151 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
152 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
153 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
154 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
155 | golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
156 | golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
157 | golang.org/x/tools v0.1.10 h1:QjFRCZxdOhBJ/UNgnBZLbNV13DlbnK0quyivTnXJM20=
158 | golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
159 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
160 | golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
161 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
162 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
163 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618=
164 | golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8=
165 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
166 | google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
167 | google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
168 | google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
169 | google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
170 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
171 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
172 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
173 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
174 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
175 | 


--------------------------------------------------------------------------------
/pkg/dnshandler/dnshandler.go:
--------------------------------------------------------------------------------
  1 | package dnshandler
  2 | 
  3 | import (
  4 | 	"context"
  5 | 	"errors"
  6 | 	"strconv"
  7 | 	"strings"
  8 | 	"sync"
  9 | 	"time"
 10 | 
 11 | 	"github.com/caffix/resolve"
 12 | 	"github.com/miekg/dns"
 13 | 	miekgdns "github.com/miekg/dns"
 14 | 	"github.com/sting8k/gowc/cmd/utils"
 15 | )
 16 | 
 17 | type DNSFactory struct {
 18 | 	KillSwitch          chan bool
 19 | 	QueryCounter        int
 20 | 	MaxRetries          int
 21 | 	QueryDict           map[string]struct{}
 22 | 	QueryPool           chan string
 23 | 	BaseResolvers       []string
 24 | 	TypeMap             map[string]uint16
 25 | 	RetryCounter        map[string]int
 26 | 	AnsMap              chan []string
 27 | 	AnsChannel          chan *miekgdns.Msg
 28 | 	SubChannel          chan []string
 29 | 	ResolveEngine       *resolve.Resolvers
 30 | 	ResolveEngineCtx    context.Context
 31 | 	ResolveEngineCancel context.CancelFunc
 32 | }
 33 | 
 34 | type Options struct {
 35 | 	Qps           int
 36 | 	MaxRetries    int
 37 | 	Timeout       int
 38 | 	BaseResolvers []string
 39 | }
 40 | 
 41 | var DefaultOptions = Options{
 42 | 	BaseResolvers: []string{"8.8.8.8", "8.8.4.4", "1.1.1.1", "1.0.0.1"},
 43 | 	MaxRetries:    2,
 44 | 	Qps:           10000,
 45 | 	Timeout:       5,
 46 | }
 47 | 
 48 | var QueryMutex = &sync.RWMutex{}
 49 | 
 50 | func InitDNSFactory(options *Options) (*DNSFactory, error) {
 51 | 	theFactory := &DNSFactory{
 52 | 		KillSwitch:    make(chan bool, 1),
 53 | 		BaseResolvers: options.BaseResolvers,
 54 | 		TypeMap: map[string]uint16{
 55 | 			"A":     miekgdns.TypeA,
 56 | 			"NS":    miekgdns.TypeNS,
 57 | 			"CNAME": miekgdns.TypeCNAME,
 58 | 		},
 59 | 		ResolveEngine: resolve.NewResolvers(),
 60 | 		AnsMap:        make(chan []string, options.Qps*3),
 61 | 		AnsChannel:    make(chan *miekgdns.Msg, options.Qps*3),
 62 | 		SubChannel:    make(chan []string, options.Qps),
 63 | 		QueryDict:     make(map[string]struct{}, 0),
 64 | 		QueryPool:     make(chan string, options.Qps),
 65 | 		QueryCounter:  0,
 66 | 		RetryCounter:  make(map[string]int, 0),
 67 | 	}
 68 | 	theFactory.ResolveEngine.AddResolvers(options.Qps, theFactory.BaseResolvers...)
 69 | 	theFactory.ResolveEngine.SetTimeout(time.Duration(options.Timeout) * time.Second)
 70 | 	theFactory.ResolveEngineCtx, theFactory.ResolveEngineCancel = context.WithCancel(context.Background())
 71 | 
 72 | 	return theFactory, nil
 73 | }
 74 | 
 75 | func (d *DNSFactory) PrepareQueryPool() {
 76 | 	go func() {
 77 | 		for domain := range d.QueryDict {
 78 | 			d.QueryPool <- domain
 79 | 		}
 80 | 	}()
 81 | }
 82 | 
 83 | func (d *DNSFactory) _ActivateQueryWithQType(domain, queryType string) {
 84 | 	d.QueryCounter += 1
 85 | 	strType := strconv.FormatUint(uint64(d.TypeMap[queryType]), 10)
 86 | 	d.RetryCounter[domain+strType] = 0
 87 | 	d.ResolveEngine.Query(d.ResolveEngineCtx, resolve.QueryMsg(domain, d.TypeMap[queryType]), d.AnsChannel)
 88 | }
 89 | 
 90 | func (d *DNSFactory) RetryQuery(loadedresp *miekgdns.Msg) {
 91 | 	domain := strings.ToLower(resolve.RemoveLastDot(loadedresp.Question[0].Name))
 92 | 	strType := strconv.FormatUint(uint64(loadedresp.Question[0].Qtype), 10)
 93 | 
 94 | 	if d.RetryCounter[domain+strType] <= d.MaxRetries {
 95 | 		d.RetryCounter[domain+strType]++
 96 | 		// go func(rx *miekgdns.Msg) {
 97 | 		switch loadedresp.Question[0].Qtype {
 98 | 		case dns.TypeA:
 99 | 			d.QueryCounter++
100 | 			go func(dm string) {
101 | 				dX := append([]string{dm}, d.GreedyQuery(dm, "A")...)
102 | 				d.SubChannel <- dX
103 | 			}(domain)
104 | 
105 | 		case dns.TypeCNAME:
106 | 			d.QueryCounter++
107 | 			go func(dm string) {
108 | 				dX := append([]string{dm}, d.GreedyQuery(dm, "CNAME")...)
109 | 				d.SubChannel <- dX
110 | 			}(domain)
111 | 		}
112 | 	} else {
113 | 		d.SubChannel <- []string{domain}
114 | 	}
115 | }
116 | 
117 | func (d *DNSFactory) ProcessPool(sectimeout int) {
118 | 	t := time.NewTicker(time.Second)
119 | 	defer t.Stop()
120 | 	lengthTimeout := time.Duration(sectimeout) * time.Second
121 | 	timedout := time.After(lengthTimeout)
122 | 
123 | 	for {
124 | 		select {
125 | 		case <-timedout: // do nothing
126 | 			if len(d.AnsChannel) > 0 {
127 | 				timedout = time.After(lengthTimeout)
128 | 			}
129 | 		case <-d.KillSwitch:
130 | 			close(d.KillSwitch)
131 | 			return
132 | 		case domainX := <-d.QueryPool:
133 | 			timedout = time.After(lengthTimeout)
134 | 
135 | 			d._ActivateQueryWithQType(domainX, "A")
136 | 			d._ActivateQueryWithQType(domainX, "CNAME")
137 | 		case ans := <-d.SubChannel:
138 | 			timedout = time.After(lengthTimeout)
139 | 			if len(ans) >= 1 {
140 | 				d.AnsMap <- ans
141 | 			}
142 | 			//noted  d.QueryCounter--
143 | 		case resp := <-d.AnsChannel:
144 | 			timedout = time.After(lengthTimeout)
145 | 			domain := strings.ToLower(resolve.RemoveLastDot(resp.Question[0].Name))
146 | 			strType := strconv.FormatUint(uint64(resp.Question[0].Qtype), 10)
147 | 			if resp.Rcode == miekgdns.RcodeSuccess || resp.Rcode == miekgdns.RcodeNameError {
148 | 				if len(resp.Answer) > 0 {
149 | 					for _, datum := range resolve.ExtractAnswers(resp) {
150 | 						if !utils.IntInSlice(datum.Type, []uint16{miekgdns.TypeA, miekgdns.TypeCNAME}) {
151 | 							continue
152 | 						}
153 | 						d.AnsMap <- []string{domain, datum.Data}
154 | 					}
155 | 				} else if resp.Rcode == miekgdns.RcodeNameError {
156 | 					d.AnsMap <- []string{domain}
157 | 				} else {
158 | 					d.RetryQuery(resp)
159 | 					continue
160 | 				}
161 | 
162 | 			} else {
163 | 				if d.RetryCounter[domain+strType] <= d.MaxRetries {
164 | 					d.ResolveEngine.Query(d.ResolveEngineCtx, resolve.QueryMsg(domain, resp.Question[0].Qtype), d.AnsChannel)
165 | 					d.RetryCounter[domain+strType]++
166 | 					continue
167 | 				} else {
168 | 					d.SubChannel <- []string{domain}
169 | 				}
170 | 
171 | 			}
172 | 		default:
173 | 			continue
174 | 			//noted d.QueryCounter--
175 | 
176 | 		}
177 | 
178 | 		// if d.QueryCounter%1 == 0 {
179 | 		// 	fmt.Printf("\rQueryRemaining %d", d.QueryCounter)
180 | 		// }
181 | 
182 | 		// if d.QueryCounter <= 0 {
183 | 		// 	return
184 | 		// }
185 | 	}
186 | }
187 | 
188 | func (d *DNSFactory) StopResolveEngine() {
189 | 	defer d.ResolveEngineCancel()
190 | 	defer d.ResolveEngine.Stop()
191 | 	defer close(d.QueryPool)
192 | 	d.QueryCounter = -1
193 | }
194 | 
195 | func (d *DNSFactory) queryPoolGetRecord(resp *miekgdns.Msg) []string {
196 | 	var result []string
197 | 	for _, record := range resp.Answer {
198 | 		if t, ok := record.(*miekgdns.NS); ok {
199 | 			result = append(result, utils.NSparse(t.String()))
200 | 		}
201 | 		if t, ok := record.(*miekgdns.A); ok {
202 | 			result = append(result, t.A.String())
203 | 		}
204 | 		if t, ok := record.(*miekgdns.CNAME); ok {
205 | 			result = append(result, utils.CNAMEparse(t.String()))
206 | 		}
207 | 	}
208 | 	return result
209 | }
210 | 
211 | func (d *DNSFactory) GreedyQuery(domain string, queryType string) []string {
212 | 	resultsPool := make([]string, 0)
213 | 
214 | 	switch queryType {
215 | 	case "NS":
216 | 		for _, resolver := range d.BaseResolvers {
217 | 			tmp, _ := d.getRecordsWithCustomNS(domain, utils.ValidateNSFmt(resolver), "NS")
218 | 			resultsPool = append(resultsPool, tmp...)
219 | 		}
220 | 	case "A":
221 | 		for _, resolver := range d.BaseResolvers {
222 | 			tmp, err := d.getRecordsWithCustomNS(domain, utils.ValidateNSFmt(resolver), "A")
223 | 			resultsPool = append(resultsPool, tmp...)
224 | 			if err == nil {
225 | 				break
226 | 			}
227 | 		}
228 | 
229 | 	case "CNAME":
230 | 		for _, resolver := range d.BaseResolvers {
231 | 			tmp, err := d.getRecordsWithCustomNS(domain, utils.ValidateNSFmt(resolver), "CNAME")
232 | 			resultsPool = append(resultsPool, tmp...)
233 | 			if err == nil {
234 | 				break
235 | 			}
236 | 		}
237 | 
238 | 	}
239 | 	return utils.RemoveDuplicates(resultsPool)
240 | 
241 | }
242 | 
243 | func (d *DNSFactory) makeQueryHeader(domain, resolver string, queryType uint16, retries int) (*miekgdns.Msg, error) {
244 | 	msg := new(miekgdns.Msg)
245 | 
246 | 	msg.Id = miekgdns.Id()
247 | 	msg.RecursionDesired = true
248 | 	msg.Question = make([]miekgdns.Question, 1)
249 | 	msg.Question[0] = miekgdns.Question{
250 | 		Name:   miekgdns.Fqdn(domain),
251 | 		Qtype:  queryType,
252 | 		Qclass: miekgdns.ClassINET,
253 | 	}
254 | 
255 | 	var err error
256 | 	var answer *miekgdns.Msg
257 | 
258 | 	for i := 0; i <= retries; i++ {
259 | 		answer, err = miekgdns.Exchange(msg, resolver)
260 | 		if err != nil {
261 | 			continue
262 | 		}
263 | 		// In case we got some error from the server, return.
264 | 		if answer != nil && answer.Rcode != miekgdns.RcodeSuccess {
265 | 			return nil, errors.New(miekgdns.RcodeToString[answer.Rcode])
266 | 		}
267 | 		return answer, err
268 | 	}
269 | 
270 | 	return answer, err
271 | }
272 | 
273 | func (d *DNSFactory) getRecordsWithCustomNS(domain, resolver, queryType string) ([]string, error) {
274 | 	var result []string
275 | 
276 | 	answer, err := d.makeQueryHeader(domain, resolver, d.TypeMap[queryType], d.MaxRetries)
277 | 	if err != nil {
278 | 		return result, err
279 | 	}
280 | 	for _, record := range answer.Answer {
281 | 		switch queryType {
282 | 		case "NS":
283 | 			if t, ok := record.(*miekgdns.NS); ok {
284 | 				result = append(result, utils.NSparse(t.String()))
285 | 			}
286 | 		case "A":
287 | 			if t, ok := record.(*miekgdns.A); ok {
288 | 				result = append(result, t.A.String())
289 | 			}
290 | 		case "CNAME":
291 | 			if t, ok := record.(*miekgdns.CNAME); ok {
292 | 				result = append(result, utils.CNAMEparse(t.String()))
293 | 			}
294 | 		}
295 | 	}
296 | 	return result, err
297 | }
298 | 


--------------------------------------------------------------------------------