├── img ├── banner.png └── elasticsearch.png ├── main.go ├── pkg ├── sources │ ├── sources.go │ ├── anubis.go │ ├── dnsrepo.go │ ├── cebaidu.go │ ├── threatminer.go │ ├── alienvault.go │ ├── threatcrowd.go │ ├── bevigil.go │ ├── subdomaincenter.go │ ├── chaos.go │ ├── threatbook.go │ ├── shodan.go │ ├── dnsdumpster.go │ ├── fullhunt.go │ ├── whoisxmlapi.go │ ├── shrewdeye.go │ ├── c99.go │ ├── dnsarchive.go │ ├── builtwith.go │ ├── virustotal.go │ ├── bufferover.go │ ├── digitalyama.go │ ├── myssl.go │ ├── chinaz.go │ ├── zoomeyeapi.go │ ├── racent.go │ ├── certspotter.go │ ├── hackertarget.go │ ├── reconcloud.go │ ├── digicert.go │ ├── waybackarchive.go │ ├── abuseipdb.go │ ├── rapiddns.go │ ├── censys.go │ ├── driftnet.go │ ├── rsecloud.go │ ├── dnsgrep.go │ ├── windvane.go │ ├── pugrecon.go │ ├── fofa.go │ ├── robtex.go │ ├── urlscan.go │ ├── quake.go │ └── hunter.go ├── llm │ ├── types.go │ ├── llm_inference.py │ ├── validator.go │ ├── downloader.go │ └── model.go ├── config │ └── paths.go ├── elastic │ └── elastic.go ├── session │ └── session.go └── active │ ├── wordlist.go │ ├── cleaner.go │ ├── deep.go │ ├── mksub.go │ └── resolvers.go ├── cmd ├── version.go ├── update.go └── track.go ├── go.mod ├── LICENSE ├── config └── config.yaml └── go.sum /img/banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samogod/samoscout/HEAD/img/banner.png -------------------------------------------------------------------------------- /img/elasticsearch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samogod/samoscout/HEAD/img/elasticsearch.png -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/samogod/samoscout/cmd" 5 | ) 6 | 7 | func main() { 8 | cmd.Execute() 9 | } 10 | -------------------------------------------------------------------------------- /pkg/sources/sources.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "github.com/samogod/samoscout/pkg/session" 6 | ) 7 | 8 | 9 | type Result struct { 10 | Type string 11 | Source string 12 | Value string 13 | Error error 14 | } 15 | 16 | 17 | type Source interface { 18 | 19 | 20 | Run(ctx context.Context, domain string, s *session.Session) <-chan Result 21 | 22 | 23 | Name() string 24 | } 25 | -------------------------------------------------------------------------------- /pkg/llm/types.go: -------------------------------------------------------------------------------- 1 | package llm 2 | 3 | type Config struct { 4 | NumPredictions int 5 | MaxRecursion int 6 | MaxTokens int 7 | Temperature float32 8 | ResolutionThreads int 9 | Device string 10 | OutputDir string 11 | Verbose bool 12 | } 13 | 14 | type ModelConfig struct { 15 | BlockSize int `json:"block_size"` 16 | VocabSize int `json:"vocab_size"` 17 | NLayer int `json:"n_layer"` 18 | NHead int `json:"n_head"` 19 | NEmbd int `json:"n_embd"` 20 | Dropout float32 `json:"dropout"` 21 | Bias bool `json:"bias"` 22 | } 23 | 24 | -------------------------------------------------------------------------------- /cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | 6 | "github.com/fatih/color" 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | const ( 11 | Version = "1.0.0" 12 | BuildDate = "2025-10-30" 13 | Author = "samogod" 14 | ) 15 | 16 | var versionCmd = &cobra.Command{ 17 | Use: "version", 18 | Short: "Show version information", 19 | Long: "Display version, build date, and author information for samoscout", 20 | Run: func(cmd *cobra.Command, args []string) { 21 | printVersionInfo() 22 | }, 23 | } 24 | 25 | func printVersionInfo() { 26 | color.Green("Current Version: %s", Version) 27 | fmt.Println() 28 | } 29 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/samogod/samoscout 2 | 3 | go 1.21 4 | 5 | require ( 6 | github.com/elastic/go-elasticsearch/v8 v8.13.0 7 | github.com/fatih/color v1.16.0 8 | github.com/lib/pq v1.10.9 9 | github.com/sirupsen/logrus v1.9.3 10 | github.com/spf13/cobra v1.8.0 11 | gopkg.in/yaml.v3 v3.0.1 12 | ) 13 | 14 | require ( 15 | github.com/elastic/elastic-transport-go/v8 v8.5.0 // indirect 16 | github.com/go-logr/logr v1.3.0 // indirect 17 | github.com/go-logr/stdr v1.2.2 // indirect 18 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 19 | github.com/mattn/go-colorable v0.1.13 // indirect 20 | github.com/mattn/go-isatty v0.0.20 // indirect 21 | github.com/spf13/pflag v1.0.5 // indirect 22 | go.opentelemetry.io/otel v1.21.0 // indirect 23 | go.opentelemetry.io/otel/metric v1.21.0 // indirect 24 | go.opentelemetry.io/otel/trace v1.21.0 // indirect 25 | golang.org/x/sys v0.14.0 // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /cmd/update.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/samogod/samoscout/pkg/update" 8 | 9 | "github.com/fatih/color" 10 | "github.com/spf13/cobra" 11 | ) 12 | 13 | var updateVerbose bool 14 | 15 | var updateCmd = &cobra.Command{ 16 | Use: "update", 17 | Short: "Update samoscout to the latest version", 18 | Long: `Update samoscout to the latest version from GitHub releases. 19 | This command will: 20 | - Check for the latest release on GitHub 21 | - Download the appropriate binary for your platform 22 | - Replace the current binary with the new version`, 23 | Example: ` samoscout update 24 | samoscout update -v`, 25 | Run: runUpdate, 26 | } 27 | 28 | func init() { 29 | updateCmd.Flags().BoolVarP(&updateVerbose, "verbose", "v", false, "enable verbose output during update") 30 | } 31 | 32 | func runUpdate(cmd *cobra.Command, args []string) { 33 | fmt.Println() 34 | 35 | if err := update.CheckAndUpdate("v"+Version, updateVerbose); err != nil { 36 | color.Red("Update failed: %v", err) 37 | os.Exit(1) 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 samet g. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /config/config.yaml: -------------------------------------------------------------------------------- 1 | # samoscout configuration file 2 | 3 | api_keys: 4 | censys: "" 5 | virustotal: "" 6 | bevigil: "" 7 | bufferover: "" 8 | builtwith: "" 9 | c99: "" 10 | certspotter: "" 11 | chaos: "" 12 | chinaz: "" 13 | cloudflare: "" 14 | digitalyama: "" 15 | dnsdb: "" 16 | dnsdumpster: "" 17 | dnsrepo: "" 18 | dnsarchive: "" 19 | driftnet: "" 20 | fofa: "" 21 | fullhunt: "" 22 | gitlab: "" 23 | hunter: "" 24 | jsmon: "" 25 | netlas: "" 26 | pugrecon: "" 27 | quake: "" 28 | redhuntlabs: "" 29 | robtex: "" 30 | rsecloud: "" 31 | securitytrails: "" 32 | shodan: "" 33 | subdomaincenter: "" 34 | threatbook: "" 35 | urlscan: "" 36 | whoisxmlapi: "" 37 | windvane: "" 38 | zoomeyeapi: "" 39 | 40 | default_settings: 41 | timeout: 10 42 | 43 | active_enumeration: 44 | enabled: false 45 | dsieve_top: 50 46 | dsieve_factor: 4 47 | output_dir: ".samoscout_active" 48 | 49 | llm_enumeration: 50 | enabled: false 51 | device: "auto" 52 | num_predictions: 500 53 | max_recursion: 5 54 | max_tokens: 10 55 | temperature: 0.0 56 | run_after_passive: true 57 | run_after_active: false 58 | 59 | database: 60 | enabled: false 61 | host: "localhost" 62 | port: 5432 63 | user: "postgres" 64 | password: "postgres" 65 | 66 | elasticsearch: 67 | enabled: true 68 | url: "http://127.0.0.1:9200" 69 | username: "elastic" 70 | password: "" 71 | index: "samoscout_httpx" -------------------------------------------------------------------------------- /pkg/sources/anubis.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type Anubis struct{} 13 | 14 | func (a *Anubis) Name() string { 15 | return "anubis" 16 | } 17 | 18 | func (a *Anubis) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 19 | results := make(chan Result) 20 | 21 | go func() { 22 | defer close(results) 23 | 24 | url := fmt.Sprintf("https://anubisdb.com/anubis/subdomains/%s", domain) 25 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 26 | if err != nil { 27 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 28 | return 29 | } 30 | 31 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 32 | req.Header.Set("Accept", "application/json") 33 | 34 | resp, err := s.Client.Do(req) 35 | if err != nil { 36 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 37 | return 38 | } 39 | defer resp.Body.Close() 40 | 41 | if resp.StatusCode != http.StatusOK { 42 | results <- Result{Source: a.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 43 | return 44 | } 45 | 46 | var subdomains []string 47 | if err := json.NewDecoder(resp.Body).Decode(&subdomains); err != nil { 48 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 49 | return 50 | } 51 | 52 | seen := make(map[string]bool) 53 | for _, subdomain := range subdomains { 54 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 55 | 56 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 57 | seen[hostname] = true 58 | 59 | select { 60 | case results <- Result{Source: a.Name(), Value: hostname, Type: "subdomain"}: 61 | case <-ctx.Done(): 62 | return 63 | } 64 | } 65 | } 66 | }() 67 | 68 | return results 69 | } 70 | -------------------------------------------------------------------------------- /pkg/sources/dnsrepo.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | type DNSRepo struct{} 14 | 15 | type dnsRepoResponse []struct { 16 | Domain string `json:"domain"` 17 | } 18 | 19 | func (d *DNSRepo) Name() string { 20 | return "dnsrepo" 21 | } 22 | 23 | func (d *DNSRepo) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 24 | results := make(chan Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | if s.Keys.DNSRepo == "" { 30 | return 31 | } 32 | 33 | apiURL := fmt.Sprintf("https://dnsrepo.noc.org/api/?apikey=%s&search=%s", s.Keys.DNSRepo, domain) 34 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 35 | if err != nil { 36 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 37 | return 38 | } 39 | 40 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36") 41 | req.Header.Set("Accept", "application/json") 42 | 43 | resp, err := s.Client.Do(req) 44 | if err != nil { 45 | results <- Result{Source: d.Name(), Error: fmt.Errorf("request failed: %w", err)} 46 | return 47 | } 48 | defer resp.Body.Close() 49 | 50 | if resp.StatusCode != http.StatusOK { 51 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 52 | return 53 | } 54 | 55 | responseData, err := io.ReadAll(resp.Body) 56 | if err != nil { 57 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to read response: %w", err)} 58 | return 59 | } 60 | 61 | var result dnsRepoResponse 62 | if err := json.Unmarshal(responseData, &result); err != nil { 63 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 64 | return 65 | } 66 | 67 | seen := make(map[string]bool) 68 | for _, sub := range result { 69 | hostname := strings.TrimSpace(strings.ToLower(strings.TrimSuffix(sub.Domain, "."))) 70 | 71 | if hostname != "" && !seen[hostname] { 72 | seen[hostname] = true 73 | 74 | select { 75 | case results <- Result{Source: d.Name(), Value: hostname, Type: "subdomain"}: 76 | case <-ctx.Done(): 77 | return 78 | } 79 | } 80 | } 81 | }() 82 | 83 | return results 84 | } 85 | -------------------------------------------------------------------------------- /pkg/sources/cebaidu.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type Cebaidu struct{} 13 | 14 | type domain struct { 15 | Domain string `json:"domain"` 16 | } 17 | 18 | type cebaiduResponse struct { 19 | Code int64 `json:"code"` 20 | Message string `json:"message"` 21 | Data []domain `json:"data"` 22 | } 23 | 24 | func (c *Cebaidu) Name() string { 25 | return "cebaidu" 26 | } 27 | 28 | func (c *Cebaidu) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 29 | results := make(chan Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | url := fmt.Sprintf("https://ce.baidu.com/index/getRelatedSites?site_address=%s", domain) 35 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 36 | if err != nil { 37 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 38 | return 39 | } 40 | 41 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 42 | req.Header.Set("Accept", "application/json") 43 | 44 | resp, err := s.Client.Do(req) 45 | if err != nil { 46 | results <- Result{Source: c.Name(), Error: fmt.Errorf("API request failed: %w", err)} 47 | return 48 | } 49 | 50 | if resp.StatusCode != http.StatusOK { 51 | resp.Body.Close() 52 | results <- Result{Source: c.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 53 | return 54 | } 55 | 56 | var response cebaiduResponse 57 | err = json.NewDecoder(resp.Body).Decode(&response) 58 | resp.Body.Close() 59 | if err != nil { 60 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 61 | return 62 | } 63 | 64 | if response.Code > 0 { 65 | results <- Result{Source: c.Name(), Error: fmt.Errorf("API error: %d, %s", response.Code, response.Message)} 66 | return 67 | } 68 | 69 | seen := make(map[string]bool) 70 | for _, domainResult := range response.Data { 71 | hostname := strings.TrimSpace(strings.ToLower(domainResult.Domain)) 72 | 73 | if hostname != "" && !seen[hostname] { 74 | seen[hostname] = true 75 | 76 | select { 77 | case results <- Result{Source: c.Name(), Value: hostname, Type: "subdomain"}: 78 | case <-ctx.Done(): 79 | return 80 | } 81 | } 82 | } 83 | }() 84 | 85 | return results 86 | } 87 | -------------------------------------------------------------------------------- /pkg/sources/threatminer.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type ThreatMiner struct{} 14 | 15 | 16 | type ThreatMinerResponse struct { 17 | StatusCode string `json:"status_code"` 18 | StatusMessage string `json:"status_message"` 19 | Results []string `json:"results"` 20 | } 21 | 22 | 23 | func (t *ThreatMiner) Name() string { 24 | return "threatminer" 25 | } 26 | 27 | 28 | func (t *ThreatMiner) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 29 | results := make(chan Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | seen := make(map[string]bool) 35 | url := fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain) 36 | 37 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 38 | if err != nil { 39 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 40 | return 41 | } 42 | 43 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 44 | 45 | resp, err := session.Client.Do(req) 46 | if err != nil { 47 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API request failed: %w", err)} 48 | return 49 | } 50 | defer resp.Body.Close() 51 | 52 | if resp.StatusCode != http.StatusOK { 53 | results <- Result{Source: t.Name(), Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} 54 | return 55 | } 56 | 57 | var response ThreatMinerResponse 58 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 59 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 60 | return 61 | } 62 | 63 | 64 | if response.StatusCode != "200" { 65 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API returned status code: %s, message: %s", response.StatusCode, response.StatusMessage)} 66 | return 67 | } 68 | 69 | 70 | for _, subdomain := range response.Results { 71 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 72 | 73 | if hostname != "" && !seen[hostname] { 74 | seen[hostname] = true 75 | 76 | select { 77 | case results <- Result{Source: t.Name(), Value: hostname, Type: "subdomain"}: 78 | case <-ctx.Done(): 79 | return 80 | } 81 | } 82 | } 83 | }() 84 | 85 | return results 86 | } 87 | -------------------------------------------------------------------------------- /pkg/sources/alienvault.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type AlienVault struct{} 13 | 14 | type AlienVaultResponse struct { 15 | Detail string `json:"detail"` 16 | Error string `json:"error"` 17 | PassiveDNS []struct { 18 | Hostname string `json:"hostname"` 19 | } `json:"passive_dns"` 20 | } 21 | 22 | func (a *AlienVault) Name() string { 23 | return "alienvault" 24 | } 25 | 26 | func (a *AlienVault) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 27 | results := make(chan Result) 28 | 29 | go func() { 30 | defer close(results) 31 | 32 | url := fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain) 33 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 34 | if err != nil { 35 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 36 | return 37 | } 38 | 39 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 40 | req.Header.Set("Accept", "application/json") 41 | 42 | resp, err := s.Client.Do(req) 43 | if err != nil { 44 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 45 | return 46 | } 47 | defer resp.Body.Close() 48 | 49 | if resp.StatusCode != http.StatusOK { 50 | results <- Result{Source: a.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 51 | return 52 | } 53 | 54 | var alienvaultResp AlienVaultResponse 55 | if err := json.NewDecoder(resp.Body).Decode(&alienvaultResp); err != nil { 56 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 57 | return 58 | } 59 | 60 | if alienvaultResp.Error != "" { 61 | results <- Result{Source: a.Name(), Error: fmt.Errorf("%s, %s", alienvaultResp.Detail, alienvaultResp.Error)} 62 | return 63 | } 64 | 65 | seen := make(map[string]bool) 66 | for _, record := range alienvaultResp.PassiveDNS { 67 | hostname := strings.TrimSpace(strings.ToLower(record.Hostname)) 68 | 69 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 70 | seen[hostname] = true 71 | 72 | select { 73 | case results <- Result{Source: a.Name(), Value: hostname, Type: "subdomain"}: 74 | case <-ctx.Done(): 75 | return 76 | } 77 | } 78 | } 79 | }() 80 | 81 | return results 82 | } 83 | -------------------------------------------------------------------------------- /pkg/config/paths.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "path/filepath" 7 | "runtime" 8 | ) 9 | 10 | // windows: C:\Users\{user}\AppData\Roaming\samoscout 11 | // macOS: ~/Library/Application Support/samoscout 12 | // linux: ~/.config/samoscout 13 | func GetConfigDir() string { 14 | var configDir string 15 | 16 | switch runtime.GOOS { 17 | case "windows": 18 | appData := os.Getenv("APPDATA") 19 | if appData == "" { 20 | home, err := os.UserHomeDir() 21 | if err != nil { 22 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 23 | } 24 | appData = filepath.Join(home, "AppData", "Roaming") 25 | } 26 | configDir = filepath.Join(appData, "samoscout") 27 | 28 | case "darwin": 29 | home, err := os.UserHomeDir() 30 | if err != nil { 31 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 32 | } 33 | configDir = filepath.Join(home, "Library", "Application Support", "samoscout") 34 | 35 | default: 36 | xdgConfig := os.Getenv("XDG_CONFIG_HOME") 37 | if xdgConfig == "" { 38 | home, err := os.UserHomeDir() 39 | if err != nil { 40 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 41 | } 42 | xdgConfig = filepath.Join(home, ".config") 43 | } 44 | configDir = filepath.Join(xdgConfig, "samoscout") 45 | } 46 | 47 | return configDir 48 | } 49 | 50 | // windows: C:\Users\{user}\AppData\Local\samoscout 51 | // macOS: ~/Library/Caches/samoscout 52 | // linux: ~/.cache/samoscout 53 | func GetCacheDir() string { 54 | var cacheDir string 55 | 56 | switch runtime.GOOS { 57 | case "windows": 58 | localAppData := os.Getenv("LOCALAPPDATA") 59 | if localAppData == "" { 60 | home, err := os.UserHomeDir() 61 | if err != nil { 62 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 63 | } 64 | localAppData = filepath.Join(home, "AppData", "Local") 65 | } 66 | cacheDir = filepath.Join(localAppData, "samoscout") 67 | 68 | case "darwin": 69 | home, err := os.UserHomeDir() 70 | if err != nil { 71 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 72 | } 73 | cacheDir = filepath.Join(home, "Library", "Caches", "samoscout") 74 | 75 | default: 76 | xdgCache := os.Getenv("XDG_CACHE_HOME") 77 | if xdgCache == "" { 78 | home, err := os.UserHomeDir() 79 | if err != nil { 80 | panic(fmt.Sprintf("failed to get user home directory: %v", err)) 81 | } 82 | xdgCache = filepath.Join(home, ".cache") 83 | } 84 | cacheDir = filepath.Join(xdgCache, "samoscout") 85 | } 86 | 87 | return cacheDir 88 | } 89 | 90 | func GetDefaultConfigPath() string { 91 | return filepath.Join(GetConfigDir(), "config.yaml") 92 | } 93 | 94 | func GetLLMCacheDir() string { 95 | return filepath.Join(GetCacheDir(), "llm") 96 | } 97 | 98 | -------------------------------------------------------------------------------- /pkg/sources/threatcrowd.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type ThreatCrowd struct{} 15 | 16 | 17 | type ThreatCrowdResponse struct { 18 | ResponseCode string `json:"response_code"` 19 | Subdomains []string `json:"subdomains"` 20 | Undercount string `json:"undercount"` 21 | } 22 | 23 | 24 | func (t *ThreatCrowd) Name() string { 25 | return "threatcrowd" 26 | } 27 | 28 | 29 | func (t *ThreatCrowd) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 30 | results := make(chan Result) 31 | 32 | go func() { 33 | defer close(results) 34 | 35 | seen := make(map[string]bool) 36 | url := fmt.Sprintf("http://ci-www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain) 37 | 38 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 39 | if err != nil { 40 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 41 | return 42 | } 43 | 44 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 45 | 46 | resp, err := session.Client.Do(req) 47 | if err != nil { 48 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API request failed: %w", err)} 49 | return 50 | } 51 | defer resp.Body.Close() 52 | 53 | if resp.StatusCode != http.StatusOK { 54 | results <- Result{Source: t.Name(), Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} 55 | return 56 | } 57 | 58 | body, err := io.ReadAll(resp.Body) 59 | if err != nil { 60 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 61 | return 62 | } 63 | 64 | var response ThreatCrowdResponse 65 | if err := json.Unmarshal(body, &response); err != nil { 66 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 67 | return 68 | } 69 | 70 | 71 | if response.ResponseCode != "1" { 72 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API returned response code: %s", response.ResponseCode)} 73 | return 74 | } 75 | 76 | 77 | for _, subdomain := range response.Subdomains { 78 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 79 | 80 | if hostname != "" && !seen[hostname] { 81 | seen[hostname] = true 82 | 83 | select { 84 | case results <- Result{Source: t.Name(), Value: hostname, Type: "subdomain"}: 85 | case <-ctx.Done(): 86 | return 87 | } 88 | } 89 | } 90 | }() 91 | 92 | return results 93 | } 94 | -------------------------------------------------------------------------------- /pkg/llm/llm_inference.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | import sys 3 | import os 4 | import json 5 | import torch 6 | from huggingface_hub import hf_hub_download 7 | from transformers import PreTrainedTokenizerFast 8 | 9 | MODEL_REPO = "HadrianSecurity/subwiz" 10 | 11 | sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) 12 | from gpt_model import GPT 13 | 14 | class LLMInference: 15 | def __init__(self): 16 | model_path = hf_hub_download(repo_id=MODEL_REPO, filename='model.pt') 17 | tokenizer_path = hf_hub_download(repo_id=MODEL_REPO, filename='tokenizer.json') 18 | 19 | self.model = GPT.from_checkpoint(model_path, device='cpu', tokenizer_path=tokenizer_path) 20 | self.model.eval() 21 | self.tokenizer = PreTrainedTokenizerFast( 22 | tokenizer_file=tokenizer_path, 23 | clean_up_tokenization_spaces=True 24 | ) 25 | 26 | def predict(self, subdomains, apex, num_predictions=500, max_tokens=10, temperature=0.0, blocked=None): 27 | blocked = blocked or [] 28 | 29 | tokenizer_input = ",".join(sorted(subdomains)) + "[DELIM]" 30 | x = self.tokenizer.encode(tokenizer_input) 31 | x = [1] * (self.model.config.block_size - len(x)) + x 32 | x = torch.tensor(x) 33 | 34 | blocked_outputs = set(blocked) 35 | 36 | predictions = self.model.generate( 37 | x, 38 | max_new_tokens=max_tokens, 39 | topn=num_predictions, 40 | temperature=temperature, 41 | blocked_outputs=blocked_outputs, 42 | ) 43 | predictions = predictions.int().tolist() 44 | 45 | results = [] 46 | for pred in predictions: 47 | decoded = self.tokenizer.decode(pred).replace(" ", "").rsplit("[DELIM]", 1) 48 | if len(decoded) > 1: 49 | subdomain = decoded[1] 50 | full_domain = subdomain + "." + apex 51 | results.append(full_domain) 52 | 53 | return results 54 | 55 | if __name__ == "__main__": 56 | try: 57 | input_json = sys.stdin.read() 58 | input_data = json.loads(input_json) 59 | 60 | llm = LLMInference() 61 | predictions = llm.predict( 62 | subdomains=input_data.get("subdomains", []), 63 | apex=input_data.get("apex", ""), 64 | num_predictions=input_data.get("num_predictions", 500), 65 | max_tokens=input_data.get("max_tokens", 10), 66 | temperature=input_data.get("temperature", 0.0), 67 | blocked=input_data.get("blocked", []) 68 | ) 69 | 70 | print(json.dumps({"predictions": predictions})) 71 | except Exception as e: 72 | print(json.dumps({"error": str(e)})) 73 | sys.exit(1) 74 | 75 | -------------------------------------------------------------------------------- /pkg/sources/bevigil.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type BeVigil struct{} 13 | 14 | type BeVigilResponse struct { 15 | Domain string `json:"domain"` 16 | Subdomains []string `json:"subdomains"` 17 | } 18 | 19 | func (b *BeVigil) Name() string { 20 | return "bevigil" 21 | } 22 | 23 | func (b *BeVigil) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 24 | results := make(chan Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | if s.Keys.BeVigil == "" { 30 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BeVigil API key not configured")} 31 | return 32 | } 33 | 34 | url := fmt.Sprintf("https://osint.bevigil.com/api/%s/subdomains/", domain) 35 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 36 | if err != nil { 37 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 38 | return 39 | } 40 | 41 | req.Header.Set("X-Access-Token", s.Keys.BeVigil) 42 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 43 | req.Header.Set("Accept", "application/json") 44 | 45 | resp, err := s.Client.Do(req) 46 | if err != nil { 47 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 48 | return 49 | } 50 | defer resp.Body.Close() 51 | 52 | if resp.StatusCode == 401 { 53 | results <- Result{Source: b.Name(), Error: fmt.Errorf("invalid BeVigil API key")} 54 | return 55 | } 56 | 57 | if resp.StatusCode == 429 { 58 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BeVigil rate limit exceeded")} 59 | return 60 | } 61 | 62 | if resp.StatusCode != http.StatusOK { 63 | results <- Result{Source: b.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 64 | return 65 | } 66 | 67 | var bevigilResp BeVigilResponse 68 | if err := json.NewDecoder(resp.Body).Decode(&bevigilResp); err != nil { 69 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 70 | return 71 | } 72 | 73 | seen := make(map[string]bool) 74 | for _, subdomain := range bevigilResp.Subdomains { 75 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 76 | 77 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 78 | seen[hostname] = true 79 | 80 | select { 81 | case results <- Result{Source: b.Name(), Value: hostname, Type: "subdomain"}: 82 | case <-ctx.Done(): 83 | return 84 | } 85 | } 86 | } 87 | }() 88 | 89 | return results 90 | } 91 | -------------------------------------------------------------------------------- /pkg/sources/subdomaincenter.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type SubdomainCenter struct{} 13 | 14 | type SubdomainCenterResponse []string 15 | 16 | func (s *SubdomainCenter) Name() string { 17 | return "subdomaincenter" 18 | } 19 | 20 | func (s *SubdomainCenter) Run(ctx context.Context, domain string, sess *session.Session) <-chan Result { 21 | results := make(chan Result) 22 | 23 | go func() { 24 | defer close(results) 25 | 26 | seen := make(map[string]bool) 27 | 28 | var url string 29 | if sess.Keys.SubdomainCenter != "" { 30 | url = fmt.Sprintf("https://api.subdomain.center/beta/?domain=%s&engine=cuttlefish&auth=%s", domain, sess.Keys.SubdomainCenter) 31 | } else { 32 | url = fmt.Sprintf("https://api.subdomain.center/?domain=%s&engine=cuttlefish", domain) 33 | } 34 | 35 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 36 | if err != nil { 37 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 38 | return 39 | } 40 | 41 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 42 | req.Header.Set("Accept", "application/json") 43 | 44 | resp, err := sess.Client.Do(req) 45 | if err != nil { 46 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 47 | return 48 | } 49 | defer resp.Body.Close() 50 | 51 | if resp.StatusCode == 401 || resp.StatusCode == 403 { 52 | results <- Result{Source: s.Name(), Error: fmt.Errorf("authentication failed - invalid API key")} 53 | return 54 | } 55 | 56 | if resp.StatusCode == 429 { 57 | results <- Result{Source: s.Name(), Error: fmt.Errorf("rate limit exceeded - consider using an API key")} 58 | return 59 | } 60 | 61 | if resp.StatusCode != http.StatusOK { 62 | results <- Result{Source: s.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 63 | return 64 | } 65 | 66 | var response SubdomainCenterResponse 67 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 68 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 69 | return 70 | } 71 | 72 | // Process subdomains 73 | for _, subdomain := range response { 74 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 75 | 76 | if hostname != "" && !seen[hostname] { 77 | seen[hostname] = true 78 | 79 | select { 80 | case results <- Result{Source: s.Name(), Value: hostname, Type: "subdomain"}: 81 | case <-ctx.Done(): 82 | return 83 | } 84 | } 85 | } 86 | }() 87 | 88 | return results 89 | } 90 | -------------------------------------------------------------------------------- /pkg/elastic/elastic.go: -------------------------------------------------------------------------------- 1 | package elastic 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "os" 9 | "strings" 10 | 11 | es8 "github.com/elastic/go-elasticsearch/v8" 12 | "github.com/elastic/go-elasticsearch/v8/esutil" 13 | ) 14 | 15 | type Config struct { 16 | URL string 17 | Username string 18 | Password string 19 | Index string 20 | } 21 | 22 | type Client struct { 23 | es *es8.Client 24 | index string 25 | } 26 | 27 | func New(cfg Config) (*Client, error) { 28 | if cfg.URL == "" { 29 | return nil, errors.New("elasticsearch URL is required") 30 | } 31 | index := cfg.Index 32 | if strings.TrimSpace(index) == "" { 33 | index = "samoscout_httpx" 34 | } 35 | 36 | es, err := es8.NewClient(es8.Config{ 37 | Addresses: []string{cfg.URL}, 38 | Username: cfg.Username, 39 | Password: cfg.Password, 40 | }) 41 | if err != nil { 42 | return nil, fmt.Errorf("failed to create elasticsearch client: %w", err) 43 | } 44 | 45 | // Lightweight ping 46 | if _, err := es.Info(); err != nil { 47 | return nil, fmt.Errorf("failed to connect to elasticsearch: %w", err) 48 | } 49 | 50 | return &Client{es: es, index: index}, nil 51 | } 52 | 53 | func (c *Client) IndexJSONLinesFile(ctx context.Context, filename string) error { 54 | f, err := os.Open(filename) 55 | if err != nil { 56 | return fmt.Errorf("failed to open jsonl file: %w", err) 57 | } 58 | defer f.Close() 59 | 60 | bi, err := esutil.NewBulkIndexer(esutil.BulkIndexerConfig{ 61 | Client: c.es, 62 | Index: c.index, 63 | NumWorkers: 4, 64 | }) 65 | if err != nil { 66 | return fmt.Errorf("failed to create bulk indexer: %w", err) 67 | } 68 | 69 | scanner := bufio.NewScanner(f) 70 | buf := make([]byte, 0, 1024*1024) 71 | scanner.Buffer(buf, 8*1024*1024) 72 | 73 | for scanner.Scan() { 74 | line := strings.TrimSpace(scanner.Text()) 75 | if line == "" { 76 | continue 77 | } 78 | 79 | item := esutil.BulkIndexerItem{ 80 | Action: "index", 81 | DocumentID: "", 82 | Body: strings.NewReader(line), 83 | OnFailure: func(ctx context.Context, item esutil.BulkIndexerItem, resp esutil.BulkIndexerResponseItem, err error) { 84 | }, 85 | } 86 | if err := bi.Add(ctx, item); err != nil { 87 | return fmt.Errorf("bulk add failed: %w", err) 88 | } 89 | } 90 | if err := scanner.Err(); err != nil { 91 | return fmt.Errorf("scanner error: %w", err) 92 | } 93 | 94 | if err := bi.Close(ctx); err != nil { 95 | return fmt.Errorf("bulk indexer close failed: %w", err) 96 | } 97 | 98 | return nil 99 | } 100 | 101 | 102 | -------------------------------------------------------------------------------- /pkg/llm/validator.go: -------------------------------------------------------------------------------- 1 | package llm 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | var ( 9 | validSubdomainRe = regexp.MustCompile( 10 | `^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?(\.[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)*$`, 11 | ) 12 | 13 | validSubdomainStartRe = regexp.MustCompile( 14 | `^([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?\.)*([A-Za-z0-9]([A-Za-z0-9-]{0,61}[A-Za-z0-9])?)?$`, 15 | ) 16 | ) 17 | 18 | type Validator struct{} 19 | 20 | func NewValidator() *Validator { 21 | return &Validator{} 22 | } 23 | 24 | func (v *Validator) IsValidSubdomain(s string) bool { 25 | s = strings.ToLower(strings.TrimSpace(s)) 26 | 27 | if s == "" || len(s) > 253 { 28 | return false 29 | } 30 | 31 | if strings.HasPrefix(s, ".") || strings.HasSuffix(s, ".") { 32 | return false 33 | } 34 | 35 | if strings.HasPrefix(s, "-") || strings.HasSuffix(s, "-") { 36 | return false 37 | } 38 | 39 | if strings.Contains(s, "..") || strings.Contains(s, "--") { 40 | return false 41 | } 42 | 43 | return validSubdomainRe.MatchString(s) 44 | } 45 | 46 | func (v *Validator) IsValidSubdomainStart(s string) bool { 47 | s = strings.TrimSpace(s) 48 | return validSubdomainStartRe.MatchString(s) 49 | } 50 | 51 | func (v *Validator) ExtractSubdomain(fullDomain, apex string) (string, bool) { 52 | fullDomain = strings.ToLower(strings.TrimSpace(fullDomain)) 53 | apex = strings.ToLower(strings.TrimSpace(apex)) 54 | 55 | if fullDomain == "" || apex == "" { 56 | return "", false 57 | } 58 | 59 | fullDomain = strings.TrimPrefix(fullDomain, "*.") 60 | 61 | if fullDomain == apex { 62 | return "", true 63 | } 64 | 65 | if !strings.HasSuffix(fullDomain, "."+apex) { 66 | return "", false 67 | } 68 | 69 | subdomain := strings.TrimSuffix(fullDomain, "."+apex) 70 | 71 | if !v.IsValidSubdomain(subdomain) { 72 | return "", false 73 | } 74 | 75 | return subdomain, true 76 | } 77 | 78 | func (v *Validator) NormalizeSubdomains(domains []string, apex string) []string { 79 | seen := make(map[string]bool) 80 | result := []string{} 81 | 82 | for _, domain := range domains { 83 | normalized := strings.ToLower(strings.TrimSpace(domain)) 84 | if normalized == "" { 85 | continue 86 | } 87 | 88 | normalized = strings.TrimPrefix(normalized, "*.") 89 | 90 | if !seen[normalized] && strings.HasSuffix(normalized, apex) { 91 | seen[normalized] = true 92 | result = append(result, normalized) 93 | } 94 | } 95 | 96 | return result 97 | } 98 | 99 | func (v *Validator) FilterBlockedDomains(predictions []string, blocked map[string]bool) []string { 100 | result := []string{} 101 | 102 | for _, pred := range predictions { 103 | normalized := strings.ToLower(strings.TrimSpace(pred)) 104 | if !blocked[normalized] { 105 | result = append(result, pred) 106 | } 107 | } 108 | 109 | return result 110 | } 111 | 112 | -------------------------------------------------------------------------------- /pkg/sources/chaos.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type Chaos struct{} 13 | 14 | type ChaosResponse struct { 15 | Domain string `json:"domain"` 16 | Subdomains []string `json:"subdomains"` 17 | Count int `json:"count"` 18 | } 19 | 20 | func (c *Chaos) Name() string { 21 | return "chaos" 22 | } 23 | 24 | func (c *Chaos) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 25 | results := make(chan Result) 26 | 27 | go func() { 28 | defer close(results) 29 | 30 | if s.Keys.Chaos == "" { 31 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Chaos API key not configured")} 32 | return 33 | } 34 | 35 | url := fmt.Sprintf("https://dns.projectdiscovery.io/dns/%s/subdomains", domain) 36 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 37 | if err != nil { 38 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 39 | return 40 | } 41 | 42 | req.Header.Set("Authorization", s.Keys.Chaos) 43 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 44 | req.Header.Set("Accept", "application/json") 45 | 46 | resp, err := s.Client.Do(req) 47 | if err != nil { 48 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 49 | return 50 | } 51 | defer resp.Body.Close() 52 | 53 | if resp.StatusCode == 401 { 54 | results <- Result{Source: c.Name(), Error: fmt.Errorf("invalid Chaos API key")} 55 | return 56 | } 57 | 58 | if resp.StatusCode == 429 { 59 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Chaos rate limit exceeded")} 60 | return 61 | } 62 | 63 | if resp.StatusCode != http.StatusOK { 64 | results <- Result{Source: c.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 65 | return 66 | } 67 | 68 | var chaosResp ChaosResponse 69 | if err := json.NewDecoder(resp.Body).Decode(&chaosResp); err != nil { 70 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 71 | return 72 | } 73 | 74 | seen := make(map[string]bool) 75 | for _, subdomain := range chaosResp.Subdomains { 76 | cleanSub := strings.TrimSpace(strings.ToLower(subdomain)) 77 | 78 | var fullSubdomain string 79 | if strings.Contains(cleanSub, ".") { 80 | fullSubdomain = cleanSub 81 | } else { 82 | fullSubdomain = fmt.Sprintf("%s.%s", cleanSub, domain) 83 | } 84 | 85 | if fullSubdomain != "" && fullSubdomain != domain && strings.HasSuffix(fullSubdomain, "."+domain) && !seen[fullSubdomain] { 86 | seen[fullSubdomain] = true 87 | 88 | select { 89 | case results <- Result{Source: c.Name(), Value: fullSubdomain, Type: "subdomain"}: 90 | case <-ctx.Done(): 91 | return 92 | } 93 | } 94 | } 95 | }() 96 | 97 | return results 98 | } 99 | -------------------------------------------------------------------------------- /pkg/session/session.go: -------------------------------------------------------------------------------- 1 | package session 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net/http" 7 | "github.com/samogod/samoscout/pkg/config" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | var DebugLog func(string, ...interface{}) 13 | 14 | type Session struct { 15 | Client *http.Client 16 | Config *config.Config 17 | Keys config.APIKeys 18 | } 19 | 20 | type LoggingTransport struct { 21 | Transport http.RoundTripper 22 | } 23 | 24 | func (t *LoggingTransport) RoundTrip(req *http.Request) (*http.Response, error) { 25 | if DebugLog != nil { 26 | DebugLog("requesting url: %s", req.URL.String()) 27 | 28 | if len(req.Header) > 0 { 29 | var headers []string 30 | for k, v := range req.Header { 31 | if k != "User-Agent" { 32 | headers = append(headers, fmt.Sprintf("%s: %s", k, strings.Join(v, ", "))) 33 | } 34 | } 35 | if len(headers) > 0 { 36 | DebugLog("request headers: %s", strings.Join(headers, " | ")) 37 | } 38 | } 39 | } 40 | 41 | resp, err := t.Transport.RoundTrip(req) 42 | 43 | if DebugLog != nil { 44 | sourceName := extractSourceName(req.URL.String()) 45 | 46 | if err != nil { 47 | DebugLog("encountered an error with source %s: %v", sourceName, err) 48 | } else { 49 | DebugLog("response for %s: status code %d", req.URL.String(), resp.StatusCode) 50 | 51 | if contentType := resp.Header.Get("Content-Type"); contentType != "" { 52 | DebugLog("response content-type: %s", contentType) 53 | } 54 | 55 | if resp.StatusCode >= 400 { 56 | DebugLog("encountered an error with source %s: unexpected status code %d received from %s", 57 | sourceName, resp.StatusCode, req.URL.String()) 58 | 59 | if resp.Body != nil { 60 | bodyBytes, readErr := io.ReadAll(io.LimitReader(resp.Body, 500)) 61 | if readErr == nil && len(bodyBytes) > 0 { 62 | DebugLog("error response body: %s", string(bodyBytes)) 63 | } 64 | } 65 | } 66 | } 67 | } 68 | 69 | return resp, err 70 | } 71 | 72 | func extractSourceName(url string) string { 73 | parts := strings.Split(url, "://") 74 | if len(parts) > 1 { 75 | domain := strings.Split(parts[1], "/")[0] 76 | domainParts := strings.Split(domain, ".") 77 | if len(domainParts) > 0 { 78 | return domainParts[0] 79 | } 80 | } 81 | 82 | return "unknown" 83 | } 84 | 85 | func New(cfg *config.Config) (*Session, error) { 86 | baseTransport := &http.Transport{ 87 | MaxIdleConns: 100, 88 | MaxIdleConnsPerHost: 100, 89 | IdleConnTimeout: 90 * time.Second, 90 | DisableKeepAlives: false, 91 | } 92 | 93 | var transport http.RoundTripper = baseTransport 94 | if DebugLog != nil { 95 | transport = &LoggingTransport{Transport: baseTransport} 96 | } 97 | 98 | client := &http.Client{ 99 | Timeout: time.Duration(cfg.DefaultSettings.Timeout*3) * time.Second, 100 | Transport: transport, 101 | } 102 | 103 | return &Session{ 104 | Client: client, 105 | Config: cfg, 106 | Keys: cfg.APIKeys, 107 | }, nil 108 | } 109 | -------------------------------------------------------------------------------- /pkg/sources/threatbook.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strconv" 10 | "strings" 11 | ) 12 | 13 | 14 | type ThreatBook struct{} 15 | 16 | 17 | type ThreatBookResponse struct { 18 | ResponseCode int64 `json:"response_code"` 19 | VerboseMsg string `json:"verbose_msg"` 20 | Data struct { 21 | Domain string `json:"domain"` 22 | SubDomains struct { 23 | Total string `json:"total"` 24 | Data []string `json:"data"` 25 | } `json:"sub_domains"` 26 | } `json:"data"` 27 | } 28 | 29 | 30 | func (t *ThreatBook) Name() string { 31 | return "threatbook" 32 | } 33 | 34 | 35 | func (t *ThreatBook) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 36 | results := make(chan Result) 37 | 38 | go func() { 39 | defer close(results) 40 | 41 | 42 | if session.Keys.ThreatBook == "" { 43 | results <- Result{Source: t.Name(), Error: fmt.Errorf("ThreatBook API key not configured")} 44 | return 45 | } 46 | 47 | seen := make(map[string]bool) 48 | url := fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain) 49 | 50 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 51 | if err != nil { 52 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 53 | return 54 | } 55 | 56 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 57 | 58 | resp, err := session.Client.Do(req) 59 | if err != nil { 60 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API request failed: %w", err)} 61 | return 62 | } 63 | defer resp.Body.Close() 64 | 65 | var response ThreatBookResponse 66 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 67 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 68 | return 69 | } 70 | 71 | 72 | if response.ResponseCode != 0 { 73 | results <- Result{Source: t.Name(), Error: fmt.Errorf("API error code %d: %s", response.ResponseCode, response.VerboseMsg)} 74 | return 75 | } 76 | 77 | 78 | total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) 79 | if err != nil { 80 | results <- Result{Source: t.Name(), Error: fmt.Errorf("failed to parse total count: %w", err)} 81 | return 82 | } 83 | 84 | 85 | if total > 0 { 86 | for _, subdomain := range response.Data.SubDomains.Data { 87 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 88 | 89 | if hostname != "" && !seen[hostname] { 90 | seen[hostname] = true 91 | 92 | select { 93 | case results <- Result{Source: t.Name(), Value: hostname, Type: "subdomain"}: 94 | case <-ctx.Done(): 95 | return 96 | } 97 | } 98 | } 99 | } 100 | }() 101 | 102 | return results 103 | } 104 | -------------------------------------------------------------------------------- /pkg/sources/shodan.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type Shodan struct{} 14 | 15 | 16 | type ShodanResponse struct { 17 | Domain string `json:"domain"` 18 | Subdomains []string `json:"subdomains"` 19 | Result int `json:"result"` 20 | Error string `json:"error"` 21 | More bool `json:"more"` 22 | } 23 | 24 | 25 | func (s *Shodan) Name() string { 26 | return "shodan" 27 | } 28 | 29 | 30 | func (s *Shodan) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 31 | results := make(chan Result) 32 | 33 | go func() { 34 | defer close(results) 35 | 36 | 37 | if session.Keys.Shodan == "" { 38 | results <- Result{Source: s.Name(), Error: fmt.Errorf("Shodan API key not configured")} 39 | return 40 | } 41 | 42 | seen := make(map[string]bool) 43 | page := 1 44 | 45 | for { 46 | select { 47 | case <-ctx.Done(): 48 | return 49 | default: 50 | } 51 | 52 | searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s&page=%d", domain, session.Keys.Shodan, page) 53 | 54 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, searchURL, nil) 55 | if err != nil { 56 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 57 | return 58 | } 59 | 60 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 61 | 62 | resp, err := session.Client.Do(req) 63 | if err != nil { 64 | results <- Result{Source: s.Name(), Error: fmt.Errorf("API request failed: %w", err)} 65 | return 66 | } 67 | 68 | var response ShodanResponse 69 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 70 | resp.Body.Close() 71 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 72 | return 73 | } 74 | resp.Body.Close() 75 | 76 | if response.Error != "" { 77 | results <- Result{Source: s.Name(), Error: fmt.Errorf("API error: %s", response.Error)} 78 | return 79 | } 80 | 81 | 82 | for _, subdomain := range response.Subdomains { 83 | 84 | var fullDomain string 85 | if strings.HasSuffix(subdomain, ".") { 86 | fullDomain = subdomain + response.Domain 87 | } else { 88 | fullDomain = subdomain + "." + response.Domain 89 | } 90 | 91 | hostname := strings.TrimSpace(strings.ToLower(fullDomain)) 92 | 93 | if hostname != "" && !seen[hostname] { 94 | seen[hostname] = true 95 | 96 | select { 97 | case results <- Result{Source: s.Name(), Value: hostname, Type: "subdomain"}: 98 | case <-ctx.Done(): 99 | return 100 | } 101 | } 102 | } 103 | 104 | if !response.More { 105 | break 106 | } 107 | page++ 108 | } 109 | }() 110 | 111 | return results 112 | } 113 | -------------------------------------------------------------------------------- /pkg/sources/dnsdumpster.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type DNSDumpster struct{} 14 | 15 | 16 | type DNSDumpsterResponse struct { 17 | A []DNSDumpsterRecord `json:"a"` 18 | Ns []DNSDumpsterRecord `json:"ns"` 19 | } 20 | 21 | type DNSDumpsterRecord struct { 22 | Host string `json:"host"` 23 | } 24 | 25 | 26 | func (d *DNSDumpster) Name() string { 27 | return "dnsdumpster" 28 | } 29 | 30 | 31 | func (d *DNSDumpster) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 32 | results := make(chan Result) 33 | 34 | go func() { 35 | defer close(results) 36 | 37 | 38 | if s.Keys.DNSDumpster == "" { 39 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DNSDumpster API key not configured")} 40 | return 41 | } 42 | 43 | 44 | url := fmt.Sprintf("https://api.dnsdumpster.com/domain/%s", domain) 45 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 46 | if err != nil { 47 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 48 | return 49 | } 50 | 51 | 52 | req.Header.Set("X-API-Key", s.Keys.DNSDumpster) 53 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 54 | req.Header.Set("Accept", "application/json") 55 | 56 | resp, err := s.Client.Do(req) 57 | if err != nil { 58 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 59 | return 60 | } 61 | defer resp.Body.Close() 62 | 63 | if resp.StatusCode == 401 { 64 | results <- Result{Source: d.Name(), Error: fmt.Errorf("invalid DNSDumpster API key")} 65 | return 66 | } 67 | 68 | if resp.StatusCode == 429 { 69 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DNSDumpster rate limit exceeded")} 70 | return 71 | } 72 | 73 | if resp.StatusCode != http.StatusOK { 74 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 75 | return 76 | } 77 | 78 | 79 | var dnsResp DNSDumpsterResponse 80 | if err := json.NewDecoder(resp.Body).Decode(&dnsResp); err != nil { 81 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 82 | return 83 | } 84 | 85 | 86 | allRecords := append(dnsResp.A, dnsResp.Ns...) 87 | seen := make(map[string]bool) 88 | 89 | for _, record := range allRecords { 90 | hostname := strings.TrimSpace(strings.ToLower(record.Host)) 91 | 92 | 93 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 94 | seen[hostname] = true 95 | 96 | select { 97 | case results <- Result{Source: d.Name(), Value: hostname, Type: "subdomain"}: 98 | case <-ctx.Done(): 99 | return 100 | } 101 | } 102 | } 103 | }() 104 | 105 | return results 106 | } 107 | -------------------------------------------------------------------------------- /pkg/llm/downloader.go: -------------------------------------------------------------------------------- 1 | package llm 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "os" 9 | "path/filepath" 10 | 11 | "github.com/samogod/samoscout/pkg/config" 12 | ) 13 | 14 | const ( 15 | HuggingFaceRepo = "HadrianSecurity/subwiz" 16 | ModelFile = "model.pt" 17 | TokenizerFile = "tokenizer.json" 18 | ConfigFile = "config.json" 19 | ) 20 | 21 | type Downloader struct { 22 | cacheDir string 23 | client *http.Client 24 | } 25 | 26 | func NewDownloader() *Downloader { 27 | cacheDir := config.GetLLMCacheDir() 28 | 29 | return &Downloader{ 30 | cacheDir: cacheDir, 31 | client: &http.Client{}, 32 | } 33 | } 34 | 35 | func (d *Downloader) DownloadModel(forceDownload bool) (string, string, error) { 36 | if err := os.MkdirAll(d.cacheDir, 0755); err != nil { 37 | return "", "", fmt.Errorf("failed to create cache directory: %w", err) 38 | } 39 | 40 | modelPath := filepath.Join(d.cacheDir, ModelFile) 41 | tokenizerPath := filepath.Join(d.cacheDir, TokenizerFile) 42 | configPath := filepath.Join(d.cacheDir, ConfigFile) 43 | 44 | if !forceDownload { 45 | if fileExists(modelPath) && fileExists(tokenizerPath) && fileExists(configPath) { 46 | return modelPath, tokenizerPath, nil 47 | } 48 | } 49 | 50 | baseURL := fmt.Sprintf("https://huggingface.co/%s/resolve/main", HuggingFaceRepo) 51 | 52 | fmt.Println("[LLM] Downloading AI model files (first run, ~100MB)...") 53 | 54 | files := []struct { 55 | name string 56 | path string 57 | url string 58 | }{ 59 | {ModelFile, modelPath, baseURL + "/" + ModelFile}, 60 | {TokenizerFile, tokenizerPath, baseURL + "/" + TokenizerFile}, 61 | {ConfigFile, configPath, baseURL + "/" + ConfigFile}, 62 | } 63 | 64 | for _, file := range files { 65 | if forceDownload || !fileExists(file.path) { 66 | fmt.Printf("[LLM] downloading %s...\n", file.name) 67 | if err := d.downloadFile(file.url, file.path); err != nil { 68 | return "", "", fmt.Errorf("failed to download %s: %w", file.name, err) 69 | } 70 | } 71 | } 72 | 73 | fmt.Printf("[LLM] Model cached at %s\n", d.cacheDir) 74 | 75 | return modelPath, tokenizerPath, nil 76 | } 77 | 78 | func (d *Downloader) downloadFile(url, dest string) error { 79 | resp, err := d.client.Get(url) 80 | if err != nil { 81 | return err 82 | } 83 | defer resp.Body.Close() 84 | 85 | if resp.StatusCode != http.StatusOK { 86 | return fmt.Errorf("HTTP %d: %s", resp.StatusCode, resp.Status) 87 | } 88 | 89 | out, err := os.Create(dest) 90 | if err != nil { 91 | return err 92 | } 93 | defer out.Close() 94 | 95 | _, err = io.Copy(out, resp.Body) 96 | return err 97 | } 98 | 99 | func (d *Downloader) LoadConfig(configPath string) (*ModelConfig, error) { 100 | data, err := os.ReadFile(configPath) 101 | if err != nil { 102 | return nil, err 103 | } 104 | 105 | var config ModelConfig 106 | if err := json.Unmarshal(data, &config); err != nil { 107 | return nil, err 108 | } 109 | 110 | return &config, nil 111 | } 112 | 113 | func fileExists(path string) bool { 114 | _, err := os.Stat(path) 115 | return err == nil 116 | } 117 | 118 | -------------------------------------------------------------------------------- /pkg/sources/fullhunt.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type FullHunt struct{} 14 | 15 | 16 | type FullHuntResponse struct { 17 | Hosts []string `json:"hosts"` 18 | Message string `json:"message"` 19 | Status int `json:"status"` 20 | } 21 | 22 | 23 | func (f *FullHunt) Name() string { 24 | return "fullhunt" 25 | } 26 | 27 | 28 | func (f *FullHunt) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 29 | results := make(chan Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | 35 | if s.Keys.FullHunt == "" { 36 | results <- Result{Source: f.Name(), Error: fmt.Errorf("FullHunt API key not configured")} 37 | return 38 | } 39 | 40 | 41 | url := fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain) 42 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 43 | if err != nil { 44 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 45 | return 46 | } 47 | 48 | 49 | req.Header.Set("X-API-KEY", s.Keys.FullHunt) 50 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 51 | req.Header.Set("Accept", "application/json") 52 | 53 | resp, err := s.Client.Do(req) 54 | if err != nil { 55 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 56 | return 57 | } 58 | defer resp.Body.Close() 59 | 60 | if resp.StatusCode == 401 { 61 | results <- Result{Source: f.Name(), Error: fmt.Errorf("invalid FullHunt API key")} 62 | return 63 | } 64 | 65 | if resp.StatusCode == 429 { 66 | results <- Result{Source: f.Name(), Error: fmt.Errorf("FullHunt rate limit exceeded")} 67 | return 68 | } 69 | 70 | if resp.StatusCode != http.StatusOK { 71 | results <- Result{Source: f.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 72 | return 73 | } 74 | 75 | 76 | var fullHuntResp FullHuntResponse 77 | if err := json.NewDecoder(resp.Body).Decode(&fullHuntResp); err != nil { 78 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 79 | return 80 | } 81 | 82 | 83 | if fullHuntResp.Status != 200 && fullHuntResp.Status != 0 { 84 | results <- Result{Source: f.Name(), Error: fmt.Errorf("FullHunt API error: %s (status: %d)", fullHuntResp.Message, fullHuntResp.Status)} 85 | return 86 | } 87 | 88 | 89 | seen := make(map[string]bool) 90 | for _, host := range fullHuntResp.Hosts { 91 | hostname := strings.TrimSpace(strings.ToLower(host)) 92 | 93 | 94 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 95 | seen[hostname] = true 96 | 97 | select { 98 | case results <- Result{Source: f.Name(), Value: hostname, Type: "subdomain"}: 99 | case <-ctx.Done(): 100 | return 101 | } 102 | } 103 | } 104 | }() 105 | 106 | return results 107 | } 108 | -------------------------------------------------------------------------------- /pkg/sources/whoisxmlapi.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type WhoisXMLAPI struct{} 14 | 15 | 16 | type WhoisXMLAPIResponse struct { 17 | Search string `json:"search"` 18 | Result WhoisXMLAPIResponseResult `json:"result"` 19 | } 20 | 21 | type WhoisXMLAPIResponseResult struct { 22 | Count int `json:"count"` 23 | Records []WhoisXMLAPIResponseRecord `json:"records"` 24 | } 25 | 26 | type WhoisXMLAPIResponseRecord struct { 27 | Domain string `json:"domain"` 28 | FirstSeen int `json:"firstSeen"` 29 | LastSeen int `json:"lastSeen"` 30 | } 31 | 32 | 33 | func (w *WhoisXMLAPI) Name() string { 34 | return "whoisxmlapi" 35 | } 36 | 37 | 38 | func (w *WhoisXMLAPI) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 39 | results := make(chan Result) 40 | 41 | go func() { 42 | defer close(results) 43 | 44 | 45 | if session.Keys.WhoisXMLAPI == "" { 46 | results <- Result{Source: w.Name(), Error: fmt.Errorf("WhoisXMLAPI API key not configured")} 47 | return 48 | } 49 | 50 | seen := make(map[string]bool) 51 | apiURL := fmt.Sprintf("https://subdomains.whoisxmlapi.com/api/v1?apiKey=%s&domainName=%s", session.Keys.WhoisXMLAPI, domain) 52 | 53 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 54 | if err != nil { 55 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 56 | return 57 | } 58 | 59 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 60 | 61 | resp, err := session.Client.Do(req) 62 | if err != nil { 63 | results <- Result{Source: w.Name(), Error: fmt.Errorf("API request failed: %w", err)} 64 | return 65 | } 66 | defer resp.Body.Close() 67 | 68 | if resp.StatusCode == 401 { 69 | results <- Result{Source: w.Name(), Error: fmt.Errorf("invalid WhoisXMLAPI API key")} 70 | return 71 | } 72 | 73 | if resp.StatusCode == 429 { 74 | results <- Result{Source: w.Name(), Error: fmt.Errorf("WhoisXMLAPI rate limit exceeded")} 75 | return 76 | } 77 | 78 | if resp.StatusCode != http.StatusOK { 79 | results <- Result{Source: w.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 80 | return 81 | } 82 | 83 | var response WhoisXMLAPIResponse 84 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 85 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 86 | return 87 | } 88 | 89 | 90 | for _, record := range response.Result.Records { 91 | hostname := strings.TrimSpace(strings.ToLower(record.Domain)) 92 | 93 | if hostname != "" && !seen[hostname] { 94 | seen[hostname] = true 95 | 96 | select { 97 | case results <- Result{Source: w.Name(), Value: hostname, Type: "subdomain"}: 98 | case <-ctx.Done(): 99 | return 100 | } 101 | } 102 | } 103 | }() 104 | 105 | return results 106 | } 107 | -------------------------------------------------------------------------------- /pkg/sources/shrewdeye.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type ShrewdEye struct{} 13 | 14 | func (s *ShrewdEye) Name() string { 15 | return "shrewdeye" 16 | } 17 | 18 | func (s *ShrewdEye) Run(ctx context.Context, domain string, sess *session.Session) <-chan Result { 19 | results := make(chan Result) 20 | 21 | go func() { 22 | defer close(results) 23 | 24 | url := fmt.Sprintf("https://shrewdeye.app/domains/%s.txt", domain) 25 | 26 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 27 | if err != nil { 28 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 29 | return 30 | } 31 | 32 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 33 | req.Header.Set("Accept", "text/plain") 34 | 35 | resp, err := sess.Client.Do(req) 36 | if err != nil { 37 | results <- Result{Source: s.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 38 | return 39 | } 40 | defer resp.Body.Close() 41 | 42 | if resp.StatusCode == 404 { 43 | return 44 | } 45 | 46 | if resp.StatusCode == 429 { 47 | results <- Result{Source: s.Name(), Error: fmt.Errorf("ShrewdEye rate limit exceeded")} 48 | return 49 | } 50 | 51 | if resp.StatusCode != http.StatusOK { 52 | results <- Result{Source: s.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 53 | return 54 | } 55 | 56 | scanner := bufio.NewScanner(resp.Body) 57 | seen := make(map[string]bool) 58 | 59 | for scanner.Scan() { 60 | line := scanner.Text() 61 | 62 | if line == "" { 63 | continue 64 | } 65 | 66 | subdomain := strings.TrimSpace(strings.ToLower(line)) 67 | 68 | if s.isValidSubdomain(subdomain, domain) && !seen[subdomain] { 69 | seen[subdomain] = true 70 | 71 | select { 72 | case results <- Result{Source: s.Name(), Value: subdomain, Type: "subdomain"}: 73 | case <-ctx.Done(): 74 | return 75 | } 76 | } 77 | } 78 | 79 | if err := scanner.Err(); err != nil { 80 | results <- Result{Source: s.Name(), Error: fmt.Errorf("scanner error: %w", err)} 81 | return 82 | } 83 | }() 84 | 85 | return results 86 | } 87 | 88 | func (s *ShrewdEye) isValidSubdomain(subdomain, domain string) bool { 89 | if subdomain == "" || subdomain == domain { 90 | return false 91 | } 92 | 93 | if !strings.HasSuffix(subdomain, "."+domain) && subdomain != domain { 94 | return false 95 | } 96 | 97 | if len(subdomain) == 0 || len(subdomain) > 253 { 98 | return false 99 | } 100 | 101 | if strings.HasPrefix(subdomain, ".") || strings.HasSuffix(subdomain, ".") { 102 | return false 103 | } 104 | 105 | if strings.Contains(subdomain, "..") { 106 | return false 107 | } 108 | 109 | if subdomain != domain && !strings.Contains(subdomain, ".") { 110 | return false 111 | } 112 | 113 | if strings.ContainsAny(subdomain, " \t\n\r") { 114 | return false 115 | } 116 | 117 | return true 118 | } 119 | -------------------------------------------------------------------------------- /pkg/active/wordlist.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "sort" 8 | "strings" 9 | ) 10 | 11 | func ExtractKeywords(subdomains []string, rootDomain string) ([]string, error) { 12 | wordCount := make(map[string]int) 13 | 14 | for _, subdomain := range subdomains { 15 | subdomain = strings.ToLower(strings.TrimSpace(subdomain)) 16 | 17 | if strings.HasSuffix(subdomain, "."+rootDomain) { 18 | subdomain = strings.TrimSuffix(subdomain, "."+rootDomain) 19 | } else if subdomain == rootDomain { 20 | continue 21 | } 22 | 23 | parts := strings.Split(subdomain, ".") 24 | for _, part := range parts { 25 | part = strings.TrimSpace(part) 26 | 27 | if part == "" || isNumericOnly(part) { 28 | continue 29 | } 30 | 31 | subParts := splitByDelimiters(part) 32 | for _, subPart := range subParts { 33 | if subPart != "" && !isNumericOnly(subPart) { 34 | wordCount[subPart]++ 35 | } 36 | } 37 | } 38 | } 39 | 40 | type wordFreq struct { 41 | word string 42 | count int 43 | } 44 | 45 | var words []wordFreq 46 | for word, count := range wordCount { 47 | words = append(words, wordFreq{word, count}) 48 | } 49 | 50 | sort.Slice(words, func(i, j int) bool { 51 | if words[i].count == words[j].count { 52 | return words[i].word < words[j].word 53 | } 54 | return words[i].count > words[j].count 55 | }) 56 | 57 | var result []string 58 | for _, w := range words { 59 | result = append(result, w.word) 60 | } 61 | 62 | return result, nil 63 | } 64 | 65 | func isNumericOnly(s string) bool { 66 | for _, r := range s { 67 | if r < '0' || r > '9' { 68 | return false 69 | } 70 | } 71 | return len(s) > 0 72 | } 73 | 74 | func splitByDelimiters(s string) []string { 75 | delimiters := []string{"-", "_", "~"} 76 | for _, delim := range delimiters { 77 | s = strings.ReplaceAll(s, delim, " ") 78 | } 79 | 80 | parts := strings.Fields(s) 81 | return parts 82 | } 83 | 84 | func WriteWordlist(keywords []string, outputPath string) error { 85 | file, err := os.Create(outputPath) 86 | if err != nil { 87 | return fmt.Errorf("failed to create wordlist file: %w", err) 88 | } 89 | defer file.Close() 90 | 91 | writer := bufio.NewWriter(file) 92 | for _, keyword := range keywords { 93 | if _, err := writer.WriteString(keyword + "\n"); err != nil { 94 | return fmt.Errorf("failed to write to wordlist: %w", err) 95 | } 96 | } 97 | 98 | return writer.Flush() 99 | } 100 | 101 | func ReadSubdomains(filePath string) ([]string, error) { 102 | file, err := os.Open(filePath) 103 | if err != nil { 104 | return nil, fmt.Errorf("failed to open file: %w", err) 105 | } 106 | defer file.Close() 107 | 108 | var subdomains []string 109 | scanner := bufio.NewScanner(file) 110 | 111 | for scanner.Scan() { 112 | line := strings.TrimSpace(scanner.Text()) 113 | if line != "" && !strings.HasPrefix(line, "#") { 114 | subdomains = append(subdomains, line) 115 | } 116 | } 117 | 118 | if err := scanner.Err(); err != nil { 119 | return nil, fmt.Errorf("error reading file: %w", err) 120 | } 121 | 122 | return subdomains, nil 123 | } 124 | 125 | -------------------------------------------------------------------------------- /pkg/sources/c99.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type C99 struct{} 13 | 14 | type C99Response struct { 15 | Success bool `json:"success"` 16 | Subdomains []struct { 17 | Subdomain string `json:"subdomain"` 18 | IP string `json:"ip"` 19 | Cloudflare bool `json:"cloudflare"` 20 | } `json:"subdomains"` 21 | Error string `json:"error"` 22 | } 23 | 24 | func (c *C99) Name() string { 25 | return "c99" 26 | } 27 | 28 | func (c *C99) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 29 | results := make(chan Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | if s.Keys.C99 == "" { 35 | results <- Result{Source: c.Name(), Error: fmt.Errorf("C99 API key not configured")} 36 | return 37 | } 38 | 39 | url := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", s.Keys.C99, domain) 40 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 41 | if err != nil { 42 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 43 | return 44 | } 45 | 46 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 47 | req.Header.Set("Accept", "application/json") 48 | 49 | resp, err := s.Client.Do(req) 50 | if err != nil { 51 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 52 | return 53 | } 54 | defer resp.Body.Close() 55 | 56 | if resp.StatusCode == 401 { 57 | results <- Result{Source: c.Name(), Error: fmt.Errorf("invalid C99 API key")} 58 | return 59 | } 60 | 61 | if resp.StatusCode == 429 { 62 | results <- Result{Source: c.Name(), Error: fmt.Errorf("C99 rate limit exceeded")} 63 | return 64 | } 65 | 66 | if resp.StatusCode != http.StatusOK { 67 | results <- Result{Source: c.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 68 | return 69 | } 70 | 71 | var c99Resp C99Response 72 | if err := json.NewDecoder(resp.Body).Decode(&c99Resp); err != nil { 73 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 74 | return 75 | } 76 | 77 | if c99Resp.Error != "" { 78 | results <- Result{Source: c.Name(), Error: fmt.Errorf("API error: %s", c99Resp.Error)} 79 | return 80 | } 81 | 82 | if !c99Resp.Success { 83 | results <- Result{Source: c.Name(), Error: fmt.Errorf("API returned success=false")} 84 | return 85 | } 86 | 87 | seen := make(map[string]bool) 88 | for _, data := range c99Resp.Subdomains { 89 | hostname := strings.TrimSpace(strings.ToLower(data.Subdomain)) 90 | 91 | if !strings.HasPrefix(hostname, ".") && hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 92 | seen[hostname] = true 93 | 94 | select { 95 | case results <- Result{Source: c.Name(), Value: hostname, Type: "subdomain"}: 96 | case <-ctx.Done(): 97 | return 98 | } 99 | } 100 | } 101 | }() 102 | 103 | return results 104 | } 105 | -------------------------------------------------------------------------------- /pkg/sources/dnsarchive.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | type DNSArchive struct{} 14 | 15 | type DNSArchiveResponse []struct { 16 | Domain string `json:"domain"` 17 | } 18 | 19 | func (d *DNSArchive) Name() string { 20 | return "DNSArchive" 21 | } 22 | 23 | func (d *DNSArchive) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 24 | results := make(chan Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | if s.Keys.DNSArchive == "" { 30 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DNSArchive API key not configured")} 31 | return 32 | } 33 | 34 | apiKeyParts := strings.Split(s.Keys.DNSArchive, ":") 35 | if len(apiKeyParts) != 2 { 36 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DNSArchive API key must be in format 'token:apikey'")} 37 | return 38 | } 39 | 40 | token := apiKeyParts[0] 41 | apiKey := apiKeyParts[1] 42 | 43 | url := fmt.Sprintf("https://dnsarchive.net/api/?apikey=%s&search=%s", apiKey, domain) 44 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 45 | if err != nil { 46 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 47 | return 48 | } 49 | 50 | req.Header.Set("X-API-Access", token) 51 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 52 | req.Header.Set("Accept", "application/json") 53 | 54 | resp, err := s.Client.Do(req) 55 | if err != nil { 56 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 57 | return 58 | } 59 | defer resp.Body.Close() 60 | 61 | if resp.StatusCode == 401 { 62 | results <- Result{Source: d.Name(), Error: fmt.Errorf("invalid DNSArchive API credentials")} 63 | return 64 | } 65 | 66 | if resp.StatusCode == 429 { 67 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DNSArchive rate limit exceeded")} 68 | return 69 | } 70 | 71 | if resp.StatusCode != http.StatusOK { 72 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 73 | return 74 | } 75 | 76 | responseData, err := io.ReadAll(resp.Body) 77 | if err != nil { 78 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 79 | return 80 | } 81 | 82 | var DNSArchiveResp DNSArchiveResponse 83 | if err := json.Unmarshal(responseData, &DNSArchiveResp); err != nil { 84 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 85 | return 86 | } 87 | 88 | seen := make(map[string]bool) 89 | for _, record := range DNSArchiveResp { 90 | 91 | hostname := strings.TrimSuffix(record.Domain, ".") 92 | hostname = strings.TrimSpace(strings.ToLower(hostname)) 93 | 94 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 95 | seen[hostname] = true 96 | 97 | select { 98 | case results <- Result{Source: d.Name(), Value: hostname, Type: "subdomain"}: 99 | case <-ctx.Done(): 100 | return 101 | } 102 | } 103 | } 104 | }() 105 | 106 | return results 107 | } 108 | -------------------------------------------------------------------------------- /pkg/sources/builtwith.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type BuiltWith struct{} 13 | 14 | type BuiltWithResponse struct { 15 | Results []BuiltWithResultItem `json:"Results"` 16 | } 17 | 18 | type BuiltWithResultItem struct { 19 | Result BuiltWithResult `json:"Result"` 20 | } 21 | 22 | type BuiltWithResult struct { 23 | Paths []BuiltWithPath `json:"Paths"` 24 | } 25 | 26 | type BuiltWithPath struct { 27 | Domain string `json:"Domain"` 28 | URL string `json:"Url"` 29 | SubDomain string `json:"SubDomain"` 30 | } 31 | 32 | func (b *BuiltWith) Name() string { 33 | return "builtwith" 34 | } 35 | 36 | func (b *BuiltWith) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 37 | results := make(chan Result) 38 | 39 | go func() { 40 | defer close(results) 41 | 42 | if s.Keys.BuiltWith == "" { 43 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BuiltWith API key not configured")} 44 | return 45 | } 46 | 47 | url := fmt.Sprintf("https://api.builtwith.com/v21/api.json?KEY=%s&HIDETEXT=yes&HIDEDL=yes&NOLIVE=yes&NOMETA=yes&NOPII=yes&NOATTR=yes&LOOKUP=%s", s.Keys.BuiltWith, domain) 48 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 49 | if err != nil { 50 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 51 | return 52 | } 53 | 54 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 55 | req.Header.Set("Accept", "application/json") 56 | 57 | resp, err := s.Client.Do(req) 58 | if err != nil { 59 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 60 | return 61 | } 62 | defer resp.Body.Close() 63 | 64 | if resp.StatusCode == 401 { 65 | results <- Result{Source: b.Name(), Error: fmt.Errorf("invalid BuiltWith API key")} 66 | return 67 | } 68 | 69 | if resp.StatusCode == 429 { 70 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BuiltWith rate limit exceeded")} 71 | return 72 | } 73 | 74 | if resp.StatusCode != http.StatusOK { 75 | results <- Result{Source: b.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 76 | return 77 | } 78 | 79 | var builtwithResp BuiltWithResponse 80 | if err := json.NewDecoder(resp.Body).Decode(&builtwithResp); err != nil { 81 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 82 | return 83 | } 84 | 85 | seen := make(map[string]bool) 86 | for _, result := range builtwithResp.Results { 87 | for _, path := range result.Result.Paths { 88 | if path.SubDomain != "" && path.Domain != "" { 89 | hostname := fmt.Sprintf("%s.%s", path.SubDomain, path.Domain) 90 | hostname = strings.TrimSpace(strings.ToLower(hostname)) 91 | 92 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 93 | seen[hostname] = true 94 | 95 | select { 96 | case results <- Result{Source: b.Name(), Value: hostname, Type: "subdomain"}: 97 | case <-ctx.Done(): 98 | return 99 | } 100 | } 101 | } 102 | } 103 | } 104 | }() 105 | 106 | return results 107 | } 108 | -------------------------------------------------------------------------------- /pkg/sources/virustotal.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type VirusTotal struct{} 14 | 15 | 16 | type VirusTotalResponse struct { 17 | Data []struct { 18 | ID string `json:"id"` 19 | Type string `json:"type"` 20 | } `json:"data"` 21 | Meta struct { 22 | Cursor string `json:"cursor"` 23 | } `json:"meta"` 24 | } 25 | 26 | 27 | func (v *VirusTotal) Name() string { 28 | return "virustotal" 29 | } 30 | 31 | 32 | func (v *VirusTotal) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 33 | results := make(chan Result) 34 | 35 | go func() { 36 | defer close(results) 37 | 38 | 39 | if s.Keys.VirusTotal == "" { 40 | results <- Result{Source: v.Name(), Error: fmt.Errorf("VirusTotal API key not configured")} 41 | return 42 | } 43 | 44 | seen := make(map[string]bool) 45 | cursor := "" 46 | 47 | 48 | for { 49 | select { 50 | case <-ctx.Done(): 51 | return 52 | default: 53 | } 54 | 55 | 56 | url := fmt.Sprintf("https://www.virustotal.com/api/v3/domains/%s/subdomains?limit=40", domain) 57 | if cursor != "" { 58 | url = fmt.Sprintf("%s&cursor=%s", url, cursor) 59 | } 60 | 61 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 62 | if err != nil { 63 | results <- Result{Source: v.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 64 | return 65 | } 66 | 67 | 68 | req.Header.Set("x-apikey", s.Keys.VirusTotal) 69 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 70 | req.Header.Set("Accept", "application/json") 71 | 72 | resp, err := s.Client.Do(req) 73 | if err != nil { 74 | results <- Result{Source: v.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 75 | return 76 | } 77 | 78 | if resp.StatusCode == 401 { 79 | resp.Body.Close() 80 | results <- Result{Source: v.Name(), Error: fmt.Errorf("invalid VirusTotal API key")} 81 | return 82 | } 83 | 84 | if resp.StatusCode == 429 { 85 | resp.Body.Close() 86 | results <- Result{Source: v.Name(), Error: fmt.Errorf("VirusTotal rate limit exceeded")} 87 | return 88 | } 89 | 90 | if resp.StatusCode != http.StatusOK { 91 | resp.Body.Close() 92 | results <- Result{Source: v.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 93 | return 94 | } 95 | 96 | 97 | var vtResp VirusTotalResponse 98 | if err := json.NewDecoder(resp.Body).Decode(&vtResp); err != nil { 99 | resp.Body.Close() 100 | results <- Result{Source: v.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 101 | return 102 | } 103 | resp.Body.Close() 104 | 105 | 106 | for _, item := range vtResp.Data { 107 | hostname := strings.TrimSpace(strings.ToLower(item.ID)) 108 | 109 | if hostname != "" && !seen[hostname] { 110 | seen[hostname] = true 111 | 112 | select { 113 | case results <- Result{Source: v.Name(), Value: hostname, Type: "subdomain"}: 114 | case <-ctx.Done(): 115 | return 116 | } 117 | } 118 | } 119 | 120 | 121 | cursor = vtResp.Meta.Cursor 122 | if cursor == "" { 123 | break 124 | } 125 | } 126 | }() 127 | 128 | return results 129 | } 130 | -------------------------------------------------------------------------------- /pkg/sources/bufferover.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type BufferOver struct{} 13 | 14 | type BufferOverResponse struct { 15 | Meta struct { 16 | Errors []string `json:"Errors"` 17 | } `json:"Meta"` 18 | FDNSA []string `json:"FDNS_A"` 19 | RDNS []string `json:"RDNS"` 20 | Results []string `json:"Results"` 21 | } 22 | 23 | func (b *BufferOver) Name() string { 24 | return "bufferover" 25 | } 26 | 27 | func (b *BufferOver) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 28 | results := make(chan Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | if s.Keys.BufferOver == "" { 34 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BufferOver API key not configured")} 35 | return 36 | } 37 | 38 | url := fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain) 39 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 40 | if err != nil { 41 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 42 | return 43 | } 44 | 45 | req.Header.Set("x-api-key", s.Keys.BufferOver) 46 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 47 | req.Header.Set("Accept", "application/json") 48 | 49 | resp, err := s.Client.Do(req) 50 | if err != nil { 51 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 52 | return 53 | } 54 | defer resp.Body.Close() 55 | 56 | if resp.StatusCode == 401 { 57 | results <- Result{Source: b.Name(), Error: fmt.Errorf("invalid BufferOver API key")} 58 | return 59 | } 60 | 61 | if resp.StatusCode == 429 { 62 | results <- Result{Source: b.Name(), Error: fmt.Errorf("BufferOver rate limit exceeded")} 63 | return 64 | } 65 | 66 | if resp.StatusCode != http.StatusOK { 67 | results <- Result{Source: b.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 68 | return 69 | } 70 | 71 | var bufferoverResp BufferOverResponse 72 | if err := json.NewDecoder(resp.Body).Decode(&bufferoverResp); err != nil { 73 | results <- Result{Source: b.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 74 | return 75 | } 76 | 77 | if len(bufferoverResp.Meta.Errors) > 0 { 78 | results <- Result{Source: b.Name(), Error: fmt.Errorf("API errors: %s", strings.Join(bufferoverResp.Meta.Errors, ", "))} 79 | return 80 | } 81 | 82 | var allSubdomains []string 83 | if len(bufferoverResp.FDNSA) > 0 { 84 | allSubdomains = bufferoverResp.FDNSA 85 | allSubdomains = append(allSubdomains, bufferoverResp.RDNS...) 86 | } else if len(bufferoverResp.Results) > 0 { 87 | allSubdomains = bufferoverResp.Results 88 | } 89 | 90 | seen := make(map[string]bool) 91 | for _, subdomain := range allSubdomains { 92 | parts := strings.Fields(subdomain) 93 | for _, part := range parts { 94 | hostname := strings.TrimSpace(strings.ToLower(part)) 95 | 96 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 97 | seen[hostname] = true 98 | 99 | select { 100 | case results <- Result{Source: b.Name(), Value: hostname, Type: "subdomain"}: 101 | case <-ctx.Done(): 102 | return 103 | } 104 | } 105 | } 106 | } 107 | }() 108 | 109 | return results 110 | } 111 | -------------------------------------------------------------------------------- /pkg/sources/digitalyama.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type DigitalYama struct{} 14 | 15 | 16 | type DigitalYamaResponse struct { 17 | Query string `json:"query"` 18 | Count int `json:"count"` 19 | Subdomains []string `json:"subdomains"` 20 | UsageSummary struct { 21 | QueryCost float64 `json:"query_cost"` 22 | CreditsRemaining float64 `json:"credits_remaining"` 23 | } `json:"usage_summary"` 24 | } 25 | 26 | 27 | type DigitalYamaErrorResponse struct { 28 | Detail []struct { 29 | Loc []string `json:"loc"` 30 | Msg string `json:"msg"` 31 | Type string `json:"type"` 32 | } `json:"detail"` 33 | } 34 | 35 | 36 | func (d *DigitalYama) Name() string { 37 | return "digitalyama" 38 | } 39 | 40 | 41 | func (d *DigitalYama) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 42 | results := make(chan Result) 43 | 44 | go func() { 45 | defer close(results) 46 | 47 | 48 | if s.Keys.DigitalYama == "" { 49 | results <- Result{Source: d.Name(), Error: fmt.Errorf("DigitalYama API key not configured")} 50 | return 51 | } 52 | 53 | 54 | url := fmt.Sprintf("https://api.digitalyama.com/subdomain_finder?domain=%s", domain) 55 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 56 | if err != nil { 57 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 58 | return 59 | } 60 | 61 | 62 | req.Header.Set("x-api-key", s.Keys.DigitalYama) 63 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 64 | req.Header.Set("Accept", "application/json") 65 | 66 | resp, err := s.Client.Do(req) 67 | if err != nil { 68 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 69 | return 70 | } 71 | defer resp.Body.Close() 72 | 73 | 74 | if resp.StatusCode != http.StatusOK { 75 | var errResponse DigitalYamaErrorResponse 76 | if err := json.NewDecoder(resp.Body).Decode(&errResponse); err != nil { 77 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 78 | return 79 | } 80 | 81 | if len(errResponse.Detail) > 0 { 82 | errMsg := errResponse.Detail[0].Msg 83 | results <- Result{Source: d.Name(), Error: fmt.Errorf("%s (HTTP %d)", errMsg, resp.StatusCode)} 84 | } else { 85 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 86 | } 87 | return 88 | } 89 | 90 | 91 | var digitalYamaResp DigitalYamaResponse 92 | if err := json.NewDecoder(resp.Body).Decode(&digitalYamaResp); err != nil { 93 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 94 | return 95 | } 96 | 97 | 98 | seen := make(map[string]bool) 99 | for _, subdomain := range digitalYamaResp.Subdomains { 100 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 101 | 102 | 103 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 104 | seen[hostname] = true 105 | 106 | select { 107 | case results <- Result{Source: d.Name(), Value: hostname, Type: "subdomain"}: 108 | case <-ctx.Done(): 109 | return 110 | } 111 | } 112 | } 113 | }() 114 | 115 | return results 116 | } 117 | -------------------------------------------------------------------------------- /pkg/sources/myssl.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type MySSL struct{} 13 | 14 | type MySSLResponse struct { 15 | Code int `json:"code"` 16 | Error string `json:"error"` 17 | Data []MySSLSubdomain `json:"data"` 18 | } 19 | 20 | type MySSLSubdomain struct { 21 | Domain string `json:"domain"` 22 | } 23 | 24 | func (m *MySSL) Name() string { 25 | return "myssl" 26 | } 27 | 28 | func (m *MySSL) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 29 | results := make(chan Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | url := fmt.Sprintf("https://myssl.com/api/v1/discover_sub_domain?domain=%s", domain) 35 | 36 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 37 | if err != nil { 38 | results <- Result{Source: m.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 39 | return 40 | } 41 | 42 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 43 | req.Header.Set("Accept", "application/json") 44 | 45 | resp, err := s.Client.Do(req) 46 | if err != nil { 47 | results <- Result{Source: m.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 48 | return 49 | } 50 | defer resp.Body.Close() 51 | 52 | if resp.StatusCode == 429 { 53 | results <- Result{Source: m.Name(), Error: fmt.Errorf("MySSL rate limit exceeded")} 54 | return 55 | } 56 | 57 | if resp.StatusCode != http.StatusOK { 58 | results <- Result{Source: m.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 59 | return 60 | } 61 | 62 | var myssLResponse MySSLResponse 63 | if err := json.NewDecoder(resp.Body).Decode(&myssLResponse); err != nil { 64 | results <- Result{Source: m.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 65 | return 66 | } 67 | 68 | if myssLResponse.Code != 0 { 69 | if len(myssLResponse.Data) == 0 { 70 | results <- Result{Source: m.Name(), Error: fmt.Errorf("API error: %s", myssLResponse.Error)} 71 | return 72 | } 73 | } 74 | 75 | seen := make(map[string]bool) 76 | 77 | for _, subdomainData := range myssLResponse.Data { 78 | subdomain := strings.TrimSpace(strings.ToLower(subdomainData.Domain)) 79 | 80 | if m.isValidSubdomain(subdomain, domain) && !seen[subdomain] { 81 | seen[subdomain] = true 82 | 83 | select { 84 | case results <- Result{Source: m.Name(), Value: subdomain, Type: "subdomain"}: 85 | case <-ctx.Done(): 86 | return 87 | } 88 | } 89 | } 90 | }() 91 | 92 | return results 93 | } 94 | 95 | func (m *MySSL) isValidSubdomain(subdomain, domain string) bool { 96 | if subdomain == "" || subdomain == domain { 97 | return false 98 | } 99 | 100 | if !strings.HasSuffix(subdomain, "."+domain) && subdomain != domain { 101 | return false 102 | } 103 | 104 | if len(subdomain) == 0 || len(subdomain) > 253 { 105 | return false 106 | } 107 | 108 | if strings.HasPrefix(subdomain, ".") || strings.HasSuffix(subdomain, ".") { 109 | return false 110 | } 111 | 112 | if strings.Contains(subdomain, "..") { 113 | return false 114 | } 115 | 116 | if subdomain != domain && !strings.Contains(subdomain, ".") { 117 | return false 118 | } 119 | 120 | if strings.ContainsAny(subdomain, " \t\n\r") { 121 | return false 122 | } 123 | 124 | return true 125 | } 126 | 127 | -------------------------------------------------------------------------------- /pkg/sources/chinaz.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | type Chinaz struct{} 14 | 15 | type ChinazResponse struct { 16 | StateCode int `json:"StateCode"` 17 | Reason string `json:"Reason"` 18 | Result ChinazResult `json:"Result"` 19 | } 20 | 21 | type ChinazResult struct { 22 | ContributingSubdomainList []ChinazSubdomain `json:"ContributingSubdomainList"` 23 | Domain string `json:"Domain"` 24 | AlexaRank int `json:"AlexaRank"` 25 | } 26 | 27 | type ChinazSubdomain struct { 28 | DataUrl string `json:"DataUrl"` 29 | Percent float64 `json:"Percent"` 30 | } 31 | 32 | func (c *Chinaz) Name() string { 33 | return "chinaz" 34 | } 35 | 36 | func (c *Chinaz) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 37 | results := make(chan Result) 38 | 39 | go func() { 40 | defer close(results) 41 | 42 | if s.Keys.Chinaz == "" { 43 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Chinaz API key not configured")} 44 | return 45 | } 46 | 47 | url := fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", s.Keys.Chinaz, domain) 48 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 49 | if err != nil { 50 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 51 | return 52 | } 53 | 54 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 55 | req.Header.Set("Accept", "application/json") 56 | 57 | resp, err := s.Client.Do(req) 58 | if err != nil { 59 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 60 | return 61 | } 62 | defer resp.Body.Close() 63 | 64 | if resp.StatusCode == 401 { 65 | results <- Result{Source: c.Name(), Error: fmt.Errorf("invalid Chinaz API key")} 66 | return 67 | } 68 | 69 | if resp.StatusCode == 429 { 70 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Chinaz rate limit exceeded")} 71 | return 72 | } 73 | 74 | if resp.StatusCode != http.StatusOK { 75 | results <- Result{Source: c.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 76 | return 77 | } 78 | 79 | body, err := io.ReadAll(resp.Body) 80 | if err != nil { 81 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 82 | return 83 | } 84 | 85 | var chinazResp ChinazResponse 86 | if err := json.Unmarshal(body, &chinazResp); err != nil { 87 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 88 | return 89 | } 90 | 91 | if chinazResp.StateCode != 0 { 92 | results <- Result{Source: c.Name(), Error: fmt.Errorf("API error: %s (StateCode: %d)", chinazResp.Reason, chinazResp.StateCode)} 93 | return 94 | } 95 | 96 | seen := make(map[string]bool) 97 | for _, subdomain := range chinazResp.Result.ContributingSubdomainList { 98 | hostname := strings.TrimSpace(strings.ToLower(subdomain.DataUrl)) 99 | 100 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 101 | seen[hostname] = true 102 | 103 | select { 104 | case results <- Result{Source: c.Name(), Value: hostname, Type: "subdomain"}: 105 | case <-ctx.Done(): 106 | return 107 | } 108 | } 109 | } 110 | }() 111 | 112 | return results 113 | } 114 | -------------------------------------------------------------------------------- /pkg/sources/zoomeyeapi.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | type ZoomEyeAPI struct{} 13 | 14 | type ZoomEyeAPIResponse struct { 15 | Status int `json:"status"` 16 | Total int `json:"total"` 17 | List []struct { 18 | Name string `json:"name"` 19 | IP []string `json:"ip"` 20 | } `json:"list"` 21 | } 22 | 23 | func (z *ZoomEyeAPI) Name() string { 24 | return "zoomeye" 25 | } 26 | 27 | func (z *ZoomEyeAPI) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 28 | results := make(chan Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | if session.Keys.ZoomEyeAPI == "" { 34 | results <- Result{Source: z.Name(), Error: fmt.Errorf("ZoomEyeAPI API key not configured")} 35 | return 36 | } 37 | 38 | apiKey := session.Keys.ZoomEyeAPI 39 | seen := make(map[string]bool) 40 | pages := 1 41 | 42 | for currentPage := 1; currentPage <= pages; currentPage++ { 43 | select { 44 | case <-ctx.Done(): 45 | return 46 | default: 47 | } 48 | 49 | apiURL := fmt.Sprintf("https://api.zoomeye.org/domain/search?q=%s&type=1&s=1000&page=%d", domain, currentPage) 50 | 51 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 52 | if err != nil { 53 | results <- Result{Source: z.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 54 | return 55 | } 56 | 57 | req.Header.Set("API-KEY", apiKey) 58 | req.Header.Set("Accept", "application/json") 59 | req.Header.Set("Content-Type", "application/json") 60 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 61 | 62 | resp, err := session.Client.Do(req) 63 | if err != nil { 64 | results <- Result{Source: z.Name(), Error: fmt.Errorf("API request failed: %w", err)} 65 | return 66 | } 67 | 68 | if resp.StatusCode == 401 { 69 | resp.Body.Close() 70 | results <- Result{Source: z.Name(), Error: fmt.Errorf("invalid ZoomEyeAPI API key")} 71 | return 72 | } 73 | 74 | if resp.StatusCode == 403 { 75 | resp.Body.Close() 76 | results <- Result{Source: z.Name(), Error: fmt.Errorf("ZoomEyeAPI access forbidden - check API key permissions")} 77 | return 78 | } 79 | 80 | if resp.StatusCode == 429 { 81 | resp.Body.Close() 82 | results <- Result{Source: z.Name(), Error: fmt.Errorf("ZoomEyeAPI rate limit exceeded")} 83 | return 84 | } 85 | 86 | if resp.StatusCode != http.StatusOK { 87 | resp.Body.Close() 88 | results <- Result{Source: z.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 89 | return 90 | } 91 | 92 | var response ZoomEyeAPIResponse 93 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 94 | resp.Body.Close() 95 | results <- Result{Source: z.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 96 | return 97 | } 98 | resp.Body.Close() 99 | 100 | if currentPage == 1 { 101 | pages = int(response.Total/1000) + 1 102 | } 103 | 104 | for _, item := range response.List { 105 | hostname := strings.TrimSpace(strings.ToLower(item.Name)) 106 | 107 | if hostname != "" && !seen[hostname] { 108 | seen[hostname] = true 109 | 110 | select { 111 | case results <- Result{Source: z.Name(), Value: hostname, Type: "subdomain"}: 112 | case <-ctx.Done(): 113 | return 114 | } 115 | } 116 | } 117 | } 118 | }() 119 | 120 | return results 121 | } 122 | -------------------------------------------------------------------------------- /pkg/sources/racent.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | type Racent struct{} 14 | 15 | type RacentResponse struct { 16 | Data struct { 17 | List []struct { 18 | DNSNames []string `json:"dnsnames"` 19 | } `json:"list"` 20 | } `json:"data"` 21 | } 22 | 23 | func (r *Racent) Name() string { 24 | return "racent" 25 | } 26 | 27 | func (r *Racent) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 28 | results := make(chan Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | url := fmt.Sprintf("https://face.racent.com/tool/query_ctlog?keyword=%s", domain) 34 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 35 | if err != nil { 36 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 37 | return 38 | } 39 | 40 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 41 | req.Header.Set("Accept", "application/json") 42 | 43 | resp, err := s.Client.Do(req) 44 | if err != nil { 45 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 46 | return 47 | } 48 | defer resp.Body.Close() 49 | 50 | if resp.StatusCode != http.StatusOK { 51 | results <- Result{Source: r.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 52 | return 53 | } 54 | 55 | body, err := io.ReadAll(resp.Body) 56 | if err != nil { 57 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 58 | return 59 | } 60 | 61 | bodyStr := string(body) 62 | if strings.Contains(bodyStr, "CTLog 查询超过限制") { 63 | results <- Result{Source: r.Name(), Error: fmt.Errorf("CTLog query limit exceeded")} 64 | return 65 | } 66 | 67 | var racentResp RacentResponse 68 | if err := json.Unmarshal(body, &racentResp); err != nil { 69 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 70 | return 71 | } 72 | 73 | seen := make(map[string]bool) 74 | 75 | for _, item := range racentResp.Data.List { 76 | for _, subdomain := range item.DNSNames { 77 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 78 | 79 | if hostname != "" && hostname != domain && 80 | strings.HasSuffix(hostname, "."+domain) && 81 | r.isValidHostname(hostname) && 82 | !seen[hostname] { 83 | seen[hostname] = true 84 | 85 | select { 86 | case results <- Result{Source: r.Name(), Value: hostname, Type: "subdomain"}: 87 | case <-ctx.Done(): 88 | return 89 | } 90 | } 91 | } 92 | } 93 | }() 94 | 95 | return results 96 | } 97 | 98 | // isValidHostname validates a hostname 99 | func (r *Racent) isValidHostname(hostname string) bool { 100 | if len(hostname) == 0 || len(hostname) > 253 { 101 | return false 102 | } 103 | 104 | // Check for leading or trailing dots 105 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 106 | return false 107 | } 108 | 109 | if strings.Contains(hostname, "..") { 110 | return false 111 | } 112 | 113 | // Must contain at least one dot 114 | if !strings.Contains(hostname, ".") { 115 | return false 116 | } 117 | 118 | // Check for valid characters 119 | for _, char := range hostname { 120 | if !((char >= 'a' && char <= 'z') || 121 | (char >= '0' && char <= '9') || 122 | char == '-' || char == '.') { 123 | return false 124 | } 125 | } 126 | 127 | // Check for leading or trailing hyphens 128 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 129 | return false 130 | } 131 | 132 | if strings.Contains(hostname, "--") { 133 | return false 134 | } 135 | 136 | return true 137 | } 138 | -------------------------------------------------------------------------------- /pkg/sources/certspotter.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type CertSpotter struct{} 14 | 15 | 16 | type CertSpotterObject struct { 17 | ID string `json:"id"` 18 | DNSNames []string `json:"dns_names"` 19 | } 20 | 21 | 22 | func (c *CertSpotter) Name() string { 23 | return "certspotter" 24 | } 25 | 26 | 27 | func (c *CertSpotter) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 28 | results := make(chan Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | 34 | if s.Keys.CertSpotter == "" { 35 | results <- Result{Source: c.Name(), Error: fmt.Errorf("CertSpotter API key not configured")} 36 | return 37 | } 38 | 39 | url := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain) 40 | response, err := c.makeRequest(ctx, url, s) 41 | if err != nil { 42 | results <- Result{Source: c.Name(), Error: err} 43 | return 44 | } 45 | 46 | seen := make(map[string]bool) 47 | c.processResults(response, domain, seen, results) 48 | 49 | if len(response) == 0 { 50 | return 51 | } 52 | 53 | lastID := response[len(response)-1].ID 54 | for { 55 | select { 56 | case <-ctx.Done(): 57 | return 58 | default: 59 | } 60 | 61 | nextURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, lastID) 62 | response, err := c.makeRequest(ctx, nextURL, s) 63 | if err != nil { 64 | results <- Result{Source: c.Name(), Error: err} 65 | return 66 | } 67 | 68 | if len(response) == 0 { 69 | break 70 | } 71 | 72 | c.processResults(response, domain, seen, results) 73 | 74 | lastID = response[len(response)-1].ID 75 | } 76 | }() 77 | 78 | return results 79 | } 80 | 81 | func (c *CertSpotter) makeRequest(ctx context.Context, url string, s *session.Session) ([]CertSpotterObject, error) { 82 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 83 | if err != nil { 84 | return nil, fmt.Errorf("failed to create request: %w", err) 85 | } 86 | 87 | req.Header.Set("Authorization", "Bearer "+s.Keys.CertSpotter) 88 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 89 | req.Header.Set("Accept", "application/json") 90 | 91 | resp, err := s.Client.Do(req) 92 | if err != nil { 93 | return nil, fmt.Errorf("failed to execute request: %w", err) 94 | } 95 | defer resp.Body.Close() 96 | 97 | if resp.StatusCode == 401 { 98 | return nil, fmt.Errorf("invalid CertSpotter API key") 99 | } 100 | 101 | if resp.StatusCode == 429 { 102 | return nil, fmt.Errorf("CertSpotter rate limit exceeded") 103 | } 104 | 105 | if resp.StatusCode != http.StatusOK { 106 | return nil, fmt.Errorf("HTTP error: %d", resp.StatusCode) 107 | } 108 | 109 | var response []CertSpotterObject 110 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 111 | return nil, fmt.Errorf("failed to decode JSON: %w", err) 112 | } 113 | 114 | return response, nil 115 | } 116 | 117 | func (c *CertSpotter) processResults(response []CertSpotterObject, domain string, seen map[string]bool, results chan<- Result) { 118 | for _, cert := range response { 119 | for _, dnsName := range cert.DNSNames { 120 | hostname := strings.TrimSpace(strings.ToLower(dnsName)) 121 | 122 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 123 | seen[hostname] = true 124 | 125 | select { 126 | case results <- Result{Source: c.Name(), Value: hostname, Type: "subdomain"}: 127 | default: 128 | } 129 | } 130 | } 131 | } 132 | } 133 | -------------------------------------------------------------------------------- /cmd/track.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "github.com/samogod/samoscout/pkg/orchestrator" 7 | "strings" 8 | "text/tabwriter" 9 | 10 | "github.com/fatih/color" 11 | "github.com/spf13/cobra" 12 | ) 13 | 14 | var ( 15 | trackStatus string 16 | trackAll bool 17 | ) 18 | 19 | var trackCmd = &cobra.Command{ 20 | Use: "track [domain]", 21 | Short: "Query subdomain tracking database", 22 | Long: `Query subdomain tracking database for a specific domain or all domains`, 23 | Run: runTrack, 24 | } 25 | 26 | func init() { 27 | trackCmd.Flags().StringVar(&trackStatus, "status", "", "filter by status (active, dead, new)") 28 | trackCmd.Flags().BoolVar(&trackAll, "all", false, "query all domains") 29 | rootCmd.AddCommand(trackCmd) 30 | } 31 | 32 | func runTrack(cmd *cobra.Command, args []string) { 33 | if !trackAll && len(args) == 0 { 34 | color.Red("Error: either provide a domain or use --all flag") 35 | cmd.Help() 36 | os.Exit(1) 37 | } 38 | 39 | if trackAll && len(args) > 0 { 40 | color.Red("Error: cannot use both domain and --all flag together") 41 | cmd.Help() 42 | os.Exit(1) 43 | } 44 | 45 | orch, err := orchestrator.NewOrchestrator(configFile) 46 | if err != nil { 47 | color.Red("Failed to initialize orchestrator: %v", err) 48 | os.Exit(1) 49 | } 50 | 51 | db := orch.GetDB() 52 | if db == nil || !db.IsEnabled() { 53 | color.Red("Error: Database is not enabled. Please enable it in config.yaml") 54 | os.Exit(1) 55 | } 56 | 57 | if trackStatus != "" { 58 | trackStatus = strings.ToUpper(trackStatus) 59 | } 60 | 61 | var records []struct { 62 | Domain string 63 | Subdomain string 64 | Status string 65 | FirstSeen string 66 | LastSeen string 67 | } 68 | 69 | if trackAll { 70 | results, err := db.QueryAllSubdomains(trackStatus) 71 | if err != nil { 72 | color.Red("Failed to query database: %v", err) 73 | os.Exit(1) 74 | } 75 | for _, r := range results { 76 | records = append(records, struct { 77 | Domain string 78 | Subdomain string 79 | Status string 80 | FirstSeen string 81 | LastSeen string 82 | }{ 83 | Domain: r.Domain, 84 | Subdomain: r.Subdomain, 85 | Status: r.Status, 86 | FirstSeen: r.FirstSeen.Format("2006-01-02 15:04:05"), 87 | LastSeen: r.LastSeen.Format("2006-01-02 15:04:05"), 88 | }) 89 | } 90 | } else { 91 | domain := args[0] 92 | results, err := db.QuerySubdomains(domain, trackStatus) 93 | if err != nil { 94 | color.Red("Failed to query database: %v", err) 95 | os.Exit(1) 96 | } 97 | 98 | if len(results) == 0 { 99 | color.Yellow("[INF] Domain %s not found in database.", domain) 100 | os.Exit(0) 101 | } 102 | 103 | for _, r := range results { 104 | records = append(records, struct { 105 | Domain string 106 | Subdomain string 107 | Status string 108 | FirstSeen string 109 | LastSeen string 110 | }{ 111 | Domain: r.Domain, 112 | Subdomain: r.Subdomain, 113 | Status: r.Status, 114 | FirstSeen: r.FirstSeen.Format("2006-01-02 15:04:05"), 115 | LastSeen: r.LastSeen.Format("2006-01-02 15:04:05"), 116 | }) 117 | } 118 | } 119 | 120 | w := tabwriter.NewWriter(os.Stdout, 0, 0, 3, ' ', 0) 121 | fmt.Fprintln(w, color.CyanString("DOMAIN\tSUBDOMAIN\tSTATUS\tFIRST_SEEN\tLAST_SEEN")) 122 | fmt.Fprintln(w, strings.Repeat("-", 100)) 123 | 124 | for _, r := range records { 125 | statusColor := color.GreenString 126 | if r.Status == "DEAD" { 127 | statusColor = color.RedString 128 | } else if r.Status == "NEW" { 129 | statusColor = color.YellowString 130 | } 131 | 132 | fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", 133 | r.Domain, 134 | r.Subdomain, 135 | statusColor(r.Status), 136 | r.FirstSeen, 137 | r.LastSeen, 138 | ) 139 | } 140 | w.Flush() 141 | 142 | color.Green("\nTotal records: %d", len(records)) 143 | } 144 | -------------------------------------------------------------------------------- /pkg/sources/hackertarget.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type HackerTarget struct{} 14 | 15 | 16 | func (h *HackerTarget) Name() string { 17 | return "hackertarget" 18 | } 19 | 20 | 21 | func (h *HackerTarget) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 22 | results := make(chan Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | 28 | url := fmt.Sprintf("https://api.hackertarget.com/hostsearch/?q=%s", domain) 29 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 30 | if err != nil { 31 | results <- Result{Source: h.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 32 | return 33 | } 34 | 35 | 36 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 37 | req.Header.Set("Accept", "text/plain") 38 | 39 | resp, err := s.Client.Do(req) 40 | if err != nil { 41 | results <- Result{Source: h.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 42 | return 43 | } 44 | defer resp.Body.Close() 45 | 46 | if resp.StatusCode == 429 { 47 | results <- Result{Source: h.Name(), Error: fmt.Errorf("HackerTarget rate limit exceeded")} 48 | return 49 | } 50 | 51 | if resp.StatusCode != http.StatusOK { 52 | results <- Result{Source: h.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 53 | return 54 | } 55 | 56 | 57 | scanner := bufio.NewScanner(resp.Body) 58 | seen := make(map[string]bool) 59 | 60 | for scanner.Scan() { 61 | line := scanner.Text() 62 | if line == "" { 63 | continue 64 | } 65 | 66 | 67 | 68 | subdomains := h.extractSubdomains(line, domain) 69 | 70 | for _, subdomain := range subdomains { 71 | if subdomain != "" && !seen[subdomain] { 72 | seen[subdomain] = true 73 | 74 | select { 75 | case results <- Result{Source: h.Name(), Value: subdomain, Type: "subdomain"}: 76 | case <-ctx.Done(): 77 | return 78 | } 79 | } 80 | } 81 | } 82 | 83 | 84 | if err := scanner.Err(); err != nil { 85 | results <- Result{Source: h.Name(), Error: fmt.Errorf("scanner error: %w", err)} 86 | } 87 | }() 88 | 89 | return results 90 | } 91 | 92 | 93 | func (h *HackerTarget) extractSubdomains(line, domain string) []string { 94 | var subdomains []string 95 | 96 | 97 | 98 | parts := strings.Split(line, ",") 99 | if len(parts) > 0 { 100 | hostname := strings.TrimSpace(strings.ToLower(parts[0])) 101 | 102 | 103 | if hostname != "" && hostname != domain && 104 | strings.HasSuffix(hostname, "."+domain) && 105 | h.isValidHostname(hostname) { 106 | subdomains = append(subdomains, hostname) 107 | } 108 | } 109 | 110 | 111 | words := strings.Fields(line) 112 | for _, word := range words { 113 | 114 | if strings.Contains(word, ".") { 115 | 116 | cleanWord := strings.Trim(word, ".,;:!?()[]{}\"'/\\") 117 | cleanWord = strings.ToLower(cleanWord) 118 | 119 | 120 | if cleanWord != "" && cleanWord != domain && 121 | strings.HasSuffix(cleanWord, "."+domain) && 122 | h.isValidHostname(cleanWord) { 123 | subdomains = append(subdomains, cleanWord) 124 | } 125 | } 126 | } 127 | 128 | return subdomains 129 | } 130 | 131 | 132 | func (h *HackerTarget) isValidHostname(hostname string) bool { 133 | if len(hostname) == 0 || len(hostname) > 253 { 134 | return false 135 | } 136 | 137 | 138 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 139 | return false 140 | } 141 | 142 | if strings.Contains(hostname, "..") { 143 | return false 144 | } 145 | 146 | 147 | if !strings.Contains(hostname, ".") { 148 | return false 149 | } 150 | 151 | return true 152 | } 153 | -------------------------------------------------------------------------------- /pkg/sources/reconcloud.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type ReconCloud struct{} 14 | 15 | 16 | type ReconCloudResponse struct { 17 | MsgType string `json:"msg_type"` 18 | RequestID string `json:"request_id"` 19 | OnCache bool `json:"on_cache"` 20 | Step string `json:"step"` 21 | CloudAssetsList []CloudAssetsList `json:"cloud_assets_list"` 22 | } 23 | 24 | 25 | type CloudAssetsList struct { 26 | Key string `json:"key"` 27 | Domain string `json:"domain"` 28 | CloudProvider string `json:"cloud_provider"` 29 | } 30 | 31 | 32 | func (r *ReconCloud) Name() string { 33 | return "reconcloud" 34 | } 35 | 36 | 37 | func (r *ReconCloud) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 38 | results := make(chan Result) 39 | 40 | go func() { 41 | defer close(results) 42 | 43 | 44 | url := fmt.Sprintf("https://recon.cloud/api/search?domain=%s", domain) 45 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 46 | if err != nil { 47 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 48 | return 49 | } 50 | 51 | 52 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 53 | req.Header.Set("Accept", "application/json") 54 | 55 | resp, err := s.Client.Do(req) 56 | if err != nil { 57 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 58 | return 59 | } 60 | defer resp.Body.Close() 61 | 62 | if resp.StatusCode == 429 { 63 | results <- Result{Source: r.Name(), Error: fmt.Errorf("ReconCloud rate limit exceeded")} 64 | return 65 | } 66 | 67 | if resp.StatusCode != http.StatusOK { 68 | results <- Result{Source: r.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 69 | return 70 | } 71 | 72 | 73 | var response ReconCloudResponse 74 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 75 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 76 | return 77 | } 78 | 79 | 80 | seen := make(map[string]bool) 81 | if len(response.CloudAssetsList) > 0 { 82 | for _, cloudAsset := range response.CloudAssetsList { 83 | hostname := strings.TrimSpace(strings.ToLower(cloudAsset.Domain)) 84 | 85 | 86 | if hostname != "" && hostname != domain && 87 | strings.HasSuffix(hostname, "."+domain) && 88 | r.isValidHostname(hostname) && !seen[hostname] { 89 | seen[hostname] = true 90 | 91 | select { 92 | case results <- Result{Source: r.Name(), Value: hostname, Type: "subdomain"}: 93 | case <-ctx.Done(): 94 | return 95 | } 96 | } 97 | } 98 | } 99 | }() 100 | 101 | return results 102 | } 103 | 104 | 105 | func (r *ReconCloud) isValidHostname(hostname string) bool { 106 | if len(hostname) == 0 || len(hostname) > 253 { 107 | return false 108 | } 109 | 110 | 111 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 112 | return false 113 | } 114 | 115 | if strings.Contains(hostname, "..") { 116 | return false 117 | } 118 | 119 | 120 | if !strings.Contains(hostname, ".") { 121 | return false 122 | } 123 | 124 | 125 | for _, char := range hostname { 126 | if !((char >= 'a' && char <= 'z') || 127 | (char >= '0' && char <= '9') || 128 | char == '-' || char == '.') { 129 | return false 130 | } 131 | } 132 | 133 | 134 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 135 | return false 136 | } 137 | 138 | if strings.Contains(hostname, "--") { 139 | return false 140 | } 141 | 142 | return true 143 | } 144 | -------------------------------------------------------------------------------- /pkg/llm/model.go: -------------------------------------------------------------------------------- 1 | package llm 2 | 3 | import ( 4 | "context" 5 | "embed" 6 | "encoding/json" 7 | "fmt" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "runtime" 12 | "strings" 13 | 14 | "github.com/samogod/samoscout/pkg/config" 15 | ) 16 | 17 | //go:embed llm_inference.py gpt_model.py 18 | var pythonScripts embed.FS 19 | 20 | type Model struct { 21 | scriptPath string 22 | config *ModelConfig 23 | } 24 | 25 | func extractPythonScripts() (string, error) { 26 | scriptsDir := filepath.Join(config.GetCacheDir(), "python_scripts") 27 | 28 | if err := os.MkdirAll(scriptsDir, 0755); err != nil { 29 | return "", fmt.Errorf("failed to create scripts directory: %w", err) 30 | } 31 | 32 | scripts := []string{"llm_inference.py", "gpt_model.py"} 33 | 34 | for _, scriptName := range scripts { 35 | scriptPath := filepath.Join(scriptsDir, scriptName) 36 | 37 | content, err := pythonScripts.ReadFile(scriptName) 38 | if err != nil { 39 | return "", fmt.Errorf("failed to read embedded script %s: %w", scriptName, err) 40 | } 41 | 42 | if err := os.WriteFile(scriptPath, content, 0755); err != nil { 43 | return "", fmt.Errorf("failed to write script %s: %w", scriptName, err) 44 | } 45 | } 46 | 47 | return filepath.Join(scriptsDir, "llm_inference.py"), nil 48 | } 49 | 50 | func LoadModel(modelPath, tokenizerPath string, device string) (*Model, error) { 51 | configPath := filepath.Join(filepath.Dir(modelPath), ConfigFile) 52 | downloader := NewDownloader() 53 | config, err := downloader.LoadConfig(configPath) 54 | if err != nil { 55 | return nil, fmt.Errorf("failed to load model config: %w", err) 56 | } 57 | 58 | scriptPath, err := extractPythonScripts() 59 | if err != nil { 60 | return nil, fmt.Errorf("failed to extract Python scripts: %w", err) 61 | } 62 | 63 | return &Model{ 64 | scriptPath: scriptPath, 65 | config: config, 66 | }, nil 67 | } 68 | 69 | func (m *Model) Close() error { 70 | return nil 71 | } 72 | 73 | type InferenceRequest struct { 74 | Subdomains []string `json:"subdomains"` 75 | Apex string `json:"apex"` 76 | NumPredictions int `json:"num_predictions"` 77 | MaxTokens int `json:"max_tokens"` 78 | Temperature float64 `json:"temperature"` 79 | Blocked []string `json:"blocked"` 80 | } 81 | 82 | type InferenceResponse struct { 83 | Predictions []string `json:"predictions"` 84 | Error string `json:"error,omitempty"` 85 | } 86 | 87 | func getPythonCommand() string { 88 | if runtime.GOOS == "windows" { 89 | return "python" 90 | } 91 | return "python3" 92 | } 93 | 94 | func (m *Model) GenerateDomains( 95 | ctx context.Context, 96 | subdomains []string, 97 | apex string, 98 | numPredictions int, 99 | maxTokens int, 100 | temperature float64, 101 | blocked []string, 102 | ) ([]string, error) { 103 | 104 | req := InferenceRequest{ 105 | Subdomains: subdomains, 106 | Apex: apex, 107 | NumPredictions: numPredictions, 108 | MaxTokens: maxTokens, 109 | Temperature: temperature, 110 | Blocked: blocked, 111 | } 112 | 113 | reqJSON, err := json.Marshal(req) 114 | if err != nil { 115 | return nil, fmt.Errorf("failed to marshal request: %w", err) 116 | } 117 | 118 | pythonCmd := getPythonCommand() 119 | cmd := exec.CommandContext(ctx, pythonCmd, m.scriptPath) 120 | cmd.Stdin = strings.NewReader(string(reqJSON)) 121 | output, err := cmd.CombinedOutput() 122 | if err != nil { 123 | return nil, fmt.Errorf("failed to run inference with '%s': %w\nOutput: %s\nHint: Ensure Python 3.7+ is installed and in PATH", pythonCmd, err, string(output)) 124 | } 125 | 126 | var resp InferenceResponse 127 | if err := json.Unmarshal(output, &resp); err != nil { 128 | return nil, fmt.Errorf("failed to parse response: %w, output: %s", err, string(output)) 129 | } 130 | 131 | if resp.Error != "" { 132 | return nil, fmt.Errorf("inference error: %s", resp.Error) 133 | } 134 | 135 | return resp.Predictions, nil 136 | } 137 | 138 | func (m *Model) GetConfig() *ModelConfig { 139 | return m.config 140 | } 141 | -------------------------------------------------------------------------------- /pkg/sources/digicert.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "regexp" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type DigiCert struct{} 15 | 16 | 17 | func (d *DigiCert) Name() string { 18 | return "digicert" 19 | } 20 | 21 | 22 | func (d *DigiCert) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 23 | results := make(chan Result) 24 | 25 | go func() { 26 | defer close(results) 27 | 28 | url := fmt.Sprintf("https://ssltools.digicert.com/chainTester/webservice/ctsearch/search?keyword=%s", domain) 29 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 30 | if err != nil { 31 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 32 | return 33 | } 34 | 35 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 36 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") 37 | 38 | resp, err := s.Client.Do(req) 39 | if err != nil { 40 | results <- Result{Source: d.Name(), Error: fmt.Errorf("API request failed: %w", err)} 41 | return 42 | } 43 | 44 | if resp.StatusCode != http.StatusOK { 45 | resp.Body.Close() 46 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 47 | return 48 | } 49 | 50 | body, err := io.ReadAll(resp.Body) 51 | resp.Body.Close() 52 | if err != nil { 53 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 54 | return 55 | } 56 | 57 | src := string(body) 58 | 59 | 60 | seen := make(map[string]bool) 61 | subdomains := d.extractSubdomains(src, domain, seen) 62 | 63 | for _, subdomain := range subdomains { 64 | select { 65 | case results <- Result{Source: d.Name(), Value: subdomain, Type: "subdomain"}: 66 | case <-ctx.Done(): 67 | return 68 | } 69 | } 70 | }() 71 | 72 | return results 73 | } 74 | 75 | 76 | func (d *DigiCert) extractSubdomains(content, domain string, seen map[string]bool) []string { 77 | var subdomains []string 78 | 79 | 80 | 81 | domainRegex := regexp.MustCompile(`([a-zA-Z0-9.-]+\.` + regexp.QuoteMeta(domain) + `)`) 82 | matches := domainRegex.FindAllStringSubmatch(content, -1) 83 | 84 | for _, match := range matches { 85 | if len(match) > 1 { 86 | hostname := strings.TrimSpace(strings.ToLower(match[1])) 87 | 88 | 89 | if d.isValidHostname(hostname) && !seen[hostname] { 90 | seen[hostname] = true 91 | subdomains = append(subdomains, hostname) 92 | } 93 | } 94 | } 95 | 96 | 97 | if len(subdomains) == 0 { 98 | generalRegex := regexp.MustCompile(`([a-zA-Z0-9.-]+\.[a-zA-Z0-9.-]+)`) 99 | generalMatches := generalRegex.FindAllStringSubmatch(content, -1) 100 | 101 | for _, match := range generalMatches { 102 | if len(match) > 1 { 103 | hostname := strings.TrimSpace(strings.ToLower(match[1])) 104 | 105 | 106 | if strings.Contains(hostname, domain) && d.isValidHostname(hostname) && !seen[hostname] { 107 | seen[hostname] = true 108 | subdomains = append(subdomains, hostname) 109 | } 110 | } 111 | } 112 | } 113 | 114 | return subdomains 115 | } 116 | 117 | 118 | func (d *DigiCert) isValidHostname(hostname string) bool { 119 | if hostname == "" { 120 | return false 121 | } 122 | 123 | 124 | if len(hostname) < 3 { 125 | return false 126 | } 127 | 128 | 129 | if strings.Contains(hostname, " ") || strings.Contains(hostname, "\t") || strings.Contains(hostname, "\n") { 130 | return false 131 | } 132 | 133 | 134 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 135 | return false 136 | } 137 | 138 | 139 | if !strings.Contains(hostname, ".") { 140 | return false 141 | } 142 | 143 | 144 | if strings.Contains(hostname, "..") { 145 | return false 146 | } 147 | 148 | 149 | if strings.Contains(hostname, "*") { 150 | return false 151 | } 152 | 153 | return true 154 | } 155 | -------------------------------------------------------------------------------- /pkg/sources/waybackarchive.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "regexp" 10 | "github.com/samogod/samoscout/pkg/session" 11 | "strings" 12 | ) 13 | 14 | 15 | type WaybackArchive struct{} 16 | 17 | 18 | func (w *WaybackArchive) Name() string { 19 | return "waybackarchive" 20 | } 21 | 22 | 23 | func (w *WaybackArchive) Run(ctx context.Context, domain string, session *session.Session) <-chan Result { 24 | results := make(chan Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | seen := make(map[string]bool) 30 | apiURL := fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain) 31 | 32 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 33 | if err != nil { 34 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 35 | return 36 | } 37 | 38 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 39 | 40 | resp, err := session.Client.Do(req) 41 | if err != nil { 42 | results <- Result{Source: w.Name(), Error: fmt.Errorf("API request failed: %w", err)} 43 | return 44 | } 45 | defer resp.Body.Close() 46 | 47 | if resp.StatusCode != http.StatusOK { 48 | results <- Result{Source: w.Name(), Error: fmt.Errorf("unexpected status code: %d", resp.StatusCode)} 49 | return 50 | } 51 | 52 | 53 | scanner := bufio.NewScanner(resp.Body) 54 | for scanner.Scan() { 55 | select { 56 | case <-ctx.Done(): 57 | return 58 | default: 59 | } 60 | 61 | line := scanner.Text() 62 | if line == "" { 63 | continue 64 | } 65 | 66 | 67 | if decodedLine, err := url.QueryUnescape(line); err == nil { 68 | line = decodedLine 69 | } 70 | 71 | 72 | subdomains := w.extractSubdomains(line, domain) 73 | 74 | for _, subdomain := range subdomains { 75 | 76 | subdomain = strings.ToLower(subdomain) 77 | subdomain = strings.TrimPrefix(subdomain, "25") 78 | subdomain = strings.TrimPrefix(subdomain, "2f") 79 | 80 | hostname := strings.TrimSpace(subdomain) 81 | 82 | if w.isValidHostname(hostname, domain) && !seen[hostname] { 83 | seen[hostname] = true 84 | 85 | select { 86 | case results <- Result{Source: w.Name(), Value: hostname, Type: "subdomain"}: 87 | case <-ctx.Done(): 88 | return 89 | } 90 | } 91 | } 92 | } 93 | 94 | if err := scanner.Err(); err != nil { 95 | results <- Result{Source: w.Name(), Error: fmt.Errorf("scanner error: %w", err)} 96 | } 97 | }() 98 | 99 | return results 100 | } 101 | 102 | 103 | func (w *WaybackArchive) extractSubdomains(line, domain string) []string { 104 | var subdomains []string 105 | 106 | 107 | domainRegex := regexp.MustCompile(`([a-zA-Z0-9.-]+\.[a-zA-Z0-9.-]+)`) 108 | matches := domainRegex.FindAllStringSubmatch(line, -1) 109 | 110 | for _, match := range matches { 111 | if len(match) > 1 { 112 | hostname := strings.TrimSpace(strings.ToLower(match[1])) 113 | 114 | 115 | if strings.HasSuffix(hostname, "."+domain) || hostname == domain { 116 | subdomains = append(subdomains, hostname) 117 | } 118 | } 119 | } 120 | 121 | return subdomains 122 | } 123 | 124 | 125 | func (w *WaybackArchive) isValidHostname(hostname, domain string) bool { 126 | if hostname == "" { 127 | return false 128 | } 129 | 130 | 131 | if strings.Contains(hostname, "..") || strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 132 | return false 133 | } 134 | 135 | 136 | if !strings.Contains(hostname, ".") { 137 | return false 138 | } 139 | 140 | 141 | if !strings.HasSuffix(hostname, "."+domain) && hostname != domain { 142 | return false 143 | } 144 | 145 | 146 | for _, char := range hostname { 147 | if !((char >= 'a' && char <= 'z') || (char >= '0' && char <= '9') || char == '.' || char == '-') { 148 | return false 149 | } 150 | } 151 | 152 | 153 | if len(hostname) > 253 { 154 | return false 155 | } 156 | 157 | return true 158 | } 159 | -------------------------------------------------------------------------------- /pkg/active/cleaner.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "regexp" 8 | "sort" 9 | ) 10 | 11 | type WordlistCleaner struct { 12 | regexes []*regexp.Regexp 13 | } 14 | 15 | func NewWordlistCleaner() *WordlistCleaner { 16 | patterns := []string{ 17 | `[\!(,%]`, // ignore noisy characters 18 | `.{100,}`, // ignore lines with more than 100 characters (overly specific) 19 | `[0-9]{4,}`, // ignore lines with 4 or more consecutive digits (likely an id) 20 | `[0-9]{3,}$`, // ignore lines where the last 3 or more characters are digits (likely an id) 21 | `[a-z0-9]{32}`, // likely MD5 hash or similar 22 | `[0-9]+[A-Z0-9]{5,}`, // number followed by 5 or more numbers and uppercase letters (almost all noise) 23 | `\/.*\/.*\/.*\/.*\/.*\/.*\/`, // ignore lines more than 6 directories deep (overly specific) 24 | `\w{8}-\w{4}-\w{4}-\w{4}-\w{12}`, // ignore UUIDs 25 | `[0-9]+[a-zA-Z]+[0-9]+[a-zA-Z]+[0-9]+`, // ignore multiple numbers and letters mixed together (likely noise) 26 | `\.(png|jpg|jpeg|gif|svg|bmp|ttf|avif|wav|mp4|aac|ajax|css|all)$`, // ignore low value filetypes 27 | `^$`, // ignores blank lines 28 | } 29 | 30 | var regexes []*regexp.Regexp 31 | for _, pattern := range patterns { 32 | re, err := regexp.Compile(pattern) 33 | if err == nil { 34 | regexes = append(regexes, re) 35 | } 36 | } 37 | 38 | return &WordlistCleaner{ 39 | regexes: regexes, 40 | } 41 | } 42 | 43 | func (wc *WordlistCleaner) CleanWordlist(words []string) []string { 44 | var filtered []string 45 | for _, word := range words { 46 | if !wc.shouldFilter(word) { 47 | filtered = append(filtered, word) 48 | } 49 | } 50 | 51 | uniqueMap := make(map[string]bool) 52 | for _, word := range filtered { 53 | uniqueMap[word] = true 54 | } 55 | 56 | var result []string 57 | for word := range uniqueMap { 58 | result = append(result, word) 59 | } 60 | sort.Strings(result) 61 | 62 | return result 63 | } 64 | 65 | func (wc *WordlistCleaner) CleanWordlistFile(inputFile, outputFile string) (int, int, error) { 66 | words, err := readWordlistFile(inputFile) 67 | if err != nil { 68 | return 0, 0, fmt.Errorf("failed to read wordlist: %w", err) 69 | } 70 | 71 | originalSize := len(words) 72 | 73 | cleaned := wc.CleanWordlist(words) 74 | newSize := len(cleaned) 75 | 76 | if err := writeWordlistFile(cleaned, outputFile); err != nil { 77 | return 0, 0, fmt.Errorf("failed to write cleaned wordlist: %w", err) 78 | } 79 | 80 | return originalSize, newSize, nil 81 | } 82 | 83 | func (wc *WordlistCleaner) shouldFilter(line string) bool { 84 | for _, re := range wc.regexes { 85 | if re.MatchString(line) { 86 | return true 87 | } 88 | } 89 | return false 90 | } 91 | 92 | func readWordlistFile(filePath string) ([]string, error) { 93 | file, err := os.Open(filePath) 94 | if err != nil { 95 | return nil, err 96 | } 97 | defer file.Close() 98 | 99 | var words []string 100 | scanner := bufio.NewScanner(file) 101 | for scanner.Scan() { 102 | line := scanner.Text() 103 | words = append(words, line) 104 | } 105 | 106 | return words, scanner.Err() 107 | } 108 | 109 | func writeWordlistFile(words []string, filePath string) error { 110 | file, err := os.Create(filePath) 111 | if err != nil { 112 | return err 113 | } 114 | defer file.Close() 115 | 116 | writer := bufio.NewWriter(file) 117 | for _, word := range words { 118 | if _, err := writer.WriteString(word + "\n"); err != nil { 119 | return err 120 | } 121 | } 122 | 123 | return writer.Flush() 124 | } 125 | 126 | func CleanAndSaveWordlist(inputFile, outputFile string, verbose bool) error { 127 | cleaner := NewWordlistCleaner() 128 | 129 | if verbose { 130 | fmt.Printf("[DBG] cleaning wordlist: %s\n", inputFile) 131 | } 132 | 133 | originalSize, newSize, err := cleaner.CleanWordlistFile(inputFile, outputFile) 134 | if err != nil { 135 | return err 136 | } 137 | 138 | removed := originalSize - newSize 139 | 140 | if verbose { 141 | fmt.Printf("[DBG] removed %d lines, wordlist now has %d lines\n", removed, newSize) 142 | } 143 | 144 | return nil 145 | } 146 | -------------------------------------------------------------------------------- /pkg/sources/abuseipdb.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "regexp" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | // AbuseIPDB struct for the AbuseIPDB source 14 | type AbuseIPDB struct{} 15 | 16 | // Regex pattern to match
  • tags 17 | var liTagPattern = regexp.MustCompile(`
  • \w[^<]*
    • `) 18 | 19 | // Name returns the name of the source 20 | func (a *AbuseIPDB) Name() string { 21 | return "abuseipdb" 22 | } 23 | 24 | // Run performs subdomain enumeration using AbuseIPDB WHOIS service 25 | func (a *AbuseIPDB) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 26 | results := make(chan Result) 27 | 28 | go func() { 29 | defer close(results) 30 | 31 | // Construct the URL 32 | url := fmt.Sprintf("https://www.abuseipdb.com/whois/%s", domain) 33 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 34 | if err != nil { 35 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 36 | return 37 | } 38 | 39 | // Set headers - AbuseIPDB requires Firefox-like user agent to avoid 403 40 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:110.0) Gecko/20100101 Firefox/110.0") 41 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") 42 | req.Header.Set("Accept-Language", "en-US,en;q=0.9") 43 | req.Header.Set("Accept-Encoding", "gzip, deflate, br") 44 | req.Header.Set("Referer", "https://www.abuseipdb.com/") 45 | 46 | // Set session cookie to bypass basic anti-bot protection 47 | req.Header.Set("Cookie", "abuseipdb_session=; XSRF-TOKEN=") 48 | 49 | resp, err := s.Client.Do(req) 50 | if err != nil { 51 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 52 | return 53 | } 54 | defer resp.Body.Close() 55 | 56 | if resp.StatusCode != http.StatusOK { 57 | results <- Result{Source: a.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 58 | return 59 | } 60 | 61 | // Read the response body 62 | body, err := io.ReadAll(resp.Body) 63 | if err != nil { 64 | results <- Result{Source: a.Name(), Error: fmt.Errorf("failed to read response body: %w", err)} 65 | return 66 | } 67 | 68 | html := string(body) 69 | 70 | // Extract subdomains from
  • tags 71 | matches := liTagPattern.FindAllString(html, -1) 72 | seen := make(map[string]bool) 73 | 74 | for _, match := range matches { 75 | // Remove
  • and
    • tags 76 | subdomain := strings.TrimSpace(match) 77 | subdomain = strings.TrimPrefix(subdomain, "
  • ") 78 | subdomain = strings.TrimSuffix(subdomain, "
    • ") 79 | subdomain = strings.TrimSpace(subdomain) 80 | 81 | // Skip if empty 82 | if subdomain == "" { 83 | continue 84 | } 85 | 86 | // Construct full domain 87 | fullDomain := subdomain + "." + domain 88 | hostname := strings.TrimSpace(strings.ToLower(fullDomain)) 89 | 90 | // Validate and send 91 | if hostname != "" && hostname != domain && 92 | strings.HasSuffix(hostname, "."+domain) && 93 | a.isValidHostname(hostname) && 94 | !seen[hostname] { 95 | seen[hostname] = true 96 | 97 | select { 98 | case results <- Result{Source: a.Name(), Value: hostname, Type: "subdomain"}: 99 | case <-ctx.Done(): 100 | return 101 | } 102 | } 103 | } 104 | }() 105 | 106 | return results 107 | } 108 | 109 | // isValidHostname validates a hostname 110 | func (a *AbuseIPDB) isValidHostname(hostname string) bool { 111 | if len(hostname) == 0 || len(hostname) > 253 { 112 | return false 113 | } 114 | 115 | // Check for leading or trailing dots 116 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 117 | return false 118 | } 119 | 120 | if strings.Contains(hostname, "..") { 121 | return false 122 | } 123 | 124 | // Must contain at least one dot 125 | if !strings.Contains(hostname, ".") { 126 | return false 127 | } 128 | 129 | // Check for valid characters 130 | for _, char := range hostname { 131 | if !((char >= 'a' && char <= 'z') || 132 | (char >= '0' && char <= '9') || 133 | char == '-' || char == '.') { 134 | return false 135 | } 136 | } 137 | 138 | // Check for leading or trailing hyphens 139 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 140 | return false 141 | } 142 | 143 | if strings.Contains(hostname, "--") { 144 | return false 145 | } 146 | 147 | return true 148 | } 149 | -------------------------------------------------------------------------------- /pkg/sources/rapiddns.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "net/http" 8 | "regexp" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | 15 | type RapidDNS struct{} 16 | 17 | 18 | var pagePattern = regexp.MustCompile(`class="page-link" href="/subdomain/[^"]+\?page=(\d+)">`) 19 | 20 | 21 | func (r *RapidDNS) Name() string { 22 | return "rapiddns" 23 | } 24 | 25 | 26 | func (r *RapidDNS) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 27 | results := make(chan Result) 28 | 29 | go func() { 30 | defer close(results) 31 | 32 | page := 1 33 | maxPages := 1 34 | seen := make(map[string]bool) 35 | 36 | for { 37 | select { 38 | case <-ctx.Done(): 39 | return 40 | default: 41 | } 42 | 43 | 44 | url := fmt.Sprintf("https://rapiddns.io/subdomain/%s?page=%d&full=1", domain, page) 45 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 46 | if err != nil { 47 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to create request for page %d: %w", page, err)} 48 | return 49 | } 50 | 51 | 52 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 53 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") 54 | 55 | resp, err := s.Client.Do(req) 56 | if err != nil { 57 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to execute request for page %d: %w", page, err)} 58 | return 59 | } 60 | defer resp.Body.Close() 61 | 62 | if resp.StatusCode != http.StatusOK { 63 | results <- Result{Source: r.Name(), Error: fmt.Errorf("HTTP error for page %d: %d", page, resp.StatusCode)} 64 | return 65 | } 66 | 67 | 68 | body, err := io.ReadAll(resp.Body) 69 | if err != nil { 70 | results <- Result{Source: r.Name(), Error: fmt.Errorf("failed to read response body for page %d: %w", page, err)} 71 | return 72 | } 73 | 74 | src := string(body) 75 | 76 | 77 | subdomains := r.extractSubdomains(src, domain) 78 | for _, subdomain := range subdomains { 79 | if !seen[subdomain] { 80 | seen[subdomain] = true 81 | 82 | select { 83 | case results <- Result{Source: r.Name(), Value: subdomain, Type: "subdomain"}: 84 | case <-ctx.Done(): 85 | return 86 | } 87 | } 88 | } 89 | 90 | 91 | if maxPages == 1 { 92 | matches := pagePattern.FindAllStringSubmatch(src, -1) 93 | if len(matches) > 0 { 94 | lastMatch := matches[len(matches)-1] 95 | if len(lastMatch) > 1 { 96 | if maxPagesFound, err := strconv.Atoi(lastMatch[1]); err == nil { 97 | maxPages = maxPagesFound 98 | } 99 | } 100 | } 101 | } 102 | 103 | 104 | if page >= maxPages { 105 | break 106 | } 107 | page++ 108 | } 109 | }() 110 | 111 | return results 112 | } 113 | 114 | 115 | func (r *RapidDNS) extractSubdomains(html, domain string) []string { 116 | var subdomains []string 117 | 118 | 119 | 120 | 121 | 122 | domainPattern := regexp.MustCompile(`(?i)([a-z0-9]([a-z0-9\-]*[a-z0-9])?\.)+` + regexp.QuoteMeta(domain)) 123 | 124 | matches := domainPattern.FindAllString(html, -1) 125 | for _, match := range matches { 126 | hostname := strings.TrimSpace(strings.ToLower(match)) 127 | 128 | 129 | if hostname != "" && hostname != domain && 130 | strings.HasSuffix(hostname, "."+domain) && 131 | r.isValidHostname(hostname) { 132 | subdomains = append(subdomains, hostname) 133 | } 134 | } 135 | 136 | return subdomains 137 | } 138 | 139 | 140 | func (r *RapidDNS) isValidHostname(hostname string) bool { 141 | if len(hostname) == 0 || len(hostname) > 253 { 142 | return false 143 | } 144 | 145 | 146 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 147 | return false 148 | } 149 | 150 | if strings.Contains(hostname, "..") { 151 | return false 152 | } 153 | 154 | 155 | if !strings.Contains(hostname, ".") { 156 | return false 157 | } 158 | 159 | 160 | for _, char := range hostname { 161 | if !((char >= 'a' && char <= 'z') || 162 | (char >= '0' && char <= '9') || 163 | char == '-' || char == '.') { 164 | return false 165 | } 166 | } 167 | 168 | 169 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 170 | return false 171 | } 172 | 173 | if strings.Contains(hostname, "--") { 174 | return false 175 | } 176 | 177 | return true 178 | } 179 | -------------------------------------------------------------------------------- /pkg/sources/censys.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strconv" 11 | "strings" 12 | ) 13 | 14 | type Censys struct{} 15 | 16 | type CensysResponse struct { 17 | Result CensysResult `json:"result"` 18 | Status string `json:"status"` 19 | } 20 | 21 | type CensysResult struct { 22 | Query string `json:"query"` 23 | Total int `json:"total"` 24 | DurationMS int `json:"duration_ms"` 25 | Hits []CensysHit `json:"hits"` 26 | Links CensysLinks `json:"links"` 27 | } 28 | 29 | type CensysHit struct { 30 | Parsed CensysParsed `json:"parsed"` 31 | Names []string `json:"names"` 32 | FingerprintSha256 string `json:"fingerprint_sha256"` 33 | } 34 | 35 | type CensysParsed struct { 36 | ValidityPeriod CensysValidityPeriod `json:"validity_period"` 37 | SubjectDN string `json:"subject_dn"` 38 | IssuerDN string `json:"issuer_dn"` 39 | } 40 | 41 | type CensysValidityPeriod struct { 42 | NotAfter string `json:"not_after"` 43 | NotBefore string `json:"not_before"` 44 | } 45 | 46 | type CensysLinks struct { 47 | Next string `json:"next"` 48 | Prev string `json:"prev"` 49 | } 50 | 51 | const ( 52 | maxCensysPages = 10 53 | maxPerPage = 100 54 | ) 55 | 56 | func (c *Censys) Name() string { 57 | return "censys" 58 | } 59 | 60 | func (c *Censys) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 61 | results := make(chan Result) 62 | 63 | go func() { 64 | defer close(results) 65 | 66 | if s.Keys.Censys == "" { 67 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Censys API key not configured")} 68 | return 69 | } 70 | 71 | cursor := "" 72 | currentPage := 1 73 | 74 | for { 75 | baseURL := "https://api.platform.censys.io/v3/global/search/certificates" 76 | u, err := url.Parse(baseURL) 77 | if err != nil { 78 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to parse URL: %w", err)} 79 | return 80 | } 81 | 82 | params := u.Query() 83 | params.Add("q", fmt.Sprintf("names: *.%s", domain)) 84 | params.Add("per_page", strconv.Itoa(maxPerPage)) 85 | if cursor != "" { 86 | params.Add("cursor", cursor) 87 | } 88 | u.RawQuery = params.Encode() 89 | 90 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) 91 | if err != nil { 92 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 93 | return 94 | } 95 | 96 | req.Header.Set("Authorization", "Bearer "+s.Keys.Censys) 97 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 98 | req.Header.Set("Accept", "application/vnd.censys.api.v3.certificate.v1+json") 99 | 100 | resp, err := s.Client.Do(req) 101 | if err != nil { 102 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 103 | return 104 | } 105 | defer resp.Body.Close() 106 | 107 | if resp.StatusCode == 401 { 108 | results <- Result{Source: c.Name(), Error: fmt.Errorf("invalid Censys API credentials")} 109 | return 110 | } 111 | 112 | if resp.StatusCode == 429 { 113 | results <- Result{Source: c.Name(), Error: fmt.Errorf("Censys rate limit exceeded")} 114 | return 115 | } 116 | 117 | if resp.StatusCode != http.StatusOK { 118 | results <- Result{Source: c.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 119 | return 120 | } 121 | 122 | var censysResp CensysResponse 123 | if err := json.NewDecoder(resp.Body).Decode(&censysResp); err != nil { 124 | results <- Result{Source: c.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 125 | return 126 | } 127 | 128 | seen := make(map[string]bool) 129 | for _, hit := range censysResp.Result.Hits { 130 | for _, name := range hit.Names { 131 | hostname := strings.TrimSpace(strings.ToLower(name)) 132 | 133 | if hostname != "" && strings.HasSuffix(hostname, "."+domain) && !seen[hostname] { 134 | seen[hostname] = true 135 | 136 | select { 137 | case results <- Result{Source: c.Name(), Value: hostname, Type: "subdomain"}: 138 | case <-ctx.Done(): 139 | return 140 | } 141 | } 142 | } 143 | } 144 | 145 | cursor = censysResp.Result.Links.Next 146 | if cursor == "" || currentPage >= maxCensysPages { 147 | break 148 | } 149 | currentPage++ 150 | } 151 | }() 152 | 153 | return results 154 | } 155 | -------------------------------------------------------------------------------- /pkg/active/deep.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "sync" 7 | ) 8 | 9 | func RunDeepEnumeration(resolvedSubdomains []string, outputDir, domain string, verbose bool) ([]string, error) { 10 | if verbose { 11 | fmt.Println("[DBG] starting deep level enumeration...") 12 | } 13 | 14 | finalSubsFile := filepath.Join(outputDir, "final_subdomains.txt") 15 | if err := writeSubdomainsToFile(resolvedSubdomains, finalSubsFile); err != nil { 16 | return nil, fmt.Errorf("failed to write final subdomains: %w", err) 17 | } 18 | 19 | if verbose { 20 | fmt.Printf("[DBG] running parallel dsieve (f3, f4, f5) on %d subdomains...\n", len(resolvedSubdomains)) 21 | } 22 | 23 | type dsieveResult struct { 24 | factor string 25 | subdomains []string 26 | err error 27 | } 28 | 29 | results := make(chan dsieveResult, 3) 30 | var wg sync.WaitGroup 31 | 32 | factors := []struct { 33 | factor string 34 | level string 35 | }{ 36 | {"3", "3"}, 37 | {"4", "4"}, 38 | {"5", "5"}, 39 | } 40 | 41 | for _, f := range factors { 42 | wg.Add(1) 43 | go func(factor, level string) { 44 | defer wg.Done() 45 | 46 | outputFile := filepath.Join(outputDir, fmt.Sprintf("dsieve_f%s_output.txt", factor)) 47 | subs, err := RunDsieve(finalSubsFile, outputFile, level, 5) 48 | 49 | results <- dsieveResult{ 50 | factor: factor, 51 | subdomains: subs, 52 | err: err, 53 | } 54 | 55 | if verbose && err == nil { 56 | fmt.Printf("[DBG] dsieve f%s: generated %d subdomains\n", factor, len(subs)) 57 | } 58 | }(f.factor, f.level) 59 | } 60 | 61 | go func() { 62 | wg.Wait() 63 | close(results) 64 | }() 65 | 66 | var f3, f4, f5 []string 67 | for result := range results { 68 | if result.err != nil { 69 | return nil, fmt.Errorf("dsieve f%s failed: %w", result.factor, result.err) 70 | } 71 | 72 | switch result.factor { 73 | case "3": 74 | f3 = result.subdomains 75 | case "4": 76 | f4 = result.subdomains 77 | case "5": 78 | f5 = result.subdomains 79 | } 80 | } 81 | 82 | fmt.Printf("[DEEP] Dsieve generated: f3=%d, f4=%d, f5=%d subdomains\n", len(f3), len(f4), len(f5)) 83 | 84 | if verbose { 85 | fmt.Println("[DBG] downloading and merging Trickest wordlists...") 86 | } 87 | 88 | level2, level3, level4plus, err := DownloadAndMergeTrickestWordlists(outputDir, verbose) 89 | if err != nil { 90 | return nil, fmt.Errorf("failed to prepare wordlists: %w", err) 91 | } 92 | 93 | normalResolverFile := filepath.Join(outputDir, "resolvers.txt") 94 | trustedResolverFile := filepath.Join(outputDir, "resolvers_trusted.txt") 95 | 96 | if verbose { 97 | fmt.Println("[DBG] running puredns bruteforce (3 levels)...") 98 | } 99 | 100 | fmt.Println("[DEEP] Starting bruteforce with level2 wordlist...") 101 | resolvedF3, err := RunPurednsBruteforce(level2, f3, normalResolverFile, trustedResolverFile, outputDir, domain, verbose) 102 | if err != nil { 103 | return nil, fmt.Errorf("bruteforce f3 failed: %w", err) 104 | } 105 | fmt.Printf("[DEEP] Level2 bruteforce: %d/%d resolved\n", len(resolvedF3), len(f3)) 106 | 107 | outputF3File := filepath.Join(outputDir, "deep_f3_resolved.txt") 108 | if err := writeSubdomainsToFile(resolvedF3, outputF3File); err != nil { 109 | return nil, fmt.Errorf("failed to write f3 results: %w", err) 110 | } 111 | 112 | fmt.Println("[DEEP] Starting bruteforce with level3 wordlist...") 113 | resolvedF4, err := RunPurednsBruteforce(level3, f4, normalResolverFile, trustedResolverFile, outputDir, domain, verbose) 114 | if err != nil { 115 | return nil, fmt.Errorf("bruteforce f4 failed: %w", err) 116 | } 117 | fmt.Printf("[DEEP] Level3 bruteforce: %d/%d resolved\n", len(resolvedF4), len(f4)) 118 | 119 | outputF4File := filepath.Join(outputDir, "deep_f4_resolved.txt") 120 | if err := writeSubdomainsToFile(resolvedF4, outputF4File); err != nil { 121 | return nil, fmt.Errorf("failed to write f4 results: %w", err) 122 | } 123 | 124 | fmt.Println("[DEEP] Starting bruteforce with level4plus wordlist...") 125 | resolvedF5, err := RunPurednsBruteforce(level4plus, f5, normalResolverFile, trustedResolverFile, outputDir, domain, verbose) 126 | if err != nil { 127 | return nil, fmt.Errorf("bruteforce f5 failed: %w", err) 128 | } 129 | fmt.Printf("[DEEP] Level4plus bruteforce: %d/%d resolved\n", len(resolvedF5), len(f5)) 130 | 131 | outputF5File := filepath.Join(outputDir, "deep_f5_resolved.txt") 132 | if err := writeSubdomainsToFile(resolvedF5, outputF5File); err != nil { 133 | return nil, fmt.Errorf("failed to write f5 results: %w", err) 134 | } 135 | 136 | allDeepSubdomains := MergeAndDeduplicate(resolvedF3, resolvedF4, resolvedF5) 137 | 138 | if verbose { 139 | fmt.Printf("[DBG] deep enumeration complete: %d total unique subdomains\n", len(allDeepSubdomains)) 140 | } 141 | 142 | return allDeepSubdomains, nil 143 | } 144 | -------------------------------------------------------------------------------- /pkg/sources/driftnet.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | "sync" 12 | "sync/atomic" 13 | ) 14 | 15 | 16 | type Driftnet struct{} 17 | 18 | const ( 19 | 20 | baseURL = "https://api.driftnet.io/v1/" 21 | 22 | summaryLimit = 10000 23 | ) 24 | 25 | 26 | type DriftnetEndpointConfig struct { 27 | 28 | Endpoint string 29 | 30 | Param string 31 | 32 | Context string 33 | } 34 | 35 | 36 | type DriftnetSummaryResponse struct { 37 | Summary struct { 38 | Other int `json:"other"` 39 | Values map[string]int `json:"values"` 40 | } `json:"summary"` 41 | } 42 | 43 | 44 | var endpoints = []DriftnetEndpointConfig{ 45 | {"ct/log", "field=host:", "cert-dns-name"}, 46 | {"scan/protocols", "field=host:", "cert-dns-name"}, 47 | {"scan/domains", "field=host:", "cert-dns-name"}, 48 | {"domain/rdns", "host=", "dns-ptr"}, 49 | } 50 | 51 | 52 | func (d *Driftnet) Name() string { 53 | return "driftnet" 54 | } 55 | 56 | 57 | func (d *Driftnet) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 58 | results := make(chan Result) 59 | 60 | go func() { 61 | defer close(results) 62 | 63 | 64 | if s.Keys.Driftnet == "" { 65 | results <- Result{Source: d.Name(), Error: fmt.Errorf("Driftnet API key not configured")} 66 | return 67 | } 68 | 69 | 70 | var wg sync.WaitGroup 71 | var errors atomic.Int32 72 | var totalResults atomic.Int32 73 | dedupe := sync.Map{} 74 | 75 | 76 | wg.Add(len(endpoints)) 77 | for i := range endpoints { 78 | go d.runSubsource(ctx, domain, s, results, &wg, &dedupe, &errors, &totalResults, endpoints[i]) 79 | } 80 | 81 | 82 | wg.Wait() 83 | }() 84 | 85 | return results 86 | } 87 | 88 | 89 | func (d *Driftnet) runSubsource(ctx context.Context, domain string, s *session.Session, results chan<- Result, wg *sync.WaitGroup, dedupe *sync.Map, errors *atomic.Int32, totalResults *atomic.Int32, epConfig DriftnetEndpointConfig) { 90 | defer wg.Done() 91 | 92 | 93 | requestURL := fmt.Sprintf("%s%s?%s%s&summarize=host&summary_context=%s&summary_limit=%d", 94 | baseURL, epConfig.Endpoint, epConfig.Param, url.QueryEscape(domain), epConfig.Context, summaryLimit) 95 | 96 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, requestURL, nil) 97 | if err != nil { 98 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request for %s: %w", epConfig.Endpoint, err)} 99 | errors.Add(1) 100 | return 101 | } 102 | 103 | 104 | req.Header.Set("Authorization", "Bearer "+s.Keys.Driftnet) 105 | req.Header.Set("Accept", "application/json") 106 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 107 | 108 | resp, err := s.Client.Do(req) 109 | if err != nil { 110 | 111 | if resp == nil || resp.StatusCode != http.StatusNoContent { 112 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to execute request for %s: %w", epConfig.Endpoint, err)} 113 | errors.Add(1) 114 | } 115 | return 116 | } 117 | defer resp.Body.Close() 118 | 119 | 120 | if resp.StatusCode == 401 { 121 | results <- Result{Source: d.Name(), Error: fmt.Errorf("invalid Driftnet API key")} 122 | errors.Add(1) 123 | return 124 | } 125 | 126 | if resp.StatusCode == 429 { 127 | results <- Result{Source: d.Name(), Error: fmt.Errorf("Driftnet rate limit exceeded")} 128 | errors.Add(1) 129 | return 130 | } 131 | 132 | 133 | if resp.StatusCode == 204 { 134 | return 135 | } 136 | 137 | if resp.StatusCode != http.StatusOK { 138 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error from %s: %d", epConfig.Endpoint, resp.StatusCode)} 139 | errors.Add(1) 140 | return 141 | } 142 | 143 | 144 | var summary DriftnetSummaryResponse 145 | if err := json.NewDecoder(resp.Body).Decode(&summary); err != nil { 146 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to decode JSON from %s: %w", epConfig.Endpoint, err)} 147 | errors.Add(1) 148 | return 149 | } 150 | 151 | 152 | for subdomain := range summary.Summary.Values { 153 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 154 | 155 | 156 | if hostname != "" && hostname != domain && strings.HasSuffix(hostname, "."+domain) { 157 | 158 | if _, present := dedupe.LoadOrStore(hostname, true); !present { 159 | totalResults.Add(1) 160 | 161 | select { 162 | case results <- Result{Source: d.Name(), Value: hostname, Type: "subdomain"}: 163 | case <-ctx.Done(): 164 | return 165 | } 166 | } 167 | } 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /pkg/sources/rsecloud.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type RSECloud struct{} 14 | 15 | 16 | type RSECloudResponse struct { 17 | Count int `json:"count"` 18 | Data []string `json:"data"` 19 | Page int `json:"page"` 20 | PageSize int `json:"pagesize"` 21 | TotalPages int `json:"total_pages"` 22 | } 23 | 24 | 25 | func (r *RSECloud) Name() string { 26 | return "rsecloud" 27 | } 28 | 29 | 30 | func (r *RSECloud) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 31 | results := make(chan Result) 32 | 33 | go func() { 34 | defer close(results) 35 | 36 | 37 | if s.Keys.RSECloud == "" { 38 | results <- Result{Source: r.Name(), Error: fmt.Errorf("RSECloud API key not configured")} 39 | return 40 | } 41 | 42 | seen := make(map[string]bool) 43 | 44 | 45 | endpoints := []string{"active", "passive"} 46 | 47 | for _, endpoint := range endpoints { 48 | select { 49 | case <-ctx.Done(): 50 | return 51 | default: 52 | } 53 | 54 | err := r.fetchSubdomains(ctx, endpoint, domain, s, results, seen) 55 | if err != nil { 56 | results <- Result{Source: r.Name(), Error: fmt.Errorf("%s endpoint failed: %w", endpoint, err)} 57 | continue 58 | } 59 | } 60 | }() 61 | 62 | return results 63 | } 64 | 65 | 66 | func (r *RSECloud) fetchSubdomains(ctx context.Context, endpoint, domain string, s *session.Session, results chan<- Result, seen map[string]bool) error { 67 | page := 1 68 | 69 | for { 70 | select { 71 | case <-ctx.Done(): 72 | return nil 73 | default: 74 | } 75 | 76 | 77 | url := fmt.Sprintf("https://api.rsecloud.com/api/v2/subdomains/%s/%s?page=%d", endpoint, domain, page) 78 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 79 | if err != nil { 80 | return fmt.Errorf("failed to create request for page %d: %w", page, err) 81 | } 82 | 83 | 84 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 85 | req.Header.Set("Content-Type", "application/json") 86 | req.Header.Set("Accept", "application/json") 87 | req.Header.Set("X-API-Key", s.Keys.RSECloud) 88 | 89 | resp, err := s.Client.Do(req) 90 | if err != nil { 91 | return fmt.Errorf("failed to execute request for page %d: %w", page, err) 92 | } 93 | defer resp.Body.Close() 94 | 95 | if resp.StatusCode == 401 { 96 | return fmt.Errorf("invalid RSECloud API key") 97 | } 98 | 99 | if resp.StatusCode == 429 { 100 | return fmt.Errorf("RSECloud rate limit exceeded") 101 | } 102 | 103 | if resp.StatusCode != http.StatusOK { 104 | return fmt.Errorf("HTTP error for page %d: %d", page, resp.StatusCode) 105 | } 106 | 107 | 108 | var rseCloudResponse RSECloudResponse 109 | if err := json.NewDecoder(resp.Body).Decode(&rseCloudResponse); err != nil { 110 | return fmt.Errorf("failed to decode JSON for page %d: %w", page, err) 111 | } 112 | 113 | 114 | for _, subdomain := range rseCloudResponse.Data { 115 | hostname := strings.TrimSpace(strings.ToLower(subdomain)) 116 | 117 | 118 | if hostname != "" && hostname != domain && 119 | strings.HasSuffix(hostname, "."+domain) && 120 | r.isValidHostname(hostname) && !seen[hostname] { 121 | seen[hostname] = true 122 | 123 | select { 124 | case results <- Result{Source: r.Name(), Value: hostname, Type: "subdomain"}: 125 | case <-ctx.Done(): 126 | return nil 127 | } 128 | } 129 | } 130 | 131 | 132 | if page >= rseCloudResponse.TotalPages || len(rseCloudResponse.Data) == 0 { 133 | break 134 | } 135 | page++ 136 | } 137 | 138 | return nil 139 | } 140 | 141 | 142 | func (r *RSECloud) isValidHostname(hostname string) bool { 143 | if len(hostname) == 0 || len(hostname) > 253 { 144 | return false 145 | } 146 | 147 | 148 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 149 | return false 150 | } 151 | 152 | if strings.Contains(hostname, "..") { 153 | return false 154 | } 155 | 156 | 157 | if !strings.Contains(hostname, ".") { 158 | return false 159 | } 160 | 161 | 162 | for _, char := range hostname { 163 | if !((char >= 'a' && char <= 'z') || 164 | (char >= '0' && char <= '9') || 165 | char == '-' || char == '.') { 166 | return false 167 | } 168 | } 169 | 170 | 171 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 172 | return false 173 | } 174 | 175 | if strings.Contains(hostname, "--") { 176 | return false 177 | } 178 | 179 | return true 180 | } 181 | -------------------------------------------------------------------------------- /pkg/sources/dnsgrep.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "regexp" 8 | "github.com/samogod/samoscout/pkg/session" 9 | "strings" 10 | ) 11 | 12 | 13 | type Dnsgrep struct{} 14 | 15 | 16 | func (d *Dnsgrep) Name() string { 17 | return "dnsgrep" 18 | } 19 | 20 | 21 | func (d *Dnsgrep) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 22 | results := make(chan Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | url := fmt.Sprintf("https://www.dnsgrep.cn/subdomain/%s", domain) 28 | 29 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 30 | if err != nil { 31 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 32 | return 33 | } 34 | 35 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 36 | req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") 37 | req.Header.Set("Accept-Language", "en-US,en;q=0.5") 38 | req.Header.Set("Accept-Encoding", "gzip, deflate") 39 | req.Header.Set("Connection", "keep-alive") 40 | req.Header.Set("Upgrade-Insecure-Requests", "1") 41 | 42 | resp, err := s.Client.Do(req) 43 | if err != nil { 44 | results <- Result{Source: d.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 45 | return 46 | } 47 | defer resp.Body.Close() 48 | 49 | if resp.StatusCode != http.StatusOK { 50 | results <- Result{Source: d.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 51 | return 52 | } 53 | 54 | body := make([]byte, 0, 32*1024) 55 | buf := make([]byte, 4096) 56 | for { 57 | select { 58 | case <-ctx.Done(): 59 | return 60 | default: 61 | } 62 | 63 | n, err := resp.Body.Read(buf) 64 | if n > 0 { 65 | body = append(body, buf[:n]...) 66 | } 67 | if err != nil { 68 | break 69 | } 70 | } 71 | 72 | d.extractSubdomains(string(body), domain, results) 73 | }() 74 | 75 | return results 76 | } 77 | 78 | 79 | func (d *Dnsgrep) extractSubdomains(responseText, domain string, results chan<- Result) { 80 | seen := make(map[string]bool) 81 | 82 | subdomainPattern := regexp.MustCompile(`[a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?\.` + regexp.QuoteMeta(domain) + `\b`) 83 | 84 | matches := subdomainPattern.FindAllString(responseText, -1) 85 | 86 | for _, match := range matches { 87 | subdomain := strings.TrimSpace(strings.ToLower(match)) 88 | 89 | if subdomain != "" && subdomain != domain && !seen[subdomain] { 90 | seen[subdomain] = true 91 | 92 | select { 93 | case results <- Result{Source: d.Name(), Value: subdomain, Type: "subdomain"}: 94 | default: 95 | } 96 | } 97 | } 98 | 99 | jsonPattern := regexp.MustCompile(`["']([a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?\.` + regexp.QuoteMeta(domain) + `)["']`) 100 | jsonMatches := jsonPattern.FindAllStringSubmatch(responseText, -1) 101 | 102 | for _, match := range jsonMatches { 103 | if len(match) >= 2 { 104 | subdomain := strings.TrimSpace(strings.ToLower(match[1])) 105 | 106 | if subdomain != "" && subdomain != domain && !seen[subdomain] { 107 | seen[subdomain] = true 108 | 109 | select { 110 | case results <- Result{Source: d.Name(), Value: subdomain, Type: "subdomain"}: 111 | default: 112 | } 113 | } 114 | } 115 | } 116 | 117 | htmlPattern := regexp.MustCompile(`(?:href|src)=["'](?:https?://)?([a-zA-Z0-9]([a-zA-Z0-9\-]*[a-zA-Z0-9])?\.` + regexp.QuoteMeta(domain) + `)`) 118 | htmlMatches := htmlPattern.FindAllStringSubmatch(responseText, -1) 119 | 120 | for _, match := range htmlMatches { 121 | if len(match) >= 2 { 122 | subdomain := strings.TrimSpace(strings.ToLower(match[1])) 123 | 124 | if subdomain != "" && subdomain != domain && !seen[subdomain] { 125 | seen[subdomain] = true 126 | 127 | select { 128 | case results <- Result{Source: d.Name(), Value: subdomain, Type: "subdomain"}: 129 | default: 130 | } 131 | } 132 | } 133 | } 134 | 135 | lines := strings.Split(responseText, "\n") 136 | 137 | for _, line := range lines { 138 | line = strings.TrimSpace(line) 139 | 140 | if strings.Contains(line, "."+domain) { 141 | lineSubdomains := subdomainPattern.FindAllString(line, -1) 142 | for _, match := range lineSubdomains { 143 | subdomain := strings.TrimSpace(strings.ToLower(match)) 144 | 145 | if subdomain != "" && subdomain != domain && !seen[subdomain] { 146 | seen[subdomain] = true 147 | 148 | select { 149 | case results <- Result{Source: d.Name(), Value: subdomain, Type: "subdomain"}: 150 | default: 151 | } 152 | } 153 | } 154 | } 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/elastic/elastic-transport-go/v8 v8.5.0 h1:v5membAl7lvQgBTexPRDBO/RdnlQX+FM9fUVDyXxvH0= 6 | github.com/elastic/elastic-transport-go/v8 v8.5.0/go.mod h1:YLHer5cj0csTzNFXoNQ8qhtGY1GTvSqPnKWKaqQE3Hk= 7 | github.com/elastic/go-elasticsearch/v8 v8.13.0 h1:YXPAWpvbYX0mWSNG9tnEpvs4h1stgMy5JUeKZECYYB8= 8 | github.com/elastic/go-elasticsearch/v8 v8.13.0/go.mod h1:DIn7HopJs4oZC/w0WoJR13uMUxtHeq92eI5bqv5CRfI= 9 | github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= 10 | github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= 11 | github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= 12 | github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= 13 | github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= 14 | github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= 15 | github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= 16 | github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= 17 | github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= 18 | github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= 19 | github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= 20 | github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= 21 | github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= 22 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 23 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 24 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 25 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 26 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 27 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 28 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 29 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 30 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 31 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 32 | github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= 33 | github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= 34 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 35 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 36 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 37 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 38 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 39 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 40 | go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= 41 | go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= 42 | go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= 43 | go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= 44 | go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= 45 | go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= 46 | go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= 47 | go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= 48 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 50 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 51 | golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= 52 | golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 53 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 54 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 55 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 56 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 57 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 58 | -------------------------------------------------------------------------------- /pkg/active/mksub.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "os" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | // Mksub - Generate subdomain combinations 13 | // Direct copy from https://github.com/trickest/mksub (MIT License) 14 | 15 | func RunMksub(wordlistFile, domain, outputFile string, verbose bool) ([]string, error) { 16 | words, err := readWordlist(wordlistFile, "") 17 | if err != nil { 18 | return nil, fmt.Errorf("failed to read wordlist: %w", err) 19 | } 20 | 21 | if verbose { 22 | fmt.Printf("[DBG] loaded %d unique words from wordlist\n", len(words)) 23 | } 24 | 25 | domains := []string{domain} 26 | 27 | if verbose { 28 | fmt.Printf("[DBG] target domain: %s\n", domain) 29 | fmt.Printf("[DBG] generating combinations: %d words × 1 domain = ~%d subdomains\n", 30 | len(words), len(words)) 31 | } 32 | 33 | subdomains := generateSubdomains(words, domains, 1, 100, verbose) 34 | 35 | if verbose { 36 | fmt.Printf("[DBG] generated %d unique subdomains\n", len(subdomains)) 37 | } 38 | 39 | if outputFile != "" { 40 | if err := writeSubdomainsList(subdomains, outputFile); err != nil { 41 | return nil, err 42 | } 43 | } 44 | 45 | return subdomains, nil 46 | } 47 | 48 | func readWordlist(filePath, regexPattern string) ([]string, error) { 49 | file, err := os.Open(filePath) 50 | if err != nil { 51 | return nil, err 52 | } 53 | defer file.Close() 54 | 55 | var re *regexp.Regexp 56 | if regexPattern != "" { 57 | re, err = regexp.Compile(regexPattern) 58 | if err != nil { 59 | return nil, fmt.Errorf("invalid regex: %w", err) 60 | } 61 | } 62 | 63 | uniqueWords := make(map[string]bool) 64 | scanner := bufio.NewScanner(file) 65 | 66 | for scanner.Scan() { 67 | word := strings.TrimSpace(scanner.Text()) 68 | if word == "" { 69 | continue 70 | } 71 | 72 | wordLower := strings.ToLower(word) 73 | 74 | if re != nil && !re.MatchString(word) { 75 | continue 76 | } 77 | 78 | uniqueWords[wordLower] = true 79 | } 80 | 81 | if err := scanner.Err(); err != nil { 82 | return nil, err 83 | } 84 | 85 | var words []string 86 | for word := range uniqueWords { 87 | words = append(words, word) 88 | } 89 | 90 | return words, nil 91 | } 92 | 93 | func generateSubdomains(words []string, domains []string, level int, threads int, verbose bool) []string { 94 | var mu sync.Mutex 95 | results := make(map[string]bool) 96 | totalProcessed := 0 97 | 98 | for _, domain := range domains { 99 | domain = strings.TrimSpace(domain) 100 | if domain == "" { 101 | continue 102 | } 103 | 104 | currentLevel := []string{domain} 105 | 106 | for l := 1; l <= level; l++ { 107 | nextLevel := make(map[string]bool) 108 | var wg sync.WaitGroup 109 | semaphore := make(chan struct{}, threads) 110 | 111 | for _, baseDomain := range currentLevel { 112 | for _, word := range words { 113 | wg.Add(1) 114 | semaphore <- struct{}{} 115 | 116 | go func(base, w string) { 117 | defer wg.Done() 118 | defer func() { <-semaphore }() 119 | 120 | subdomain := fmt.Sprintf("%s.%s", w, base) 121 | 122 | mu.Lock() 123 | results[subdomain] = true 124 | nextLevel[subdomain] = true 125 | totalProcessed++ 126 | 127 | if verbose && totalProcessed%10000 == 0 { 128 | fmt.Printf("[DBG] progress: %d subdomains generated...\r", totalProcessed) 129 | } 130 | mu.Unlock() 131 | }(baseDomain, word) 132 | } 133 | } 134 | 135 | wg.Wait() 136 | 137 | if verbose { 138 | fmt.Printf("\n") 139 | } 140 | 141 | currentLevel = make([]string, 0, len(nextLevel)) 142 | for sub := range nextLevel { 143 | currentLevel = append(currentLevel, sub) 144 | } 145 | } 146 | } 147 | 148 | var subdomains []string 149 | for sub := range results { 150 | subdomains = append(subdomains, sub) 151 | } 152 | 153 | return subdomains 154 | } 155 | 156 | func readDomainsList(filePath string) ([]string, error) { 157 | file, err := os.Open(filePath) 158 | if err != nil { 159 | return nil, err 160 | } 161 | defer file.Close() 162 | 163 | var domains []string 164 | scanner := bufio.NewScanner(file) 165 | 166 | for scanner.Scan() { 167 | line := strings.TrimSpace(scanner.Text()) 168 | if line != "" && !strings.HasPrefix(line, "#") { 169 | domains = append(domains, line) 170 | } 171 | } 172 | 173 | return domains, scanner.Err() 174 | } 175 | 176 | func writeSubdomainsList(subdomains []string, filePath string) error { 177 | file, err := os.Create(filePath) 178 | if err != nil { 179 | return err 180 | } 181 | defer file.Close() 182 | 183 | writer := bufio.NewWriter(file) 184 | for _, sub := range subdomains { 185 | if _, err := writer.WriteString(sub + "\n"); err != nil { 186 | return err 187 | } 188 | } 189 | 190 | return writer.Flush() 191 | } 192 | 193 | -------------------------------------------------------------------------------- /pkg/sources/windvane.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type Windvane struct{} 15 | 16 | 17 | type windvaneRequest struct { 18 | Domain string `json:"domain"` 19 | PageRequest pageRequest `json:"page_request"` 20 | } 21 | 22 | 23 | type pageRequest struct { 24 | Page int `json:"page"` 25 | Count int `json:"count"` 26 | } 27 | 28 | 29 | type windvaneResponse struct { 30 | Code int `json:"code"` 31 | Msg string `json:"msg"` 32 | Data *data `json:"data"` 33 | ServerTime string `json:"server_time"` 34 | Version string `json:"version"` 35 | } 36 | 37 | 38 | type data struct { 39 | List []subdomain `json:"list"` 40 | PageResponse pageResponse `json:"page_response"` 41 | } 42 | 43 | 44 | type subdomain struct { 45 | LastUpdatedAt string `json:"last_updated_at"` 46 | Domain string `json:"domain"` 47 | UUID string `json:"uuid"` 48 | } 49 | 50 | 51 | type pageResponse struct { 52 | Total string `json:"total"` 53 | Count string `json:"count"` 54 | TotalPage string `json:"total_page"` 55 | } 56 | 57 | const ( 58 | windvaneBaseURL = "https://windvane.lichoin.com/trpc.backendhub.public.WindvaneService" 59 | windvanePageSize = 1000 60 | windvaneMaxResults = 50000 61 | ) 62 | 63 | 64 | func (w *Windvane) Name() string { 65 | return "windvane" 66 | } 67 | 68 | 69 | func (w *Windvane) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 70 | results := make(chan Result) 71 | 72 | go func() { 73 | defer close(results) 74 | 75 | 76 | if s.Keys.Windvane == "" { 77 | results <- Result{Source: w.Name(), Error: fmt.Errorf("Windvane API key not configured")} 78 | return 79 | } 80 | 81 | 82 | seen := make(map[string]bool) 83 | page := 1 84 | totalFetched := 0 85 | maxPages := windvaneMaxResults / windvanePageSize 86 | 87 | for totalFetched < windvaneMaxResults { 88 | select { 89 | case <-ctx.Done(): 90 | return 91 | default: 92 | } 93 | 94 | 95 | requestBody := windvaneRequest{ 96 | Domain: domain, 97 | PageRequest: pageRequest{ 98 | Page: page, 99 | Count: windvanePageSize, 100 | }, 101 | } 102 | 103 | jsonData, err := json.Marshal(requestBody) 104 | if err != nil { 105 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to marshal request: %w", err)} 106 | return 107 | } 108 | 109 | url := windvaneBaseURL + "/ListSubDomain" 110 | 111 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewBuffer(jsonData)) 112 | if err != nil { 113 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 114 | return 115 | } 116 | 117 | 118 | req.Header.Set("Content-Type", "application/json") 119 | req.Header.Set("X-Api-Key", s.Keys.Windvane) 120 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 121 | 122 | resp, err := s.Client.Do(req) 123 | if err != nil { 124 | results <- Result{Source: w.Name(), Error: fmt.Errorf("API request failed: %w", err)} 125 | return 126 | } 127 | 128 | if resp.StatusCode != http.StatusOK { 129 | resp.Body.Close() 130 | results <- Result{Source: w.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 131 | return 132 | } 133 | 134 | 135 | var windvaneResp windvaneResponse 136 | if err := json.NewDecoder(resp.Body).Decode(&windvaneResp); err != nil { 137 | resp.Body.Close() 138 | results <- Result{Source: w.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 139 | return 140 | } 141 | resp.Body.Close() 142 | 143 | 144 | if windvaneResp.Code != 0 { 145 | results <- Result{Source: w.Name(), Error: fmt.Errorf("API error %d: %s", windvaneResp.Code, windvaneResp.Msg)} 146 | return 147 | } 148 | 149 | 150 | if windvaneResp.Data == nil || len(windvaneResp.Data.List) == 0 { 151 | break 152 | } 153 | 154 | 155 | for _, sub := range windvaneResp.Data.List { 156 | hostname := strings.TrimSpace(strings.ToLower(sub.Domain)) 157 | 158 | if hostname == "" { 159 | continue 160 | } 161 | 162 | 163 | if !seen[hostname] { 164 | seen[hostname] = true 165 | totalFetched++ 166 | 167 | select { 168 | case results <- Result{Source: w.Name(), Value: hostname, Type: "subdomain"}: 169 | case <-ctx.Done(): 170 | return 171 | } 172 | } 173 | } 174 | 175 | 176 | if len(windvaneResp.Data.List) < windvanePageSize { 177 | break 178 | } 179 | 180 | 181 | if page >= maxPages { 182 | break 183 | } 184 | 185 | page++ 186 | } 187 | }() 188 | 189 | return results 190 | } 191 | -------------------------------------------------------------------------------- /pkg/sources/pugrecon.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type PugRecon struct{} 15 | 16 | 17 | type PugReconResult struct { 18 | Name string `json:"name"` 19 | } 20 | 21 | 22 | type PugReconAPIResponse struct { 23 | Results []PugReconResult `json:"results"` 24 | QuotaRemaining int `json:"quota_remaining"` 25 | Limited bool `json:"limited"` 26 | TotalResults int `json:"total_results"` 27 | Message string `json:"message"` 28 | } 29 | 30 | 31 | func (p *PugRecon) Name() string { 32 | return "pugrecon" 33 | } 34 | 35 | 36 | func (p *PugRecon) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 37 | results := make(chan Result) 38 | 39 | go func() { 40 | defer close(results) 41 | 42 | 43 | if s.Keys.PugRecon == "" { 44 | results <- Result{Source: p.Name(), Error: fmt.Errorf("PugRecon API key not configured")} 45 | return 46 | } 47 | 48 | 49 | postData := map[string]string{"domain_name": domain} 50 | bodyBytes, err := json.Marshal(postData) 51 | if err != nil { 52 | results <- Result{Source: p.Name(), Error: fmt.Errorf("failed to marshal request body: %w", err)} 53 | return 54 | } 55 | 56 | 57 | apiURL := "https://pugrecon.com/api/v1/domains" 58 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader(bodyBytes)) 59 | if err != nil { 60 | results <- Result{Source: p.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 61 | return 62 | } 63 | 64 | 65 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 66 | req.Header.Set("Authorization", "Bearer "+s.Keys.PugRecon) 67 | req.Header.Set("Content-Type", "application/json") 68 | req.Header.Set("Accept", "application/json") 69 | 70 | resp, err := s.Client.Do(req) 71 | if err != nil { 72 | results <- Result{Source: p.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 73 | return 74 | } 75 | defer resp.Body.Close() 76 | 77 | 78 | if resp.StatusCode == 401 { 79 | results <- Result{Source: p.Name(), Error: fmt.Errorf("invalid PugRecon API key")} 80 | return 81 | } 82 | 83 | if resp.StatusCode == 429 { 84 | results <- Result{Source: p.Name(), Error: fmt.Errorf("PugRecon rate limit exceeded")} 85 | return 86 | } 87 | 88 | if resp.StatusCode != http.StatusOK { 89 | 90 | var apiResp PugReconAPIResponse 91 | if json.NewDecoder(resp.Body).Decode(&apiResp) == nil && apiResp.Message != "" { 92 | results <- Result{Source: p.Name(), Error: fmt.Errorf("HTTP error %d: %s", resp.StatusCode, apiResp.Message)} 93 | } else { 94 | results <- Result{Source: p.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 95 | } 96 | return 97 | } 98 | 99 | 100 | var response PugReconAPIResponse 101 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 102 | results <- Result{Source: p.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 103 | return 104 | } 105 | 106 | 107 | if response.Message != "" && len(response.Results) == 0 { 108 | results <- Result{Source: p.Name(), Error: fmt.Errorf("API error: %s", response.Message)} 109 | return 110 | } 111 | 112 | 113 | seen := make(map[string]bool) 114 | for _, result := range response.Results { 115 | hostname := strings.TrimSpace(strings.ToLower(result.Name)) 116 | 117 | 118 | if hostname != "" && hostname != domain && 119 | strings.HasSuffix(hostname, "."+domain) && 120 | p.isValidHostname(hostname) && !seen[hostname] { 121 | seen[hostname] = true 122 | 123 | select { 124 | case results <- Result{Source: p.Name(), Value: hostname, Type: "subdomain"}: 125 | case <-ctx.Done(): 126 | return 127 | } 128 | } 129 | } 130 | }() 131 | 132 | return results 133 | } 134 | 135 | 136 | func (p *PugRecon) isValidHostname(hostname string) bool { 137 | if len(hostname) == 0 || len(hostname) > 253 { 138 | return false 139 | } 140 | 141 | 142 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 143 | return false 144 | } 145 | 146 | if strings.Contains(hostname, "..") { 147 | return false 148 | } 149 | 150 | 151 | if !strings.Contains(hostname, ".") { 152 | return false 153 | } 154 | 155 | 156 | for _, char := range hostname { 157 | if !((char >= 'a' && char <= 'z') || 158 | (char >= '0' && char <= '9') || 159 | char == '-' || char == '.') { 160 | return false 161 | } 162 | } 163 | 164 | 165 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 166 | return false 167 | } 168 | 169 | if strings.Contains(hostname, "--") { 170 | return false 171 | } 172 | 173 | return true 174 | } 175 | -------------------------------------------------------------------------------- /pkg/sources/fofa.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "regexp" 10 | "github.com/samogod/samoscout/pkg/session" 11 | "strings" 12 | ) 13 | 14 | 15 | type Fofa struct{} 16 | 17 | 18 | type FofaResponse struct { 19 | Error bool `json:"error"` 20 | ErrMsg string `json:"errmsg"` 21 | Size int `json:"size"` 22 | Results []string `json:"results"` 23 | } 24 | 25 | 26 | func (f *Fofa) Name() string { 27 | return "fofa" 28 | } 29 | 30 | 31 | func (f *Fofa) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 32 | results := make(chan Result) 33 | 34 | go func() { 35 | defer close(results) 36 | 37 | 38 | if s.Keys.Fofa == "" { 39 | results <- Result{Source: f.Name(), Error: fmt.Errorf("Fofa API key not configured")} 40 | return 41 | } 42 | 43 | 44 | apiKeyParts := strings.Split(s.Keys.Fofa, ":") 45 | if len(apiKeyParts) != 2 { 46 | results <- Result{Source: f.Name(), Error: fmt.Errorf("Fofa API key must be in format 'username:secret'")} 47 | return 48 | } 49 | 50 | username := apiKeyParts[0] 51 | secret := apiKeyParts[1] 52 | 53 | 54 | query := fmt.Sprintf("domain=\"%s\"", domain) 55 | qbase64 := base64.StdEncoding.EncodeToString([]byte(query)) 56 | 57 | 58 | url := fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host&page=1&size=10000&email=%s&key=%s&qbase64=%s", 59 | username, secret, qbase64) 60 | 61 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 62 | if err != nil { 63 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 64 | return 65 | } 66 | 67 | 68 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 69 | req.Header.Set("Accept", "application/json") 70 | 71 | resp, err := s.Client.Do(req) 72 | if err != nil { 73 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 74 | return 75 | } 76 | defer resp.Body.Close() 77 | 78 | if resp.StatusCode == 401 { 79 | results <- Result{Source: f.Name(), Error: fmt.Errorf("invalid Fofa API credentials")} 80 | return 81 | } 82 | 83 | if resp.StatusCode == 429 { 84 | results <- Result{Source: f.Name(), Error: fmt.Errorf("Fofa rate limit exceeded")} 85 | return 86 | } 87 | 88 | if resp.StatusCode != http.StatusOK { 89 | results <- Result{Source: f.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 90 | return 91 | } 92 | 93 | 94 | var fofaResp FofaResponse 95 | if err := json.NewDecoder(resp.Body).Decode(&fofaResp); err != nil { 96 | results <- Result{Source: f.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 97 | return 98 | } 99 | 100 | 101 | if fofaResp.Error { 102 | results <- Result{Source: f.Name(), Error: fmt.Errorf("Fofa API error: %s", fofaResp.ErrMsg)} 103 | return 104 | } 105 | 106 | 107 | seen := make(map[string]bool) 108 | if fofaResp.Size > 0 { 109 | for _, subdomain := range fofaResp.Results { 110 | hostname := f.cleanHostname(subdomain, domain) 111 | if hostname != "" && !seen[hostname] { 112 | seen[hostname] = true 113 | 114 | select { 115 | case results <- Result{Source: f.Name(), Value: hostname, Type: "subdomain"}: 116 | case <-ctx.Done(): 117 | return 118 | } 119 | } 120 | } 121 | } 122 | }() 123 | 124 | return results 125 | } 126 | 127 | 128 | func (f *Fofa) cleanHostname(raw, domain string) string { 129 | hostname := strings.TrimSpace(strings.ToLower(raw)) 130 | 131 | 132 | if strings.HasPrefix(hostname, "http://") { 133 | hostname = hostname[7:] 134 | } else if strings.HasPrefix(hostname, "https://") { 135 | hostname = hostname[8:] 136 | } 137 | 138 | 139 | re := regexp.MustCompile(`:\d+$`) 140 | if re.MatchString(hostname) { 141 | hostname = re.ReplaceAllString(hostname, "") 142 | } 143 | 144 | 145 | if slashIndex := strings.Index(hostname, "/"); slashIndex > 0 { 146 | hostname = hostname[:slashIndex] 147 | } 148 | 149 | 150 | if questionIndex := strings.Index(hostname, "?"); questionIndex > 0 { 151 | hostname = hostname[:questionIndex] 152 | } 153 | 154 | 155 | hostname = strings.Trim(hostname, ".,;:!?()[]{}\"'/\\") 156 | 157 | 158 | if hostname != "" && hostname != domain && 159 | strings.HasSuffix(hostname, "."+domain) && 160 | f.isValidHostname(hostname) { 161 | return hostname 162 | } 163 | 164 | return "" 165 | } 166 | 167 | 168 | func (f *Fofa) isValidHostname(hostname string) bool { 169 | if len(hostname) == 0 || len(hostname) > 253 { 170 | return false 171 | } 172 | 173 | 174 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 175 | return false 176 | } 177 | 178 | if strings.Contains(hostname, "..") { 179 | return false 180 | } 181 | 182 | 183 | if !strings.Contains(hostname, ".") { 184 | return false 185 | } 186 | 187 | return true 188 | } 189 | -------------------------------------------------------------------------------- /pkg/sources/robtex.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "net/http" 10 | "github.com/samogod/samoscout/pkg/session" 11 | "strings" 12 | ) 13 | 14 | 15 | type Robtex struct{} 16 | 17 | 18 | type RobtexResult struct { 19 | Rrname string `json:"rrname"` 20 | Rrdata string `json:"rrdata"` 21 | Rrtype string `json:"rrtype"` 22 | } 23 | 24 | const ( 25 | addrRecord = "A" 26 | iPv6AddrRecord = "AAAA" 27 | robtexBaseURL = "https://proapi.robtex.com/pdns" 28 | ) 29 | 30 | 31 | func (r *Robtex) Name() string { 32 | return "robtex" 33 | } 34 | 35 | 36 | func (r *Robtex) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 37 | results := make(chan Result) 38 | 39 | go func() { 40 | defer close(results) 41 | 42 | 43 | if s.Keys.Robtex == "" { 44 | results <- Result{Source: r.Name(), Error: fmt.Errorf("Robtex API key not configured")} 45 | return 46 | } 47 | 48 | seen := make(map[string]bool) 49 | 50 | 51 | forwardURL := fmt.Sprintf("%s/forward/%s?key=%s", robtexBaseURL, domain, s.Keys.Robtex) 52 | ips, err := r.enumerate(ctx, forwardURL, s) 53 | if err != nil { 54 | results <- Result{Source: r.Name(), Error: fmt.Errorf("forward lookup failed: %w", err)} 55 | return 56 | } 57 | 58 | 59 | for _, result := range ips { 60 | if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { 61 | select { 62 | case <-ctx.Done(): 63 | return 64 | default: 65 | } 66 | 67 | 68 | reverseURL := fmt.Sprintf("%s/reverse/%s?key=%s", robtexBaseURL, result.Rrdata, s.Keys.Robtex) 69 | domains, err := r.enumerate(ctx, reverseURL, s) 70 | if err != nil { 71 | results <- Result{Source: r.Name(), Error: fmt.Errorf("reverse lookup failed for %s: %w", result.Rrdata, err)} 72 | continue 73 | } 74 | 75 | 76 | for _, domainResult := range domains { 77 | hostname := strings.TrimSpace(strings.ToLower(domainResult.Rrdata)) 78 | 79 | 80 | if hostname != "" && hostname != domain && 81 | strings.HasSuffix(hostname, "."+domain) && 82 | r.isValidHostname(hostname) && !seen[hostname] { 83 | seen[hostname] = true 84 | 85 | select { 86 | case results <- Result{Source: r.Name(), Value: hostname, Type: "subdomain"}: 87 | case <-ctx.Done(): 88 | return 89 | } 90 | } 91 | } 92 | } 93 | } 94 | }() 95 | 96 | return results 97 | } 98 | 99 | 100 | func (r *Robtex) enumerate(ctx context.Context, targetURL string, s *session.Session) ([]RobtexResult, error) { 101 | var results []RobtexResult 102 | 103 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, targetURL, nil) 104 | if err != nil { 105 | return results, fmt.Errorf("failed to create request: %w", err) 106 | } 107 | 108 | 109 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 110 | req.Header.Set("Accept", "application/x-ndjson") 111 | req.Header.Set("Content-Type", "application/x-ndjson") 112 | 113 | resp, err := s.Client.Do(req) 114 | if err != nil { 115 | return results, fmt.Errorf("failed to execute request: %w", err) 116 | } 117 | defer resp.Body.Close() 118 | 119 | if resp.StatusCode == 401 { 120 | return results, fmt.Errorf("invalid Robtex API key") 121 | } 122 | 123 | if resp.StatusCode == 429 { 124 | return results, fmt.Errorf("Robtex rate limit exceeded") 125 | } 126 | 127 | if resp.StatusCode != http.StatusOK { 128 | return results, fmt.Errorf("HTTP error: %d", resp.StatusCode) 129 | } 130 | 131 | 132 | scanner := bufio.NewScanner(resp.Body) 133 | for scanner.Scan() { 134 | line := scanner.Text() 135 | if line == "" { 136 | continue 137 | } 138 | 139 | var response RobtexResult 140 | if err := json.NewDecoder(bytes.NewBufferString(line)).Decode(&response); err != nil { 141 | continue 142 | } 143 | 144 | results = append(results, response) 145 | } 146 | 147 | if err := scanner.Err(); err != nil { 148 | return results, fmt.Errorf("error reading response: %w", err) 149 | } 150 | 151 | return results, nil 152 | } 153 | 154 | 155 | func (r *Robtex) isValidHostname(hostname string) bool { 156 | if len(hostname) == 0 || len(hostname) > 253 { 157 | return false 158 | } 159 | 160 | 161 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 162 | return false 163 | } 164 | 165 | if strings.Contains(hostname, "..") { 166 | return false 167 | } 168 | 169 | 170 | if !strings.Contains(hostname, ".") { 171 | return false 172 | } 173 | 174 | 175 | for _, char := range hostname { 176 | if !((char >= 'a' && char <= 'z') || 177 | (char >= '0' && char <= '9') || 178 | char == '-' || char == '.') { 179 | return false 180 | } 181 | } 182 | 183 | 184 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 185 | return false 186 | } 187 | 188 | if strings.Contains(hostname, "--") { 189 | return false 190 | } 191 | 192 | return true 193 | } 194 | -------------------------------------------------------------------------------- /pkg/active/resolvers.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "net/http" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | const ( 14 | ResolverTrickest = "https://raw.githubusercontent.com/trickest/resolvers/main/resolvers.txt" 15 | ResolverTrickestTrusted = "https://raw.githubusercontent.com/trickest/resolvers/main/resolvers-trusted.txt" 16 | ResolverPublicDNS = "https://public-dns.info/nameservers.txt" 17 | ) 18 | 19 | func DownloadResolvers(outputDir string, verbose bool) (string, string, error) { 20 | normalResolverFile := filepath.Join(outputDir, "resolvers.txt") 21 | trustedResolverFile := filepath.Join(outputDir, "resolvers_trusted.txt") 22 | 23 | normalExists := isResolverFileFresh(normalResolverFile) 24 | trustedExists := isResolverFileFresh(trustedResolverFile) 25 | 26 | if normalExists && trustedExists { 27 | if verbose { 28 | fmt.Println("[DBG] using cached resolver files (fresh within 24h)") 29 | } 30 | return normalResolverFile, trustedResolverFile, nil 31 | } 32 | 33 | if verbose { 34 | fmt.Println("[DBG] resolver cache expired or missing, downloading fresh resolvers...") 35 | } 36 | 37 | normalURLs := []string{ 38 | ResolverTrickest, 39 | ResolverPublicDNS, 40 | } 41 | 42 | var normalResolvers []string 43 | normalSet := make(map[string]bool) 44 | 45 | for i, url := range normalURLs { 46 | if verbose { 47 | fmt.Printf("[DBG] downloading normal resolver list %d/2 from %s\n", i+1, url) 48 | } 49 | 50 | resolvers, err := downloadResolverList(url) 51 | if err != nil { 52 | if verbose { 53 | fmt.Printf("[DBG] warning: failed to download %s: %v\n", url, err) 54 | } 55 | continue 56 | } 57 | 58 | for _, resolver := range resolvers { 59 | if !normalSet[resolver] { 60 | normalSet[resolver] = true 61 | normalResolvers = append(normalResolvers, resolver) 62 | } 63 | } 64 | 65 | if verbose { 66 | fmt.Printf("[DBG] downloaded %d resolvers from source %d/2\n", len(resolvers), i+1) 67 | } 68 | } 69 | 70 | if verbose { 71 | fmt.Println("[DBG] downloading trusted resolver list from trickest...") 72 | } 73 | 74 | trustedResolvers, err := downloadResolverList(ResolverTrickestTrusted) 75 | if err != nil { 76 | if verbose { 77 | fmt.Printf("[DBG] warning: failed to download trusted resolvers: %v\n", err) 78 | } 79 | trustedResolvers = []string{} 80 | } else if verbose { 81 | fmt.Printf("[DBG] downloaded %d trusted resolvers\n", len(trustedResolvers)) 82 | } 83 | 84 | if len(normalResolvers) == 0 && len(trustedResolvers) == 0 { 85 | return "", "", fmt.Errorf("no resolvers downloaded from any source") 86 | } 87 | 88 | if len(normalResolvers) > 0 { 89 | if err := writeResolvers(normalResolvers, normalResolverFile); err != nil { 90 | return "", "", fmt.Errorf("failed to write normal resolvers: %w", err) 91 | } 92 | if verbose { 93 | fmt.Printf("[DBG] %d normal resolvers saved to %s\n", len(normalResolvers), normalResolverFile) 94 | } 95 | } 96 | 97 | if len(trustedResolvers) > 0 { 98 | if err := writeResolvers(trustedResolvers, trustedResolverFile); err != nil { 99 | return "", "", fmt.Errorf("failed to write trusted resolvers: %w", err) 100 | } 101 | if verbose { 102 | fmt.Printf("[DBG] %d trusted resolvers saved to %s\n", len(trustedResolvers), trustedResolverFile) 103 | } 104 | } 105 | 106 | return normalResolverFile, trustedResolverFile, nil 107 | } 108 | 109 | func isResolverFileFresh(filePath string) bool { 110 | info, err := os.Stat(filePath) 111 | if err != nil { 112 | return false 113 | } 114 | 115 | age := time.Since(info.ModTime()) 116 | return age < 24*time.Hour 117 | } 118 | 119 | func downloadResolverList(url string) ([]string, error) { 120 | resp, err := http.Get(url) 121 | if err != nil { 122 | return nil, err 123 | } 124 | defer resp.Body.Close() 125 | 126 | if resp.StatusCode != http.StatusOK { 127 | return nil, fmt.Errorf("HTTP %d", resp.StatusCode) 128 | } 129 | 130 | var resolvers []string 131 | scanner := bufio.NewScanner(resp.Body) 132 | 133 | for scanner.Scan() { 134 | line := strings.TrimSpace(scanner.Text()) 135 | 136 | if line == "" || strings.HasPrefix(line, "#") { 137 | continue 138 | } 139 | 140 | if isValidResolver(line) { 141 | resolvers = append(resolvers, line) 142 | } 143 | } 144 | 145 | return resolvers, scanner.Err() 146 | } 147 | 148 | func isValidResolver(resolver string) bool { 149 | if resolver == "" { 150 | return false 151 | } 152 | 153 | for _, char := range resolver { 154 | if !((char >= '0' && char <= '9') || char == '.' || char == ':') { 155 | return false 156 | } 157 | } 158 | 159 | return true 160 | } 161 | 162 | func writeResolvers(resolvers []string, filePath string) error { 163 | file, err := os.Create(filePath) 164 | if err != nil { 165 | return err 166 | } 167 | defer file.Close() 168 | 169 | writer := bufio.NewWriter(file) 170 | for _, resolver := range resolvers { 171 | if _, err := writer.WriteString(resolver + "\n"); err != nil { 172 | return err 173 | } 174 | } 175 | 176 | return writer.Flush() 177 | } 178 | -------------------------------------------------------------------------------- /pkg/sources/urlscan.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | 16 | type URLScan struct{} 17 | 18 | const ( 19 | urlscanBaseURL = "https://urlscan.io/api/v1/" 20 | urlscanPageSize = 100 21 | urlscanMaxResults = 10000 22 | maxPageRetries = 3 23 | backoffBase = 2 * time.Second 24 | ) 25 | 26 | 27 | type searchResponse struct { 28 | Total int `json:"total"` 29 | Results []struct { 30 | Task struct { 31 | URL string `json:"url"` 32 | } `json:"task"` 33 | Page struct { 34 | Domain string `json:"domain"` 35 | } `json:"page"` 36 | } `json:"results"` 37 | } 38 | 39 | 40 | func (u *URLScan) Name() string { 41 | return "urlscan" 42 | } 43 | 44 | 45 | func (u *URLScan) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 46 | results := make(chan Result) 47 | 48 | go func() { 49 | defer close(results) 50 | 51 | 52 | if s.Keys.URLScan == "" { 53 | results <- Result{Source: u.Name(), Error: fmt.Errorf("URLScan API key not configured")} 54 | return 55 | } 56 | 57 | 58 | seen := make(map[string]bool) 59 | var errors, resultCount atomic.Int32 60 | totalFetched := 0 61 | 62 | headers := map[string]string{ 63 | "accept": "application/json", 64 | "API-Key": s.Keys.URLScan, 65 | } 66 | 67 | page := 0 68 | for totalFetched < urlscanMaxResults { 69 | select { 70 | case <-ctx.Done(): 71 | return 72 | default: 73 | } 74 | 75 | 76 | q := "domain:" + domain 77 | apiURL := urlscanBaseURL + "search/?" + 78 | "q=" + url.QueryEscape(q) + 79 | "&size=" + url.QueryEscape(fmt.Sprintf("%d", urlscanPageSize)) 80 | 81 | 82 | if page > 0 { 83 | offset := page * urlscanPageSize 84 | apiURL += "&search_after=" + url.QueryEscape(fmt.Sprintf("%d", offset)) 85 | } 86 | 87 | 88 | var resp *http.Response 89 | var err error 90 | backoff := backoffBase 91 | 92 | 93 | for attempt := 0; attempt <= maxPageRetries; attempt++ { 94 | req, reqErr := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) 95 | if reqErr != nil { 96 | results <- Result{Source: u.Name(), Error: fmt.Errorf("failed to create request: %w", reqErr)} 97 | return 98 | } 99 | 100 | 101 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 102 | for key, value := range headers { 103 | req.Header.Set(key, value) 104 | } 105 | 106 | resp, err = s.Client.Do(req) 107 | if err != nil { 108 | results <- Result{Source: u.Name(), Error: fmt.Errorf("API request failed: %w", err)} 109 | errors.Add(1) 110 | break 111 | } 112 | 113 | if resp.StatusCode == http.StatusOK { 114 | break 115 | } 116 | 117 | resp.Body.Close() 118 | 119 | 120 | if resp.StatusCode == http.StatusTooManyRequests || (resp.StatusCode >= 500 && resp.StatusCode < 600) { 121 | select { 122 | case <-time.After(backoff): 123 | backoff *= 2 124 | continue 125 | case <-ctx.Done(): 126 | return 127 | } 128 | } 129 | 130 | 131 | err = fmt.Errorf("URLScan API returned status %d", resp.StatusCode) 132 | results <- Result{Source: u.Name(), Error: err} 133 | errors.Add(1) 134 | break 135 | } 136 | 137 | if err != nil { 138 | break 139 | } 140 | 141 | 142 | var sr searchResponse 143 | if err := json.NewDecoder(resp.Body).Decode(&sr); err != nil { 144 | resp.Body.Close() 145 | results <- Result{Source: u.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 146 | errors.Add(1) 147 | break 148 | } 149 | resp.Body.Close() 150 | 151 | 152 | if len(sr.Results) == 0 { 153 | break 154 | } 155 | 156 | 157 | for _, r := range sr.Results { 158 | if totalFetched >= urlscanMaxResults { 159 | break 160 | } 161 | 162 | host := strings.ToLower(strings.TrimSpace(r.Page.Domain)) 163 | 164 | 165 | if host == "" && r.Task.URL != "" { 166 | if parsed, parseErr := url.Parse(r.Task.URL); parseErr == nil && parsed != nil { 167 | host = strings.ToLower(parsed.Hostname()) 168 | } 169 | } 170 | 171 | if host == "" { 172 | continue 173 | } 174 | 175 | 176 | host = strings.TrimPrefix(host, "www.") 177 | 178 | 179 | if !strings.HasSuffix(host, "."+domain) && host != domain { 180 | continue 181 | } 182 | 183 | 184 | if !seen[host] { 185 | seen[host] = true 186 | resultCount.Add(1) 187 | totalFetched++ 188 | 189 | select { 190 | case results <- Result{Source: u.Name(), Value: host, Type: "subdomain"}: 191 | case <-ctx.Done(): 192 | return 193 | } 194 | } 195 | } 196 | 197 | 198 | if len(sr.Results) < urlscanPageSize { 199 | break 200 | } 201 | 202 | page++ 203 | } 204 | }() 205 | 206 | return results 207 | } 208 | -------------------------------------------------------------------------------- /pkg/sources/quake.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type Quake struct{} 15 | 16 | 17 | type QuakeResults struct { 18 | Code int `json:"code"` 19 | Message string `json:"message"` 20 | Data []struct { 21 | Service struct { 22 | HTTP struct { 23 | Host string `json:"host"` 24 | } `json:"http"` 25 | } `json:"service"` 26 | } `json:"data"` 27 | Meta struct { 28 | Pagination struct { 29 | Total int `json:"total"` 30 | } `json:"pagination"` 31 | } `json:"meta"` 32 | } 33 | 34 | 35 | func (q *Quake) Name() string { 36 | return "quake" 37 | } 38 | 39 | 40 | func (q *Quake) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 41 | results := make(chan Result) 42 | 43 | go func() { 44 | defer close(results) 45 | 46 | 47 | if s.Keys.Quake == "" { 48 | results <- Result{Source: q.Name(), Error: fmt.Errorf("Quake API key not configured")} 49 | return 50 | } 51 | 52 | 53 | var pageSize = 500 54 | var start = 0 55 | var totalResults = -1 56 | seen := make(map[string]bool) 57 | 58 | for { 59 | select { 60 | case <-ctx.Done(): 61 | return 62 | default: 63 | } 64 | 65 | 66 | requestBody := fmt.Sprintf(`{"query":"domain: %s", "include":["service.http.host"], "latest": true, "size":%d, "start":%d}`, 67 | domain, pageSize, start) 68 | 69 | 70 | apiURL := "https://quake.360.net/api/v3/search/quake_service" 71 | req, err := http.NewRequestWithContext(ctx, http.MethodPost, apiURL, bytes.NewReader([]byte(requestBody))) 72 | if err != nil { 73 | results <- Result{Source: q.Name(), Error: fmt.Errorf("failed to create request: %w", err)} 74 | return 75 | } 76 | 77 | 78 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 79 | req.Header.Set("Content-Type", "application/json") 80 | req.Header.Set("X-QuakeToken", s.Keys.Quake) 81 | 82 | resp, err := s.Client.Do(req) 83 | if err != nil { 84 | results <- Result{Source: q.Name(), Error: fmt.Errorf("failed to execute request: %w", err)} 85 | return 86 | } 87 | defer resp.Body.Close() 88 | 89 | if resp.StatusCode == 401 { 90 | results <- Result{Source: q.Name(), Error: fmt.Errorf("invalid Quake API key")} 91 | return 92 | } 93 | 94 | if resp.StatusCode == 429 { 95 | results <- Result{Source: q.Name(), Error: fmt.Errorf("Quake rate limit exceeded")} 96 | return 97 | } 98 | 99 | if resp.StatusCode != http.StatusOK { 100 | results <- Result{Source: q.Name(), Error: fmt.Errorf("HTTP error: %d", resp.StatusCode)} 101 | return 102 | } 103 | 104 | 105 | var response QuakeResults 106 | if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { 107 | results <- Result{Source: q.Name(), Error: fmt.Errorf("failed to decode JSON: %w", err)} 108 | return 109 | } 110 | 111 | 112 | if response.Code != 0 { 113 | results <- Result{Source: q.Name(), Error: fmt.Errorf("Quake API error: %s (code: %d)", response.Message, response.Code)} 114 | return 115 | } 116 | 117 | 118 | if totalResults == -1 { 119 | totalResults = response.Meta.Pagination.Total 120 | } 121 | 122 | 123 | for _, quakeDomain := range response.Data { 124 | hostname := strings.TrimSpace(strings.ToLower(quakeDomain.Service.HTTP.Host)) 125 | 126 | 127 | if strings.Contains(hostname, "暂无权限") { 128 | continue 129 | } 130 | 131 | 132 | if hostname != "" && hostname != domain && 133 | strings.HasSuffix(hostname, "."+domain) && 134 | q.isValidHostname(hostname) && !seen[hostname] { 135 | seen[hostname] = true 136 | 137 | select { 138 | case results <- Result{Source: q.Name(), Value: hostname, Type: "subdomain"}: 139 | case <-ctx.Done(): 140 | return 141 | } 142 | } 143 | } 144 | 145 | 146 | if len(response.Data) == 0 || start+pageSize >= totalResults { 147 | break 148 | } 149 | 150 | start += pageSize 151 | } 152 | }() 153 | 154 | return results 155 | } 156 | 157 | 158 | func (q *Quake) isValidHostname(hostname string) bool { 159 | if len(hostname) == 0 || len(hostname) > 253 { 160 | return false 161 | } 162 | 163 | 164 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 165 | return false 166 | } 167 | 168 | if strings.Contains(hostname, "..") { 169 | return false 170 | } 171 | 172 | 173 | if !strings.Contains(hostname, ".") { 174 | return false 175 | } 176 | 177 | 178 | for _, char := range hostname { 179 | if !((char >= 'a' && char <= 'z') || 180 | (char >= '0' && char <= '9') || 181 | char == '-' || char == '.') { 182 | return false 183 | } 184 | } 185 | 186 | 187 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 188 | return false 189 | } 190 | 191 | if strings.Contains(hostname, "--") { 192 | return false 193 | } 194 | 195 | return true 196 | } 197 | -------------------------------------------------------------------------------- /pkg/sources/hunter.go: -------------------------------------------------------------------------------- 1 | package sources 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "encoding/json" 7 | "fmt" 8 | "net/http" 9 | "github.com/samogod/samoscout/pkg/session" 10 | "strings" 11 | ) 12 | 13 | 14 | type Hunter struct{} 15 | 16 | 17 | type HunterResponse struct { 18 | Code int `json:"code"` 19 | Data HunterData `json:"data"` 20 | Message string `json:"message"` 21 | } 22 | 23 | type HunterData struct { 24 | InfoArr []HunterInfo `json:"arr"` 25 | Total int `json:"total"` 26 | } 27 | 28 | type HunterInfo struct { 29 | URL string `json:"url"` 30 | IP string `json:"ip"` 31 | Port int `json:"port"` 32 | Domain string `json:"domain"` 33 | Protocol string `json:"protocol"` 34 | } 35 | 36 | 37 | func (h *Hunter) Name() string { 38 | return "hunter" 39 | } 40 | 41 | 42 | func (h *Hunter) Run(ctx context.Context, domain string, s *session.Session) <-chan Result { 43 | results := make(chan Result) 44 | 45 | go func() { 46 | defer close(results) 47 | 48 | 49 | if s.Keys.Hunter == "" { 50 | results <- Result{Source: h.Name(), Error: fmt.Errorf("Hunter API key not configured")} 51 | return 52 | } 53 | 54 | 55 | var pages = 1 56 | seen := make(map[string]bool) 57 | 58 | for currentPage := 1; currentPage <= pages; currentPage++ { 59 | select { 60 | case <-ctx.Done(): 61 | return 62 | default: 63 | } 64 | 65 | 66 | query := fmt.Sprintf("domain=\"%s\"", domain) 67 | qbase64 := base64.URLEncoding.EncodeToString([]byte(query)) 68 | 69 | 70 | url := fmt.Sprintf("https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3", 71 | s.Keys.Hunter, qbase64, currentPage) 72 | 73 | req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 74 | if err != nil { 75 | results <- Result{Source: h.Name(), Error: fmt.Errorf("failed to create request for page %d: %w", currentPage, err)} 76 | return 77 | } 78 | 79 | 80 | req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36") 81 | req.Header.Set("Accept", "application/json") 82 | 83 | resp, err := s.Client.Do(req) 84 | if err != nil { 85 | results <- Result{Source: h.Name(), Error: fmt.Errorf("failed to execute request for page %d: %w", currentPage, err)} 86 | return 87 | } 88 | defer resp.Body.Close() 89 | 90 | if resp.StatusCode == 401 { 91 | results <- Result{Source: h.Name(), Error: fmt.Errorf("invalid Hunter API key")} 92 | return 93 | } 94 | 95 | if resp.StatusCode == 429 { 96 | results <- Result{Source: h.Name(), Error: fmt.Errorf("Hunter rate limit exceeded")} 97 | return 98 | } 99 | 100 | if resp.StatusCode != http.StatusOK { 101 | results <- Result{Source: h.Name(), Error: fmt.Errorf("HTTP error for page %d: %d", currentPage, resp.StatusCode)} 102 | return 103 | } 104 | 105 | 106 | var hunterResp HunterResponse 107 | if err := json.NewDecoder(resp.Body).Decode(&hunterResp); err != nil { 108 | results <- Result{Source: h.Name(), Error: fmt.Errorf("failed to decode JSON for page %d: %w", currentPage, err)} 109 | return 110 | } 111 | 112 | 113 | if hunterResp.Code == 401 || hunterResp.Code == 400 { 114 | results <- Result{Source: h.Name(), Error: fmt.Errorf("Hunter API error: %s (code: %d)", hunterResp.Message, hunterResp.Code)} 115 | return 116 | } 117 | 118 | 119 | if hunterResp.Data.Total > 0 { 120 | for _, hunterInfo := range hunterResp.Data.InfoArr { 121 | hostname := strings.TrimSpace(strings.ToLower(hunterInfo.Domain)) 122 | 123 | 124 | if hostname != "" && hostname != domain && 125 | strings.HasSuffix(hostname, "."+domain) && 126 | h.isValidHostname(hostname) && !seen[hostname] { 127 | seen[hostname] = true 128 | 129 | select { 130 | case results <- Result{Source: h.Name(), Value: hostname, Type: "subdomain"}: 131 | case <-ctx.Done(): 132 | return 133 | } 134 | } 135 | } 136 | 137 | 138 | if currentPage == 1 { 139 | pages = hunterResp.Data.Total/100 + 1 140 | if pages > 10 { 141 | pages = 10 142 | } 143 | } 144 | } 145 | } 146 | }() 147 | 148 | return results 149 | } 150 | 151 | 152 | func (h *Hunter) isValidHostname(hostname string) bool { 153 | if len(hostname) == 0 || len(hostname) > 253 { 154 | return false 155 | } 156 | 157 | 158 | if strings.HasPrefix(hostname, ".") || strings.HasSuffix(hostname, ".") { 159 | return false 160 | } 161 | 162 | if strings.Contains(hostname, "..") { 163 | return false 164 | } 165 | 166 | 167 | if !strings.Contains(hostname, ".") { 168 | return false 169 | } 170 | 171 | 172 | for _, char := range hostname { 173 | if !((char >= 'a' && char <= 'z') || 174 | (char >= '0' && char <= '9') || 175 | char == '-' || char == '.') { 176 | return false 177 | } 178 | } 179 | 180 | 181 | if strings.HasPrefix(hostname, "-") || strings.HasSuffix(hostname, "-") { 182 | return false 183 | } 184 | 185 | if strings.Contains(hostname, "--") { 186 | return false 187 | } 188 | 189 | return true 190 | } 191 | --------------------------------------------------------------------------------