├── pkg ├── subscraping │ ├── doc.go │ ├── utils.go │ ├── sources │ │ ├── chaos │ │ │ └── chaos.go │ │ ├── riddler │ │ │ └── riddler.go │ │ ├── hackertarget │ │ │ └── hackertarget.go │ │ ├── anubis │ │ │ └── anubis.go │ │ ├── rapiddns │ │ │ └── rapiddns.go │ │ ├── sublist3r │ │ │ └── subllist3r.go │ │ ├── archiveis │ │ │ └── archiveis.go │ │ ├── threatcrowd │ │ │ └── threatcrowd.go │ │ ├── github │ │ │ ├── tokenmanager.go │ │ │ └── github.go │ │ ├── fullhunt │ │ │ └── fullhunt.go │ │ ├── sonarsearch │ │ │ └── sonarsearch.go │ │ ├── threatminer │ │ │ └── threatminer.go │ │ ├── virustotal │ │ │ └── virustotal.go │ │ ├── waybackarchive │ │ │ └── waybackarchive.go │ │ ├── chinaz │ │ │ └── chinaz.go │ │ ├── securitytrails │ │ │ └── securitytrails.go │ │ ├── alienvault │ │ │ └── alienvault.go │ │ ├── dnsdb │ │ │ └── dnsdb.go │ │ ├── c99 │ │ │ └── c99.go │ │ ├── passivetotal │ │ │ └── passivetotal.go │ │ ├── zoomeyeapi │ │ │ └── zoomeyeapi.go │ │ ├── sitedossier │ │ │ └── sitedossier.go │ │ ├── threatbook │ │ │ └── threatbook.go │ │ ├── quake │ │ │ └── quake.go │ │ ├── shodan │ │ │ └── shodan.go │ │ ├── robtex │ │ │ └── robtext.go │ │ ├── censys │ │ │ └── censys.go │ │ ├── bufferover │ │ │ └── bufferover.go │ │ ├── certspotter │ │ │ └── certspotter.go │ │ ├── dnsdumpster │ │ │ └── dnsdumpster.go │ │ ├── commoncrawl │ │ │ └── commoncrawl.go │ │ ├── fofa │ │ │ └── fofa.go │ │ ├── spyse │ │ │ └── spyse.go │ │ ├── intelx │ │ │ └── intelx.go │ │ ├── crtsh │ │ │ └── crtsh.go │ │ ├── zoomeye │ │ │ └── zoomeye.go │ │ ├── hunter │ │ │ └── hunter.go │ │ └── binaryedge │ │ │ └── binaryedge.go │ ├── types.go │ └── agent.go ├── runner │ ├── doc.go │ ├── util.go │ ├── validate.go │ ├── initialize.go │ ├── banner.go │ ├── runner.go │ ├── outputter.go │ └── config.go ├── passive │ ├── doc.go │ ├── passive.go │ └── sources.go ├── resolve │ ├── doc.go │ ├── client.go │ └── resolve.go ├── subTakeOver │ ├── assets │ │ └── assets.go │ ├── dns.go │ ├── requests.go │ ├── subTakeOver.go │ └── fingerprint.go ├── active │ ├── struct.go │ ├── subdata.go │ ├── device │ │ ├── struct.go │ │ └── device.go │ ├── retry.go │ ├── active_test.go │ ├── options.go │ ├── statusdb │ │ └── db.go │ ├── active.go │ ├── send.go │ └── recv.go ├── goflags │ ├── insertionorderedmap.go │ ├── string_slice.go │ ├── runtime_map.go │ └── normalized_slice.go ├── enum │ ├── zone_test.go │ └── zone.go ├── net │ ├── requests.go │ └── network.go └── util │ └── util.go ├── .gitignore ├── cmd └── Starmap.go ├── Makefile ├── CHANGELOG.md ├── LICENSE ├── .github └── workflows │ └── build.yml ├── test.go ├── go.mod └── README.md /pkg/subscraping/doc.go: -------------------------------------------------------------------------------- 1 | // Package subscraping contains the logic of scraping agents 2 | package subscraping 3 | -------------------------------------------------------------------------------- /pkg/runner/doc.go: -------------------------------------------------------------------------------- 1 | // Package runner implements the mechanism to drive the 2 | // subdomain enumeration process 3 | package runner 4 | -------------------------------------------------------------------------------- /pkg/passive/doc.go: -------------------------------------------------------------------------------- 1 | // Package passive provides capability for doing passive subdomain 2 | // enumeration on targets. 3 | package passive 4 | -------------------------------------------------------------------------------- /pkg/resolve/doc.go: -------------------------------------------------------------------------------- 1 | // Package resolve is used to handle resolving records 2 | // It also handles wildcard subdomains and rotating resolvers. 3 | package resolve 4 | -------------------------------------------------------------------------------- /pkg/subTakeOver/assets/assets.go: -------------------------------------------------------------------------------- 1 | package assets 2 | 3 | import _ "embed" 4 | 5 | //文件的内容嵌入为slice of byte，也就是一个字节数组 6 | 7 | //go:embed fingerprints.json 8 | var Fingerprints []byte 9 | 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.db 3 | .idea 4 | .DS_Store 5 | 6 | # Test binary, built with `go test -c` 7 | *.test 8 | 9 | # Output of the go coverage tool, specifically when used with LiteIDE 10 | *.out 11 | -------------------------------------------------------------------------------- /pkg/active/struct.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "github.com/google/gopacket/layers" 5 | ) 6 | 7 | // RecvResult 接收结果数据结构 8 | type RecvResult struct { 9 | Subdomain string 10 | Answers []layers.DNSResourceRecord 11 | ResponseCode layers.DNSResponseCode 12 | } 13 | -------------------------------------------------------------------------------- /pkg/active/subdata.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | _ "embed" 5 | "strings" 6 | ) 7 | 8 | //go:embed data/subnext.txt 9 | var subnext string 10 | 11 | //go:embed data/subdomain.txt 12 | var subdomain string 13 | 14 | func GetDefaultSubdomainData() []string { 15 | return strings.Split(subdomain, "\n") 16 | } 17 | 18 | func GetDefaultSubNextData() []string { 19 | return strings.Split(subnext, "\n") 20 | } 21 | -------------------------------------------------------------------------------- /cmd/Starmap.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "github.com/ZhuriLab/Starmap/pkg/runner" 6 | "github.com/projectdiscovery/gologger" 7 | ) 8 | 9 | func main() { 10 | // Parse the command line flags and read config files 11 | options := runner.ParseOptions() 12 | 13 | newRunner, err := runner.NewRunner(options) 14 | if err != nil { 15 | gologger.Fatal().Msgf("Could not create runner: %s\n", err) 16 | } 17 | 18 | err = newRunner.RunEnumeration(context.Background()) 19 | if err != nil { 20 | gologger.Fatal().Msgf("Could not run enumeration: %s\n", err) 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Go build flags 2 | LDFLAGS=-ldflags "-s -w" 3 | 4 | default: 5 | go build ${LDFLAGS} -o "Starmap" cmd/Starmap.go 6 | 7 | # Compile Server - Windows x64 8 | windows: 9 | export GOOS=windows;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap.exe" cmd/Starmap.go 10 | 11 | # Compile Server - Linux x64 12 | linux: 13 | export GOOS=linux;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap" cmd/Starmap.go 14 | 15 | # Compile Server - Darwin x64 16 | darwin: 17 | export GOOS=darwin;export GOARCH=amd64;go build ${LDFLAGS} -o "Starmap" cmd/Starmap.go 18 | 19 | # clean 20 | clean: 21 | rm -rf ${DIR} 22 | -------------------------------------------------------------------------------- /pkg/active/device/struct.go: -------------------------------------------------------------------------------- 1 | package device 2 | 3 | import ( 4 | "net" 5 | ) 6 | 7 | type SelfMac net.HardwareAddr 8 | 9 | func (d SelfMac) String() string { 10 | n := (net.HardwareAddr)(d) 11 | return n.String() 12 | } 13 | func (d SelfMac) MarshalYAML() (interface{}, error) { 14 | n := (net.HardwareAddr)(d) 15 | return n.String(), nil 16 | } 17 | func (d SelfMac) HardwareAddr() net.HardwareAddr { 18 | n := (net.HardwareAddr)(d) 19 | return n 20 | } 21 | 22 | 23 | type EtherTable struct { 24 | SrcIp net.IP `yaml:"src_ip"` 25 | Device string `yaml:"device"` 26 | SrcMac SelfMac `yaml:"src_mac"` 27 | DstMac SelfMac `yaml:"dst_mac"` 28 | } 29 | 30 | -------------------------------------------------------------------------------- /pkg/subTakeOver/dns.go: -------------------------------------------------------------------------------- 1 | package subTakeOver 2 | 3 | 4 | //func nxdomain(subdomain string) bool { 5 | // // initialize global pseudo random generators 6 | // rand.Seed(time.Now().UTC().UnixNano()) 7 | // 8 | // c := dns.Client{} 9 | // m := dns.Msg{} 10 | // 11 | // dnsService := DefaultBaselineResolvers[rand.Intn(len(DefaultBaselineResolvers))] 12 | // 13 | // m.SetQuestion(dns.Fqdn(subdomain), dns.TypeNS) 14 | // r, _, err := c.Exchange(&m, dnsService + ":53") 15 | // 16 | // if err != nil { 17 | // return false 18 | // } 19 | // 20 | // if strings.Contains(r.String(), "NXDOMAIN") { 21 | // return true 22 | // } 23 | // 24 | // return false 25 | //} 26 | 27 | -------------------------------------------------------------------------------- /pkg/goflags/insertionorderedmap.go: -------------------------------------------------------------------------------- 1 | package goflags 2 | 3 | type InsertionOrderedMap struct { 4 | values map[string]*FlagData 5 | keys []string `yaml:"-"` 6 | } 7 | 8 | func (insertionOrderedMap *InsertionOrderedMap) forEach(fn func(key string, data *FlagData)) { 9 | for _, key := range insertionOrderedMap.keys { 10 | fn(key, insertionOrderedMap.values[key]) 11 | } 12 | } 13 | 14 | func (insertionOrderedMap *InsertionOrderedMap) Set(key string, value *FlagData) { 15 | _, present := insertionOrderedMap.values[key] 16 | insertionOrderedMap.values[key] = value 17 | if !present { 18 | insertionOrderedMap.keys = append(insertionOrderedMap.keys, key) 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /pkg/enum/zone_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017-2021 Jeff Foley. All rights reserved. 2 | // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 | 4 | package enum 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | ) 10 | 11 | // https://github.com/vulhub/vulhub/blob/master/dns/dns-zone-transfer/README.zh-cn.md 12 | 13 | func TestZoneTransfer(t *testing.T) { 14 | 15 | a, err := ZoneTransfer("vulhub.org", "", "192.168.102.102") 16 | if err != nil { 17 | t.Errorf("Error in creating ZoneTransfer: %v", err) 18 | } 19 | 20 | for _, out := range a { 21 | fmt.Printf("name: %s, Source: %s, tag: %s, domain: %s , rec: %v\r\n", out.Name, out.Source, out.Tag, out.Domain, out.Records) 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /pkg/active/retry.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "context" 5 | "github.com/ZhuriLab/Starmap/pkg/active/statusdb" 6 | "sync/atomic" 7 | "time" 8 | ) 9 | 10 | func (r *runner) retry(ctx context.Context) { 11 | t := time.NewTicker(1 * time.Second) 12 | defer t.Stop() 13 | 14 | for { 15 | select { 16 | case <-ctx.Done(): 17 | return 18 | case <-t.C: 19 | // 循环检测超时的队列 20 | now := time.Now() 21 | r.hm.Scan(func(key string, v statusdb.Item) error { 22 | if r.maxRetry > 0 && v.Retry > r.maxRetry { 23 | r.hm.Del(key) 24 | atomic.AddUint64(&r.faildIndex, 1) 25 | return nil 26 | } 27 | if int64(now.Sub(v.Time)) >= r.timeout { 28 | // 重新发送 29 | r.sender <- key 30 | } 31 | return nil 32 | }) 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /pkg/subscraping/utils.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "regexp" 5 | "sync" 6 | ) 7 | 8 | var subdomainExtractorMutex = &sync.Mutex{} 9 | 10 | // NewSubdomainExtractor creates a new regular expression to extract 11 | // subdomains from text based on the given domain. 12 | func NewSubdomainExtractor(domain string) (*regexp.Regexp, error) { 13 | subdomainExtractorMutex.Lock() 14 | defer subdomainExtractorMutex.Unlock() 15 | extractor, err := regexp.Compile(`[a-zA-Z0-9\*_.-]+\.` + domain) 16 | if err != nil { 17 | return nil, err 18 | } 19 | return extractor, nil 20 | } 21 | 22 | // Exists check if a key exist in a slice 23 | func Exists(values []string, key string) bool { 24 | for _, v := range values { 25 | if v == key { 26 | return true 27 | } 28 | } 29 | return false 30 | } 31 | -------------------------------------------------------------------------------- /pkg/runner/util.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/pkg/errors" 7 | "github.com/projectdiscovery/fileutil" 8 | ) 9 | 10 | var ( 11 | ErrEmptyInput = errors.New("empty data") 12 | ) 13 | 14 | func loadFromFile(file string) ([]string, error) { 15 | chanItems, err := fileutil.ReadFile(file) 16 | if err != nil { 17 | return nil, err 18 | } 19 | var items []string 20 | for item := range chanItems { 21 | var err error 22 | item, err = sanitize(item) 23 | if errors.Is(err, ErrEmptyInput) { 24 | continue 25 | } 26 | items = append(items, item) 27 | } 28 | return items, nil 29 | } 30 | 31 | func sanitize(data string) (string, error) { 32 | data = strings.Trim(data, "\n\t\"' ") 33 | if data == "" { 34 | return "", ErrEmptyInput 35 | } 36 | return data, nil 37 | } 38 | -------------------------------------------------------------------------------- /pkg/net/requests.go: -------------------------------------------------------------------------------- 1 | package net 2 | 3 | // DNSAnswer is the type used by Amass to represent a DNS record. 4 | type DNSAnswer struct { 5 | Name string `json:"name"` 6 | Type int `json:"type"` 7 | TTL int `json:"TTL"` 8 | Data string `json:"data"` 9 | } 10 | 11 | // DNSRequest handles data needed throughout Service processing of a DNS name. 12 | type DNSRequest struct { 13 | Name string 14 | Domain string 15 | Records []DNSAnswer 16 | Tag string 17 | Source string 18 | } 19 | 20 | // Request tag types. 21 | const ( 22 | NONE = "none" 23 | ALT = "alt" 24 | GUESS = "guess" 25 | ARCHIVE = "archive" 26 | API = "api" 27 | AXFR = "axfr" 28 | BRUTE = "brute" 29 | CERT = "cert" 30 | CRAWL = "crawl" 31 | DNS = "dns" 32 | RIR = "rir" 33 | EXTERNAL = "ext" 34 | SCRAPE = "scrape" 35 | ) 36 | -------------------------------------------------------------------------------- /pkg/active/active_test.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "fmt" 5 | "github.com/ZhuriLab/Starmap/pkg/resolve" 6 | "testing" 7 | ) 8 | 9 | func TestEnum(t *testing.T) { 10 | uniqueMap := make(map[string]resolve.HostEntry) 11 | resolvers := []string{ 12 | "114.114.114.114", 13 | } 14 | uniqueMap, _ = Enum("baidu.com", uniqueMap, false, "", 2, "", resolvers, nil, 30) 15 | for k, v := range uniqueMap { 16 | fmt.Println(k, v) 17 | } 18 | 19 | } 20 | 21 | func TestVerify(t *testing.T) { 22 | uniqueMap := make(map[string]resolve.HostEntry) 23 | 24 | hostEntry := resolve.HostEntry{Host: "www.baidu.com", Source: ""} 25 | 26 | uniqueMap["www.baidu.com"] = hostEntry 27 | 28 | fmt.Println(uniqueMap) 29 | 30 | resolvers := []string{ 31 | "114.114.114.114", 32 | } 33 | 34 | uniqueMap, _ = Verify(uniqueMap, true, resolvers, nil, 30) 35 | for k, v := range uniqueMap { 36 | fmt.Println(k, v) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /pkg/subTakeOver/requests.go: -------------------------------------------------------------------------------- 1 | package subTakeOver 2 | 3 | import ( 4 | "crypto/tls" 5 | "github.com/valyala/fasthttp" 6 | "time" 7 | ) 8 | 9 | func get(url string, ssl bool, timeout int) (body []byte) { 10 | req := fasthttp.AcquireRequest() 11 | req.SetRequestURI(site(url, ssl)) 12 | req.Header.Add("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") 13 | req.Header.Add("Connection", "close") 14 | resp := fasthttp.AcquireResponse() 15 | 16 | client := &fasthttp.Client{TLSConfig: &tls.Config{InsecureSkipVerify: true}} 17 | err := client.DoTimeout(req, resp, time.Duration(timeout)*time.Second) 18 | 19 | if err != nil { 20 | return nil 21 | } 22 | 23 | return resp.Body() 24 | } 25 | 26 | func site(url string, ssl bool) (site string) { 27 | site = "http://" + url 28 | if ssl { 29 | site = "https://" + url 30 | } 31 | 32 | return site 33 | } 34 | -------------------------------------------------------------------------------- /pkg/goflags/string_slice.go: -------------------------------------------------------------------------------- 1 | package goflags 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | // StringSlice is a slice of strings 8 | type StringSlice []string 9 | 10 | func (stringSlice StringSlice) String() string { 11 | return stringSlice.createStringArrayDefaultValue() 12 | } 13 | 14 | // Set appends a value to the string slice. 15 | func (stringSlice *StringSlice) Set(value string) error { 16 | *stringSlice = append(*stringSlice, value) 17 | return nil 18 | } 19 | 20 | func (stringSlice *StringSlice) createStringArrayDefaultValue() string { 21 | defaultBuilder := &strings.Builder{} 22 | defaultBuilder.WriteString("[") 23 | for i, k := range *stringSlice { 24 | defaultBuilder.WriteString("\"") 25 | defaultBuilder.WriteString(k) 26 | defaultBuilder.WriteString("\"") 27 | if i != len(*stringSlice)-1 { 28 | defaultBuilder.WriteString(", ") 29 | } 30 | } 31 | defaultBuilder.WriteString("]") 32 | return defaultBuilder.String() 33 | } 34 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.2.0 4 | - [feat] 增加dns域传送尝试 5 | - [feat] 尝试内存优化 6 | - [fix] 修复向关闭的 channel 发送数据导致程序异常退出 7 | 8 | ## v0.1.1 9 | - [feat] 增加 henter、quake 10 | 11 | ## v0.1.0 12 | - [fix] 修复一处 goroutine 泄露 13 | 14 | ## v0.0.9 15 | - [fix] 修复泛解析时 map 没有初始化的 bug 16 | 17 | ## v0.0.8 18 | - [feat] 优化泛解析，添加参数 mI, 爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析(默认 100) 19 | 20 | ## v0.0.7 21 | - [fix] 修复静默模式下还会输出 banner 的 bug 22 | 23 | ## v0.0.6 24 | - [fix] 修复-s 指定源不生效的 bug 25 | - [feat] 网络空间引擎搜集子域时，同时获取子域的 ip、 开放的端口 26 | - [x] shodan 27 | - [x] fofa 28 | - [x] zoomeyeapi 29 | 30 | - 主动爆破、泛解析过滤改为默认不使用，使用时请添加 -b/-rW 参数 31 | 32 | ## v0.0.5 33 | - 子域名爆破时泛解析过滤 34 | - 参考 https://github.com/boy-hack/ksubdomain/issues/5 35 | 36 | ## v0.0.4 37 | - 修复修改二进制文件名可能读取不到配置文件的 bug 38 | 39 | ## v0.0.3 40 | - 参考 [subjack](https://github.com/haccer/subjack) 添加子域名接管检测 41 | - 合并 [subfinder](https://github.com/projectdiscovery/subfinder) v2.5.0 42 | 43 | ## v0.0.2 44 | - [subfinder](https://github.com/projectdiscovery/subfinder) 和 [ksubdomain](https://github.com/boy-hack/ksubdomain) 初步融合 45 | 46 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 ZhuriLab 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/chaos/chaos.go: -------------------------------------------------------------------------------- 1 | // Package chaos logic 2 | package chaos 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | "github.com/projectdiscovery/chaos-client/pkg/chaos" 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Run function returns all subdomains found with the service 16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 17 | results := make(chan subscraping.Result) 18 | 19 | go func() { 20 | defer close(results) 21 | 22 | if session.Keys.Chaos == "" { 23 | return 24 | } 25 | 26 | chaosClient := chaos.New(session.Keys.Chaos) 27 | for result := range chaosClient.GetSubdomains(&chaos.SubdomainsRequest{ 28 | Domain: domain, 29 | }) { 30 | if result.Error != nil { 31 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: result.Error} 32 | break 33 | } 34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: fmt.Sprintf("%s.%s", result.Subdomain, domain)} 35 | } 36 | }() 37 | 38 | return results 39 | } 40 | 41 | // Name returns the name of the source 42 | func (s *Source) Name() string { 43 | return "chaos" 44 | } 45 | -------------------------------------------------------------------------------- /pkg/active/options.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "github.com/ZhuriLab/Starmap/pkg/resolve" 5 | "github.com/projectdiscovery/gologger" 6 | "strconv" 7 | ) 8 | 9 | type Options struct { 10 | Rate int64 11 | Domain string 12 | FileName string // 字典文件名 13 | Resolvers []string 14 | Output string // 输出文件名 15 | Silent bool 16 | WildcardIPs map[string]struct{} 17 | WildcardIPsAc map[string]struct{} 18 | MaxIPs int 19 | TimeOut int 20 | Retry int 21 | Method string // verify模式 enum模式 test模式 22 | Level int 23 | LevelDomains []string 24 | UniqueMap map[string]resolve.HostEntry 25 | } 26 | 27 | func Band2Rate(bandWith string) int64 { 28 | suffix := string(bandWith[len(bandWith)-1]) 29 | rate, _ := strconv.ParseInt(string(bandWith[0:len(bandWith)-1]), 10, 64) 30 | switch suffix { 31 | case "G": 32 | fallthrough 33 | case "g": 34 | rate *= 1000000000 35 | case "M": 36 | fallthrough 37 | case "m": 38 | rate *= 1000000 39 | case "K": 40 | fallthrough 41 | case "k": 42 | rate *= 1000 43 | default: 44 | gologger.Fatal().Msgf("unknown bandwith suffix '%s' (supported suffixes are G,M and K)\n", suffix) 45 | } 46 | packSize := int64(80) // 一个DNS包大概有74byte 47 | rate = rate / packSize 48 | return rate 49 | } 50 | 51 | -------------------------------------------------------------------------------- /pkg/resolve/client.go: -------------------------------------------------------------------------------- 1 | package resolve 2 | 3 | import ( 4 | "github.com/projectdiscovery/dnsx/libs/dnsx" 5 | ) 6 | 7 | // DefaultResolvers contains the default list of resolvers known to be good 8 | var DefaultResolvers = []string { 9 | "1.1.1.1", // Cloudflare 10 | "1.0.0.1", // Cloudlfare secondary 11 | "8.8.8.8", // Google 12 | "8.8.4.4", // Google secondary 13 | "9.9.9.9", // Quad9 14 | "9.9.9.10", // Quad9 Secondary 15 | "77.88.8.8", // Yandex Primary 16 | "77.88.8.1", // Yandex Secondary 17 | "208.67.222.222", // Cisco OpenDNS 18 | "208.67.220.220", // OpenDNS Secondary 19 | } 20 | 21 | // DefaultResolversCN contains the default list of resolvers known to be good 22 | var DefaultResolversCN = []string{ 23 | "223.5.5.5", // AliDNS 24 | "223.6.6.6", // AliDNS 25 | "119.29.29.29", // DNSPod 26 | "114.114.114.114", // 114DNS 27 | "114.114.115.115", // 114DNS 28 | "101.226.4.6", // DNS 派 29 | "117.50.11.11", // One(微步) DNS 30 | "52.80.66.66", // One(微步) DNS 31 | "1.2.4.8", // CNNIC 32 | "210.2.4.8", // CNNIC 33 | } 34 | 35 | // Resolver is a struct for resolving DNS names 36 | type Resolver struct { 37 | DNSClient *dnsx.DNSX 38 | Resolvers []string 39 | } 40 | 41 | // New creates a new resolver struct with the default resolvers 42 | func New() *Resolver { 43 | return &Resolver{ 44 | Resolvers: []string{}, 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/riddler/riddler.go: -------------------------------------------------------------------------------- 1 | // Package riddler logic 2 | package riddler 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Run function returns all subdomains found with the service 16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 17 | results := make(chan subscraping.Result) 18 | 19 | go func() { 20 | defer close(results) 21 | 22 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://riddler.io/search?q=pld:%s&view_type=data_table", domain)) 23 | if err != nil { 24 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 25 | session.DiscardHTTPResponse(resp) 26 | return 27 | } 28 | 29 | scanner := bufio.NewScanner(resp.Body) 30 | for scanner.Scan() { 31 | line := scanner.Text() 32 | if line == "" { 33 | continue 34 | } 35 | subdomain := session.Extractor.FindString(line) 36 | if subdomain != "" { 37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 38 | } 39 | } 40 | resp.Body.Close() 41 | }() 42 | 43 | return results 44 | } 45 | 46 | // Name returns the name of the source 47 | func (s *Source) Name() string { 48 | return "riddler" 49 | } 50 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/hackertarget/hackertarget.go: -------------------------------------------------------------------------------- 1 | // Package hackertarget logic 2 | package hackertarget 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Run function returns all subdomains found with the service 16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 17 | results := make(chan subscraping.Result) 18 | 19 | go func() { 20 | defer close(results) 21 | 22 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://api.hackertarget.com/hostsearch/?q=%s", domain)) 23 | if err != nil { 24 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 25 | session.DiscardHTTPResponse(resp) 26 | return 27 | } 28 | 29 | defer resp.Body.Close() 30 | 31 | scanner := bufio.NewScanner(resp.Body) 32 | for scanner.Scan() { 33 | line := scanner.Text() 34 | if line == "" { 35 | continue 36 | } 37 | match := session.Extractor.FindAllString(line, -1) 38 | for _, subdomain := range match { 39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 40 | } 41 | } 42 | }() 43 | 44 | return results 45 | } 46 | 47 | // Name returns the name of the source 48 | func (s *Source) Name() string { 49 | return "hackertarget" 50 | } 51 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/anubis/anubis.go: -------------------------------------------------------------------------------- 1 | // Package anubis logic 2 | package anubis 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Run function returns all subdomains found with the service 16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 17 | results := make(chan subscraping.Result) 18 | 19 | go func() { 20 | defer close(results) 21 | 22 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://jonlu.ca/anubis/subdomains/%s", domain)) 23 | if err != nil { 24 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 25 | session.DiscardHTTPResponse(resp) 26 | return 27 | } 28 | 29 | var subdomains []string 30 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) 31 | if err != nil { 32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 33 | resp.Body.Close() 34 | return 35 | } 36 | 37 | resp.Body.Close() 38 | 39 | for _, record := range subdomains { 40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} 41 | } 42 | }() 43 | 44 | return results 45 | } 46 | 47 | // Name returns the name of the source 48 | func (s *Source) Name() string { 49 | return "anubis" 50 | } 51 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/rapiddns/rapiddns.go: -------------------------------------------------------------------------------- 1 | // Package rapiddns is a RapidDNS Scraping Engine in Golang 2 | package rapiddns 3 | 4 | import ( 5 | "context" 6 | "io" 7 | 8 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 9 | ) 10 | 11 | // Source is the passive scraping agent 12 | type Source struct{} 13 | 14 | // Run function returns all subdomains found with the service 15 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 16 | results := make(chan subscraping.Result) 17 | 18 | go func() { 19 | defer close(results) 20 | 21 | resp, err := session.SimpleGet(ctx, "https://rapiddns.io/subdomain/"+domain+"?full=1") 22 | if err != nil { 23 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 24 | session.DiscardHTTPResponse(resp) 25 | return 26 | } 27 | 28 | body, err := io.ReadAll(resp.Body) 29 | if err != nil { 30 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 31 | resp.Body.Close() 32 | return 33 | } 34 | 35 | resp.Body.Close() 36 | 37 | src := string(body) 38 | 39 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { 40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 41 | } 42 | }() 43 | 44 | return results 45 | } 46 | 47 | // Name returns the name of the source 48 | func (s *Source) Name() string { 49 | return "rapiddns" 50 | } 51 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/sublist3r/subllist3r.go: -------------------------------------------------------------------------------- 1 | // Package sublist3r logic 2 | package sublist3r 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Run function returns all subdomains found with the service 16 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 17 | results := make(chan subscraping.Result) 18 | 19 | go func() { 20 | defer close(results) 21 | 22 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.sublist3r.com/search.php?domain=%s", domain)) 23 | if err != nil { 24 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 25 | session.DiscardHTTPResponse(resp) 26 | return 27 | } 28 | 29 | var subdomains []string 30 | err = json.NewDecoder(resp.Body).Decode(&subdomains) 31 | if err != nil { 32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 33 | resp.Body.Close() 34 | return 35 | } 36 | 37 | resp.Body.Close() 38 | 39 | for _, subdomain := range subdomains { 40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 41 | } 42 | }() 43 | 44 | return results 45 | } 46 | 47 | // Name returns the name of the source 48 | func (s *Source) Name() string { 49 | return "sublist3r" 50 | } 51 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/archiveis/archiveis.go: -------------------------------------------------------------------------------- 1 | // Package archiveis is a Archiveis Scraping Engine in Golang 2 | package archiveis 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // Source is the passive scraping agent 13 | type Source struct{} 14 | 15 | // Name returns the name of the source 16 | func (s *Source) Name() string { 17 | return "archiveis" 18 | } 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://archive.is/*.%s", domain)) 28 | 29 | if err != nil { 30 | results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} 31 | session.DiscardHTTPResponse(resp) 32 | return 33 | } 34 | 35 | body, err := io.ReadAll(resp.Body) 36 | if err != nil { 37 | results <- subscraping.Result{Source: "archiveis", Type: subscraping.Error, Error: err} 38 | resp.Body.Close() 39 | return 40 | } 41 | 42 | resp.Body.Close() 43 | 44 | src := string(body) 45 | 46 | for _, subdomain := range session.Extractor.FindAllString(src, -1) { 47 | results <- subscraping.Result{Source: "archiveis", Type: subscraping.Subdomain, Value: subdomain} 48 | } 49 | }() 50 | 51 | return results 52 | } 53 | -------------------------------------------------------------------------------- /pkg/active/statusdb/db.go: -------------------------------------------------------------------------------- 1 | package statusdb 2 | 3 | import ( 4 | "sync" 5 | "sync/atomic" 6 | "time" 7 | ) 8 | 9 | type Item struct { 10 | Domain string // 查询域名 11 | Dns string // 查询dns 12 | Time time.Time // 发送时间 13 | Retry int // 重试次数 14 | DomainLevel int // 域名层级 15 | } 16 | 17 | type StatusDb struct { 18 | Items sync.Map 19 | length int64 20 | } 21 | 22 | // 内存简易读写数据库，自带锁机制 23 | func CreateMemoryDB() *StatusDb { 24 | db := &StatusDb{ 25 | Items: sync.Map{}, 26 | length: 0, 27 | } 28 | return db 29 | } 30 | func (r *StatusDb) Add(domain string, tableData Item) { 31 | r.Items.Store(domain, tableData) 32 | atomic.AddInt64(&r.length, 1) 33 | } 34 | func (r *StatusDb) Set(domain string, tableData Item) { 35 | r.Items.Store(domain, tableData) 36 | } 37 | func (r *StatusDb) Get(domain string) (Item, bool) { 38 | v, ok := r.Items.Load(domain) 39 | if !ok { 40 | return Item{}, false 41 | } 42 | return v.(Item), ok 43 | } 44 | func (r *StatusDb) Length() int64 { 45 | return r.length 46 | } 47 | func (r *StatusDb) Del(domain string) { 48 | //r.Mu.Lock() 49 | //defer r.Mu.Unlock() 50 | _, ok := r.Items.LoadAndDelete(domain) 51 | if ok { 52 | atomic.AddInt64(&r.length, -1) 53 | } 54 | } 55 | 56 | func (r *StatusDb) Scan(f func(key string, value Item) error) { 57 | r.Items.Range(func(key, value interface{}) bool { 58 | k := key.(string) 59 | item := value.(Item) 60 | f(k, item) 61 | return true 62 | }) 63 | } 64 | func (r *StatusDb) Close() { 65 | 66 | } 67 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/threatcrowd/threatcrowd.go: -------------------------------------------------------------------------------- 1 | // Package threatcrowd logic 2 | package threatcrowd 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | type response struct { 14 | Subdomains []string `json:"subdomains"` 15 | } 16 | 17 | // Source is the passive scraping agent 18 | type Source struct{} 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.threatcrowd.org/searchApi/v2/domain/report/?domain=%s", domain)) 28 | if err != nil { 29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 30 | session.DiscardHTTPResponse(resp) 31 | return 32 | } 33 | 34 | defer resp.Body.Close() 35 | 36 | var data response 37 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 38 | if err != nil { 39 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 40 | return 41 | } 42 | 43 | for _, subdomain := range data.Subdomains { 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 45 | } 46 | }() 47 | 48 | return results 49 | } 50 | 51 | // Name returns the name of the source 52 | func (s *Source) Name() string { 53 | return "threatcrowd" 54 | } 55 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/github/tokenmanager.go: -------------------------------------------------------------------------------- 1 | package github 2 | 3 | import "time" 4 | 5 | // Token struct 6 | type Token struct { 7 | Hash string 8 | RetryAfter int64 9 | ExceededTime time.Time 10 | } 11 | 12 | // Tokens is the internal struct to manage the current token 13 | // and the pool 14 | type Tokens struct { 15 | current int 16 | pool []Token 17 | } 18 | 19 | // NewTokenManager initialize the tokens pool 20 | func NewTokenManager(keys []string) *Tokens { 21 | pool := []Token{} 22 | for _, key := range keys { 23 | t := Token{Hash: key, ExceededTime: time.Time{}, RetryAfter: 0} 24 | pool = append(pool, t) 25 | } 26 | 27 | return &Tokens{ 28 | current: 0, 29 | pool: pool, 30 | } 31 | } 32 | 33 | func (r *Tokens) setCurrentTokenExceeded(retryAfter int64) { 34 | if r.current >= len(r.pool) { 35 | r.current %= len(r.pool) 36 | } 37 | if r.pool[r.current].RetryAfter == 0 { 38 | r.pool[r.current].ExceededTime = time.Now() 39 | r.pool[r.current].RetryAfter = retryAfter 40 | } 41 | } 42 | 43 | // Get returns a new token from the token pool 44 | func (r *Tokens) Get() *Token { 45 | resetExceededTokens(r) 46 | 47 | if r.current >= len(r.pool) { 48 | r.current %= len(r.pool) 49 | } 50 | 51 | result := &r.pool[r.current] 52 | r.current++ 53 | 54 | return result 55 | } 56 | 57 | func resetExceededTokens(r *Tokens) { 58 | for i, token := range r.pool { 59 | if token.RetryAfter > 0 { 60 | if int64(time.Since(token.ExceededTime)/time.Second) > token.RetryAfter { 61 | r.pool[i].ExceededTime = time.Time{} 62 | r.pool[i].RetryAfter = 0 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/fullhunt/fullhunt.go: -------------------------------------------------------------------------------- 1 | package fullhunt 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | 7 | jsoniter "github.com/json-iterator/go" 8 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 9 | ) 10 | 11 | //fullhunt response 12 | type fullHuntResponse struct { 13 | Hosts []string `json:"hosts"` 14 | Message string `json:"message"` 15 | Status int `json:"status"` 16 | } 17 | 18 | // Source is the passive scraping agent 19 | type Source struct{} 20 | 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | resp, err := session.Get(ctx, fmt.Sprintf("https://fullhunt.io/api/v1/domain/%s/subdomains", domain), "", map[string]string{"X-API-KEY": session.Keys.FullHunt}) 28 | if err != nil { 29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 30 | session.DiscardHTTPResponse(resp) 31 | return 32 | } 33 | 34 | var response fullHuntResponse 35 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 36 | if err != nil { 37 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 38 | resp.Body.Close() 39 | return 40 | } 41 | resp.Body.Close() 42 | for _, record := range response.Hosts { 43 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record} 44 | } 45 | }() 46 | return results 47 | } 48 | 49 | // Name returns the name of the source 50 | func (s *Source) Name() string { 51 | return "fullhunt" 52 | } 53 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/sonarsearch/sonarsearch.go: -------------------------------------------------------------------------------- 1 | // Package sonarsearch logic 2 | package sonarsearch 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "strconv" 9 | 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct{} 15 | 16 | // Run function returns all subdomains found with the service 17 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 18 | results := make(chan subscraping.Result) 19 | go func() { 20 | defer close(results) 21 | 22 | getURL := fmt.Sprintf("https://sonar.omnisint.io/subdomains/%s?page=", domain) 23 | page := 0 24 | var subdomains []string 25 | for { 26 | resp, err := session.SimpleGet(ctx, getURL+strconv.Itoa(page)) 27 | if err != nil { 28 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 29 | session.DiscardHTTPResponse(resp) 30 | return 31 | } 32 | 33 | if err := json.NewDecoder(resp.Body).Decode(&subdomains); err != nil { 34 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 35 | resp.Body.Close() 36 | return 37 | } 38 | resp.Body.Close() 39 | 40 | if len(subdomains) == 0 { 41 | return 42 | } 43 | 44 | for _, subdomain := range subdomains { 45 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 46 | } 47 | 48 | page++ 49 | } 50 | }() 51 | 52 | return results 53 | } 54 | 55 | // Name returns the name of the source 56 | func (s *Source) Name() string { 57 | return "sonarsearch" 58 | } 59 | -------------------------------------------------------------------------------- /pkg/runner/validate.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "errors" 5 | "github.com/projectdiscovery/gologger" 6 | "github.com/projectdiscovery/gologger/formatter" 7 | "github.com/projectdiscovery/gologger/levels" 8 | ) 9 | 10 | // validateOptions validates the configuration options passed 11 | func (options *Options) validateOptions() error { 12 | // Check if domain, list of domains, or stdin info was provided. 13 | // If none was provided, then return. 14 | if len(options.Domain) == 0 && options.DomainsFile == "" && !options.Stdin { 15 | return errors.New("no input list provided") 16 | } 17 | 18 | // Both verbose and silent flags were used 19 | if options.Verbose && options.Silent { 20 | return errors.New("both verbose and silent mode specified") 21 | } 22 | 23 | // Validate threads and options 24 | if options.Threads == 0 { 25 | return errors.New("threads cannot be zero") 26 | } 27 | if options.Timeout == 0 { 28 | return errors.New("timeout cannot be zero") 29 | } 30 | 31 | // Always remove wildcard with hostip 32 | if options.HostIP && !options.RemoveWildcard { 33 | return errors.New("hostip flag must be used with RemoveWildcard option") 34 | } 35 | 36 | return nil 37 | } 38 | 39 | // ConfigureOutput configures the output on the screen 40 | func (options *Options) ConfigureOutput() { 41 | // If the user desires verbose output, show verbose output 42 | if options.Verbose { 43 | gologger.DefaultLogger.SetMaxLevel(levels.LevelVerbose) 44 | } 45 | if options.NoColor { 46 | gologger.DefaultLogger.SetFormatter(formatter.NewCLI(true)) 47 | } 48 | if options.Silent { 49 | gologger.DefaultLogger.SetMaxLevel(levels.LevelSilent) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/threatminer/threatminer.go: -------------------------------------------------------------------------------- 1 | // Package threatminer logic 2 | package threatminer 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | type response struct { 14 | StatusCode string `json:"status_code"` 15 | StatusMessage string `json:"status_message"` 16 | Results []string `json:"results"` 17 | } 18 | 19 | // Source is the passive scraping agent 20 | type Source struct{} 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatminer.org/v2/domain.php?q=%s&rt=5", domain)) 30 | if err != nil { 31 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 32 | session.DiscardHTTPResponse(resp) 33 | return 34 | } 35 | 36 | defer resp.Body.Close() 37 | 38 | var data response 39 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | return 43 | } 44 | 45 | for _, subdomain := range data.Results { 46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 47 | } 48 | }() 49 | 50 | return results 51 | } 52 | 53 | // Name returns the name of the source 54 | func (s *Source) Name() string { 55 | return "threatminer" 56 | } 57 | -------------------------------------------------------------------------------- /pkg/passive/passive.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "sync" 7 | "time" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // EnumerateSubdomains enumerates all the subdomains for a given domain 13 | func (a *Agent) EnumerateSubdomains(domain string, keys *subscraping.Keys, proxy string, rateLimit, timeout int, maxEnumTime time.Duration) chan subscraping.Result { 14 | results := make(chan subscraping.Result) 15 | go func() { 16 | session, err := subscraping.NewSession(domain, keys, proxy, rateLimit, timeout) 17 | if err != nil { 18 | results <- subscraping.Result{Type: subscraping.Error, Error: fmt.Errorf("could not init passive session for %s: %s", domain, err)} 19 | } 20 | 21 | ctx, cancel := context.WithTimeout(context.Background(), maxEnumTime) 22 | 23 | timeTaken := make(map[string]string) 24 | timeTakenMutex := &sync.Mutex{} 25 | 26 | wg := &sync.WaitGroup{} 27 | // Run each source in parallel on the target domain 28 | for source, runner := range a.sources { 29 | wg.Add(1) 30 | 31 | now := time.Now() 32 | go func(source string, runner subscraping.Source) { 33 | for resp := range runner.Run(ctx, domain, session) { 34 | results <- resp 35 | } 36 | 37 | duration := time.Since(now) 38 | timeTakenMutex.Lock() 39 | timeTaken[source] = fmt.Sprintf("Source took %s for enumeration\n", duration) 40 | timeTakenMutex.Unlock() 41 | 42 | wg.Done() 43 | }(source, runner) 44 | } 45 | wg.Wait() 46 | 47 | //for source, data := range timeTaken { 48 | // gologger.Verbose().Label(source).Msg(data) 49 | //} 50 | 51 | close(results) 52 | cancel() 53 | }() 54 | 55 | return results 56 | } 57 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/virustotal/virustotal.go: -------------------------------------------------------------------------------- 1 | // Package virustotal logic 2 | package virustotal 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | type response struct { 13 | Subdomains []string `json:"subdomains"` 14 | } 15 | 16 | // Source is the passive scraping agent 17 | type Source struct{} 18 | 19 | // Run function returns all subdomains found with the service 20 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 21 | results := make(chan subscraping.Result) 22 | 23 | go func() { 24 | defer close(results) 25 | 26 | if session.Keys.Virustotal == "" { 27 | return 28 | } 29 | 30 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://www.virustotal.com/vtapi/v2/domain/report?apikey=%s&domain=%s", session.Keys.Virustotal, domain)) 31 | if err != nil { 32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 33 | session.DiscardHTTPResponse(resp) 34 | return 35 | } 36 | 37 | var data response 38 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 39 | if err != nil { 40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 41 | resp.Body.Close() 42 | return 43 | } 44 | 45 | resp.Body.Close() 46 | 47 | for _, subdomain := range data.Subdomains { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 49 | } 50 | }() 51 | 52 | return results 53 | } 54 | 55 | // Name returns the name of the source 56 | func (s *Source) Name() string { 57 | return "virustotal" 58 | } 59 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/waybackarchive/waybackarchive.go: -------------------------------------------------------------------------------- 1 | // Package waybackarchive logic 2 | package waybackarchive 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "net/url" 9 | "strings" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 12 | ) 13 | 14 | // Source is the passive scraping agent 15 | type Source struct{} 16 | 17 | // Run function returns all subdomains found with the service 18 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 19 | results := make(chan subscraping.Result) 20 | 21 | go func() { 22 | defer close(results) 23 | 24 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("http://web.archive.org/cdx/search/cdx?url=*.%s/*&output=txt&fl=original&collapse=urlkey", domain)) 25 | if err != nil { 26 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 27 | session.DiscardHTTPResponse(resp) 28 | return 29 | } 30 | 31 | defer resp.Body.Close() 32 | 33 | scanner := bufio.NewScanner(resp.Body) 34 | for scanner.Scan() { 35 | line := scanner.Text() 36 | if line == "" { 37 | continue 38 | } 39 | line, _ = url.QueryUnescape(line) 40 | subdomain := session.Extractor.FindString(line) 41 | if subdomain != "" { 42 | // fix for triple encoded URL 43 | subdomain = strings.ToLower(subdomain) 44 | subdomain = strings.TrimPrefix(subdomain, "25") 45 | subdomain = strings.TrimPrefix(subdomain, "2f") 46 | 47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 48 | } 49 | } 50 | }() 51 | 52 | return results 53 | } 54 | 55 | // Name returns the name of the source 56 | func (s *Source) Name() string { 57 | return "waybackarchive" 58 | } 59 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: 🎉 Build Binary 2 | on: 3 | create: 4 | tags: 5 | - v* 6 | workflow_dispatch: 7 | jobs: 8 | 9 | build: 10 | name: Build 11 | runs-on: ${{ matrix.os }} 12 | strategy: 13 | matrix: 14 | os: 15 | - ubuntu-latest 16 | - macos-latest 17 | - windows-latest 18 | steps: 19 | - name: Set up Go 1.17 20 | uses: actions/setup-go@v2 21 | with: 22 | go-version: 1.17 23 | id: go 24 | 25 | - name: Check out code into the Go module directory 26 | uses: actions/checkout@v2 27 | 28 | - name: Set up libpcap-dev 29 | if: matrix.os == 'ubuntu-latest' 30 | run: sudo apt-get install libpcap-dev gcc -y 31 | 32 | - name: Get dependencies 33 | run: go mod download 34 | 35 | - name: Build On Linux 36 | run: | 37 | go build -o Starmap-linux cmd/Starmap.go 38 | chmod +x Starmap-linux 39 | if: matrix.os == 'ubuntu-latest' 40 | env: 41 | GOENABLE: 1 42 | CGO_LDFLAGS: "-Wl,-static -L/usr/lib/x86_64-linux-gnu/libpcap.a -lpcap -Wl,-Bdynamic" 43 | - name: Build On Darwin 44 | run: | 45 | go build -o Starmap-darwin cmd/Starmap.go 46 | chmod +x Starmap-darwin 47 | if: matrix.os == 'macos-latest' 48 | - name: Build On Windows 49 | run: | 50 | go build -o Starmap-windows.exe cmd/Starmap.go 51 | if: matrix.os == 'windows-latest' 52 | env: 53 | GOOS: windows 54 | GOENABLE: 1 55 | - name: Release 56 | uses: softprops/action-gh-release@master 57 | with: 58 | files: Starmap-* 59 | env: 60 | GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" -------------------------------------------------------------------------------- /pkg/subscraping/sources/chinaz/chinaz.go: -------------------------------------------------------------------------------- 1 | package chinaz 2 | 3 | // chinaz http://my.chinaz.com/ChinazAPI/DataCenter/MyDataApi 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct{} 15 | 16 | // Run function returns all subdomains found with the service 17 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 18 | results := make(chan subscraping.Result) 19 | 20 | go func() { 21 | defer close(results) 22 | 23 | if session.Keys.Chinaz == "" { 24 | return 25 | } 26 | 27 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://apidatav2.chinaz.com/single/alexa?key=%s&domain=%s", session.Keys.Chinaz, domain)) 28 | if err != nil { 29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 30 | session.DiscardHTTPResponse(resp) 31 | return 32 | } 33 | 34 | body, err := io.ReadAll(resp.Body) 35 | 36 | resp.Body.Close() 37 | 38 | SubdomainList := jsoniter.Get(body, "Result").Get("ContributingSubdomainList") 39 | 40 | if SubdomainList.ToBool() { 41 | _data := []byte(SubdomainList.ToString()) 42 | for i := 0; i < SubdomainList.Size(); i++ { 43 | subdomain := jsoniter.Get(_data, i, "DataUrl").ToString() 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 45 | } 46 | } else { 47 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 48 | return 49 | } 50 | }() 51 | 52 | return results 53 | } 54 | 55 | // Name returns the name of the source 56 | func (s *Source) Name() string { 57 | return "chinaz" 58 | } 59 | -------------------------------------------------------------------------------- /pkg/goflags/runtime_map.go: -------------------------------------------------------------------------------- 1 | package goflags 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/projectdiscovery/stringsutil" 9 | ) 10 | 11 | const ( 12 | kvSep = "=" 13 | ) 14 | 15 | // RuntimeMap is a runtime only map of interfaces 16 | type RuntimeMap struct { 17 | kv map[string]interface{} 18 | } 19 | 20 | func (runtimeMap RuntimeMap) String() string { 21 | defaultBuilder := &strings.Builder{} 22 | defaultBuilder.WriteString("{") 23 | 24 | var items string 25 | for k, v := range runtimeMap.kv { 26 | items += fmt.Sprintf("\"%s\"=\"%s\"%s", k, v, kvSep) 27 | } 28 | defaultBuilder.WriteString(stringsutil.TrimSuffixAny(items, ",", "=")) 29 | defaultBuilder.WriteString("}") 30 | return defaultBuilder.String() 31 | } 32 | 33 | // Set inserts a value to the map. Format: key=value 34 | func (runtimeMap *RuntimeMap) Set(value string) error { 35 | if runtimeMap.kv == nil { 36 | runtimeMap.kv = make(map[string]interface{}) 37 | } 38 | k, v := stringsutil.Before(value, kvSep), stringsutil.After(value, kvSep) 39 | // note: 40 | // - inserting multiple times the same key will override the previous value 41 | // - empty string is legitimate value 42 | if k != "" { 43 | runtimeMap.kv[k] = v 44 | } 45 | return nil 46 | } 47 | 48 | // Del removes the specified key 49 | func (runtimeMap *RuntimeMap) Del(key string) error { 50 | if runtimeMap.kv == nil { 51 | return errors.New("empty runtime map") 52 | } 53 | delete(runtimeMap.kv, key) 54 | return nil 55 | } 56 | 57 | // IsEmpty specifies if the underlying map is empty 58 | func (runtimeMap *RuntimeMap) IsEmpty() bool { 59 | return runtimeMap.kv == nil || len(runtimeMap.kv) == 0 60 | } 61 | 62 | // AsMap returns the internal map as reference - changes are allowed 63 | func (runtimeMap *RuntimeMap) AsMap() map[string]interface{} { 64 | return runtimeMap.kv 65 | } 66 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/securitytrails/securitytrails.go: -------------------------------------------------------------------------------- 1 | // Package securitytrails logic 2 | package securitytrails 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | type response struct { 14 | Subdomains []string `json:"subdomains"` 15 | } 16 | 17 | // Source is the passive scraping agent 18 | type Source struct{} 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | if session.Keys.Securitytrails == "" { 28 | return 29 | } 30 | 31 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.securitytrails.com/v1/domain/%s/subdomains", domain), "", map[string]string{"APIKEY": session.Keys.Securitytrails}) 32 | if err != nil { 33 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 34 | session.DiscardHTTPResponse(resp) 35 | return 36 | } 37 | 38 | var securityTrailsResponse response 39 | err = jsoniter.NewDecoder(resp.Body).Decode(&securityTrailsResponse) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | resp.Body.Close() 43 | return 44 | } 45 | 46 | resp.Body.Close() 47 | 48 | for _, subdomain := range securityTrailsResponse.Subdomains { 49 | if strings.HasSuffix(subdomain, ".") { 50 | subdomain += domain 51 | } else { 52 | subdomain = subdomain + "." + domain 53 | } 54 | 55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 56 | } 57 | }() 58 | 59 | return results 60 | } 61 | 62 | // Name returns the name of the source 63 | func (s *Source) Name() string { 64 | return "securitytrails" 65 | } 66 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/alienvault/alienvault.go: -------------------------------------------------------------------------------- 1 | // Package alienvault logic 2 | package alienvault 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | type alienvaultResponse struct { 13 | Detail string `json:"detail"` 14 | Error string `json:"error"` 15 | PassiveDNS []struct { 16 | Hostname string `json:"hostname"` 17 | } `json:"passive_dns"` 18 | } 19 | 20 | // Source is the passive scraping agent 21 | type Source struct{} 22 | 23 | // Run function returns all subdomains found with the service 24 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 25 | results := make(chan subscraping.Result) 26 | 27 | go func() { 28 | defer close(results) 29 | 30 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://otx.alienvault.com/api/v1/indicators/domain/%s/passive_dns", domain)) 31 | if err != nil && resp == nil { 32 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 33 | session.DiscardHTTPResponse(resp) 34 | return 35 | } 36 | 37 | var response alienvaultResponse 38 | // Get the response body and decode 39 | err = json.NewDecoder(resp.Body).Decode(&response) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | resp.Body.Close() 43 | return 44 | } 45 | resp.Body.Close() 46 | 47 | if response.Error != "" { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s, %s", response.Detail, response.Error)} 49 | return 50 | } 51 | 52 | for _, record := range response.PassiveDNS { 53 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: record.Hostname} 54 | } 55 | }() 56 | 57 | return results 58 | } 59 | 60 | // Name returns the name of the source 61 | func (s *Source) Name() string { 62 | return "alienvault" 63 | } 64 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/dnsdb/dnsdb.go: -------------------------------------------------------------------------------- 1 | // Package dnsdb logic 2 | package dnsdb 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "context" 8 | "fmt" 9 | "strings" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | ) 14 | 15 | type dnsdbResponse struct { 16 | Name string `json:"rrname"` 17 | } 18 | 19 | // Source is the passive scraping agent 20 | type Source struct{} 21 | 22 | // Run function returns all subdomains found with the service 23 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 24 | results := make(chan subscraping.Result) 25 | 26 | go func() { 27 | defer close(results) 28 | 29 | if session.Keys.DNSDB == "" { 30 | return 31 | } 32 | 33 | headers := map[string]string{ 34 | "X-API-KEY": session.Keys.DNSDB, 35 | "Accept": "application/json", 36 | "Content-Type": "application/json", 37 | } 38 | 39 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.dnsdb.info/lookup/rrset/name/*.%s?limit=1000000000000", domain), "", headers) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | session.DiscardHTTPResponse(resp) 43 | return 44 | } 45 | 46 | scanner := bufio.NewScanner(resp.Body) 47 | for scanner.Scan() { 48 | line := scanner.Text() 49 | if line == "" { 50 | continue 51 | } 52 | var response dnsdbResponse 53 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) 54 | if err != nil { 55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 56 | return 57 | } 58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: strings.TrimSuffix(response.Name, ".")} 59 | } 60 | resp.Body.Close() 61 | }() 62 | return results 63 | } 64 | 65 | // Name returns the name of the source 66 | func (s *Source) Name() string { 67 | return "DNSDB" 68 | } 69 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/c99/c99.go: -------------------------------------------------------------------------------- 1 | // Package c99 logic 2 | package c99 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | // Source is the passive scraping agent 14 | type Source struct{} 15 | 16 | type dnsdbLookupResponse struct { 17 | Success bool `json:"success"` 18 | Subdomains []struct { 19 | Subdomain string `json:"subdomain"` 20 | IP string `json:"ip"` 21 | Cloudflare bool `json:"cloudflare"` 22 | } `json:"subdomains"` 23 | Error string `json:"error"` 24 | } 25 | 26 | // Run function returns all subdomains found with the service 27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 28 | results := make(chan subscraping.Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | if session.Keys.C99 == "" { 34 | return 35 | } 36 | 37 | searchURL := fmt.Sprintf("https://api.c99.nl/subdomainfinder?key=%s&domain=%s&json", session.Keys.C99, domain) 38 | resp, err := session.SimpleGet(ctx, searchURL) 39 | if err != nil { 40 | session.DiscardHTTPResponse(resp) 41 | return 42 | } 43 | 44 | defer resp.Body.Close() 45 | 46 | var response dnsdbLookupResponse 47 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 48 | if err != nil { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 50 | return 51 | } 52 | 53 | if response.Error != "" { 54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", response.Error)} 55 | return 56 | } 57 | 58 | for _, data := range response.Subdomains { 59 | if !strings.HasPrefix(data.Subdomain, ".") { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data.Subdomain} 61 | } 62 | } 63 | }() 64 | 65 | return results 66 | } 67 | 68 | // Name returns the name of the source 69 | func (s *Source) Name() string { 70 | return "c99" 71 | } 72 | -------------------------------------------------------------------------------- /pkg/subTakeOver/subTakeOver.go: -------------------------------------------------------------------------------- 1 | package subTakeOver 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/ZhuriLab/Starmap/pkg/resolve" 7 | "github.com/ZhuriLab/Starmap/pkg/subTakeOver/assets" 8 | "github.com/projectdiscovery/gologger" 9 | "sync" 10 | ) 11 | 12 | type Options struct { 13 | Timeout int 14 | Ssl bool 15 | All bool 16 | Verbose bool 17 | Fingerprints []Fingerprints 18 | } 19 | 20 | // Process Start processing subTakeOver from the defined options. 21 | func Process(uniqueMap map[string]resolve.HostEntry, all, verbose bool) map[string]resolve.HostEntry{ 22 | hostEntrys := make(chan resolve.HostEntry, 99) 23 | 24 | var data []Fingerprints 25 | err := json.Unmarshal(assets.Fingerprints, &data) 26 | if err != nil { 27 | gologger.Fatal().Msgf("%s", err) 28 | } 29 | 30 | o := &Options { 31 | Timeout : 10, 32 | Ssl: false, 33 | All: all, 34 | Fingerprints: data, 35 | Verbose: verbose, 36 | } 37 | 38 | wg := new(sync.WaitGroup) 39 | 40 | for i := 0; i < 30; i++ { 41 | wg.Add(1) 42 | go func() { 43 | for hostEntry := range hostEntrys { 44 | 45 | if all { 46 | service := Identify(hostEntry.Host, hostEntry.CNames, o, "", Fingerprints{}) 47 | if service != "" { 48 | gologger.Info().Label(service).Msgf(hostEntry.Host) 49 | hostEntry.TakeOver = true 50 | uniqueMap[hostEntry.Host] = hostEntry 51 | } 52 | 53 | if service == "" && o.Verbose { 54 | gologger.Info().Label("[Not Vulnerable]").Msgf(hostEntry.Host) 55 | } 56 | 57 | } else { 58 | // 仅测试 cname 匹配的 url 59 | if ok, cname, fingerprint := VerifyCNAME(hostEntry.CNames, o.Fingerprints); ok { 60 | service := Identify(hostEntry.Host, hostEntry.CNames, o, cname, fingerprint) 61 | if service != "" { 62 | gologger.Info().Label(service).Msgf(hostEntry.Host) 63 | hostEntry.TakeOver = true 64 | uniqueMap[hostEntry.Host] = hostEntry 65 | } 66 | 67 | if service == "" && o.Verbose { 68 | gologger.Info().Label("[Not Vulnerable]").Msgf(hostEntry.Host) 69 | } 70 | 71 | } 72 | } 73 | } 74 | wg.Done() 75 | }() 76 | } 77 | 78 | for _, result := range uniqueMap { 79 | hostEntrys <- result 80 | fmt.Println(result) 81 | } 82 | 83 | close(hostEntrys) 84 | wg.Wait() 85 | 86 | return uniqueMap 87 | } 88 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/passivetotal/passivetotal.go: -------------------------------------------------------------------------------- 1 | // Package passivetotal logic 2 | package passivetotal 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "regexp" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | type response struct { 14 | Subdomains []string `json:"subdomains"` 15 | } 16 | 17 | // Source is the passive scraping agent 18 | type Source struct{} 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | if session.Keys.PassiveTotalUsername == "" || session.Keys.PassiveTotalPassword == "" { 28 | return 29 | } 30 | 31 | // Create JSON Get body 32 | var request = []byte(`{"query":"` + domain + `"}`) 33 | 34 | resp, err := session.HTTPRequest( 35 | ctx, 36 | "GET", 37 | "https://api.passivetotal.org/v2/enrichment/subdomains", 38 | "", 39 | map[string]string{"Content-Type": "application/json"}, 40 | bytes.NewBuffer(request), 41 | subscraping.BasicAuth{Username: session.Keys.PassiveTotalUsername, Password: session.Keys.PassiveTotalPassword}, 42 | ) 43 | if err != nil { 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 45 | session.DiscardHTTPResponse(resp) 46 | return 47 | } 48 | 49 | var data response 50 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 51 | if err != nil { 52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 53 | resp.Body.Close() 54 | return 55 | } 56 | resp.Body.Close() 57 | 58 | for _, subdomain := range data.Subdomains { 59 | // skip entries like xxx.xxx.xxx.xxx\032domain.tld 60 | if passiveTotalFilterRegex.MatchString(subdomain) { 61 | continue 62 | } 63 | finalSubdomain := subdomain + "." + domain 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: finalSubdomain} 65 | } 66 | }() 67 | 68 | return results 69 | } 70 | 71 | // Name returns the name of the source 72 | func (s *Source) Name() string { 73 | return "passivetotal" 74 | } 75 | 76 | var passiveTotalFilterRegex *regexp.Regexp = regexp.MustCompile(`^(?:\d{1,3}\.){3}\d{1,3}\\032`) 77 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/zoomeyeapi/zoomeyeapi.go: -------------------------------------------------------------------------------- 1 | package zoomeyeapi 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "net/http" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | // search results 13 | type zoomeyeResults struct { 14 | Status int `json:"status"` 15 | Total int `json:"total"` 16 | List []struct { 17 | Name string `json:"name"` 18 | Ip []string `json:"ip"` 19 | } `json:"list"` 20 | } 21 | 22 | // Source is the passive scraping agent 23 | type Source struct{} 24 | 25 | // Run function returns all subdomains found with the service 26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 27 | results := make(chan subscraping.Result) 28 | 29 | go func() { 30 | defer close(results) 31 | 32 | if session.Keys.ZoomEyeKey == "" { 33 | return 34 | } 35 | 36 | headers := map[string]string{ 37 | "API-KEY": session.Keys.ZoomEyeKey, 38 | "Accept": "application/json", 39 | "Content-Type": "application/json", 40 | } 41 | var pages = 1 42 | for currentPage := 1; currentPage <= pages; currentPage++ { 43 | api := fmt.Sprintf("https://api.zoomeye.org/domain/search?q=%s&type=1&s=1000&page=%d", domain, currentPage) 44 | resp, err := session.Get(ctx, api, "", headers) 45 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden 46 | if err != nil { 47 | if !isForbidden { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 49 | session.DiscardHTTPResponse(resp) 50 | } 51 | return 52 | } 53 | 54 | var res zoomeyeResults 55 | err = json.NewDecoder(resp.Body).Decode(&res) 56 | 57 | if err != nil { 58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 59 | _ = resp.Body.Close() 60 | return 61 | } 62 | _ = resp.Body.Close() 63 | pages = int(res.Total/1000) + 1 64 | for _, r := range res.List { 65 | //fmt.Println(r.Ip) 66 | ipPorts := make(map[string][]int) 67 | 68 | for _, ip := range r.Ip { 69 | ipPorts[ip] = nil 70 | } 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Name, IpPorts: ipPorts} 72 | } 73 | } 74 | }() 75 | 76 | return results 77 | } 78 | 79 | // Name returns the name of the source 80 | func (s *Source) Name() string { 81 | return "zoomeyeapi" 82 | } 83 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/sitedossier/sitedossier.go: -------------------------------------------------------------------------------- 1 | // Package sitedossier logic 2 | package sitedossier 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | "math/rand" 9 | "net/http" 10 | "regexp" 11 | "time" 12 | 13 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 14 | ) 15 | 16 | // SleepRandIntn is the integer value to get the pseudo-random number 17 | // to sleep before find the next match 18 | const SleepRandIntn = 5 19 | 20 | var reNext = regexp.MustCompile(``) 21 | 22 | type agent struct { 23 | results chan subscraping.Result 24 | session *subscraping.Session 25 | } 26 | 27 | func (a *agent) enumerate(ctx context.Context, baseURL string) { 28 | select { 29 | case <-ctx.Done(): 30 | return 31 | default: 32 | } 33 | 34 | resp, err := a.session.SimpleGet(ctx, baseURL) 35 | isnotfound := resp != nil && resp.StatusCode == http.StatusNotFound 36 | if err != nil && !isnotfound { 37 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} 38 | a.session.DiscardHTTPResponse(resp) 39 | return 40 | } 41 | 42 | body, err := io.ReadAll(resp.Body) 43 | if err != nil { 44 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Error, Error: err} 45 | resp.Body.Close() 46 | return 47 | } 48 | resp.Body.Close() 49 | 50 | src := string(body) 51 | for _, match := range a.session.Extractor.FindAllString(src, -1) { 52 | a.results <- subscraping.Result{Source: "sitedossier", Type: subscraping.Subdomain, Value: match} 53 | } 54 | 55 | match1 := reNext.FindStringSubmatch(src) 56 | time.Sleep(time.Duration((3 + rand.Intn(SleepRandIntn))) * time.Second) 57 | 58 | if len(match1) > 0 { 59 | a.enumerate(ctx, "http://www.sitedossier.com"+match1[1]) 60 | } 61 | } 62 | 63 | // Source is the passive scraping agent 64 | type Source struct{} 65 | 66 | // Run function returns all subdomains found with the service 67 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 68 | results := make(chan subscraping.Result) 69 | 70 | a := agent{ 71 | session: session, 72 | results: results, 73 | } 74 | 75 | go func() { 76 | a.enumerate(ctx, fmt.Sprintf("http://www.sitedossier.com/parentdomain/%s", domain)) 77 | close(a.results) 78 | }() 79 | 80 | return a.results 81 | } 82 | 83 | // Name returns the name of the source 84 | func (s *Source) Name() string { 85 | return "sitedossier" 86 | } 87 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/threatbook/threatbook.go: -------------------------------------------------------------------------------- 1 | // Package threatbook logic 2 | package threatbook 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strconv" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 11 | ) 12 | 13 | type threatBookResponse struct { 14 | ResponseCode int64 `json:"response_code"` 15 | VerboseMsg string `json:"verbose_msg"` 16 | Data struct { 17 | Domain string `json:"domain"` 18 | SubDomains struct { 19 | Total string `json:"total"` 20 | Data []string `json:"data"` 21 | } `json:"sub_domains"` 22 | } `json:"data"` 23 | } 24 | 25 | // Source is the passive scraping agent 26 | type Source struct{} 27 | 28 | // Run function returns all subdomains found with the service 29 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 30 | results := make(chan subscraping.Result) 31 | 32 | go func() { 33 | defer close(results) 34 | 35 | if session.Keys.ThreatBook == "" { 36 | return 37 | } 38 | 39 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://api.threatbook.cn/v3/domain/sub_domains?apikey=%s&resource=%s", session.Keys.ThreatBook, domain)) 40 | if err != nil && resp == nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | session.DiscardHTTPResponse(resp) 43 | return 44 | } 45 | 46 | var response threatBookResponse 47 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 48 | if err != nil { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 50 | resp.Body.Close() 51 | return 52 | } 53 | resp.Body.Close() 54 | 55 | if response.ResponseCode != 0 { 56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("code %d, %s", response.ResponseCode, response.VerboseMsg)} 57 | return 58 | } 59 | 60 | total, err := strconv.ParseInt(response.Data.SubDomains.Total, 10, 64) 61 | if err != nil { 62 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 63 | return 64 | } 65 | 66 | if total > 0 { 67 | for _, subdomain := range response.Data.SubDomains.Data { 68 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 69 | } 70 | } 71 | }() 72 | 73 | return results 74 | } 75 | 76 | // Name returns the name of the source 77 | func (s *Source) Name() string { 78 | return "threatbook" 79 | } 80 | -------------------------------------------------------------------------------- /pkg/active/active.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "context" 5 | "github.com/ZhuriLab/Starmap/pkg/resolve" 6 | "github.com/ZhuriLab/Starmap/pkg/util" 7 | "github.com/projectdiscovery/gologger" 8 | ) 9 | 10 | func Enum(domain string, uniqueMap map[string]resolve.HostEntry, silent bool, fileName string, level int, levelDict string, resolvers []string, wildcardIPs map[string]struct{}, maxIPs int) (map[string]resolve.HostEntry, map[string]struct{}) { 11 | gologger.Info().Msgf("Start DNS blasting of %s", domain) 12 | var levelDomains []string 13 | if levelDict != "" { 14 | dl, err := util.LinesInFile(levelDict) 15 | if err != nil { 16 | gologger.Fatal().Msgf("读取domain文件失败:%s,请检查--level-dict参数\n", err.Error()) 17 | } 18 | levelDomains = dl 19 | } else { 20 | levelDomains = GetDefaultSubNextData() 21 | } 22 | 23 | opt := &Options{ 24 | Rate: Band2Rate("2m"), 25 | Domain: domain, 26 | FileName: fileName, 27 | Resolvers: resolvers, 28 | Output: "", 29 | Silent: silent, 30 | WildcardIPs: wildcardIPs, 31 | MaxIPs: maxIPs, 32 | TimeOut: 5, 33 | Retry: 6, 34 | Level: level, // 枚举几级域名，默认为2，二级域名, 35 | LevelDomains: levelDomains, // 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认 36 | Method: "enum", 37 | } 38 | 39 | ctx := context.Background() 40 | 41 | r, err := New(opt) 42 | 43 | if err != nil { 44 | gologger.Fatal().Msgf("%s", err) 45 | } 46 | 47 | enumMap, wildcardIPs := r.RunEnumeration(uniqueMap, ctx) 48 | 49 | r.Close() 50 | 51 | return enumMap, wildcardIPs 52 | } 53 | 54 | func Verify(uniqueMap map[string]resolve.HostEntry, silent bool, resolvers []string, wildcardIPs map[string]struct{}, maxIPs int) (map[string]resolve.HostEntry, map[string]struct{}, []string) { 55 | gologger.Info().Msgf("Start to verify the collected sub domain name results, a total of %d", len(uniqueMap)) 56 | 57 | opt := &Options{ 58 | Rate: Band2Rate("2m"), 59 | Domain: "", 60 | UniqueMap: uniqueMap, 61 | Resolvers: resolvers, 62 | Output: "", 63 | Silent: silent, 64 | WildcardIPs: wildcardIPs, 65 | MaxIPs: maxIPs, 66 | TimeOut: 5, 67 | Retry: 6, 68 | Method: "verify", 69 | } 70 | ctx := context.Background() 71 | 72 | r, err := New(opt) 73 | if err != nil { 74 | gologger.Fatal().Msgf("%s", err) 75 | } 76 | 77 | AuniqueMap, wildcardIPs, unanswers := r.RunEnumerationVerify(ctx) 78 | 79 | r.Close() 80 | 81 | return AuniqueMap, wildcardIPs, unanswers 82 | } 83 | -------------------------------------------------------------------------------- /pkg/active/send.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "github.com/ZhuriLab/Starmap/pkg/active/device" 5 | "github.com/ZhuriLab/Starmap/pkg/active/statusdb" 6 | "github.com/google/gopacket" 7 | "github.com/google/gopacket/layers" 8 | "github.com/google/gopacket/pcap" 9 | "github.com/projectdiscovery/gologger" 10 | "net" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | func (r *runner) sendCycle() { 16 | for domain := range r.sender { 17 | r.limit.Take() 18 | v, ok := r.hm.Get(domain) 19 | if !ok { 20 | v = statusdb.Item{ 21 | Domain: domain, 22 | Dns: r.choseDns(), 23 | Time: time.Now(), 24 | Retry: 0, 25 | DomainLevel: 0, 26 | } 27 | r.hm.Add(domain, v) 28 | } else { 29 | v.Retry += 1 30 | v.Time = time.Now() 31 | v.Dns = r.choseDns() 32 | r.hm.Set(domain, v) 33 | } 34 | send(domain, v.Dns, r.ether, r.dnsid, uint16(r.freeport), r.handle) 35 | atomic.AddUint64(&r.sendIndex, 1) 36 | } 37 | } 38 | func send(domain string, dnsname string, ether *device.EtherTable, dnsid uint16, freeport uint16, handle *pcap.Handle) { 39 | DstIp := net.ParseIP(dnsname).To4() 40 | eth := &layers.Ethernet{ 41 | SrcMAC: ether.SrcMac.HardwareAddr(), 42 | DstMAC: ether.DstMac.HardwareAddr(), 43 | EthernetType: layers.EthernetTypeIPv4, 44 | } 45 | // Our IPv4 header 46 | ip := &layers.IPv4{ 47 | Version: 4, 48 | IHL: 5, 49 | TOS: 0, 50 | Length: 0, // FIX 51 | Id: 0, 52 | Flags: layers.IPv4DontFragment, 53 | FragOffset: 0, 54 | TTL: 255, 55 | Protocol: layers.IPProtocolUDP, 56 | Checksum: 0, 57 | SrcIP: ether.SrcIp, 58 | DstIP: DstIp, 59 | } 60 | // Our UDP header 61 | udp := &layers.UDP{ 62 | SrcPort: layers.UDPPort(freeport), 63 | DstPort: layers.UDPPort(53), 64 | } 65 | // Our DNS header 66 | dns := &layers.DNS{ 67 | ID: dnsid, 68 | QDCount: 1, 69 | RD: true, //递归查询标识 70 | } 71 | dns.Questions = append(dns.Questions, 72 | layers.DNSQuestion{ 73 | Name: []byte(domain), 74 | Type: layers.DNSTypeA, 75 | Class: layers.DNSClassIN, 76 | }) 77 | // Our UDP header 78 | _ = udp.SetNetworkLayerForChecksum(ip) 79 | buf := gopacket.NewSerializeBuffer() 80 | err := gopacket.SerializeLayers( 81 | buf, 82 | gopacket.SerializeOptions{ 83 | ComputeChecksums: true, // automatically compute checksums 84 | FixLengths: true, 85 | }, 86 | eth, ip, udp, dns, 87 | ) 88 | if err != nil { 89 | gologger.Warning().Msgf("SerializeLayers faild:%s\n", err.Error()) 90 | } 91 | err = handle.WritePacketData(buf.Bytes()) 92 | if err != nil { 93 | gologger.Warning().Msgf("WritePacketDate error:%s\n", err.Error()) 94 | } 95 | } -------------------------------------------------------------------------------- /pkg/subscraping/sources/quake/quake.go: -------------------------------------------------------------------------------- 1 | package quake 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | jsoniter "github.com/json-iterator/go" 11 | ) 12 | 13 | type quakeResults struct { 14 | Code int `json:"code"` 15 | Message string `json:"message"` 16 | Data []struct { 17 | Service struct { 18 | HTTP struct { 19 | // Title string `json:"title"` 20 | Host string `json:"host"` 21 | // StatusCode int `json:"status_code"` 22 | // ResponseHeaders string `json:"response_headers"` 23 | } `json:"http"` 24 | } 25 | } 26 | Meta struct { 27 | Pagination struct { 28 | Total int `json:"total"` 29 | } `json:"pagination"` 30 | } `json:"meta"` 31 | } 32 | 33 | // Source is the passive scraping agent 34 | type Source struct{} 35 | 36 | // Run function returns all subdomains found with the service 37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 38 | results := make(chan subscraping.Result) 39 | go func() { 40 | defer close(results) 41 | 42 | if session.Keys.Quake == "" { 43 | return 44 | } 45 | 46 | // quake api doc https://quake.360.cn/quake/#/help 47 | var requestBody = []byte(`{"query":"domain: *.` + domain + `", "start":0, "size":1000}`) 48 | resp, err := session.Post(ctx, "https://quake.360.cn/api/v3/search/quake_service", "", map[string]string{"Content-Type": "application/json", "X-QuakeToken": session.Keys.Quake}, bytes.NewReader(requestBody)) 49 | if err != nil { 50 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 51 | session.DiscardHTTPResponse(resp) 52 | return 53 | } 54 | 55 | var response quakeResults 56 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 57 | if err != nil { 58 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 59 | resp.Body.Close() 60 | return 61 | } 62 | resp.Body.Close() 63 | 64 | if response.Code != 0 { 65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} 66 | return 67 | } 68 | 69 | if response.Meta.Pagination.Total > 0 { 70 | for _, quakeDomain := range response.Data { 71 | subdomain := quakeDomain.Service.HTTP.Host 72 | if strings.ContainsAny(subdomain, "暂无权限") { 73 | subdomain = "" 74 | } 75 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 76 | } 77 | } 78 | }() 79 | 80 | return results 81 | } 82 | 83 | // Name returns the name of the source 84 | func (s *Source) Name() string { 85 | return "quake" 86 | } 87 | -------------------------------------------------------------------------------- /pkg/runner/initialize.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/ZhuriLab/Starmap/pkg/passive" 5 | "github.com/ZhuriLab/Starmap/pkg/resolve" 6 | "github.com/ZhuriLab/Starmap/pkg/util" 7 | "net" 8 | "strings" 9 | ) 10 | 11 | // initializePassiveEngine creates the passive engine and loads sources etc 12 | func (r *Runner) initializePassiveEngine() { 13 | var sources, exclusions []string 14 | 15 | if len(r.options.ExcludeSources) > 0 { 16 | exclusions = r.options.ExcludeSources 17 | } else { 18 | exclusions = append(exclusions, r.options.YAMLConfig.ExcludeSources...) 19 | } 20 | 21 | switch { 22 | // Use all sources if asked by the user 23 | case r.options.All: 24 | sources = append(sources, r.options.YAMLConfig.AllSources...) 25 | // If only recursive sources are wanted, use them only. 26 | case r.options.OnlyRecursive: 27 | sources = append(sources, r.options.YAMLConfig.Recursive...) 28 | // Otherwise, use the CLI/YAML sources 29 | default: 30 | if len(r.options.Sources) == 0 { 31 | sources = append(sources, r.options.YAMLConfig.Sources...) 32 | } else { 33 | sources = r.options.Sources 34 | } 35 | } 36 | r.passiveAgent = passive.New(sources, exclusions) 37 | } 38 | 39 | // initializeActiveEngine creates the resolver used to resolve the found subdomains 40 | func (r *Runner) initializeActiveEngine() error { 41 | var resolvers []string 42 | 43 | // If the file has been provided, read resolvers from the file 44 | if r.options.ResolverList != "" { 45 | var err error 46 | resolvers, err = loadFromFile(r.options.ResolverList) 47 | if err != nil { 48 | return err 49 | } 50 | } 51 | 52 | if len(r.options.Resolvers) > 0 { 53 | resolvers = append(resolvers, r.options.Resolvers...) 54 | } else if r.options.DNS == "in" { 55 | resolvers = append(resolvers, resolve.DefaultResolvers...) 56 | } else if r.options.DNS == "cn" { 57 | resolvers = append(resolvers, resolve.DefaultResolversCN...) 58 | } else if r.options.DNS == "all" { 59 | resolvers = append(resolve.DefaultResolvers, resolve.DefaultResolversCN...) 60 | } else if r.options.DNS == "conf" { 61 | resolvers = append(resolvers, r.options.YAMLConfig.Resolvers...) 62 | } 63 | 64 | resolvers = util.RemoveDuplicateElement(resolvers) 65 | 66 | // Add default 53 UDP port if missing 67 | for i, resolver := range resolvers { 68 | if !strings.Contains(resolver, ":") { 69 | resolvers[i] = net.JoinHostPort(resolver, "53") 70 | } 71 | } 72 | 73 | r.Resolvers = resolvers 74 | 75 | //r.resolverClient = resolve.New() 76 | //var err error 77 | //r.resolverClient.DNSClient, err = dnsx.New(dnsx.Options{BaseResolvers: resolvers, MaxRetries: 5}) 78 | //if err != nil { 79 | // return nil 80 | //} 81 | 82 | return nil 83 | } 84 | -------------------------------------------------------------------------------- /pkg/goflags/normalized_slice.go: -------------------------------------------------------------------------------- 1 | package goflags 2 | 3 | import ( 4 | "strings" 5 | 6 | "github.com/pkg/errors" 7 | ) 8 | 9 | var quotes = []rune{'"', '\'', '`'} 10 | 11 | // NormalizedStringSlice is a slice of strings 12 | type NormalizedStringSlice []string 13 | 14 | func (normalizedStringSlice NormalizedStringSlice) String() string { 15 | return normalizedStringSlice.createStringArrayDefaultValue() 16 | } 17 | 18 | //Set appends a value to the string slice. 19 | func (normalizedStringSlice *NormalizedStringSlice) Set(value string) error { 20 | if slice, err := ToNormalizedStringSlice(value); err != nil { 21 | return err 22 | } else { 23 | *normalizedStringSlice = append(*normalizedStringSlice, slice...) 24 | return nil 25 | } 26 | } 27 | 28 | func (normalizedStringSlice *NormalizedStringSlice) createStringArrayDefaultValue() string { 29 | defaultBuilder := &strings.Builder{} 30 | defaultBuilder.WriteString("[") 31 | for i, k := range *normalizedStringSlice { 32 | defaultBuilder.WriteString("\"") 33 | defaultBuilder.WriteString(k) 34 | defaultBuilder.WriteString("\"") 35 | if i != len(*normalizedStringSlice)-1 { 36 | defaultBuilder.WriteString(", ") 37 | } 38 | } 39 | defaultBuilder.WriteString("]") 40 | return defaultBuilder.String() 41 | } 42 | 43 | func ToNormalizedStringSlice(value string) ([]string, error) { 44 | var result []string 45 | 46 | addPartToResult := func(part string) { 47 | if strings.TrimSpace(part) != "" { 48 | result = append(result, strings.TrimSpace(strings.Trim(strings.TrimSpace(strings.ToLower(part)), string(quotes)))) 49 | } 50 | } 51 | 52 | index := 0 53 | for index < len(value) { 54 | char := rune(value[index]) 55 | if isQuote, quote := isQuote(char); isQuote { 56 | quoteFound, part := searchPart(value[index+1:], quote) 57 | 58 | if !quoteFound { 59 | return nil, errors.New("Unclosed quote in path") 60 | } 61 | 62 | index += len(part) + 2 63 | 64 | addPartToResult(part) 65 | } else { 66 | commaFound, part := searchPart(value[index:], ',') 67 | 68 | if commaFound { 69 | index += len(part) + 1 70 | } else { 71 | index += len(part) 72 | } 73 | 74 | addPartToResult(part) 75 | } 76 | } 77 | 78 | return result, nil 79 | } 80 | 81 | func isQuote(char rune) (bool, rune) { 82 | for _, quote := range quotes { 83 | if quote == char { 84 | return true, quote 85 | } 86 | } 87 | return false, 0 88 | } 89 | 90 | func searchPart(value string, stop rune) (bool, string) { 91 | var result string 92 | for _, char := range value { 93 | if char != stop { 94 | result += string(char) 95 | } else { 96 | return true, result 97 | } 98 | } 99 | return false, result 100 | } 101 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/shodan/shodan.go: -------------------------------------------------------------------------------- 1 | // Package shodan logic 2 | package shodan 3 | 4 | import ( 5 | "context" 6 | "encoding/json" 7 | "fmt" 8 | "github.com/projectdiscovery/gologger" 9 | "io" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 12 | jsoniter "github.com/json-iterator/go" 13 | ) 14 | 15 | // Source is the passive scraping agent 16 | type Source struct{} 17 | 18 | 19 | type respone struct { 20 | Subdomain string `json:"subdomain"` 21 | Type string `json:"type"` 22 | IP string `json:"value"` 23 | Ports []int `json:"ports"` 24 | Tags []string `json:"tags"` 25 | } 26 | 27 | // Run function returns all subdomains found with the service 28 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 29 | results := make(chan subscraping.Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | if session.Keys.Shodan == "" { 35 | return 36 | } 37 | searchURL := fmt.Sprintf("https://api.shodan.io/dns/domain/%s?key=%s", domain, session.Keys.Shodan) 38 | resp, err := session.SimpleGet(ctx, searchURL) 39 | if err != nil { 40 | session.DiscardHTTPResponse(resp) 41 | gologger.Debug().Msg(err.Error()) 42 | return 43 | } 44 | 45 | body, err := io.ReadAll(resp.Body) 46 | defer resp.Body.Close() 47 | 48 | if err != nil { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 50 | return 51 | } 52 | 53 | if jsoniter.Get(body, "error").ToString() != "" { 54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%v", jsoniter.Get(body, "error").ToString())} 55 | return 56 | } 57 | 58 | subdomains := jsoniter.Get(body, "data") 59 | 60 | for i := 0; i < subdomains.Size(); i++ { 61 | 62 | var data respone 63 | json.Unmarshal([]byte(subdomains.Get(i).ToString()), &data) 64 | 65 | var sub string 66 | prefix := data.Subdomain 67 | var ( 68 | ip string 69 | ports []int 70 | ) 71 | 72 | if prefix != "" { 73 | sub = prefix + "." + domain 74 | } else { 75 | sub = domain 76 | } 77 | 78 | if data.Type == "A" { 79 | ip = data.IP 80 | } 81 | 82 | if data.Ports != nil { 83 | ports = data.Ports 84 | } 85 | 86 | ipPorts := make(map[string][]int) 87 | ipPorts[ip] = ports 88 | 89 | results <- subscraping.Result{ 90 | Source: s.Name(), 91 | Type: subscraping.Subdomain, 92 | Value: sub, 93 | IpPorts: ipPorts, 94 | } 95 | } 96 | }() 97 | 98 | return results 99 | } 100 | 101 | // Name returns the name of the source 102 | func (s *Source) Name() string { 103 | return "shodan" 104 | } 105 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/robtex/robtext.go: -------------------------------------------------------------------------------- 1 | // Package robtex logic 2 | package robtex 3 | 4 | import ( 5 | "bufio" 6 | "bytes" 7 | "context" 8 | "fmt" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 12 | ) 13 | 14 | const ( 15 | addrRecord = "A" 16 | iPv6AddrRecord = "AAAA" 17 | baseURL = "https://proapi.robtex.com/pdns" 18 | ) 19 | 20 | // Source is the passive scraping agent 21 | type Source struct{} 22 | 23 | type result struct { 24 | Rrname string `json:"rrname"` 25 | Rrdata string `json:"rrdata"` 26 | Rrtype string `json:"rrtype"` 27 | } 28 | 29 | // Run function returns all subdomains found with the service 30 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 31 | results := make(chan subscraping.Result) 32 | 33 | go func() { 34 | defer close(results) 35 | 36 | if session.Keys.Robtex == "" { 37 | return 38 | } 39 | 40 | headers := map[string]string{"Content-Type": "application/x-ndjson"} 41 | 42 | ips, err := enumerate(ctx, session, fmt.Sprintf("%s/forward/%s?key=%s", baseURL, domain, session.Keys.Robtex), headers) 43 | if err != nil { 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 45 | return 46 | } 47 | 48 | for _, result := range ips { 49 | if result.Rrtype == addrRecord || result.Rrtype == iPv6AddrRecord { 50 | domains, err := enumerate(ctx, session, fmt.Sprintf("%s/reverse/%s?key=%s", baseURL, result.Rrdata, session.Keys.Robtex), headers) 51 | if err != nil { 52 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 53 | return 54 | } 55 | for _, result := range domains { 56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: result.Rrdata} 57 | } 58 | } 59 | } 60 | }() 61 | return results 62 | } 63 | 64 | func enumerate(ctx context.Context, session *subscraping.Session, targetURL string, headers map[string]string) ([]result, error) { 65 | var results []result 66 | 67 | resp, err := session.Get(ctx, targetURL, "", headers) 68 | if err != nil { 69 | session.DiscardHTTPResponse(resp) 70 | return results, err 71 | } 72 | 73 | scanner := bufio.NewScanner(resp.Body) 74 | for scanner.Scan() { 75 | line := scanner.Text() 76 | if line == "" { 77 | continue 78 | } 79 | var response result 80 | err = jsoniter.NewDecoder(bytes.NewBufferString(line)).Decode(&response) 81 | if err != nil { 82 | return results, err 83 | } 84 | 85 | results = append(results, response) 86 | } 87 | 88 | resp.Body.Close() 89 | 90 | return results, nil 91 | } 92 | 93 | // Name returns the name of the source 94 | func (s *Source) Name() string { 95 | return "robtex" 96 | } 97 | -------------------------------------------------------------------------------- /test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "github.com/ZhuriLab/Starmap/pkg/passive" 7 | "github.com/ZhuriLab/Starmap/pkg/resolve" 8 | "github.com/ZhuriLab/Starmap/pkg/runner" 9 | "log" 10 | ) 11 | 12 | /** 13 | @author: yhy 14 | @since: 2022/8/17 15 | @desc: //TODO 16 | **/ 17 | 18 | func Starmap(domain string) (error, *map[string]resolve.HostEntry, []string) { 19 | config, _ := runner.UnmarshalRead("/Users/yhy/.config/Starmap/config.yaml") 20 | 21 | config.Recursive = resolve.DefaultResolvers 22 | config.Sources = passive.DefaultSources 23 | config.AllSources = passive.DefaultAllSources 24 | config.Recursive = passive.DefaultRecursiveSources 25 | 26 | var verify bool 27 | 28 | options := &runner.Options{ 29 | Threads: 10, // Thread controls the number of threads to use for active enumerations 30 | Timeout: 30, // Timeout is the seconds to wait for sources to respond 31 | MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration 32 | Resolvers: resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config 33 | Sources: passive.DefaultSources, // Use the default list of passive sources 34 | AllSources: passive.DefaultAllSources, // Use the default list of all passive sources 35 | Recursive: passive.DefaultRecursiveSources, // Use the default list of recursive sources 36 | 37 | YAMLConfig: config, // 读取自定义配置文件 38 | All: true, 39 | Verbose: verify, 40 | Brute: true, 41 | Verify: true, // 验证找到的域名 42 | RemoveWildcard: true, // 泛解析过滤 43 | MaxIps: 100, // 爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析 44 | DNS: "cn", // dns 服务器区域选择，根据目标选择不同区域得到的结果不同，国内网站的话，选择 cn，dns 爆破结果比较多 45 | BruteWordlist: "", // 爆破子域的域名字典，不填则使用内置的 46 | Level: 2, // 枚举几级域名，默认为二级域名 47 | LevelDic: "", // 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认 48 | Takeover: false, // 子域名接管检测 49 | SAll: false, // 子域名接管检测中请求全部 url，默认只对匹配的 cname 进行检测 50 | } 51 | 52 | runnerInstance, err := runner.NewRunner(options) 53 | 54 | err, uniqueMap, unanswers := runnerInstance.EnumerateSingleDomain(context.Background(), domain, nil) 55 | 56 | if err != nil { 57 | return err, nil, nil 58 | } 59 | 60 | return nil, uniqueMap, unanswers 61 | } 62 | 63 | func main() { 64 | err, subdomains, unanswers := Starmap("moresec.cn") 65 | 66 | if err != nil { // 运行失败，丢给其他机器扫描 67 | log.Fatalln(err) 68 | } 69 | 70 | for _, hostEntry := range *subdomains { 71 | fmt.Printf("%v \n", hostEntry) 72 | //delete(*subdomains, sub) 73 | } 74 | 75 | //*subdomains = nil 76 | 77 | fmt.Println(len(*subdomains)) 78 | fmt.Println(unanswers) 79 | } 80 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/censys/censys.go: -------------------------------------------------------------------------------- 1 | // Package censys logic 2 | package censys 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "strconv" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | jsoniter "github.com/json-iterator/go" 11 | ) 12 | 13 | const maxCensysPages = 10 14 | 15 | type resultsq struct { 16 | Data []string `json:"parsed.extensions.subject_alt_name.dns_names"` 17 | Data1 []string `json:"parsed.names"` 18 | } 19 | 20 | type response struct { 21 | Results []resultsq `json:"results"` 22 | Metadata struct { 23 | Pages int `json:"pages"` 24 | } `json:"metadata"` 25 | } 26 | 27 | // Source is the passive scraping agent 28 | type Source struct{} 29 | 30 | // Run function returns all subdomains found with the service 31 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 32 | results := make(chan subscraping.Result) 33 | 34 | go func() { 35 | defer close(results) 36 | 37 | if session.Keys.CensysToken == "" || session.Keys.CensysSecret == "" { 38 | return 39 | } 40 | 41 | currentPage := 1 42 | for { 43 | var request = []byte(`{"query":"` + domain + `", "page":` + strconv.Itoa(currentPage) + `, "fields":["parsed.names","parsed.extensions.subject_alt_name.dns_names"], "flatten":true}`) 44 | 45 | resp, err := session.HTTPRequest( 46 | ctx, 47 | "POST", 48 | "https://search.censys.io/api/v1/search/certificates", 49 | "", 50 | map[string]string{"Content-Type": "application/json", "Accept": "application/json"}, 51 | bytes.NewReader(request), 52 | subscraping.BasicAuth{Username: session.Keys.CensysToken, Password: session.Keys.CensysSecret}, 53 | ) 54 | 55 | if err != nil { 56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 57 | session.DiscardHTTPResponse(resp) 58 | return 59 | } 60 | 61 | var censysResponse response 62 | err = jsoniter.NewDecoder(resp.Body).Decode(&censysResponse) 63 | if err != nil { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 65 | resp.Body.Close() 66 | return 67 | } 68 | 69 | resp.Body.Close() 70 | 71 | for _, res := range censysResponse.Results { 72 | for _, part := range res.Data { 73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} 74 | } 75 | for _, part := range res.Data1 { 76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: part} 77 | } 78 | } 79 | 80 | // Exit the censys enumeration if max pages is reached 81 | if currentPage >= censysResponse.Metadata.Pages || currentPage >= maxCensysPages { 82 | break 83 | } 84 | 85 | currentPage++ 86 | } 87 | }() 88 | 89 | return results 90 | } 91 | 92 | // Name returns the name of the source 93 | func (s *Source) Name() string { 94 | return "censys" 95 | } 96 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/bufferover/bufferover.go: -------------------------------------------------------------------------------- 1 | // Package bufferover is a bufferover Scraping Engine in Golang 2 | package bufferover 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "strings" 8 | 9 | jsoniter "github.com/json-iterator/go" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 12 | ) 13 | 14 | type response struct { 15 | Meta struct { 16 | Errors []string `json:"Errors"` 17 | } `json:"Meta"` 18 | FDNSA []string `json:"FDNS_A"` 19 | RDNS []string `json:"RDNS"` 20 | Results []string `json:"Results"` 21 | } 22 | 23 | // Source is the passive scraping agent 24 | type Source struct{} 25 | 26 | // Run function returns all subdomains found with the service 27 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 28 | results := make(chan subscraping.Result) 29 | 30 | go func() { 31 | defer close(results) 32 | 33 | // DNS data does not rely on an API key. 34 | s.getData(ctx, fmt.Sprintf("https://dns.bufferover.run/dns?q=.%s", domain), "", session, results) 35 | 36 | if session.Keys.Bufferover == "" { 37 | return 38 | } 39 | 40 | // TLS data relies on an API key. 41 | s.getData(ctx, fmt.Sprintf("https://tls.bufferover.run/dns?q=.%s", domain), session.Keys.Bufferover, session, results) 42 | }() 43 | 44 | return results 45 | } 46 | 47 | func (s *Source) getData(ctx context.Context, sourceURL string, apiKey string, session *subscraping.Session, results chan subscraping.Result) { 48 | resp, err := session.Get(ctx, sourceURL, "", map[string]string{"x-api-key": apiKey}) 49 | 50 | if err != nil && resp == nil { 51 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 52 | session.DiscardHTTPResponse(resp) 53 | return 54 | } 55 | 56 | var bufforesponse response 57 | err = jsoniter.NewDecoder(resp.Body).Decode(&bufforesponse) 58 | if err != nil { 59 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 60 | resp.Body.Close() 61 | return 62 | } 63 | 64 | resp.Body.Close() 65 | 66 | metaErrors := bufforesponse.Meta.Errors 67 | 68 | if len(metaErrors) > 0 { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", strings.Join(metaErrors, ", "))} 70 | return 71 | } 72 | 73 | var subdomains []string 74 | 75 | if len(bufforesponse.FDNSA) > 0 { 76 | subdomains = bufforesponse.FDNSA 77 | subdomains = append(subdomains, bufforesponse.RDNS...) 78 | } else if len(bufforesponse.Results) > 0 { 79 | subdomains = bufforesponse.Results 80 | } 81 | 82 | for _, subdomain := range subdomains { 83 | for _, value := range session.Extractor.FindAllString(subdomain, -1) { 84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: value} 85 | } 86 | } 87 | } 88 | 89 | // Name returns the name of the source 90 | func (s *Source) Name() string { 91 | return "bufferover" 92 | } 93 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/certspotter/certspotter.go: -------------------------------------------------------------------------------- 1 | // Package certspotter logic 2 | package certspotter 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | 8 | jsoniter "github.com/json-iterator/go" 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | ) 11 | 12 | type certspotterObject struct { 13 | ID string `json:"id"` 14 | DNSNames []string `json:"dns_names"` 15 | } 16 | 17 | // Source is the passive scraping agent 18 | type Source struct{} 19 | 20 | // Run function returns all subdomains found with the service 21 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 22 | results := make(chan subscraping.Result) 23 | 24 | go func() { 25 | defer close(results) 26 | 27 | if session.Keys.Certspotter == "" { 28 | return 29 | } 30 | 31 | resp, err := session.Get(ctx, fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names", domain), "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) 32 | if err != nil { 33 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 34 | session.DiscardHTTPResponse(resp) 35 | return 36 | } 37 | 38 | var response []certspotterObject 39 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 40 | if err != nil { 41 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 42 | resp.Body.Close() 43 | return 44 | } 45 | resp.Body.Close() 46 | 47 | for _, cert := range response { 48 | for _, subdomain := range cert.DNSNames { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 50 | } 51 | } 52 | 53 | // if the number of responses is zero, close the channel and return. 54 | if len(response) == 0 { 55 | return 56 | } 57 | 58 | id := response[len(response)-1].ID 59 | for { 60 | reqURL := fmt.Sprintf("https://api.certspotter.com/v1/issuances?domain=%s&include_subdomains=true&expand=dns_names&after=%s", domain, id) 61 | 62 | resp, err := session.Get(ctx, reqURL, "", map[string]string{"Authorization": "Bearer " + session.Keys.Certspotter}) 63 | if err != nil { 64 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 65 | return 66 | } 67 | 68 | var response []certspotterObject 69 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 70 | if err != nil { 71 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 72 | resp.Body.Close() 73 | return 74 | } 75 | resp.Body.Close() 76 | 77 | if len(response) == 0 { 78 | break 79 | } 80 | 81 | for _, cert := range response { 82 | for _, subdomain := range cert.DNSNames { 83 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 84 | } 85 | } 86 | 87 | id = response[len(response)-1].ID 88 | } 89 | }() 90 | 91 | return results 92 | } 93 | 94 | // Name returns the name of the source 95 | func (s *Source) Name() string { 96 | return "certspotter" 97 | } 98 | -------------------------------------------------------------------------------- /pkg/util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "bufio" 5 | "github.com/ZhuriLab/Starmap/pkg/resolve" 6 | "math/rand" 7 | "net" 8 | "os" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | func RandomStr(n int) string { 14 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyz1234567890") 15 | rand.Seed(time.Now().UnixNano()) 16 | b := make([]rune, n) 17 | for i := range b { 18 | b[i] = letterRunes[rand.Intn(len(letterRunes))] 19 | } 20 | return string(b) 21 | } 22 | 23 | // LinesInFile 读取文件 返回每行的数组 24 | func LinesInFile(fileName string) ([]string, error) { 25 | result := []string{} 26 | f, err := os.Open(fileName) 27 | if err != nil { 28 | return result, err 29 | } 30 | defer f.Close() 31 | scanner := bufio.NewScanner(f) 32 | for scanner.Scan() { 33 | line := scanner.Text() 34 | if line != "" { 35 | result = append(result, line) 36 | } 37 | } 38 | return result, nil 39 | } 40 | 41 | func MergeMap(map1, map2 map[string]resolve.HostEntry) map[string]resolve.HostEntry { 42 | map3 := make(map[string]resolve.HostEntry) 43 | 44 | for i, v := range map1 { 45 | for j, w := range map2 { 46 | if i == j { 47 | map3[i] = w 48 | 49 | } else { 50 | if _, ok := map3[i]; !ok { 51 | map3[i] = v 52 | } 53 | if _, ok := map3[j]; !ok { 54 | map3[j] = w 55 | } 56 | } 57 | } 58 | } 59 | return map3 60 | } 61 | 62 | func MergeIpPortMap(map1, map2 map[string][]int) map[string][]int { 63 | map3 := make(map[string][]int) 64 | 65 | for i, v := range map1 { 66 | for j, w := range map2 { 67 | if i == j { 68 | map3[i] = w 69 | 70 | } else { 71 | if _, ok := map3[i]; !ok { 72 | map3[i] = v 73 | } 74 | if _, ok := map3[j]; !ok { 75 | map3[j] = w 76 | } 77 | } 78 | } 79 | } 80 | return map3 81 | } 82 | 83 | // RemoveDuplicateElement 数组去重 84 | func RemoveDuplicateElement(strs []string) []string { 85 | result := make([]string, 0, len(strs)) 86 | temp := map[string]struct{}{} 87 | for _, item := range strs { 88 | if item != "" { 89 | if _, ok := temp[item]; !ok { 90 | temp[item] = struct{}{} 91 | result = append(result, item) 92 | } 93 | } 94 | 95 | } 96 | return result 97 | } 98 | 99 | // In 判断一个字符串是否在另一个字符数组里面，存在返回true 100 | func In(target string, strs []string) bool { 101 | target = strings.TrimSpace(target) 102 | for _, element := range strs { 103 | if strings.Contains(target, element) { 104 | return true 105 | } 106 | } 107 | return false 108 | } 109 | 110 | // InInt 判断一个字符串是否在另一个字符数组里面，存在返回true 111 | func InInt(target int, strs []int) bool { 112 | for _, element := range strs { 113 | if target == element { 114 | return true 115 | } 116 | } 117 | return false 118 | } 119 | 120 | // IsInnerIP 判断是否为内网IP 121 | func IsInnerIP(ip string) bool { 122 | IP := net.ParseIP(ip) 123 | if ip4 := IP.To4(); ip4 != nil { 124 | if IP.IsLoopback() || IP.IsLinkLocalMulticast() || IP.IsLinkLocalUnicast() { 125 | return true 126 | } 127 | return ip4.IsPrivate() 128 | } 129 | 130 | return false 131 | } 132 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/ZhuriLab/Starmap 2 | 3 | go 1.17 4 | 5 | require ( 6 | github.com/caffix/resolve v0.5.4 7 | github.com/cnf/structhash v0.0.0-20201127153200-e1b16c1ebc08 8 | github.com/corpix/uarand v0.1.1 9 | github.com/google/gopacket v1.1.19 10 | github.com/hako/durafmt v0.0.0-20210608085754-5c1018a4e16b 11 | github.com/json-iterator/go v1.1.12 12 | github.com/lib/pq v1.10.4 13 | github.com/logrusorgru/aurora v2.0.3+incompatible 14 | github.com/miekg/dns v1.1.46 15 | github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 16 | github.com/pkg/errors v0.9.1 17 | github.com/projectdiscovery/chaos-client v0.2.0 18 | github.com/projectdiscovery/dnsx v1.0.7 19 | github.com/projectdiscovery/fileutil v0.0.0-20210926202739-6050d0acf73c 20 | github.com/projectdiscovery/gologger v1.1.4 21 | github.com/projectdiscovery/stringsutil v0.0.0-20210830151154-f567170afdd9 22 | github.com/rs/xid v1.3.0 23 | github.com/spyse-com/go-spyse v1.2.4 24 | github.com/tomnomnom/linkheader v0.0.0-20180905144013-02ca5825eb80 25 | github.com/valyala/fasthttp v1.34.0 26 | go.uber.org/ratelimit v0.2.0 27 | gopkg.in/yaml.v2 v2.4.0 28 | gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b 29 | ) 30 | 31 | require ( 32 | github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect 33 | github.com/andres-erbsen/clock v0.0.0-20160526145045-9e14626cd129 // indirect 34 | github.com/andybalholm/brotli v1.0.4 // indirect 35 | github.com/caffix/queue v0.1.3 // indirect 36 | github.com/caffix/stringset v0.1.0 // indirect 37 | github.com/cespare/xxhash/v2 v2.1.2 // indirect 38 | github.com/dgraph-io/badger v1.6.2 // indirect 39 | github.com/dgraph-io/ristretto v0.1.0 // indirect 40 | github.com/dustin/go-humanize v1.0.0 // indirect 41 | github.com/golang/glog v1.0.0 // indirect 42 | github.com/golang/protobuf v1.5.2 // indirect 43 | github.com/karrick/godirwalk v1.16.1 // indirect 44 | github.com/klauspost/compress v1.15.0 // indirect 45 | github.com/mitchellh/mapstructure v1.4.1 // indirect 46 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 47 | github.com/modern-go/reflect2 v1.0.2 // indirect 48 | github.com/projectdiscovery/blackrock v0.0.0-20210415162320-b38689ae3a2e // indirect 49 | github.com/projectdiscovery/httputil v0.0.0-20210906072657-f3a099cb20bc // indirect 50 | github.com/projectdiscovery/iputil v0.0.0-20210804143329-3a30fcde43f3 // indirect 51 | github.com/projectdiscovery/mapcidr v0.0.8 // indirect 52 | github.com/projectdiscovery/retryabledns v1.0.13-0.20210927160332-db15799e2e4d // indirect 53 | github.com/projectdiscovery/retryablehttp-go v1.0.2 // indirect 54 | github.com/valyala/bytebufferpool v1.0.0 // indirect 55 | golang.org/x/mod v0.5.1 // indirect 56 | golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect 57 | golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5 // indirect 58 | golang.org/x/text v0.3.7 // indirect 59 | golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect 60 | golang.org/x/tools v0.1.9 // indirect 61 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 62 | google.golang.org/protobuf v1.27.1 // indirect 63 | ) 64 | -------------------------------------------------------------------------------- /pkg/subscraping/types.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "regexp" 7 | 8 | "go.uber.org/ratelimit" 9 | ) 10 | 11 | // BasicAuth request's Authorization header 12 | type BasicAuth struct { 13 | Username string 14 | Password string 15 | } 16 | 17 | // Source is an interface inherited by each passive source 18 | type Source interface { 19 | // Run takes a domain as argument and a session object 20 | // which contains the extractor for subdomains, http client 21 | // and other stuff. 22 | Run(context.Context, string, *Session) <-chan Result 23 | // Name returns the name of the source 24 | Name() string 25 | } 26 | 27 | // Session is the option passed to the source, an option is created 28 | // uniquely for each source. 29 | type Session struct { 30 | // Extractor is the regex for subdomains created for each domain 31 | Extractor *regexp.Regexp 32 | // Keys is the API keys for the application 33 | Keys *Keys 34 | // Client is the current http client 35 | Client *http.Client 36 | // Rate limit instance 37 | RateLimiter ratelimit.Limiter 38 | } 39 | 40 | // Keys contains the current API Keys we have in store 41 | type Keys struct { 42 | Binaryedge string `json:"binaryedge"` 43 | Bufferover string `json:"bufferover"` 44 | C99 string `json:"c99"` 45 | CensysToken string `json:"censysUsername"` 46 | CensysSecret string `json:"censysPassword"` 47 | Certspotter string `json:"certspotter"` 48 | Chaos string `json:"chaos"` 49 | Chinaz string `json:"chinaz"` 50 | DNSDB string `json:"dnsdb"` 51 | GitHub []string `json:"github"` 52 | IntelXHost string `json:"intelXHost"` 53 | IntelXKey string `json:"intelXKey"` 54 | PassiveTotalUsername string `json:"passivetotal_username"` 55 | PassiveTotalPassword string `json:"passivetotal_password"` 56 | Robtex string `json:"robtex"` 57 | Securitytrails string `json:"securitytrails"` 58 | Shodan string `json:"shodan"` 59 | Spyse string `json:"spyse"` 60 | ThreatBook string `json:"threatbook"` 61 | URLScan string `json:"urlscan"` 62 | Virustotal string `json:"virustotal"` 63 | ZoomEyeUsername string `json:"zoomeye_username"` 64 | ZoomEyePassword string `json:"zoomeye_password"` 65 | ZoomEyeKey string `json:"zoomeye_key"` 66 | FofaUsername string `json:"fofa_username"` 67 | FofaSecret string `json:"fofa_secret"` 68 | FullHunt string `json:"fullhunt"` 69 | Quake string `json:"quake"` 70 | Hunter string `json:"hunter"` 71 | } 72 | 73 | // Result is a result structure returned by a source 74 | type Result struct { 75 | Type ResultType 76 | Source string 77 | Value string 78 | Error error 79 | IpPorts map[string][]int 80 | } 81 | 82 | // ResultType is the type of result returned by the source 83 | type ResultType int 84 | 85 | // Types of results returned by the source 86 | const ( 87 | Subdomain ResultType = iota 88 | Error 89 | ) 90 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/dnsdumpster/dnsdumpster.go: -------------------------------------------------------------------------------- 1 | // Package dnsdumpster logic 2 | package dnsdumpster 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "io" 8 | "net/url" 9 | "regexp" 10 | "strings" 11 | 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | ) 14 | 15 | // CSRFSubMatchLength CSRF regex submatch length 16 | const CSRFSubMatchLength = 2 17 | 18 | var re = regexp.MustCompile("") 19 | 20 | // getCSRFToken gets the CSRF Token from the page 21 | func getCSRFToken(page string) string { 22 | if subs := re.FindStringSubmatch(page); len(subs) == CSRFSubMatchLength { 23 | return strings.TrimSpace(subs[1]) 24 | } 25 | return "" 26 | } 27 | 28 | // postForm posts a form for a domain and returns the response 29 | func postForm(ctx context.Context, session *subscraping.Session, token, domain string) (string, error) { 30 | params := url.Values{ 31 | "csrfmiddlewaretoken": {token}, 32 | "targetip": {domain}, 33 | "user": {"free"}, 34 | } 35 | 36 | resp, err := session.HTTPRequest( 37 | ctx, 38 | "POST", 39 | "https://dnsdumpster.com/", 40 | fmt.Sprintf("csrftoken=%s; Domain=dnsdumpster.com", token), 41 | map[string]string{ 42 | "Content-Type": "application/x-www-form-urlencoded", 43 | "Referer": "https://dnsdumpster.com", 44 | "X-CSRF-Token": token, 45 | }, 46 | strings.NewReader(params.Encode()), 47 | subscraping.BasicAuth{}, 48 | ) 49 | 50 | if err != nil { 51 | session.DiscardHTTPResponse(resp) 52 | return "", err 53 | } 54 | 55 | // Now, grab the entire page 56 | in, err := io.ReadAll(resp.Body) 57 | resp.Body.Close() 58 | return string(in), err 59 | } 60 | 61 | // Source is the passive scraping agent 62 | type Source struct{} 63 | 64 | // Run function returns all subdomains found with the service 65 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 66 | results := make(chan subscraping.Result) 67 | 68 | go func() { 69 | defer close(results) 70 | 71 | resp, err := session.SimpleGet(ctx, "https://dnsdumpster.com/") 72 | if err != nil { 73 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 74 | session.DiscardHTTPResponse(resp) 75 | return 76 | } 77 | 78 | body, err := io.ReadAll(resp.Body) 79 | if err != nil { 80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 81 | resp.Body.Close() 82 | return 83 | } 84 | resp.Body.Close() 85 | 86 | csrfToken := getCSRFToken(string(body)) 87 | data, err := postForm(ctx, session, csrfToken, domain) 88 | if err != nil { 89 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 90 | return 91 | } 92 | 93 | for _, subdomain := range session.Extractor.FindAllString(data, -1) { 94 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 95 | } 96 | }() 97 | 98 | return results 99 | } 100 | 101 | // Name returns the name of the source 102 | func (s *Source) Name() string { 103 | return "dnsdumpster" 104 | } 105 | -------------------------------------------------------------------------------- /pkg/runner/banner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "github.com/ZhuriLab/Starmap/pkg/passive" 5 | "github.com/ZhuriLab/Starmap/pkg/resolve" 6 | "github.com/logrusorgru/aurora" 7 | "github.com/projectdiscovery/gologger" 8 | ) 9 | 10 | const banner = ` 11 | ███████████████╗█████╗██████╗███╗ ███╗█████╗██████╗ 12 | ██╔════╚══██╔══██╔══████╔══██████╗ ██████╔══████╔══██╗ 13 | ███████╗ ██║ █████████████╔██╔████╔███████████████╔╝ 14 | ╚════██║ ██║ ██╔══████╔══████║╚██╔╝████╔══████╔═══╝ 15 | ███████║ ██║ ██║ ████║ ████║ ╚═╝ ████║ ████║ 16 | ╚══════╝ ╚═╝ ╚═╝ ╚═╚═╝ ╚═╚═╝ ╚═╚═╝ ╚═╚═╝ 17 | ` 18 | 19 | // Version is the current version of Starmap 20 | const Version = `v0.2.0` 21 | 22 | // showBanner is used to show the banner to the user 23 | func showBanner() { 24 | gologger.Print().Msgf("%s\n", aurora.Blue(banner)) 25 | gologger.Print().Msgf("\t\t\t\t%s\n", aurora.Red(Version)) 26 | gologger.Print().Msgf("\t\t\t%s\n\n", aurora.Green("https://github.com/ZhuriLab/Starmap")) 27 | 28 | gologger.Print().Msgf(aurora.Red("Use with caution. You are responsible for your actions").String()) 29 | gologger.Print().Msgf(aurora.Red("Developers assume no liability and are not responsible for any misuse or damage.").String()) 30 | gologger.Print().Msgf(aurora.Red("By using Starmap, you also agree to the terms of the APIs used.\n").String()) 31 | } 32 | 33 | // normalRunTasks runs the normal startup tasks 34 | func (options *Options) normalRunTasks() { 35 | configFile, err := UnmarshalRead(options.Config) 36 | if err != nil { 37 | gologger.Fatal().Msgf("Could not read configuration file %s: %s\n", options.Config, err) 38 | } 39 | 40 | // If we have a different version of subfinder installed 41 | // previously, use the new iteration of config file. 42 | 43 | if configFile.Version != Version { 44 | configFile.Sources = passive.DefaultSources 45 | configFile.AllSources = passive.DefaultAllSources 46 | configFile.Recursive = passive.DefaultRecursiveSources 47 | configFile.Version = Version 48 | 49 | err = configFile.MarshalWrite(options.Config) 50 | if err != nil { 51 | gologger.Fatal().Msgf("Could not update configuration file to %s: %s\n", options.Config, err) 52 | } 53 | } 54 | options.YAMLConfig = configFile 55 | } 56 | 57 | // firstRunTasks runs some housekeeping tasks done 58 | // when the program is ran for the first time 59 | func (options *Options) firstRunTasks() { 60 | // Create the configuration file and display information 61 | // about it to the user. 62 | config := Providers{ 63 | // Use the default list of resolvers by marshaling it to the config 64 | Resolvers: resolve.DefaultResolvers, 65 | // Use the default list of passive sources 66 | Sources: passive.DefaultSources, 67 | // Use the default list of all passive sources 68 | AllSources: passive.DefaultAllSources, 69 | // Use the default list of recursive sources 70 | Recursive: passive.DefaultRecursiveSources, 71 | Version: Version, 72 | } 73 | 74 | err := config.MarshalWrite(options.Config) 75 | 76 | if err != nil { 77 | gologger.Fatal().Msgf("Could not write configuration file to %s: %s\n", options.Config, err) 78 | } 79 | options.YAMLConfig = config 80 | 81 | gologger.Info().Msgf("Configuration file saved to %s\n", options.Config) 82 | } 83 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/commoncrawl/commoncrawl.go: -------------------------------------------------------------------------------- 1 | // Package commoncrawl logic 2 | package commoncrawl 3 | 4 | import ( 5 | "bufio" 6 | "context" 7 | "fmt" 8 | "net/url" 9 | "strings" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | ) 14 | 15 | const indexURL = "https://index.commoncrawl.org/collinfo.json" 16 | 17 | type indexResponse struct { 18 | ID string `json:"id"` 19 | APIURL string `json:"cdx-api"` 20 | } 21 | 22 | // Source is the passive scraping agent 23 | type Source struct{} 24 | 25 | var years = [...]string{"2020", "2019", "2018", "2017"} 26 | 27 | // Run function returns all subdomains found with the service 28 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 29 | results := make(chan subscraping.Result) 30 | 31 | go func() { 32 | defer close(results) 33 | 34 | resp, err := session.SimpleGet(ctx, indexURL) 35 | if err != nil { 36 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 37 | session.DiscardHTTPResponse(resp) 38 | return 39 | } 40 | 41 | var indexes []indexResponse 42 | err = jsoniter.NewDecoder(resp.Body).Decode(&indexes) 43 | if err != nil { 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 45 | resp.Body.Close() 46 | return 47 | } 48 | resp.Body.Close() 49 | 50 | searchIndexes := make(map[string]string) 51 | for _, year := range years { 52 | for _, index := range indexes { 53 | if strings.Contains(index.ID, year) { 54 | if _, ok := searchIndexes[year]; !ok { 55 | searchIndexes[year] = index.APIURL 56 | break 57 | } 58 | } 59 | } 60 | } 61 | 62 | for _, apiURL := range searchIndexes { 63 | further := s.getSubdomains(ctx, apiURL, domain, session, results) 64 | if !further { 65 | break 66 | } 67 | } 68 | }() 69 | 70 | return results 71 | } 72 | 73 | // Name returns the name of the source 74 | func (s *Source) Name() string { 75 | return "commoncrawl" 76 | } 77 | 78 | func (s *Source) getSubdomains(ctx context.Context, searchURL, domain string, session *subscraping.Session, results chan subscraping.Result) bool { 79 | for { 80 | select { 81 | case <-ctx.Done(): 82 | return false 83 | default: 84 | var headers = map[string]string{"Host": "index.commoncrawl.org"} 85 | resp, err := session.Get(ctx, fmt.Sprintf("%s?url=*.%s", searchURL, domain), "", headers) 86 | if err != nil { 87 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 88 | session.DiscardHTTPResponse(resp) 89 | return false 90 | } 91 | 92 | scanner := bufio.NewScanner(resp.Body) 93 | for scanner.Scan() { 94 | line := scanner.Text() 95 | if line == "" { 96 | continue 97 | } 98 | line, _ = url.QueryUnescape(line) 99 | subdomain := session.Extractor.FindString(line) 100 | if subdomain != "" { 101 | // fix for triple encoded URL 102 | subdomain = strings.ToLower(subdomain) 103 | subdomain = strings.TrimPrefix(subdomain, "25") 104 | subdomain = strings.TrimPrefix(subdomain, "2f") 105 | 106 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 107 | } 108 | } 109 | resp.Body.Close() 110 | return true 111 | } 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/fofa/fofa.go: -------------------------------------------------------------------------------- 1 | // Package fofa logic 2 | package fofa 3 | 4 | import ( 5 | "context" 6 | "encoding/base64" 7 | "fmt" 8 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 9 | "github.com/ZhuriLab/Starmap/pkg/util" 10 | jsoniter "github.com/json-iterator/go" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | type fofaResponse struct { 16 | Error bool `json:"error"` 17 | ErrMsg string `json:"errmsg"` 18 | Size int `json:"size"` 19 | Results []interface{} `json:"results"` 20 | } 21 | 22 | // Source is the passive scraping agent 23 | type Source struct{} 24 | 25 | // Run function returns all subdomains found with the service 26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 27 | results := make(chan subscraping.Result) 28 | 29 | go func() { 30 | defer close(results) 31 | 32 | if session.Keys.FofaUsername == "" || session.Keys.FofaSecret == "" { 33 | return 34 | } 35 | 36 | // fofa api doc https://fofa.info/static_pages/api_help 37 | qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) 38 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://fofa.info/api/v1/search/all?full=true&fields=host,ip,port&page=1&size=10000&email=%s&key=%s&qbase64=%s", session.Keys.FofaUsername, session.Keys.FofaSecret, qbase64)) 39 | if err != nil && resp == nil { 40 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 41 | session.DiscardHTTPResponse(resp) 42 | return 43 | } 44 | 45 | var response fofaResponse 46 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 47 | if err != nil { 48 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 49 | resp.Body.Close() 50 | return 51 | } 52 | resp.Body.Close() 53 | 54 | if response.Error { 55 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.ErrMsg)} 56 | return 57 | } 58 | 59 | 60 | if response.Size > 0 { 61 | subdomains := make(map[string][]string) 62 | 63 | ipPortsTmp := make(map[string][]int) 64 | for _, result := range response.Results { 65 | 66 | tmp := fmt.Sprintf("%v", result) 67 | tmp = strings.ReplaceAll(tmp, "[", "") 68 | tmp = strings.ReplaceAll(tmp, "]", "") 69 | 70 | res := strings.Split(tmp, " ") 71 | 72 | subdomain := res[0] 73 | if strings.HasPrefix(strings.ToLower(subdomain), "http://") || strings.HasPrefix(strings.ToLower(subdomain), "https://") { 74 | subdomain = subdomain[strings.Index(subdomain, "//")+2:] 75 | } 76 | port, _ := strconv.Atoi(res[2]) 77 | 78 | if !util.InInt(port, ipPortsTmp[res[1]]) { 79 | ipPortsTmp[res[1]] = append(ipPortsTmp[res[1]], port) 80 | } 81 | 82 | subdomains[subdomain] = append(subdomains[subdomain], res[1]) 83 | } 84 | 85 | for subdomain, ips := range subdomains { 86 | ipPorts := make(map[string][]int) 87 | for _, ip := range ips { 88 | ipPorts[ip] = ipPortsTmp[ip] 89 | } 90 | 91 | results <- subscraping.Result{ 92 | Source: s.Name(), 93 | Type: subscraping.Subdomain, 94 | Value: subdomain, 95 | IpPorts: ipPorts, 96 | } 97 | } 98 | } 99 | }() 100 | 101 | return results 102 | } 103 | 104 | // Name returns the name of the source 105 | func (s *Source) Name() string { 106 | return "fofa" 107 | } 108 | -------------------------------------------------------------------------------- /pkg/active/recv.go: -------------------------------------------------------------------------------- 1 | package active 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "github.com/ZhuriLab/Starmap/pkg/util" 8 | "github.com/google/gopacket" 9 | "github.com/google/gopacket/layers" 10 | "github.com/google/gopacket/pcap" 11 | "sync/atomic" 12 | "time" 13 | ) 14 | 15 | func (r *runner) recvChanel(ctx context.Context, tag bool) error { 16 | defer close(r.recver) 17 | var ( 18 | snapshotLen = 65536 19 | timeout = -1 * time.Second 20 | err error 21 | ) 22 | inactive, err := pcap.NewInactiveHandle(r.ether.Device) 23 | if err != nil { 24 | return errors.New(fmt.Sprintf("pcap.NewInactiveHandle:%s", err.Error())) 25 | } 26 | err = inactive.SetSnapLen(snapshotLen) 27 | if err != nil { 28 | return errors.New(fmt.Sprintf("inactive.SetSnapLen:%s", err.Error())) 29 | } 30 | defer inactive.CleanUp() 31 | if err = inactive.SetTimeout(timeout); err != nil { 32 | return errors.New(fmt.Sprintf("inactive.SetTimeout:%s", err.Error())) 33 | } 34 | err = inactive.SetImmediateMode(true) 35 | if err != nil { 36 | return err 37 | } 38 | handle, err := inactive.Activate() 39 | if err != nil { 40 | return errors.New(fmt.Sprintf("inactive.Activate():%s", err.Error())) 41 | } 42 | defer handle.Close() 43 | 44 | err = handle.SetBPFFilter(fmt.Sprintf("udp and src port 53 and dst port %d", r.freeport)) 45 | if err != nil { 46 | return errors.New(fmt.Sprintf("SetBPFFilter Faild:%s", err.Error())) 47 | } 48 | 49 | // Listening 50 | 51 | var udp layers.UDP 52 | var dns layers.DNS 53 | var eth layers.Ethernet 54 | var ipv4 layers.IPv4 55 | var ipv6 layers.IPv6 56 | 57 | parser := gopacket.NewDecodingLayerParser( 58 | layers.LayerTypeEthernet, ð, &ipv4, &ipv6, &udp, &dns) 59 | 60 | var data []byte 61 | var decoded []gopacket.LayerType 62 | for { 63 | select { 64 | case <-ctx.Done(): 65 | return nil 66 | default: 67 | data, _, err = handle.ReadPacketData() 68 | if err != nil { 69 | continue 70 | } 71 | err = parser.DecodeLayers(data, &decoded) 72 | if err != nil { 73 | continue 74 | } 75 | if !dns.QR { 76 | continue 77 | } 78 | if dns.ID != r.dnsid { 79 | continue 80 | } 81 | atomic.AddUint64(&r.recvIndex, 1) 82 | if len(dns.Questions) == 0 { 83 | continue 84 | } 85 | domain := string(dns.Questions[0].Name) 86 | 87 | r.hm.Del(domain) 88 | 89 | if dns.ANCount > 0 { 90 | var ips []string 91 | for _, answers := range dns.Answers { 92 | if answers.Class == layers.DNSClassIN { 93 | if answers.IP != nil { 94 | ips = append(ips, answers.IP.String()) 95 | if util.IsInnerIP(answers.IP.String()) { 96 | r.unanswers = append(r.unanswers, domain) 97 | } 98 | } 99 | } 100 | } 101 | 102 | if len(ips) == 0 { 103 | continue 104 | } 105 | 106 | var skip bool 107 | for _, ip := range ips { 108 | // Ignore the host if it exists in wildcard ips map 109 | if _, ok := r.options.WildcardIPs[ip]; ok { 110 | skip = true 111 | break 112 | } 113 | } 114 | 115 | // 不是泛解析出的 ip 的记录 116 | if !skip { 117 | atomic.AddUint64(&r.successIndex, 1) 118 | r.recver <- RecvResult{ 119 | Subdomain: domain, 120 | Answers: dns.Answers, 121 | ResponseCode: dns.ResponseCode, 122 | } 123 | } 124 | } else if tag { 125 | r.unanswers = append(r.unanswers, domain) 126 | } 127 | 128 | } 129 | } 130 | } 131 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/spyse/spyse.go: -------------------------------------------------------------------------------- 1 | // Package spyse logic 2 | package spyse 3 | 4 | import ( 5 | "context" 6 | 7 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 8 | spyse "github.com/spyse-com/go-spyse/pkg" 9 | ) 10 | 11 | const searchMethodResultsLimit = 10000 12 | 13 | // Source is the passive scraping agent 14 | type Source struct{} 15 | 16 | // Run function returns all subdomains found with the service 17 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 18 | results := make(chan subscraping.Result) 19 | 20 | go func() { 21 | defer close(results) 22 | 23 | if session.Keys.Spyse == "" { 24 | return 25 | } 26 | 27 | client, err := spyse.NewClient(session.Keys.Spyse, nil) 28 | if err != nil { 29 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 30 | return 31 | } 32 | 33 | domainSvc := spyse.NewDomainService(client) 34 | 35 | var searchDomain = "." + domain 36 | var subdomainsSearchParams spyse.QueryBuilder 37 | 38 | subdomainsSearchParams.AppendParam(spyse.QueryParam{ 39 | Name: domainSvc.Params().Name.Name, 40 | Operator: domainSvc.Params().Name.Operator.EndsWith, 41 | Value: searchDomain, 42 | }) 43 | 44 | totalResults, err := domainSvc.SearchCount(ctx, subdomainsSearchParams.Query) 45 | if err != nil { 46 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 47 | return 48 | } 49 | 50 | if totalResults == 0 { 51 | return 52 | } 53 | 54 | // The default "Search" method returns only first 10 000 subdomains 55 | // To obtain more than 10 000 subdomains the "Scroll" method should be using 56 | // Note: The "Scroll" method is only available for "PRO" customers, so we need to check 57 | // quota.IsScrollSearchEnabled param 58 | if totalResults > searchMethodResultsLimit && client.Account().IsScrollSearchEnabled { 59 | var scrollID string 60 | var scrollResults *spyse.DomainScrollResponse 61 | 62 | for { 63 | scrollResults, err = domainSvc.ScrollSearch(ctx, subdomainsSearchParams.Query, scrollID) 64 | if err != nil { 65 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 66 | return 67 | } 68 | if len(scrollResults.Items) > 0 { 69 | scrollID = scrollResults.SearchID 70 | 71 | for i := range scrollResults.Items { 72 | results <- subscraping.Result{ 73 | Source: s.Name(), 74 | Type: subscraping.Subdomain, 75 | Value: scrollResults.Items[i].Name, 76 | } 77 | } 78 | } 79 | } 80 | } else { 81 | var limit = 100 82 | var searchResults []spyse.Domain 83 | 84 | for offset := 0; int64(offset) < totalResults && int64(offset) < searchMethodResultsLimit; offset += limit { 85 | searchResults, err = domainSvc.Search(ctx, subdomainsSearchParams.Query, limit, offset) 86 | if err != nil { 87 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 88 | return 89 | } 90 | 91 | for i := range searchResults { 92 | results <- subscraping.Result{ 93 | Source: s.Name(), 94 | Type: subscraping.Subdomain, 95 | Value: searchResults[i].Name, 96 | } 97 | } 98 | } 99 | } 100 | }() 101 | 102 | return results 103 | } 104 | 105 | // Name returns the name of the source 106 | func (s *Source) Name() string { 107 | return "spyse" 108 | } 109 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/intelx/intelx.go: -------------------------------------------------------------------------------- 1 | // Package intelx logic 2 | package intelx 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "fmt" 9 | "io" 10 | 11 | jsoniter "github.com/json-iterator/go" 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | ) 14 | 15 | type searchResponseType struct { 16 | ID string `json:"id"` 17 | Status int `json:"status"` 18 | } 19 | 20 | type selectorType struct { 21 | Selectvalue string `json:"selectorvalue"` 22 | } 23 | 24 | type searchResultType struct { 25 | Selectors []selectorType `json:"selectors"` 26 | Status int `json:"status"` 27 | } 28 | 29 | type requestBody struct { 30 | Term string 31 | Maxresults int 32 | Media int 33 | Target int 34 | Terminate []int 35 | Timeout int 36 | } 37 | 38 | // Source is the passive scraping agent 39 | type Source struct{} 40 | 41 | // Run function returns all subdomains found with the service 42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 43 | results := make(chan subscraping.Result) 44 | 45 | go func() { 46 | defer close(results) 47 | 48 | if session.Keys.IntelXKey == "" || session.Keys.IntelXHost == "" { 49 | return 50 | } 51 | 52 | searchURL := fmt.Sprintf("https://%s/phonebook/search?k=%s", session.Keys.IntelXHost, session.Keys.IntelXKey) 53 | reqBody := requestBody{ 54 | Term: domain, 55 | Maxresults: 100000, 56 | Media: 0, 57 | Target: 1, 58 | Timeout: 20, 59 | } 60 | 61 | body, err := json.Marshal(reqBody) 62 | if err != nil { 63 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 64 | return 65 | } 66 | 67 | resp, err := session.SimplePost(ctx, searchURL, "application/json", bytes.NewBuffer(body)) 68 | if err != nil { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 70 | session.DiscardHTTPResponse(resp) 71 | return 72 | } 73 | 74 | var response searchResponseType 75 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 76 | if err != nil { 77 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 78 | resp.Body.Close() 79 | return 80 | } 81 | 82 | resp.Body.Close() 83 | 84 | resultsURL := fmt.Sprintf("https://%s/phonebook/search/result?k=%s&id=%s&limit=10000", session.Keys.IntelXHost, session.Keys.IntelXKey, response.ID) 85 | status := 0 86 | for status == 0 || status == 3 { 87 | resp, err = session.Get(ctx, resultsURL, "", nil) 88 | if err != nil { 89 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 90 | session.DiscardHTTPResponse(resp) 91 | return 92 | } 93 | var response searchResultType 94 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 95 | if err != nil { 96 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 97 | resp.Body.Close() 98 | return 99 | } 100 | 101 | _, err = io.ReadAll(resp.Body) 102 | if err != nil { 103 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 104 | resp.Body.Close() 105 | return 106 | } 107 | resp.Body.Close() 108 | 109 | status = response.Status 110 | for _, hostname := range response.Selectors { 111 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: hostname.Selectvalue} 112 | } 113 | } 114 | }() 115 | 116 | return results 117 | } 118 | 119 | // Name returns the name of the source 120 | func (s *Source) Name() string { 121 | return "intelx" 122 | } 123 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/crtsh/crtsh.go: -------------------------------------------------------------------------------- 1 | // Package crtsh logic 2 | package crtsh 3 | 4 | import ( 5 | "context" 6 | "database/sql" 7 | "fmt" 8 | "strings" 9 | 10 | jsoniter "github.com/json-iterator/go" 11 | 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | // postgres driver 14 | _ "github.com/lib/pq" 15 | ) 16 | 17 | type subdomain struct { 18 | ID int `json:"id"` 19 | NameValue string `json:"name_value"` 20 | } 21 | 22 | // Source is the passive scraping agent 23 | type Source struct{} 24 | 25 | // Run function returns all subdomains found with the service 26 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 27 | results := make(chan subscraping.Result) 28 | 29 | go func() { 30 | defer close(results) 31 | count := s.getSubdomainsFromSQL(domain, results) 32 | if count > 0 { 33 | return 34 | } 35 | _ = s.getSubdomainsFromHTTP(ctx, domain, session, results) 36 | }() 37 | 38 | return results 39 | } 40 | 41 | func (s *Source) getSubdomainsFromSQL(domain string, results chan subscraping.Result) int { 42 | db, err := sql.Open("postgres", "host=crt.sh user=guest dbname=certwatch sslmode=disable binary_parameters=yes") 43 | if err != nil { 44 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 45 | return 0 46 | } 47 | 48 | defer db.Close() 49 | 50 | pattern := "%." + domain 51 | query := `SELECT DISTINCT ci.NAME_VALUE as domain FROM certificate_identity ci 52 | WHERE reverse(lower(ci.NAME_VALUE)) LIKE reverse(lower($1)) 53 | ORDER BY ci.NAME_VALUE` 54 | rows, err := db.Query(query, pattern) 55 | if err != nil { 56 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 57 | return 0 58 | } 59 | if err := rows.Err(); err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | return 0 62 | } 63 | 64 | var count int 65 | var data string 66 | // Parse all the rows getting subdomains 67 | for rows.Next() { 68 | err := rows.Scan(&data) 69 | if err != nil { 70 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 71 | return count 72 | } 73 | count++ 74 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: data} 75 | } 76 | return count 77 | } 78 | 79 | func (s *Source) getSubdomainsFromHTTP(ctx context.Context, domain string, session *subscraping.Session, results chan subscraping.Result) bool { 80 | resp, err := session.SimpleGet(ctx, fmt.Sprintf("https://crt.sh/?q=%%25.%s&output=json", domain)) 81 | if err != nil { 82 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 83 | session.DiscardHTTPResponse(resp) 84 | return false 85 | } 86 | 87 | var subdomains []subdomain 88 | err = jsoniter.NewDecoder(resp.Body).Decode(&subdomains) 89 | if err != nil { 90 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 91 | resp.Body.Close() 92 | return false 93 | } 94 | 95 | resp.Body.Close() 96 | 97 | for _, subdomain := range subdomains { 98 | if strings.Contains(subdomain.NameValue, "\n") { 99 | nameValues := strings.Split(subdomain.NameValue, "\n") 100 | for _, nameValue := range nameValues { 101 | if nameValue != "" { 102 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: nameValue} 103 | } 104 | } 105 | 106 | } else { 107 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain.NameValue} 108 | } 109 | } 110 | 111 | return true 112 | } 113 | 114 | // Name returns the name of the source 115 | func (s *Source) Name() string { 116 | return "crtsh" 117 | } 118 | -------------------------------------------------------------------------------- /pkg/resolve/resolve.go: -------------------------------------------------------------------------------- 1 | package resolve 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "sync" 7 | "time" 8 | 9 | "github.com/rs/xid" 10 | ) 11 | 12 | 13 | // ResolutionPool is a pool of resolvers created for resolving subdomains 14 | // for a given host. 15 | type ResolutionPool struct { 16 | *Resolver 17 | Tasks chan HostEntry 18 | Results chan Result 19 | wg *sync.WaitGroup 20 | removeWildcard bool 21 | 22 | wildcardIPs map[string]struct{} 23 | } 24 | 25 | // HostEntry defines a host with the source 26 | type HostEntry struct { 27 | Host string `json:"host"` 28 | Source string `json:"source"` 29 | IpPorts map[string][]int `json:"ip_ports"` 30 | CNames []string `json:"cnames"` 31 | TakeOver bool `json:"take_over"` 32 | } 33 | 34 | // Result contains the result for a host resolution 35 | type Result struct { 36 | Type ResultType 37 | Host string 38 | IP string 39 | Error error 40 | Source string 41 | } 42 | 43 | // ResultType is the type of result found 44 | type ResultType int 45 | 46 | // Types of data result can return 47 | const ( 48 | Subdomain ResultType = iota 49 | Error 50 | ) 51 | 52 | //// NewResolutionPool creates a pool of resolvers for resolving subdomains of a given domain 53 | //func (r *Resolver) NewResolutionPool(workers int, removeWildcard bool) *ResolutionPool { 54 | // resolutionPool := &ResolutionPool{ 55 | // Resolver: r, 56 | // Tasks: make(chan HostEntry), 57 | // Results: make(chan Result), 58 | // wg: &sync.WaitGroup{}, 59 | // removeWildcard: removeWildcard, 60 | // wildcardIPs: make(map[string]struct{}), 61 | // } 62 | // 63 | // go func() { 64 | // for i := 0; i < workers; i++ { 65 | // resolutionPool.wg.Add(1) 66 | // go resolutionPool.resolveWorker() 67 | // } 68 | // resolutionPool.wg.Wait() 69 | // close(resolutionPool.Results) 70 | // }() 71 | // 72 | // return resolutionPool 73 | //} 74 | 75 | // InitWildcards inits the wildcard ips array 76 | func InitWildcards(r *Resolver, domain string, resolvers []string, maxWildcardChecks int) (error, map[string]struct{}) { 77 | // 随机多少个域名 78 | if maxWildcardChecks == 0 { 79 | maxWildcardChecks = len(resolvers)*2 80 | } 81 | 82 | wildcardIPs := make(map[string]struct{}) 83 | for i := 0; i < maxWildcardChecks; i++ { 84 | rand.Seed(time.Now().UTC().UnixNano()) 85 | uid := xid.New().String() 86 | 87 | hosts, _ := r.DNSClient.Lookup(uid + "." + domain) 88 | 89 | if len(hosts) == 0 { 90 | return fmt.Errorf("%s is not a wildcard domain", domain), nil 91 | } 92 | // Append all wildcard ips found for domains 93 | // Append all wildcard ips found for domains 94 | for _, host := range hosts { 95 | wildcardIPs[host] = struct{}{} 96 | } 97 | } 98 | return nil, wildcardIPs 99 | } 100 | 101 | //func (r *ResolutionPool) resolveWorker() { 102 | // for task := range r.Tasks { 103 | // if !r.removeWildcard { 104 | // r.Results <- Result{Type: Subdomain, Host: task.Host, IP: "", Source: task.Source} 105 | // continue 106 | // } 107 | // 108 | // fmt.Println(task) 109 | // hosts, err := r.DNSClient.Lookup(task.Host) 110 | // if err != nil { 111 | // r.Results <- Result{Type: Error, Host: task.Host, Source: task.Source, Error: err} 112 | // continue 113 | // } 114 | // 115 | // if len(hosts) == 0 { 116 | // continue 117 | // } 118 | // 119 | // var skip bool 120 | // for _, host := range hosts { 121 | // // Ignore the host if it exists in wildcard ips map 122 | // if _, ok := r.wildcardIPs[host]; ok { 123 | // skip = true 124 | // break 125 | // } 126 | // } 127 | // 128 | // if !skip { 129 | // r.Results <- Result{Type: Subdomain, Host: task.Host, IP: hosts[0], Source: task.Source} 130 | // } 131 | // } 132 | // r.wg.Done() 133 | //} 134 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/zoomeye/zoomeye.go: -------------------------------------------------------------------------------- 1 | // Package zoomeye logic 2 | package zoomeye 3 | 4 | import ( 5 | "bytes" 6 | "context" 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "net/http" 11 | 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 13 | ) 14 | 15 | // zoomAuth holds the ZoomEye credentials 16 | type zoomAuth struct { 17 | User string `json:"username"` 18 | Pass string `json:"password"` 19 | } 20 | 21 | type loginResp struct { 22 | JWT string `json:"access_token"` 23 | } 24 | 25 | // search results 26 | type zoomeyeResults struct { 27 | Matches []struct { 28 | Site string `json:"site"` 29 | Domains []string `json:"domains"` 30 | } `json:"matches"` 31 | } 32 | 33 | // Source is the passive scraping agent 34 | type Source struct{} 35 | 36 | // Run function returns all subdomains found with the service 37 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 38 | results := make(chan subscraping.Result) 39 | 40 | go func() { 41 | defer close(results) 42 | 43 | if session.Keys.ZoomEyeUsername == "" || session.Keys.ZoomEyePassword == "" { 44 | return 45 | } 46 | 47 | jwt, err := doLogin(ctx, session) 48 | if err != nil { 49 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 50 | return 51 | } 52 | // check if jwt is null 53 | if jwt == "" { 54 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: errors.New("could not log into zoomeye")} 55 | return 56 | } 57 | 58 | headers := map[string]string{ 59 | "Authorization": fmt.Sprintf("JWT %s", jwt), 60 | "Accept": "application/json", 61 | "Content-Type": "application/json", 62 | } 63 | for currentPage := 0; currentPage <= 100; currentPage++ { 64 | api := fmt.Sprintf("https://api.zoomeye.org/web/search?query=hostname:%s&page=%d", domain, currentPage) 65 | resp, err := session.Get(ctx, api, "", headers) 66 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden 67 | if err != nil { 68 | if !isForbidden && currentPage == 0 { 69 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 70 | session.DiscardHTTPResponse(resp) 71 | } 72 | return 73 | } 74 | 75 | var res zoomeyeResults 76 | err = json.NewDecoder(resp.Body).Decode(&res) 77 | if err != nil { 78 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 79 | resp.Body.Close() 80 | return 81 | } 82 | resp.Body.Close() 83 | 84 | for _, r := range res.Matches { 85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: r.Site} 86 | for _, domain := range r.Domains { 87 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: domain} 88 | } 89 | } 90 | } 91 | }() 92 | 93 | return results 94 | } 95 | 96 | // doLogin performs authentication on the ZoomEye API 97 | func doLogin(ctx context.Context, session *subscraping.Session) (string, error) { 98 | creds := &zoomAuth{ 99 | User: session.Keys.ZoomEyeUsername, 100 | Pass: session.Keys.ZoomEyePassword, 101 | } 102 | body, err := json.Marshal(&creds) 103 | if err != nil { 104 | return "", err 105 | } 106 | resp, err := session.SimplePost(ctx, "https://api.zoomeye.org/user/login", "application/json", bytes.NewBuffer(body)) 107 | if err != nil { 108 | session.DiscardHTTPResponse(resp) 109 | return "", err 110 | } 111 | 112 | defer resp.Body.Close() 113 | 114 | var login loginResp 115 | err = json.NewDecoder(resp.Body).Decode(&login) 116 | if err != nil { 117 | return "", err 118 | } 119 | return login.JWT, nil 120 | } 121 | 122 | // Name returns the name of the source 123 | func (s *Source) Name() string { 124 | return "zoomeye" 125 | } 126 | -------------------------------------------------------------------------------- /pkg/runner/runner.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "io" 7 | "os" 8 | "path" 9 | "strings" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/passive" 12 | "github.com/ZhuriLab/Starmap/pkg/resolve" 13 | "github.com/pkg/errors" 14 | "github.com/projectdiscovery/gologger" 15 | ) 16 | 17 | // Runner is an instance of the subdomain enumeration 18 | // client used to orchestrate the whole process. 19 | type Runner struct { 20 | options *Options 21 | passiveAgent *passive.Agent 22 | resolverClient *resolve.Resolver 23 | Resolvers []string 24 | } 25 | 26 | // NewRunner creates a new runner struct instance by parsing 27 | // the configuration options, configuring sources, reading lists 28 | // and setting up loggers, etc. 29 | func NewRunner(options *Options) (*Runner, error) { 30 | runner := &Runner{options: options} 31 | 32 | // Initialize the passive subdomain enumeration engine 33 | runner.initializePassiveEngine() 34 | 35 | // Initialize the active subdomain enumeration engine 36 | err := runner.initializeActiveEngine() 37 | if err != nil { 38 | return nil, err 39 | } 40 | 41 | return runner, nil 42 | } 43 | 44 | // RunEnumeration runs the subdomain enumeration flow on the targets specified 45 | func (r *Runner) RunEnumeration(ctx context.Context) error { 46 | outputs := []io.Writer{r.options.Output} 47 | 48 | if len(r.options.Domain) > 0 { 49 | domainsReader := strings.NewReader(strings.Join(r.options.Domain, "\n")) 50 | return r.EnumerateMultipleDomains(ctx, domainsReader, outputs) 51 | } 52 | 53 | // If we have multiple domains as input, 54 | if r.options.DomainsFile != "" { 55 | f, err := os.Open(r.options.DomainsFile) 56 | if err != nil { 57 | return err 58 | } 59 | err = r.EnumerateMultipleDomains(ctx, f, outputs) 60 | f.Close() 61 | return err 62 | } 63 | 64 | // If we have STDIN input, treat it as multiple domains 65 | if r.options.Stdin { 66 | return r.EnumerateMultipleDomains(ctx, os.Stdin, outputs) 67 | } 68 | return nil 69 | } 70 | 71 | // EnumerateMultipleDomains enumerates subdomains for multiple domains 72 | // We keep enumerating subdomains for a given domain until we reach an error 73 | func (r *Runner) EnumerateMultipleDomains(ctx context.Context, reader io.Reader, outputs []io.Writer) error { 74 | scanner := bufio.NewScanner(reader) 75 | for scanner.Scan() { 76 | domain, err := sanitize(scanner.Text()) 77 | if errors.Is(err, ErrEmptyInput) { 78 | continue 79 | } 80 | 81 | var file *os.File 82 | // If the user has specified an output file, use that output file instead 83 | // of creating a new output file for each domain. Else create a new file 84 | // for each domain in the directory. 85 | if r.options.OutputFile != "" { 86 | outputter := NewOutputter(r.options.JSON) 87 | file, err = outputter.createFile(r.options.OutputFile, true) 88 | if err != nil { 89 | gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) 90 | return err 91 | } 92 | 93 | err, _, _ = r.EnumerateSingleDomain(ctx, domain, append(outputs, file)) 94 | 95 | file.Close() 96 | } else if r.options.OutputDirectory != "" { 97 | outputFile := path.Join(r.options.OutputDirectory, domain) 98 | if r.options.JSON { 99 | outputFile += ".json" 100 | } else { 101 | outputFile += ".txt" 102 | } 103 | 104 | outputter := NewOutputter(r.options.JSON) 105 | file, err = outputter.createFile(outputFile, false) 106 | if err != nil { 107 | gologger.Error().Msgf("Could not create file %s for %s: %s\n", r.options.OutputFile, r.options.Domain, err) 108 | return err 109 | } 110 | 111 | err, _, _ = r.EnumerateSingleDomain(ctx, domain, append(outputs, file)) 112 | 113 | file.Close() 114 | } else { 115 | err, _, _ = r.EnumerateSingleDomain(ctx, domain, outputs) 116 | } 117 | if err != nil { 118 | return err 119 | } 120 | } 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /pkg/subTakeOver/fingerprint.go: -------------------------------------------------------------------------------- 1 | package subTakeOver 2 | 3 | import ( 4 | "bytes" 5 | "strings" 6 | ) 7 | 8 | type Fingerprints struct { 9 | Service string `json:"service"` 10 | Cname []string `json:"cname"` 11 | Fingerprint []string `json:"fingerprint"` 12 | Nxdomain bool `json:"nxdomain"` 13 | CheckAll bool `json:"checkall"` 14 | } 15 | 16 | 17 | /* 18 | * Triage step to check whether the CNAME matches 19 | * the fingerprinted CNAME of a vulnerable cloud service. 20 | */ 21 | 22 | func VerifyCNAME(cnames []string, fingerprints []Fingerprints) (match bool, cname string, fingerprint Fingerprints) { 23 | 24 | match = false 25 | if len(cname) == 0 { 26 | return match, "", Fingerprints{} 27 | } 28 | 29 | cname = cnames[len(cnames)-1] 30 | VERIFY: 31 | for n := range fingerprints { 32 | for c := range fingerprints[n].Cname { 33 | if strings.Contains(cname, fingerprints[n].Cname[c]) { 34 | match = true 35 | fingerprint = fingerprints[n] 36 | break VERIFY 37 | } 38 | } 39 | } 40 | 41 | return match, cname, fingerprint 42 | } 43 | 44 | /* 45 | * This function aims to identify whether the subdomain 46 | * is attached to a vulnerable cloud service and able to 47 | * be taken over. 48 | */ 49 | func Identify(subdomain string, cnames []string, o *Options, cname string, fingerprint Fingerprints) (service string) { 50 | 51 | if cname == "" { 52 | if len(cnames) > 0 { 53 | cname = cnames[len(cnames)-1] 54 | } 55 | } 56 | 57 | if len(cname) <= 3 { 58 | cname = "" 59 | } 60 | 61 | //nx := nxdomain(subdomain) 62 | // 63 | //if nx { 64 | // dead := available.Domain(cname) 65 | // if dead { 66 | // service = "Domain Available - " + cname 67 | // return service 68 | // } 69 | // 70 | // // 报告存在 NXDOMAIN 的子域名 71 | // // Option to always print the CNAME and not check if it's available to be registered. 72 | // if o.Manual && cname != "" { 73 | // service = "Dead Domain - " + cname 74 | // return service 75 | // } 76 | //} 77 | 78 | if o.Ssl != true && fingerprint.Service == "cloudfront" { 79 | o.Ssl = true 80 | } 81 | 82 | body := get(subdomain, o.Ssl, o.Timeout) 83 | 84 | // 只对匹配的 cname 进行检测, 前面已经检测过 cname 匹配了，这里直接进行 body 内容匹配 85 | if !o.All { 86 | // 如果字典中的 nxdomain 为 ture(这种情况时，对应指纹字典中没有 fingerprint) 87 | //只会进行 cname 指纹匹配, 命中则可以进行子域名接管 88 | if fingerprint.Nxdomain { 89 | return fingerprint.Service 90 | } 91 | 92 | if body == nil { 93 | return "" 94 | } 95 | 96 | for n := range fingerprint.Fingerprint { 97 | if bytes.Contains(body, []byte(fingerprint.Fingerprint[n])) { 98 | return fingerprint.Service 99 | } 100 | } 101 | } else { 102 | fingerprints := o.Fingerprints 103 | 104 | for f := range fingerprints { // 不看 cname ，只对 body 内容进行检测 105 | 106 | if fingerprints[f].Nxdomain { 107 | for n := range fingerprints[f].Cname { 108 | if strings.Contains(cname, fingerprints[f].Cname[n]) { 109 | return fingerprints[f].Service 110 | } 111 | } 112 | } 113 | 114 | if body == nil { 115 | return "" 116 | } 117 | 118 | if o.Ssl != true && fingerprints[f].Service == "cloudfront" { 119 | o.Ssl = true 120 | body = get(subdomain, o.Ssl, o.Timeout) 121 | if body == nil { 122 | return "" 123 | } 124 | } 125 | 126 | if fingerprints[f].CheckAll { // 指纹中指定 CheckAll ，cname 、body 内容都检查 127 | for c := range fingerprints[f].Cname { 128 | if strings.Contains(cname, fingerprints[f].Cname[c]) { 129 | // cname 匹配的情况下 ，再检测cname 对应的指纹是否匹配 130 | for n := range fingerprints[f].Fingerprint { 131 | if bytes.Contains(body, []byte(fingerprints[f].Fingerprint[n])) { 132 | return fingerprints[f].Service 133 | } 134 | } 135 | } 136 | 137 | } 138 | } else { 139 | for n := range fingerprints[f].Fingerprint { 140 | if bytes.Contains(body, []byte(fingerprints[f].Fingerprint[n])) { 141 | return fingerprints[f].Service 142 | } 143 | } 144 | } 145 | } 146 | 147 | } 148 | 149 | return "" 150 | } 151 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/hunter/hunter.go: -------------------------------------------------------------------------------- 1 | package hunter 2 | 3 | import ( 4 | "context" 5 | "encoding/base64" 6 | "fmt" 7 | "time" 8 | 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 10 | jsoniter "github.com/json-iterator/go" 11 | ) 12 | 13 | type hunterResults struct { 14 | Code int `json:"code"` 15 | Data struct { 16 | AccountType string `json:"account_type"` 17 | Total int `json:"total"` 18 | Time int `json:"time"` 19 | Arr []struct { 20 | IsRisk string `json:"is_risk"` 21 | URL string `json:"url"` 22 | IP string `json:"ip"` 23 | Port int `json:"port"` 24 | WebTitle string `json:"web_title"` 25 | Domain string `json:"domain"` 26 | IsRiskProtocol string `json:"is_risk_protocol"` 27 | Protocol string `json:"protocol"` 28 | BaseProtocol string `json:"base_protocol"` 29 | StatusCode int `json:"status_code"` 30 | Component []struct { 31 | Name string `json:"name"` 32 | Version string `json:"version"` 33 | } `json:"component"` 34 | Os string `json:"os"` 35 | Company string `json:"company"` 36 | Number string `json:"number"` 37 | Country string `json:"country"` 38 | Province string `json:"province"` 39 | City string `json:"city"` 40 | UpdatedAt string `json:"updated_at"` 41 | IsWeb string `json:"is_web"` 42 | AsOrg string `json:"as_org"` 43 | Isp string `json:"isp"` 44 | Banner string `json:"banner"` 45 | } `json:"arr"` 46 | ConsumeQuota string `json:"consume_quota"` 47 | RestQuota string `json:"rest_quota"` 48 | SyntaxPrompt string `json:"syntax_prompt"` 49 | } `json:"data"` 50 | Message string `json:"message"` 51 | } 52 | 53 | // Source is the passive scraping agent 54 | type Source struct{} 55 | 56 | // Run function returns all subdomains found with the service 57 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 58 | results := make(chan subscraping.Result) 59 | go func() { 60 | defer close(results) 61 | 62 | if session.Keys.Hunter == "" { 63 | return 64 | } 65 | 66 | // hunter api doc https://hunter.qianxin.com/home/helpCenter?r=5-1-2 67 | api := "https://hunter.qianxin.com/openApi/search?api-key=%s&search=%s&page=%d&page_size=100&is_web=3&start_time=\"%d-01-01+00:00:00\"&end_time=\"%d-12-31+00:00:00\"" 68 | 69 | qbase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("domain=\"%s\"", domain))) 70 | 71 | url := fmt.Sprintf(api, session.Keys.Hunter, qbase64, 1, time.Now().Year()-1, time.Now().Year()) 72 | 73 | resp, err := session.SimpleGet(ctx, url) 74 | 75 | if err != nil { 76 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 77 | session.DiscardHTTPResponse(resp) 78 | return 79 | } 80 | 81 | var response hunterResults 82 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 83 | if err != nil { 84 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 85 | resp.Body.Close() 86 | return 87 | } 88 | resp.Body.Close() 89 | 90 | if response.Code != 200 { 91 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("%s", response.Message)} 92 | return 93 | } 94 | 95 | if response.Data.Total > 0 { 96 | page := response.Data.Total/100 + 1 97 | 98 | for i := 0; i < page; i++ { 99 | for _, hunterDomain := range response.Data.Arr { 100 | subdomain := hunterDomain.Domain 101 | if subdomain != "" { 102 | 103 | ipPorts := make(map[string][]int) 104 | ipPorts[hunterDomain.IP] = []int{hunterDomain.Port} 105 | 106 | results <- subscraping.Result{ 107 | Source: s.Name(), 108 | Type: subscraping.Subdomain, 109 | Value: subdomain, 110 | IpPorts: ipPorts, 111 | } 112 | } 113 | 114 | } 115 | } 116 | 117 | } 118 | }() 119 | 120 | return results 121 | } 122 | 123 | // Name returns the name of the source 124 | func (s *Source) Name() string { 125 | return "hunter" 126 | } 127 | -------------------------------------------------------------------------------- /pkg/active/device/device.go: -------------------------------------------------------------------------------- 1 | package device 2 | 3 | import ( 4 | "context" 5 | "github.com/ZhuriLab/Starmap/pkg/util" 6 | "github.com/google/gopacket" 7 | "github.com/google/gopacket/layers" 8 | "github.com/google/gopacket/pcap" 9 | "github.com/projectdiscovery/gologger" 10 | "net" 11 | "time" 12 | ) 13 | 14 | func AutoGetDevices() *EtherTable { 15 | domain := util.RandomStr(6) + ".baidu.com" 16 | signal := make(chan *EtherTable) 17 | devices, err := pcap.FindAllDevs() 18 | if err != nil { 19 | gologger.Fatal().Msgf("获取网络设备失败:%s\n", err.Error()) 20 | } 21 | data := make(map[string]net.IP) 22 | keys := []string{} 23 | for _, d := range devices { 24 | for _, address := range d.Addresses { 25 | ip := address.IP 26 | if ip.To4() != nil && !ip.IsLoopback() { 27 | data[d.Name] = ip 28 | keys = append(keys, d.Name) 29 | } 30 | } 31 | } 32 | ctx := context.Background() 33 | // 在初始上下文的基础上创建一个有取消功能的上下文 34 | ctx, cancel := context.WithCancel(ctx) 35 | for _, drviceName := range keys { 36 | go func(drviceName string, domain string, ctx context.Context) { 37 | var ( 38 | snapshot_len int32 = 1024 39 | promiscuous bool = false 40 | timeout time.Duration = -1 * time.Second 41 | handle *pcap.Handle 42 | ) 43 | var err error 44 | handle, err = pcap.OpenLive( 45 | drviceName, 46 | snapshot_len, 47 | promiscuous, 48 | timeout, 49 | ) 50 | if err != nil { 51 | gologger.Error().Msgf("pcap打开失败:%s\n", err.Error()) 52 | return 53 | } 54 | defer handle.Close() 55 | // Use the handle as a packet source to process all packets 56 | packetSource := gopacket.NewPacketSource(handle, handle.LinkType()) 57 | for { 58 | select { 59 | case <-ctx.Done(): 60 | return 61 | default: 62 | packet, err := packetSource.NextPacket() 63 | if err != nil { 64 | continue 65 | } 66 | if dnsLayer := packet.Layer(layers.LayerTypeDNS); dnsLayer != nil { 67 | dns, _ := dnsLayer.(*layers.DNS) 68 | if !dns.QR { 69 | continue 70 | } 71 | for _, v := range dns.Questions { 72 | if string(v.Name) == domain { 73 | ethLayer := packet.Layer(layers.LayerTypeEthernet) 74 | if ethLayer != nil { 75 | eth := ethLayer.(*layers.Ethernet) 76 | etherTable := EtherTable{ 77 | SrcIp: data[drviceName], 78 | Device: drviceName, 79 | SrcMac: SelfMac(eth.DstMAC), 80 | DstMac: SelfMac(eth.SrcMAC), 81 | } 82 | signal <- ðerTable 83 | return 84 | } 85 | } 86 | } 87 | } 88 | } 89 | } 90 | }(drviceName, domain, ctx) 91 | } 92 | for { 93 | select { 94 | case c := <-signal: 95 | cancel() 96 | return c 97 | default: 98 | _, _ = net.LookupHost(domain) 99 | time.Sleep(time.Second * 1) 100 | } 101 | } 102 | } 103 | func GetIpv4Devices() (keys []string, data map[string]net.IP) { 104 | devices, err := pcap.FindAllDevs() 105 | data = make(map[string]net.IP) 106 | if err != nil { 107 | gologger.Fatal().Msgf("获取网络设备失败:%s\n", err.Error()) 108 | } 109 | for _, d := range devices { 110 | for _, address := range d.Addresses { 111 | ip := address.IP 112 | if ip.To4() != nil && !ip.IsLoopback() { 113 | gologger.Print().Msgf(" [%d] Name: %s\n", len(keys), d.Name) 114 | gologger.Print().Msgf(" Description: %s\n", d.Description) 115 | gologger.Print().Msgf(" Devices addresses: %s\n", d.Description) 116 | gologger.Print().Msgf(" IP address: %s\n", ip) 117 | gologger.Print().Msgf(" Subnet mask: %s\n\n", address.Netmask.String()) 118 | data[d.Name] = ip 119 | keys = append(keys, d.Name) 120 | } 121 | } 122 | } 123 | return 124 | } 125 | func PcapInit(devicename string) (*pcap.Handle, error) { 126 | var ( 127 | snapshot_len int32 = 1024 128 | //promiscuous bool = false 129 | err error 130 | timeout time.Duration = -1 * time.Second 131 | ) 132 | handle, err := pcap.OpenLive(devicename, snapshot_len, false, timeout) 133 | if err != nil { 134 | gologger.Fatal().Msgf("pcap初始化失败:%s\n", err.Error()) 135 | return nil, err 136 | } 137 | return handle, nil 138 | } 139 | -------------------------------------------------------------------------------- /pkg/subscraping/agent.go: -------------------------------------------------------------------------------- 1 | package subscraping 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "net/http" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/corpix/uarand" 13 | "github.com/projectdiscovery/gologger" 14 | "go.uber.org/ratelimit" 15 | ) 16 | 17 | // NewSession creates a new session object for a domain 18 | func NewSession(domain string, keys *Keys, proxy string, rateLimit, timeout int) (*Session, error) { 19 | Transport := &http.Transport{ 20 | MaxIdleConns: 100, 21 | MaxIdleConnsPerHost: 100, 22 | TLSClientConfig: &tls.Config{ 23 | InsecureSkipVerify: true, 24 | }, 25 | } 26 | 27 | // Add proxy 28 | if proxy != "" { 29 | proxyURL, _ := url.Parse(proxy) 30 | if proxyURL == nil { 31 | // Log warning but continue anyways 32 | gologger.Warning().Msgf("Invalid proxy '%s' provided", proxy) 33 | } else { 34 | Transport.Proxy = http.ProxyURL(proxyURL) 35 | } 36 | } 37 | 38 | client := &http.Client{ 39 | Transport: Transport, 40 | Timeout: time.Duration(timeout) * time.Second, 41 | } 42 | 43 | session := &Session{ 44 | Client: client, 45 | Keys: keys, 46 | } 47 | 48 | // Initiate rate limit instance 49 | if rateLimit > 0 { 50 | session.RateLimiter = ratelimit.New(rateLimit) 51 | } else { 52 | session.RateLimiter = ratelimit.NewUnlimited() 53 | } 54 | 55 | // Create a new extractor object for the current domain 56 | extractor, err := NewSubdomainExtractor(domain) 57 | session.Extractor = extractor 58 | 59 | return session, err 60 | } 61 | 62 | // Get makes a GET request to a URL with extended parameters 63 | func (s *Session) Get(ctx context.Context, getURL, cookies string, headers map[string]string) (*http.Response, error) { 64 | return s.HTTPRequest(ctx, http.MethodGet, getURL, cookies, headers, nil, BasicAuth{}) 65 | } 66 | 67 | // SimpleGet makes a simple GET request to a URL 68 | func (s *Session) SimpleGet(ctx context.Context, getURL string) (*http.Response, error) { 69 | return s.HTTPRequest(ctx, http.MethodGet, getURL, "", map[string]string{}, nil, BasicAuth{}) 70 | } 71 | 72 | // Post makes a POST request to a URL with extended parameters 73 | func (s *Session) Post(ctx context.Context, postURL, cookies string, headers map[string]string, body io.Reader) (*http.Response, error) { 74 | return s.HTTPRequest(ctx, http.MethodPost, postURL, cookies, headers, body, BasicAuth{}) 75 | } 76 | 77 | // SimplePost makes a simple POST request to a URL 78 | func (s *Session) SimplePost(ctx context.Context, postURL, contentType string, body io.Reader) (*http.Response, error) { 79 | return s.HTTPRequest(ctx, http.MethodPost, postURL, "", map[string]string{"Content-Type": contentType}, body, BasicAuth{}) 80 | } 81 | 82 | // HTTPRequest makes any HTTP request to a URL with extended parameters 83 | func (s *Session) HTTPRequest(ctx context.Context, method, requestURL, cookies string, headers map[string]string, body io.Reader, basicAuth BasicAuth) (*http.Response, error) { 84 | req, err := http.NewRequestWithContext(ctx, method, requestURL, body) 85 | if err != nil { 86 | return nil, err 87 | } 88 | 89 | req.Header.Set("User-Agent", uarand.GetRandom()) 90 | req.Header.Set("Accept", "*/*") 91 | req.Header.Set("Accept-Language", "en") 92 | req.Header.Set("Connection", "close") 93 | 94 | if basicAuth.Username != "" || basicAuth.Password != "" { 95 | req.SetBasicAuth(basicAuth.Username, basicAuth.Password) 96 | } 97 | 98 | if cookies != "" { 99 | req.Header.Set("Cookie", cookies) 100 | } 101 | 102 | for key, value := range headers { 103 | req.Header.Set(key, value) 104 | } 105 | 106 | s.RateLimiter.Take() 107 | 108 | return httpRequestWrapper(s.Client, req) 109 | } 110 | 111 | // DiscardHTTPResponse discards the response content by demand 112 | func (s *Session) DiscardHTTPResponse(response *http.Response) { 113 | if response != nil { 114 | _, err := io.Copy(io.Discard, response.Body) 115 | if err != nil { 116 | gologger.Warning().Msgf("Could not discard response body: %s\n", err) 117 | return 118 | } 119 | response.Body.Close() 120 | } 121 | } 122 | 123 | func httpRequestWrapper(client *http.Client, request *http.Request) (*http.Response, error) { 124 | resp, err := client.Do(request) 125 | if err != nil { 126 | return nil, err 127 | } 128 | 129 | if resp.StatusCode != http.StatusOK { 130 | requestURL, _ := url.QueryUnescape(request.URL.String()) 131 | return resp, fmt.Errorf("unexpected status code %d received from %s", resp.StatusCode, requestURL) 132 | } 133 | return resp, nil 134 | } 135 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/binaryedge/binaryedge.go: -------------------------------------------------------------------------------- 1 | // Package binaryedge logic 2 | package binaryedge 3 | 4 | import ( 5 | "context" 6 | "fmt" 7 | "math" 8 | "net/url" 9 | "strconv" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 12 | jsoniter "github.com/json-iterator/go" 13 | ) 14 | 15 | const ( 16 | v1 = "v1" 17 | v2 = "v2" 18 | baseAPIURLFmt = "https://api.binaryedge.io/%s/query/domains/subdomain/%s" 19 | v2SubscriptionURL = "https://api.binaryedge.io/v2/user/subscription" 20 | v1PageSizeParam = "pagesize" 21 | pageParam = "page" 22 | firstPage = 1 23 | maxV1PageSize = 10000 24 | ) 25 | 26 | type subdomainsResponse struct { 27 | Message string `json:"message"` 28 | Title string `json:"title"` 29 | Status interface{} `json:"status"` // string for v1, int for v2 30 | Subdomains []string `json:"events"` 31 | Page int `json:"page"` 32 | PageSize int `json:"pagesize"` 33 | Total int `json:"total"` 34 | } 35 | 36 | // Source is the passive scraping agent 37 | type Source struct{} 38 | 39 | // Run function returns all subdomains found with the service 40 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 41 | results := make(chan subscraping.Result) 42 | 43 | go func() { 44 | defer close(results) 45 | 46 | if session.Keys.Binaryedge == "" { 47 | return 48 | } 49 | 50 | var baseURL string 51 | 52 | authHeader := map[string]string{"X-Key": session.Keys.Binaryedge} 53 | 54 | if isV2(ctx, session, authHeader) { 55 | baseURL = fmt.Sprintf(baseAPIURLFmt, v2, domain) 56 | } else { 57 | authHeader = map[string]string{"X-Token": session.Keys.Binaryedge} 58 | v1URLWithPageSize, err := addURLParam(fmt.Sprintf(baseAPIURLFmt, v1, domain), v1PageSizeParam, strconv.Itoa(maxV1PageSize)) 59 | if err != nil { 60 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 61 | return 62 | } 63 | baseURL = v1URLWithPageSize.String() 64 | } 65 | 66 | if baseURL == "" { 67 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf("can't get API URL")} 68 | return 69 | } 70 | 71 | s.enumerate(ctx, session, baseURL, firstPage, authHeader, results) 72 | }() 73 | 74 | return results 75 | } 76 | 77 | func (s *Source) enumerate(ctx context.Context, session *subscraping.Session, baseURL string, page int, authHeader map[string]string, results chan subscraping.Result) { 78 | pageURL, err := addURLParam(baseURL, pageParam, strconv.Itoa(page)) 79 | if err != nil { 80 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 81 | return 82 | } 83 | 84 | resp, err := session.Get(ctx, pageURL.String(), "", authHeader) 85 | if err != nil && resp == nil { 86 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 87 | session.DiscardHTTPResponse(resp) 88 | return 89 | } 90 | 91 | var response subdomainsResponse 92 | err = jsoniter.NewDecoder(resp.Body).Decode(&response) 93 | if err != nil { 94 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 95 | resp.Body.Close() 96 | return 97 | } 98 | 99 | // Check error messages 100 | if response.Message != "" && response.Status != nil { 101 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: fmt.Errorf(response.Message)} 102 | } 103 | 104 | resp.Body.Close() 105 | 106 | for _, subdomain := range response.Subdomains { 107 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Subdomain, Value: subdomain} 108 | } 109 | 110 | totalPages := int(math.Ceil(float64(response.Total) / float64(response.PageSize))) 111 | nextPage := response.Page + 1 112 | for currentPage := nextPage; currentPage <= totalPages; currentPage++ { 113 | s.enumerate(ctx, session, baseURL, currentPage, authHeader, results) 114 | } 115 | } 116 | 117 | // Name returns the name of the source 118 | func (s *Source) Name() string { 119 | return "binaryedge" 120 | } 121 | 122 | func isV2(ctx context.Context, session *subscraping.Session, authHeader map[string]string) bool { 123 | resp, err := session.Get(ctx, v2SubscriptionURL, "", authHeader) 124 | if err != nil { 125 | session.DiscardHTTPResponse(resp) 126 | return false 127 | } 128 | 129 | resp.Body.Close() 130 | 131 | return true 132 | } 133 | 134 | func addURLParam(targetURL, name, value string) (*url.URL, error) { 135 | u, err := url.Parse(targetURL) 136 | if err != nil { 137 | return u, err 138 | } 139 | q, _ := url.ParseQuery(u.RawQuery) 140 | q.Add(name, value) 141 | u.RawQuery = q.Encode() 142 | 143 | return u, nil 144 | } 145 | -------------------------------------------------------------------------------- /pkg/enum/zone.go: -------------------------------------------------------------------------------- 1 | package enum 2 | 3 | /** 4 | @author: yhy 5 | @since: 2022/7/20 6 | @desc: //TODO 7 | **/ 8 | import ( 9 | "context" 10 | "fmt" 11 | "net" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | "github.com/caffix/resolve" 17 | "github.com/miekg/dns" 18 | ) 19 | 20 | // DNSAnswer is the type used by Amass to represent a DNS record. 21 | type DNSAnswer struct { 22 | Name string `json:"name"` 23 | Type int `json:"type"` 24 | TTL int `json:"TTL"` 25 | Data string `json:"data"` 26 | } 27 | 28 | // DNSRequest handles data needed throughout Service processing of a DNS name. 29 | type DNSRequest struct { 30 | Name string 31 | Domain string 32 | Records []DNSAnswer 33 | Tag string 34 | Source string 35 | } 36 | 37 | // ZoneTransfer attempts a DNS zone transfer using the provided server. 38 | // The returned slice contains all the records discovered from the zone transfer. 39 | func ZoneTransfer(sub, domain, server string) ([]*DNSRequest, error) { 40 | var results []*DNSRequest 41 | 42 | // Set the maximum time allowed for making the connection 43 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) 44 | defer cancel() 45 | 46 | addr := net.JoinHostPort(server, "53") 47 | conn, err := DialContext(ctx, "tcp", addr) 48 | if err != nil { 49 | return results, fmt.Errorf("zone xfr error: Failed to obtain TCP connection to [%s]: %v", addr, err) 50 | } 51 | defer conn.Close() 52 | 53 | xfr := &dns.Transfer{ 54 | Conn: &dns.Conn{Conn: conn}, 55 | ReadTimeout: 15 * time.Second, 56 | } 57 | 58 | m := &dns.Msg{} 59 | m.SetAxfr(dns.Fqdn(sub)) 60 | 61 | in, err := xfr.In(m, "") 62 | if err != nil { 63 | return results, fmt.Errorf("DNS zone transfer error for [%s]: %v", addr, err) 64 | } 65 | 66 | for en := range in { 67 | reqs := getXfrRequests(en, domain) 68 | if reqs == nil { 69 | continue 70 | } 71 | 72 | results = append(results, reqs...) 73 | } 74 | return results, nil 75 | } 76 | 77 | func getXfrRequests(en *dns.Envelope, domain string) []*DNSRequest { 78 | if en.Error != nil { 79 | return nil 80 | } 81 | 82 | reqs := make(map[string]*DNSRequest) 83 | for _, a := range en.RR { 84 | var record DNSAnswer 85 | 86 | switch v := a.(type) { 87 | case *dns.CNAME: 88 | record.Type = int(dns.TypeCNAME) 89 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 90 | record.Data = resolve.RemoveLastDot(v.Target) 91 | case *dns.A: 92 | record.Type = int(dns.TypeA) 93 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 94 | record.Data = v.A.String() 95 | case *dns.AAAA: 96 | record.Type = int(dns.TypeAAAA) 97 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 98 | record.Data = v.AAAA.String() 99 | case *dns.PTR: 100 | record.Type = int(dns.TypePTR) 101 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 102 | record.Data = resolve.RemoveLastDot(v.Ptr) 103 | case *dns.NS: 104 | record.Type = int(dns.TypeNS) 105 | record.Name = realName(v.Hdr) 106 | record.Data = resolve.RemoveLastDot(v.Ns) 107 | case *dns.MX: 108 | record.Type = int(dns.TypeMX) 109 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 110 | record.Data = resolve.RemoveLastDot(v.Mx) 111 | case *dns.TXT: 112 | record.Type = int(dns.TypeTXT) 113 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 114 | for _, piece := range v.Txt { 115 | record.Data += piece + " " 116 | } 117 | case *dns.SOA: 118 | record.Type = int(dns.TypeSOA) 119 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 120 | record.Data = v.Ns + " " + v.Mbox 121 | case *dns.SPF: 122 | record.Type = int(dns.TypeSPF) 123 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 124 | for _, piece := range v.Txt { 125 | record.Data += piece + " " 126 | } 127 | case *dns.SRV: 128 | record.Type = int(dns.TypeSRV) 129 | record.Name = resolve.RemoveLastDot(v.Hdr.Name) 130 | record.Data = resolve.RemoveLastDot(v.Target) 131 | default: 132 | continue 133 | } 134 | 135 | if r, found := reqs[record.Name]; found { 136 | r.Records = append(r.Records, record) 137 | } else { 138 | reqs[record.Name] = &DNSRequest{ 139 | Name: record.Name, 140 | Domain: domain, 141 | Records: []DNSAnswer{record}, 142 | Tag: "axfr", 143 | Source: "DNS Zone XFR", 144 | } 145 | } 146 | } 147 | 148 | var requests []*DNSRequest 149 | for _, r := range reqs { 150 | requests = append(requests, r) 151 | } 152 | return requests 153 | } 154 | 155 | func realName(hdr dns.RR_Header) string { 156 | pieces := strings.Split(hdr.Name, " ") 157 | 158 | return resolve.RemoveLastDot(pieces[len(pieces)-1]) 159 | } 160 | 161 | var LocalAddr net.Addr 162 | 163 | // DialContext performs the dial using global variables (e.g. LocalAddr). 164 | func DialContext(ctx context.Context, network, addr string) (net.Conn, error) { 165 | d := &net.Dialer{DualStack: true} 166 | 167 | _, p, err := net.SplitHostPort(addr) 168 | if err != nil { 169 | return nil, err 170 | } 171 | 172 | port, err := strconv.Atoi(p) 173 | if err != nil { 174 | return nil, err 175 | } 176 | 177 | if LocalAddr != nil { 178 | addr, _, err := net.ParseCIDR(LocalAddr.String()) 179 | 180 | if err == nil && strings.HasPrefix(network, "tcp") { 181 | d.LocalAddr = &net.TCPAddr{ 182 | IP: addr, 183 | Port: port, 184 | } 185 | } else if err == nil && strings.HasPrefix(network, "udp") { 186 | d.LocalAddr = &net.UDPAddr{ 187 | IP: addr, 188 | Port: port, 189 | } 190 | } 191 | } 192 | 193 | return d.DialContext(ctx, network, addr) 194 | } 195 | -------------------------------------------------------------------------------- /pkg/runner/outputter.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/ZhuriLab/Starmap/pkg/resolve" 12 | jsoniter "github.com/json-iterator/go" 13 | ) 14 | 15 | // OutPutter outputs content to writers. 16 | type OutPutter struct { 17 | JSON bool 18 | } 19 | 20 | type jsonResult struct { 21 | Host string `json:"host"` 22 | IP string `json:"ip"` 23 | Source string `json:"source"` 24 | } 25 | 26 | type jsonSourceResult struct { 27 | Host string `json:"host"` 28 | Sources []string `json:"sources"` 29 | } 30 | 31 | // NewOutputter creates a new Outputter 32 | func NewOutputter(json bool) *OutPutter { 33 | return &OutPutter{JSON: json} 34 | } 35 | 36 | func (o *OutPutter) createFile(filename string, appendtoFile bool) (*os.File, error) { 37 | if filename == "" { 38 | return nil, errors.New("empty filename") 39 | } 40 | 41 | dir := filepath.Dir(filename) 42 | 43 | if dir != "" { 44 | if _, err := os.Stat(dir); os.IsNotExist(err) { 45 | err := os.MkdirAll(dir, os.ModePerm) 46 | if err != nil { 47 | return nil, err 48 | } 49 | } 50 | } 51 | 52 | var file *os.File 53 | var err error 54 | if appendtoFile { 55 | file, err = os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 56 | } else { 57 | file, err = os.Create(filename) 58 | } 59 | if err != nil { 60 | return nil, err 61 | } 62 | 63 | return file, nil 64 | } 65 | 66 | // WriteHostIP writes the output list of subdomain to an io.Writer 67 | func (o *OutPutter) WriteHostIP(results map[string]resolve.Result, writer io.Writer) error { 68 | var err error 69 | if o.JSON { 70 | err = writeJSONHostIP(results, writer) 71 | } else { 72 | err = writePlainHostIP(results, writer) 73 | } 74 | return err 75 | } 76 | 77 | func writePlainHostIP(results map[string]resolve.Result, writer io.Writer) error { 78 | bufwriter := bufio.NewWriter(writer) 79 | sb := &strings.Builder{} 80 | 81 | for _, result := range results { 82 | sb.WriteString(result.Host) 83 | sb.WriteString(",") 84 | sb.WriteString(result.IP) 85 | sb.WriteString(",") 86 | sb.WriteString(result.Source) 87 | sb.WriteString("\n") 88 | 89 | _, err := bufwriter.WriteString(sb.String()) 90 | if err != nil { 91 | bufwriter.Flush() 92 | return err 93 | } 94 | sb.Reset() 95 | } 96 | return bufwriter.Flush() 97 | } 98 | 99 | func writeJSONHostIP(results map[string]resolve.Result, writer io.Writer) error { 100 | encoder := jsoniter.NewEncoder(writer) 101 | 102 | var data jsonResult 103 | 104 | for _, result := range results { 105 | data.Host = result.Host 106 | data.IP = result.IP 107 | data.Source = result.Source 108 | 109 | err := encoder.Encode(&data) 110 | if err != nil { 111 | return err 112 | } 113 | } 114 | return nil 115 | } 116 | 117 | // WriteHostNoWildcard writes the output list of subdomain with nW flag to an io.Writer 118 | func (o *OutPutter) WriteHostNoWildcard(results map[string]resolve.Result, writer io.Writer) error { 119 | hosts := make(map[string]resolve.HostEntry) 120 | for host, result := range results { 121 | hosts[host] = resolve.HostEntry{Host: result.Host, Source: result.Source} 122 | } 123 | 124 | return o.WriteHost(hosts, writer) 125 | } 126 | 127 | // WriteHost writes the output list of subdomain to an io.Writer 128 | func (o *OutPutter) WriteHost(results map[string]resolve.HostEntry, writer io.Writer) error { 129 | var err error 130 | if o.JSON { 131 | err = writeJSONHost(results, writer) 132 | } else { 133 | err = writePlainHost(results, writer) 134 | } 135 | return err 136 | } 137 | 138 | func writePlainHost(results map[string]resolve.HostEntry, writer io.Writer) error { 139 | bufwriter := bufio.NewWriter(writer) 140 | sb := &strings.Builder{} 141 | 142 | for _, result := range results { 143 | sb.WriteString(result.Host) 144 | sb.WriteString("\n") 145 | 146 | _, err := bufwriter.WriteString(sb.String()) 147 | if err != nil { 148 | bufwriter.Flush() 149 | return err 150 | } 151 | sb.Reset() 152 | } 153 | return bufwriter.Flush() 154 | } 155 | 156 | func writeJSONHost(results map[string]resolve.HostEntry, writer io.Writer) error { 157 | encoder := jsoniter.NewEncoder(writer) 158 | 159 | for _, result := range results { 160 | err := encoder.Encode(result) 161 | if err != nil { 162 | return err 163 | } 164 | } 165 | return nil 166 | } 167 | 168 | // WriteSourceHost writes the output list of subdomain to an io.Writer 169 | func (o *OutPutter) WriteSourceHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error { 170 | var err error 171 | if o.JSON { 172 | err = writeSourceJSONHost(sourceMap, writer) 173 | } else { 174 | err = writeSourcePlainHost(sourceMap, writer) 175 | } 176 | return err 177 | } 178 | 179 | func writeSourceJSONHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error { 180 | encoder := jsoniter.NewEncoder(writer) 181 | 182 | var data jsonSourceResult 183 | 184 | for host, sources := range sourceMap { 185 | data.Host = host 186 | keys := make([]string, 0, len(sources)) 187 | for source := range sources { 188 | keys = append(keys, source) 189 | } 190 | data.Sources = keys 191 | 192 | err := encoder.Encode(&data) 193 | if err != nil { 194 | return err 195 | } 196 | } 197 | return nil 198 | } 199 | 200 | func writeSourcePlainHost(sourceMap map[string]map[string]struct{}, writer io.Writer) error { 201 | bufwriter := bufio.NewWriter(writer) 202 | sb := &strings.Builder{} 203 | 204 | for host, sources := range sourceMap { 205 | sb.WriteString(host) 206 | sb.WriteString(",[") 207 | sourcesString := "" 208 | for source := range sources { 209 | sourcesString += source + "," 210 | } 211 | sb.WriteString(strings.Trim(sourcesString, ", ")) 212 | sb.WriteString("]\n") 213 | 214 | _, err := bufwriter.WriteString(sb.String()) 215 | if err != nil { 216 | bufwriter.Flush() 217 | return err 218 | } 219 | sb.Reset() 220 | } 221 | return bufwriter.Flush() 222 | } 223 | -------------------------------------------------------------------------------- /pkg/subscraping/sources/github/github.go: -------------------------------------------------------------------------------- 1 | // Package github GitHub search package 2 | // Based on gwen001's https://github.com/gwen001/github-search github-subdomains 3 | package github 4 | 5 | import ( 6 | "bufio" 7 | "context" 8 | "fmt" 9 | "net/http" 10 | "net/url" 11 | "regexp" 12 | "strconv" 13 | "strings" 14 | "time" 15 | 16 | jsoniter "github.com/json-iterator/go" 17 | 18 | "github.com/projectdiscovery/gologger" 19 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 20 | "github.com/tomnomnom/linkheader" 21 | ) 22 | 23 | type textMatch struct { 24 | Fragment string `json:"fragment"` 25 | } 26 | 27 | type item struct { 28 | Name string `json:"name"` 29 | HTMLURL string `json:"html_url"` 30 | TextMatches []textMatch `json:"text_matches"` 31 | } 32 | 33 | type response struct { 34 | TotalCount int `json:"total_count"` 35 | Items []item `json:"items"` 36 | } 37 | 38 | // Source is the passive scraping agent 39 | type Source struct{} 40 | 41 | // Run function returns all subdomains found with the service 42 | func (s *Source) Run(ctx context.Context, domain string, session *subscraping.Session) <-chan subscraping.Result { 43 | results := make(chan subscraping.Result) 44 | 45 | go func() { 46 | defer close(results) 47 | 48 | if len(session.Keys.GitHub) == 0 { 49 | return 50 | } 51 | 52 | tokens := NewTokenManager(session.Keys.GitHub) 53 | 54 | searchURL := fmt.Sprintf("https://api.github.com/search/code?per_page=100&q=%s&sort=created&order=asc", domain) 55 | s.enumerate(ctx, searchURL, domainRegexp(domain), tokens, session, results) 56 | }() 57 | 58 | return results 59 | } 60 | 61 | func (s *Source) enumerate(ctx context.Context, searchURL string, domainRegexp *regexp.Regexp, tokens *Tokens, session *subscraping.Session, results chan subscraping.Result) { 62 | select { 63 | case <-ctx.Done(): 64 | return 65 | default: 66 | } 67 | 68 | token := tokens.Get() 69 | 70 | if token.RetryAfter > 0 { 71 | if len(tokens.pool) == 1 { 72 | gologger.Verbose().Label(s.Name()).Msgf("GitHub Search request rate limit exceeded, waiting for %d seconds before retry... \n", token.RetryAfter) 73 | time.Sleep(time.Duration(token.RetryAfter) * time.Second) 74 | } else { 75 | token = tokens.Get() 76 | } 77 | } 78 | 79 | headers := map[string]string{"Accept": "application/vnd.github.v3.text-match+json", "Authorization": "token " + token.Hash} 80 | 81 | // Initial request to GitHub search 82 | resp, err := session.Get(ctx, searchURL, "", headers) 83 | isForbidden := resp != nil && resp.StatusCode == http.StatusForbidden 84 | if err != nil && !isForbidden { 85 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 86 | session.DiscardHTTPResponse(resp) 87 | return 88 | } 89 | 90 | // Retry enumerarion after Retry-After seconds on rate limit abuse detected 91 | ratelimitRemaining, _ := strconv.ParseInt(resp.Header.Get("X-Ratelimit-Remaining"), 10, 64) 92 | if isForbidden && ratelimitRemaining == 0 { 93 | retryAfterSeconds, _ := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64) 94 | tokens.setCurrentTokenExceeded(retryAfterSeconds) 95 | resp.Body.Close() 96 | 97 | s.enumerate(ctx, searchURL, domainRegexp, tokens, session, results) 98 | } 99 | 100 | var data response 101 | 102 | // Marshall json response 103 | err = jsoniter.NewDecoder(resp.Body).Decode(&data) 104 | if err != nil { 105 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 106 | resp.Body.Close() 107 | return 108 | } 109 | 110 | resp.Body.Close() 111 | 112 | err = proccesItems(ctx, data.Items, domainRegexp, s.Name(), session, results) 113 | if err != nil { 114 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 115 | return 116 | } 117 | 118 | // Links header, first, next, last... 119 | linksHeader := linkheader.Parse(resp.Header.Get("Link")) 120 | // Process the next link recursively 121 | for _, link := range linksHeader { 122 | if link.Rel == "next" { 123 | nextURL, err := url.QueryUnescape(link.URL) 124 | if err != nil { 125 | results <- subscraping.Result{Source: s.Name(), Type: subscraping.Error, Error: err} 126 | return 127 | } 128 | s.enumerate(ctx, nextURL, domainRegexp, tokens, session, results) 129 | } 130 | } 131 | } 132 | 133 | // proccesItems procceses github response items 134 | func proccesItems(ctx context.Context, items []item, domainRegexp *regexp.Regexp, name string, session *subscraping.Session, results chan subscraping.Result) error { 135 | for _, item := range items { 136 | // find subdomains in code 137 | resp, err := session.SimpleGet(ctx, rawURL(item.HTMLURL)) 138 | if err != nil { 139 | if resp != nil && resp.StatusCode != http.StatusNotFound { 140 | session.DiscardHTTPResponse(resp) 141 | } 142 | return err 143 | } 144 | 145 | if resp.StatusCode == http.StatusOK { 146 | scanner := bufio.NewScanner(resp.Body) 147 | for scanner.Scan() { 148 | line := scanner.Text() 149 | if line == "" { 150 | continue 151 | } 152 | for _, subdomain := range domainRegexp.FindAllString(normalizeContent(line), -1) { 153 | results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} 154 | } 155 | } 156 | resp.Body.Close() 157 | } 158 | 159 | // find subdomains in text matches 160 | for _, textMatch := range item.TextMatches { 161 | for _, subdomain := range domainRegexp.FindAllString(normalizeContent(textMatch.Fragment), -1) { 162 | results <- subscraping.Result{Source: name, Type: subscraping.Subdomain, Value: subdomain} 163 | } 164 | } 165 | } 166 | return nil 167 | } 168 | 169 | // Normalize content before matching, query unescape, remove tabs and new line chars 170 | func normalizeContent(content string) string { 171 | normalizedContent, _ := url.QueryUnescape(content) 172 | normalizedContent = strings.ReplaceAll(normalizedContent, "\\t", "") 173 | normalizedContent = strings.ReplaceAll(normalizedContent, "\\n", "") 174 | return normalizedContent 175 | } 176 | 177 | // Raw URL to get the files code and match for subdomains 178 | func rawURL(htmlURL string) string { 179 | domain := strings.ReplaceAll(htmlURL, "https://github.com/", "https://raw.githubusercontent.com/") 180 | return strings.ReplaceAll(domain, "/blob/", "/") 181 | } 182 | 183 | // DomainRegexp regular expression to match subdomains in github files code 184 | func domainRegexp(domain string) *regexp.Regexp { 185 | rdomain := strings.ReplaceAll(domain, ".", "\\.") 186 | return regexp.MustCompile("(\\w[a-zA-Z0-9][a-zA-Z0-9-\\.]*)" + rdomain) 187 | } 188 | 189 | // Name returns the name of the source 190 | func (s *Source) Name() string { 191 | return "github" 192 | } 193 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🌟 Starmap 2 | 3 | ![Starmap](https://socialify.git.ci/ZhuriLab/Starmap/image?description=1&font=Inter&forks=1&issues=1&logo=https%3A%2F%2Favatars.githubusercontent.com%2Fu%2F69614236%3Fs%3D200%26v%3D4&name=1&owner=1&pattern=Circuit%20Board&pulls=1&stargazers=1&theme=Light) 4 | 5 | - [Amass](https://github.com/OWASP/Amass/) 虽然搜集的方法多，但太笨重，不方便集成，目标多了会内存爆炸 6 | - [subfinder](https://github.com/projectdiscovery/subfinder) 非常方便集成，但是只有被动的方式 7 | - [ksubdomain](https://github.com/boy-hack/ksubdomain) 仅主动爆破，以及验证 8 | 9 | 遂以 subfinder 为基础，融合 ksubdomain、 Amass 的一些优点进行二次开发的一款子域名收集工具，可以很方便作为 go 库集成进入项目中。并增加了以下功能： 10 | 11 | - 子域名接管检测功能 12 | - 网络空间扫描引擎被动获取子域时，增加ip、端口开放收集 13 | - shodan 获取 ip\端口 14 | - fofa 获取 ip\端口 15 | - zoomeyeapi 获取 ip 16 | 17 | # 🍺 Installation 18 | 下载二进制 https://github.com/ZhuriLab/Starmap/releases 19 | 20 | 安装`libpcap`环境 21 | - Windows 下载 npcap 驱动: https://npcap.com/#download (ksubdomain 推荐下载的winpcap驱动存在一点问题，我在虚拟机中跑不出任何东西，改用 npcap 驱动可以) 22 | - Linux 已经静态编译打包`libpcap`，无需其他操作 23 | - MacOS 自带`libpcap`,无需其他操作 24 | 25 | # 🔅 Usage 26 | 27 | 1. 被动模式运行并保存到文件（只保存域名） 28 | 29 | ```bash 30 | Starmap -d baidu.com -o res.txt 31 | ``` 32 | 33 | 2. 被动加主动爆破, 过滤泛解析 json 格式输出（json 输出更丰富） 34 | 35 | ```bash 36 | Starmap -d baidu.com -b -rW -oJ -o res.json 37 | ``` 38 | 39 | 3. 其他选项 40 | 41 | ```bash 42 | Flags: 43 | INPUT: 44 | -d, -domain string[] domains to find subdomains for 45 | 枚举的目标域名 46 | -dL, -list string file containing list of domains for subdomain discovery 47 | 枚举的域名列表的文件 48 | 49 | SOURCE: 50 | -s, -sources string[] specific sources to use for discovery (-s crtsh,github) 51 | 被动使用的源 52 | -recursive use only recursive sources 53 | 仅使用递归源 54 | -all Use all sources (slow) for enumeration 55 | 使用所有源进行枚举 56 | -es, -exclude-sources string[] sources to exclude from enumeration (-es archiveis,zoomeye) 57 | 被动枚举中排除使用的源列表 58 | 59 | OUTPUT: 60 | -o, -output string file to write output to 61 | 输出文件名 62 | -oJ, -json write output in JSONL(ines) format 63 | Json格式输出，该选项输出内容丰富,输出到文件需要配合 -o res.json 64 | 65 | CONFIGURATION: 66 | -config string flag config file 67 | 自定义API密钥等的配置文件位置 (default "/Users/用户名/.config/Starmap/config.yaml") 68 | -proxy string http proxy to use with subfinder 69 | 指定被动api获取子域名时的代理 70 | 71 | DEBUG: 72 | -silent show only subdomains in output 73 | 使用后屏幕将仅输出结果域名 74 | -version show version of Starmap 75 | 输出当前版本 76 | -v show verbose output 77 | 显示详细输出 78 | 79 | DNS BRUTE FORCING SUBDOMAIN: 80 | -w string Path to a different wordlist file for brute forcing 81 | dns 爆破使用的字典 82 | -ld string Multilevel subdomain dictionary(level > 2 use) 83 | dns 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认 84 | -l int Number of blasting subdomain layers 85 | 枚举几级域名，默认为二级域名 (default 2) 86 | -n int Number of DNS forced subdomains 87 | dns爆破每个域名的次数，默认跑一次 (default 1) 88 | -brute Use DNS brute forcing subdomain(default false) 89 | 被动加 dns 主动爆破(默认不使用) 90 | -verify DNS authentication survival, Export only verified domain names 91 | 验证被动获取的域名，使用后仅输出验证存活的域名 92 | -dns string DNS server, cn:China dns, in:International, all:(cn+in DNS), conf:(read ./config/Starmap/config.yaml), Select according to the target. 93 | DNS服务器，默认国内的服务器(cn)(cn: 表示使用国内的 dns, in:国外 dns，all: 全部内置 dns, conf: 从配置文件 ./config/Starmap/config.yaml获取)，根据目标选择 (default "cn") 94 | -rW, -active Domain name pan resolution filtering 95 | 爆破时过滤泛解析(default false) 96 | -mW int Number of random domain names during universal resolution detection(default len(resolvers)*2) 97 | 泛解析检测时的随机域名数量(default len(resolvers)*2) 98 | 99 | SUBDOMAIN TAKEOVER: 100 | -takeover Scan subdomain takeover (default False). 101 | 子域名接管检测 (默认：false) 102 | -sa subdomain take over: Request to test each URL (by default, only the URL matching CNAME is requested to test). 103 | 子域名接管检测：请求测试每个URL（默认情况下，仅请求测试与CNAME匹配的URL） 104 | ``` 105 | 106 | # 🎉 Starmap Go library 107 | 108 | ```go 109 | package main 110 | 111 | import ( 112 | "bytes" 113 | "context" 114 | "fmt" 115 | "github.com/ZhuriLab/Starmap/pkg/passive" 116 | "github.com/ZhuriLab/Starmap/pkg/resolve" 117 | "github.com/ZhuriLab/Starmap/pkg/runner" 118 | "io" 119 | "io/ioutil" 120 | "log" 121 | ) 122 | 123 | // 作为 go library 集成 124 | func main() { 125 | 126 | config, _ := runner.UnmarshalRead("/Users/yhy/.config/Starmap/config.yaml") 127 | 128 | config.Recursive = resolve.DefaultResolvers 129 | config.Sources = passive.DefaultSources 130 | config.AllSources = passive.DefaultAllSources 131 | config.Recursive = passive.DefaultRecursiveSources 132 | 133 | options := &runner.Options{ 134 | Threads: 10, // Thread controls the number of threads to use for active enumerations 135 | Timeout: 30, // Timeout is the seconds to wait for sources to respond 136 | MaxEnumerationTime: 10, // MaxEnumerationTime is the maximum amount of time in mins to wait for enumeration 137 | Resolvers: resolve.DefaultResolvers, // Use the default list of resolvers by marshaling it to the config 138 | Sources: passive.DefaultSources, // Use the default list of passive sources 139 | AllSources: passive.DefaultAllSources, // Use the default list of all passive sources 140 | Recursive: passive.DefaultRecursiveSources, // Use the default list of recursive sources 141 | 142 | YAMLConfig: config, // 读取自定义配置文件 143 | All: true, 144 | Verbose: false, 145 | Brute: true, 146 | Verify: true, // 验证找到的域名 147 | RemoveWildcard: true, // 泛解析过滤 148 | MaxIps: 100, // 爆破时如果超出一定数量的域名指向同一个 ip，则认为是泛解析 149 | Silent: false, // 是否为静默模式，只输出找到的域名 150 | DNS: "cn", // dns 服务器区域选择，根据目标选择不同区域得到的结果不同，国内网站的话，选择 cn，dns 爆破结果比较多 151 | BruteWordlist: "", // 爆破子域的域名字典，不填则使用内置的 152 | Level: 2, // 枚举几级域名，默认为二级域名 153 | LevelDic: "", // 枚举多级域名的字典文件，当level大于2时候使用，不填则会默认 154 | Takeover: false, // 子域名接管检测 155 | SAll: false, // 子域名接管检测中请求全部 url，默认只对匹配的 cname 进行检测 156 | } 157 | 158 | options.ConfigureOutput() 159 | 160 | runnerInstance, err := runner.NewRunner(options) 161 | 162 | buf := bytes.Buffer{} 163 | err, subdomains := runnerInstance.EnumerateSingleDomain(context.Background(), "baidu.com", []io.Writer{&buf}) 164 | if err != nil { 165 | log.Fatal(err) 166 | } 167 | 168 | 169 | data, err := ioutil.ReadAll(&buf) 170 | if err != nil { 171 | log.Fatal(err) 172 | } 173 | 174 | // 只输出域名 175 | fmt.Printf("%s", data) 176 | 177 | // 输出详细信息 178 | /* 179 | Host string `json:"host"` 180 | Source string `json:"source"` 181 | IpPorts map[string][]int `json:"ip_ports"` 182 | CNames []string `json:"cnames"` 183 | TakeOver bool `json:"take_over"` 184 | */ 185 | for _, result := range subdomains { 186 | fmt.Println(result.Source, result.Host, result.IpPorts, result.CNames, result.TakeOver) 187 | } 188 | } 189 | ``` 190 | 191 | # 📌 TODO 192 | 193 | - [ ] [Amass](https://github.com/OWASP/Amass/) 中的子域名检测技术 194 | - [x] 子域名接管检测 195 | 196 | # 💡 Tips 197 | - 指定不同的 dns ，获取到的结果会不同。比如：如果目标是国内的网站，选择国内的 dns 得到的子域名结果可能会比较多 198 | - 提示 `pcap打开失败:vnic0: You don't have permission to capture on that device ((cannot open BPF device) /dev/bpf0: Permission denied)`等错误 199 | 200 | 执行`sudo chmod 777 /dev/bpf*` 或 `sudo ./Starmap` 201 | 202 | # 👀 参考 203 | - [subfinder](https://github.com/projectdiscovery/subfinder) 204 | - [ksubdomain](https://github.com/boy-hack/ksubdomain) 205 | - [Amass](https://github.com/OWASP/Amass) 206 | 207 | # 📄 免责声明 208 | 本工具仅面向合法授权的企业安全建设行为，在使用本工具进行检测时，您应确保该行为符合当地的法律法规，并且已经取得了足够的授权。 209 | 210 | 如您在使用本工具的过程中存在任何非法行为，您需自行承担相应后果，作者将不承担任何法律及连带责任。 211 | 212 | 在使用本工具前，请您务必审慎阅读、充分理解各条款内容，限制、免责条款或者其他涉及您重大权益的条款可能会以加粗、加下划线等形式提示您重点注意。 除非您已充分阅读、完全理解并接受本协议所有条款，否则，请您不要使用本工具。您的使用行为或者您以其他任何明示或者默示方式表示接受本协议的，即视为您已阅读并同意本协议的约束。 -------------------------------------------------------------------------------- /pkg/passive/sources.go: -------------------------------------------------------------------------------- 1 | package passive 2 | 3 | import ( 4 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 5 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/alienvault" 6 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/anubis" 7 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/archiveis" 8 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/binaryedge" 9 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/bufferover" 10 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/c99" 11 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/censys" 12 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/certspotter" 13 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/chaos" 14 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/chinaz" 15 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/commoncrawl" 16 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/crtsh" 17 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/dnsdb" 18 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/dnsdumpster" 19 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/fofa" 20 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/fullhunt" 21 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/github" 22 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/hackertarget" 23 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/hunter" 24 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/intelx" 25 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/passivetotal" 26 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/quake" 27 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/rapiddns" 28 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/riddler" 29 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/robtex" 30 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/securitytrails" 31 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/shodan" 32 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/sitedossier" 33 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/sonarsearch" 34 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/spyse" 35 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/sublist3r" 36 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/threatbook" 37 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/threatcrowd" 38 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/threatminer" 39 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/virustotal" 40 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/waybackarchive" 41 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/zoomeye" 42 | "github.com/ZhuriLab/Starmap/pkg/subscraping/sources/zoomeyeapi" 43 | ) 44 | 45 | // DefaultSources contains the list of fast sources used by default. 46 | var DefaultSources = []string{ 47 | "alienvault", 48 | "anubis", 49 | "bufferover", 50 | "c99", 51 | "certspotter", 52 | "censys", 53 | "chaos", 54 | "chinaz", 55 | "crtsh", 56 | "dnsdumpster", 57 | "hackertarget", 58 | "intelx", 59 | "passivetotal", 60 | "robtex", 61 | "riddler", 62 | "securitytrails", 63 | "shodan", 64 | "spyse", 65 | "sublist3r", 66 | "threatcrowd", 67 | "threatminer", 68 | "virustotal", 69 | "fofa", 70 | "fullhunt", 71 | "quake", 72 | "hunter", 73 | } 74 | 75 | // DefaultRecursiveSources contains list of default recursive sources 76 | var DefaultRecursiveSources = []string{ 77 | "alienvault", 78 | "binaryedge", 79 | "bufferover", 80 | "certspotter", 81 | "crtsh", 82 | "dnsdumpster", 83 | "hackertarget", 84 | "passivetotal", 85 | "securitytrails", 86 | "sonarsearch", 87 | "sublist3r", 88 | "virustotal", 89 | } 90 | 91 | // DefaultAllSources contains list of all sources 92 | var DefaultAllSources = []string{ 93 | "alienvault", 94 | "anubis", 95 | "archiveis", 96 | "binaryedge", 97 | "bufferover", 98 | "c99", 99 | "censys", 100 | "certspotter", 101 | "chaos", 102 | "commoncrawl", 103 | "crtsh", 104 | "dnsdumpster", 105 | "dnsdb", 106 | "github", 107 | "hackertarget", 108 | "intelx", 109 | "passivetotal", 110 | "rapiddns", 111 | "riddler", 112 | "robtex", 113 | "securitytrails", 114 | "shodan", 115 | "sitedossier", 116 | "sonarsearch", 117 | "spyse", 118 | "sublist3r", 119 | "threatbook", 120 | "threatcrowd", 121 | "threatminer", 122 | "virustotal", 123 | "waybackarchive", 124 | "zoomeye", 125 | "zoomeyeapi", 126 | "fofa", 127 | "quake", 128 | "hunter", 129 | "fullhunt", 130 | } 131 | 132 | // Agent is a struct for running passive subdomain enumeration 133 | // against a given host. It wraps subscraping package and provides 134 | // a layer to build upon. 135 | type Agent struct { 136 | sources map[string]subscraping.Source 137 | } 138 | 139 | // New creates a new agent for passive subdomain discovery 140 | func New(sources, exclusions []string) *Agent { 141 | // Create the agent, insert the sources and remove the excluded sources 142 | agent := &Agent{sources: make(map[string]subscraping.Source)} 143 | 144 | agent.addSources(sources) 145 | agent.removeSources(exclusions) 146 | 147 | return agent 148 | } 149 | 150 | // addSources adds the given list of sources to the source array 151 | func (a *Agent) addSources(sources []string) { 152 | for _, source := range sources { 153 | switch source { 154 | case "alienvault": 155 | a.sources[source] = &alienvault.Source{} 156 | case "anubis": 157 | a.sources[source] = &anubis.Source{} 158 | case "archiveis": 159 | a.sources[source] = &archiveis.Source{} 160 | case "binaryedge": 161 | a.sources[source] = &binaryedge.Source{} 162 | case "bufferover": 163 | a.sources[source] = &bufferover.Source{} 164 | case "c99": 165 | a.sources[source] = &c99.Source{} 166 | case "censys": 167 | a.sources[source] = &censys.Source{} 168 | case "certspotter": 169 | a.sources[source] = &certspotter.Source{} 170 | case "chaos": 171 | a.sources[source] = &chaos.Source{} 172 | case "chinaz": 173 | a.sources[source] = &chinaz.Source{} 174 | case "commoncrawl": 175 | a.sources[source] = &commoncrawl.Source{} 176 | case "crtsh": 177 | a.sources[source] = &crtsh.Source{} 178 | case "dnsdumpster": 179 | a.sources[source] = &dnsdumpster.Source{} 180 | case "dnsdb": 181 | a.sources[source] = &dnsdb.Source{} 182 | case "github": 183 | a.sources[source] = &github.Source{} 184 | case "hackertarget": 185 | a.sources[source] = &hackertarget.Source{} 186 | case "intelx": 187 | a.sources[source] = &intelx.Source{} 188 | case "passivetotal": 189 | a.sources[source] = &passivetotal.Source{} 190 | case "rapiddns": 191 | a.sources[source] = &rapiddns.Source{} 192 | case "riddler": 193 | a.sources[source] = &riddler.Source{} 194 | case "robtex": 195 | a.sources[source] = &robtex.Source{} 196 | case "securitytrails": 197 | a.sources[source] = &securitytrails.Source{} 198 | case "shodan": 199 | a.sources[source] = &shodan.Source{} 200 | case "sitedossier": 201 | a.sources[source] = &sitedossier.Source{} 202 | case "sonarsearch": 203 | a.sources[source] = &sonarsearch.Source{} 204 | case "spyse": 205 | a.sources[source] = &spyse.Source{} 206 | case "sublist3r": 207 | a.sources[source] = &sublist3r.Source{} 208 | case "threatbook": 209 | a.sources[source] = &threatbook.Source{} 210 | case "threatcrowd": 211 | a.sources[source] = &threatcrowd.Source{} 212 | case "threatminer": 213 | a.sources[source] = &threatminer.Source{} 214 | case "virustotal": 215 | a.sources[source] = &virustotal.Source{} 216 | case "waybackarchive": 217 | a.sources[source] = &waybackarchive.Source{} 218 | case "zoomeye": 219 | a.sources[source] = &zoomeye.Source{} 220 | case "zoomeyeapi": 221 | a.sources[source] = &zoomeyeapi.Source{} 222 | case "fofa": 223 | a.sources[source] = &fofa.Source{} 224 | case "fullhunt": 225 | a.sources[source] = &fullhunt.Source{} 226 | case "quake": 227 | a.sources[source] = &quake.Source{} 228 | case "hunter": 229 | a.sources[source] = &hunter.Source{} 230 | } 231 | } 232 | } 233 | 234 | // removeSources deletes the given sources from the source map 235 | func (a *Agent) removeSources(sources []string) { 236 | for _, source := range sources { 237 | delete(a.sources, source) 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /pkg/runner/config.go: -------------------------------------------------------------------------------- 1 | package runner 2 | 3 | import ( 4 | "math/rand" 5 | "os" 6 | "strings" 7 | 8 | "github.com/ZhuriLab/Starmap/pkg/subscraping" 9 | "gopkg.in/yaml.v3" 10 | ) 11 | 12 | // MultipleKeyPartsLength is the max length for multiple keys 13 | const MultipleKeyPartsLength = 2 14 | 15 | // YAMLIndentCharLength number of chars for identation on write YAML to file 16 | const YAMLIndentCharLength = 4 17 | 18 | // Providers contains the providers stored in the configuration file 19 | type Providers struct { 20 | // Resolvers contains the list of resolvers to use while resolving 21 | Resolvers []string `yaml:"resolvers,omitempty"` 22 | // Sources contains a list of sources to use for enumeration 23 | Sources []string `yaml:"sources,omitempty"` 24 | // AllSources contains the list of all sources for enumeration (slow) 25 | AllSources []string `yaml:"all-sources,omitempty"` 26 | // Recrusive contains the list of recursive subdomain enum sources 27 | Recursive []string `yaml:"recursive,omitempty"` 28 | // ExcludeSources contains the sources to not include in the enumeration process 29 | ExcludeSources []string `yaml:"exclude-sources,omitempty"` 30 | // API keys for different sources 31 | Bufferover []string `yaml:"bufferover"` 32 | Binaryedge []string `yaml:"binaryedge"` 33 | C99 []string `yaml:"c99"` 34 | Censys []string `yaml:"censys"` 35 | Certspotter []string `yaml:"certspotter"` 36 | Chaos []string `yaml:"chaos"` 37 | Chinaz []string `yaml:"chinaz"` 38 | DNSDB []string `yaml:"dnsdb"` 39 | GitHub []string `yaml:"github"` 40 | IntelX []string `yaml:"intelx"` 41 | PassiveTotal []string `yaml:"passivetotal"` 42 | Robtex []string `yaml:"robtex"` 43 | SecurityTrails []string `yaml:"securitytrails"` 44 | Shodan []string `yaml:"shodan"` 45 | Spyse []string `yaml:"spyse"` 46 | ThreatBook []string `yaml:"threatbook"` 47 | URLScan []string `yaml:"urlscan"` 48 | Virustotal []string `yaml:"virustotal"` 49 | ZoomEye []string `yaml:"zoomeye"` 50 | ZoomEyeApi []string `yaml:"zoomeyeapi"` 51 | Fofa []string `yaml:"fofa"` 52 | FullHunt []string `json:"fullhunt"` 53 | Quake []string `yaml:"quake"` 54 | Hunter []string `yaml:"hunter"` 55 | // Version indicates the version of subfinder installed. 56 | Version string `yaml:"Starmap-version"` 57 | } 58 | 59 | // GetConfigDirectory gets the subfinder config directory for a user 60 | func GetConfigDirectory() (string, error) { 61 | var config string 62 | 63 | directory, err := os.UserHomeDir() 64 | if err != nil { 65 | return config, err 66 | } 67 | config = directory + "/.config/Starmap" 68 | 69 | // Create All directory for subfinder even if they exist 70 | err = os.MkdirAll(config, os.ModePerm) 71 | if err != nil { 72 | return config, err 73 | } 74 | 75 | return config, nil 76 | } 77 | 78 | // MarshalTo writes the marshaled yaml config to disk 79 | func (c *Providers) MarshalTo(file string) error { 80 | f, err := os.Create(file) 81 | if err != nil { 82 | return err 83 | } 84 | defer f.Close() 85 | 86 | return yaml.NewEncoder(f).Encode(c) 87 | } 88 | 89 | // MarshalTo writes the marshaled yaml config to disk 90 | func (c *Providers) UnmarshalFrom(file string) error { 91 | f, err := os.Open(file) 92 | if err != nil { 93 | return err 94 | } 95 | defer f.Close() 96 | 97 | return yaml.NewDecoder(f).Decode(c) 98 | } 99 | 100 | // CheckConfigExists checks if the config file exists in the given path 101 | func CheckConfigExists(configPath string) bool { 102 | if _, err := os.Stat(configPath); err == nil { 103 | return true 104 | } else if os.IsNotExist(err) { 105 | return false 106 | } 107 | return false 108 | } 109 | 110 | // MarshalWrite writes the marshaled yaml config to disk 111 | func (c *Providers) MarshalWrite(file string) error { 112 | f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0755) 113 | if err != nil { 114 | return err 115 | } 116 | 117 | // Indent the spaces too 118 | enc := yaml.NewEncoder(f) 119 | enc.SetIndent(YAMLIndentCharLength) 120 | err = enc.Encode(&c) 121 | f.Close() 122 | return err 123 | } 124 | 125 | // UnmarshalRead reads the unmarshalled config yaml file from disk 126 | func UnmarshalRead(file string) (Providers, error) { 127 | config := Providers{} 128 | 129 | f, err := os.Open(file) 130 | if err != nil { 131 | return config, err 132 | } 133 | err = yaml.NewDecoder(f).Decode(&config) 134 | f.Close() 135 | return config, err 136 | } 137 | 138 | // GetKeys gets the API keys from config file and creates a Keys struct 139 | // We use random selection of api keys from the list of keys supplied. 140 | // Keys that require 2 options are separated by colon (:). 141 | func (c *Providers) GetKeys() subscraping.Keys { 142 | keys := subscraping.Keys{} 143 | 144 | if len(c.Binaryedge) > 0 { 145 | keys.Binaryedge = c.Binaryedge[rand.Intn(len(c.Binaryedge))] 146 | } 147 | if len(c.C99) > 0 { 148 | keys.C99 = c.C99[rand.Intn(len(c.C99))] 149 | } 150 | 151 | if len(c.Bufferover) > 0 { 152 | keys.Bufferover = c.Bufferover[rand.Intn(len(c.Bufferover))] 153 | } 154 | 155 | if len(c.Censys) > 0 { 156 | censysKeys := c.Censys[rand.Intn(len(c.Censys))] 157 | parts := strings.Split(censysKeys, ":") 158 | if len(parts) == MultipleKeyPartsLength { 159 | keys.CensysToken = parts[0] 160 | keys.CensysSecret = parts[1] 161 | } 162 | } 163 | 164 | if len(c.Certspotter) > 0 { 165 | keys.Certspotter = c.Certspotter[rand.Intn(len(c.Certspotter))] 166 | } 167 | if len(c.Chaos) > 0 { 168 | keys.Chaos = c.Chaos[rand.Intn(len(c.Chaos))] 169 | } 170 | if len(c.Chinaz) > 0 { 171 | keys.Chinaz = c.Chinaz[rand.Intn(len(c.Chinaz))] 172 | } 173 | if (len(c.DNSDB)) > 0 { 174 | keys.DNSDB = c.DNSDB[rand.Intn(len(c.DNSDB))] 175 | } 176 | if (len(c.GitHub)) > 0 { 177 | keys.GitHub = c.GitHub 178 | } 179 | 180 | if len(c.IntelX) > 0 { 181 | intelxKeys := c.IntelX[rand.Intn(len(c.IntelX))] 182 | parts := strings.Split(intelxKeys, ":") 183 | if len(parts) == MultipleKeyPartsLength { 184 | keys.IntelXHost = parts[0] 185 | keys.IntelXKey = parts[1] 186 | } 187 | } 188 | 189 | if len(c.PassiveTotal) > 0 { 190 | passiveTotalKeys := c.PassiveTotal[rand.Intn(len(c.PassiveTotal))] 191 | parts := strings.Split(passiveTotalKeys, ":") 192 | if len(parts) == MultipleKeyPartsLength { 193 | keys.PassiveTotalUsername = parts[0] 194 | keys.PassiveTotalPassword = parts[1] 195 | } 196 | } 197 | 198 | if len(c.Robtex) > 0 { 199 | keys.Robtex = c.Robtex[rand.Intn(len(c.Robtex))] 200 | } 201 | 202 | if len(c.SecurityTrails) > 0 { 203 | keys.Securitytrails = c.SecurityTrails[rand.Intn(len(c.SecurityTrails))] 204 | } 205 | if len(c.Shodan) > 0 { 206 | keys.Shodan = c.Shodan[rand.Intn(len(c.Shodan))] 207 | } 208 | if len(c.Spyse) > 0 { 209 | keys.Spyse = c.Spyse[rand.Intn(len(c.Spyse))] 210 | } 211 | if len(c.ThreatBook) > 0 { 212 | keys.ThreatBook = c.ThreatBook[rand.Intn(len(c.ThreatBook))] 213 | } 214 | if len(c.URLScan) > 0 { 215 | keys.URLScan = c.URLScan[rand.Intn(len(c.URLScan))] 216 | } 217 | if len(c.Virustotal) > 0 { 218 | keys.Virustotal = c.Virustotal[rand.Intn(len(c.Virustotal))] 219 | } 220 | if len(c.ZoomEye) > 0 { 221 | zoomEyeKeys := c.ZoomEye[rand.Intn(len(c.ZoomEye))] 222 | parts := strings.Split(zoomEyeKeys, ":") 223 | if len(parts) == MultipleKeyPartsLength { 224 | keys.ZoomEyeUsername = parts[0] 225 | keys.ZoomEyePassword = parts[1] 226 | } 227 | } 228 | if len(c.ZoomEyeApi) > 0 { 229 | keys.ZoomEyeKey = c.ZoomEyeApi[rand.Intn(len(c.ZoomEyeApi))] 230 | } 231 | if len(c.Fofa) > 0 { 232 | fofaKeys := c.Fofa[rand.Intn(len(c.Fofa))] 233 | parts := strings.Split(fofaKeys, ":") 234 | if len(parts) == MultipleKeyPartsLength { 235 | keys.FofaUsername = parts[0] 236 | keys.FofaSecret = parts[1] 237 | } 238 | } 239 | if len(c.FullHunt) > 0 { 240 | keys.FullHunt = c.FullHunt[rand.Intn(len(c.FullHunt))] 241 | } 242 | 243 | if len(c.Quake) > 0 { 244 | keys.Quake = c.Quake[rand.Intn(len(c.Quake))] 245 | } 246 | 247 | if len(c.Hunter) > 0 { 248 | keys.Hunter = c.Hunter[rand.Intn(len(c.Hunter))] 249 | } 250 | 251 | return keys 252 | } 253 | -------------------------------------------------------------------------------- /pkg/net/network.go: -------------------------------------------------------------------------------- 1 | // Copyright 2017 Jeff Foley. All rights reserved. 2 | // Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file. 3 | 4 | package net 5 | 6 | import ( 7 | "bytes" 8 | "context" 9 | "math/big" 10 | "net" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | // IPv4RE is a regular expression that will match an IPv4 address. 16 | const IPv4RE = "((25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)[.]){3}(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)" 17 | 18 | // ReservedCIDRDescription is the description used for reserved address ranges. 19 | const ReservedCIDRDescription = "Reserved Network Address Blocks" 20 | 21 | // LocalAddr is the global option for specifying the network interface. 22 | var LocalAddr net.Addr 23 | 24 | // ReservedCIDRs includes all the networks that are reserved for special use. 25 | var ReservedCIDRs = []string{ 26 | "192.168.0.0/16", 27 | "172.16.0.0/12", 28 | "10.0.0.0/8", 29 | "127.0.0.0/8", 30 | "224.0.0.0/4", 31 | "240.0.0.0/4", 32 | "100.64.0.0/10", 33 | "198.18.0.0/15", 34 | "169.254.0.0/16", 35 | "192.88.99.0/24", 36 | "192.0.0.0/24", 37 | "192.0.2.0/24", 38 | "192.94.77.0/24", 39 | "192.94.78.0/24", 40 | "192.52.193.0/24", 41 | "192.12.109.0/24", 42 | "192.31.196.0/24", 43 | "192.0.0.0/29", 44 | } 45 | 46 | // The reserved network address ranges 47 | var reservedAddrRanges []*net.IPNet 48 | 49 | func init() { 50 | for _, cidr := range ReservedCIDRs { 51 | if _, ipnet, err := net.ParseCIDR(cidr); err == nil { 52 | reservedAddrRanges = append(reservedAddrRanges, ipnet) 53 | } 54 | } 55 | } 56 | 57 | // DialContext performs the dial using global variables (e.g. LocalAddr). 58 | func DialContext(ctx context.Context, network, addr string) (net.Conn, error) { 59 | d := &net.Dialer{DualStack: true} 60 | 61 | _, p, err := net.SplitHostPort(addr) 62 | if err != nil { 63 | return nil, err 64 | } 65 | 66 | port, err := strconv.Atoi(p) 67 | if err != nil { 68 | return nil, err 69 | } 70 | 71 | if LocalAddr != nil { 72 | addr, _, err := net.ParseCIDR(LocalAddr.String()) 73 | 74 | if err == nil && strings.HasPrefix(network, "tcp") { 75 | d.LocalAddr = &net.TCPAddr{ 76 | IP: addr, 77 | Port: port, 78 | } 79 | } else if err == nil && strings.HasPrefix(network, "udp") { 80 | d.LocalAddr = &net.UDPAddr{ 81 | IP: addr, 82 | Port: port, 83 | } 84 | } 85 | } 86 | 87 | return d.DialContext(ctx, network, addr) 88 | } 89 | 90 | // IsIPv4 returns true when the provided net.IP address is an IPv4 address. 91 | func IsIPv4(ip net.IP) bool { 92 | return strings.Count(ip.String(), ":") < 2 93 | } 94 | 95 | // IsIPv6 returns true when the provided net.IP address is an IPv6 address. 96 | func IsIPv6(ip net.IP) bool { 97 | return strings.Count(ip.String(), ":") >= 2 98 | } 99 | 100 | // IsReservedAddress checks if the addr parameter is within one of the address ranges in the ReservedCIDRs slice. 101 | func IsReservedAddress(addr string) (bool, string) { 102 | ip := net.ParseIP(addr) 103 | if ip == nil { 104 | return false, "" 105 | } 106 | 107 | var cidr string 108 | for _, block := range reservedAddrRanges { 109 | if block.Contains(ip) { 110 | cidr = block.String() 111 | break 112 | } 113 | } 114 | 115 | if cidr != "" { 116 | return true, cidr 117 | } 118 | return false, "" 119 | } 120 | 121 | // FirstLast return the first and last IP address of the provided CIDR/netblock. 122 | func FirstLast(cidr *net.IPNet) (net.IP, net.IP) { 123 | firstIP := cidr.IP 124 | prefixLen, bits := cidr.Mask.Size() 125 | 126 | if prefixLen == bits { 127 | lastIP := make([]byte, len(firstIP)) 128 | copy(lastIP, firstIP) 129 | return firstIP, lastIP 130 | } 131 | 132 | firstIPInt, bits := ipToInt(firstIP) 133 | hostLen := uint(bits) - uint(prefixLen) 134 | lastIPInt := big.NewInt(1) 135 | 136 | lastIPInt.Lsh(lastIPInt, hostLen) 137 | lastIPInt.Sub(lastIPInt, big.NewInt(1)) 138 | lastIPInt.Or(lastIPInt, firstIPInt) 139 | 140 | return firstIP, intToIP(lastIPInt, bits) 141 | } 142 | 143 | // Range2CIDR turns an IP range into a CIDR. 144 | func Range2CIDR(first, last net.IP) *net.IPNet { 145 | startip, m := ipToInt(first) 146 | endip, _ := ipToInt(last) 147 | newip := big.NewInt(1) 148 | mask := big.NewInt(1) 149 | one := big.NewInt(1) 150 | 151 | if startip.Cmp(endip) == 1 { 152 | return nil 153 | } 154 | 155 | max := uint(m) 156 | var bits uint = 1 157 | newip.Set(startip) 158 | tmp := new(big.Int) 159 | for bits < max { 160 | tmp.Rsh(startip, bits) 161 | tmp.Lsh(tmp, bits) 162 | 163 | newip.Or(startip, mask) 164 | if newip.Cmp(endip) == 1 || tmp.Cmp(startip) != 0 { 165 | bits-- 166 | mask.Rsh(mask, 1) 167 | break 168 | } 169 | 170 | bits++ 171 | tmp.Lsh(mask, 1) 172 | mask.Add(tmp, one) 173 | } 174 | 175 | cidrstr := first.String() + "/" + strconv.Itoa(int(max-bits)) 176 | _, ipnet, _ := net.ParseCIDR(cidrstr) 177 | 178 | return ipnet 179 | } 180 | 181 | // AllHosts returns a slice containing all the IP addresses within 182 | // the CIDR provided by the parameter. This implementation was 183 | // obtained/modified from the following: 184 | // https://gist.github.com/kotakanbe/d3059af990252ba89a82 185 | func AllHosts(cidr *net.IPNet) []net.IP { 186 | var ips []net.IP 187 | 188 | for ip := cidr.IP.Mask(cidr.Mask); cidr.Contains(ip); IPInc(ip) { 189 | addr := net.ParseIP(ip.String()) 190 | 191 | ips = append(ips, addr) 192 | } 193 | 194 | if len(ips) > 2 { 195 | // Remove network address and broadcast address 196 | ips = ips[1 : len(ips)-1] 197 | } 198 | return ips 199 | } 200 | 201 | // RangeHosts returns all the IP addresses (inclusive) between 202 | // the start and stop addresses provided by the parameters. 203 | func RangeHosts(start, end net.IP) []net.IP { 204 | var ips []net.IP 205 | 206 | if start == nil || end == nil { 207 | return ips 208 | } 209 | 210 | start16 := start.To16() 211 | end16 := end.To16() 212 | // Check that the end address is higher than the start address 213 | if r := bytes.Compare(end16, start16); r < 0 { 214 | return ips 215 | } else if r == 0 { 216 | return []net.IP{start} 217 | } 218 | 219 | stop := net.ParseIP(end.String()) 220 | IPInc(stop) 221 | 222 | for ip := net.ParseIP(start.String()); !ip.Equal(stop); IPInc(ip) { 223 | if addr := net.ParseIP(ip.String()); addr != nil { 224 | ips = append(ips, addr) 225 | } 226 | } 227 | 228 | return ips 229 | } 230 | 231 | // CIDRSubset returns a subset of the IP addresses contained within 232 | // the cidr parameter with num elements around the addr element. 233 | func CIDRSubset(cidr *net.IPNet, addr string, num int) []net.IP { 234 | first := net.ParseIP(addr) 235 | 236 | if !cidr.Contains(first) { 237 | return []net.IP{first} 238 | } 239 | 240 | offset := num / 2 241 | // Get the first address 242 | for i := 0; i < offset; i++ { 243 | IPDec(first) 244 | // Check that it is still within the CIDR 245 | if !cidr.Contains(first) { 246 | IPInc(first) 247 | break 248 | } 249 | } 250 | // Get the last address 251 | last := net.ParseIP(addr) 252 | for i := 0; i < offset; i++ { 253 | IPInc(last) 254 | // Check that it is still within the CIDR 255 | if !cidr.Contains(last) { 256 | IPDec(last) 257 | break 258 | } 259 | } 260 | // Check that the addresses are not the same 261 | if first.Equal(last) { 262 | return []net.IP{first} 263 | } 264 | // Return the IP addresses within the range 265 | return RangeHosts(first, last) 266 | } 267 | 268 | // IPInc increments the IP address provided. 269 | func IPInc(ip net.IP) { 270 | for j := len(ip) - 1; j >= 0; j-- { 271 | ip[j]++ 272 | if ip[j] > 0 { 273 | break 274 | } 275 | } 276 | } 277 | 278 | // IPDec decrements the IP address provided. 279 | func IPDec(ip net.IP) { 280 | for j := len(ip) - 1; j >= 0; j-- { 281 | if ip[j] > 0 { 282 | ip[j]-- 283 | break 284 | } 285 | ip[j]-- 286 | } 287 | } 288 | 289 | func ipToInt(ip net.IP) (*big.Int, int) { 290 | val := big.NewInt(1) 291 | 292 | val.SetBytes([]byte(ip)) 293 | if IsIPv4(ip) { 294 | return val, 32 295 | } else if IsIPv6(ip) { 296 | return val, 128 297 | } 298 | 299 | return val, 0 300 | } 301 | 302 | func intToIP(ipInt *big.Int, bits int) net.IP { 303 | ipBytes := ipInt.Bytes() 304 | ret := make([]byte, bits/8) 305 | 306 | // Pack our IP bytes into the end of the return array, 307 | // since big.Int.Bytes() removes front zero padding 308 | for i := 1; i <= len(ipBytes); i++ { 309 | ret[len(ret)-i] = ipBytes[len(ipBytes)-i] 310 | } 311 | 312 | return net.IP(ret) 313 | } 314 | --------------------------------------------------------------------------------