├── .github └── workflows │ └── build.yml ├── .gitignore ├── README.md ├── config.example.json ├── linkdeep.go ├── makefile ├── miner ├── common.go ├── config.go ├── fofa.go └── github.go └── miner_test.go /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build Releases 2 | 3 | on: 4 | release: 5 | types: [ created ] 6 | 7 | jobs: 8 | 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v2 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@v2 16 | with: 17 | go-version: 1.15 18 | 19 | - name: GetVersion 20 | id: getVersion 21 | uses: actions/github-script@v4.0.2 22 | with: 23 | script: | 24 | const version = context.ref.replace("refs/tags/", "") 25 | core.setOutput('version', version); 26 | 27 | - name: Install dependencies 28 | run: | 29 | go get -u github.com/spf13/cobra 30 | go get -u github.com/inconshreveable/mousetrap 31 | go get -u github.com/hluwa/simplethreadpool 32 | 33 | - name: Make 34 | run: make 35 | 36 | - name: Rename 37 | run: | 38 | mv dist/linkdeep_linux32 dist/linkdeep_${{ steps.getVersion.outputs.version }}_linux32 39 | mv dist/linkdeep_linux64 dist/linkdeep_${{ steps.getVersion.outputs.version }}_linux64 40 | mv dist/linkdeep_macos dist/linkdeep_${{ steps.getVersion.outputs.version }}_macos 41 | mv dist/linkdeep_win32.exe dist/linkdeep_${{ steps.getVersion.outputs.version }}_win32.exe 42 | mv dist/linkdeep_win64.exe dist/linkdeep_${{ steps.getVersion.outputs.version }}_win64.exe 43 | 44 | - name: Publish 45 | uses: softprops/action-gh-release@v1 46 | if: startsWith(github.ref, 'refs/tags/') 47 | with: 48 | files: | 49 | dist/linkdeep_${{ steps.getVersion.outputs.version }}_linux32 50 | dist/linkdeep_${{ steps.getVersion.outputs.version }}_linux64 51 | dist/linkdeep_${{ steps.getVersion.outputs.version }}_macos 52 | dist/linkdeep_${{ steps.getVersion.outputs.version }}_win32.exe 53 | dist/linkdeep_${{ steps.getVersion.outputs.version }}_win64.exe 54 | env: 55 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 56 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | config.json 2 | output.txt 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## LinkDeep 2 | 3 | LinkDeep is a useful tool for discover deeplink from internet. Enter the uri or facts(scheme, host, path) to generate 4 | uri start deeplink discovering from internet big-data. 5 | 6 | ### Usage: 7 | 8 | linkdeep [target] [flags] 9 | 10 | ### Flags: 11 | 12 | ``` 13 | -c, --config string config file path (default "config.json") 14 | -h, --help help for linkdeep 15 | --host string host for deeplink 16 | -o, --output string output file path (default "output.txt") 17 | --path string path for deeplink 18 | -x, --proxy string proxy for global default 19 | --proxy-fofa string proxy for fofa 20 | --scheme string scheme for deeplink 21 | ``` 22 | -------------------------------------------------------------------------------- /config.example.json: -------------------------------------------------------------------------------- 1 | { 2 | "fofa": { 3 | "email": "xxxx@xxx.com", 4 | "key": "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 5 | "maxCount": 500, 6 | "threadCount": 10, 7 | "proxy": "http://127.0.0.1:8888/" 8 | }, 9 | "proxy": "" 10 | } -------------------------------------------------------------------------------- /linkdeep.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./miner" 5 | "errors" 6 | "fmt" 7 | "github.com/spf13/cobra" 8 | "io/ioutil" 9 | "log" 10 | "strings" 11 | ) 12 | 13 | var ( 14 | miners = []func(string) ([]string, error){miner.GithubMiner, miner.FofaMiner} 15 | proxyFofa = "" 16 | proxy = "" 17 | scheme = "" 18 | host = "" 19 | path = "" 20 | output = "" 21 | command = &cobra.Command{ 22 | Use: "linkdeep [target]", 23 | Short: "Automation discovering from public internet for deeplink.", 24 | Long: `LinkDeep is a useful tool for discover deeplink from internet. 25 | Enter the uri or facts(scheme, host, path) to generate uri 26 | start deeplink discovering from internet big-data.`, 27 | RunE: linkdeep, 28 | Args: func(cmd *cobra.Command, args []string) error { 29 | if len(args) < 1 && scheme == "" && host == "" && path == "" { 30 | return errors.New("requires input target uri or set flags for uri fact") 31 | } 32 | return nil 33 | }, 34 | } 35 | ) 36 | 37 | func initConfig() { 38 | config := miner.GetConfig() 39 | if proxy != "" { 40 | config.Proxy = proxy 41 | } 42 | if proxyFofa != "" { 43 | config.Fofa.Proxy = proxyFofa 44 | } 45 | } 46 | 47 | func generateUri() (uri string) { 48 | uri = "" 49 | if scheme != "" { 50 | uri = fmt.Sprintf("%s://", scheme) 51 | } 52 | if host != "" { 53 | if uri == "" { 54 | uri = "://" 55 | } 56 | uri = fmt.Sprintf("%s%s/", uri, host) 57 | if path != "" { 58 | uri = fmt.Sprintf("%s%s", uri, path) 59 | } 60 | } else if path != "" { 61 | uri = fmt.Sprintf("%s/%s", uri, path) 62 | } 63 | return 64 | } 65 | 66 | func linkdeep(_ *cobra.Command, args []string) error { 67 | initConfig() 68 | 69 | var uri string 70 | if len(args) > 0 { 71 | uri = args[0] 72 | } else { 73 | uri = generateUri() 74 | } 75 | 76 | log.Printf("Start mining for %s\n", uri) 77 | var links []string 78 | for _, m := range miners { 79 | l, e := m(uri) 80 | if e != nil { 81 | log.Printf("[E] %s\n", e) 82 | } 83 | if len(l) > 0 { 84 | links = append(links, l...) 85 | } 86 | } 87 | 88 | for _, links := range links { 89 | log.Println(links) 90 | } 91 | if output != "" { 92 | log.Printf("Save %d link into %s\n", len(links), output) 93 | return ioutil.WriteFile(output, []byte(strings.Join(links, "\n")), 0777) 94 | } 95 | 96 | return nil 97 | } 98 | 99 | func initCommand() { 100 | command.Flags().StringVarP(&miner.ConfigPath, "config", "c", "config.json", "config file path") 101 | command.Flags().StringVarP(&proxyFofa, "proxy-fofa", "", "", "proxy for fofa") 102 | command.Flags().StringVarP(&proxy, "proxy", "x", "", "proxy for global default") 103 | command.Flags().StringVarP(&scheme, "scheme", "", "", "scheme for deeplink") 104 | command.Flags().StringVarP(&host, "host", "", "", "host for deeplink") 105 | command.Flags().StringVarP(&path, "path", "", "", "path for deeplink") 106 | command.Flags().StringVarP(&output, "output", "o", "output.txt", "output file path") 107 | } 108 | 109 | func main() { 110 | initCommand() 111 | _ = command.Execute() 112 | } 113 | -------------------------------------------------------------------------------- /makefile: -------------------------------------------------------------------------------- 1 | source = linkdeep.go miner/common.go miner/config.go miner/fofa.go 2 | 3 | all: dist 4 | 5 | dist: dist/linkdeep_linux64 dist/linkdeep_linux32 dist/linkdeep_win64.exe dist/linkdeep_win32.exe dist/linkdeep_macos 6 | 7 | dist/linkdeep_linux64: $(source) 8 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o dist/linkdeep_linux64 9 | 10 | dist/linkdeep_linux32: $(source) 11 | CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -o dist/linkdeep_linux32 12 | 13 | dist/linkdeep_win64.exe: $(source) 14 | CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o dist/linkdeep_win64.exe 15 | 16 | dist/linkdeep_win32.exe: $(source) 17 | CGO_ENABLED=0 GOOS=windows GOARCH=386 go build -o dist/linkdeep_win32.exe 18 | 19 | dist/linkdeep_macos: $(source) 20 | CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o dist/linkdeep_macos 21 | 22 | clean: 23 | rm -rf dist -------------------------------------------------------------------------------- /miner/common.go: -------------------------------------------------------------------------------- 1 | package miner 2 | 3 | import ( 4 | "io/ioutil" 5 | "net/http" 6 | "net/url" 7 | "regexp" 8 | "strings" 9 | "sync" 10 | ) 11 | 12 | var httpClient *http.Client 13 | var httpMu sync.Mutex 14 | 15 | func GetHttpClient() *http.Client { 16 | httpMu.Lock() 17 | defer httpMu.Unlock() 18 | if httpClient == nil { 19 | proxy := func(req *http.Request) (*url.URL, error) { 20 | proxy := GetConfig().Proxy 21 | if strings.HasSuffix(req.URL.Host, "fofa.so") { 22 | proxy = GetConfig().GetFofaProxy() 23 | } else if strings.HasSuffix(req.URL.Host, "github.com") { 24 | proxy = GetConfig().GetGithubProxy() 25 | } 26 | if proxy != "" { 27 | return url.Parse(proxy) 28 | } 29 | return nil, nil 30 | } 31 | transport := &http.Transport{Proxy: proxy} 32 | httpClient = &http.Client{Transport: transport} 33 | } 34 | return httpClient 35 | } 36 | 37 | func SimpleGet(u string) (body []byte, err error) { 38 | return CustomGet(u, nil) 39 | } 40 | 41 | func CustomGet(u string, preFunc func(r *http.Request)) (body []byte, err error) { 42 | client := GetHttpClient() 43 | req, err := http.NewRequest("GET", u, nil) 44 | if err != nil { 45 | return 46 | } 47 | 48 | if preFunc != nil { 49 | preFunc(req) 50 | } 51 | 52 | resp, err := client.Do(req) 53 | if err != nil { 54 | return 55 | } 56 | return ioutil.ReadAll(resp.Body) 57 | } 58 | 59 | var re *regexp.Regexp 60 | 61 | func init() { 62 | re, _ = regexp.Compile("[a-zA-Z0-9\\-_]+://[-A-Za-z0-9+&@#/%?=~_|!:,.;]+[-A-Za-z0-9+&@#/%=~_|]") 63 | } 64 | 65 | func MatchLinks(content string, prefix string) (links []string) { 66 | matches := re.FindAllString(content, -1) 67 | for _, s := range matches { 68 | if strings.Contains(s, prefix) { 69 | links = append(links, s) 70 | } 71 | } 72 | return links 73 | } 74 | 75 | func RemoveRep(slc []string) []string { 76 | var result []string 77 | tempMap := map[string]byte{} 78 | for _, e := range slc { 79 | l := len(tempMap) 80 | tempMap[e] = 0 81 | if len(tempMap) != l { 82 | result = append(result, e) 83 | } 84 | } 85 | return result 86 | } 87 | -------------------------------------------------------------------------------- /miner/config.go: -------------------------------------------------------------------------------- 1 | package miner 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "sync" 8 | ) 9 | 10 | var config *Config 11 | var mu sync.Mutex 12 | 13 | type MinerConfig struct { 14 | MaxCount int `json:"maxCount"` 15 | ThreadCount int `json:"threadCount"` 16 | Proxy string `json:"proxy"` 17 | } 18 | 19 | type Config struct { 20 | Fofa struct { 21 | MinerConfig 22 | Email string `json:"email"` 23 | Key string `json:"key"` 24 | } `json:"fofa"` 25 | Github struct { 26 | MinerConfig 27 | Token string `json:"token"` 28 | } `json:"github"` 29 | Proxy string `json:"proxy"` 30 | } 31 | 32 | func (c *Config) GetFofaProxy() string { 33 | if c.Fofa.Proxy != "" { 34 | return c.Fofa.Proxy 35 | } else { 36 | return c.Proxy 37 | } 38 | } 39 | 40 | func (c *Config) GetGithubProxy() string { 41 | if c.Github.Proxy != "" { 42 | return c.Github.Proxy 43 | } else { 44 | return c.Proxy 45 | } 46 | } 47 | 48 | var ConfigPath = "config.json" 49 | 50 | func GetConfig() *Config { 51 | mu.Lock() 52 | defer mu.Unlock() 53 | if config == nil { 54 | content, err := ioutil.ReadFile(ConfigPath) 55 | if err != nil { 56 | panic(fmt.Sprintf("Unable read config.json: %s", err)) 57 | } 58 | config = NewConfig() 59 | err = json.Unmarshal(content, config) 60 | if err != nil { 61 | panic(fmt.Sprintf("Unable unmarshal config: %s", err)) 62 | } 63 | 64 | if config.Fofa.MaxCount > 10000 { 65 | config.Fofa.MaxCount = 10000 66 | } 67 | } 68 | return config 69 | } 70 | 71 | func NewConfig() *Config { 72 | return &Config{ 73 | Fofa: struct { 74 | MinerConfig 75 | Email string `json:"email"` 76 | Key string `json:"key"` 77 | }{ 78 | MinerConfig: MinerConfig{ 79 | MaxCount: 100, 80 | ThreadCount: 5, 81 | }, 82 | }, 83 | Github: struct { 84 | MinerConfig 85 | Token string `json:"token"` 86 | }{ 87 | MinerConfig: MinerConfig{ 88 | MaxCount: 1000, 89 | ThreadCount: 20, 90 | }, 91 | }, 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /miner/fofa.go: -------------------------------------------------------------------------------- 1 | package miner 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/hluwa/simplethreadpool" 9 | "html" 10 | "log" 11 | "sync" 12 | ) 13 | 14 | type FofaResponse struct { 15 | Error bool `json:"error"` 16 | Errmsg string `json:"errmsg"` 17 | } 18 | 19 | type FofaSearchResult struct { 20 | FofaResponse 21 | Mode string `json:"mode"` 22 | Error bool `json:"error"` 23 | Query string `json:"query"` 24 | Page int `json:"page"` 25 | Size int `json:"size"` 26 | Results []string `json:"results"` 27 | } 28 | 29 | func FofaSearchBody(keyword string) (hosts []string, err error) { 30 | qBase64 := base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("body=\"%s\"", keyword))) 31 | maxCount := GetConfig().Fofa.MaxCount 32 | if maxCount == 0 { 33 | maxCount = 100 34 | } 35 | url := fmt.Sprintf("https://fofa.info/api/v1/search/all?fields=host&full=true&qbase64=%s&email=%s&key=%s&size=%d", 36 | qBase64, GetConfig().Fofa.Email, GetConfig().Fofa.Key, maxCount) 37 | respBody, err := SimpleGet(url) 38 | if err != nil { 39 | return 40 | } 41 | var result FofaSearchResult 42 | err = json.Unmarshal(respBody, &result) 43 | if err != nil { 44 | return 45 | } 46 | 47 | if result.Error { 48 | err = errors.New(result.Errmsg) 49 | return 50 | } 51 | 52 | log.Printf("[*] FOFA found %d item, return %d hosts.\n", result.Size, len(result.Results)) 53 | hosts = result.Results 54 | return 55 | } 56 | 57 | func FofaFetchContent(host string) (content string, err error) { 58 | url := fmt.Sprintf("https://fofa.info/result/website?host=%s", host) 59 | respBody, err := SimpleGet(url) 60 | if err != nil { 61 | return 62 | } 63 | 64 | return string(respBody), nil 65 | } 66 | 67 | func handleContent(content string) string { 68 | return html.UnescapeString(content) 69 | } 70 | 71 | func FofaMiner(format string) (links []string, err error) { 72 | config := GetConfig().Fofa 73 | log.Printf("[*] Starting discover from FOFA, threadCount=%d, maxCount=%d\n", config.ThreadCount, config.MaxCount) 74 | hosts, err := FofaSearchBody(format) 75 | if err != nil { 76 | return 77 | } 78 | log.Printf("[*] FOFA found %d host.\n", len(hosts)) 79 | 80 | var mu sync.Mutex 81 | makeF := func(host string) func() { 82 | return func() { 83 | content, err := FofaFetchContent(host) 84 | if err != nil { 85 | log.Printf("[*] (%s) fetch content failed as %s.\n", host, err) 86 | } else { 87 | content = handleContent(content) 88 | l := MatchLinks(content, format) 89 | log.Printf("[*] (%s) fetch content size %d, matched %d links\n", host, len(content), len(l)) 90 | mu.Lock() 91 | defer mu.Unlock() 92 | links = append(links, l...) 93 | } 94 | } 95 | } 96 | pool := simplethreadpool.NewSimpleThreadPool(config.ThreadCount) 97 | for _, host := range hosts { 98 | pool.Put(makeF(host)) 99 | } 100 | pool.Sync() 101 | 102 | links = RemoveRep(links) 103 | log.Printf("[*] FOFA found %d link.\n", len(links)) 104 | return 105 | } 106 | -------------------------------------------------------------------------------- /miner/github.go: -------------------------------------------------------------------------------- 1 | package miner 2 | 3 | import ( 4 | "encoding/base64" 5 | "encoding/json" 6 | "errors" 7 | "fmt" 8 | "github.com/hluwa/simplethreadpool" 9 | "log" 10 | "net/http" 11 | "net/url" 12 | "strconv" 13 | "sync" 14 | ) 15 | 16 | type GithubAuthResponse struct { 17 | Message string `json:"message"` 18 | } 19 | 20 | type GithubSearchResponse struct { 21 | TotalCount int `json:"total_count"` 22 | IncompleteResults bool `json:"incomplete_results"` 23 | Items []struct { 24 | Url string `json:"url"` 25 | } `json:"items"` 26 | } 27 | 28 | type GithubContentResponse struct { 29 | Content string `json:"content"` 30 | Encoding string `json:"encoding"` 31 | } 32 | 33 | func GithubGet(u string) ([]byte, error) { 34 | return CustomGet(u, func(r *http.Request) { 35 | r.Header.Set("Authorization", fmt.Sprintf("token %s", GetConfig().Github.Token)) 36 | }) 37 | } 38 | 39 | func GithubAuth() error { 40 | content, err := GithubGet("https://api.github.com") 41 | if err != nil { 42 | return err 43 | } 44 | var resp GithubAuthResponse 45 | err = json.Unmarshal(content, &resp) 46 | if err != nil { 47 | return err 48 | } 49 | 50 | if resp.Message != "" { 51 | log.Printf("[*] Github auth failed at %s\n", resp.Message) 52 | return err 53 | } 54 | 55 | return nil 56 | } 57 | 58 | func GithubSearch(query string) (result []string, err error) { 59 | page := 0 60 | failed := 0 61 | for ; len(result) < GetConfig().Github.MaxCount; page++ { 62 | perPage := GetConfig().Github.MaxCount - len(result) 63 | if perPage > 100 { 64 | perPage = 100 65 | } 66 | params := url.Values{} 67 | params.Set("q", query) 68 | params.Set("per_page", strconv.Itoa(perPage)) 69 | params.Set("page", strconv.Itoa(page)) 70 | params.Set("sort", "") 71 | content, err := GithubGet(fmt.Sprintf("https://api.github.com/search/code?%s", params.Encode())) 72 | if err != nil { 73 | if failed > 5 { 74 | break 75 | } else { 76 | failed++ 77 | continue 78 | } 79 | 80 | } 81 | var resp GithubSearchResponse 82 | err = json.Unmarshal(content, &resp) 83 | if err != nil || len(resp.Items) <= 0 { 84 | break 85 | } 86 | 87 | for _, item := range resp.Items { 88 | result = append(result, item.Url) 89 | } 90 | 91 | if resp.IncompleteResults { 92 | break 93 | } 94 | } 95 | return result, err 96 | } 97 | 98 | func GithubGetContent(url string) (content string, err error) { 99 | body, err := GithubGet(url) 100 | if err != nil { 101 | return 102 | } 103 | 104 | var resp GithubContentResponse 105 | err = json.Unmarshal(body, &resp) 106 | if err != nil { 107 | return 108 | } 109 | 110 | if resp.Content == "" { 111 | return "", errors.New(fmt.Sprintf("cannot fetch content, body at %s", string(body))) 112 | } 113 | 114 | if resp.Encoding != "base64" { 115 | return "", errors.New(fmt.Sprintf("unimplemented encoding at %s", resp.Encoding)) 116 | } 117 | 118 | body, err = base64.StdEncoding.DecodeString(resp.Content) 119 | if err != nil { 120 | return 121 | } 122 | 123 | return string(body), nil 124 | } 125 | 126 | func GithubMiner(format string) (links []string, err error) { 127 | if err = GithubAuth(); err != nil { 128 | return nil, errors.New(fmt.Sprintf("github is cannot auth at %s", err)) 129 | } 130 | config := GetConfig().Github 131 | log.Printf("[*] Starting discover from Github, threadCount=%d, maxCount=%d\n", config.ThreadCount, config.MaxCount) 132 | urls, err := GithubSearch(fmt.Sprintf("\"%s\"", format)) 133 | if err != nil { 134 | return 135 | } 136 | 137 | var mu sync.Mutex 138 | makeF := func(u string) func() { 139 | return func() { 140 | content, err := GithubGetContent(u) 141 | if err != nil { 142 | log.Printf("[*] (%s) fetch content failed as %s.\n", u, err) 143 | } else { 144 | l := MatchLinks(content, format) 145 | log.Printf("[*] (%s) fetch content size %d, matched %d links\n", u, len(content), len(l)) 146 | mu.Lock() 147 | defer mu.Unlock() 148 | links = append(links, l...) 149 | } 150 | } 151 | } 152 | pool := simplethreadpool.NewSimpleThreadPool(GetConfig().Github.ThreadCount) 153 | for _, u := range urls { 154 | pool.Put(makeF(u)) 155 | } 156 | pool.Sync() 157 | links = RemoveRep(links) 158 | log.Printf("[*] Github found %d link.\n", len(links)) 159 | return 160 | } 161 | -------------------------------------------------------------------------------- /miner_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "./miner" 5 | "fmt" 6 | "testing" 7 | ) 8 | 9 | func Mining(name string, f func(string) ([]string, error), t *testing.T) ([]string, error) { 10 | links, e := f("tencent://message/") 11 | if e != nil { 12 | t.Fatal(e) 13 | } 14 | 15 | if len(links) <= 0 { 16 | t.Fatal(fmt.Sprintf("Unable discover from %s.", name)) 17 | } 18 | return links, e 19 | } 20 | 21 | func TestFofa(t *testing.T) { 22 | _, _ = Mining("FOFA", miner.FofaMiner, t) 23 | //for _, links := range links { 24 | // t.Log(links) 25 | //} 26 | } 27 | 28 | func TestGithub(t *testing.T) { 29 | _, _ = Mining("Github", miner.GithubMiner, t) 30 | //for _, links := range links { 31 | // t.Log(links) 32 | //} 33 | } 34 | --------------------------------------------------------------------------------