├── .gitignore ├── assets ├── terminal.png └── tld_chase.gif ├── .github └── workflows │ └── release.yml ├── cmd └── subchase │ ├── subchase_test.go │ └── subchase.go ├── LICENSE ├── go.mod ├── README.md └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | sites_cache 2 | video.tape 3 | -------------------------------------------------------------------------------- /assets/terminal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/w0ltage/subchase/HEAD/assets/terminal.png -------------------------------------------------------------------------------- /assets/tld_chase.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/w0ltage/subchase/HEAD/assets/tld_chase.gif -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release subchase binaries 2 | 3 | on: 4 | release: 5 | types: [created] 6 | 7 | jobs: 8 | releases-matrix: 9 | name: Release Go Binary 10 | runs-on: ubuntu-latest 11 | strategy: 12 | matrix: 13 | goos: [linux, darwin, windows] 14 | goarch: [amd64] 15 | steps: 16 | - uses: actions/checkout@v3 17 | - name: Run tests 18 | run: go test -v -p=1 -timeout=0 ./... 19 | - uses: wangyoucao577/go-release-action@v1.38 20 | with: 21 | github_token: ${{ secrets.GITHUB_TOKEN }} 22 | goos: ${{ matrix.goos }} 23 | goarch: ${{ matrix.goarch }} 24 | project_path: "./cmd/subchase/" 25 | binary_name: "subchase" 26 | ldflags: "-s -w" 27 | extra_files: LICENSE README.md 28 | -------------------------------------------------------------------------------- /cmd/subchase/subchase_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "reflect" 5 | "testing" 6 | ) 7 | 8 | func Test_processFoundDomains(t *testing.T) { 9 | 10 | t.Run("Bring to lowercase, remove duplicates and schemes", func(t *testing.T) { 11 | got := []string{"patent.google.com", "FONTS.gOOgle.com", "support.google.com", "https://support.google.com"} 12 | result := processFoundDomains(got) 13 | 14 | // Convert got slice to []reflect.Value 15 | var expected []reflect.Value 16 | for _, str := range got { 17 | expected = append(expected, reflect.ValueOf(str)) 18 | } 19 | 20 | if reflect.DeepEqual(result, expected) { 21 | t.Errorf("Got %v, expected %v, result %v", got, expected, result) 22 | } 23 | }) 24 | 25 | t.Run("Passed empty []string slice", func(t *testing.T) { 26 | got := []string{} 27 | result := processFoundDomains(got) 28 | 29 | if len(result) != 0 { 30 | t.Errorf("Got %v, expected length %v, result length of slice %v", got, 0, len(result)) 31 | } 32 | }) 33 | } 34 | 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 tokiakasu 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/tokiakasu/subchase 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/gocolly/colly v1.2.0 7 | github.com/leaanthony/spinner v0.5.4 8 | ) 9 | 10 | require ( 11 | github.com/PuerkitoBio/goquery v1.8.1 // indirect 12 | github.com/andybalholm/cascadia v1.3.1 // indirect 13 | github.com/antchfx/htmlquery v1.3.0 // indirect 14 | github.com/antchfx/xmlquery v1.3.16 // indirect 15 | github.com/antchfx/xpath v1.2.4 // indirect 16 | github.com/fatih/color v1.7.0 // indirect 17 | github.com/gobwas/glob v0.2.3 // indirect 18 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 19 | github.com/golang/protobuf v1.3.1 // indirect 20 | github.com/kennygrant/sanitize v1.2.4 // indirect 21 | github.com/leaanthony/synx v0.1.0 // indirect 22 | github.com/leaanthony/wincursor v0.1.0 // indirect 23 | github.com/mattn/go-colorable v0.0.9 // indirect 24 | github.com/mattn/go-isatty v0.0.4 // indirect 25 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect 26 | github.com/temoto/robotstxt v1.1.2 // indirect 27 | golang.org/x/net v0.7.0 // indirect 28 | golang.org/x/sys v0.5.0 // indirect 29 | golang.org/x/text v0.7.0 // indirect 30 | google.golang.org/appengine v1.6.7 // indirect 31 | ) 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | terminal 3 |
4 |

5 | 6 |

7 | Notes • 8 | Installation • 9 | Usage • 10 | Running subchase • 11 | To-Do functionality 12 |

13 | 14 | `subchase` is a subdomain discovery tool that returns (almost always) valid subdomains for websites by analyzing search results from Google and Yandex search engines. The goal of `subchase` is not to find all subdomains, but to find a few subdomains that were not found by other tools. 15 | 16 | # Notes 17 | 18 | - There are false positives in the results. Methods to filter results have not yet been implemented. 19 | - The results may vary from run to run. 20 | - This is usually due to captchas that cannot be bypassed, and the frequency of which cannot be predicted. 21 | 22 | # Installation 23 | 24 | Install `go v1.20`, then add `$HOME/go/bin` to PATH and run the following command to get the repo 25 | 26 | ```sh 27 | go install -v github.com/tokiakasu/subchase/cmd/subchase@latest 28 | ``` 29 | 30 | # Usage 31 | 32 | ```sh 33 | $ subchase -h 34 | 35 | Usage of subchase: 36 | -d string 37 | Specify the domain whose subdomains to look for (ex: -d google.com) 38 | -json 39 | Ouput as JSON 40 | -silent 41 | Remove startup banner 42 | ``` 43 | 44 | # Running subchase 45 | 46 | To run the tool on a target, just use the following command. 47 | 48 | ```console 49 | $ subchase -d google.com 50 | __ __ 51 | _______ __/ /_ _____/ /_ ____ _________ 52 | / ___/ / / / __ \/ ___/ __ \/ __ `/ ___/ _ \ 53 | (__ ) /_/ / /_/ / /__/ / / / /_/ (__ ) __/ 54 | /____/\__,_/_.___/\___/_/ /_/\__,_/____/\___/ v0.1.0 55 | 56 | earthengine.google.com 57 | meet.google.com 58 | classroom.google.com 59 | passwords.google.com 60 | cloud.google.com 61 | jibe.google.com 62 | books.google.com 63 | messages.google.com 64 | adsense.google.com 65 | sites.google.com 66 | images.google.com 67 | support.google.com 68 | careers.google.com 69 | ads.google.com 70 | store.google.com 71 | checks.google.com 72 | asia.google.com 73 | firebase.google.com 74 | accounts.google.com 75 | mydevices.google.com 76 | myactivity.google.com 77 | mymaps.google.com 78 | atap.google.com 79 | forms.google.com 80 | admin.sites.google.com 81 | about.artsandculture.google.com 82 | ipv4.google.com 83 | ipv6.google.com 84 | assistant.google.com 85 | fonts.google.com 86 | ``` 87 | 88 | ## Chase subdomains by top-level domain (TLD) 89 | 90 | Specify only a TLD instead of a subdomain 91 | 92 |

93 | top-level_domain_chase 94 |
95 |

96 | 97 | # To-Do functionality 98 | 99 | - [x] Add option to output results in JSON 100 | - [ ] Add option to output content-length along with domains 101 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/PuerkitoBio/goquery v1.8.1 h1:uQxhNlArOIdbrH1tr0UXwdVFgDcZDrZVdcpygAcwmWM= 2 | github.com/PuerkitoBio/goquery v1.8.1/go.mod h1:Q8ICL1kNUJ2sXGoAhPGUdYDJvgQgHzJsnnd3H7Ho5jQ= 3 | github.com/andybalholm/cascadia v1.3.1 h1:nhxRkql1kdYCc8Snf7D5/D3spOX+dBgjA6u8x004T2c= 4 | github.com/andybalholm/cascadia v1.3.1/go.mod h1:R4bJ1UQfqADjvDa4P6HZHLh/3OxWWEqc0Sk8XGwHqvA= 5 | github.com/antchfx/htmlquery v1.3.0 h1:5I5yNFOVI+egyia5F2s/5Do2nFWxJz41Tr3DyfKD25E= 6 | github.com/antchfx/htmlquery v1.3.0/go.mod h1:zKPDVTMhfOmcwxheXUsx4rKJy8KEY/PU6eXr/2SebQ8= 7 | github.com/antchfx/xmlquery v1.3.16 h1:OCevguHq93z9Y4vb9xpRmU4Cc9lMVoiMkMbBNZVDeBM= 8 | github.com/antchfx/xmlquery v1.3.16/go.mod h1:Afkq4JIeXut75taLSuI31ISJ/zeq+3jG7TunF7noreA= 9 | github.com/antchfx/xpath v1.2.3/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= 10 | github.com/antchfx/xpath v1.2.4 h1:dW1HB/JxKvGtJ9WyVGJ0sIoEcqftV3SqIstujI+B9XY= 11 | github.com/antchfx/xpath v1.2.4/go.mod h1:i54GszH55fYfBmoZXapTHN8T8tkcHfRgLyVwwqzXNcs= 12 | github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= 13 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= 15 | github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= 16 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 17 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 18 | github.com/gocolly/colly v1.2.0 h1:qRz9YAn8FIH0qzgNUw+HT9UN7wm1oF9OBAilwEWpyrI= 19 | github.com/gocolly/colly v1.2.0/go.mod h1:Hof5T3ZswNVsOHYmba1u03W65HDWgpV5HifSuueE0EA= 20 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= 21 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= 22 | github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= 23 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 24 | github.com/kennygrant/sanitize v1.2.4 h1:gN25/otpP5vAsO2djbMhF/LQX6R7+O1TB4yv8NzpJ3o= 25 | github.com/kennygrant/sanitize v1.2.4/go.mod h1:LGsjYYtgxbetdg5owWB2mpgUL6e2nfw2eObZ0u0qvak= 26 | github.com/leaanthony/spinner v0.5.4 h1:XA2ElQgqwCg3gkTR6bJD+amKZ/VU0Ou94Vhv3W+GQug= 27 | github.com/leaanthony/spinner v0.5.4/go.mod h1:oHlrvWicr++CVV7ALWYi+qHk/XNA91D9IJ48IqmpVUo= 28 | github.com/leaanthony/synx v0.1.0 h1:R0lmg2w6VMb8XcotOwAe5DLyzwjLrskNkwU7LLWsyL8= 29 | github.com/leaanthony/synx v0.1.0/go.mod h1:Iz7eybeeG8bdq640iR+CwYb8p+9EOsgMWghkSRyZcqs= 30 | github.com/leaanthony/wincursor v0.1.0 h1:Dsyp68QcF5cCs65AMBmxoYNEm0n8K7mMchG6a8fYxf8= 31 | github.com/leaanthony/wincursor v0.1.0/go.mod h1:7TVwwrzSH/2Y9gLOGH+VhA+bZhoWXBRgbGNTMk+yimE= 32 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 33 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 34 | github.com/mattn/go-isatty v0.0.4 h1:bnP0vzxcAdeI1zdubAl5PjU6zsERjGZb7raWodagDYs= 35 | github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 36 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 37 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 38 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d h1:hrujxIzL1woJ7AwssoOcM/tq5JjjG2yYOc8odClEiXA= 39 | github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d/go.mod h1:uugorj2VCxiV1x+LzaIdVa9b4S4qGAcH6cbhh4qVxOU= 40 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 41 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 42 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 43 | github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fxg= 44 | github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= 45 | github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= 46 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 47 | golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= 48 | golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= 49 | golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= 50 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 51 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 52 | golang.org/x/net v0.0.0-20210916014120-12bc252f5db8/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 53 | golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= 54 | golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= 55 | golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= 56 | golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= 57 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 58 | golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 59 | golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 60 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 61 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 62 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 63 | golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 64 | golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 65 | golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 66 | golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 67 | golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= 68 | golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 69 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 70 | golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= 71 | golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= 72 | golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= 73 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 74 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 75 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 76 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 77 | golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= 78 | golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 79 | golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= 80 | golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= 81 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 82 | golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 83 | golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= 84 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 85 | google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= 86 | google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= 87 | -------------------------------------------------------------------------------- /cmd/subchase/subchase.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "crypto/tls" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "log" 9 | "net/http" 10 | "net/url" 11 | "os" 12 | "reflect" 13 | "strings" 14 | "time" 15 | 16 | "github.com/gocolly/colly" 17 | // "github.com/gocolly/colly/debug" 18 | "github.com/gocolly/colly/extensions" 19 | "github.com/leaanthony/spinner" 20 | ) 21 | 22 | // The `void` type is defined as an empty struct. 23 | // It is used as the value type for the map (`set`) to create 24 | // a set-like data structure where only unique elements are stored. 25 | type void struct{} 26 | 27 | type OutputJSON struct { 28 | Domains []string `json:"domains"` 29 | } 30 | 31 | const codename = "subchase" 32 | const version = "v0.3.0" 33 | 34 | func main() { 35 | var givenDomain string 36 | var quiet bool 37 | var jsonFlag bool 38 | 39 | flag.StringVar(&givenDomain, "d", "", "Specify the domain whose subdomains to look for (ex: -d google.com)") 40 | flag.BoolVar(&quiet, "silent", false, "Remove startup banner") 41 | flag.BoolVar(&jsonFlag, "json", false, "Output as JSON") 42 | flag.Parse() 43 | 44 | if !quiet { 45 | showBanner() 46 | } 47 | 48 | if givenDomain == "" { 49 | log.Printf("No domain is passed to '-d' option\n\n") 50 | flag.Usage() 51 | os.Exit(1) 52 | } 53 | 54 | // Collect domains from search engines into []string 55 | rawDomains := findDomains(givenDomain) 56 | 57 | if len(rawDomains) == 0 { 58 | log.Printf("No subdomains of %q was found", givenDomain) 59 | } 60 | 61 | // Bring elements in rawDomains slice to lower case 62 | // + remove duplicates and schemes 63 | domains := processFoundDomains(rawDomains) 64 | 65 | if jsonFlag { 66 | data := sliceToJSON(domains) 67 | fmt.Println(string(data)) 68 | } else { 69 | // Iterate through slice of unique domains 70 | for i := 0; i < len(domains); i++ { 71 | domain := domains[i] 72 | fmt.Println(domain.Interface()) 73 | } 74 | } 75 | } 76 | 77 | func findDomains(givenDomain string) []string { 78 | var domains []string 79 | 80 | loading_spinner := spinner.New("Collecting domains from Google and Yandex") 81 | loading_spinner.Start() 82 | 83 | // Instantiate default collector 84 | collector := colly.NewCollector( 85 | colly.Async(true), 86 | // colly.CacheDir("./sites_cache"), 87 | // colly.Debugger(&debug.LogDebugger{}), 88 | colly.DetectCharset(), 89 | colly.UserAgent("Mozilla/5.0 (X11; Linux x86_64; rv:109.0) Gecko/20100101 Firefox/98.0"), 90 | ) 91 | 92 | collector.Limit(&colly.LimitRule{ 93 | Parallelism: 2, 94 | RandomDelay: 5 * time.Second, 95 | }) 96 | 97 | // Setting the max TLS version to 1.2 98 | // Without specifying the maximum version of TLS 1.2, 99 | // requests get a response "403 Forbidden". 100 | collector.WithTransport(&http.Transport{ 101 | TLSClientConfig: &tls.Config{ 102 | MaxVersion: tls.VersionTLS12, 103 | }, 104 | }) 105 | 106 | // Referer sets valid Referer HTTP header to requests 107 | extensions.Referer(collector) 108 | // extensions.RandomUserAgent(collector) 109 | 110 | // Add headers to requests to imitate Firefox 111 | collector.OnRequest(func(r *colly.Request) { 112 | r.Headers.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8") 113 | r.Headers.Add("Accept-Language", "en-US,en;q=0.5") 114 | r.Headers.Add("Accept-Encoding", "gzip") 115 | r.Headers.Add("DNT", "1") 116 | r.Headers.Add("Connection", "keep-alive") 117 | r.Headers.Add("Upgrade-Insecure-Requests", "1") 118 | r.Headers.Add("Sec-Fetch-Dest", "document") 119 | r.Headers.Add("Sec-Fetch-Mode", "navigate") 120 | r.Headers.Add("Sec-Fetch-Site", "same-origin") 121 | r.Headers.Add("Sec-Fetch-User", "?1") 122 | }) 123 | 124 | // Set error handler 125 | collector.OnError(func(r *colly.Response, err error) { 126 | if r.StatusCode == http.StatusTooManyRequests { 127 | message := fmt.Sprintf("Google got tired of requests and started replying %q.\nRestart %q after a couple of minutes.", err, codename) 128 | loading_spinner.Error(message) 129 | } else { 130 | message := fmt.Sprintln("Request URL:", r.Request.URL, "failed with response:", r, "\nError:", err) 131 | loading_spinner.Error(message) 132 | } 133 | }) 134 | 135 | // Extract domains from Google search results 136 | collector.OnHTML("#center_col cite", func(e *colly.HTMLElement) { 137 | domSelection := e.DOM 138 | link := domSelection.Contents().First().Text() 139 | 140 | if strings.Contains(link, givenDomain) { 141 | message := fmt.Sprintf("Found %q", link) 142 | loading_spinner.UpdateMessage(message) 143 | 144 | domains = append(domains, link) 145 | } 146 | }) 147 | 148 | // Find and visit next Google search results page 149 | collector.OnHTML("#pnnext[href]", func(e *colly.HTMLElement) { 150 | link := e.Attr("href") 151 | e.Request.Visit(link) 152 | }) 153 | 154 | // Extract domains from Yandex search results 155 | collector.OnHTML("a.Link.Link_theme_outer", func(e *colly.HTMLElement) { 156 | link := e.Attr("href") 157 | message := fmt.Sprintf("Found %q", link) 158 | loading_spinner.UpdateMessage(message) 159 | 160 | domains = append(domains, link) 161 | }) 162 | 163 | // Find and visit next Yandex search results page 164 | collector.OnHTML(".Pager-Item_type_next", func(e *colly.HTMLElement) { 165 | link := e.Attr("href") 166 | e.Request.Visit(link) 167 | }) 168 | 169 | // Checks for YandexSmartCaptcha 170 | collector.OnHTML("#checkbox-captcha-form", func(e *colly.HTMLElement) { 171 | loading_spinner.UpdateMessage("Yandex captured us with SmartCaptcha :(") 172 | }) 173 | 174 | googleQuery := "https://www.google.com/search?q=site:*." + givenDomain 175 | collector.Visit(googleQuery) 176 | 177 | // Yandex sucks at search by TLD 178 | if strings.ContainsAny(".", givenDomain) { 179 | yandexQuery := "https://yandex.com/search/?text=site:" + givenDomain + "&lr=100" 180 | 181 | collector.Visit(yandexQuery + "&lang=en") 182 | collector.Visit(yandexQuery + "&lang=ru") 183 | } else { 184 | loading_spinner.UpdateMessage("Search by TLD detected. Switching to Google only.") 185 | } 186 | 187 | collector.Wait() 188 | 189 | loading_spinner.UpdateMessage("Finished") 190 | loading_spinner.Success() 191 | 192 | return domains 193 | } 194 | 195 | // Bring domains to lowercase 196 | // and remove duplicates + schemes 197 | func processFoundDomains(domains []string) []reflect.Value { 198 | set := make(map[string]void) 199 | 200 | for _, element := range domains { 201 | element = strings.ToLower(element) 202 | 203 | if strings.Contains(element, "http") { 204 | u, _ := url.Parse(element) 205 | set[u.Host] = void{} 206 | 207 | } else { 208 | set[element] = void{} 209 | } 210 | } 211 | 212 | result := reflect.ValueOf(set).MapKeys() 213 | return result 214 | } 215 | 216 | func showBanner() { 217 | fmt.Printf( 218 | ` __ __ 219 | _______ __/ /_ _____/ /_ ____ _________ 220 | / ___/ / / / __ \/ ___/ __ \/ __ %c/ ___/ _ \ 221 | (__ ) /_/ / /_/ / /__/ / / / /_/ (__ ) __/ 222 | /____/\__,_/_.___/\___/_/ /_/\__,_/____/\___/ %v 223 | 224 | `, '`', version) 225 | } 226 | 227 | func sliceToJSON(values []reflect.Value) []byte { 228 | var strings []string 229 | 230 | for _, value := range values { 231 | strings = append(strings, value.String()) 232 | } 233 | 234 | outputJSON := OutputJSON{ 235 | Domains: strings, 236 | } 237 | 238 | jsonData, err := json.Marshal(outputJSON) 239 | if err != nil { 240 | log.Println("Error marhaling to JSON:", err) 241 | os.Exit(1) 242 | } 243 | 244 | return jsonData 245 | } 246 | --------------------------------------------------------------------------------