├── .github ├── FUNDING.yml └── workflows │ └── release.yaml ├── .gitignore ├── demo.gif ├── internal └── app │ └── scout │ └── version │ └── version.go ├── pkg ├── wordlist │ ├── default.go │ ├── wordlist.go │ ├── reader.go │ ├── file.go │ └── readcloser.go └── scan │ ├── vhost_scanner_test.go │ ├── vhost_options.go │ ├── url_options.go │ ├── url_scanner_test.go │ ├── vhost_scanner.go │ └── url_scanner.go ├── Makefile ├── cmd └── scout │ ├── main.go │ ├── version.go │ ├── root.go │ ├── vhost.go │ └── url.go ├── scripts ├── import-wordlists.sh ├── build.sh └── install.sh ├── .goreleaser.yaml ├── go.mod ├── assets └── vhost.txt ├── LICENSE ├── README.md └── go.sum /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: [liamg] 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /scout 2 | /.idea 3 | /bin 4 | -------------------------------------------------------------------------------- /demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/liamg/scout/HEAD/demo.gif -------------------------------------------------------------------------------- /internal/app/scout/version/version.go: -------------------------------------------------------------------------------- 1 | package version 2 | 3 | var Version = "0.0.0" 4 | -------------------------------------------------------------------------------- /pkg/wordlist/default.go: -------------------------------------------------------------------------------- 1 | package wordlist 2 | 3 | func Default() Wordlist { 4 | return nil 5 | } 6 | -------------------------------------------------------------------------------- /pkg/wordlist/wordlist.go: -------------------------------------------------------------------------------- 1 | package wordlist 2 | 3 | type Wordlist interface { 4 | Next() (string, error) 5 | } 6 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | default: build 3 | 4 | build: 5 | ./scripts/build.sh 6 | 7 | test: 8 | GO111MODULE=on go test -v -race -timeout 30m ./... 9 | 10 | import-wordlists: 11 | ./scripts/import-wordlists.sh 12 | -------------------------------------------------------------------------------- /cmd/scout/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | 10 | if err := rootCmd.Execute(); err != nil { 11 | fmt.Println(err) 12 | os.Exit(1) 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /pkg/wordlist/reader.go: -------------------------------------------------------------------------------- 1 | package wordlist 2 | 3 | import ( 4 | "io" 5 | "io/ioutil" 6 | ) 7 | 8 | // FromReader creates a wordlist from a reader 9 | func FromReader(r io.Reader) Wordlist { 10 | return FromReadCloser(ioutil.NopCloser(r)) 11 | } 12 | -------------------------------------------------------------------------------- /cmd/scout/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/spf13/cobra" 5 | ) 6 | 7 | var versionCmd = &cobra.Command{ 8 | Use: "version", 9 | Short: "Display scout version.", 10 | Run: func(cmd *cobra.Command, args []string) { 11 | 12 | }, 13 | } 14 | 15 | func init() { 16 | rootCmd.AddCommand(versionCmd) 17 | } 18 | -------------------------------------------------------------------------------- /scripts/import-wordlists.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -x 4 | 5 | go install github.com/go-bindata/go-bindata/go-bindata@latest 6 | go-bindata -o internal/app/scout/data/wordlists.go assets/ 7 | cp internal/app/scout/data/wordlists.go internal/app/scout/data/wordlists.go.old 8 | sed -e 's/package main/package data/g' internal/app/scout/data/wordlists.go.old > internal/app/scout/data/wordlists.go 9 | 10 | rm internal/app/scout/data/wordlists.go.old -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | build: 10 | name: releasing 11 | runs-on: ubuntu-22.04 12 | 13 | steps: 14 | - uses: actions/checkout@v3 15 | with: 16 | fetch-depth: 0 17 | 18 | - uses: actions/setup-go@v3 19 | with: 20 | go-version: "1.19" 21 | 22 | - uses: goreleaser/goreleaser-action@v3 23 | with: 24 | version: latest 25 | args: release --rm-dist 26 | env: 27 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | builds: 2 | - id: scout 3 | main: ./cmd/scout 4 | binary: scout 5 | ldflags: 6 | - "-X github.com/liamg/scout/internal/app/scout/version.Version=${TAG}" 7 | goos: 8 | - linux 9 | - darwin 10 | - windows 11 | goarch: 12 | - "amd64" 13 | - "arm64" 14 | changelog: 15 | sort: asc 16 | filters: 17 | exclude: 18 | - "^docs:" 19 | - "^test:" 20 | 21 | archives: 22 | - format: binary 23 | name_template: "{{ .Binary}}-{{ .Os }}-{{ .Arch }}" 24 | 25 | release: 26 | prerelease: auto 27 | github: 28 | owner: liamg 29 | name: scout -------------------------------------------------------------------------------- /pkg/wordlist/file.go: -------------------------------------------------------------------------------- 1 | package wordlist 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | ) 8 | 9 | type ReaderWordlist struct { 10 | handle io.ReadCloser 11 | scanner *bufio.Scanner 12 | } 13 | 14 | // FromFile creates a WordList from a file on disk. The file should includes words separated by new lines. 15 | func FromFile(path string) (Wordlist, error) { 16 | fileWordlist := &ReaderWordlist{} 17 | var err error 18 | fileWordlist.handle, err = os.Open(path) 19 | if err != nil { 20 | return nil, err 21 | } 22 | fileWordlist.scanner = bufio.NewScanner(fileWordlist.handle) 23 | return fileWordlist, err 24 | } 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/liamg/scout 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/avast/retry-go v2.4.3+incompatible 7 | github.com/liamg/tml v0.2.0 8 | github.com/sirupsen/logrus v1.4.2 9 | github.com/spf13/cobra v0.0.5 10 | github.com/stretchr/testify v1.3.0 11 | ) 12 | 13 | require ( 14 | github.com/davecgh/go-spew v1.1.1 // indirect 15 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 16 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | github.com/spf13/pflag v1.0.3 // indirect 19 | golang.org/x/sys v0.2.0 // indirect 20 | ) 21 | -------------------------------------------------------------------------------- /pkg/wordlist/readcloser.go: -------------------------------------------------------------------------------- 1 | package wordlist 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | ) 7 | 8 | // FromReadCloser creates a wordlist from a readcloser 9 | func FromReadCloser(rc io.ReadCloser) Wordlist { 10 | fileWordlist := &ReaderWordlist{} 11 | fileWordlist.handle = rc 12 | fileWordlist.scanner = bufio.NewScanner(fileWordlist.handle) 13 | return fileWordlist 14 | } 15 | 16 | func (fw *ReaderWordlist) Next() (string, error) { 17 | if !fw.scanner.Scan() { 18 | defer func() { _ = fw.handle.Close() }() 19 | if err := fw.scanner.Err(); err != nil { 20 | return "", err 21 | } 22 | return "", io.EOF 23 | } 24 | return fw.scanner.Text(), nil 25 | } 26 | -------------------------------------------------------------------------------- /assets/vhost.txt: -------------------------------------------------------------------------------- 1 | a 2 | account 3 | accounts 4 | admin 5 | admin-dev 6 | alpha 7 | alpha-www 8 | api 9 | app 10 | archive 11 | b 12 | backup 13 | beta 14 | beta-www 15 | blog 16 | c 17 | chat 18 | clients 19 | code 20 | d 21 | dev 22 | development 23 | dev-www 24 | download 25 | e 26 | email 27 | f 28 | g 29 | h 30 | i 31 | j 32 | k 33 | l 34 | local 35 | m 36 | mail 37 | mobile 38 | n 39 | o 40 | p 41 | panel 42 | q 43 | r 44 | s 45 | sandbox 46 | secret 47 | secure 48 | staging 49 | staging-www 50 | status 51 | t 52 | test 53 | testing 54 | test-www 55 | u 56 | uat 57 | upload 58 | v 59 | v1 60 | v2 61 | v3 62 | v4 63 | v5 64 | v6 65 | v7 66 | v8 67 | v9 68 | w 69 | web 70 | www 71 | www2 72 | www3 73 | x 74 | y 75 | z -------------------------------------------------------------------------------- /scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BINARY=scout 3 | TAG=${TRAVIS_TAG:-development} 4 | GO111MODULE=on 5 | mkdir -p bin/darwin 6 | GOOS=darwin GOARCH=amd64 go build -o bin/darwin/${BINARY}-darwin-amd64 -ldflags "-X github.com/liamg/scout/internal/app/scout/version.Version=${TAG}" ./cmd/scout/ 7 | GOOS=darwin GOARCH=arm64 go build -o bin/darwin/${BINARY}-darwin-arm64 -ldflags "-X github.com/liamg/scout/internal/app/scout/version.Version=${TAG}" ./cmd/scout/ 8 | mkdir -p bin/linux 9 | GOOS=linux GOARCH=amd64 go build -o bin/linux/${BINARY}-linux-amd64 -ldflags "-X github.com/liamg/scout/internal/app/scout/version.Version=${TAG}" ./cmd/scout/ 10 | mkdir -p bin/windows 11 | GOOS=windows GOARCH=amd64 go build -o bin/windows/${BINARY}-windows-amd64.exe -ldflags "-X github.com/liamg/scout/internal/app/scout/version.Version=${TAG}" ./cmd/scout/ -------------------------------------------------------------------------------- /scripts/install.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | echo "Determining platform..." 6 | platform=$(uname | tr '[:upper:]' '[:lower:]') 7 | echo "Finding latest release..." 8 | asset=$(curl --silent https://api.github.com/repos/liamg/scout/releases/latest | jq -r ".assets[] | select(.name | contains(\"${platform}\")) | .url") 9 | echo "Downloading latest release for your platform..." 10 | curl -s -L -H "Accept: application/octet-stream" "${asset}" --output ./scout 11 | echo "Installing scout..." 12 | chmod +x ./scout 13 | installdir="${HOME}/bin/" 14 | if [ "$EUID" -eq 0 ]; then 15 | installdir="/usr/local/bin/" 16 | fi 17 | mkdir -p $installdir 18 | mv ./scout "${installdir}/scout" 19 | which scout &> /dev/null || (echo "Please add ${installdir} to your PATH to complete installation!" && exit 1) 20 | echo "Installation complete!" 21 | -------------------------------------------------------------------------------- /pkg/scan/vhost_scanner_test.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "net/url" 7 | "strconv" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestVHOSTScanner(t *testing.T) { 16 | 17 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 18 | 19 | switch r.Host { 20 | case "site.eg", "admin.site.eg": 21 | w.WriteHeader(http.StatusOK) 22 | default: 23 | w.WriteHeader(http.StatusNotFound) 24 | } 25 | })) 26 | defer server.Close() 27 | 28 | parsed, err := url.Parse(server.URL) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | parts := strings.Split(parsed.Host, ":") 34 | 35 | host := parts[0] 36 | port, err := strconv.Atoi(parts[1]) 37 | if err != nil { 38 | t.Error(err) 39 | } 40 | 41 | scanner := NewVHOSTScanner(&VHOSTOptions{ 42 | BaseDomain: "site.eg", 43 | IP: host, 44 | Port: port, 45 | Parallelism: 1, 46 | }) 47 | 48 | results, err := scanner.Scan() 49 | if err != nil { 50 | t.Fatal(err) 51 | } 52 | 53 | require.Equal(t, len(results), 1) 54 | assert.Equal(t, results[0], "admin.site.eg") 55 | 56 | } 57 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /pkg/scan/vhost_options.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "bytes" 5 | "time" 6 | 7 | "github.com/liamg/scout/internal/app/scout/data" 8 | "github.com/liamg/scout/pkg/wordlist" 9 | ) 10 | 11 | type VHOSTOptions struct { 12 | BaseDomain string // target url 13 | Timeout time.Duration // http request timeout 14 | Parallelism int // parallel routines 15 | ResultChan chan VHOSTResult // chan to return results on - otherwise will be returned in slice 16 | BusyChan chan string // chan to use to update current job 17 | Wordlist wordlist.Wordlist 18 | SkipSSLVerification bool 19 | UseSSL bool 20 | IP string 21 | Port int 22 | ContentHashing bool 23 | } 24 | 25 | type VHOSTResult struct { 26 | VHOST string 27 | StatusCode int 28 | } 29 | 30 | var DefaultVHOSTOptions = VHOSTOptions{ 31 | Timeout: time.Second * 5, 32 | Parallelism: 10, 33 | } 34 | 35 | func (opt *VHOSTOptions) Inherit() { 36 | if opt.Timeout == 0 { 37 | opt.Timeout = DefaultVHOSTOptions.Timeout 38 | } 39 | if opt.Parallelism == 0 { 40 | opt.Parallelism = DefaultVHOSTOptions.Parallelism 41 | } 42 | if opt.Wordlist == nil { 43 | wordlistBytes, err := data.Asset("assets/vhost.txt") 44 | if err != nil { 45 | wordlistBytes = []byte{} 46 | } 47 | opt.Wordlist = wordlist.FromReader(bytes.NewReader(wordlistBytes)) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /cmd/scout/root.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "strconv" 7 | 8 | "github.com/liamg/scout/internal/app/scout/version" 9 | "github.com/spf13/cobra" 10 | ) 11 | 12 | var parallelism = 10 13 | var noColours = false 14 | var wordlistPath string 15 | var debug bool 16 | var skipSSLVerification bool 17 | var positiveStatusCodes = []int{ 18 | http.StatusOK, 19 | http.StatusBadRequest, 20 | http.StatusInternalServerError, 21 | http.StatusMethodNotAllowed, 22 | http.StatusNoContent, 23 | http.StatusUnauthorized, 24 | http.StatusForbidden, 25 | http.StatusFound, 26 | http.StatusMovedPermanently, 27 | } 28 | 29 | var rootCmd = &cobra.Command{ 30 | Use: "scout", 31 | Short: "Scout is a portable URL fuzzer and spider", 32 | Long: `A fast and portable url fuzzer and spider - see https://github.com/liamg/scout for more information`, 33 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 34 | fmt.Printf(` 35 | __ 36 | ______________ __ __/ /_ 37 | / ___/ ___/ __ \/ / / / __/ %s 38 | (__ ) /__/ /_/ / /_/ / /_ http://github.com/liamg/scout 39 | /____/\___/\____/\__,_/\__/ 40 | 41 | `, version.Version) 42 | }, 43 | } 44 | 45 | func init() { 46 | for _, code := range positiveStatusCodes { 47 | statusCodes = append(statusCodes, strconv.Itoa(code)) 48 | } 49 | 50 | rootCmd.PersistentFlags().IntVarP(¶llelism, "parallelism", "r", parallelism, "Parallel routines to use for sending requests.") 51 | rootCmd.PersistentFlags().BoolVarP(&noColours, "no-colours", "n", noColours, "Disable coloured output.") 52 | rootCmd.PersistentFlags().StringVarP(&wordlistPath, "wordlist", "w", wordlistPath, "Path to wordlist file. If this is not specified an internal wordlist will be used.") 53 | rootCmd.PersistentFlags().BoolVarP(&debug, "debug", "d", debug, "Enable debug logging.") 54 | rootCmd.PersistentFlags().BoolVarP(&skipSSLVerification, "skip-ssl-verify", "k", skipSSLVerification, "Skip SSL certificate verification.") 55 | } 56 | -------------------------------------------------------------------------------- /pkg/scan/url_options.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "net/url" 5 | "strings" 6 | "time" 7 | 8 | "github.com/liamg/scout/pkg/wordlist" 9 | ) 10 | 11 | type URLOption func(s *URLScanner) 12 | 13 | // WithTargetURL sets the url to initiate scans from 14 | func WithTargetURL(target url.URL) URLOption { 15 | return func(s *URLScanner) { 16 | s.targetURL = target 17 | } 18 | } // target url 19 | 20 | // WithProxy sets the url to initiate scans from 21 | func WithProxy(proxy *url.URL) URLOption { 22 | return func(s *URLScanner) { 23 | s.proxy = proxy 24 | } 25 | } // target url 26 | 27 | // WithPositiveStatusCodes provides status codes that indicate the existence of a file/directory 28 | func WithPositiveStatusCodes(codes []int) URLOption { 29 | return func(s *URLScanner) { 30 | s.positiveStatusCodes = codes 31 | } 32 | } 33 | 34 | // WithNegativeLengths provides lengths which should be ignored 35 | func WithNegativeLengths(lengths []int) URLOption { 36 | return func(s *URLScanner) { 37 | s.negativeLengths = lengths 38 | } 39 | } 40 | 41 | func WithTimeout(timeout time.Duration) URLOption { 42 | return func(s *URLScanner) { 43 | s.timeout = timeout 44 | } 45 | } // http request timeout 46 | 47 | func WithParallelism(routines int) URLOption { 48 | return func(s *URLScanner) { 49 | s.parallelism = routines 50 | } 51 | } // parallel routines 52 | 53 | func WithResultChan(c chan URLResult) URLOption { 54 | return func(s *URLScanner) { 55 | s.resultChan = c 56 | } 57 | } // chan to return results on - otherwise will be returned in slice 58 | 59 | func WithBusyChan(c chan string) URLOption { 60 | return func(s *URLScanner) { 61 | s.busyChan = c 62 | } 63 | } // chan to use to update current job 64 | 65 | func WithWordlist(w wordlist.Wordlist) URLOption { 66 | return func(s *URLScanner) { 67 | s.words = w 68 | } 69 | } 70 | 71 | // you must include the . 72 | func WithExtensions(extensions []string) URLOption { 73 | return func(s *URLScanner) { 74 | s.extensions = extensions 75 | } 76 | } 77 | func WithIncludeNoExtension(includeNoExtension bool) URLOption { 78 | return func(s *URLScanner) { 79 | s.includeNoExtension = includeNoExtension 80 | } 81 | } 82 | 83 | func WithFilename(filename string) URLOption { 84 | return func(s *URLScanner) { 85 | s.filename = filename 86 | } 87 | } 88 | func WithSpidering(spider bool) URLOption { 89 | return func(s *URLScanner) { 90 | s.enableSpidering = spider 91 | } 92 | } 93 | 94 | func WithSkipSSLVerification(skipSSL bool) URLOption { 95 | return func(s *URLScanner) { 96 | s.skipSSLVerification = skipSSL 97 | } 98 | } 99 | func WithBackupExtensions(backupExtensions []string) URLOption { 100 | return func(s *URLScanner) { 101 | s.backupExtensions = backupExtensions 102 | } 103 | } 104 | 105 | func WithExtraHeaders(headers []string) URLOption { 106 | return func(s *URLScanner) { 107 | s.extraHeaders = append(s.extraHeaders, headers...) 108 | } 109 | } 110 | 111 | func WithMethod(method string) URLOption { 112 | return func(s *URLScanner) { 113 | s.method = strings.ToUpper(method) 114 | } 115 | } 116 | 117 | type URLResult struct { 118 | URL url.URL 119 | StatusCode int 120 | Size int 121 | } 122 | -------------------------------------------------------------------------------- /pkg/scan/url_scanner_test.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "bytes" 5 | "net/http" 6 | "net/http/httptest" 7 | "net/url" 8 | "testing" 9 | 10 | "github.com/liamg/scout/pkg/wordlist" 11 | 12 | "github.com/stretchr/testify/assert" 13 | "github.com/stretchr/testify/require" 14 | ) 15 | 16 | func TestURLScanner(t *testing.T) { 17 | 18 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 19 | switch r.URL.Path { 20 | case "/login.php": 21 | w.WriteHeader(http.StatusOK) 22 | default: 23 | w.WriteHeader(http.StatusNotFound) 24 | } 25 | })) 26 | defer server.Close() 27 | 28 | parsed, err := url.Parse(server.URL) 29 | if err != nil { 30 | t.Fatal(err) 31 | } 32 | 33 | options := []URLOption{ 34 | WithTargetURL(*parsed), 35 | WithParallelism(2), 36 | WithWordlist(wordlist.FromReader(bytes.NewReader([]byte("login.php\nsomething.php")))), 37 | } 38 | 39 | scanner := NewURLScanner(options...) 40 | 41 | results, err := scanner.Scan() 42 | if err != nil { 43 | t.Fatal(err) 44 | } 45 | 46 | require.Equal(t, 1, len(results)) 47 | assert.Equal(t, results[0].String(), server.URL+"/login.php") 48 | 49 | } 50 | 51 | func TestURLScannerWithRedirects(t *testing.T) { 52 | 53 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 54 | switch r.URL.Path { 55 | case "/very-secret-file.php": 56 | w.WriteHeader(http.StatusOK) 57 | case "/login.php": 58 | http.Redirect(w, r, "/very-secret-file.php", http.StatusFound) 59 | default: 60 | w.WriteHeader(http.StatusNotFound) 61 | } 62 | })) 63 | defer server.Close() 64 | 65 | parsed, err := url.Parse(server.URL) 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | 70 | options := []URLOption{ 71 | WithTargetURL(*parsed), 72 | WithParallelism(1), 73 | WithWordlist(wordlist.FromReader(bytes.NewReader([]byte("login.php")))), 74 | WithPositiveStatusCodes([]int{http.StatusOK}), 75 | } 76 | 77 | scanner := NewURLScanner(options...) 78 | 79 | results, err := scanner.Scan() 80 | if err != nil { 81 | t.Fatal(err) 82 | } 83 | 84 | require.Equal(t, 1, len(results)) 85 | assert.Equal(t, results[0].String(), server.URL+"/very-secret-file.php") 86 | 87 | } 88 | 89 | func TestURLScannerWithBackupFile(t *testing.T) { 90 | 91 | server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 92 | switch r.URL.Path { 93 | case "/login.php": 94 | w.WriteHeader(http.StatusOK) 95 | case "/login.php~": 96 | w.WriteHeader(http.StatusOK) 97 | default: 98 | w.WriteHeader(http.StatusNotFound) 99 | } 100 | })) 101 | defer server.Close() 102 | 103 | parsed, err := url.Parse(server.URL) 104 | if err != nil { 105 | t.Fatal(err) 106 | } 107 | 108 | options := []URLOption{ 109 | WithTargetURL(*parsed), 110 | WithParallelism(1), 111 | WithWordlist(wordlist.FromReader(bytes.NewReader([]byte("login")))), 112 | } 113 | 114 | scanner := NewURLScanner(options...) 115 | 116 | results, err := scanner.Scan() 117 | if err != nil { 118 | t.Fatal(err) 119 | } 120 | 121 | require.Equal(t, 2, len(results)) 122 | assert.Equal(t, results[0].String(), server.URL+"/login.php") 123 | assert.Equal(t, results[1].String(), server.URL+"/login.php~") 124 | 125 | } 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scout 2 | 3 | [![Travis Build Status](https://travis-ci.org/liamg/scout.svg?branch=master)](https://travis-ci.org/liamg/scout) 4 | 5 | Scout is a URL fuzzer and spider for discovering undisclosed VHOSTS, files and directories on a web server. 6 | 7 |

8 | 9 |

10 | 11 | A full word list is included in the binary, meaning maximum portability and minimal configuration. Aim and fire! 12 | 13 | ## Usage 14 | 15 | ```bash 16 | 17 | Usage: 18 | scout [command] 19 | 20 | Available Commands: 21 | help Help about any command 22 | url Discover URLs on a given web server. 23 | version Display scout version. 24 | vhost Discover VHOSTs on a given web server. 25 | 26 | Flags: 27 | -d, --debug Enable debug logging. 28 | -h, --help help for scout 29 | -n, --no-colours Disable coloured output. 30 | -p, --parallelism int Parallel routines to use for sending requests. (default 10) 31 | -k, --skip-ssl-verify Skip SSL certificate verification. 32 | -w, --wordlist string Path to wordlist file. If this is not specified an internal wordlist will be used. 33 | 34 | ``` 35 | 36 | ### Discover URLs 37 | 38 | #### Flags 39 | 40 | ##### `-x, --extensions` 41 | 42 | File extensions to detect. (default `php,htm,html,txt`]) 43 | 44 | ##### `-f, --filename` 45 | 46 | Filename to seek in the directory being searched. Useful when all directories report 404 status. 47 | 48 | ##### `-H, --header` 49 | 50 | Extra header to send with requests e.g. `-H "Cookie: PHPSESSID=blah"` 51 | 52 | ##### `-c, --status-codes` 53 | 54 | HTTP status codes which indicate a positive find. (default `200,400,403,500,405,204,401,301,302`) 55 | 56 | ##### `-m, --method` 57 | 58 | HTTP method to use. 59 | 60 | ##### `-s, --spider` 61 | 62 | Scan page content for links and confirm their existence. 63 | 64 | #### Full example 65 | 66 | ```bash 67 | $ scout url http://192.168.1.1 68 | 69 | [+] Target URL http://192.168.1.1 70 | [+] Routines 10 71 | [+] Extensions php,htm,html 72 | [+] Positive Codes 200,302,301,400,403,500,405,204,401,301,302 73 | 74 | [302] http://192.168.1.1/css 75 | [302] http://192.168.1.1/js 76 | [302] http://192.168.1.1/language 77 | [302] http://192.168.1.1/style 78 | [302] http://192.168.1.1/help 79 | [401] http://192.168.1.1/index.htm 80 | [302] http://192.168.1.1/image 81 | [200] http://192.168.1.1/log.htm 82 | [302] http://192.168.1.1/script 83 | [401] http://192.168.1.1/top.html 84 | [200] http://192.168.1.1/shares 85 | [200] http://192.168.1.1/shares.php 86 | [200] http://192.168.1.1/shares.htm 87 | [200] http://192.168.1.1/shares.html 88 | [401] http://192.168.1.1/traffic.htm 89 | [401] http://192.168.1.1/reboot.htm 90 | [302] http://192.168.1.1/debug 91 | [401] http://192.168.1.1/debug.htm 92 | [401] http://192.168.1.1/debug.html 93 | [401] http://192.168.1.1/start.htm 94 | 95 | Scan complete. 28 results found. 96 | 97 | ``` 98 | 99 | ### Discover VHOSTs 100 | 101 | ```bash 102 | $ scout vhost https://google.com 103 | 104 | [+] Base Domain google.com 105 | [+] Routines 10 106 | [+] IP - 107 | [+] Port - 108 | [+] Using SSL true 109 | 110 | account.google.com 111 | accounts.google.com 112 | blog.google.com 113 | code.google.com 114 | dev.google.com 115 | local.google.com 116 | m.google.com 117 | mail.google.com 118 | mobile.google.com 119 | www.google.com 120 | admin.google.com 121 | chat.google.com 122 | 123 | Scan complete. 12 results found. 124 | 125 | ``` 126 | 127 | ## Installation 128 | 129 | ```bash 130 | curl -s "https://raw.githubusercontent.com/liamg/scout/master/scripts/install.sh" | bash 131 | ``` 132 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= 3 | github.com/avast/retry-go v2.4.3+incompatible h1:c/FTk2POrEQyZfaHBMkMrXdu3/6IESJUHwu8r3k1JEU= 4 | github.com/avast/retry-go v2.4.3+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= 5 | github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= 6 | github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= 7 | github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= 8 | github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= 9 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 11 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 12 | github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= 13 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 14 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 15 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 16 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 17 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 18 | github.com/liamg/tml v0.2.0 h1:Ab4Qs+gWfy5TJ0OPk7OSVwgdPZmRD978LIHgsdqg56A= 19 | github.com/liamg/tml v0.2.0/go.mod h1:0h4EAV/zBOsqI91EWONedjRpO8O0itjGJVd+wG5eC+E= 20 | github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= 21 | github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= 22 | github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= 23 | github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= 24 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 25 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 26 | github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= 27 | github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= 28 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 29 | github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= 30 | github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= 31 | github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= 32 | github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= 33 | github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= 34 | github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= 35 | github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= 36 | github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 39 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 40 | github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= 41 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 42 | github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= 43 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= 44 | golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 45 | golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 46 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 47 | golang.org/x/sys v0.2.0 h1:ljd4t30dBnAvMZaQCevtY0xLLD0A+bRZXbgLMLU1F/A= 48 | golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 49 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 50 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 51 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 52 | -------------------------------------------------------------------------------- /cmd/scout/vhost.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/url" 8 | "os" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/liamg/scout/pkg/scan" 13 | "github.com/liamg/scout/pkg/wordlist" 14 | "github.com/liamg/tml" 15 | "github.com/sirupsen/logrus" 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | var ip string 20 | var port int 21 | var useSSL bool 22 | var contentHashing bool 23 | 24 | var vhostCmd = &cobra.Command{ 25 | Use: "vhost [base_domain]", 26 | Short: "Discover VHOSTs on a given web server.", 27 | Long: "Scout will discover VHOSTs as subdomains of the provided base domain.", 28 | Run: func(cmd *cobra.Command, args []string) { 29 | 30 | log.SetOutput(ioutil.Discard) 31 | 32 | if debug { 33 | logrus.SetLevel(logrus.DebugLevel) 34 | } 35 | 36 | if noColours { 37 | tml.DisableFormatting() 38 | } 39 | 40 | if len(args) == 0 { 41 | tml.Println("Error: You must specify a base domain.") 42 | os.Exit(1) 43 | } 44 | 45 | baseDomain := args[0] 46 | 47 | if strings.HasPrefix(baseDomain, "https://") { 48 | useSSL = true 49 | } 50 | 51 | if parsedURL, err := url.Parse(args[0]); err == nil && parsedURL.Host != "" { 52 | baseDomain = parsedURL.Host 53 | } 54 | 55 | if strings.Contains(baseDomain, "/") { 56 | baseDomain = baseDomain[:strings.Index(baseDomain, "/")] 57 | } 58 | 59 | resultChan := make(chan scan.VHOSTResult) 60 | busyChan := make(chan string, 0x400) 61 | 62 | var intStatusCodes []int 63 | 64 | for _, code := range statusCodes { 65 | i, err := strconv.Atoi(code) 66 | if err != nil { 67 | tml.Printf("Error: Invalid status code entered: %s.\n", code) 68 | os.Exit(1) 69 | } 70 | intStatusCodes = append(intStatusCodes, i) 71 | } 72 | 73 | ipStr := ip 74 | if ipStr == "" { 75 | ipStr = "-" 76 | } 77 | 78 | portStr := strconv.Itoa(port) 79 | if port == 0 { 80 | portStr = "-" 81 | } 82 | 83 | options := &scan.VHOSTOptions{ 84 | BaseDomain: baseDomain, 85 | Parallelism: parallelism, 86 | ResultChan: resultChan, 87 | BusyChan: busyChan, 88 | UseSSL: useSSL, 89 | IP: ip, 90 | Port: port, 91 | ContentHashing: contentHashing, 92 | } 93 | if wordlistPath != "" { 94 | var err error 95 | options.Wordlist, err = wordlist.FromFile(wordlistPath) 96 | if err != nil { 97 | tml.Printf("Error: %s\n", err) 98 | os.Exit(1) 99 | } 100 | } 101 | options.Inherit() 102 | 103 | tml.Printf( 104 | `[+] Base Domain %s 105 | [+] Routines %d 106 | [+] IP %s 107 | [+] Port %s 108 | [+] Using SSL %t 109 | 110 | `, 111 | options.BaseDomain, 112 | options.Parallelism, 113 | ipStr, 114 | portStr, 115 | options.UseSSL, 116 | ) 117 | 118 | scanner := scan.NewVHOSTScanner(options) 119 | 120 | waitChan := make(chan struct{}) 121 | 122 | genericOutputChan := make(chan string) 123 | importantOutputChan := make(chan string) 124 | 125 | go func() { 126 | for result := range resultChan { 127 | importantOutputChan <- tml.Sprintf("%s\n", result.VHOST) 128 | } 129 | close(waitChan) 130 | }() 131 | 132 | go func() { 133 | defer func() { 134 | _ = recover() 135 | }() 136 | for uri := range busyChan { 137 | genericOutputChan <- tml.Sprintf("Checking %s...", uri) 138 | } 139 | }() 140 | 141 | outChan := make(chan struct{}) 142 | go func() { 143 | 144 | defer close(outChan) 145 | 146 | for { 147 | select { 148 | case output := <-importantOutputChan: 149 | clearLine() 150 | fmt.Printf(output) 151 | FLUSH: 152 | for { 153 | select { 154 | case str := <-genericOutputChan: 155 | if str == "" { 156 | break FLUSH 157 | } 158 | default: 159 | break FLUSH 160 | } 161 | } 162 | case <-waitChan: 163 | return 164 | case output := <-genericOutputChan: 165 | clearLine() 166 | fmt.Printf(output) 167 | } 168 | } 169 | 170 | }() 171 | 172 | results, err := scanner.Scan() 173 | if err != nil { 174 | clearLine() 175 | tml.Printf("Error: %s\n", err) 176 | os.Exit(1) 177 | } 178 | logrus.Debug("Waiting for output to flush...") 179 | <-waitChan 180 | close(genericOutputChan) 181 | <-outChan 182 | 183 | clearLine() 184 | tml.Printf("\nScan complete. %d results found.\n\n", len(results)) 185 | 186 | }, 187 | } 188 | 189 | func init() { 190 | 191 | vhostCmd.Flags().BoolVar(&useSSL, "ssl", useSSL, "Use HTTPS when connecting to the server.") 192 | vhostCmd.Flags().StringVar(&ip, "ip", ip, "IP address to connect to - defaults to the DNS A record for the base domain.") 193 | vhostCmd.Flags().IntVar(&port, "port", port, "Port to connect to - defaults to 80 or 443 if --ssl is set.") 194 | vhostCmd.Flags().BoolVar(&contentHashing, "hash-contents", contentHashing, "Hash each response body to detect differences for catch-all scenarios.") 195 | 196 | rootCmd.AddCommand(vhostCmd) 197 | } 198 | -------------------------------------------------------------------------------- /pkg/scan/vhost_scanner.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "context" 5 | "crypto/md5" 6 | "crypto/tls" 7 | "fmt" 8 | "io" 9 | "io/ioutil" 10 | "net" 11 | "net/http" 12 | "sync" 13 | "time" 14 | 15 | "github.com/avast/retry-go" 16 | "github.com/sirupsen/logrus" 17 | ) 18 | 19 | type VHOSTScanner struct { 20 | client *http.Client 21 | options *VHOSTOptions 22 | badCode int 23 | badHash string 24 | } 25 | 26 | func NewVHOSTScanner(opt *VHOSTOptions) *VHOSTScanner { 27 | 28 | if opt == nil { 29 | opt = &DefaultVHOSTOptions 30 | } 31 | 32 | opt.Inherit() 33 | 34 | client := &http.Client{ 35 | Timeout: opt.Timeout, 36 | Transport: http.DefaultTransport, 37 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 38 | return http.ErrUseLastResponse 39 | }, 40 | } 41 | 42 | client.Transport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 43 | 44 | return &VHOSTScanner{ 45 | options: opt, 46 | client: client, 47 | } 48 | } 49 | 50 | func (scanner *VHOSTScanner) forceRequestsToIP(ip net.IP) { 51 | dialer := &net.Dialer{ 52 | Timeout: scanner.options.Timeout, 53 | } 54 | scanner.client.Transport.(*http.Transport).DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { 55 | port := scanner.options.Port 56 | if port == 0 { 57 | if scanner.options.UseSSL { 58 | port = 443 59 | } else { 60 | port = 80 61 | } 62 | } 63 | return dialer.DialContext(ctx, network, fmt.Sprintf("%s:%d", ip, port)) 64 | } 65 | } 66 | 67 | func md5Hash(input string) string { 68 | hash := md5.New() 69 | io.WriteString(hash, input) 70 | return fmt.Sprintf("%x", hash.Sum(nil)) 71 | } 72 | 73 | func (scanner *VHOSTScanner) Scan() ([]string, error) { 74 | 75 | logrus.Debug("Looking up base domain...") 76 | 77 | ip := scanner.options.IP 78 | if ip != "" { 79 | if parsed := net.ParseIP(ip); parsed == nil { 80 | return nil, fmt.Errorf("invalid IP address specified: %s", ip) 81 | } 82 | } else { 83 | ips, err := net.LookupIP(scanner.options.BaseDomain) 84 | if err != nil { 85 | return nil, fmt.Errorf("failed to resolve base domain: %s", err) 86 | } 87 | if len(ips) == 0 { 88 | return nil, fmt.Errorf("failed to resolve base domain: no A record found") 89 | } 90 | ip = ips[0].String() 91 | } 92 | 93 | scanner.forceRequestsToIP(net.ParseIP(ip)) 94 | 95 | badVHOST := fmt.Sprintf("%s.%s", md5Hash(time.Now().String()), scanner.options.BaseDomain) 96 | 97 | url := "http://" + badVHOST 98 | 99 | if scanner.options.UseSSL { 100 | url = "https://" + badVHOST 101 | } 102 | 103 | resp, err := scanner.client.Get(url) 104 | if err != nil { 105 | return nil, err 106 | } 107 | scanner.badCode = resp.StatusCode 108 | if scanner.options.ContentHashing { 109 | data, err := ioutil.ReadAll(resp.Body) 110 | if err != nil { 111 | return nil, err 112 | } 113 | scanner.badHash = md5Hash(string(data)) 114 | } else { 115 | io.Copy(ioutil.Discard, resp.Body) 116 | } 117 | resp.Body.Close() 118 | 119 | jobs := make(chan string, scanner.options.Parallelism) 120 | results := make(chan VHOSTResult, scanner.options.Parallelism) 121 | 122 | wg := sync.WaitGroup{} 123 | 124 | logrus.Debug("Starting workers...") 125 | 126 | for i := 0; i < scanner.options.Parallelism; i++ { 127 | wg.Add(1) 128 | go func() { 129 | scanner.worker(jobs, results) 130 | wg.Done() 131 | }() 132 | } 133 | 134 | logrus.Debugf("Started %d workers!", scanner.options.Parallelism) 135 | 136 | logrus.Debug("Starting results gatherer...") 137 | 138 | waitChan := make(chan struct{}) 139 | var foundVHOSTs []string 140 | 141 | go func() { 142 | for result := range results { 143 | if scanner.options.ResultChan != nil { 144 | scanner.options.ResultChan <- result 145 | } 146 | foundVHOSTs = append(foundVHOSTs, result.VHOST) 147 | } 148 | if scanner.options.ResultChan != nil { 149 | close(scanner.options.ResultChan) 150 | } 151 | close(waitChan) 152 | }() 153 | 154 | logrus.Debug("Adding jobs...") 155 | 156 | for { 157 | if word, err := scanner.options.Wordlist.Next(); err != nil { 158 | if err != io.EOF { 159 | return nil, err 160 | } 161 | break 162 | } else { 163 | if word == "" { 164 | continue 165 | } 166 | vhost := word + "." + scanner.options.BaseDomain 167 | jobs <- vhost 168 | } 169 | } 170 | 171 | close(jobs) 172 | 173 | logrus.Debug("Waiting for workers to complete...") 174 | 175 | wg.Wait() 176 | close(results) 177 | 178 | logrus.Debug("Waiting for results...") 179 | 180 | <-waitChan 181 | 182 | if scanner.options.BusyChan != nil { 183 | close(scanner.options.BusyChan) 184 | } 185 | 186 | logrus.Debug("Complete!") 187 | 188 | return foundVHOSTs, nil 189 | } 190 | 191 | func (scanner *VHOSTScanner) worker(jobs <-chan string, results chan<- VHOSTResult) { 192 | for j := range jobs { 193 | if result := scanner.checkVHOST(j); result != nil { 194 | results <- *result 195 | } 196 | } 197 | } 198 | 199 | // hit a url - is it one of certain response codes? leave connections open! 200 | func (scanner *VHOSTScanner) checkVHOST(vhost string) *VHOSTResult { 201 | 202 | if scanner.options.BusyChan != nil { 203 | scanner.options.BusyChan <- vhost 204 | } 205 | 206 | var code int 207 | var hash string 208 | 209 | url := "http://" + vhost 210 | 211 | if scanner.options.UseSSL { 212 | url = "https://" + vhost 213 | } 214 | 215 | if err := retry.Do(func() error { 216 | resp, err := scanner.client.Get(url) 217 | if err != nil { 218 | return nil 219 | } 220 | 221 | if scanner.options.ContentHashing { 222 | data, err := ioutil.ReadAll(resp.Body) 223 | if err != nil { 224 | return err 225 | } 226 | hash = md5Hash(string(data)) 227 | } else { 228 | io.Copy(ioutil.Discard, resp.Body) 229 | } 230 | resp.Body.Close() 231 | 232 | code = resp.StatusCode 233 | return nil 234 | }, retry.Attempts(10), retry.DelayType(retry.BackOffDelay)); err != nil { 235 | return nil 236 | } 237 | 238 | if code != scanner.badCode || (scanner.options.ContentHashing && hash != scanner.badHash) { 239 | return &VHOSTResult{ 240 | StatusCode: code, 241 | VHOST: vhost, 242 | } 243 | } 244 | 245 | return nil 246 | } 247 | -------------------------------------------------------------------------------- /cmd/scout/url.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "log" 7 | "net/url" 8 | "os" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/liamg/scout/pkg/scan" 13 | "github.com/liamg/scout/pkg/wordlist" 14 | "github.com/liamg/tml" 15 | "github.com/sirupsen/logrus" 16 | "github.com/spf13/cobra" 17 | ) 18 | 19 | var statusCodes []string 20 | var hideStatusCodes []string 21 | var filename string 22 | var headers []string 23 | var extensions = []string{"php", "htm", "html", "txt"} 24 | var includeNoExtension bool 25 | var enableSpidering bool 26 | var ignoredLengths []int 27 | 28 | var urlCmd = &cobra.Command{ 29 | Use: "url [url]", 30 | Short: "Discover URLs on a given web server.", 31 | Long: "Scout will discover URLs relative to the provided one.", 32 | Run: func(cmd *cobra.Command, args []string) { 33 | 34 | log.SetOutput(ioutil.Discard) 35 | 36 | if debug { 37 | logrus.SetLevel(logrus.DebugLevel) 38 | } 39 | 40 | if noColours { 41 | tml.DisableFormatting() 42 | } 43 | 44 | if len(args) == 0 { 45 | tml.Println("Error: You must specify a target URL.") 46 | os.Exit(1) 47 | } 48 | 49 | parsedURL, err := url.ParseRequestURI(args[0]) 50 | if err != nil { 51 | tml.Printf("Error: Invalid URL: %s\n", err) 52 | os.Exit(1) 53 | } 54 | 55 | resultChan := make(chan scan.URLResult) 56 | busyChan := make(chan string, 0x400) 57 | 58 | var intStatusCodes []int 59 | var filteredStatusCodes []string 60 | 61 | for _, code := range statusCodes { 62 | 63 | var skip bool 64 | for _, ignoreCode := range hideStatusCodes { 65 | if ignoreCode == code { 66 | skip = true 67 | break 68 | } 69 | } 70 | if skip { 71 | continue 72 | } 73 | 74 | i, err := strconv.Atoi(code) 75 | if err != nil { 76 | tml.Printf("Error: Invalid status code entered: %s.\n", code) 77 | os.Exit(1) 78 | } 79 | filteredStatusCodes = append(filteredStatusCodes, code) 80 | intStatusCodes = append(intStatusCodes, i) 81 | } 82 | 83 | options := []scan.URLOption{ 84 | scan.WithPositiveStatusCodes(intStatusCodes), 85 | scan.WithNegativeLengths(ignoredLengths), 86 | scan.WithTargetURL(*parsedURL), 87 | scan.WithResultChan(resultChan), 88 | scan.WithBusyChan(busyChan), 89 | scan.WithParallelism(parallelism), 90 | scan.WithExtensions(extensions), 91 | scan.WithIncludeNoExtension(includeNoExtension), 92 | scan.WithFilename(filename), 93 | scan.WithSkipSSLVerification(skipSSLVerification), 94 | scan.WithExtraHeaders(headers), 95 | scan.WithSpidering(enableSpidering), 96 | } 97 | 98 | if wordlistPath != "" { 99 | words, err := wordlist.FromFile(wordlistPath) 100 | if err != nil { 101 | tml.Printf("Error: %s\n", err) 102 | os.Exit(1) 103 | } 104 | options = append(options, scan.WithWordlist(words)) 105 | } 106 | 107 | tml.Printf( 108 | `[+] Target URL %s 109 | [+] Routines %d 110 | [+] Extensions %s 111 | [+] Positive Codes %s 112 | [+] Spider %t 113 | 114 | `, 115 | parsedURL.String(), 116 | parallelism, 117 | strings.Join(extensions, ","), 118 | strings.Join(filteredStatusCodes, ","), 119 | enableSpidering, 120 | ) 121 | 122 | scanner := scan.NewURLScanner(options...) 123 | 124 | waitChan := make(chan struct{}) 125 | 126 | genericOutputChan := make(chan string) 127 | importantOutputChan := make(chan string) 128 | 129 | go func() { 130 | for result := range resultChan { 131 | importantOutputChan <- tml.Sprintf("[%d] [%d] %s\n", result.StatusCode, result.Size, result.URL.String()) 132 | } 133 | close(waitChan) 134 | }() 135 | 136 | go func() { 137 | defer func() { 138 | _ = recover() 139 | }() 140 | for uri := range busyChan { 141 | genericOutputChan <- tml.Sprintf("Checking %s...", uri) 142 | } 143 | }() 144 | 145 | outChan := make(chan struct{}) 146 | go func() { 147 | 148 | defer close(outChan) 149 | 150 | for { 151 | select { 152 | case output := <-importantOutputChan: 153 | clearLine() 154 | fmt.Print(output) 155 | FLUSH: 156 | for { 157 | select { 158 | case str := <-genericOutputChan: 159 | if str == "" { 160 | break FLUSH 161 | } 162 | default: 163 | break FLUSH 164 | } 165 | } 166 | case <-waitChan: 167 | return 168 | case output := <-genericOutputChan: 169 | clearLine() 170 | fmt.Print(output) 171 | } 172 | } 173 | 174 | }() 175 | 176 | results, err := scanner.Scan() 177 | if err != nil { 178 | clearLine() 179 | tml.Printf("Error: %s\n", err) 180 | os.Exit(1) 181 | } 182 | logrus.Debug("Waiting for output to flush...") 183 | <-waitChan 184 | close(genericOutputChan) 185 | <-outChan 186 | 187 | clearLine() 188 | tml.Printf("\nScan complete. %d results found.\n\n", len(results)) 189 | 190 | }, 191 | } 192 | 193 | func clearLine() { 194 | fmt.Printf("\033[2K\r") 195 | } 196 | 197 | func init() { 198 | urlCmd.Flags().StringVarP(&filename, "filename", "f", filename, "Filename to seek in the directory being searched. Useful when all directories report 404 status.") 199 | urlCmd.Flags().StringSliceVarP(&statusCodes, "status-codes", "c", statusCodes, "HTTP status codes which indicate a positive find.") 200 | urlCmd.Flags().StringSliceVarP(&hideStatusCodes, "hide-status-codes", "z", hideStatusCodes, "HTTP status codes which should be hidden.") 201 | urlCmd.Flags().StringSliceVarP(&extensions, "extensions", "x", extensions, "File extensions to detect.") 202 | urlCmd.Flags().BoolVarP(&includeNoExtension, "include-no-extension", "X", includeNoExtension, "Include URLs with no extension.") 203 | urlCmd.Flags().StringSliceVarP(&headers, "header", "H", headers, "Extra header to send with requests (can be specified multiple times).") 204 | urlCmd.Flags().BoolVarP(&enableSpidering, "spider", "s", enableSpidering, "Spider links within page content") 205 | urlCmd.Flags().IntSliceVarP(&ignoredLengths, "hide-lengths", "l", ignoredLengths, "Hide results with these content lengths") 206 | 207 | rootCmd.AddCommand(urlCmd) 208 | } 209 | -------------------------------------------------------------------------------- /pkg/scan/url_scanner.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "bytes" 5 | "crypto/tls" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "net/http" 10 | "net/url" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "sync/atomic" 15 | "time" 16 | 17 | "github.com/liamg/scout/internal/app/scout/data" 18 | 19 | "github.com/liamg/scout/pkg/wordlist" 20 | 21 | "github.com/avast/retry-go" 22 | "github.com/sirupsen/logrus" 23 | ) 24 | 25 | type URLScanner struct { 26 | client *http.Client 27 | jobChan chan URLJob 28 | targetURL url.URL // target url 29 | positiveStatusCodes []int // status codes that indicate the existance of a file/directory 30 | timeout time.Duration // http request timeout 31 | parallelism int // parallel routines 32 | resultChan chan URLResult // chan to return results on - otherwise will be returned in slice 33 | busyChan chan string // chan to use to update current job 34 | words wordlist.Wordlist 35 | extensions []string 36 | includeNoExtension bool 37 | filename string 38 | skipSSLVerification bool 39 | backupExtensions []string 40 | extraHeaders []string 41 | enableSpidering bool 42 | checked map[string]struct{} 43 | checkMutex sync.Mutex 44 | queueChan chan URLJob 45 | jobsLoaded int32 46 | proxy *url.URL 47 | method string 48 | negativeLengths []int 49 | } 50 | 51 | type URLJob struct { 52 | URL string 53 | BasicOnly bool // don;t bother adding .BAK etc. 54 | } 55 | 56 | const MaxURLs = 1000 57 | 58 | func NewURLScanner(options ...URLOption) *URLScanner { 59 | 60 | scanner := &URLScanner{ 61 | checked: make(map[string]struct{}), 62 | positiveStatusCodes: []int{ 63 | http.StatusOK, 64 | http.StatusBadRequest, 65 | http.StatusInternalServerError, 66 | http.StatusMethodNotAllowed, 67 | http.StatusNoContent, 68 | http.StatusUnauthorized, 69 | http.StatusForbidden, 70 | http.StatusFound, 71 | http.StatusMovedPermanently, 72 | }, 73 | timeout: time.Second * 5, 74 | parallelism: 10, 75 | extensions: []string{"php", "htm", "html", "txt"}, 76 | includeNoExtension: false, 77 | backupExtensions: []string{"~", ".bak", ".BAK", ".old", ".backup", ".txt", ".OLD", ".BACKUP", "1", "2", "_", ".1", ".2"}, 78 | enableSpidering: false, 79 | method: "GET", 80 | } 81 | 82 | for _, option := range options { 83 | option(scanner) 84 | } 85 | 86 | scanner.queueChan = make(chan URLJob, MaxURLs*scanner.parallelism) 87 | 88 | if scanner.words == nil { 89 | wordlistBytes, err := data.Asset("assets/wordlist.txt") 90 | if err != nil { 91 | wordlistBytes = []byte{} 92 | } 93 | scanner.words = wordlist.FromReader(bytes.NewReader(wordlistBytes)) 94 | } 95 | 96 | scanner.client = &http.Client{ 97 | Timeout: scanner.timeout, 98 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 99 | return http.ErrUseLastResponse 100 | }, 101 | } 102 | 103 | if scanner.proxy != nil { 104 | scanner.client.Transport = &http.Transport{Proxy: http.ProxyURL(scanner.proxy)} 105 | } 106 | 107 | if scanner.skipSSLVerification { 108 | http.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 109 | scanner.client.Transport = http.DefaultTransport 110 | } 111 | 112 | return scanner 113 | } 114 | 115 | func (scanner *URLScanner) Scan() ([]url.URL, error) { 116 | 117 | atomic.StoreInt32(&(scanner.jobsLoaded), 0) 118 | 119 | scanner.jobChan = make(chan URLJob, scanner.parallelism) 120 | results := make(chan URLResult, scanner.parallelism) 121 | 122 | wg := sync.WaitGroup{} 123 | 124 | logrus.Debug("Starting workers...") 125 | 126 | for i := 0; i < scanner.parallelism; i++ { 127 | wg.Add(1) 128 | go func() { 129 | scanner.worker(results) 130 | wg.Done() 131 | }() 132 | } 133 | 134 | logrus.Debugf("Started %d workers!", scanner.parallelism) 135 | 136 | logrus.Debug("Starting results gatherer...") 137 | 138 | waitChan := make(chan struct{}) 139 | var foundURLs []url.URL 140 | 141 | go func() { 142 | for result := range results { 143 | if scanner.resultChan != nil { 144 | scanner.resultChan <- result 145 | } 146 | foundURLs = append(foundURLs, result.URL) 147 | } 148 | if scanner.resultChan != nil { 149 | close(scanner.resultChan) 150 | } 151 | close(waitChan) 152 | }() 153 | 154 | logrus.Debug("Adding jobs...") 155 | 156 | // add urls to scan... 157 | prefix := scanner.targetURL.String() 158 | if !strings.HasSuffix(prefix, "/") { 159 | prefix = prefix + "/" 160 | } 161 | 162 | scanner.jobChan <- URLJob{URL: prefix} 163 | 164 | for { 165 | if word, err := scanner.words.Next(); err != nil { 166 | if err != io.EOF { 167 | return nil, err 168 | } 169 | break 170 | } else { 171 | if word == "" { 172 | continue 173 | } 174 | uri := prefix + word 175 | 176 | if scanner.filename != "" { 177 | scanner.jobChan <- URLJob{URL: uri + "/" + scanner.filename, BasicOnly: true} 178 | } else { 179 | if scanner.includeNoExtension { 180 | scanner.jobChan <- URLJob{URL: uri, BasicOnly: true} 181 | } 182 | if !strings.HasSuffix(uri, ".htaccess") && !strings.HasSuffix(uri, ".htpasswd") { 183 | for _, ext := range scanner.extensions { 184 | scanner.jobChan <- URLJob{URL: uri + "." + ext} 185 | } 186 | } 187 | } 188 | } 189 | } 190 | 191 | atomic.StoreInt32(&(scanner.jobsLoaded), 1) 192 | 193 | logrus.Debug("Waiting for workers to complete...") 194 | 195 | wg.Wait() 196 | close(scanner.jobChan) 197 | close(results) 198 | 199 | logrus.Debug("Waiting for results...") 200 | 201 | <-waitChan 202 | 203 | if scanner.busyChan != nil { 204 | close(scanner.busyChan) 205 | } 206 | 207 | logrus.Debug("Complete!") 208 | 209 | return foundURLs, nil 210 | } 211 | 212 | func (scanner *URLScanner) worker(results chan<- URLResult) { 213 | 214 | for { 215 | select { 216 | case job := <-scanner.jobChan: 217 | if result := scanner.checkURL(job); result != nil { 218 | results <- *result 219 | } 220 | EXTRA: 221 | for { 222 | select { 223 | case extra := <-scanner.queueChan: 224 | if result := scanner.checkURL(extra); result != nil { 225 | results <- *result 226 | } 227 | default: 228 | break EXTRA 229 | } 230 | } 231 | default: 232 | if atomic.LoadInt32(&scanner.jobsLoaded) > 0 { 233 | return 234 | } 235 | 236 | time.Sleep(time.Millisecond * 100) 237 | } 238 | } 239 | } 240 | 241 | func (scanner *URLScanner) queue(job URLJob) { 242 | scanner.queueChan <- job 243 | } 244 | 245 | func (scanner *URLScanner) visited(uri string) bool { 246 | scanner.checkMutex.Lock() 247 | defer scanner.checkMutex.Unlock() 248 | if _, ok := scanner.checked[uri]; ok { 249 | return true 250 | } 251 | scanner.checked[uri] = struct{}{} 252 | return false 253 | } 254 | 255 | func (scanner *URLScanner) clean(url string) string { 256 | if strings.Contains(url, "#") { 257 | return strings.Split(url, "#")[0] 258 | } 259 | return url 260 | } 261 | 262 | // hit a url - is it one of certain response codes? leave connections open! 263 | func (scanner *URLScanner) checkURL(job URLJob) *URLResult { 264 | 265 | job.URL = scanner.clean(job.URL) 266 | 267 | if scanner.visited(job.URL) { 268 | return nil 269 | } 270 | 271 | if scanner.busyChan != nil { 272 | scanner.busyChan <- job.URL 273 | } 274 | 275 | var code int 276 | var location string 277 | var result *URLResult 278 | 279 | if err := retry.Do(func() error { 280 | 281 | req, err := http.NewRequest(scanner.method, job.URL, nil) 282 | if err != nil { 283 | return err 284 | } 285 | 286 | for _, header := range scanner.extraHeaders { 287 | parts := strings.SplitN(header, ":", 2) 288 | if len(parts) == 2 { 289 | if strings.ToLower(parts[0]) == "host" { 290 | req.Host = strings.TrimSpace(parts[1]) 291 | } 292 | req.Header.Set(parts[0], strings.TrimPrefix(parts[1], " ")) 293 | } 294 | } 295 | 296 | resp, err := scanner.client.Do(req) 297 | if err != nil { 298 | return nil 299 | } 300 | defer func() { _ = resp.Body.Close() }() 301 | 302 | code = resp.StatusCode 303 | location = resp.Header.Get("Location") 304 | 305 | if location != "" { 306 | if parsed, err := url.Parse(job.URL); err == nil { 307 | if relative, err := url.Parse(location); err == nil { 308 | target := parsed.ResolveReference(relative) 309 | if target.Host == parsed.Host { 310 | scanner.queue(URLJob{URL: target.String()}) 311 | } 312 | } 313 | } 314 | } 315 | 316 | for _, status := range scanner.positiveStatusCodes { 317 | if status == code { 318 | parsedURL, err := url.Parse(job.URL) 319 | if err != nil { 320 | return nil 321 | } 322 | 323 | if !job.BasicOnly && !strings.Contains(job.URL, "/.htpasswd") && !strings.Contains(job.URL, "/.htaccess") { 324 | for _, ext := range scanner.backupExtensions { 325 | bUrl := job.URL + ext 326 | if strings.Contains(job.URL, "?") { 327 | bits := strings.SplitN(job.URL, "?", 2) 328 | bUrl = strings.Join(bits, ext+"?") 329 | } 330 | scanner.queue(URLJob{URL: bUrl, BasicOnly: true}) 331 | } 332 | } 333 | 334 | contentType := resp.Header.Get("Content-Type") 335 | 336 | size := -1 337 | 338 | if scanner.enableSpidering && (contentType == "" || strings.Contains(contentType, "html")) { 339 | body, err := ioutil.ReadAll(resp.Body) 340 | if err == nil { 341 | for _, link := range findLinks(job.URL, body) { 342 | scanner.queue(URLJob{URL: link}) 343 | } 344 | } 345 | size = len(body) 346 | } 347 | 348 | if size == -1 { 349 | contentLength := resp.Header.Get("Content-Length") 350 | if contentLength != "" { 351 | size, _ = strconv.Atoi(contentLength) 352 | } else { 353 | cdata, _ := ioutil.ReadAll(resp.Body) 354 | size = len(cdata) 355 | cdata = nil 356 | } 357 | } 358 | 359 | for _, length := range scanner.negativeLengths { 360 | if length == size { 361 | return nil 362 | } 363 | } 364 | 365 | result = &URLResult{ 366 | StatusCode: code, 367 | URL: *parsedURL, 368 | Size: size, 369 | } 370 | 371 | break 372 | } 373 | } 374 | 375 | return nil 376 | 377 | }, retry.Attempts(10), retry.DelayType(retry.BackOffDelay)); err != nil { 378 | return nil 379 | } 380 | 381 | return result 382 | } 383 | 384 | var linkAttributes = []string{"src", "href"} 385 | 386 | func findLinks(currentURL string, html []byte) []string { 387 | 388 | base, err := url.Parse(currentURL) 389 | if err != nil { 390 | return nil 391 | } 392 | 393 | var results []string 394 | 395 | source := string(html) 396 | 397 | var bestIndex int 398 | var bestAttr string 399 | var link string 400 | 401 | for source != "" { 402 | bestIndex = -1 403 | for _, attr := range linkAttributes { 404 | index := strings.Index(source, fmt.Sprintf("%s=", attr)) 405 | if index < bestIndex || bestIndex == -1 { 406 | bestIndex = index 407 | bestAttr = attr 408 | } 409 | } 410 | if bestIndex == -1 { 411 | break 412 | } 413 | source = source[bestIndex+len(bestAttr)+1:] 414 | switch source[0] { 415 | case '"': 416 | source = source[1:] 417 | index := strings.Index(source, "\"") 418 | if index == -1 { 419 | continue 420 | } 421 | link = source[:index] 422 | case '\'': 423 | source = source[1:] 424 | index := strings.Index(source, "'") 425 | if index == -1 { 426 | continue 427 | } 428 | link = source[:index] 429 | default: 430 | spaceIndex := strings.Index(source, " ") 431 | bIndex := strings.Index(source, ">") 432 | if spaceIndex == -1 && bIndex == -1 { 433 | continue 434 | } 435 | if spaceIndex == -1 { 436 | link = source[:bIndex] 437 | } else if bIndex == -1 { 438 | link = source[:spaceIndex] 439 | } else { 440 | if spaceIndex < bIndex { 441 | bIndex = spaceIndex 442 | } 443 | link = source[:bIndex] 444 | } 445 | } 446 | 447 | u, err := url.Parse(link) 448 | if err != nil { 449 | return nil 450 | } 451 | 452 | target := base.ResolveReference(u) 453 | 454 | if target.Host != base.Host { 455 | continue 456 | } 457 | 458 | results = append(results, target.String()) 459 | } 460 | 461 | return results 462 | } 463 | --------------------------------------------------------------------------------