├── pkg ├── cmd │ ├── testdata │ │ ├── out │ │ │ └── .gitkeep │ │ ├── .gitignore │ │ ├── dict.txt │ │ ├── dict2.txt │ │ ├── out.txt │ │ └── out2.txt │ ├── output.go │ ├── termination │ │ ├── handler.go │ │ └── handler_test.go │ ├── root.go │ ├── version.go │ ├── result_view_integration_test.go │ ├── result_view.go │ ├── flags.go │ ├── root_integration_test.go │ ├── result_diff_integration_test.go │ ├── generate_dictionary.go │ ├── result_diff.go │ ├── dictionary_integration_test.go │ ├── config.go │ ├── scan.go │ └── scan_integration_test.go ├── scan │ ├── testdata │ │ ├── dictionary2.txt │ │ ├── dictionary1.txt │ │ └── one_element_dictionary.txt │ ├── output │ │ ├── testdata │ │ │ └── unwriteable.txt │ │ ├── conversion.go │ │ ├── nullsaver.go │ │ ├── nullsaver_test.go │ │ ├── saver.go │ │ └── saver_integration_test.go │ ├── filter.go │ ├── doer.go │ ├── summarizer │ │ ├── tree.go │ │ ├── tree │ │ │ ├── result_tree.go │ │ │ └── result_tree_test.go │ │ ├── result_summarizer.go │ │ └── result_summarizer_integration_test.go │ ├── producer.go │ ├── client │ │ ├── user_agent_internal_test.go │ │ ├── request_cache_internal_test.go │ │ ├── cookie │ │ │ ├── jar.go │ │ │ └── jar_test.go │ │ ├── headers_internal_test.go │ │ ├── user_agent.go │ │ ├── headers.go │ │ ├── request_cache.go │ │ ├── client.go │ │ └── client_test.go │ ├── config.go │ ├── filter │ │ ├── http.go │ │ └── http_test.go │ ├── producer │ │ ├── dictionary.go │ │ ├── reproducer.go │ │ ├── dictionary_test.go │ │ └── reproducer_test.go │ ├── scanner.go │ └── scanner_integration_test.go ├── result │ ├── testdata │ │ ├── invalidout.txt │ │ └── out.txt │ ├── load_integration_linux_test.go │ ├── load_integration_darwin_test.go │ ├── load.go │ └── load_integration_test.go ├── dictionary │ ├── testdata │ │ ├── directory_to_generate_dictionary │ │ │ ├── myfile.php │ │ │ └── subfolder │ │ │ │ ├── image.jpg │ │ │ │ ├── image2.gif │ │ │ │ └── subsubfolder │ │ │ │ ├── myfile.php │ │ │ │ └── myfile2.php │ │ └── dict.txt │ ├── doer.go │ ├── generator_test.go │ ├── generator.go │ ├── dictionary.go │ └── dictionary_integration_test.go └── common │ ├── urlpath │ ├── pathutil.go │ ├── join.go │ ├── pathutil_test.go │ └── join_test.go │ ├── must.go │ ├── test │ ├── util.go │ ├── rand.go │ ├── logger.go │ └── server.go │ └── must_test.go ├── resources └── tests │ ├── dictionary.txt │ ├── out.txt │ └── out2.txt ├── cmd ├── testserver │ └── main.go └── dirstalk │ └── main.go ├── .gitignore ├── Dockerfile ├── .goreleaser.yml ├── go.mod ├── .golangci.yml ├── LICENSE.md ├── .github └── workflows │ └── ci.yml ├── Makefile ├── go.sum ├── functional-tests.sh └── README.md /pkg/cmd/testdata/out/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/scan/testdata/dictionary2.txt: -------------------------------------------------------------------------------- 1 | /home -------------------------------------------------------------------------------- /pkg/result/testdata/invalidout.txt: -------------------------------------------------------------------------------- 1 | {omg/ -------------------------------------------------------------------------------- /pkg/scan/output/testdata/unwriteable.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /resources/tests/dictionary.txt: -------------------------------------------------------------------------------- 1 | home 2 | index -------------------------------------------------------------------------------- /pkg/scan/testdata/dictionary1.txt: -------------------------------------------------------------------------------- 1 | /home 2 | /about -------------------------------------------------------------------------------- /pkg/scan/testdata/one_element_dictionary.txt: -------------------------------------------------------------------------------- 1 | mypath -------------------------------------------------------------------------------- /pkg/cmd/testdata/.gitignore: -------------------------------------------------------------------------------- 1 | out/* 2 | !out/.gitkeep 3 | -------------------------------------------------------------------------------- /pkg/cmd/testdata/dict.txt: -------------------------------------------------------------------------------- 1 | home 2 | home/index.php 3 | blabla -------------------------------------------------------------------------------- /pkg/dictionary/testdata/directory_to_generate_dictionary/myfile.php: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/cmd/testdata/dict2.txt: -------------------------------------------------------------------------------- 1 | test/ 2 | home 3 | home/index.php 4 | blabla 5 | -------------------------------------------------------------------------------- /pkg/dictionary/testdata/directory_to_generate_dictionary/subfolder/image.jpg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/dictionary/testdata/directory_to_generate_dictionary/subfolder/image2.gif: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/dictionary/testdata/directory_to_generate_dictionary/subfolder/subsubfolder/myfile.php: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/dictionary/testdata/directory_to_generate_dictionary/subfolder/subsubfolder/myfile2.php: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /pkg/scan/filter.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | type ResultFilter interface { 4 | ShouldIgnore(Result) bool 5 | } 6 | -------------------------------------------------------------------------------- /pkg/scan/doer.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import "net/http" 4 | 5 | type Doer interface { 6 | Do(*http.Request) (*http.Response, error) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/dictionary/doer.go: -------------------------------------------------------------------------------- 1 | package dictionary 2 | 3 | import "net/http" 4 | 5 | type Doer interface { 6 | Do(*http.Request) (*http.Response, error) 7 | } 8 | -------------------------------------------------------------------------------- /pkg/dictionary/testdata/dict.txt: -------------------------------------------------------------------------------- 1 | # a comment 2 | home 3 | home/index.php 4 | # another comment 5 | blabla 6 | 7 | # yet another comment 8 | 9 | # 10 | -------------------------------------------------------------------------------- /pkg/cmd/output.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import "github.com/stefanoj3/dirstalk/pkg/scan" 4 | 5 | type OutputSaver interface { 6 | Save(scan.Result) error 7 | Close() error 8 | } 9 | -------------------------------------------------------------------------------- /pkg/common/urlpath/pathutil.go: -------------------------------------------------------------------------------- 1 | package urlpath 2 | 3 | import ( 4 | "path" 5 | ) 6 | 7 | func HasExtension(p string) bool { 8 | return len(path.Ext(p)) > 0 9 | } 10 | -------------------------------------------------------------------------------- /pkg/scan/summarizer/tree.go: -------------------------------------------------------------------------------- 1 | package summarizer 2 | 3 | import ( 4 | "github.com/stefanoj3/dirstalk/pkg/scan" 5 | ) 6 | 7 | type ResultTree interface { 8 | String(results []scan.Result) string 9 | } 10 | -------------------------------------------------------------------------------- /pkg/scan/output/conversion.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "encoding/json" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/scan" 7 | ) 8 | 9 | func convertResultToRawData(r scan.Result) ([]byte, error) { 10 | return json.Marshal(r) 11 | } 12 | -------------------------------------------------------------------------------- /pkg/scan/producer.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import "context" 4 | 5 | type Producer interface { 6 | Produce(ctx context.Context) <-chan Target 7 | } 8 | 9 | type ReProducer interface { 10 | Reproduce(ctx context.Context) func(r Result) <-chan Target 11 | } 12 | -------------------------------------------------------------------------------- /pkg/common/must.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import "github.com/pkg/errors" 4 | 5 | // Must accepts an error in input, if the error is not nil it panics 6 | // It helps to simplify code where no error is expected. 7 | func Must(err error) { 8 | if err != nil { 9 | panic(errors.WithStack(err)) 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /pkg/scan/client/user_agent_internal_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestDecorateTransportUserAgent(t *testing.T) { 10 | transport, err := decorateTransportWithUserAgentDecorator(nil, "") 11 | assert.Nil(t, transport) 12 | assert.Error(t, err) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/scan/client/request_cache_internal_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestRequestCacheTransportDecorator(t *testing.T) { 10 | transport, err := decorateTransportWithRequestCacheDecorator(nil) 11 | assert.Nil(t, transport) 12 | assert.Error(t, err) 13 | } 14 | -------------------------------------------------------------------------------- /pkg/scan/output/nullsaver.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import "github.com/stefanoj3/dirstalk/pkg/scan" 4 | 5 | func NewNullSaver() NullSaver { 6 | return NullSaver{} 7 | } 8 | 9 | type NullSaver struct{} 10 | 11 | func (n NullSaver) Save(r scan.Result) error { 12 | return nil 13 | } 14 | 15 | func (n NullSaver) Close() error { 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /pkg/common/test/util.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "net/url" 5 | ) 6 | 7 | type TestingT interface { 8 | Fatalf(format string, args ...interface{}) 9 | } 10 | 11 | func MustParseURL(t TestingT, rawurl string) *url.URL { 12 | u, err := url.Parse(rawurl) 13 | if err != nil { 14 | t.Fatalf("failed to parse url: %s", rawurl) 15 | } 16 | 17 | return u 18 | } 19 | -------------------------------------------------------------------------------- /cmd/testserver/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "net/http" 5 | ) 6 | 7 | func main() { 8 | handlerFunc := func(writer http.ResponseWriter, request *http.Request) {} 9 | 10 | http.HandleFunc("/home", handlerFunc) 11 | http.HandleFunc("/index", handlerFunc) 12 | http.HandleFunc("/index/home", handlerFunc) 13 | 14 | if err := http.ListenAndServe(":8080", nil); err != nil { 15 | panic(err) 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pkg/result/load_integration_linux_test.go: -------------------------------------------------------------------------------- 1 | package result_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/result" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLoadResultsFromFileShouldErrForInvalidPath(t *testing.T) { 11 | _, err := result.LoadResultsFromFile("/root/123/abc") 12 | assert.Error(t, err) 13 | 14 | assert.Contains(t, err.Error(), "permission denied") 15 | } 16 | -------------------------------------------------------------------------------- /pkg/common/test/rand.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | ) 7 | 8 | var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890") 9 | 10 | func RandStringRunes(n int) string { 11 | rand.Seed(time.Now().UnixNano()) 12 | 13 | b := make([]rune, n) 14 | for i := range b { 15 | b[i] = letterRunes[rand.Intn(len(letterRunes))] //nolint 16 | } 17 | 18 | return string(b) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/result/load_integration_darwin_test.go: -------------------------------------------------------------------------------- 1 | package result_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/result" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestLoadResultsFromFileShouldErrForInvalidPath(t *testing.T) { 11 | _, err := result.LoadResultsFromFile("/root/123/abc") 12 | assert.Error(t, err) 13 | 14 | assert.Contains(t, err.Error(), "no such file or directory") 15 | } 16 | -------------------------------------------------------------------------------- /pkg/scan/client/cookie/jar.go: -------------------------------------------------------------------------------- 1 | package cookie 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | ) 7 | 8 | func NewStatelessJar(cookies []*http.Cookie) StatelessJar { 9 | return StatelessJar{cookies: cookies} 10 | } 11 | 12 | type StatelessJar struct { 13 | cookies []*http.Cookie 14 | } 15 | 16 | func (s StatelessJar) SetCookies(_ *url.URL, _ []*http.Cookie) { 17 | } 18 | 19 | func (s StatelessJar) Cookies(_ *url.URL) []*http.Cookie { 20 | return s.cookies 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | gin-bin 26 | 27 | _book 28 | /vendor 29 | 30 | /dist 31 | 32 | coverage.txt 33 | 34 | *.out 35 | -------------------------------------------------------------------------------- /pkg/scan/output/nullsaver_test.go: -------------------------------------------------------------------------------- 1 | package output_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/scan" 7 | "github.com/stefanoj3/dirstalk/pkg/scan/output" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNullSaver(t *testing.T) { 12 | t.Parallel() 13 | 14 | sut := output.NewNullSaver() 15 | 16 | assert.NoError(t, sut.Save(scan.Result{})) 17 | assert.NoError(t, sut.Close()) 18 | assert.NoError(t, sut.Save(scan.Result{})) 19 | } 20 | -------------------------------------------------------------------------------- /pkg/common/must_test.go: -------------------------------------------------------------------------------- 1 | package common_test 2 | 3 | import ( 4 | "errors" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/common" 8 | ) 9 | 10 | func TestMustShouldNotPanicForNoErr(t *testing.T) { 11 | defer func() { 12 | if r := recover(); r != nil { 13 | t.Fatal("no panic expected") 14 | } 15 | }() 16 | 17 | common.Must(nil) 18 | } 19 | 20 | func TestMustShouldPanicOnErr(t *testing.T) { 21 | defer func() { 22 | if r := recover(); r == nil { 23 | t.Fatal("panic expected") 24 | } 25 | }() 26 | 27 | common.Must(errors.New("my error")) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/common/urlpath/join.go: -------------------------------------------------------------------------------- 1 | package urlpath 2 | 3 | import ( 4 | "path" 5 | "strings" 6 | ) 7 | 8 | // Join joins any number of path elements into a single path, adding a 9 | // separating slash if necessary. The result is Cleaned; in particular, 10 | // all empty strings are ignored. 11 | // If the last element end in a slash it will preserve it. 12 | func Join(elem ...string) string { 13 | joined := path.Join(elem...) 14 | 15 | last := elem[len(elem)-1] 16 | 17 | if strings.HasSuffix(last, "/") && !strings.HasSuffix(joined, "/") { 18 | joined += "/" 19 | } 20 | 21 | return joined 22 | } 23 | -------------------------------------------------------------------------------- /pkg/scan/client/headers_internal_test.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestDecorateTransportHeaderShouldFailWithNilDecorated(t *testing.T) { 11 | transport, err := decorateTransportWithHeadersDecorator(nil, map[string]string{}) 12 | assert.Nil(t, transport) 13 | assert.Error(t, err) 14 | } 15 | 16 | func TestDecorateTransportHeaderShouldFailWithNilHeaderMap(t *testing.T) { 17 | transport, err := decorateTransportWithHeadersDecorator(http.DefaultTransport, nil) 18 | assert.Nil(t, transport) 19 | assert.Error(t, err) 20 | } 21 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15-alpine3.12 as builder 2 | 3 | RUN adduser -D -g '' dirstalkuser 4 | RUN apk add --update make git ca-certificates 5 | 6 | RUN mkdir -p $GOPATH/src/github.com/stefanoj3/dirstalk 7 | ADD . $GOPATH/src/github.com/stefanoj3/dirstalk 8 | WORKDIR $GOPATH/src/github.com/stefanoj3/dirstalk 9 | 10 | RUN make dep 11 | RUN make build 12 | 13 | FROM scratch 14 | 15 | COPY --from=builder /etc/passwd /etc/passwd 16 | COPY --from=builder /go/src/github.com/stefanoj3/dirstalk/dist/dirstalk /bin/dirstalk 17 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ 18 | 19 | USER dirstalkuser 20 | CMD ["dirstalk"] -------------------------------------------------------------------------------- /pkg/cmd/termination/handler.go: -------------------------------------------------------------------------------- 1 | package termination 2 | 3 | import "sync" 4 | 5 | func NewTerminationHandler(attempts int) *Handler { 6 | return &Handler{terminationAttemptsLeft: attempts} 7 | } 8 | 9 | type Handler struct { 10 | terminationAttemptsLeft int 11 | mx sync.RWMutex 12 | } 13 | 14 | func (h *Handler) SignalTermination() { 15 | h.mx.Lock() 16 | defer h.mx.Unlock() 17 | 18 | if h.terminationAttemptsLeft <= 0 { 19 | return 20 | } 21 | 22 | h.terminationAttemptsLeft-- 23 | } 24 | 25 | func (h *Handler) ShouldTerminate() bool { 26 | h.mx.RLock() 27 | defer h.mx.RUnlock() 28 | 29 | return h.terminationAttemptsLeft <= 0 30 | } 31 | -------------------------------------------------------------------------------- /pkg/scan/client/user_agent.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | ) 7 | 8 | func decorateTransportWithUserAgentDecorator(decorated http.RoundTripper, userAgent string) (*userAgentTransportDecorator, error) { 9 | if decorated == nil { 10 | return nil, errors.New("decorated round tripper is nil") 11 | } 12 | 13 | return &userAgentTransportDecorator{decorated: decorated, userAgent: userAgent}, nil 14 | } 15 | 16 | type userAgentTransportDecorator struct { 17 | decorated http.RoundTripper 18 | userAgent string 19 | } 20 | 21 | func (u *userAgentTransportDecorator) RoundTrip(r *http.Request) (*http.Response, error) { 22 | r.Header.Set("User-Agent", u.userAgent) 23 | 24 | return u.decorated.RoundTrip(r) 25 | } 26 | -------------------------------------------------------------------------------- /pkg/common/test/logger.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "bytes" 5 | "sync" 6 | 7 | "github.com/sirupsen/logrus" 8 | ) 9 | 10 | type ThreadSafeBuffer struct { 11 | buf *bytes.Buffer 12 | rw sync.RWMutex 13 | } 14 | 15 | func (t *ThreadSafeBuffer) String() string { 16 | t.rw.RLock() 17 | defer t.rw.RUnlock() 18 | 19 | return t.buf.String() 20 | } 21 | 22 | func (t *ThreadSafeBuffer) Write(p []byte) (n int, err error) { 23 | t.rw.Lock() 24 | defer t.rw.Unlock() 25 | 26 | return t.buf.Write(p) 27 | } 28 | 29 | func NewLogger() (*logrus.Logger, *ThreadSafeBuffer) { 30 | b := &bytes.Buffer{} 31 | tsb := &ThreadSafeBuffer{buf: b} 32 | 33 | l := logrus.New() 34 | l.SetLevel(logrus.DebugLevel) 35 | l.SetOutput(tsb) 36 | 37 | return l, tsb 38 | } 39 | -------------------------------------------------------------------------------- /pkg/cmd/root.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | ) 7 | 8 | func NewRootCommand(logger *logrus.Logger) *cobra.Command { 9 | var verbose bool 10 | 11 | cmd := &cobra.Command{ 12 | Use: "dirstalk", 13 | Short: "Stalk the given url trying to enumerate files and folders", 14 | Long: `dirstalk is a tool that attempts to enumerate files and folders starting from a given URL`, 15 | PersistentPreRun: func(cmd *cobra.Command, args []string) { 16 | if verbose { 17 | logger.SetLevel(logrus.DebugLevel) 18 | } 19 | }, 20 | } 21 | 22 | cmd.PersistentFlags().BoolVarP( 23 | &verbose, 24 | flagRootVerbose, 25 | flagRootVerboseShort, 26 | false, 27 | "verbose mode", 28 | ) 29 | 30 | return cmd 31 | } 32 | -------------------------------------------------------------------------------- /pkg/cmd/version.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/spf13/cobra" 8 | ) 9 | 10 | var ( 11 | Version string 12 | BuildTime string 13 | ) 14 | 15 | //nolint:gochecknoinits 16 | func init() { 17 | if Version == "" { 18 | Version = "dev" 19 | } 20 | 21 | if BuildTime == "" { 22 | BuildTime = "now" 23 | } 24 | } 25 | 26 | func NewVersionCommand(out io.Writer) *cobra.Command { 27 | cmd := &cobra.Command{ 28 | Use: "version", 29 | Short: "Print the current version", 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | _, _ = fmt.Fprintln( 32 | out, 33 | fmt.Sprintf("Version: %s", Version), 34 | ) 35 | 36 | _, _ = fmt.Fprintln( 37 | out, 38 | fmt.Sprintf("Built at: %s", BuildTime), 39 | ) 40 | 41 | return nil 42 | }, 43 | } 44 | 45 | return cmd 46 | } 47 | -------------------------------------------------------------------------------- /.goreleaser.yml: -------------------------------------------------------------------------------- 1 | builds: 2 | - 3 | env: 4 | - CGO_ENABLED=0 5 | 6 | main: ./cmd/dirstalk/main.go 7 | 8 | goos: 9 | - freebsd 10 | - linux 11 | - darwin 12 | - openbsd 13 | 14 | goarch: 15 | - amd64 16 | - arm 17 | - arm64 18 | goarm: 19 | - 5 20 | - 6 21 | - 7 22 | 23 | ldflags: 24 | - -X github.com/stefanoj3/dirstalk/pkg/cmd.Version={{.Version}} -X github.com/stefanoj3/dirstalk/pkg/cmd.BuildTime={{.Timestamp}} 25 | 26 | archive: 27 | replacements: 28 | darwin: Darwin 29 | linux: Linux 30 | openbsd: OpenBSD 31 | freebsd: FreeBSD 32 | android: Android 33 | 386: i386 34 | amd64: x86_64 35 | 36 | checksum: 37 | name_template: 'checksums.txt' 38 | changelog: 39 | sort: asc 40 | filters: 41 | exclude: 42 | - '^docs:' 43 | - '^test:' 44 | -------------------------------------------------------------------------------- /pkg/scan/client/headers.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | "net/http" 6 | ) 7 | 8 | func decorateTransportWithHeadersDecorator(decorated http.RoundTripper, headers map[string]string) (*headersTransportDecorator, error) { 9 | if decorated == nil { 10 | return nil, errors.New("decorated round tripper is nil") 11 | } 12 | 13 | if headers == nil { 14 | return nil, errors.New("headers is nil") 15 | } 16 | 17 | return &headersTransportDecorator{decorated: decorated, headers: headers}, nil 18 | } 19 | 20 | type headersTransportDecorator struct { 21 | decorated http.RoundTripper 22 | headers map[string]string 23 | } 24 | 25 | func (h *headersTransportDecorator) RoundTrip(r *http.Request) (*http.Response, error) { 26 | for key, value := range h.headers { 27 | r.Header.Set(key, value) 28 | } 29 | 30 | return h.decorated.RoundTrip(r) 31 | } 32 | -------------------------------------------------------------------------------- /cmd/dirstalk/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/sirupsen/logrus" 5 | "github.com/spf13/cobra" 6 | "github.com/stefanoj3/dirstalk/pkg/cmd" 7 | ) 8 | 9 | func main() { 10 | logger := logrus.New() 11 | logger.Formatter = &logrus.TextFormatter{DisableTimestamp: true} 12 | 13 | dirStalkCmd := createCommand(logger) 14 | 15 | if err := dirStalkCmd.Execute(); err != nil { 16 | logger.WithField("err", err).Fatal("Execution error") 17 | } 18 | } 19 | 20 | func createCommand(logger *logrus.Logger) *cobra.Command { 21 | dirStalkCmd := cmd.NewRootCommand(logger) 22 | 23 | dirStalkCmd.AddCommand(cmd.NewScanCommand(logger)) 24 | dirStalkCmd.AddCommand(cmd.NewResultViewCommand(logger.Out)) 25 | dirStalkCmd.AddCommand(cmd.NewResultDiffCommand(logger.Out)) 26 | dirStalkCmd.AddCommand(cmd.NewGenerateDictionaryCommand(logger.Out)) 27 | dirStalkCmd.AddCommand(cmd.NewVersionCommand(logger.Out)) 28 | 29 | return dirStalkCmd 30 | } 31 | -------------------------------------------------------------------------------- /pkg/scan/config.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | ) 7 | 8 | // Config represents the configuration needed to perform a scan. 9 | type Config struct { 10 | DictionaryPath string 11 | DictionaryTimeoutInMilliseconds int 12 | HTTPMethods []string 13 | HTTPStatusesToIgnore []int 14 | Threads int 15 | TimeoutInMilliseconds int 16 | CacheRequests bool 17 | ScanDepth int 18 | Socks5Url *url.URL 19 | UserAgent string 20 | UseCookieJar bool 21 | Cookies []*http.Cookie 22 | Headers map[string]string 23 | Out string 24 | ShouldSkipSSLCertificatesValidation bool 25 | IgnoreEmpty20xResponses bool 26 | } 27 | -------------------------------------------------------------------------------- /pkg/scan/client/cookie/jar_test.go: -------------------------------------------------------------------------------- 1 | package cookie_test 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "testing" 7 | 8 | "github.com/stefanoj3/dirstalk/pkg/scan/client/cookie" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestStatelessJarShouldWorkWithNilCookies(t *testing.T) { 13 | assert.Nil(t, cookie.NewStatelessJar(nil).Cookies(nil)) 14 | } 15 | 16 | func TestStatelessJarShouldBeStateless(t *testing.T) { 17 | cookies := []*http.Cookie{ 18 | { 19 | Name: "a_cookie_name", 20 | Value: "a_cookie_value", 21 | }, 22 | } 23 | 24 | jar := cookie.NewStatelessJar(cookies) 25 | 26 | u, err := url.Parse("http://github.com/stefanoj3") 27 | assert.NoError(t, err) 28 | 29 | assert.Equal(t, cookies, jar.Cookies(u)) 30 | 31 | jar.SetCookies( 32 | u, 33 | []*http.Cookie{ 34 | { 35 | Name: "another_cookie_name", 36 | Value: "another_cookie_value", 37 | }, 38 | }, 39 | ) 40 | 41 | assert.Equal(t, cookies, jar.Cookies(u)) 42 | } 43 | -------------------------------------------------------------------------------- /pkg/scan/filter/http.go: -------------------------------------------------------------------------------- 1 | package filter 2 | 3 | import ( 4 | "github.com/stefanoj3/dirstalk/pkg/scan" 5 | ) 6 | 7 | func NewHTTPStatusResultFilter(httpStatusesToIgnore []int, ignoreEmptyBody bool) HTTPStatusResultFilter { 8 | httpStatusesToIgnoreMap := make(map[int]struct{}, len(httpStatusesToIgnore)) 9 | for _, statusToIgnore := range httpStatusesToIgnore { 10 | httpStatusesToIgnoreMap[statusToIgnore] = struct{}{} 11 | } 12 | 13 | return HTTPStatusResultFilter{httpStatusesToIgnoreMap: httpStatusesToIgnoreMap, ignoreEmptyBody: ignoreEmptyBody} 14 | } 15 | 16 | type HTTPStatusResultFilter struct { 17 | httpStatusesToIgnoreMap map[int]struct{} 18 | ignoreEmptyBody bool 19 | } 20 | 21 | func (f HTTPStatusResultFilter) ShouldIgnore(result scan.Result) bool { 22 | if f.ignoreEmptyBody && result.StatusCode/100 == 2 && result.ContentLength == 0 { 23 | return true 24 | } 25 | 26 | _, found := f.httpStatusesToIgnoreMap[result.StatusCode] 27 | 28 | return found 29 | } 30 | -------------------------------------------------------------------------------- /pkg/cmd/testdata/out.txt: -------------------------------------------------------------------------------- 1 | {"Target":{"Path":"partners","Method":"GET","Depth":3},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 2 | {"Target":{"Path":"s","Method":"GET","Depth":3},"StatusCode":400,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/s","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 3 | {"Target":{"Path":"adview","Method":"GET","Depth":3},"StatusCode":204,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/adview","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 4 | {"Target":{"Path":"partners/terms","Method":"GET","Depth":2},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners/terms","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 5 | -------------------------------------------------------------------------------- /pkg/cmd/testdata/out2.txt: -------------------------------------------------------------------------------- 1 | {"Target":{"Path":"partners","Method":"GET","Depth":3},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 2 | {"Target":{"Path":"s","Method":"GET","Depth":3},"StatusCode":400,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/s","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 3 | {"Target":{"Path":"adview","Method":"GET","Depth":3},"StatusCode":204,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/adview","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 4 | {"Target":{"Path":"partners/terms","Method":"GET","Depth":2},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners/123","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 5 | -------------------------------------------------------------------------------- /resources/tests/out.txt: -------------------------------------------------------------------------------- 1 | {"Target":{"Path":"partners","Method":"GET","Depth":3},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 2 | {"Target":{"Path":"s","Method":"GET","Depth":3},"StatusCode":400,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/s","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 3 | {"Target":{"Path":"adview","Method":"GET","Depth":3},"StatusCode":204,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/adview","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 4 | {"Target":{"Path":"partners/terms","Method":"GET","Depth":2},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners/terms","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 5 | -------------------------------------------------------------------------------- /resources/tests/out2.txt: -------------------------------------------------------------------------------- 1 | {"Target":{"Path":"partners","Method":"GET","Depth":3},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 2 | {"Target":{"Path":"s","Method":"GET","Depth":3},"StatusCode":400,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/s","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 3 | {"Target":{"Path":"adview","Method":"GET","Depth":3},"StatusCode":204,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/adview","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 4 | {"Target":{"Path":"partners/terms","Method":"GET","Depth":2},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners/123","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 5 | -------------------------------------------------------------------------------- /pkg/result/testdata/out.txt: -------------------------------------------------------------------------------- 1 | {"Target":{"Path":"partners","Method":"GET","Depth":3},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 2 | {"Target":{"Path":"s","Method":"GET","Depth":3},"StatusCode":400,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/s","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 3 | {"Target":{"Path":"adview","Method":"GET","Depth":3},"StatusCode":204,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/adview","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 4 | {"Target":{"Path":"partners/terms","Method":"GET","Depth":2},"StatusCode":200,"URL":{"Scheme":"https","Opaque":"","User":null,"Host":"www.brucewillisdiesinarmageddon.co.de","Path":"/partners/terms","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":""}} 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/stefanoj3/dirstalk 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/DiSiqueira/GoTree v0.0.0-20180907134536-53a8e837f295 7 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 8 | github.com/pkg/errors v0.9.1 9 | github.com/sergi/go-diff v1.2.0 10 | github.com/sirupsen/logrus v1.8.1 11 | github.com/spf13/cobra v1.4.0 12 | github.com/stretchr/testify v1.7.1 13 | golang.org/x/net v0.0.0-20220516155154-20f960328961 14 | ) 15 | 16 | require ( 17 | github.com/davecgh/go-spew v1.1.1 // indirect 18 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 19 | github.com/kr/pretty v0.3.0 // indirect 20 | github.com/pmezard/go-difflib v1.0.0 // indirect 21 | github.com/rogpeppe/go-internal v1.8.1 // indirect 22 | github.com/spf13/pflag v1.0.5 // indirect 23 | golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a // indirect 24 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect 25 | gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 // indirect 26 | ) 27 | -------------------------------------------------------------------------------- /.golangci.yml: -------------------------------------------------------------------------------- 1 | 2 | # See https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml 3 | run: 4 | tests: true 5 | deadline: 5m 6 | 7 | linters-settings: 8 | errcheck: 9 | check-type-assertions: true 10 | check-blank: true 11 | gocyclo: 12 | min-complexity: 20 13 | dupl: 14 | threshold: 100 15 | misspell: 16 | locale: US 17 | unused: 18 | check-exported: false 19 | unparam: 20 | check-exported: true 21 | 22 | linters: 23 | enable-all: true 24 | disable: 25 | - lll 26 | - maligned 27 | - gochecknoglobals 28 | - stylecheck 29 | - golint 30 | - goconst 31 | - funlen 32 | - gomnd 33 | - gosimple 34 | - goerr113 35 | - gofumpt 36 | 37 | issues: 38 | exclude-use-default: false 39 | exclude-rules: 40 | - path: _test\.go 41 | linters: 42 | - noctx 43 | 44 | # output configuration options 45 | output: 46 | # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" 47 | format: colored-line-number -------------------------------------------------------------------------------- /pkg/scan/producer/dictionary.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/scan" 7 | ) 8 | 9 | func NewDictionaryProducer( 10 | methods []string, 11 | dictionary []string, 12 | depth int, 13 | ) *DictionaryProducer { 14 | return &DictionaryProducer{ 15 | methods: methods, 16 | dictionary: dictionary, 17 | depth: depth, 18 | } 19 | } 20 | 21 | type DictionaryProducer struct { 22 | methods []string 23 | dictionary []string 24 | depth int 25 | } 26 | 27 | func (p *DictionaryProducer) Produce(ctx context.Context) <-chan scan.Target { 28 | targets := make(chan scan.Target, 10) 29 | 30 | go func() { 31 | defer close(targets) 32 | 33 | for _, entry := range p.dictionary { 34 | for _, method := range p.methods { 35 | select { 36 | case <-ctx.Done(): 37 | return 38 | default: 39 | targets <- scan.Target{ 40 | Path: entry, 41 | Method: method, 42 | Depth: p.depth, 43 | } 44 | } 45 | } 46 | } 47 | }() 48 | 49 | return targets 50 | } 51 | -------------------------------------------------------------------------------- /pkg/cmd/termination/handler_test.go: -------------------------------------------------------------------------------- 1 | package termination_test 2 | 3 | import ( 4 | "sync" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/cmd/termination" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestHandler(t *testing.T) { 12 | handler := termination.NewTerminationHandler(3) 13 | 14 | handler.SignalTermination() 15 | assert.False(t, handler.ShouldTerminate()) 16 | 17 | handler.SignalTermination() 18 | assert.False(t, handler.ShouldTerminate()) 19 | 20 | handler.SignalTermination() 21 | assert.True(t, handler.ShouldTerminate()) 22 | 23 | handler.SignalTermination() 24 | assert.True(t, handler.ShouldTerminate()) 25 | } 26 | 27 | func TestHandlerShouldWorkWithMultipleRoutines(_ *testing.T) { 28 | handler := termination.NewTerminationHandler(10) 29 | 30 | const workers = 1000 31 | 32 | wg := sync.WaitGroup{} 33 | wg.Add(workers) 34 | 35 | for i := 0; i < workers; i++ { 36 | go func() { 37 | defer wg.Done() 38 | 39 | handler.ShouldTerminate() 40 | handler.SignalTermination() 41 | }() 42 | } 43 | 44 | wg.Wait() 45 | } 46 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018-2019 Stefano Gabryel 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /pkg/scan/output/saver.go: -------------------------------------------------------------------------------- 1 | package output 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/stefanoj3/dirstalk/pkg/scan" 10 | ) 11 | 12 | var ( 13 | errNilWriteCloser = errors.New("Saver: writeCloser is nil") 14 | ) 15 | 16 | func NewFileSaver(path string) (Saver, error) { 17 | file, err := os.Create(path) 18 | if err != nil { 19 | return Saver{}, errors.Wrapf(err, "failed to create file `%s` for output", path) 20 | } 21 | 22 | return Saver{writeCloser: file}, nil 23 | } 24 | 25 | type Saver struct { 26 | writeCloser io.WriteCloser 27 | } 28 | 29 | func (f Saver) Save(r scan.Result) error { 30 | if f.writeCloser == nil { 31 | return errNilWriteCloser 32 | } 33 | 34 | rawResult, err := convertResultToRawData(r) 35 | if err != nil { 36 | return errors.Wrap(err, "Saver: failed to convert result") 37 | } 38 | 39 | _, err = fmt.Fprintln(f.writeCloser, string(rawResult)) 40 | 41 | return errors.Wrapf(err, "Saver: failed to write result: %s", rawResult) 42 | } 43 | 44 | func (f Saver) Close() error { 45 | if f.writeCloser == nil { 46 | return errNilWriteCloser 47 | } 48 | 49 | return f.writeCloser.Close() 50 | } 51 | -------------------------------------------------------------------------------- /pkg/scan/summarizer/tree/result_tree.go: -------------------------------------------------------------------------------- 1 | package tree 2 | 3 | import ( 4 | "sort" 5 | "strings" 6 | 7 | gotree "github.com/DiSiqueira/GoTree" 8 | "github.com/stefanoj3/dirstalk/pkg/scan" 9 | ) 10 | 11 | func NewResultTreeProducer() ResultTreeProducer { 12 | return ResultTreeProducer{} 13 | } 14 | 15 | type ResultTreeProducer struct{} 16 | 17 | func (s ResultTreeProducer) String(results []scan.Result) string { 18 | sort.Slice(results, func(i, j int) bool { 19 | return results[i].Target.Path < results[j].Target.Path 20 | }) 21 | 22 | root := gotree.New("/") 23 | 24 | for _, r := range results { 25 | currentBranch := root 26 | 27 | parts := strings.Split(r.URL.Path, "/") 28 | for _, p := range parts { 29 | if len(p) == 0 { 30 | continue 31 | } 32 | 33 | found := false 34 | 35 | for _, item := range currentBranch.Items() { 36 | if item.Text() != p { 37 | continue 38 | } 39 | 40 | currentBranch = item 41 | found = true 42 | 43 | break 44 | } 45 | 46 | if found { 47 | continue 48 | } 49 | 50 | newTree := gotree.New(p) 51 | currentBranch.AddTree(newTree) 52 | currentBranch = newTree 53 | } 54 | } 55 | 56 | return root.Print() 57 | } 58 | -------------------------------------------------------------------------------- /pkg/common/urlpath/pathutil_test.go: -------------------------------------------------------------------------------- 1 | package urlpath_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/common/urlpath" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestHasExtension(t *testing.T) { 11 | testCases := []struct { 12 | path string 13 | expectedResult bool 14 | }{ 15 | { 16 | path: "images/image.jpg", 17 | expectedResult: true, 18 | }, 19 | { 20 | path: "file.pdf", 21 | expectedResult: true, 22 | }, 23 | { 24 | path: "home/page.php", 25 | expectedResult: true, 26 | }, 27 | { 28 | path: "src/code.cpp", 29 | expectedResult: true, 30 | }, 31 | { 32 | path: "src/code.h", 33 | expectedResult: true, 34 | }, 35 | { 36 | path: "folder/script.sh", 37 | expectedResult: true, 38 | }, 39 | { 40 | path: "myfile", 41 | expectedResult: false, 42 | }, 43 | { 44 | path: "myfolder/myfile", 45 | expectedResult: false, 46 | }, 47 | } 48 | 49 | for _, tc := range testCases { 50 | tc := tc 51 | t.Run(tc.path, func(t *testing.T) { 52 | t.Parallel() 53 | 54 | assert.Equal(t, tc.expectedResult, urlpath.HasExtension(tc.path)) 55 | }) 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /pkg/scan/client/request_cache.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "net/http" 7 | "sync" 8 | ) 9 | 10 | var ( 11 | // ErrRequestRedundant this error is returned when trying to perform the 12 | // same request (method, host, path) more than one time. 13 | ErrRequestRedundant = errors.New("this request has been made already") 14 | ) 15 | 16 | func decorateTransportWithRequestCacheDecorator(decorated http.RoundTripper) (*requestCacheTransportDecorator, error) { 17 | if decorated == nil { 18 | return nil, errors.New("decorated round tripper is nil") 19 | } 20 | 21 | return &requestCacheTransportDecorator{decorated: decorated}, nil 22 | } 23 | 24 | type requestCacheTransportDecorator struct { 25 | decorated http.RoundTripper 26 | requestMap sync.Map 27 | } 28 | 29 | func (u *requestCacheTransportDecorator) RoundTrip(r *http.Request) (*http.Response, error) { 30 | key := u.keyForRequest(r) 31 | 32 | _, found := u.requestMap.Load(key) 33 | if found { 34 | return nil, ErrRequestRedundant 35 | } 36 | 37 | u.requestMap.Store(key, struct{}{}) 38 | 39 | return u.decorated.RoundTrip(r) 40 | } 41 | 42 | func (u *requestCacheTransportDecorator) keyForRequest(r *http.Request) string { 43 | return fmt.Sprintf("%s~%s~%s", r.Method, r.Host, r.URL.Path) 44 | } 45 | -------------------------------------------------------------------------------- /pkg/cmd/result_view_integration_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stefanoj3/dirstalk/pkg/common/test" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestResultViewShouldErrWhenCalledWithoutResultFlag(t *testing.T) { 11 | logger, _ := test.NewLogger() 12 | 13 | c := createCommand(logger) 14 | assert.NotNil(t, c) 15 | 16 | err := executeCommand(c, "result.view") 17 | assert.Error(t, err) 18 | 19 | assert.Contains(t, err.Error(), "result-file") 20 | assert.Contains(t, err.Error(), "not set") 21 | } 22 | 23 | func TestResultViewShouldErrWhenCalledWithInvalidPath(t *testing.T) { 24 | logger, _ := test.NewLogger() 25 | 26 | c := createCommand(logger) 27 | assert.NotNil(t, c) 28 | 29 | err := executeCommand(c, "result.view", "-r", "/root/123/abc") 30 | assert.Error(t, err) 31 | 32 | assert.Contains(t, err.Error(), "failed to load results from") 33 | } 34 | 35 | func TestResultView(t *testing.T) { 36 | logger, loggerBuffer := test.NewLogger() 37 | 38 | c := createCommand(logger) 39 | assert.NotNil(t, c) 40 | 41 | err := executeCommand(c, "result.view", "-r", "testdata/out.txt") 42 | assert.NoError(t, err) 43 | 44 | expected := `/ 45 | ├── adview 46 | ├── partners 47 | │ └── terms 48 | └── s 49 | ` 50 | 51 | assert.Contains(t, loggerBuffer.String(), expected) 52 | } 53 | -------------------------------------------------------------------------------- /pkg/result/load.go: -------------------------------------------------------------------------------- 1 | package result 2 | 3 | import ( 4 | "bufio" 5 | "encoding/json" 6 | "os" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/stefanoj3/dirstalk/pkg/scan" 10 | ) 11 | 12 | func LoadResultsFromFile(resultFilePath string) ([]scan.Result, error) { 13 | file, err := os.Open(resultFilePath) // #nosec 14 | if err != nil { 15 | return nil, errors.Wrapf(err, "failed to open %s", resultFilePath) 16 | } 17 | 18 | defer file.Close() //nolint 19 | 20 | fileInfo, err := file.Stat() 21 | if err != nil { 22 | return nil, errors.Wrapf(err, "failed to read properties of %s", resultFilePath) 23 | } 24 | 25 | if fileInfo.IsDir() { 26 | return nil, errors.Errorf("`%s` is a directory, you need to specify a valid result file", resultFilePath) 27 | } 28 | 29 | fileScanner := bufio.NewScanner(file) 30 | 31 | lineCounter := 0 32 | results := make([]scan.Result, 0, 10) 33 | 34 | for fileScanner.Scan() { 35 | lineCounter++ 36 | 37 | r := scan.Result{} 38 | 39 | if err := json.Unmarshal(fileScanner.Bytes(), &r); err != nil { 40 | return nil, errors.Wrapf(err, "unable to read line %d", lineCounter) 41 | } 42 | 43 | results = append(results, r) 44 | } 45 | 46 | if err := fileScanner.Err(); err != nil { 47 | return nil, errors.Wrap(err, "an error occurred while reading the result file") 48 | } 49 | 50 | return results, nil 51 | } 52 | -------------------------------------------------------------------------------- /pkg/cmd/result_view.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/spf13/cobra" 9 | "github.com/stefanoj3/dirstalk/pkg/common" 10 | "github.com/stefanoj3/dirstalk/pkg/result" 11 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer/tree" 12 | ) 13 | 14 | func NewResultViewCommand(out io.Writer) *cobra.Command { 15 | cmd := &cobra.Command{ 16 | Use: "result.view", 17 | Short: "Read a scan output file and render the folder tree", 18 | RunE: buildResultViewCmd(out), 19 | } 20 | 21 | cmd.Flags().StringP( 22 | flagResultViewResultFile, 23 | flagResultViewResultFileShort, 24 | "", 25 | "result file to read", 26 | ) 27 | common.Must(cmd.MarkFlagFilename(flagResultViewResultFile)) 28 | common.Must(cmd.MarkFlagRequired(flagResultViewResultFile)) 29 | 30 | return cmd 31 | } 32 | 33 | func buildResultViewCmd(out io.Writer) func(cmd *cobra.Command, args []string) error { 34 | return func(cmd *cobra.Command, args []string) error { 35 | resultFilePath := cmd.Flag(flagResultViewResultFile).Value.String() 36 | 37 | results, err := result.LoadResultsFromFile(resultFilePath) 38 | if err != nil { 39 | return errors.Wrapf(err, "failed to load results from %s", resultFilePath) 40 | } 41 | 42 | treeAsString := tree.NewResultTreeProducer().String(results) 43 | 44 | _, err = fmt.Fprintln(out, treeAsString) 45 | 46 | return errors.Wrap(err, "failed to print result tree") 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | on: [push, pull_request] 3 | jobs: 4 | 5 | ci: 6 | strategy: 7 | matrix: 8 | go-version: [1.18.x] 9 | platform: [macOS-latest, ubuntu-18.04] 10 | name: Continuous Integration on go ${{ matrix.go-version }}/${{ matrix.platform }} 11 | runs-on: ${{ matrix.platform }} 12 | steps: 13 | - name: Set up Go 14 | uses: actions/setup-go@v1 15 | with: 16 | go-version: ${{ matrix.go-version }} 17 | id: go 18 | 19 | - name: Checkout code 20 | uses: actions/checkout@v1 21 | with: 22 | fetch-depth: 1 23 | 24 | - name: Debug 25 | run: | 26 | echo "pwd:" 27 | pwd 28 | echo "HOME:" 29 | echo ${HOME} 30 | echo "GITHUB_WORKSPACE:" 31 | echo ${GITHUB_WORKSPACE} 32 | echo "GOPATH:" 33 | echo ${GOPATH} 34 | echo "GOROOT:" 35 | echo ${GOROOT} 36 | 37 | - name: Get dependencies 38 | run: make dep 39 | 40 | - name: Check codestyle 41 | if: matrix.platform == 'ubuntu-18.04' 42 | run: | 43 | export PATH=$PATH:$(go env GOPATH)/bin 44 | make check 45 | 46 | - name: Tests 47 | run: CI=true make tests 48 | 49 | - name: Functional Tests 50 | run: make functional-tests 51 | 52 | - name: Codecov coverage upload 53 | if: matrix.platform == 'ubuntu-18.04' 54 | uses: codecov/codecov-action@v1.0.2 55 | with: 56 | token: ${{secrets.CODECOV_TOKEN}} 57 | file: ./coverage.txt 58 | -------------------------------------------------------------------------------- /pkg/common/test/server.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "sync" 7 | ) 8 | 9 | func NewServerWithAssertion(handler http.HandlerFunc) (*httptest.Server, *ServerAssertion) { 10 | serverAssertion := &ServerAssertion{} 11 | 12 | server := httptest.NewServer(serverAssertion.wrap(handler)) 13 | 14 | return server, serverAssertion 15 | } 16 | 17 | func NewTSLServerWithAssertion(handler http.HandlerFunc) (*httptest.Server, *ServerAssertion) { 18 | serverAssertion := &ServerAssertion{} 19 | 20 | server := httptest.NewUnstartedServer(serverAssertion.wrap(handler)) 21 | server.StartTLS() 22 | 23 | return server, serverAssertion 24 | } 25 | 26 | type ServerAssertion struct { 27 | requests []http.Request 28 | requestsMx sync.RWMutex 29 | } 30 | 31 | func (s *ServerAssertion) wrap(handler http.HandlerFunc) http.HandlerFunc { 32 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 33 | handler(w, r) 34 | 35 | s.requestsMx.Lock() 36 | defer s.requestsMx.Unlock() 37 | 38 | s.requests = append(s.requests, *r) 39 | }) 40 | } 41 | 42 | func (s *ServerAssertion) Range(fn func(index int, r http.Request)) { 43 | s.requestsMx.RLock() 44 | defer s.requestsMx.RUnlock() 45 | 46 | for i, r := range s.requests { 47 | fn(i, r) 48 | } 49 | } 50 | 51 | func (s *ServerAssertion) At(index int, fn func(r http.Request)) { 52 | s.requestsMx.RLock() 53 | defer s.requestsMx.RUnlock() 54 | 55 | fn(s.requests[index]) 56 | } 57 | 58 | func (s *ServerAssertion) Len() int { 59 | s.requestsMx.RLock() 60 | defer s.requestsMx.RUnlock() 61 | 62 | return len(s.requests) 63 | } 64 | -------------------------------------------------------------------------------- /pkg/common/urlpath/join_test.go: -------------------------------------------------------------------------------- 1 | package urlpath_test 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/stefanoj3/dirstalk/pkg/common/urlpath" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestJoin(t *testing.T) { 13 | testCases := []struct { 14 | input []string 15 | expectedOutput string 16 | }{ 17 | { 18 | input: []string{"/home"}, 19 | expectedOutput: "/home", 20 | }, 21 | { 22 | input: []string{"/home/"}, 23 | expectedOutput: "/home/", 24 | }, 25 | { 26 | input: []string{"/home/", "test"}, 27 | expectedOutput: "/home/test", 28 | }, 29 | { 30 | input: []string{"/home/", "/test"}, 31 | expectedOutput: "/home/test", 32 | }, 33 | { 34 | input: []string{"/home/", "/test/"}, 35 | expectedOutput: "/home/test/", 36 | }, 37 | { 38 | input: []string{"/home", "/test/"}, 39 | expectedOutput: "/home/test/", 40 | }, 41 | { 42 | input: []string{"/home", "test/"}, 43 | expectedOutput: "/home/test/", 44 | }, 45 | { 46 | input: []string{"/home", "test"}, 47 | expectedOutput: "/home/test", 48 | }, 49 | } 50 | 51 | for _, tc := range testCases { 52 | tc := tc // Pinning ranged variable, more info: https://github.com/kyoh86/scopelint 53 | 54 | scenario := fmt.Sprintf( 55 | "Input: `%s`, Expected output: `%s`", 56 | strings.Join(tc.input, ","), 57 | tc.expectedOutput, 58 | ) 59 | 60 | t.Run(scenario, func(t *testing.T) { 61 | t.Parallel() 62 | 63 | output := urlpath.Join(tc.input...) 64 | 65 | assert.Equal(t, tc.expectedOutput, output) 66 | }) 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /pkg/scan/producer/reproducer.go: -------------------------------------------------------------------------------- 1 | package producer 2 | 3 | import ( 4 | "context" 5 | "sync" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/common/urlpath" 8 | "github.com/stefanoj3/dirstalk/pkg/scan" 9 | ) 10 | 11 | const defaultChannelBuffer = 25 12 | 13 | func NewReProducer( 14 | producer scan.Producer, 15 | ) *ReProducer { 16 | return &ReProducer{producer: producer} 17 | } 18 | 19 | type ReProducer struct { 20 | producer scan.Producer 21 | } 22 | 23 | // Reproduce will check if it is possible to go deeper on the result provided, if so will. 24 | func (r *ReProducer) Reproduce(ctx context.Context) func(r scan.Result) <-chan scan.Target { 25 | return r.buildReproducer(ctx) 26 | } 27 | 28 | func (r *ReProducer) buildReproducer(ctx context.Context) func(result scan.Result) <-chan scan.Target { 29 | resultRegistry := sync.Map{} 30 | 31 | return func(result scan.Result) <-chan scan.Target { 32 | resultChannel := make(chan scan.Target, defaultChannelBuffer) 33 | 34 | go func() { 35 | defer close(resultChannel) 36 | 37 | if result.Target.Depth <= 0 { 38 | return 39 | } 40 | 41 | // no point in appending to a filename 42 | if urlpath.HasExtension(result.Target.Path) { 43 | return 44 | } 45 | 46 | _, inRegistry := resultRegistry.Load(result.Target.Path) 47 | if inRegistry { 48 | return 49 | } 50 | 51 | resultRegistry.Store(result.Target.Path, nil) 52 | 53 | for target := range r.producer.Produce(ctx) { 54 | newTarget := result.Target 55 | newTarget.Depth-- 56 | newTarget.Path = urlpath.Join(newTarget.Path, target.Path) 57 | newTarget.Method = target.Method 58 | 59 | resultChannel <- newTarget 60 | } 61 | }() 62 | 63 | return resultChannel 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /pkg/cmd/flags.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | const ( 4 | // Root flags. 5 | flagRootVerbose = "verbose" 6 | flagRootVerboseShort = "v" 7 | 8 | // Scan flags. 9 | flagScanDictionary = "dictionary" 10 | flagScanDictionaryShort = "d" 11 | flagScanDictionaryGetTimeout = "dictionary-get-timeout" 12 | flagScanHTTPMethods = "http-methods" 13 | flagScanHTTPStatusesToIgnore = "http-statuses-to-ignore" 14 | flagScanHTTPTimeout = "http-timeout" 15 | flagScanHTTPCacheRequests = "http-cache-requests" 16 | flagScanScanDepth = "scan-depth" 17 | flagScanThreads = "threads" 18 | flagScanThreadsShort = "t" 19 | flagScanSocks5Host = "socks5" 20 | flagScanUserAgent = "user-agent" 21 | flagScanCookieJar = "use-cookie-jar" 22 | flagScanCookie = "cookie" 23 | flagScanHeader = "header" 24 | flagScanResultOutput = "out" 25 | flagShouldSkipSSLCertificatesValidation = "no-check-certificate" 26 | 27 | flagIgnore20xWithEmptyBody = "ignore-empty-body" 28 | 29 | // Generate dictionary flags. 30 | flagDictionaryGenerateOutput = "out" 31 | flagDictionaryGenerateOutputShort = "o" 32 | flagDictionaryGenerateAbsolutePathOnly = "absolute-only" 33 | 34 | // Result view flags. 35 | flagResultViewResultFile = "result-file" 36 | flagResultViewResultFileShort = "r" 37 | 38 | // Result diff flags. 39 | flagResultDiffFirstFile = "first" 40 | flagResultDiffFirstFileShort = "f" 41 | flagResultDiffSecondFile = "second" 42 | flagResultDiffSecondFileShort = "s" 43 | ) 44 | -------------------------------------------------------------------------------- /pkg/dictionary/generator_test.go: -------------------------------------------------------------------------------- 1 | package dictionary_test 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/dictionary" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestAbsolutePathsGenerator(t *testing.T) { 12 | t.Parallel() 13 | 14 | b := &bytes.Buffer{} 15 | 16 | dictionaryGenerator := dictionary.NewGenerator(b) 17 | 18 | err := dictionaryGenerator.GenerateDictionaryFrom( 19 | "./testdata/directory_to_generate_dictionary", 20 | true, 21 | ) 22 | assert.NoError(t, err) 23 | 24 | expectedOutput := `testdata/directory_to_generate_dictionary/myfile.php 25 | testdata/directory_to_generate_dictionary/subfolder/image.jpg 26 | testdata/directory_to_generate_dictionary/subfolder/image2.gif 27 | testdata/directory_to_generate_dictionary/subfolder/subsubfolder/myfile.php 28 | testdata/directory_to_generate_dictionary/subfolder/subsubfolder/myfile2.php 29 | ` 30 | 31 | assert.Equal(t, expectedOutput, b.String()) 32 | } 33 | 34 | func TestFilenamePathsGenerator(t *testing.T) { 35 | t.Parallel() 36 | 37 | b := &bytes.Buffer{} 38 | 39 | dictionaryGenerator := dictionary.NewGenerator(b) 40 | 41 | err := dictionaryGenerator.GenerateDictionaryFrom( 42 | "testdata/directory_to_generate_dictionary", 43 | false, 44 | ) 45 | assert.NoError(t, err) 46 | 47 | expectedOutput := `directory_to_generate_dictionary 48 | myfile.php 49 | subfolder 50 | image.jpg 51 | image2.gif 52 | subsubfolder 53 | myfile2.php 54 | ` 55 | 56 | assert.Equal(t, expectedOutput, b.String()) 57 | } 58 | 59 | func BenchmarkGenerateDictionaryFrom(b *testing.B) { 60 | buf := &bytes.Buffer{} 61 | 62 | dictionaryGenerator := dictionary.NewGenerator(buf) 63 | 64 | for i := 0; i < b.N; i++ { 65 | //nolint:errcheck 66 | _ = dictionaryGenerator.GenerateDictionaryFrom( 67 | "testdata/directory_to_generate_dictionary", 68 | false, 69 | ) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /pkg/cmd/root_integration_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "bytes" 5 | "os" 6 | "strings" 7 | "testing" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/spf13/cobra" 11 | "github.com/stefanoj3/dirstalk/pkg/cmd" 12 | "github.com/stefanoj3/dirstalk/pkg/common/test" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestRootCommand(t *testing.T) { 17 | logger, _ := test.NewLogger() 18 | 19 | c := createCommand(logger) 20 | assert.NotNil(t, c) 21 | 22 | err := executeCommand(c) 23 | assert.NoError(t, err) 24 | } 25 | 26 | func TestVersionCommand(t *testing.T) { 27 | logger, buf := test.NewLogger() 28 | 29 | c := createCommand(logger) 30 | assert.NotNil(t, c) 31 | 32 | err := executeCommand(c, "version") 33 | assert.NoError(t, err) 34 | 35 | // Ensure the command ran and produced some of the expected output 36 | // it is not in the scope of this test to ensure the correct output 37 | assert.Contains(t, buf.String(), "Version: ") 38 | } 39 | 40 | func executeCommand(root *cobra.Command, args ...string) (err error) { 41 | buf := new(bytes.Buffer) 42 | root.SetOutput(buf) 43 | 44 | a := []string{""} 45 | os.Args = append(a, args...) //nolint 46 | 47 | _, err = root.ExecuteC() 48 | 49 | return err 50 | } 51 | 52 | func removeTestFile(path string) { 53 | if !strings.Contains(path, "testdata") { 54 | return 55 | } 56 | 57 | _ = os.Remove(path) //nolint:errcheck 58 | } 59 | 60 | func createCommand(logger *logrus.Logger) *cobra.Command { 61 | dirStalkCmd := cmd.NewRootCommand(logger) 62 | 63 | dirStalkCmd.AddCommand(cmd.NewScanCommand(logger)) 64 | dirStalkCmd.AddCommand(cmd.NewResultViewCommand(logger.Out)) 65 | dirStalkCmd.AddCommand(cmd.NewResultDiffCommand(logger.Out)) 66 | dirStalkCmd.AddCommand(cmd.NewGenerateDictionaryCommand(logger.Out)) 67 | dirStalkCmd.AddCommand(cmd.NewVersionCommand(logger.Out)) 68 | 69 | return dirStalkCmd 70 | } 71 | -------------------------------------------------------------------------------- /pkg/dictionary/generator.go: -------------------------------------------------------------------------------- 1 | package dictionary 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/pkg/errors" 10 | ) 11 | 12 | func NewGenerator(out io.Writer) *Generator { 13 | return &Generator{out: out} 14 | } 15 | 16 | type Generator struct { 17 | out io.Writer 18 | } 19 | 20 | func (g *Generator) GenerateDictionaryFrom(path string, absoluteOnly bool) error { 21 | var ( 22 | dictionary []string 23 | err error 24 | ) 25 | 26 | if absoluteOnly { 27 | dictionary, err = findAbsolutePaths(path) 28 | } else { 29 | dictionary, err = findFileNames(path) 30 | } 31 | 32 | if err != nil { 33 | return errors.Wrap(err, "failed to generate dictionary") 34 | } 35 | 36 | for _, entry := range dictionary { 37 | _, err = fmt.Fprintln(g.out, entry) 38 | if err != nil { 39 | return errors.Wrap(err, "failed to write to buffer") 40 | } 41 | } 42 | 43 | return nil 44 | } 45 | 46 | func findAbsolutePaths(root string) ([]string, error) { 47 | var files []string 48 | 49 | err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { 50 | if err != nil { 51 | return errors.Wrap(err, "findAbsolutePaths: failed to walk") 52 | } 53 | 54 | if !info.IsDir() { 55 | files = append(files, p) 56 | } 57 | 58 | return nil 59 | }) 60 | 61 | return files, err 62 | } 63 | 64 | func findFileNames(root string) ([]string, error) { 65 | var files []string 66 | 67 | filesByKey := make(map[string]bool) 68 | 69 | err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { 70 | if err != nil { 71 | return errors.Wrap(err, "findFileNames: failed to walk") 72 | } 73 | 74 | if _, ok := filesByKey[info.Name()]; !ok { 75 | filesByKey[info.Name()] = true 76 | files = append(files, info.Name()) 77 | 78 | return nil 79 | } 80 | 81 | return nil 82 | }) 83 | 84 | return files, err 85 | } 86 | -------------------------------------------------------------------------------- /pkg/scan/producer/dictionary_test.go: -------------------------------------------------------------------------------- 1 | package producer_test 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "testing" 7 | 8 | "github.com/stefanoj3/dirstalk/pkg/scan" 9 | "github.com/stefanoj3/dirstalk/pkg/scan/producer" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestDictionaryProducerShouldProduce(t *testing.T) { 14 | t.Parallel() 15 | 16 | const depth = 4 17 | 18 | sut := producer.NewDictionaryProducer( 19 | []string{http.MethodGet, http.MethodPost}, 20 | []string{"/home", "/about"}, 21 | depth, 22 | ) 23 | 24 | results := make([]scan.Target, 0, 4) 25 | 26 | producerChannel := sut.Produce(context.Background()) 27 | for r := range producerChannel { 28 | results = append(results, r) 29 | } 30 | 31 | assert.Len(t, results, 4) 32 | 33 | expectedResults := []scan.Target{ 34 | { 35 | Depth: depth, 36 | Path: "/home", 37 | Method: http.MethodGet, 38 | }, 39 | { 40 | Depth: depth, 41 | Path: "/home", 42 | Method: http.MethodPost, 43 | }, 44 | { 45 | Depth: depth, 46 | Path: "/about", 47 | Method: http.MethodGet, 48 | }, 49 | { 50 | Depth: depth, 51 | Path: "/about", 52 | Method: http.MethodPost, 53 | }, 54 | } 55 | 56 | assert.Equal(t, expectedResults, results) 57 | } 58 | 59 | func TestDictionaryProducerCanBeCanceled(t *testing.T) { 60 | t.Parallel() 61 | 62 | const depth = 4 63 | 64 | sut := producer.NewDictionaryProducer( 65 | []string{http.MethodGet, http.MethodPost, http.MethodPatch, http.MethodDelete}, 66 | []string{"/home", "/about", "/index", "/search", "/tomato"}, 67 | depth, 68 | ) 69 | 70 | ctx, cancelFunc := context.WithCancel(context.Background()) 71 | 72 | producerChannel := sut.Produce(ctx) 73 | 74 | cancelFunc() 75 | 76 | resultsCount := 0 77 | 78 | for range producerChannel { 79 | resultsCount++ 80 | } 81 | 82 | // 11 is the size of the producer buffer 83 | assert.True(t, resultsCount <= 11) 84 | } 85 | -------------------------------------------------------------------------------- /pkg/dictionary/dictionary.go: -------------------------------------------------------------------------------- 1 | package dictionary 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "net/http" 7 | "os" 8 | "strings" 9 | 10 | "github.com/pkg/errors" 11 | ) 12 | 13 | const commentPrefix = "#" 14 | 15 | func NewDictionaryFrom(path string, doer Doer) ([]string, error) { 16 | if strings.HasPrefix(path, "http") { 17 | return newDictionaryFromRemoteFile(path, doer) 18 | } 19 | 20 | return newDictionaryFromLocalFile(path) 21 | } 22 | 23 | func newDictionaryFromLocalFile(path string) ([]string, error) { 24 | file, err := os.Open(path) // #nosec 25 | if err != nil { 26 | return nil, errors.Wrapf(err, "dictionary: unable to open: %s", path) 27 | } 28 | 29 | defer file.Close() //nolint 30 | 31 | return dictionaryFromReader(file), nil 32 | } 33 | 34 | func dictionaryFromReader(reader io.Reader) []string { 35 | entries := make([]string, 0) 36 | scanner := bufio.NewScanner(reader) 37 | 38 | for scanner.Scan() { 39 | line := scanner.Text() 40 | if len(line) == 0 { 41 | continue 42 | } 43 | 44 | if isAComment(line) { 45 | continue 46 | } 47 | 48 | entries = append(entries, line) 49 | } 50 | 51 | return entries 52 | } 53 | 54 | func newDictionaryFromRemoteFile(path string, doer Doer) ([]string, error) { 55 | req, err := http.NewRequest(http.MethodGet, path, nil) //nolint 56 | if err != nil { 57 | return nil, errors.Wrapf(err, "dictionary: failed to build request for `%s`", path) 58 | } 59 | 60 | res, err := doer.Do(req) 61 | if err != nil { 62 | return nil, errors.Wrapf(err, "dictionary: failed to get `%s`", path) 63 | } 64 | 65 | defer res.Body.Close() //nolint:errcheck 66 | 67 | statusCode := res.StatusCode 68 | if statusCode > 299 || statusCode < 200 { 69 | return nil, errors.Errorf( 70 | "dictionary: failed to retrieve from `%s`, status code %d", 71 | path, 72 | statusCode, 73 | ) 74 | } 75 | 76 | return dictionaryFromReader(res.Body), nil 77 | } 78 | 79 | func isAComment(line string) bool { 80 | return line[0:1] == commentPrefix 81 | } 82 | -------------------------------------------------------------------------------- /pkg/cmd/result_diff_integration_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/common/test" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestNewResultDiff(t *testing.T) { 12 | logger, loggerBuffer := test.NewLogger() 13 | 14 | c := createCommand(logger) 15 | assert.NotNil(t, c) 16 | 17 | err := executeCommand(c, "result.diff", "-f", "testdata/out.txt", "-s", "testdata/out2.txt") 18 | assert.NoError(t, err) 19 | 20 | // to keep compatibility with other systems open, the language should take care to use the correct newline symbol 21 | newlineSymbol := fmt.Sprintln() 22 | 23 | expected := "/" + newlineSymbol + 24 | "├── adview" + newlineSymbol + 25 | "├── partners" + newlineSymbol + 26 | "│ └── \x1b[31mterms\x1b[0m\x1b[32m123\x1b[0m" + newlineSymbol + 27 | "└── s" 28 | 29 | assert.Contains(t, loggerBuffer.String(), expected) 30 | } 31 | 32 | func TestNewResultDiffShouldErrWithInvalidFirstFile(t *testing.T) { 33 | logger, _ := test.NewLogger() 34 | 35 | c := createCommand(logger) 36 | assert.NotNil(t, c) 37 | 38 | err := executeCommand(c, "result.diff", "-f", "/root/123/bla", "-s", "testdata/out2.txt") 39 | assert.Error(t, err) 40 | 41 | assert.Contains(t, err.Error(), "/root/123/bla") 42 | } 43 | 44 | func TestNewResultDiffShouldErrWithInvalidSecondFile(t *testing.T) { 45 | logger, _ := test.NewLogger() 46 | 47 | c := createCommand(logger) 48 | assert.NotNil(t, c) 49 | 50 | err := executeCommand(c, "result.diff", "-f", "testdata/out2.txt", "-s", "/root/123/bla") 51 | assert.Error(t, err) 52 | 53 | assert.Contains(t, err.Error(), "/root/123/bla") 54 | } 55 | 56 | func TestDiffForSameFileShouldErr(t *testing.T) { 57 | logger, _ := test.NewLogger() 58 | 59 | c := createCommand(logger) 60 | assert.NotNil(t, c) 61 | 62 | err := executeCommand(c, "result.diff", "-f", "testdata/out.txt", "-s", "testdata/out.txt") 63 | assert.Error(t, err) 64 | assert.Contains(t, err.Error(), "no diffs found") 65 | } 66 | -------------------------------------------------------------------------------- /pkg/scan/filter/http_test.go: -------------------------------------------------------------------------------- 1 | package filter_test 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/stefanoj3/dirstalk/pkg/scan" 10 | "github.com/stefanoj3/dirstalk/pkg/scan/filter" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestHTTPStatusResultFilter(t *testing.T) { 15 | testCases := []struct { 16 | statusCodesToIgnore []int 17 | result scan.Result 18 | expectedResult bool 19 | }{ 20 | { 21 | statusCodesToIgnore: []int{http.StatusCreated, http.StatusNotFound}, 22 | result: scan.Result{StatusCode: http.StatusOK}, 23 | expectedResult: false, 24 | }, 25 | { 26 | statusCodesToIgnore: []int{http.StatusCreated, http.StatusNotFound}, 27 | result: scan.Result{StatusCode: http.StatusNotFound}, 28 | expectedResult: true, 29 | }, 30 | { 31 | statusCodesToIgnore: []int{}, 32 | result: scan.Result{StatusCode: http.StatusNotFound}, 33 | expectedResult: false, 34 | }, 35 | { 36 | statusCodesToIgnore: []int{}, 37 | result: scan.Result{StatusCode: http.StatusOK}, 38 | expectedResult: false, 39 | }, 40 | } 41 | 42 | for _, tc := range testCases { 43 | tc := tc // Pinning ranged variable, more info: https://github.com/kyoh86/scopelint 44 | 45 | scenario := fmt.Sprintf("ignored: %v, result: %d", tc.statusCodesToIgnore, tc.result.StatusCode) 46 | 47 | t.Run(scenario, func(t *testing.T) { 48 | t.Parallel() 49 | 50 | actual := filter.NewHTTPStatusResultFilter(tc.statusCodesToIgnore, false).ShouldIgnore(tc.result) 51 | assert.Equal(t, tc.expectedResult, actual) 52 | }) 53 | } 54 | } 55 | 56 | func TestHTTPStatusResultFilterShouldWorkConcurrently(_ *testing.T) { 57 | sut := filter.NewHTTPStatusResultFilter(nil, false) 58 | 59 | wg := sync.WaitGroup{} 60 | 61 | for i := 0; i < 1000; i++ { 62 | wg.Add(1) 63 | 64 | go func(i int) { 65 | sut.ShouldIgnore(scan.Result{StatusCode: i}) 66 | wg.Done() 67 | }(i) 68 | } 69 | 70 | wg.Wait() 71 | } 72 | -------------------------------------------------------------------------------- /pkg/result/load_integration_test.go: -------------------------------------------------------------------------------- 1 | package result_test 2 | 3 | import ( 4 | "net/url" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/result" 8 | "github.com/stefanoj3/dirstalk/pkg/scan" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestLoadResultsFromFile(t *testing.T) { 13 | results, err := result.LoadResultsFromFile("testdata/out.txt") 14 | assert.NoError(t, err) 15 | 16 | expectedResults := []scan.Result{ 17 | { 18 | Target: scan.Target{Path: "partners", Method: "GET", Depth: 3}, 19 | StatusCode: 200, 20 | URL: url.URL{ 21 | Scheme: "https", 22 | User: (*url.Userinfo)(nil), 23 | Host: "www.brucewillisdiesinarmageddon.co.de", 24 | Path: "/partners", 25 | }, 26 | }, 27 | { 28 | Target: scan.Target{Path: "s", Method: "GET", Depth: 3}, 29 | StatusCode: 400, 30 | URL: url.URL{ 31 | Scheme: "https", 32 | User: (*url.Userinfo)(nil), 33 | Host: "www.brucewillisdiesinarmageddon.co.de", 34 | Path: "/s", 35 | }, 36 | }, 37 | { 38 | Target: scan.Target{Path: "adview", Method: "GET", Depth: 3}, 39 | StatusCode: 204, 40 | URL: url.URL{ 41 | Scheme: "https", 42 | User: (*url.Userinfo)(nil), 43 | Host: "www.brucewillisdiesinarmageddon.co.de", 44 | Path: "/adview", 45 | }, 46 | }, 47 | { 48 | Target: scan.Target{Path: "partners/terms", Method: "GET", Depth: 2}, 49 | StatusCode: 200, 50 | URL: url.URL{ 51 | Scheme: "https", 52 | User: (*url.Userinfo)(nil), 53 | Host: "www.brucewillisdiesinarmageddon.co.de", 54 | Path: "/partners/terms", 55 | }, 56 | }, 57 | } 58 | 59 | assert.Equal(t, expectedResults, results) 60 | } 61 | 62 | func TestLoadResultsFromFileShouldErrForDirectories(t *testing.T) { 63 | _, err := result.LoadResultsFromFile("testdata/") 64 | assert.Error(t, err) 65 | 66 | assert.Contains(t, err.Error(), "is a directory") 67 | } 68 | 69 | func TestLoadResultsFromFileShouldErrForInvalidFileFormat(t *testing.T) { 70 | _, err := result.LoadResultsFromFile("testdata/invalidout.txt") 71 | assert.Error(t, err) 72 | 73 | assert.Contains(t, err.Error(), "unable to read line") 74 | } 75 | -------------------------------------------------------------------------------- /pkg/cmd/generate_dictionary.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | 8 | "github.com/pkg/errors" 9 | "github.com/spf13/cobra" 10 | "github.com/stefanoj3/dirstalk/pkg/dictionary" 11 | ) 12 | 13 | func NewGenerateDictionaryCommand(out io.Writer) *cobra.Command { 14 | cmd := &cobra.Command{ 15 | Use: "dictionary.generate [path]", 16 | Short: "Generate a dictionary from the given folder", 17 | RunE: buildGenerateDictionaryFunc(out), 18 | } 19 | 20 | cmd.Flags().StringP( 21 | flagDictionaryGenerateOutput, 22 | flagDictionaryGenerateOutputShort, 23 | "", 24 | fmt.Sprintf("where to write the dictionary"), 25 | ) 26 | 27 | cmd.Flags().BoolP( 28 | flagDictionaryGenerateAbsolutePathOnly, 29 | "", 30 | false, 31 | "determines if the dictionary should contain only the absolute path of the files", 32 | ) 33 | 34 | return cmd 35 | } 36 | 37 | func buildGenerateDictionaryFunc(out io.Writer) func(cmd *cobra.Command, args []string) error { 38 | f := func(cmd *cobra.Command, args []string) error { 39 | p, err := getPath(args) 40 | if err != nil { 41 | return err 42 | } 43 | 44 | out, err := getOutputForDictionaryGenerator(cmd, out) 45 | if err != nil { 46 | return err 47 | } 48 | 49 | absolutePathOnly, err := cmd.Flags().GetBool(flagDictionaryGenerateAbsolutePathOnly) 50 | if err != nil { 51 | return errors.Wrapf(err, "failed to retrieve %s flag", flagDictionaryGenerateAbsolutePathOnly) 52 | } 53 | 54 | generator := dictionary.NewGenerator(out) 55 | 56 | return generator.GenerateDictionaryFrom(p, absolutePathOnly) 57 | } 58 | 59 | return f 60 | } 61 | 62 | func getOutputForDictionaryGenerator(cmd *cobra.Command, out io.Writer) (io.Writer, error) { 63 | output := cmd.Flag(flagDictionaryGenerateOutput).Value.String() 64 | if output == "" { 65 | return out, nil 66 | } 67 | 68 | file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY, 0600) //nolint:gosec 69 | if err != nil { 70 | return nil, errors.Wrap(err, "cannot write on the path provided") 71 | } 72 | 73 | return file, nil 74 | } 75 | 76 | func getPath(args []string) (string, error) { 77 | if len(args) == 0 { 78 | return "", errors.New("no path provided") 79 | } 80 | 81 | path := args[0] 82 | 83 | fileInfo, err := os.Stat(path) 84 | if err != nil { 85 | return "", errors.Wrap(err, "unable to use the provided path") 86 | } 87 | 88 | if !fileInfo.IsDir() { 89 | return "", errors.New("the path should be a directory") 90 | } 91 | 92 | return path, nil 93 | } 94 | -------------------------------------------------------------------------------- /pkg/cmd/result_diff.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | 7 | "github.com/pkg/errors" 8 | "github.com/sergi/go-diff/diffmatchpatch" 9 | "github.com/spf13/cobra" 10 | "github.com/stefanoj3/dirstalk/pkg/common" 11 | "github.com/stefanoj3/dirstalk/pkg/result" 12 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer/tree" 13 | ) 14 | 15 | func NewResultDiffCommand(out io.Writer) *cobra.Command { 16 | cmd := &cobra.Command{ 17 | Use: "result.diff", 18 | Short: "Prints differences between 2 result files", 19 | RunE: buildResultDiffCmd(out), 20 | } 21 | 22 | cmd.Flags().StringP( 23 | flagResultDiffFirstFile, 24 | flagResultDiffFirstFileShort, 25 | "", 26 | "first result file to read", 27 | ) 28 | common.Must(cmd.MarkFlagFilename(flagResultDiffFirstFile)) 29 | common.Must(cmd.MarkFlagRequired(flagResultDiffFirstFile)) 30 | 31 | cmd.Flags().StringP( 32 | flagResultDiffSecondFile, 33 | flagResultDiffSecondFileShort, 34 | "", 35 | "second result file to read", 36 | ) 37 | common.Must(cmd.MarkFlagFilename(flagResultDiffSecondFile)) 38 | common.Must(cmd.MarkFlagRequired(flagResultDiffSecondFile)) 39 | 40 | return cmd 41 | } 42 | 43 | func buildResultDiffCmd(out io.Writer) func(cmd *cobra.Command, args []string) error { 44 | return func(cmd *cobra.Command, args []string) error { 45 | firstResultFilePath := cmd.Flag(flagResultDiffFirstFile).Value.String() 46 | 47 | resultsFirst, err := result.LoadResultsFromFile(firstResultFilePath) 48 | if err != nil { 49 | return errors.Wrapf(err, "failed to load results from %s", firstResultFilePath) 50 | } 51 | 52 | secondResultFilePath := cmd.Flag(flagResultDiffSecondFile).Value.String() 53 | 54 | resultsSecond, err := result.LoadResultsFromFile(secondResultFilePath) 55 | if err != nil { 56 | return errors.Wrapf(err, "failed to load results from %s", secondResultFilePath) 57 | } 58 | 59 | treeProducer := tree.NewResultTreeProducer() 60 | 61 | differ := diffmatchpatch.New() 62 | diffs := differ.DiffMain( 63 | treeProducer.String(resultsFirst), 64 | treeProducer.String(resultsSecond), 65 | false, 66 | ) 67 | 68 | if isEqual(diffs) { 69 | return errors.New("no diffs found") 70 | } 71 | 72 | _, err = fmt.Fprintln(out, differ.DiffPrettyText(diffs)) 73 | 74 | return errors.Wrap(err, "failed to print results diff") 75 | } 76 | } 77 | 78 | func isEqual(diffs []diffmatchpatch.Diff) bool { 79 | if len(diffs) != 1 { 80 | return false 81 | } 82 | 83 | return diffs[0].Type == diffmatchpatch.DiffEqual 84 | } 85 | -------------------------------------------------------------------------------- /pkg/cmd/dictionary_integration_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/common/test" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestDictionaryGenerateCommand(t *testing.T) { 12 | logger, _ := test.NewLogger() 13 | 14 | c := createCommand(logger) 15 | assert.NotNil(t, c) 16 | 17 | testFilePath := "testdata/" + test.RandStringRunes(10) 18 | defer removeTestFile(testFilePath) 19 | err := executeCommand(c, "dictionary.generate", ".", "-o", testFilePath) 20 | assert.NoError(t, err) 21 | 22 | //nolint:gosec 23 | content, err := ioutil.ReadFile(testFilePath) 24 | assert.NoError(t, err) 25 | 26 | // Ensure the command ran and produced some of the expected output 27 | // it is not in the scope of this test to ensure the correct output 28 | assert.Contains(t, string(content), "root_integration_test.go") 29 | } 30 | 31 | func TestDictionaryGenerateCommandShouldErrWhenNoTargetIsProvided(t *testing.T) { 32 | logger, _ := test.NewLogger() 33 | 34 | c := createCommand(logger) 35 | assert.NotNil(t, c) 36 | 37 | err := executeCommand(c, "dictionary.generate") 38 | assert.Error(t, err) 39 | 40 | assert.Contains(t, err.Error(), "no path provided") 41 | } 42 | 43 | func TestDictionaryGenerateShouldFailWhenAFilePathIsProvidedInsteadOfADirectory(t *testing.T) { 44 | logger, _ := test.NewLogger() 45 | 46 | c := createCommand(logger) 47 | assert.NotNil(t, c) 48 | 49 | testFilePath := "testdata/" + test.RandStringRunes(10) 50 | defer removeTestFile(testFilePath) 51 | 52 | err := executeCommand(c, "dictionary.generate", "./root_integration_test.go") 53 | assert.Error(t, err) 54 | 55 | assert.Contains(t, err.Error(), "the path should be a directory") 56 | } 57 | 58 | func TestGenerateDictionaryWithoutOutputPath(t *testing.T) { 59 | logger, loggerBuffer := test.NewLogger() 60 | 61 | c := createCommand(logger) 62 | assert.NotNil(t, c) 63 | 64 | err := executeCommand(c, "dictionary.generate", ".") 65 | assert.NoError(t, err) 66 | 67 | assert.Contains(t, loggerBuffer.String(), "root_integration_test.go") 68 | } 69 | 70 | func TestGenerateDictionaryWithInvalidDirectory(t *testing.T) { 71 | logger, _ := test.NewLogger() 72 | 73 | fakePath := "./" + test.RandStringRunes(10) 74 | c := createCommand(logger) 75 | assert.NotNil(t, c) 76 | 77 | err := executeCommand(c, "dictionary.generate", fakePath) 78 | assert.Error(t, err) 79 | 80 | assert.Contains(t, err.Error(), "unable to use the provided path") 81 | assert.Contains(t, err.Error(), fakePath) 82 | } 83 | -------------------------------------------------------------------------------- /pkg/scan/summarizer/result_summarizer.go: -------------------------------------------------------------------------------- 1 | package summarizer 2 | 3 | import ( 4 | "fmt" 5 | "net/http" 6 | "sort" 7 | "sync" 8 | 9 | "github.com/sirupsen/logrus" 10 | "github.com/stefanoj3/dirstalk/pkg/scan" 11 | ) 12 | 13 | const ( 14 | breakingText = "Found something breaking" 15 | foundText = "Found" 16 | ) 17 | 18 | func NewResultSummarizer(treePrinter ResultTree, logger *logrus.Logger) *ResultSummarizer { 19 | return &ResultSummarizer{ 20 | treePrinter: treePrinter, 21 | logger: logger, 22 | resultMap: make(map[string]struct{}), 23 | } 24 | } 25 | 26 | type ResultSummarizer struct { 27 | treePrinter ResultTree 28 | logger *logrus.Logger 29 | results []scan.Result 30 | resultMap map[string]struct{} 31 | mux sync.RWMutex 32 | } 33 | 34 | func (s *ResultSummarizer) Add(result scan.Result) { 35 | s.mux.Lock() 36 | defer s.mux.Unlock() 37 | 38 | key := keyForResult(result) 39 | 40 | if _, found := s.resultMap[key]; found { 41 | return 42 | } 43 | 44 | s.log(result) 45 | 46 | s.resultMap[key] = struct{}{} 47 | 48 | s.results = append(s.results, result) 49 | } 50 | 51 | func (s *ResultSummarizer) Summarize() { 52 | s.mux.Lock() 53 | defer s.mux.Unlock() 54 | 55 | sort.Slice(s.results, func(i, j int) bool { 56 | return s.results[i].Target.Path < s.results[j].Target.Path 57 | }) 58 | 59 | s.printSummary() 60 | s.printTree() 61 | 62 | for _, r := range s.results { 63 | _, _ = fmt.Fprintln( 64 | s.logger.Out, 65 | fmt.Sprintf( 66 | "%s [%d] [%s]", 67 | r.URL.String(), 68 | r.StatusCode, 69 | r.Target.Method, 70 | ), 71 | ) 72 | } 73 | } 74 | 75 | func (s *ResultSummarizer) printSummary() { 76 | _, _ = fmt.Fprintln( 77 | s.logger.Out, 78 | fmt.Sprintf("%d results found", len(s.results)), 79 | ) 80 | } 81 | 82 | func (s *ResultSummarizer) printTree() { 83 | _, _ = fmt.Fprintln(s.logger.Out, s.treePrinter.String(s.results)) 84 | } 85 | 86 | func (s *ResultSummarizer) log(result scan.Result) { 87 | statusCode := result.StatusCode 88 | 89 | l := s.logger.WithFields(logrus.Fields{ 90 | "status-code": statusCode, 91 | "method": result.Target.Method, 92 | "url": result.URL.String(), 93 | }) 94 | 95 | if statusCode >= http.StatusInternalServerError { 96 | l.Warn(breakingText) 97 | } else { 98 | l.Info(foundText) 99 | } 100 | } 101 | 102 | func keyForResult(result scan.Result) string { 103 | return fmt.Sprintf("%s~%s", result.URL.String(), result.Target.Method) 104 | } 105 | -------------------------------------------------------------------------------- /pkg/scan/output/saver_integration_test.go: -------------------------------------------------------------------------------- 1 | package output_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "os" 6 | "sync" 7 | "testing" 8 | 9 | "github.com/stefanoj3/dirstalk/pkg/common/test" 10 | "github.com/stefanoj3/dirstalk/pkg/scan" 11 | "github.com/stefanoj3/dirstalk/pkg/scan/output" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | func TestFileSaverShouldErrWhenInvalidPath(t *testing.T) { 16 | saver, err := output.NewFileSaver("/root/123/bla.txt") 17 | assert.Error(t, err) 18 | assert.Contains(t, err.Error(), "failed to create file") 19 | 20 | err = saver.Save(scan.Result{}) 21 | assert.Error(t, err) 22 | assert.Contains(t, err.Error(), "writeCloser is nil") 23 | 24 | err = saver.Close() 25 | assert.Error(t, err) 26 | assert.Contains(t, err.Error(), "writeCloser is nil") 27 | } 28 | 29 | func TestFileSaverShouldWriteResults(t *testing.T) { 30 | filename := test.RandStringRunes(10) 31 | filename = "testdata/" + filename + ".txt" 32 | 33 | defer func() { 34 | err := os.Remove(filename) 35 | if err != nil { 36 | t.Fatalf("%s failed to clean up file created during tests: %s", err, filename) 37 | } 38 | }() 39 | 40 | saver, err := output.NewFileSaver(filename) 41 | assert.NoError(t, err) 42 | 43 | err = saver.Save(scan.Result{}) 44 | assert.NoError(t, err) 45 | 46 | err = saver.Close() 47 | assert.NoError(t, err) 48 | 49 | //nolint:gosec 50 | file, err := os.Open(filename) 51 | assert.NoError(t, err) 52 | 53 | b, err := ioutil.ReadAll(file) 54 | assert.NoError(t, err) 55 | 56 | assert.NoError(t, file.Close()) 57 | 58 | expected := `{"Target":{"Path":"","Method":"","Depth":0},"StatusCode":0,"URL":{"Scheme":"","Opaque":"","User":null,"Host":"","Path":"","RawPath":"","ForceQuery":false,"RawQuery":"","Fragment":"","RawFragment":""},"ContentLength":0} 59 | ` 60 | assert.Equal( 61 | t, 62 | expected, 63 | string(b), 64 | ) 65 | } 66 | 67 | func TestFileSaverShouldWorkConcurrently(t *testing.T) { 68 | filename := test.RandStringRunes(10) 69 | filename = "testdata/" + filename + ".txt" 70 | 71 | defer func() { 72 | err := os.Remove(filename) 73 | if err != nil { 74 | t.Fatalf("%s failed to clean up file created during tests: %s", err, filename) 75 | } 76 | }() 77 | 78 | saver, err := output.NewFileSaver(filename) 79 | assert.NoError(t, err) 80 | 81 | wg := sync.WaitGroup{} 82 | 83 | const workers = 1000 84 | 85 | wg.Add(workers) 86 | 87 | for i := 0; i < workers; i++ { 88 | go func() { 89 | err := saver.Save(scan.Result{}) 90 | if err != nil { 91 | panic(err) 92 | } 93 | 94 | wg.Done() 95 | }() 96 | } 97 | 98 | wg.Wait() 99 | 100 | // checking that the file is there 101 | //nolint:gosec 102 | file, err := os.Open(filename) 103 | assert.NoError(t, err) 104 | assert.NoError(t, file.Close()) 105 | } 106 | -------------------------------------------------------------------------------- /pkg/scan/client/client.go: -------------------------------------------------------------------------------- 1 | package client 2 | 3 | import ( 4 | "context" 5 | "crypto/tls" 6 | "net" 7 | "net/http" 8 | "net/http/cookiejar" 9 | "net/url" 10 | "time" 11 | 12 | "github.com/pkg/errors" 13 | "github.com/stefanoj3/dirstalk/pkg/scan/client/cookie" 14 | "golang.org/x/net/proxy" 15 | ) 16 | 17 | func NewClientFromConfig( 18 | timeoutInMilliseconds int, 19 | socks5Url *url.URL, 20 | userAgent string, 21 | useCookieJar bool, 22 | cookies []*http.Cookie, 23 | headers map[string]string, 24 | shouldCacheRequests bool, 25 | shouldSkipSSLCertificatesValidation bool, 26 | u *url.URL, 27 | ) (*http.Client, error) { 28 | transport := buildTransport(shouldSkipSSLCertificatesValidation) 29 | 30 | c := &http.Client{ 31 | Timeout: time.Millisecond * time.Duration(timeoutInMilliseconds), 32 | Transport: transport, 33 | CheckRedirect: func(req *http.Request, via []*http.Request) error { 34 | return http.ErrUseLastResponse 35 | }, 36 | } 37 | 38 | if useCookieJar { 39 | jar, err := cookiejar.New(nil) 40 | if err != nil { 41 | return nil, errors.Wrap(err, "NewClientFromConfig: failed to create cookie jar") 42 | } 43 | 44 | c.Jar = jar 45 | } 46 | 47 | if c.Jar != nil { 48 | c.Jar.SetCookies(u, cookies) 49 | } 50 | 51 | if len(cookies) > 0 && c.Jar == nil { 52 | c.Jar = cookie.NewStatelessJar(cookies) 53 | } 54 | 55 | if socks5Url != nil { 56 | tbDialer, err := proxy.FromURL(socks5Url, proxy.Direct) 57 | if err != nil { 58 | return nil, errors.Wrap(err, "NewClientFromConfig: failed to create socks5 proxy") 59 | } 60 | 61 | transport.DialContext = func(ctx context.Context, network, addr string) (conn net.Conn, e error) { 62 | return tbDialer.Dial(network, addr) 63 | } 64 | } 65 | 66 | var err error 67 | 68 | c.Transport, err = decorateTransportWithUserAgentDecorator(c.Transport, userAgent) 69 | if err != nil { 70 | return nil, errors.Wrap(err, "NewClientFromConfig: failed to decorate transport") 71 | } 72 | 73 | if len(headers) > 0 { 74 | c.Transport, err = decorateTransportWithHeadersDecorator(c.Transport, headers) 75 | if err != nil { 76 | return nil, errors.Wrap(err, "NewClientFromConfig: failed to decorate transport") 77 | } 78 | } 79 | 80 | if shouldCacheRequests { 81 | c.Transport, err = decorateTransportWithRequestCacheDecorator(c.Transport) 82 | if err != nil { 83 | return nil, errors.Wrap(err, "NewClientFromConfig: failed to decorate transport") 84 | } 85 | } 86 | 87 | return c, nil 88 | } 89 | 90 | func buildTransport(shouldSkipSSLCertificatesValidation bool) *http.Transport { 91 | transport := http.Transport{ 92 | MaxIdleConns: 100, 93 | IdleConnTimeout: 90 * time.Second, 94 | TLSHandshakeTimeout: 10 * time.Second, 95 | ExpectContinueTimeout: 1 * time.Second, 96 | } 97 | 98 | if shouldSkipSSLCertificatesValidation { 99 | //nolint:gosec 100 | transport.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} 101 | } 102 | 103 | return &transport 104 | } 105 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SRC_DIRS=cmd pkg 2 | 3 | TESTARGS=-v -race -cover -timeout 20s -cpu 24 4 | 5 | VERSION=$(shell git describe || git rev-parse HEAD) 6 | DATE=$(shell date +%s) 7 | LD_FLAGS=-extldflags '-static' -X github.com/stefanoj3/dirstalk/pkg/cmd.Version=$(VERSION) -X github.com/stefanoj3/dirstalk/pkg/cmd.BuildTime=$(DATE) 8 | 9 | ifeq ($(CI), true) 10 | TESTARGS=-v -race -coverprofile=coverage.txt -covermode=atomic -timeout 20s -cpu 24 11 | endif 12 | 13 | .PHONY: dep 14 | ## Fetch dependencies 15 | dep: 16 | @go mod download 17 | 18 | .PHONY: tests 19 | ## Execute tests 20 | tests: 21 | @echo "Executing tests" 22 | @CGO_ENABLED=1 go test $(TESTARGS) ./... 23 | 24 | .PHONY: functional-tests 25 | ## Execute functional test 26 | functional-tests: build build-testserver 27 | ./functional-tests.sh 28 | 29 | .PHONY: check 30 | ## Run checks against the codebase 31 | check: 32 | docker run -t --rm -v $(PWD):/app -w /app golangci/golangci-lint:v1.30-alpine golangci-lint run -v 33 | 34 | .PHONY: fix 35 | ## Run goimports against the source code 36 | fix: 37 | docker run --rm -v $(pwd):/data cytopia/goimports -w . 38 | 39 | .PHONY: fmt 40 | ## Run fmt against the source code 41 | fmt: 42 | gofmt -s -w $(SRC_DIRS) 43 | 44 | .PHONY: release-snapshot 45 | ## Creates a release snapshot - requires goreleaser to be available in the $PATH 46 | release-snapshot: 47 | @echo "Creating release snapshot..." 48 | @goreleaser --snapshot --rm-dist 49 | 50 | .PHONY: release 51 | ## Creates a release - requires goreleaser to be available in the $PATH 52 | release: 53 | @echo "Creating release ..." 54 | @goreleaser release --skip-publish --rm-dist 55 | 56 | .PHONY: build 57 | ## Builds binary from source 58 | build: 59 | CGO_ENABLED=0 go build -a -installsuffix cgo -ldflags "$(LD_FLAGS)" -o dist/dirstalk cmd/dirstalk/main.go 60 | 61 | .PHONY: build-testserver 62 | ## Builds binary for testserver used for the functional tests 63 | build-testserver: 64 | go build -o dist/testserver cmd/testserver/main.go 65 | 66 | .PHONY: out-find 67 | ## Search for .out file (profiling) in the repo 68 | out-find: 69 | @echo "Searching for *.out files" 70 | find . -name '*.out' 71 | 72 | .PHONY: out-delete 73 | ## Delete *.out files from the repo 74 | out-delete: 75 | @echo "Delete *.out files" 76 | find . -name '*.out' -delete 77 | 78 | .PHONY: help 79 | HELP_WIDTH=" " 80 | ## Display makefile help 81 | help: 82 | @printf "Usage\n"; 83 | @awk '{ \ 84 | if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ 85 | helpCommand = substr($$0, index($$0, ":") + 2); \ 86 | if (helpMessage) { \ 87 | printf " \033[32m%-20s\033[0m %s\n", \ 88 | helpCommand, helpMessage; \ 89 | helpMessage = ""; \ 90 | } \ 91 | } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ 92 | helpCommand = substr($$0, 0, index($$0, ":")); \ 93 | if (helpMessage) { \ 94 | printf " \033[32m%-20s\033[0m %s\n", \ 95 | helpCommand, helpMessage; \ 96 | helpMessage = ""; \ 97 | } \ 98 | } else if ($$0 ~ /^##/) { \ 99 | if (helpMessage) { \ 100 | helpMessage = helpMessage"\n"${HELP_WIDTH}substr($$0, 3); \ 101 | } else { \ 102 | helpMessage = substr($$0, 3); \ 103 | } \ 104 | } else { \ 105 | if (helpMessage) { \ 106 | print "\n"${HELP_WIDTH}helpMessage"\n" \ 107 | } \ 108 | helpMessage = ""; \ 109 | } \ 110 | }' \ 111 | $(MAKEFILE_LIST) 112 | -------------------------------------------------------------------------------- /pkg/scan/producer/reproducer_test.go: -------------------------------------------------------------------------------- 1 | package producer_test 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "sort" 7 | "testing" 8 | 9 | "github.com/stefanoj3/dirstalk/pkg/common/test" 10 | "github.com/stefanoj3/dirstalk/pkg/scan" 11 | "github.com/stefanoj3/dirstalk/pkg/scan/producer" 12 | "github.com/stretchr/testify/assert" 13 | ) 14 | 15 | var benchData interface{} 16 | 17 | func TestNewReProducer(t *testing.T) { 18 | t.Parallel() 19 | 20 | methods := []string{http.MethodGet, http.MethodPost} 21 | dictionary := []string{"/home", "/about"} 22 | 23 | dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1) 24 | 25 | sut := producer.NewReProducer(dictionaryProducer) 26 | 27 | result := scan.NewResult( 28 | scan.Target{ 29 | Path: "/home", 30 | Method: http.MethodGet, 31 | Depth: 1, 32 | }, 33 | &http.Response{ 34 | StatusCode: http.StatusOK, 35 | Request: &http.Request{ 36 | URL: test.MustParseURL(t, "http://mysite/contacts"), 37 | }, 38 | }, 39 | ) 40 | 41 | reproducerFunc := sut.Reproduce(context.Background()) 42 | reproducerChannel := reproducerFunc(result) 43 | 44 | targets := make([]scan.Target, 0, 10) 45 | for tar := range reproducerChannel { 46 | targets = append(targets, tar) 47 | } 48 | 49 | sort.Slice(targets, func(i, j int) bool { 50 | return targets[i].Path < targets[j].Path && targets[i].Method < targets[j].Method 51 | }) 52 | 53 | assert.Len(t, targets, 4) 54 | 55 | expectedTargets := []scan.Target{ 56 | { 57 | Path: "/home/home", 58 | Method: http.MethodGet, 59 | Depth: 0, 60 | }, 61 | { 62 | Path: "/home/about", 63 | Method: http.MethodGet, 64 | Depth: 0, 65 | }, 66 | { 67 | Path: "/home/home", 68 | Method: http.MethodPost, 69 | Depth: 0, 70 | }, 71 | { 72 | Path: "/home/about", 73 | Method: http.MethodPost, 74 | Depth: 0, 75 | }, 76 | } 77 | assert.Equal(t, expectedTargets, targets) 78 | 79 | // reproducing again on the same result should not yield more targets 80 | reproducerChannel = reproducerFunc(result) 81 | 82 | targets = make([]scan.Target, 0) 83 | for tar := range reproducerChannel { 84 | targets = append(targets, tar) 85 | } 86 | 87 | assert.Len(t, targets, 0) 88 | } 89 | 90 | func TestReProducerShouldProduceNothingForDepthZero(t *testing.T) { 91 | t.Parallel() 92 | 93 | methods := []string{http.MethodGet, http.MethodPost} 94 | dictionary := []string{"/home", "/about"} 95 | 96 | dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1) 97 | 98 | sut := producer.NewReProducer(dictionaryProducer) 99 | 100 | result := scan.NewResult( 101 | scan.Target{ 102 | Path: "/home", 103 | Method: http.MethodGet, 104 | Depth: 0, 105 | }, 106 | &http.Response{ 107 | StatusCode: http.StatusOK, 108 | Request: &http.Request{ 109 | URL: test.MustParseURL(t, "http://mysite/contacts"), 110 | }, 111 | }, 112 | ) 113 | 114 | reproducerFunc := sut.Reproduce(context.Background()) 115 | reproducerChannel := reproducerFunc(result) 116 | 117 | targets := make([]scan.Target, 0) 118 | for tar := range reproducerChannel { 119 | targets = append(targets, tar) 120 | } 121 | 122 | assert.Len(t, targets, 0) 123 | } 124 | 125 | func BenchmarkReProducer(b *testing.B) { 126 | methods := []string{http.MethodGet, http.MethodPost} 127 | dictionary := []string{"/home", "/about"} 128 | 129 | dictionaryProducer := producer.NewDictionaryProducer(methods, dictionary, 1) 130 | 131 | sut := producer.NewReProducer(dictionaryProducer) 132 | 133 | result := scan.NewResult( 134 | scan.Target{ 135 | Path: "/home", 136 | Method: http.MethodGet, 137 | Depth: 1, 138 | }, 139 | &http.Response{ 140 | StatusCode: http.StatusOK, 141 | Request: &http.Request{ 142 | URL: test.MustParseURL(b, "http://mysite/contacts"), 143 | }, 144 | }, 145 | ) 146 | 147 | b.ResetTimer() 148 | 149 | targets := make([]scan.Target, 0, 10) 150 | 151 | for i := 0; i < b.N; i++ { 152 | reproducerFunc := sut.Reproduce(context.Background()) 153 | reproducerChannel := reproducerFunc(result) 154 | 155 | for tar := range reproducerChannel { 156 | targets = append(targets, tar) 157 | } 158 | } 159 | 160 | benchData = targets 161 | } 162 | -------------------------------------------------------------------------------- /pkg/dictionary/dictionary_integration_test.go: -------------------------------------------------------------------------------- 1 | package dictionary_test 2 | 3 | import ( 4 | "net/http" 5 | "net/http/httptest" 6 | "os" 7 | "path/filepath" 8 | "strings" 9 | "testing" 10 | "time" 11 | 12 | "github.com/stefanoj3/dirstalk/pkg/common/test" 13 | "github.com/stefanoj3/dirstalk/pkg/dictionary" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestDictionaryFromFile(t *testing.T) { 18 | entries, err := dictionary.NewDictionaryFrom("testdata/dict.txt", &http.Client{}) 19 | assert.NoError(t, err) 20 | 21 | expectedValue := []string{ 22 | "home", 23 | "home/index.php", 24 | "blabla", 25 | } 26 | assert.Equal(t, expectedValue, entries) 27 | } 28 | 29 | func TestShouldFailToCreateDictionaryFromInvalidPath(t *testing.T) { 30 | _, err := dictionary.NewDictionaryFrom("http:///home/\n", &http.Client{}) 31 | assert.Error(t, err) 32 | } 33 | 34 | func TestDictionaryFromAbsolutePath(t *testing.T) { 35 | path, err := filepath.Abs("testdata/dict.txt") 36 | assert.NoError(t, err) 37 | 38 | entries, err := dictionary.NewDictionaryFrom(path, &http.Client{}) 39 | assert.NoError(t, err) 40 | 41 | expectedValue := []string{ 42 | "home", 43 | "home/index.php", 44 | "blabla", 45 | } 46 | assert.Equal(t, expectedValue, entries) 47 | } 48 | 49 | func TestDictionaryWithUnableToReadFolderShouldFail(t *testing.T) { 50 | newFolderPath := "testdata/" + test.RandStringRunes(10) 51 | 52 | err := os.Mkdir(newFolderPath, 0200) 53 | assert.NoError(t, err) 54 | 55 | defer removeTestDirectory(t, newFolderPath) 56 | 57 | _, err = dictionary.NewDictionaryFrom(newFolderPath, &http.Client{}) 58 | assert.Error(t, err) 59 | assert.Contains(t, err.Error(), "permission denied") 60 | } 61 | 62 | func TestDictionaryFromFileWithInvalidPath(t *testing.T) { 63 | t.Parallel() 64 | 65 | d, err := dictionary.NewDictionaryFrom("testdata/gibberish_nonexisting_file", &http.Client{}) 66 | assert.Error(t, err) 67 | assert.Nil(t, d) 68 | 69 | assert.Contains(t, err.Error(), "unable to open") 70 | } 71 | 72 | func TestNewDictionaryFromRemoteFile(t *testing.T) { 73 | srv := httptest.NewServer( 74 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 75 | dict := `/home 76 | /about 77 | /contacts 78 | something 79 | potato 80 | ` 81 | w.WriteHeader(http.StatusOK) 82 | _, _ = w.Write([]byte(dict)) //nolint:errcheck 83 | }), 84 | ) 85 | defer srv.Close() 86 | 87 | entries, err := dictionary.NewDictionaryFrom(srv.URL, &http.Client{}) 88 | assert.NoError(t, err) 89 | 90 | expectedValue := []string{ 91 | "/home", 92 | "/about", 93 | "/contacts", 94 | "something", 95 | "potato", 96 | } 97 | assert.Equal(t, expectedValue, entries) 98 | } 99 | 100 | func TestNewDictionaryFromRemoteFileWillReturnErrorWhenRequestTimeout(t *testing.T) { 101 | srv := httptest.NewServer( 102 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 103 | time.Sleep(time.Millisecond) // out of paranoia - we dont want unstable tests 104 | 105 | w.WriteHeader(http.StatusOK) 106 | _, _ = w.Write([]byte("/home")) //nolint:errcheck 107 | }), 108 | ) 109 | defer srv.Close() 110 | 111 | entries, err := dictionary.NewDictionaryFrom( 112 | srv.URL, 113 | &http.Client{ 114 | Timeout: time.Microsecond, 115 | }, 116 | ) 117 | assert.Error(t, err) 118 | assert.Contains(t, err.Error(), "failed to get") 119 | assert.Contains(t, err.Error(), "Timeout") 120 | 121 | assert.Nil(t, entries) 122 | } 123 | 124 | func TestNewDictionaryFromRemoteShouldFailWhenRemoteReturnNon200Status(t *testing.T) { 125 | srv := httptest.NewServer( 126 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 127 | w.WriteHeader(http.StatusForbidden) 128 | }), 129 | ) 130 | defer srv.Close() 131 | 132 | entries, err := dictionary.NewDictionaryFrom(srv.URL, &http.Client{}) 133 | assert.Error(t, err) 134 | assert.Contains(t, err.Error(), srv.URL) 135 | assert.Contains(t, err.Error(), "status code 403") 136 | 137 | assert.Nil(t, entries) 138 | } 139 | 140 | func removeTestDirectory(t *testing.T, path string) { 141 | if !strings.Contains(path, "testdata") { 142 | t.Fatalf("cannot delete `%s`, it is not in a `testdata` folder", path) 143 | 144 | return 145 | } 146 | 147 | stats, err := os.Stat(path) 148 | if err != nil { 149 | t.Fatalf("failed to read `%s` properties", path) 150 | } 151 | 152 | if !stats.IsDir() { 153 | t.Fatalf("cannot delete `%s`, it is not a directory", path) 154 | } 155 | 156 | err = os.Remove(path) 157 | if err != nil { 158 | t.Fatalf("failed to remove `%s`: %s", path, err.Error()) 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /pkg/cmd/config.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "strings" 7 | "time" 8 | 9 | "github.com/pkg/errors" 10 | "github.com/spf13/cobra" 11 | "github.com/stefanoj3/dirstalk/pkg/scan" 12 | ) 13 | 14 | const failedToReadPropertyError = "failed to read %s" 15 | 16 | func scanConfigFromCmd(cmd *cobra.Command) (*scan.Config, error) { 17 | c := &scan.Config{} 18 | 19 | var err error 20 | 21 | c.DictionaryPath = cmd.Flag(flagScanDictionary).Value.String() 22 | 23 | if c.DictionaryTimeoutInMilliseconds, err = cmd.Flags().GetInt(flagScanDictionaryGetTimeout); err != nil { 24 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanDictionaryGetTimeout) 25 | } 26 | 27 | if c.HTTPMethods, err = cmd.Flags().GetStringSlice(flagScanHTTPMethods); err != nil { 28 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanHTTPMethods) 29 | } 30 | 31 | if c.HTTPStatusesToIgnore, err = cmd.Flags().GetIntSlice(flagScanHTTPStatusesToIgnore); err != nil { 32 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanHTTPStatusesToIgnore) 33 | } 34 | 35 | if c.Threads, err = cmd.Flags().GetInt(flagScanThreads); err != nil { 36 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanThreads) 37 | } 38 | 39 | if c.TimeoutInMilliseconds, err = cmd.Flags().GetInt(flagScanHTTPTimeout); err != nil { 40 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanHTTPTimeout) 41 | } 42 | 43 | if c.CacheRequests, err = cmd.Flags().GetBool(flagScanHTTPCacheRequests); err != nil { 44 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanHTTPCacheRequests) 45 | } 46 | 47 | if c.ScanDepth, err = cmd.Flags().GetInt(flagScanScanDepth); err != nil { 48 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanScanDepth) 49 | } 50 | 51 | socks5Host := cmd.Flag(flagScanSocks5Host).Value.String() 52 | if len(socks5Host) > 0 { 53 | if c.Socks5Url, err = url.Parse("socks5://" + socks5Host); err != nil { 54 | return nil, errors.Wrapf(err, "invalid value for %s", flagScanSocks5Host) 55 | } 56 | } 57 | 58 | c.UserAgent = cmd.Flag(flagScanUserAgent).Value.String() 59 | 60 | if c.UseCookieJar, err = cmd.Flags().GetBool(flagScanCookieJar); err != nil { 61 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanCookieJar) 62 | } 63 | 64 | rawCookies, err := cmd.Flags().GetStringArray(flagScanCookie) 65 | if err != nil { 66 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanCookie) 67 | } 68 | 69 | if c.Cookies, err = rawCookiesToCookies(rawCookies); err != nil { 70 | return nil, errors.Wrap(err, "failed to convert rawCookies to objects") 71 | } 72 | 73 | rawHeaders, err := cmd.Flags().GetStringArray(flagScanHeader) 74 | if err != nil { 75 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagScanHeader) 76 | } 77 | 78 | if c.Headers, err = rawHeadersToHeaders(rawHeaders); err != nil { 79 | return nil, errors.Wrapf(err, "failed to convert rawHeaders (%v)", rawHeaders) 80 | } 81 | 82 | c.Out = cmd.Flag(flagScanResultOutput).Value.String() 83 | 84 | c.ShouldSkipSSLCertificatesValidation, err = cmd.Flags().GetBool(flagShouldSkipSSLCertificatesValidation) 85 | if err != nil { 86 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagShouldSkipSSLCertificatesValidation) 87 | } 88 | 89 | c.IgnoreEmpty20xResponses, err = cmd.Flags().GetBool(flagIgnore20xWithEmptyBody) 90 | if err != nil { 91 | return nil, errors.Wrapf(err, failedToReadPropertyError, flagIgnore20xWithEmptyBody) 92 | } 93 | 94 | return c, nil 95 | } 96 | 97 | func rawHeadersToHeaders(rawHeaders []string) (map[string]string, error) { 98 | headers := make(map[string]string, len(rawHeaders)*2) 99 | 100 | for _, rawHeader := range rawHeaders { 101 | parts := strings.Split(rawHeader, ":") 102 | if len(parts) != 2 { 103 | return nil, errors.Errorf("header is in invalid format: %s", rawHeader) 104 | } 105 | 106 | headers[parts[0]] = parts[1] 107 | } 108 | 109 | return headers, nil 110 | } 111 | 112 | func rawCookiesToCookies(rawCookies []string) ([]*http.Cookie, error) { 113 | cookies := make([]*http.Cookie, 0, len(rawCookies)) 114 | 115 | for _, rawCookie := range rawCookies { 116 | parts := strings.Split(rawCookie, "=") 117 | if len(parts) != 2 { 118 | return nil, errors.Errorf("cookie format is invalid: %s", rawCookie) 119 | } 120 | 121 | cookies = append( 122 | cookies, 123 | &http.Cookie{ 124 | Name: parts[0], 125 | Value: parts[1], 126 | Expires: time.Now().AddDate(0, 0, 2), 127 | }, 128 | ) 129 | } 130 | 131 | return cookies, nil 132 | } 133 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/DiSiqueira/GoTree v0.0.0-20180907134536-53a8e837f295 h1:94+Tj6lJzlPeTZntnEDdOekoENcLb4gQvSQLNM7srMA= 2 | github.com/DiSiqueira/GoTree v0.0.0-20180907134536-53a8e837f295/go.mod h1:e0aH495YLkrsIe9fhedd6aSR6fgU/qhKvtroi6y7G/M= 3 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= 4 | github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 6 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= 11 | github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= 12 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 13 | github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= 14 | github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= 15 | github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= 16 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 17 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 18 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 19 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 20 | github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= 21 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 22 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 23 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 24 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 25 | github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= 26 | github.com/rogpeppe/go-internal v1.8.1 h1:geMPLpDpQOgVyCg5z5GoRwLHepNdb71NXb67XFkP+Eg= 27 | github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= 28 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 29 | github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= 30 | github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= 31 | github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= 32 | github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= 33 | github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= 34 | github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= 35 | github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= 36 | github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= 37 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 38 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 39 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 40 | github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= 41 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 42 | golang.org/x/net v0.0.0-20220516155154-20f960328961 h1:+W/iTMPG0EL7aW+/atntZwZrvSRIj3m3yX414dSULUU= 43 | golang.org/x/net v0.0.0-20220516155154-20f960328961/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= 44 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 45 | golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a h1:N2T1jUrTQE9Re6TFF5PhvEHXHCguynGhKjWVsIUt5cY= 46 | golang.org/x/sys v0.0.0-20220513210249-45d2b4557a2a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 47 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 48 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 49 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 50 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 51 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 52 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 53 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 54 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 55 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 56 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 57 | gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99 h1:dbuHpmKjkDzSOMKAWl10QNlgaZUd3V1q99xc81tt2Kc= 58 | gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 59 | -------------------------------------------------------------------------------- /functional-tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ################################################################################################################ 4 | ## The purpose of this script is to make sure dirstalk basic functionalities are working as expected 5 | ################################################################################################################ 6 | 7 | ################################### 8 | ## function to assert that the given string contains the given substring 9 | ## example usage: assert_contains "error" "my_special_error: blabla" "an error is expected for XY" 10 | ################################### 11 | function assert_contains { 12 | local actual=$1 13 | local contains=$2 14 | local msg=$3 15 | 16 | if ! echo "$actual" | grep "$contains" > /dev/null; then 17 | echo "ERROR: $msg" 18 | echo "Failed to assert that $actual contains $contains" 19 | exit 1; 20 | fi 21 | echo "Assertion passing" 22 | } 23 | 24 | ################################### 25 | ## function to assert that the given string does not contain the given substring 26 | ## example usage: assert_contains "error" "my_special_error: blabla" "an error is expected for XY" 27 | ################################### 28 | function assert_not_contains { 29 | local actual=$1 30 | local contains=$2 31 | local msg=$3 32 | 33 | if printf -- '%s' "$actual" | egrep -q -- "$contains"; then 34 | echo "ERROR: $msg" 35 | echo "Failed to assert that $actual does not contain: $contains" 36 | exit 1; 37 | fi 38 | echo "Assertion passing" 39 | } 40 | 41 | ## Starting test server running on the 8080 port 42 | echo "Starting test server" 43 | ./dist/testserver& 44 | SERVER_PID=$! 45 | sleep 1 46 | echo "Done" 47 | 48 | function finish { 49 | echo "Killing test server $SERVER_PID" 50 | kill -9 "$SERVER_PID" 51 | echo "Done" 52 | } 53 | trap finish EXIT 54 | 55 | ## Tests 56 | 57 | ROOT_RESULT=$(./dist/dirstalk 2>&1); 58 | assert_contains "$ROOT_RESULT" "dirstalk is a tool that attempts" "description is expected" 59 | assert_contains "$ROOT_RESULT" "Usage" "description is expected" 60 | 61 | VERSION_RESULT=$(./dist/dirstalk version 2>&1); 62 | assert_contains "$VERSION_RESULT" "Version" "the version is expected to be printed when calling the version command" 63 | assert_contains "$VERSION_RESULT" "Built" "the build time is expected to be printed when calling the version command" 64 | assert_contains "$VERSION_RESULT" "Built" "the build time is expected to be printed when calling the version command" 65 | 66 | SCAN_RESULT=$(./dist/dirstalk scan 2>&1 || true); 67 | assert_contains "$SCAN_RESULT" "error" "an error is expected when no argument is passed" 68 | 69 | SCAN_RESULT=$(./dist/dirstalk scan -d resources/tests/dictionary.txt http://localhost:8080 2>&1); 70 | assert_contains "$SCAN_RESULT" "/index" "result expected when performing scan" 71 | assert_contains "$SCAN_RESULT" "/index/home" "result expected when performing scan" 72 | assert_contains "$SCAN_RESULT" "3 results found" "a recap was expected when performing a scan" 73 | assert_contains "$SCAN_RESULT" "├── home" "a recap was expected when performing a scan" 74 | assert_contains "$SCAN_RESULT" "└── index" "a recap was expected when performing a scan" 75 | assert_contains "$SCAN_RESULT" " └── home" "a recap was expected when performing a scan" 76 | 77 | assert_not_contains "$SCAN_RESULT" "error" "no error is expected for a successful scan" 78 | 79 | SCAN_RESULT=$(./dist/dirstalk scan -h 2>&1); 80 | assert_contains "$SCAN_RESULT" "\-\-dictionary" "dictionary help is expected to be printed" 81 | assert_contains "$SCAN_RESULT" "\-\-cookie" "cookie help is expected to be printed" 82 | assert_contains "$SCAN_RESULT" "\-\-header" "header help is expected to be printed" 83 | assert_contains "$SCAN_RESULT" "\-\-http-cache-requests" "http-cache-requests help is expected to be printed" 84 | assert_contains "$SCAN_RESULT" "\-\-http-methods" "http-methods help is expected to be printed" 85 | assert_contains "$SCAN_RESULT" "\-\-http-statuses-to-ignore" "http-statuses-to-ignore help is expected to be printed" 86 | assert_contains "$SCAN_RESULT" "\-\-http-timeout" "http-timeout help is expected to be printed" 87 | assert_contains "$SCAN_RESULT" "\-\-socks5" "socks5 help is expected to be printed" 88 | assert_contains "$SCAN_RESULT" "\-\-threads" "threads help is expected to be printed" 89 | assert_contains "$SCAN_RESULT" "\-\-user-agent" "user-agent help is expected to be printed" 90 | assert_contains "$SCAN_RESULT" "\-\-scan-depth" "scan-depth help is expected to be printed" 91 | 92 | assert_not_contains "$SCAN_RESULT" "error" "no error is expected when priting scan help" 93 | 94 | DICTIONARY_GENERATE_RESULT=$(./dist/dirstalk dictionary.generate resources/tests 2>&1); 95 | assert_contains "$DICTIONARY_GENERATE_RESULT" "dictionary.txt" "dictionary generation should contains a file in the folder" 96 | assert_not_contains "$DICTIONARY_GENERATE_RESULT" "error" "no error is expected when generating a dictionary successfully" 97 | 98 | RESULT_VIEW_RESULT=$(./dist/dirstalk result.view -r resources/tests/out.txt 2>&1); 99 | assert_contains "$RESULT_VIEW_RESULT" "├── adview" "result output should contain tree output" 100 | assert_contains "$RESULT_VIEW_RESULT" "├── partners" "result output should contain tree output" 101 | assert_contains "$RESULT_VIEW_RESULT" "│ └── terms" "result output should contain tree output" 102 | assert_contains "$RESULT_VIEW_RESULT" "└── s" "result output should contain tree output" 103 | assert_not_contains "$RESULT_VIEW_RESULT" "error" "no error is expected when displaying a result" 104 | 105 | RESULT_DIFF_RESULT=$(./dist/dirstalk result.diff -f resources/tests/out.txt -s resources/tests/out2.txt 2>&1); 106 | assert_contains "$RESULT_DIFF_RESULT" "├── adview" "result output should contain diff" 107 | assert_contains "$RESULT_DIFF_RESULT" "├── partners" "result output should contain diff" 108 | assert_contains "$RESULT_DIFF_RESULT" $(echo "│ └── \x1b[31mterms\x1b[0m\x1b[32m123\x1b[0m") "result output should contain diff" 109 | assert_contains "$RESULT_DIFF_RESULT" "└── s" "result output should contain diff" 110 | assert_not_contains "$RESULT_DIFF_RESULT" "error" "no error is expected when displaying a result" 111 | 112 | RESULT_DIFF_RESULT=$(./dist/dirstalk result.diff -f resources/tests/out.txt -s resources/tests/out.txt 2>&1 || true); 113 | assert_contains "$RESULT_DIFF_RESULT" "no diffs found" 114 | assert_contains "$RESULT_DIFF_RESULT" "error" "error is expected when content is the same" 115 | -------------------------------------------------------------------------------- /pkg/scan/scanner.go: -------------------------------------------------------------------------------- 1 | package scan 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "net/url" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/sirupsen/logrus" 11 | "github.com/stefanoj3/dirstalk/pkg/common/urlpath" 12 | "github.com/stefanoj3/dirstalk/pkg/scan/client" 13 | ) 14 | 15 | // Target represents the target to scan. 16 | type Target struct { 17 | Path string 18 | Method string 19 | Depth int 20 | } 21 | 22 | // Result represents the result of the scan of a single URL. 23 | type Result struct { 24 | Target Target 25 | StatusCode int 26 | URL url.URL 27 | ContentLength int64 28 | } 29 | 30 | // NewResult creates a new instance of the Result entity based on the Target and Response. 31 | func NewResult(target Target, response *http.Response) Result { 32 | return Result{ 33 | Target: target, 34 | StatusCode: response.StatusCode, 35 | URL: *response.Request.URL, 36 | ContentLength: response.ContentLength, 37 | } 38 | } 39 | 40 | func NewScanner( 41 | httpClient Doer, 42 | producer Producer, 43 | reproducer ReProducer, 44 | resultFilter ResultFilter, 45 | logger *logrus.Logger, 46 | ) *Scanner { 47 | return &Scanner{ 48 | httpClient: httpClient, 49 | producer: producer, 50 | reproducer: reproducer, 51 | resultFilter: resultFilter, 52 | logger: logger, 53 | } 54 | } 55 | 56 | type Scanner struct { 57 | httpClient Doer 58 | producer Producer 59 | reproducer ReProducer 60 | resultFilter ResultFilter 61 | logger *logrus.Logger 62 | } 63 | 64 | func (s *Scanner) Scan(ctx context.Context, baseURL *url.URL, workers int) <-chan Result { 65 | resultChannel := make(chan Result, workers) 66 | 67 | u := normalizeBaseURL(*baseURL) 68 | 69 | wg := sync.WaitGroup{} 70 | 71 | producerChannel := s.producer.Produce(ctx) 72 | reproducer := s.reproducer.Reproduce(ctx) 73 | 74 | wg.Add(workers) 75 | 76 | for i := 0; i < workers; i++ { 77 | go func() { 78 | defer wg.Done() 79 | 80 | for { 81 | select { 82 | case <-ctx.Done(): 83 | s.logger.Debug("terminating worker: context cancellation") 84 | case target, ok := <-producerChannel: 85 | if !ok { 86 | s.logger.Debug("terminating worker: producer channel closed") 87 | 88 | return 89 | } 90 | 91 | s.processTarget(ctx, u, target, reproducer, resultChannel) 92 | } 93 | } 94 | }() 95 | } 96 | 97 | go func() { 98 | wg.Wait() 99 | close(resultChannel) 100 | }() 101 | 102 | return resultChannel 103 | } 104 | 105 | func (s *Scanner) processTarget( 106 | ctx context.Context, 107 | baseURL url.URL, 108 | target Target, 109 | reproducer func(r Result) <-chan Target, 110 | results chan<- Result, 111 | ) { 112 | l := s.logger.WithFields(logrus.Fields{ 113 | "method": target.Method, 114 | "depth": target.Depth, 115 | "path": target.Path, 116 | }) 117 | 118 | l.Debug("Working") 119 | 120 | u := buildURL(baseURL, target) 121 | 122 | req, err := http.NewRequestWithContext(ctx, target.Method, u.String(), nil) 123 | if err != nil { 124 | l.WithError(err).Error("failed to build request") 125 | 126 | return 127 | } 128 | 129 | s.processRequest(ctx, l, req, target, results, reproducer, baseURL) 130 | } 131 | 132 | func (s *Scanner) processRequest( 133 | ctx context.Context, 134 | l *logrus.Entry, 135 | req *http.Request, 136 | target Target, 137 | results chan<- Result, 138 | reproducer func(r Result) <-chan Target, 139 | baseURL url.URL, 140 | ) { 141 | res, err := s.httpClient.Do(req) 142 | if err != nil && strings.Contains(err.Error(), client.ErrRequestRedundant.Error()) { 143 | l.WithError(err).Debug("skipping, request was already made") 144 | 145 | return 146 | } 147 | 148 | if err != nil { 149 | l.WithError(err).Error("failed to perform request") 150 | 151 | return 152 | } 153 | 154 | if err := res.Body.Close(); err != nil { 155 | l.WithError(err).Warn("failed to close response body") 156 | } 157 | 158 | result := NewResult(target, res) 159 | 160 | if s.resultFilter.ShouldIgnore(result) { 161 | return 162 | } 163 | 164 | results <- result 165 | 166 | redirectTarget, shouldRedirect := s.shouldRedirect(l, req, res, target.Depth) 167 | if shouldRedirect { 168 | s.processTarget(ctx, baseURL, redirectTarget, reproducer, results) 169 | } 170 | 171 | for newTarget := range reproducer(result) { 172 | s.processTarget(ctx, baseURL, newTarget, reproducer, results) 173 | } 174 | } 175 | 176 | func (s *Scanner) shouldRedirect(l *logrus.Entry, req *http.Request, res *http.Response, targetDepth int) (Target, bool) { 177 | if targetDepth == 0 { 178 | l.Debug("depth is 0, not following any redirect") 179 | 180 | return Target{}, false 181 | } 182 | 183 | redirectMethod := req.Method 184 | location := res.Header.Get("Location") 185 | 186 | if location == "" { 187 | return Target{}, false 188 | } 189 | 190 | redirectStatusCodes := map[int]bool{ 191 | http.StatusMovedPermanently: true, 192 | http.StatusFound: true, 193 | http.StatusSeeOther: true, 194 | http.StatusTemporaryRedirect: false, 195 | http.StatusPermanentRedirect: false, 196 | } 197 | 198 | shouldOverrideRequestMethod, shouldRedirect := redirectStatusCodes[res.StatusCode] 199 | if !shouldRedirect { 200 | return Target{}, false 201 | } 202 | 203 | // RFC 2616 allowed automatic redirection only with GET and 204 | // HEAD requests. RFC 7231 lifts this restriction, but we still 205 | // restrict other methods to GET to maintain compatibility. 206 | // See Issue 18570. 207 | if shouldOverrideRequestMethod { 208 | if req.Method != "GET" && req.Method != "HEAD" { 209 | redirectMethod = "GET" 210 | } 211 | } 212 | 213 | u, err := url.Parse(location) 214 | if err != nil { 215 | l.WithError(err). 216 | WithField("location", location). 217 | Warn("failed to parse location for redirect") 218 | 219 | return Target{}, false 220 | } 221 | 222 | if u.Host != "" && u.Host != req.Host { 223 | l.Debug("skipping redirect, pointing to a different host") 224 | 225 | return Target{}, false 226 | } 227 | 228 | return Target{ 229 | Path: u.Path, 230 | Method: redirectMethod, 231 | Depth: targetDepth - 1, 232 | }, true 233 | } 234 | 235 | func normalizeBaseURL(baseURL url.URL) url.URL { 236 | if strings.HasSuffix(baseURL.Path, "/") { 237 | return baseURL 238 | } 239 | 240 | baseURL.Path += "/" 241 | 242 | return baseURL 243 | } 244 | 245 | func buildURL(baseURL url.URL, target Target) url.URL { 246 | baseURL.Path = urlpath.Join(baseURL.Path, target.Path) 247 | 248 | return baseURL 249 | } 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dirstalk 2 | ![](https://github.com/stefanoj3/dirstalk/workflows/CI/badge.svg) 3 | [![codecov](https://codecov.io/gh/stefanoj3/dirstalk/branch/master/graph/badge.svg)](https://codecov.io/gh/stefanoj3/dirstalk) 4 | [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/stefanoj3/dirstalk/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/stefanoj3/dirstalk/?branch=master) 5 | ![Docker Pulls](https://img.shields.io/docker/pulls/stefanoj3/dirstalk.svg) 6 | ![GitHub](https://img.shields.io/github/license/stefanoj3/dirstalk.svg) 7 | 8 | Dirstalk is a multi threaded application designed to brute force paths on web servers. 9 | 10 | The tool contains functionalities similar to the ones offered by 11 | [dirbuster](https://www.owasp.org/index.php/Category:OWASP_DirBuster_Project) 12 | and [dirb](https://tools.kali.org/web-applications/dirb). 13 | 14 | Here you can see it in action: 15 | [![asciicast](https://asciinema.org/a/ehvNAUetjWbNExQegA2KPaHuY.svg)](https://asciinema.org/a/ehvNAUetjWbNExQegA2KPaHuY) 16 | 17 | ## Contents 18 | - [How to use it](#-how-to-use-it) 19 | - [Scan](#scan) 20 | - [Useful resources](#useful-resources) 21 | - [Dictionary generator](#dictionary-generator) 22 | - [Download](#-download) 23 | - [Development](#-development) 24 | - [License](https://github.com/stefanoj3/dirstalk/blob/master/LICENSE.md) 25 | 26 | ## [↑](#contents) How to use it 27 | 28 | The application is self-documenting, launching `dirstalk -h` will return all the available commands with a 29 | short description, you can get the help for each command by doing `distalk -h`. 30 | 31 | EG `dirstalk result.diff -h` 32 | 33 | ### Scan 34 | 35 | To perform a scan you need to provide at least a dictionary and a URL: 36 | ```shell script 37 | dirstalk scan http://someaddress.url/ --dictionary mydictionary.txt 38 | ``` 39 | 40 | As mentioned before, to see all the flags available for the scan command you can 41 | just call the command with the `-h` flag: 42 | ```shell script 43 | dirstalk scan -h 44 | ``` 45 | 46 | ##### Example of how you can customize a scan: 47 | ```shell script 48 | dirstalk scan http://someaddress.url/ \ 49 | --dictionary mydictionary.txt \ 50 | --http-methods GET,POST \ 51 | --http-timeout 10000 \ 52 | --scan-depth 10 \ 53 | --threads 10 \ 54 | --socks5 127.0.0.1:9150 \ 55 | --cookie name=value \ 56 | --use-cookie-jar \ 57 | --user-agent my_user_agent \ 58 | --header "Authorization: Bearer 123" 59 | ``` 60 | 61 | 62 | ##### Currently available flags: 63 | ```shell script 64 | --cookie stringArray cookie to add to each request; eg name=value (can be specified multiple times) 65 | -d, --dictionary string dictionary to use for the scan (path to local file or remote url) 66 | --header stringArray header to add to each request; eg name=value (can be specified multiple times) 67 | -h, --help help for scan 68 | --http-cache-requests cache requests to avoid performing the same request multiple times within the same scan (EG if the server reply with the same redirect location multiple times, dirstalk will follow it only once) (default true) 69 | --http-methods strings comma separated list of http methods to use; eg: GET,POST,PUT (default [GET]) 70 | --http-statuses-to-ignore ints comma separated list of http statuses to ignore when showing and processing results; eg: 404,301 (default [404]) 71 | --http-timeout int timeout in milliseconds (default 5000) 72 | --out string path where to store result output 73 | --scan-depth int scan depth (default 3) 74 | --socks5 string socks5 host to use 75 | -t, --threads int amount of threads for concurrent requests (default 3) 76 | --use-cookie-jar enables the use of a cookie jar: it will retain any cookie sent from the server and send them for the following requests 77 | --user-agent string user agent to use for http requests 78 | ``` 79 | 80 | ##### Useful resources 81 | - [here](https://github.com/dustyfresh/dictionaries/tree/master/DirBuster-Lists) you can find dictionaries that can be used with dirstalk 82 | - [tordock](https://github.com/stefanoj3/tordock) is a containerized Tor SOCKS5 that you can use easily with dirstalk 83 | (just `docker run -d -p 127.0.0.1:9150:9150 stefanoj3/tordock:latest` and then when launching a 84 | scan specify the following flag: `--socks5 127.0.0.1:9150`) 85 | 86 | ### Dictionary generator 87 | Dirstalk can also produce it's own dictionaries, useful for example if you 88 | want to check if a specific set of files is available on a given web server. 89 | 90 | ##### Example: 91 | ```shell script 92 | dirstalk dictionary.generate /path/to/local/files --out mydictionary.txt 93 | ``` 94 | The result will be printed to the stdout if no out flag is specified. 95 | 96 | ## [↑](#contents) Download 97 | You can download a release from [here](https://github.com/stefanoj3/dirstalk/releases) 98 | or you can use a docker image. (eg `docker run stefanoj3/dirstalk dirstalk `) 99 | 100 | If you are using an arch based linux distribution you can fetch it via AUR: https://aur.archlinux.org/packages/dirstalk/ 101 | 102 | Example: 103 | ```shell script 104 | yay -S aur/dirstalk 105 | ``` 106 | 107 | 108 | ## [↑](#contents) Development 109 | All you need to do local development is to have [make](https://www.gnu.org/software/make/) 110 | and [golang](https://golang.org/) available and the GOPATH correctly configured. 111 | 112 | Then you can just clone the project, enter the folder and: 113 | ```shell script 114 | make dep # to fetch dependencies 115 | make tests # to run the test suite 116 | make check # to check for any code style issue 117 | make fix # to automatically fix the code style using goimports 118 | make build # to build an executable for your host OS (not tested under windows) 119 | ``` 120 | 121 | ```shell script 122 | make help 123 | ``` 124 | will print a description of every command available in the Makefile. 125 | 126 | Wanna add a functionality? fix a bug? fork and create a PR. 127 | 128 | ## [↑](#contents) Plans for the future 129 | - Add support for rotating SOCKS5 proxies 130 | - Scan a website pages looking for links to bruteforce 131 | - Expose a webserver that can be used to launch scans and check their status 132 | - Introduce metrics that can give a sense of how much of the dictionary was found on the remote server 133 | -------------------------------------------------------------------------------- /pkg/scan/summarizer/result_summarizer_integration_test.go: -------------------------------------------------------------------------------- 1 | package summarizer_test 2 | 3 | import ( 4 | "net/http" 5 | "sync" 6 | "testing" 7 | 8 | "github.com/sirupsen/logrus" 9 | "github.com/stefanoj3/dirstalk/pkg/common/test" 10 | "github.com/stefanoj3/dirstalk/pkg/scan" 11 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer" 12 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer/tree" 13 | "github.com/stretchr/testify/assert" 14 | ) 15 | 16 | func TestResultSummarizerShouldSummarizeResults(t *testing.T) { 17 | logger, loggerBuffer := test.NewLogger() 18 | logger.SetLevel(logrus.FatalLevel) 19 | 20 | sut := summarizer.NewResultSummarizer(tree.NewResultTreeProducer(), logger) 21 | 22 | sut.Add( 23 | scan.NewResult( 24 | scan.Target{ 25 | Method: http.MethodPost, 26 | Path: "/home", 27 | }, 28 | &http.Response{ 29 | StatusCode: http.StatusCreated, 30 | Request: &http.Request{ 31 | URL: test.MustParseURL(t, "http://mysite/home"), 32 | }, 33 | }, 34 | ), 35 | ) 36 | 37 | sut.Add( 38 | scan.NewResult( 39 | scan.Target{ 40 | Method: http.MethodPost, 41 | Path: "/home/hidden", 42 | }, 43 | &http.Response{ 44 | StatusCode: http.StatusCreated, 45 | Request: &http.Request{ 46 | URL: test.MustParseURL(t, "http://mysite/home/hidden"), 47 | }, 48 | }, 49 | ), 50 | ) 51 | 52 | sut.Add( 53 | scan.NewResult( 54 | scan.Target{ 55 | Method: http.MethodGet, 56 | Path: "/home/about", 57 | }, 58 | &http.Response{ 59 | StatusCode: http.StatusOK, 60 | Request: &http.Request{ 61 | URL: test.MustParseURL(t, "http://mysite/home/about"), 62 | }, 63 | }, 64 | ), 65 | ) 66 | 67 | sut.Add( 68 | scan.NewResult( 69 | scan.Target{ 70 | Method: http.MethodGet, 71 | Path: "/home/about/me", 72 | }, 73 | &http.Response{ 74 | StatusCode: http.StatusOK, 75 | Request: &http.Request{ 76 | URL: test.MustParseURL(t, "http://mysite/home/about/me"), 77 | }, 78 | }, 79 | ), 80 | ) 81 | 82 | sut.Add( 83 | scan.NewResult( 84 | scan.Target{ 85 | Method: http.MethodGet, 86 | Path: "/home/home", 87 | }, 88 | &http.Response{ 89 | StatusCode: http.StatusOK, 90 | Request: &http.Request{ 91 | URL: test.MustParseURL(t, "http://mysite/home/home"), 92 | }, 93 | }, 94 | ), 95 | ) 96 | 97 | sut.Add( 98 | scan.NewResult( 99 | scan.Target{ 100 | Method: http.MethodGet, 101 | Path: "/contacts", 102 | }, 103 | &http.Response{ 104 | StatusCode: http.StatusOK, 105 | Request: &http.Request{ 106 | URL: test.MustParseURL(t, "http://mysite/contacts"), 107 | }, 108 | }, 109 | ), 110 | ) 111 | 112 | sut.Add( 113 | scan.NewResult( 114 | scan.Target{ 115 | Method: http.MethodGet, 116 | Path: "/gibberish", 117 | }, 118 | &http.Response{ 119 | StatusCode: http.StatusNotFound, 120 | Request: &http.Request{ 121 | URL: test.MustParseURL(t, "http://mysite/gibberish"), 122 | }, 123 | }, 124 | ), 125 | ) 126 | 127 | sut.Add( 128 | scan.NewResult( 129 | scan.Target{ 130 | Method: http.MethodGet, 131 | Path: "/path/to/my/files", 132 | }, 133 | &http.Response{ 134 | StatusCode: http.StatusOK, 135 | Request: &http.Request{ 136 | URL: test.MustParseURL(t, "http://mysite/path/to/my/files"), 137 | }, 138 | }, 139 | ), 140 | ) 141 | 142 | // Adding multiple times the same result should not change the outcome 143 | wg := &sync.WaitGroup{} 144 | 145 | const workers = 10 146 | 147 | wg.Add(workers) 148 | 149 | for i := 0; i < workers; i++ { 150 | go func() { 151 | defer wg.Done() 152 | 153 | sut.Add( 154 | scan.NewResult( 155 | scan.Target{ 156 | Method: http.MethodGet, 157 | Path: "/path/to/my/files", 158 | }, 159 | &http.Response{ 160 | StatusCode: http.StatusOK, 161 | Request: &http.Request{ 162 | URL: test.MustParseURL(t, "http://mysite/path/to/my/files"), 163 | }, 164 | }, 165 | ), 166 | ) 167 | }() 168 | } 169 | 170 | wg.Wait() 171 | 172 | sut.Summarize() 173 | 174 | expectedResult := `8 results found 175 | / 176 | ├── contacts 177 | ├── gibberish 178 | ├── home 179 | │ ├── about 180 | │ │ └── me 181 | │ ├── hidden 182 | │ └── home 183 | └── path 184 | └── to 185 | └── my 186 | └── files 187 | 188 | http://mysite/contacts [200] [GET] 189 | http://mysite/gibberish [404] [GET] 190 | http://mysite/home [201] [POST] 191 | http://mysite/home/about [200] [GET] 192 | http://mysite/home/about/me [200] [GET] 193 | http://mysite/home/hidden [201] [POST] 194 | http://mysite/home/home [200] [GET] 195 | http://mysite/path/to/my/files [200] [GET] 196 | ` 197 | assert.Equal(t, expectedResult, loggerBuffer.String()) 198 | } 199 | 200 | func TestResultSummarizerShouldLogResults(t *testing.T) { 201 | testCases := []struct { 202 | result scan.Result 203 | expectedToContain []string 204 | }{ 205 | { 206 | result: scan.NewResult( 207 | scan.Target{ 208 | Method: http.MethodPost, 209 | Path: "/home", 210 | }, 211 | &http.Response{ 212 | StatusCode: http.StatusOK, 213 | Request: &http.Request{ 214 | URL: test.MustParseURL(t, "http://mysite/home"), 215 | }, 216 | }, 217 | ), 218 | expectedToContain: []string{ 219 | "Found", 220 | "method=POST", 221 | "status-code=200", 222 | `url="http://mysite/home"`, 223 | }, 224 | }, 225 | { 226 | result: scan.NewResult( 227 | scan.Target{ 228 | Method: http.MethodGet, 229 | Path: "/index", 230 | }, 231 | &http.Response{ 232 | StatusCode: http.StatusBadGateway, 233 | Request: &http.Request{ 234 | URL: test.MustParseURL(t, "http://mysite/index"), 235 | }, 236 | }, 237 | ), 238 | expectedToContain: []string{ 239 | "Found something breaking", 240 | "method=GET", 241 | "status-code=502", 242 | `url="http://mysite/index"`, 243 | }, 244 | }, 245 | { 246 | result: scan.NewResult( 247 | scan.Target{ 248 | Method: http.MethodGet, 249 | Path: "/gibberish", 250 | }, 251 | &http.Response{ 252 | StatusCode: http.StatusNotFound, 253 | Request: &http.Request{ 254 | URL: test.MustParseURL(t, "http://mysite/gibberish"), 255 | }, 256 | }, 257 | ), 258 | expectedToContain: []string{ 259 | "Found", 260 | "method=GET", 261 | "status-code=404", 262 | `url="http://mysite/gibberish"`, 263 | }, 264 | }, 265 | } 266 | 267 | for _, tc := range testCases { 268 | tc := tc // Pinning ranged variable, more info: https://github.com/kyoh86/scopelint 269 | t.Run(tc.result.Target.Path, func(t *testing.T) { 270 | t.Parallel() 271 | logger, loggerBuffer := test.NewLogger() 272 | sut := summarizer.NewResultSummarizer(tree.NewResultTreeProducer(), logger) 273 | 274 | sut.Add(tc.result) 275 | 276 | bufferAsString := loggerBuffer.String() 277 | for _, expectedToContain := range tc.expectedToContain { 278 | assert.Contains(t, bufferAsString, expectedToContain) 279 | } 280 | }) 281 | } 282 | } 283 | -------------------------------------------------------------------------------- /pkg/scan/client/client_test.go: -------------------------------------------------------------------------------- 1 | package client_test 2 | 3 | import ( 4 | "net/http" 5 | "net/url" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stefanoj3/dirstalk/pkg/common/test" 10 | "github.com/stefanoj3/dirstalk/pkg/scan/client" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestWhenRemoteIsTooSlowClientShouldTimeout(t *testing.T) { 15 | testServer, _ := test.NewServerWithAssertion( 16 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 17 | time.Sleep(time.Millisecond * 100) 18 | }), 19 | ) 20 | defer testServer.Close() 21 | 22 | c, err := client.NewClientFromConfig( 23 | 10, 24 | nil, 25 | "", 26 | false, 27 | nil, 28 | nil, 29 | true, 30 | false, 31 | nil, 32 | ) 33 | assert.NoError(t, err) 34 | 35 | res, err := c.Get(testServer.URL) //nolint 36 | assert.Error(t, err) 37 | assert.Nil(t, res) 38 | 39 | assert.Contains(t, err.Error(), "exceeded") 40 | } 41 | 42 | func TestShouldForwardProvidedCookiesWhenUsingJar(t *testing.T) { 43 | const ( 44 | serverCookieName = "server_cookie_name" 45 | serverCookieValue = "server_cookie_value" 46 | ) 47 | 48 | testServer, serverAssertion := test.NewServerWithAssertion( 49 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 50 | http.SetCookie( 51 | w, 52 | &http.Cookie{ 53 | Name: serverCookieName, 54 | Value: serverCookieValue, 55 | Expires: time.Now().AddDate(0, 1, 0), 56 | }, 57 | ) 58 | }), 59 | ) 60 | defer testServer.Close() 61 | 62 | u, err := url.Parse(testServer.URL) 63 | assert.NoError(t, err) 64 | 65 | cookies := []*http.Cookie{ 66 | { 67 | Name: "a_cookie_name", 68 | Value: "a_cookie_value", 69 | }, 70 | } 71 | 72 | c, err := client.NewClientFromConfig( 73 | 100, 74 | nil, 75 | "", 76 | true, 77 | cookies, 78 | map[string]string{}, 79 | false, 80 | false, 81 | u, 82 | ) 83 | assert.NoError(t, err) 84 | 85 | res, err := c.Get(testServer.URL) 86 | assert.NoError(t, err) 87 | assert.NotNil(t, res) 88 | 89 | defer res.Body.Close() //nolint:errcheck 90 | 91 | assert.Equal(t, 1, serverAssertion.Len()) 92 | 93 | serverAssertion.At(0, func(r http.Request) { 94 | assert.Equal(t, 1, len(r.Cookies())) 95 | 96 | assert.Equal(t, r.Cookies()[0].Name, cookies[0].Name) 97 | assert.Equal(t, r.Cookies()[0].Value, cookies[0].Value) 98 | assert.Equal(t, r.Cookies()[0].Expires, cookies[0].Expires) 99 | }) 100 | 101 | res, err = c.Get(testServer.URL) 102 | assert.NoError(t, err) 103 | assert.NotNil(t, res) 104 | 105 | defer res.Body.Close() //nolint:errcheck 106 | 107 | assert.Equal(t, 2, serverAssertion.Len()) 108 | 109 | serverAssertion.At(1, func(r http.Request) { 110 | assert.Equal(t, 2, len(r.Cookies())) 111 | 112 | assert.Equal(t, r.Cookies()[0].Name, cookies[0].Name) 113 | assert.Equal(t, r.Cookies()[0].Value, cookies[0].Value) 114 | assert.Equal(t, r.Cookies()[0].Expires, cookies[0].Expires) 115 | 116 | assert.Equal(t, r.Cookies()[1].Name, serverCookieName) 117 | assert.Equal(t, r.Cookies()[1].Value, serverCookieValue) 118 | }) 119 | } 120 | 121 | func TestShouldForwardCookiesWhenJarIsDisabled(t *testing.T) { 122 | testServer, serverAssertion := test.NewServerWithAssertion( 123 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 124 | ) 125 | defer testServer.Close() 126 | 127 | u, err := url.Parse(testServer.URL) 128 | assert.NoError(t, err) 129 | 130 | cookies := []*http.Cookie{ 131 | { 132 | Name: "a_cookie_name", 133 | Value: "a_cookie_value", 134 | }, 135 | } 136 | 137 | c, err := client.NewClientFromConfig( 138 | 100, 139 | nil, 140 | "", 141 | false, 142 | cookies, 143 | map[string]string{}, 144 | true, 145 | false, 146 | u, 147 | ) 148 | assert.NoError(t, err) 149 | 150 | res, err := c.Get(testServer.URL) 151 | assert.NoError(t, err) 152 | assert.NotNil(t, res) 153 | 154 | defer res.Body.Close() //nolint:errcheck 155 | 156 | assert.Equal(t, 1, serverAssertion.Len()) 157 | 158 | serverAssertion.At(0, func(r http.Request) { 159 | assert.Equal(t, 1, len(r.Cookies())) 160 | 161 | assert.Equal(t, r.Cookies()[0].Name, cookies[0].Name) 162 | assert.Equal(t, r.Cookies()[0].Value, cookies[0].Value) 163 | assert.Equal(t, r.Cookies()[0].Expires, cookies[0].Expires) 164 | }) 165 | } 166 | 167 | func TestShouldForwardProvidedHeader(t *testing.T) { 168 | const ( 169 | headerName = "my_header_name" 170 | headerValue = "my_header_value_123" 171 | ) 172 | 173 | testServer, serverAssertion := test.NewServerWithAssertion( 174 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 175 | ) 176 | defer testServer.Close() 177 | 178 | u, err := url.Parse(testServer.URL) 179 | assert.NoError(t, err) 180 | 181 | c, err := client.NewClientFromConfig( 182 | 100, 183 | nil, 184 | "", 185 | false, 186 | nil, 187 | map[string]string{headerName: headerValue}, 188 | true, 189 | false, 190 | u, 191 | ) 192 | assert.NoError(t, err) 193 | 194 | res, err := c.Get(testServer.URL) 195 | assert.NoError(t, err) 196 | assert.NotNil(t, res) 197 | 198 | defer res.Body.Close() //nolint:errcheck 199 | 200 | assert.Equal(t, 1, serverAssertion.Len()) 201 | 202 | serverAssertion.At(0, func(r http.Request) { 203 | assert.Equal(t, headerValue, r.Header.Get(headerName)) 204 | }) 205 | } 206 | 207 | func TestShouldFailToCreateAClientWithInvalidSocks5Url(t *testing.T) { 208 | u := url.URL{Scheme: "potatoscheme"} 209 | 210 | c, err := client.NewClientFromConfig( 211 | 100, 212 | &u, 213 | "", 214 | false, 215 | nil, 216 | map[string]string{}, 217 | true, 218 | false, 219 | nil, 220 | ) 221 | assert.Nil(t, c) 222 | assert.Error(t, err) 223 | 224 | assert.Contains(t, err.Error(), "unknown scheme") 225 | } 226 | 227 | func TestShouldNotRepeatTheSameRequestTwice(t *testing.T) { 228 | testServer, serverAssertion := test.NewServerWithAssertion( 229 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 230 | ) 231 | defer testServer.Close() 232 | 233 | u, err := url.Parse(testServer.URL) 234 | assert.NoError(t, err) 235 | 236 | c, err := client.NewClientFromConfig( 237 | 100, 238 | nil, 239 | "", 240 | false, 241 | nil, 242 | nil, 243 | true, 244 | false, 245 | u, 246 | ) 247 | assert.NoError(t, err) 248 | 249 | req, err := http.NewRequest(http.MethodGet, u.String(), nil) 250 | assert.NoError(t, err) 251 | 252 | res, err := c.Do(req) 253 | assert.NoError(t, err) 254 | 255 | res.Body.Close() //nolint:errcheck,gosec 256 | 257 | assert.Equal(t, http.StatusOK, res.StatusCode) 258 | 259 | res, err = c.Do(req) //nolint 260 | assert.Contains(t, err.Error(), client.ErrRequestRedundant.Error()) 261 | assert.Nil(t, res) 262 | 263 | assert.Equal(t, 1, serverAssertion.Len()) 264 | } 265 | 266 | func TestShouldFailToCommunicateWithServerHavingInvalidSSLCertificates(t *testing.T) { 267 | testServer, serverAssertion := test.NewTSLServerWithAssertion( 268 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 269 | ) 270 | defer testServer.Close() 271 | 272 | u, err := url.Parse(testServer.URL) 273 | assert.NoError(t, err) 274 | 275 | c, err := client.NewClientFromConfig( 276 | 1500, 277 | nil, 278 | "", 279 | false, 280 | nil, 281 | nil, 282 | true, 283 | false, 284 | u, 285 | ) 286 | assert.NoError(t, err) 287 | 288 | req, err := http.NewRequest(http.MethodGet, u.String(), nil) 289 | assert.NoError(t, err) 290 | 291 | res, err := c.Do(req) //nolint:bodyclose 292 | assert.Error(t, err) 293 | assert.Nil(t, res) 294 | 295 | assert.Contains(t, err.Error(), "certificate") 296 | 297 | // the request should NOT hit the handler 298 | assert.Equal(t, 0, serverAssertion.Len()) 299 | } 300 | 301 | func TestShouldBeAbleToSkipSSLCertificatesCheck(t *testing.T) { 302 | testServer, serverAssertion := test.NewTSLServerWithAssertion( 303 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 304 | w.WriteHeader(http.StatusNoContent) 305 | }), 306 | ) 307 | defer testServer.Close() 308 | 309 | u, err := url.Parse(testServer.URL) 310 | assert.NoError(t, err) 311 | 312 | c, err := client.NewClientFromConfig( 313 | 1500, 314 | nil, 315 | "", 316 | false, 317 | nil, 318 | nil, 319 | true, 320 | true, 321 | u, 322 | ) 323 | assert.NoError(t, err) 324 | 325 | req, err := http.NewRequest(http.MethodGet, u.String(), nil) 326 | assert.NoError(t, err) 327 | 328 | res, err := c.Do(req) 329 | assert.NoError(t, err) 330 | 331 | res.Body.Close() //nolint:errcheck,gosec 332 | 333 | assert.Equal(t, http.StatusNoContent, res.StatusCode) 334 | 335 | // the request should hit the handler 336 | assert.Equal(t, 1, serverAssertion.Len()) 337 | } 338 | -------------------------------------------------------------------------------- /pkg/cmd/scan.go: -------------------------------------------------------------------------------- 1 | package cmd 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/http" 7 | "net/url" 8 | "os" 9 | "os/signal" 10 | 11 | "github.com/pkg/errors" 12 | "github.com/sirupsen/logrus" 13 | "github.com/spf13/cobra" 14 | "github.com/stefanoj3/dirstalk/pkg/cmd/termination" 15 | "github.com/stefanoj3/dirstalk/pkg/common" 16 | "github.com/stefanoj3/dirstalk/pkg/dictionary" 17 | "github.com/stefanoj3/dirstalk/pkg/scan" 18 | "github.com/stefanoj3/dirstalk/pkg/scan/client" 19 | "github.com/stefanoj3/dirstalk/pkg/scan/filter" 20 | "github.com/stefanoj3/dirstalk/pkg/scan/output" 21 | "github.com/stefanoj3/dirstalk/pkg/scan/producer" 22 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer" 23 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer/tree" 24 | ) 25 | 26 | func NewScanCommand(logger *logrus.Logger) *cobra.Command { 27 | cmd := &cobra.Command{ 28 | Use: "scan [url]", 29 | Short: "Scan the given URL", 30 | RunE: buildScanFunction(logger), 31 | } 32 | 33 | cmd.Flags().StringP( 34 | flagScanDictionary, 35 | flagScanDictionaryShort, 36 | "", 37 | "dictionary to use for the scan (path to local file or remote url)", 38 | ) 39 | common.Must(cmd.MarkFlagFilename(flagScanDictionary)) 40 | common.Must(cmd.MarkFlagRequired(flagScanDictionary)) 41 | 42 | cmd.Flags().IntP( 43 | flagScanDictionaryGetTimeout, 44 | "", 45 | 50000, 46 | "timeout in milliseconds (used when fetching remote dictionary)", 47 | ) 48 | 49 | cmd.Flags().StringSlice( 50 | flagScanHTTPMethods, 51 | []string{"GET"}, 52 | "comma separated list of http methods to use; eg: GET,POST,PUT", 53 | ) 54 | 55 | cmd.Flags().IntSlice( 56 | flagScanHTTPStatusesToIgnore, 57 | []int{http.StatusNotFound}, 58 | "comma separated list of http statuses to ignore when showing and processing results; eg: 404,301", 59 | ) 60 | 61 | cmd.Flags().IntP( 62 | flagScanThreads, 63 | flagScanThreadsShort, 64 | 3, 65 | "amount of threads for concurrent requests", 66 | ) 67 | 68 | cmd.Flags().IntP( 69 | flagScanHTTPTimeout, 70 | "", 71 | 5000, 72 | "timeout in milliseconds", 73 | ) 74 | 75 | cmd.Flags().BoolP( 76 | flagScanHTTPCacheRequests, 77 | "", 78 | true, 79 | "cache requests to avoid performing the same request multiple times within the same scan (EG if the "+ 80 | "server reply with the same redirect location multiple times, dirstalk will follow it only once)", 81 | ) 82 | 83 | cmd.Flags().IntP( 84 | flagScanScanDepth, 85 | "", 86 | 3, 87 | "scan depth", 88 | ) 89 | 90 | cmd.Flags().StringP( 91 | flagScanSocks5Host, 92 | "", 93 | "", 94 | "socks5 host to use", 95 | ) 96 | 97 | cmd.Flags().StringP( 98 | flagScanUserAgent, 99 | "", 100 | "", 101 | "user agent to use for http requests", 102 | ) 103 | 104 | cmd.Flags().BoolP( 105 | flagScanCookieJar, 106 | "", 107 | false, 108 | "enables the use of a cookie jar: it will retain any cookie sent "+ 109 | "from the server and send them for the following requests", 110 | ) 111 | 112 | cmd.Flags().StringArray( 113 | flagScanCookie, 114 | []string{}, 115 | "cookie to add to each request; eg name=value (can be specified multiple times)", 116 | ) 117 | 118 | cmd.Flags().StringArray( 119 | flagScanHeader, 120 | []string{}, 121 | "header to add to each request; eg name=value (can be specified multiple times)", 122 | ) 123 | 124 | cmd.Flags().String( 125 | flagScanResultOutput, 126 | "", 127 | "path where to store result output", 128 | ) 129 | 130 | cmd.Flags().Bool( 131 | flagShouldSkipSSLCertificatesValidation, 132 | false, 133 | "to skip checking the validity of SSL certificates", 134 | ) 135 | 136 | cmd.Flags().Bool( 137 | flagIgnore20xWithEmptyBody, 138 | false, 139 | "ignore HTTP 20x responses with empty body", 140 | ) 141 | 142 | return cmd 143 | } 144 | 145 | func buildScanFunction(logger *logrus.Logger) func(cmd *cobra.Command, args []string) error { 146 | f := func(cmd *cobra.Command, args []string) error { 147 | u, err := getURL(args) 148 | if err != nil { 149 | return err 150 | } 151 | 152 | cnf, err := scanConfigFromCmd(cmd) 153 | if err != nil { 154 | return errors.Wrap(err, "failed to build config") 155 | } 156 | 157 | return startScan(logger, cnf, u) 158 | } 159 | 160 | return f 161 | } 162 | 163 | func getURL(args []string) (*url.URL, error) { 164 | if len(args) == 0 { 165 | return nil, errors.New("no URL provided") 166 | } 167 | 168 | arg := args[0] 169 | 170 | u, err := url.ParseRequestURI(arg) 171 | if err != nil { 172 | return nil, errors.Wrap(err, "the first argument must be a valid url") 173 | } 174 | 175 | return u, nil 176 | } 177 | 178 | // startScan is a convenience method that wires together all the dependencies needed to start a scan. 179 | func startScan(logger *logrus.Logger, cnf *scan.Config, u *url.URL) error { 180 | dict, err := buildDictionary(cnf, u) 181 | if err != nil { 182 | return err 183 | } 184 | 185 | s, err := buildScanner(cnf, dict, u, logger) 186 | if err != nil { 187 | return err 188 | } 189 | 190 | logger.WithFields(logrus.Fields{ 191 | "url": u.String(), 192 | "threads": cnf.Threads, 193 | "dictionary-length": len(dict), 194 | "scan-depth": cnf.ScanDepth, 195 | "timeout": cnf.TimeoutInMilliseconds, 196 | "socks5": cnf.Socks5Url, 197 | "cookies": stringifyCookies(cnf.Cookies), 198 | "cookie-jar": cnf.UseCookieJar, 199 | "headers": stringifyHeaders(cnf.Headers), 200 | "user-agent": cnf.UserAgent, 201 | }).Info("Starting scan") 202 | 203 | resultSummarizer := summarizer.NewResultSummarizer(tree.NewResultTreeProducer(), logger) 204 | 205 | osSigint := make(chan os.Signal, 1) 206 | signal.Notify(osSigint, os.Interrupt) 207 | 208 | outputSaver, err := newOutputSaver(cnf.Out) 209 | if err != nil { 210 | return errors.Wrap(err, "failed to create output saver") 211 | } 212 | 213 | defer func() { 214 | resultSummarizer.Summarize() 215 | 216 | err := outputSaver.Close() 217 | if err != nil { 218 | logger.WithError(err).Error("failed to close output file") 219 | } 220 | 221 | logger.Info("Finished scan") 222 | }() 223 | 224 | ctx, cancellationFunc := context.WithCancel(context.Background()) 225 | defer cancellationFunc() 226 | 227 | resultsChannel := s.Scan(ctx, u, cnf.Threads) 228 | 229 | terminationHandler := termination.NewTerminationHandler(2) 230 | 231 | for { 232 | select { 233 | case <-osSigint: 234 | terminationHandler.SignalTermination() 235 | cancellationFunc() 236 | 237 | if terminationHandler.ShouldTerminate() { 238 | logger.Info("Received sigint, terminating...") 239 | 240 | return nil 241 | } 242 | 243 | logger.Info( 244 | "Received sigint, trying to shutdown gracefully, another SIGNINT will terminate the application", 245 | ) 246 | case result, ok := <-resultsChannel: 247 | if !ok { 248 | logger.Debug("result channel is being closed, scan should be complete") 249 | 250 | return nil 251 | } 252 | 253 | resultSummarizer.Add(result) 254 | 255 | if err := outputSaver.Save(result); err != nil { 256 | return errors.Wrap(err, "failed to add output to file") 257 | } 258 | } 259 | } 260 | } 261 | 262 | func buildScanner(cnf *scan.Config, dict []string, u *url.URL, logger *logrus.Logger) (*scan.Scanner, error) { 263 | targetProducer := producer.NewDictionaryProducer(cnf.HTTPMethods, dict, cnf.ScanDepth) 264 | reproducer := producer.NewReProducer(targetProducer) 265 | 266 | resultFilter := filter.NewHTTPStatusResultFilter(cnf.HTTPStatusesToIgnore, cnf.IgnoreEmpty20xResponses) 267 | 268 | scannerClient, err := buildScannerClient(cnf, u) 269 | if err != nil { 270 | return nil, err 271 | } 272 | 273 | s := scan.NewScanner( 274 | scannerClient, 275 | targetProducer, 276 | reproducer, 277 | resultFilter, 278 | logger, 279 | ) 280 | 281 | return s, nil 282 | } 283 | 284 | func buildDictionary(cnf *scan.Config, u *url.URL) ([]string, error) { 285 | c, err := buildDictionaryClient(cnf, u) 286 | if err != nil { 287 | return nil, err 288 | } 289 | 290 | dict, err := dictionary.NewDictionaryFrom(cnf.DictionaryPath, c) 291 | if err != nil { 292 | return nil, errors.Wrap(err, "failed to build dictionary") 293 | } 294 | 295 | return dict, nil 296 | } 297 | 298 | func buildScannerClient(cnf *scan.Config, u *url.URL) (*http.Client, error) { 299 | c, err := client.NewClientFromConfig( 300 | cnf.TimeoutInMilliseconds, 301 | cnf.Socks5Url, 302 | cnf.UserAgent, 303 | cnf.UseCookieJar, 304 | cnf.Cookies, 305 | cnf.Headers, 306 | cnf.CacheRequests, 307 | cnf.ShouldSkipSSLCertificatesValidation, 308 | u, 309 | ) 310 | if err != nil { 311 | return nil, errors.Wrap(err, "failed to build scanner client") 312 | } 313 | 314 | return c, nil 315 | } 316 | 317 | func buildDictionaryClient(cnf *scan.Config, u *url.URL) (*http.Client, error) { 318 | c, err := client.NewClientFromConfig( 319 | cnf.DictionaryTimeoutInMilliseconds, 320 | cnf.Socks5Url, 321 | cnf.UserAgent, 322 | cnf.UseCookieJar, 323 | cnf.Cookies, 324 | cnf.Headers, 325 | cnf.CacheRequests, 326 | cnf.ShouldSkipSSLCertificatesValidation, 327 | u, 328 | ) 329 | if err != nil { 330 | return nil, errors.Wrap(err, "failed to build dictionary client") 331 | } 332 | 333 | return c, nil 334 | } 335 | 336 | func newOutputSaver(path string) (OutputSaver, error) { 337 | if path == "" { 338 | return output.NewNullSaver(), nil 339 | } 340 | 341 | return output.NewFileSaver(path) 342 | } 343 | 344 | func stringifyCookies(cookies []*http.Cookie) string { 345 | result := "" 346 | 347 | for _, cookie := range cookies { 348 | result += fmt.Sprintf("{%s=%s}", cookie.Name, cookie.Value) 349 | } 350 | 351 | return result 352 | } 353 | 354 | func stringifyHeaders(headers map[string]string) string { 355 | result := "" 356 | 357 | for name, value := range headers { 358 | result += fmt.Sprintf("{%s:%s}", name, value) 359 | } 360 | 361 | return result 362 | } 363 | -------------------------------------------------------------------------------- /pkg/scan/summarizer/tree/result_tree_test.go: -------------------------------------------------------------------------------- 1 | package tree_test 2 | 3 | import ( 4 | "net/http" 5 | "testing" 6 | 7 | "github.com/stefanoj3/dirstalk/pkg/common/test" 8 | "github.com/stefanoj3/dirstalk/pkg/scan" 9 | "github.com/stefanoj3/dirstalk/pkg/scan/summarizer/tree" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var testResult string 14 | 15 | func TestNewResultTreePrinter(t *testing.T) { 16 | results := []scan.Result{ 17 | scan.NewResult( 18 | scan.Target{ 19 | Method: http.MethodPost, 20 | Path: "/", 21 | }, 22 | &http.Response{ 23 | StatusCode: http.StatusCreated, 24 | Request: &http.Request{ 25 | URL: test.MustParseURL(t, "http://mysite/"), 26 | }, 27 | }, 28 | ), 29 | scan.NewResult( 30 | scan.Target{ 31 | Method: http.MethodPost, 32 | Path: "/home", 33 | }, 34 | &http.Response{ 35 | StatusCode: http.StatusCreated, 36 | Request: &http.Request{ 37 | URL: test.MustParseURL(t, "http://mysite/home"), 38 | }, 39 | }, 40 | ), 41 | scan.NewResult( 42 | scan.Target{ 43 | Method: http.MethodPost, 44 | Path: "/home/123/", 45 | }, 46 | &http.Response{ 47 | StatusCode: http.StatusCreated, 48 | Request: &http.Request{ 49 | URL: test.MustParseURL(t, "http://mysite/home/123"), 50 | }, 51 | }, 52 | ), 53 | scan.NewResult( 54 | scan.Target{ 55 | Method: http.MethodPost, 56 | Path: "/about", 57 | }, 58 | &http.Response{ 59 | StatusCode: http.StatusCreated, 60 | Request: &http.Request{ 61 | URL: test.MustParseURL(t, "http://mysite/about"), 62 | }, 63 | }, 64 | ), 65 | } 66 | 67 | actual := tree.NewResultTreeProducer().String(results) 68 | 69 | expected := `/ 70 | ├── about 71 | └── home 72 | └── 123 73 | ` 74 | 75 | assert.Equal(t, expected, actual) 76 | } 77 | 78 | func BenchmarkResultTree(b *testing.B) { 79 | results := []scan.Result{ 80 | scan.NewResult( 81 | scan.Target{ 82 | Method: http.MethodPost, 83 | Path: "/", 84 | }, 85 | &http.Response{ 86 | StatusCode: http.StatusCreated, 87 | Request: &http.Request{ 88 | URL: test.MustParseURL(b, "/"), 89 | }, 90 | }, 91 | ), 92 | scan.NewResult( 93 | scan.Target{ 94 | Method: http.MethodPost, 95 | Path: "/home", 96 | }, 97 | &http.Response{ 98 | StatusCode: http.StatusCreated, 99 | Request: &http.Request{ 100 | URL: test.MustParseURL(b, "http://mysite/home"), 101 | }, 102 | }, 103 | ), 104 | scan.NewResult( 105 | scan.Target{ 106 | Method: http.MethodPost, 107 | Path: "/home/123", 108 | }, 109 | &http.Response{ 110 | StatusCode: http.StatusCreated, 111 | Request: &http.Request{ 112 | URL: test.MustParseURL(b, "http://mysite/home/123"), 113 | }, 114 | }, 115 | ), 116 | scan.NewResult( 117 | scan.Target{ 118 | Method: http.MethodPost, 119 | Path: "/about", 120 | }, 121 | &http.Response{ 122 | StatusCode: http.StatusCreated, 123 | Request: &http.Request{ 124 | URL: test.MustParseURL(b, "http://mysite/about"), 125 | }, 126 | }, 127 | ), 128 | scan.NewResult( 129 | scan.Target{ 130 | Method: http.MethodPost, 131 | Path: "/about", 132 | }, 133 | &http.Response{ 134 | StatusCode: http.StatusCreated, 135 | Request: &http.Request{ 136 | URL: test.MustParseURL(b, "http://mysite/about"), 137 | }, 138 | }, 139 | ), 140 | scan.NewResult( 141 | scan.Target{ 142 | Method: http.MethodPost, 143 | Path: "/about", 144 | }, 145 | &http.Response{ 146 | StatusCode: http.StatusCreated, 147 | Request: &http.Request{ 148 | URL: test.MustParseURL(b, "http://mysite/about"), 149 | }, 150 | }, 151 | ), 152 | scan.NewResult( 153 | scan.Target{ 154 | Method: http.MethodPost, 155 | Path: "/about/1", 156 | }, 157 | &http.Response{ 158 | StatusCode: http.StatusCreated, 159 | Request: &http.Request{ 160 | URL: test.MustParseURL(b, "/about/1"), 161 | }, 162 | }, 163 | ), 164 | scan.NewResult( 165 | scan.Target{ 166 | Method: http.MethodPost, 167 | Path: "/about/12", 168 | }, 169 | &http.Response{ 170 | StatusCode: http.StatusCreated, 171 | Request: &http.Request{ 172 | URL: test.MustParseURL(b, "/about/12"), 173 | }, 174 | }, 175 | ), 176 | scan.NewResult( 177 | scan.Target{ 178 | Method: http.MethodPost, 179 | Path: "/about/123", 180 | }, 181 | &http.Response{ 182 | StatusCode: http.StatusCreated, 183 | Request: &http.Request{ 184 | URL: test.MustParseURL(b, "/about/123"), 185 | }, 186 | }, 187 | ), 188 | scan.NewResult( 189 | scan.Target{ 190 | Method: http.MethodPost, 191 | Path: "/about/1/2/3", 192 | }, 193 | &http.Response{ 194 | StatusCode: http.StatusCreated, 195 | Request: &http.Request{ 196 | URL: test.MustParseURL(b, "/about/1/2/3"), 197 | }, 198 | }, 199 | ), 200 | scan.NewResult( 201 | scan.Target{ 202 | Method: http.MethodPost, 203 | Path: "/about/1/2/a", 204 | }, 205 | &http.Response{ 206 | StatusCode: http.StatusCreated, 207 | Request: &http.Request{ 208 | URL: test.MustParseURL(b, "/about/1/2/a"), 209 | }, 210 | }, 211 | ), 212 | scan.NewResult( 213 | scan.Target{ 214 | Method: http.MethodPost, 215 | Path: "/about/1/2/b", 216 | }, 217 | &http.Response{ 218 | StatusCode: http.StatusCreated, 219 | Request: &http.Request{ 220 | URL: test.MustParseURL(b, "/about/1/2/b"), 221 | }, 222 | }, 223 | ), 224 | scan.NewResult( 225 | scan.Target{ 226 | Method: http.MethodPost, 227 | Path: "/about/1/2/b/c/d/e", 228 | }, 229 | &http.Response{ 230 | StatusCode: http.StatusCreated, 231 | Request: &http.Request{ 232 | URL: test.MustParseURL(b, "/about/1/2/b/c/d/e"), 233 | }, 234 | }, 235 | ), 236 | scan.NewResult( 237 | scan.Target{ 238 | Method: http.MethodPost, 239 | Path: "/about/1/2/b/c/f/e", 240 | }, 241 | &http.Response{ 242 | StatusCode: http.StatusCreated, 243 | Request: &http.Request{ 244 | URL: test.MustParseURL(b, "/about/1/2/b/c/f/e"), 245 | }, 246 | }, 247 | ), 248 | scan.NewResult( 249 | scan.Target{ 250 | Method: http.MethodPost, 251 | Path: "/about/1/2/a/c/f/e", 252 | }, 253 | &http.Response{ 254 | StatusCode: http.StatusCreated, 255 | Request: &http.Request{ 256 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e"), 257 | }, 258 | }, 259 | ), 260 | scan.NewResult( 261 | scan.Target{ 262 | Method: http.MethodPost, 263 | Path: "/about/1/2/a/c/f/e/g/h/i/l/m/n/o/p/q", 264 | }, 265 | &http.Response{ 266 | StatusCode: http.StatusCreated, 267 | Request: &http.Request{ 268 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/i/l/m/n/o/p/q"), 269 | }, 270 | }, 271 | ), 272 | scan.NewResult( 273 | scan.Target{ 274 | Method: http.MethodPost, 275 | Path: "/about/1/2/a/c/f/e/g/h/1/l/m/n/o/p/q", 276 | }, 277 | &http.Response{ 278 | StatusCode: http.StatusCreated, 279 | Request: &http.Request{ 280 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/1/l/m/n/o/p/q"), 281 | }, 282 | }, 283 | ), 284 | scan.NewResult( 285 | scan.Target{ 286 | Method: http.MethodPost, 287 | Path: "/about/1/2/a/c/f/e/g/h/2/l/m/n/o/p/q", 288 | }, 289 | &http.Response{ 290 | StatusCode: http.StatusCreated, 291 | Request: &http.Request{ 292 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/2/l/m/n/o/p/q"), 293 | }, 294 | }, 295 | ), 296 | scan.NewResult( 297 | scan.Target{ 298 | Method: http.MethodPost, 299 | Path: "/about/1/2/a/c/f/e/g/h/3/l/m/n/o/p/q", 300 | }, 301 | &http.Response{ 302 | StatusCode: http.StatusCreated, 303 | Request: &http.Request{ 304 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/3/l/m/n/o/p/q"), 305 | }, 306 | }, 307 | ), 308 | scan.NewResult( 309 | scan.Target{ 310 | Method: http.MethodPost, 311 | Path: "/about/1/2/a/c/f/e/g/h/4/l/m/n/o/p/q", 312 | }, 313 | &http.Response{ 314 | StatusCode: http.StatusCreated, 315 | Request: &http.Request{ 316 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/l/m/n/o/p/q"), 317 | }, 318 | }, 319 | ), 320 | scan.NewResult( 321 | scan.Target{ 322 | Method: http.MethodPost, 323 | Path: "/about/1/2/a/c/f/e/g/h/4/1/m/n/o/p/q", 324 | }, 325 | &http.Response{ 326 | StatusCode: http.StatusCreated, 327 | Request: &http.Request{ 328 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/1/m/n/o/p/q"), 329 | }, 330 | }, 331 | ), 332 | scan.NewResult( 333 | scan.Target{ 334 | Method: http.MethodPost, 335 | Path: "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/q", 336 | }, 337 | &http.Response{ 338 | StatusCode: http.StatusCreated, 339 | Request: &http.Request{ 340 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/q"), 341 | }, 342 | }, 343 | ), 344 | scan.NewResult( 345 | scan.Target{ 346 | Method: http.MethodPost, 347 | Path: "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/u", 348 | }, 349 | &http.Response{ 350 | StatusCode: http.StatusCreated, 351 | Request: &http.Request{ 352 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/u"), 353 | }, 354 | }, 355 | ), 356 | scan.NewResult( 357 | scan.Target{ 358 | Method: http.MethodPost, 359 | Path: "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/z", 360 | }, 361 | &http.Response{ 362 | StatusCode: http.StatusCreated, 363 | Request: &http.Request{ 364 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/z"), 365 | }, 366 | }, 367 | ), 368 | scan.NewResult( 369 | scan.Target{ 370 | Method: http.MethodPost, 371 | Path: "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/z/1", 372 | }, 373 | &http.Response{ 374 | StatusCode: http.StatusCreated, 375 | Request: &http.Request{ 376 | URL: test.MustParseURL(b, "/about/1/2/a/c/f/e/g/h/4/2/m/n/o/p/z/1"), 377 | }, 378 | }, 379 | ), 380 | scan.NewResult( 381 | scan.Target{ 382 | Method: http.MethodPost, 383 | Path: "/somepage", 384 | }, 385 | &http.Response{ 386 | StatusCode: http.StatusCreated, 387 | Request: &http.Request{ 388 | URL: test.MustParseURL(b, "/somepage"), 389 | }, 390 | }, 391 | ), 392 | scan.NewResult( 393 | scan.Target{ 394 | Method: http.MethodPost, 395 | Path: "/anotherpage", 396 | }, 397 | &http.Response{ 398 | StatusCode: http.StatusCreated, 399 | Request: &http.Request{ 400 | URL: test.MustParseURL(b, "/anotherpage"), 401 | }, 402 | }, 403 | ), 404 | scan.NewResult( 405 | scan.Target{ 406 | Method: http.MethodPost, 407 | Path: "/anotherpage2", 408 | }, 409 | &http.Response{ 410 | StatusCode: http.StatusCreated, 411 | Request: &http.Request{ 412 | URL: test.MustParseURL(b, "/anotherpage2"), 413 | }, 414 | }, 415 | ), 416 | } 417 | 418 | b.ResetTimer() 419 | 420 | for i := 0; i < b.N; i++ { 421 | testResult = tree.NewResultTreeProducer().String(results) 422 | } 423 | 424 | testResult += "1" 425 | } 426 | -------------------------------------------------------------------------------- /pkg/scan/scanner_integration_test.go: -------------------------------------------------------------------------------- 1 | package scan_test 2 | 3 | import ( 4 | "context" 5 | "net/http" 6 | "testing" 7 | "time" 8 | 9 | "github.com/stefanoj3/dirstalk/pkg/common/test" 10 | "github.com/stefanoj3/dirstalk/pkg/scan" 11 | "github.com/stefanoj3/dirstalk/pkg/scan/client" 12 | "github.com/stefanoj3/dirstalk/pkg/scan/filter" 13 | "github.com/stefanoj3/dirstalk/pkg/scan/producer" 14 | "github.com/stretchr/testify/assert" 15 | ) 16 | 17 | func TestScanningWithEmptyProducerWillProduceNoResults(t *testing.T) { 18 | logger, _ := test.NewLogger() 19 | 20 | prod := producer.NewDictionaryProducer([]string{}, []string{}, 1) 21 | c := &http.Client{Timeout: time.Microsecond} 22 | sut := scan.NewScanner( 23 | c, 24 | prod, 25 | producer.NewReProducer(prod), 26 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 27 | logger, 28 | ) 29 | 30 | results := sut.Scan(context.Background(), test.MustParseURL(t, "http://localhost/"), 10) 31 | 32 | for r := range results { 33 | t.Fatalf("No results expected, got %s", r.Target.Path) 34 | } 35 | } 36 | 37 | func TestScannerWillLogAnErrorWithInvalidDictionary(t *testing.T) { 38 | logger, loggerBuffer := test.NewLogger() 39 | 40 | prod := producer.NewDictionaryProducer( 41 | []string{"\n"}, 42 | []string{"/home"}, 43 | 1, 44 | ) 45 | 46 | testServer, serverAssertion := test.NewServerWithAssertion( 47 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 48 | ) 49 | defer testServer.Close() 50 | 51 | c, err := client.NewClientFromConfig( 52 | 1000, 53 | nil, 54 | "", 55 | false, 56 | nil, 57 | nil, 58 | true, 59 | false, 60 | test.MustParseURL(t, testServer.URL), 61 | ) 62 | assert.NoError(t, err) 63 | 64 | sut := scan.NewScanner( 65 | c, 66 | prod, 67 | producer.NewReProducer(prod), 68 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 69 | logger, 70 | ) 71 | 72 | results := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 10) 73 | 74 | for r := range results { 75 | t.Fatalf("No results expected, got %s", r.Target.Path) 76 | } 77 | 78 | assert.Contains(t, loggerBuffer.String(), "failed to build request") 79 | assert.Contains(t, loggerBuffer.String(), "invalid method") 80 | assert.Equal(t, 0, serverAssertion.Len()) 81 | } 82 | 83 | func TestScannerWillNotRedirectIfStatusCodeIsInvalid(t *testing.T) { 84 | logger, loggerBuffer := test.NewLogger() 85 | 86 | prod := producer.NewDictionaryProducer( 87 | []string{http.MethodGet}, 88 | []string{"/home"}, 89 | 3, 90 | ) 91 | 92 | testServer, serverAssertion := test.NewServerWithAssertion( 93 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 94 | w.Header().Add("location", "/potato") 95 | if r.URL.Path == "/home" { 96 | w.WriteHeader(http.StatusOK) 97 | 98 | return 99 | } 100 | 101 | w.WriteHeader(http.StatusNotFound) 102 | }), 103 | ) 104 | defer testServer.Close() 105 | 106 | c, err := client.NewClientFromConfig( 107 | 1000, 108 | nil, 109 | "", 110 | false, 111 | nil, 112 | nil, 113 | true, 114 | false, 115 | test.MustParseURL(t, testServer.URL), 116 | ) 117 | assert.NoError(t, err) 118 | 119 | sut := scan.NewScanner( 120 | c, 121 | prod, 122 | producer.NewReProducer(prod), 123 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 124 | logger, 125 | ) 126 | 127 | results := make([]scan.Result, 0, 2) 128 | resultsChannel := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 10) 129 | 130 | for r := range resultsChannel { 131 | results = append(results, r) 132 | } 133 | 134 | expectedsResults := []scan.Result{ 135 | { 136 | Target: scan.Target{Path: "/home", Method: http.MethodGet, Depth: 3}, 137 | StatusCode: http.StatusOK, 138 | URL: *test.MustParseURL(t, testServer.URL+"/home"), 139 | }, 140 | } 141 | 142 | assert.Equal(t, expectedsResults, results) 143 | 144 | assert.Contains(t, loggerBuffer.String(), "/home") 145 | assert.Contains(t, loggerBuffer.String(), "/home/home") 146 | assert.Equal(t, 2, serverAssertion.Len()) 147 | } 148 | 149 | func TestScannerWillChangeMethodForRedirect(t *testing.T) { 150 | logger, loggerBuffer := test.NewLogger() 151 | 152 | prod := producer.NewDictionaryProducer( 153 | []string{http.MethodPatch}, 154 | []string{"/home"}, 155 | 3, 156 | ) 157 | 158 | testServer, serverAssertion := test.NewServerWithAssertion( 159 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 160 | if r.URL.Path == "/home" { 161 | http.Redirect(w, r, "/potato", http.StatusMovedPermanently) 162 | 163 | return 164 | } 165 | 166 | if r.URL.Path == "/potato" { 167 | w.WriteHeader(http.StatusCreated) 168 | 169 | return 170 | } 171 | 172 | w.WriteHeader(http.StatusNotFound) 173 | }), 174 | ) 175 | defer testServer.Close() 176 | 177 | c, err := client.NewClientFromConfig( 178 | 1000, 179 | nil, 180 | "", 181 | false, 182 | nil, 183 | nil, 184 | true, 185 | false, 186 | test.MustParseURL(t, testServer.URL), 187 | ) 188 | assert.NoError(t, err) 189 | 190 | sut := scan.NewScanner( 191 | c, 192 | prod, 193 | producer.NewReProducer(prod), 194 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 195 | logger, 196 | ) 197 | 198 | results := make([]scan.Result, 0, 3) 199 | resultsChannel := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 1) 200 | 201 | for r := range resultsChannel { 202 | results = append(results, r) 203 | } 204 | 205 | expectedResults := []scan.Result{ 206 | { 207 | Target: scan.Target{Path: "/home", Method: http.MethodPatch, Depth: 3}, 208 | StatusCode: http.StatusMovedPermanently, 209 | URL: *test.MustParseURL(t, testServer.URL+"/home"), 210 | }, 211 | { 212 | Target: scan.Target{Path: "/potato", Method: http.MethodGet, Depth: 2}, 213 | StatusCode: http.StatusCreated, 214 | URL: *test.MustParseURL(t, testServer.URL+"/potato"), 215 | }, 216 | } 217 | 218 | assert.Equal(t, expectedResults, results) 219 | 220 | assert.NotContains(t, loggerBuffer.String(), "error") 221 | assert.Equal(t, 4, serverAssertion.Len()) 222 | } 223 | 224 | func TestScannerWhenOutOfDepthWillNotFollowRedirect(t *testing.T) { 225 | logger, loggerBuffer := test.NewLogger() 226 | 227 | prod := producer.NewDictionaryProducer( 228 | []string{http.MethodPatch}, 229 | []string{"/home"}, 230 | 0, 231 | ) 232 | 233 | testServer, serverAssertion := test.NewServerWithAssertion( 234 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 235 | if r.URL.Path == "/home" { 236 | http.Redirect(w, r, "/potato", http.StatusMovedPermanently) 237 | 238 | return 239 | } 240 | 241 | w.WriteHeader(http.StatusNotFound) 242 | }), 243 | ) 244 | defer testServer.Close() 245 | 246 | c, err := client.NewClientFromConfig( 247 | 1000, 248 | nil, 249 | "", 250 | false, 251 | nil, 252 | nil, 253 | true, 254 | false, 255 | test.MustParseURL(t, testServer.URL), 256 | ) 257 | assert.NoError(t, err) 258 | 259 | sut := scan.NewScanner( 260 | c, 261 | prod, 262 | producer.NewReProducer(prod), 263 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 264 | logger, 265 | ) 266 | 267 | results := make([]scan.Result, 0, 1) 268 | resultsChannel := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 1) 269 | 270 | for r := range resultsChannel { 271 | results = append(results, r) 272 | } 273 | 274 | expectedResults := []scan.Result{ 275 | { 276 | Target: scan.Target{Path: "/home", Method: http.MethodPatch, Depth: 0}, 277 | StatusCode: http.StatusMovedPermanently, 278 | URL: *test.MustParseURL(t, testServer.URL+"/home"), 279 | }, 280 | } 281 | 282 | assert.Equal(t, expectedResults, results) 283 | 284 | loggerBufferAsString := loggerBuffer.String() 285 | assert.Contains(t, loggerBufferAsString, "/home") 286 | assert.Contains(t, loggerBufferAsString, "depth is 0, not following any redirect") 287 | assert.NotContains(t, loggerBufferAsString, "error") 288 | assert.Equal(t, 1, serverAssertion.Len()) 289 | } 290 | 291 | func TestScannerWillSkipRedirectWhenLocationHostIsDifferent(t *testing.T) { 292 | logger, loggerBuffer := test.NewLogger() 293 | 294 | prod := producer.NewDictionaryProducer( 295 | []string{http.MethodPatch}, 296 | []string{"/home"}, 297 | 3, 298 | ) 299 | 300 | testServer, serverAssertion := test.NewServerWithAssertion( 301 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 302 | if r.URL.Path == "/home" { 303 | http.Redirect(w, r, "http://gibberish/potato", http.StatusMovedPermanently) 304 | 305 | return 306 | } 307 | 308 | w.WriteHeader(http.StatusNotFound) 309 | }), 310 | ) 311 | defer testServer.Close() 312 | 313 | c, err := client.NewClientFromConfig( 314 | 1000, 315 | nil, 316 | "", 317 | false, 318 | nil, 319 | nil, 320 | true, 321 | false, 322 | test.MustParseURL(t, testServer.URL), 323 | ) 324 | assert.NoError(t, err) 325 | 326 | sut := scan.NewScanner( 327 | c, 328 | prod, 329 | producer.NewReProducer(prod), 330 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 331 | logger, 332 | ) 333 | 334 | results := make([]scan.Result, 0, 2) 335 | resultsChannel := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 1) 336 | 337 | for r := range resultsChannel { 338 | results = append(results, r) 339 | } 340 | 341 | expectedResults := []scan.Result{ 342 | { 343 | Target: scan.Target{Path: "/home", Method: http.MethodPatch, Depth: 3}, 344 | StatusCode: http.StatusMovedPermanently, 345 | URL: *test.MustParseURL(t, testServer.URL+"/home"), 346 | }, 347 | } 348 | 349 | assert.Equal(t, expectedResults, results) 350 | 351 | loggerBufferAsString := loggerBuffer.String() 352 | assert.Contains(t, loggerBufferAsString, "skipping redirect, pointing to a different host") 353 | assert.NotContains(t, loggerBufferAsString, "error") 354 | assert.Equal(t, 2, serverAssertion.Len()) 355 | } 356 | 357 | func TestScannerWillIgnoreRequestRedundantError(t *testing.T) { 358 | logger, loggerBuffer := test.NewLogger() 359 | 360 | prod := producer.NewDictionaryProducer( 361 | []string{http.MethodGet}, 362 | []string{"/home", "/home"}, // twice the same entry to trick the client into doing the same request twice 363 | 3, 364 | ) 365 | 366 | testServer, serverAssertion := test.NewServerWithAssertion( 367 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 368 | w.WriteHeader(http.StatusNotFound) 369 | }), 370 | ) 371 | defer testServer.Close() 372 | 373 | c, err := client.NewClientFromConfig( 374 | 1000, 375 | nil, 376 | "", 377 | false, 378 | nil, 379 | nil, 380 | true, 381 | false, 382 | test.MustParseURL(t, testServer.URL), 383 | ) 384 | assert.NoError(t, err) 385 | 386 | sut := scan.NewScanner( 387 | c, 388 | prod, 389 | producer.NewReProducer(prod), 390 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 391 | logger, 392 | ) 393 | 394 | results := make([]scan.Result, 0, 1) 395 | resultsChannel := sut.Scan(context.Background(), test.MustParseURL(t, testServer.URL), 1) 396 | 397 | for r := range resultsChannel { 398 | results = append(results, r) 399 | } 400 | 401 | assert.Equal(t, 0, len(results)) 402 | 403 | loggerBufferAsString := loggerBuffer.String() 404 | assert.Contains(t, loggerBufferAsString, "/home") 405 | assert.Contains(t, loggerBufferAsString, "this request has been made already") 406 | assert.Equal(t, 1, serverAssertion.Len()) 407 | } 408 | 409 | func TestCanCancelScanUsingContext(t *testing.T) { 410 | logger, _ := test.NewLogger() 411 | 412 | prod := producer.NewDictionaryProducer( 413 | []string{http.MethodGet, http.MethodPatch, http.MethodDelete, http.MethodPost, http.MethodPut}, 414 | []string{"/home", "/index", "/about", "/search", "/jobs", "robots.txt", "/subscription", "/orders"}, 415 | 200000000, 416 | ) 417 | 418 | testServer, serverAssertion := test.NewServerWithAssertion( 419 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 420 | w.WriteHeader(http.StatusOK) 421 | }), 422 | ) 423 | defer testServer.Close() 424 | 425 | // the depth of the dictionary and the fact that the server returns always a 426 | // http.StatusOK should keep this test running forever in case the cancellation would not work 427 | 428 | c, err := client.NewClientFromConfig( 429 | 1000, 430 | nil, 431 | "", 432 | false, 433 | nil, 434 | nil, 435 | true, 436 | false, 437 | test.MustParseURL(t, testServer.URL), 438 | ) 439 | assert.NoError(t, err) 440 | 441 | sut := scan.NewScanner( 442 | c, 443 | prod, 444 | producer.NewReProducer(prod), 445 | filter.NewHTTPStatusResultFilter([]int{http.StatusNotFound}, false), 446 | logger, 447 | ) 448 | 449 | ctx := context.Background() 450 | ctx, cancelFunc := context.WithCancel(ctx) 451 | 452 | resultsChannel := sut.Scan(ctx, test.MustParseURL(t, testServer.URL), 100) 453 | 454 | go func() { 455 | time.Sleep(50 * time.Millisecond) 456 | cancelFunc() 457 | }() 458 | 459 | done := make(chan struct{}) 460 | 461 | go func() { 462 | for range resultsChannel { 463 | 464 | } 465 | done <- struct{}{} 466 | }() 467 | 468 | select { 469 | case <-done: 470 | t.Log("result channel closed") 471 | case <-time.After(time.Second * 8): 472 | t.Fatalf("the scan should have terminated by now, something is wrong with the context cancellation") 473 | } 474 | 475 | assert.True(t, serverAssertion.Len() > 1) 476 | } 477 | -------------------------------------------------------------------------------- /pkg/cmd/scan_integration_test.go: -------------------------------------------------------------------------------- 1 | package cmd_test 2 | 3 | import ( 4 | "io/ioutil" 5 | "net" 6 | "net/http" 7 | "net/http/httptest" 8 | "os" 9 | "sync" 10 | "syscall" 11 | "testing" 12 | "time" 13 | 14 | "github.com/armon/go-socks5" 15 | "github.com/stefanoj3/dirstalk/pkg/common/test" 16 | "github.com/stretchr/testify/assert" 17 | ) 18 | 19 | const socks5TestServerHost = "127.0.0.1:8899" 20 | 21 | func TestScanCommand(t *testing.T) { 22 | logger, loggerBuffer := test.NewLogger() 23 | 24 | c := createCommand(logger) 25 | assert.NotNil(t, c) 26 | 27 | testServer, serverAssertion := test.NewServerWithAssertion( 28 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 29 | if r.URL.Path == "/test/" { 30 | w.WriteHeader(http.StatusOK) 31 | 32 | return 33 | } 34 | if r.URL.Path == "/potato" { 35 | w.WriteHeader(http.StatusOK) 36 | 37 | return 38 | } 39 | 40 | if r.URL.Path == "/test/test/" { 41 | http.Redirect(w, r, "/potato", http.StatusMovedPermanently) 42 | 43 | return 44 | } 45 | 46 | w.WriteHeader(http.StatusNotFound) 47 | }), 48 | ) 49 | defer testServer.Close() 50 | 51 | err := executeCommand( 52 | c, 53 | "scan", 54 | testServer.URL, 55 | "--dictionary", 56 | "testdata/dict2.txt", 57 | "-v", 58 | "--http-statuses-to-ignore", 59 | "404", 60 | "--http-timeout", 61 | "300", 62 | ) 63 | assert.NoError(t, err) 64 | 65 | assert.Equal(t, 17, serverAssertion.Len()) 66 | 67 | requestsMap := map[string]string{} 68 | 69 | serverAssertion.Range(func(_ int, r http.Request) { 70 | requestsMap[r.URL.Path] = r.Method 71 | }) 72 | 73 | expectedRequests := map[string]string{ 74 | "/test/": http.MethodGet, 75 | "/test/home": http.MethodGet, 76 | "/test/blabla": http.MethodGet, 77 | "/test/home/index.php": http.MethodGet, 78 | "/potato": http.MethodGet, 79 | 80 | "/potato/test/": http.MethodGet, 81 | "/potato/home": http.MethodGet, 82 | "/potato/home/index.php": http.MethodGet, 83 | "/potato/blabla": http.MethodGet, 84 | 85 | "/test/test/test/": http.MethodGet, 86 | "/test/test/home": http.MethodGet, 87 | "/test/test/home/index.php": http.MethodGet, 88 | "/test/test/blabla": http.MethodGet, 89 | 90 | "/test/test/": http.MethodGet, 91 | 92 | "/home": http.MethodGet, 93 | "/blabla": http.MethodGet, 94 | "/home/index.php": http.MethodGet, 95 | } 96 | 97 | assert.Equal(t, expectedRequests, requestsMap) 98 | 99 | expectedResultTree := `/ 100 | ├── potato 101 | └── test 102 | └── test 103 | 104 | ` 105 | 106 | assert.Contains(t, loggerBuffer.String(), expectedResultTree) 107 | } 108 | 109 | func TestScanShouldWriteOutput(t *testing.T) { 110 | logger, _ := test.NewLogger() 111 | 112 | c := createCommand(logger) 113 | assert.NotNil(t, c) 114 | 115 | testServer, _ := test.NewServerWithAssertion( 116 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 117 | if r.URL.Path == "/home" { 118 | w.WriteHeader(http.StatusOK) 119 | 120 | return 121 | } 122 | 123 | w.WriteHeader(http.StatusNotFound) 124 | }), 125 | ) 126 | defer testServer.Close() 127 | 128 | outputFilename := test.RandStringRunes(10) 129 | outputFilename = "testdata/out/" + outputFilename + ".txt" 130 | 131 | defer func() { 132 | err := os.Remove(outputFilename) 133 | if err != nil { 134 | panic("failed to remove file create during test: " + err.Error()) 135 | } 136 | }() 137 | 138 | err := executeCommand( 139 | c, 140 | "scan", 141 | testServer.URL, 142 | "--dictionary", 143 | "testdata/dict2.txt", 144 | "--out", 145 | outputFilename, 146 | ) 147 | assert.NoError(t, err) 148 | 149 | //nolint:gosec 150 | file, err := os.Open(outputFilename) 151 | assert.NoError(t, err) 152 | 153 | b, err := ioutil.ReadAll(file) 154 | assert.NoError(t, err, "failed to read file content") 155 | 156 | assert.Contains(t, string(b), testServer.Listener.Addr().String()) 157 | 158 | assert.NoError(t, file.Close(), "failed to close file") 159 | } 160 | 161 | func TestScanInvalidOutputFileShouldErr(t *testing.T) { 162 | logger, _ := test.NewLogger() 163 | 164 | c := createCommand(logger) 165 | assert.NotNil(t, c) 166 | 167 | testServer, _ := test.NewServerWithAssertion( 168 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 169 | if r.URL.Path == "/home" { 170 | w.WriteHeader(http.StatusOK) 171 | 172 | return 173 | } 174 | 175 | w.WriteHeader(http.StatusNotFound) 176 | }), 177 | ) 178 | defer testServer.Close() 179 | 180 | err := executeCommand( 181 | c, 182 | "scan", 183 | testServer.URL, 184 | "--dictionary", 185 | "testdata/dict2.txt", 186 | "--out", 187 | "/root/blabla/123/gibberish/123", 188 | ) 189 | assert.Error(t, err) 190 | assert.Contains(t, err.Error(), "failed to create output saver") 191 | } 192 | 193 | func TestScanWithInvalidStatusesToIgnoreShouldErr(t *testing.T) { 194 | logger, _ := test.NewLogger() 195 | 196 | c := createCommand(logger) 197 | assert.NotNil(t, c) 198 | 199 | testServer, serverAssertion := test.NewServerWithAssertion( 200 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 201 | ) 202 | defer testServer.Close() 203 | 204 | err := executeCommand( 205 | c, 206 | "scan", 207 | testServer.URL, 208 | "--dictionary", 209 | "testdata/dict2.txt", 210 | "-v", 211 | "--http-statuses-to-ignore", 212 | "300,gibberish,404", 213 | "--http-timeout", 214 | "300", 215 | ) 216 | assert.Error(t, err) 217 | assert.Contains(t, err.Error(), "strconv.Atoi: parsing") 218 | assert.Contains(t, err.Error(), "gibberish") 219 | 220 | assert.Equal(t, 0, serverAssertion.Len()) 221 | } 222 | 223 | func TestScanWithNoTargetShouldErr(t *testing.T) { 224 | logger, _ := test.NewLogger() 225 | 226 | c := createCommand(logger) 227 | assert.NotNil(t, c) 228 | 229 | err := executeCommand(c, "scan", "--dictionary", "testdata/dict2.txt") 230 | assert.Error(t, err) 231 | assert.Contains(t, err.Error(), "no URL provided") 232 | } 233 | 234 | func TestScanWithInvalidTargetShouldErr(t *testing.T) { 235 | logger, _ := test.NewLogger() 236 | 237 | c := createCommand(logger) 238 | assert.NotNil(t, c) 239 | 240 | err := executeCommand(c, "scan", "--dictionary", "testdata/dict2.txt", "localhost%%2") 241 | assert.Error(t, err) 242 | assert.Contains(t, err.Error(), "invalid URI") 243 | } 244 | 245 | func TestScanCommandCanBeInterrupted(t *testing.T) { 246 | logger, loggerBuffer := test.NewLogger() 247 | 248 | c := createCommand(logger) 249 | assert.NotNil(t, c) 250 | 251 | testServer, _ := test.NewServerWithAssertion( 252 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 253 | time.Sleep(time.Millisecond * 650) 254 | 255 | if r.URL.Path == "/test/" { 256 | w.WriteHeader(http.StatusOK) 257 | 258 | return 259 | } 260 | 261 | w.WriteHeader(http.StatusNotFound) 262 | }), 263 | ) 264 | defer testServer.Close() 265 | 266 | go func() { 267 | time.Sleep(time.Millisecond * 200) 268 | 269 | _ = syscall.Kill(syscall.Getpid(), syscall.SIGINT) //nolint:errcheck 270 | }() 271 | 272 | err := executeCommand( 273 | c, 274 | "scan", 275 | testServer.URL, 276 | "--dictionary", 277 | "testdata/dict2.txt", 278 | "-v", 279 | "--http-timeout", 280 | "900", 281 | ) 282 | assert.NoError(t, err) 283 | 284 | assert.Contains(t, loggerBuffer.String(), "Received sigint") 285 | } 286 | 287 | func TestScanWithRemoteDictionary(t *testing.T) { 288 | logger, _ := test.NewLogger() 289 | 290 | c := createCommand(logger) 291 | assert.NotNil(t, c) 292 | 293 | dictionaryServer := httptest.NewServer( 294 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 295 | dict := `home 296 | home/index.php 297 | blabla 298 | ` 299 | w.WriteHeader(http.StatusOK) 300 | _, _ = w.Write([]byte(dict)) //nolint:errcheck 301 | }), 302 | ) 303 | defer dictionaryServer.Close() 304 | 305 | testServer, serverAssertion := test.NewServerWithAssertion( 306 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 307 | w.WriteHeader(http.StatusNotFound) 308 | }), 309 | ) 310 | defer testServer.Close() 311 | 312 | err := executeCommand( 313 | c, 314 | "scan", 315 | testServer.URL, 316 | "--dictionary", 317 | dictionaryServer.URL, 318 | "--http-timeout", 319 | "300", 320 | ) 321 | assert.NoError(t, err) 322 | 323 | assert.Equal(t, 3, serverAssertion.Len()) 324 | } 325 | 326 | func TestScanWithUserAgentFlag(t *testing.T) { 327 | const testUserAgent = "my_test_user_agent" 328 | 329 | logger, loggerBuffer := test.NewLogger() 330 | 331 | c := createCommand(logger) 332 | assert.NotNil(t, c) 333 | 334 | testServer, serverAssertion := test.NewServerWithAssertion( 335 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 336 | w.WriteHeader(http.StatusNotFound) 337 | }), 338 | ) 339 | defer testServer.Close() 340 | 341 | err := executeCommand( 342 | c, 343 | "scan", 344 | testServer.URL, 345 | "--user-agent", 346 | testUserAgent, 347 | "--dictionary", 348 | "testdata/dict.txt", 349 | "--http-timeout", 350 | "300", 351 | ) 352 | assert.NoError(t, err) 353 | 354 | assert.Equal(t, 3, serverAssertion.Len()) 355 | serverAssertion.Range(func(_ int, r http.Request) { 356 | assert.Equal(t, testUserAgent, r.Header.Get("User-Agent")) 357 | }) 358 | 359 | // to ensure we print the user agent to the cli 360 | assert.Contains(t, loggerBuffer.String(), testUserAgent) 361 | } 362 | 363 | func TestScanWithCookies(t *testing.T) { 364 | logger, loggerBuffer := test.NewLogger() 365 | 366 | c := createCommand(logger) 367 | assert.NotNil(t, c) 368 | 369 | testServer, serverAssertion := test.NewServerWithAssertion( 370 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 371 | ) 372 | defer testServer.Close() 373 | 374 | err := executeCommand( 375 | c, 376 | "scan", 377 | testServer.URL, 378 | "--cookie", 379 | "name1=val1", 380 | "--cookie", 381 | "name2=val2", 382 | "--dictionary", 383 | "testdata/dict.txt", 384 | "--http-timeout", 385 | "300", 386 | ) 387 | assert.NoError(t, err) 388 | 389 | serverAssertion.Range(func(_ int, r http.Request) { 390 | assert.Equal(t, 2, len(r.Cookies())) 391 | 392 | assert.Equal(t, r.Cookies()[0].Name, "name1") 393 | assert.Equal(t, r.Cookies()[0].Value, "val1") 394 | 395 | assert.Equal(t, r.Cookies()[1].Name, "name2") 396 | assert.Equal(t, r.Cookies()[1].Value, "val2") 397 | }) 398 | 399 | // to ensure we print the cookies to the cli 400 | assert.Contains(t, loggerBuffer.String(), "name1=val1") 401 | assert.Contains(t, loggerBuffer.String(), "name2=val2") 402 | } 403 | 404 | func TestWhenProvidingCookiesInWrongFormatShouldErr(t *testing.T) { 405 | const malformedCookie = "gibberish" 406 | 407 | logger, _ := test.NewLogger() 408 | 409 | c := createCommand(logger) 410 | assert.NotNil(t, c) 411 | 412 | testServer, serverAssertion := test.NewServerWithAssertion( 413 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 414 | w.WriteHeader(http.StatusNotFound) 415 | }), 416 | ) 417 | defer testServer.Close() 418 | 419 | err := executeCommand( 420 | c, 421 | "scan", 422 | testServer.URL, 423 | "--cookie", 424 | malformedCookie, 425 | "--dictionary", 426 | "testdata/dict.txt", 427 | ) 428 | assert.Error(t, err) 429 | assert.Contains(t, err.Error(), "cookie format is invalid") 430 | assert.Contains(t, err.Error(), malformedCookie) 431 | 432 | assert.Equal(t, 0, serverAssertion.Len()) 433 | } 434 | 435 | func TestScanWithCookieJar(t *testing.T) { 436 | const ( 437 | serverCookieName = "server_cookie_name" 438 | serverCookieValue = "server_cookie_value" 439 | ) 440 | 441 | logger, _ := test.NewLogger() 442 | 443 | c := createCommand(logger) 444 | assert.NotNil(t, c) 445 | 446 | once := sync.Once{} 447 | testServer, serverAssertion := test.NewServerWithAssertion( 448 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 449 | once.Do(func() { 450 | http.SetCookie( 451 | w, 452 | &http.Cookie{ 453 | Name: serverCookieName, 454 | Value: serverCookieValue, 455 | Expires: time.Now().AddDate(0, 1, 0), 456 | }, 457 | ) 458 | }) 459 | }), 460 | ) 461 | 462 | defer testServer.Close() 463 | 464 | err := executeCommand( 465 | c, 466 | "scan", 467 | testServer.URL, 468 | "--use-cookie-jar", 469 | "--dictionary", 470 | "testdata/dict.txt", 471 | "--http-timeout", 472 | "300", 473 | "-t", 474 | "1", 475 | ) 476 | assert.NoError(t, err) 477 | 478 | serverAssertion.Range(func(index int, r http.Request) { 479 | if index == 0 { // first request should have no cookies 480 | assert.Equal(t, 0, len(r.Cookies())) 481 | 482 | return 483 | } 484 | 485 | assert.Equal(t, 1, len(r.Cookies())) 486 | assert.Equal(t, r.Cookies()[0].Name, serverCookieName) 487 | assert.Equal(t, r.Cookies()[0].Value, serverCookieValue) 488 | }) 489 | } 490 | 491 | func TestScanWithUnknownFlagShouldErr(t *testing.T) { 492 | logger, _ := test.NewLogger() 493 | 494 | c := createCommand(logger) 495 | assert.NotNil(t, c) 496 | 497 | testServer, serverAssertion := test.NewServerWithAssertion( 498 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 499 | ) 500 | defer testServer.Close() 501 | 502 | err := executeCommand( 503 | c, 504 | "scan", 505 | testServer.URL, 506 | "--gibberishflag", 507 | "--dictionary", 508 | "testdata/dict.txt", 509 | ) 510 | assert.Error(t, err) 511 | assert.Contains(t, err.Error(), "unknown flag") 512 | 513 | assert.Equal(t, 0, serverAssertion.Len()) 514 | } 515 | 516 | func TestScanWithHeaders(t *testing.T) { 517 | logger, loggerBuffer := test.NewLogger() 518 | 519 | c := createCommand(logger) 520 | assert.NotNil(t, c) 521 | 522 | testServer, serverAssertion := test.NewServerWithAssertion( 523 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 524 | ) 525 | defer testServer.Close() 526 | 527 | err := executeCommand( 528 | c, 529 | "scan", 530 | testServer.URL, 531 | "--header", 532 | "Accept-Language: en-US,en;q=0.5", 533 | "--header", 534 | `"Authorization: Bearer 123"`, 535 | "--dictionary", 536 | "testdata/dict.txt", 537 | "--http-timeout", 538 | "300", 539 | ) 540 | assert.NoError(t, err) 541 | 542 | serverAssertion.Range(func(_ int, r http.Request) { 543 | assert.Equal(t, 2, len(r.Header)) 544 | 545 | assert.Equal(t, "en-US,en;q=0.5", r.Header.Get("Accept-Language")) 546 | assert.Equal(t, "Bearer 123", r.Header.Get("Authorization")) 547 | }) 548 | 549 | // to ensure we print the headers to the cli 550 | assert.Contains(t, loggerBuffer.String(), "Accept-Language") 551 | assert.Contains(t, loggerBuffer.String(), "Authorization") 552 | assert.Contains(t, loggerBuffer.String(), "Bearer 123") 553 | } 554 | 555 | func TestScanWithMalformedHeaderShouldErr(t *testing.T) { 556 | const malformedHeader = "gibberish" 557 | 558 | logger, _ := test.NewLogger() 559 | 560 | c := createCommand(logger) 561 | assert.NotNil(t, c) 562 | 563 | testServer, serverAssertion := test.NewServerWithAssertion( 564 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {}), 565 | ) 566 | defer testServer.Close() 567 | 568 | err := executeCommand( 569 | c, 570 | "scan", 571 | testServer.URL, 572 | "--header", 573 | "Accept-Language: en-US,en;q=0.5", 574 | "--header", 575 | malformedHeader, 576 | "--dictionary", 577 | "testdata/dict.txt", 578 | ) 579 | assert.Error(t, err) 580 | assert.Contains(t, err.Error(), malformedHeader) 581 | assert.Contains(t, err.Error(), "header is in invalid format") 582 | 583 | assert.Equal(t, 0, serverAssertion.Len()) 584 | } 585 | 586 | func TestStartScanWithSocks5ShouldFindResultsWhenAServerIsAvailable(t *testing.T) { 587 | logger, _ := test.NewLogger() 588 | 589 | c := createCommand(logger) 590 | assert.NotNil(t, c) 591 | 592 | testServer, serverAssertion := test.NewServerWithAssertion( 593 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 594 | w.WriteHeader(http.StatusNotFound) 595 | }), 596 | ) 597 | defer testServer.Close() 598 | 599 | socks5Server := startSocks5TestServer(t) 600 | defer socks5Server.Close() //nolint:errcheck 601 | 602 | err := executeCommand( 603 | c, 604 | "scan", 605 | testServer.URL, 606 | "--dictionary", 607 | "testdata/dict.txt", 608 | "-v", 609 | "--http-timeout", 610 | "300", 611 | "--socks5", 612 | socks5TestServerHost, 613 | ) 614 | assert.NoError(t, err) 615 | 616 | assert.Equal(t, 3, serverAssertion.Len()) 617 | } 618 | 619 | func TestShouldFailToScanWithAnUnreachableSocks5Server(t *testing.T) { 620 | logger, loggerBuffer := test.NewLogger() 621 | 622 | c := createCommand(logger) 623 | assert.NotNil(t, c) 624 | 625 | testServer, serverAssertion := test.NewServerWithAssertion( 626 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 627 | w.WriteHeader(http.StatusNotFound) 628 | }), 629 | ) 630 | defer testServer.Close() 631 | 632 | socks5Server := startSocks5TestServer(t) 633 | defer socks5Server.Close() //nolint:errcheck 634 | 635 | err := executeCommand( 636 | c, 637 | "scan", 638 | testServer.URL, 639 | "--dictionary", 640 | "testdata/dict.txt", 641 | "-v", 642 | "--http-timeout", 643 | "300", 644 | "--socks5", 645 | "127.0.0.1:9555", // invalid 646 | ) 647 | assert.NoError(t, err) 648 | 649 | assert.Equal(t, 0, serverAssertion.Len()) 650 | assert.Contains(t, loggerBuffer.String(), "failed to perform request") 651 | assert.Contains(t, loggerBuffer.String(), "socks connect tcp") 652 | assert.Contains(t, loggerBuffer.String(), "connect: connection refused") 653 | } 654 | 655 | func TestShouldFailToStartWithAnInvalidSocks5Address(t *testing.T) { 656 | logger, _ := test.NewLogger() 657 | 658 | c := createCommand(logger) 659 | assert.NotNil(t, c) 660 | 661 | testServer, serverAssertion := test.NewServerWithAssertion( 662 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 663 | w.WriteHeader(http.StatusNotFound) 664 | }), 665 | ) 666 | defer testServer.Close() 667 | 668 | err := executeCommand( 669 | c, 670 | "scan", 671 | testServer.URL, 672 | "--dictionary", 673 | "testdata/dict.txt", 674 | "-v", 675 | "--http-timeout", 676 | "300", 677 | "--socks5", 678 | "localhost%%2", // invalid 679 | ) 680 | assert.Error(t, err) 681 | assert.Contains(t, err.Error(), "invalid URL escape") 682 | 683 | assert.Equal(t, 0, serverAssertion.Len()) 684 | } 685 | 686 | func TestScanShouldFailToCommunicateWithServerHavingInvalidSSLCertificates(t *testing.T) { 687 | logger, loggerBuffer := test.NewLogger() 688 | 689 | c := createCommand(logger) 690 | assert.NotNil(t, c) 691 | 692 | testServer, serverAssertion := test.NewTSLServerWithAssertion( 693 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 694 | w.WriteHeader(http.StatusOK) 695 | }), 696 | ) 697 | defer testServer.Close() 698 | 699 | err := executeCommand( 700 | c, 701 | "scan", 702 | testServer.URL, 703 | "--dictionary", 704 | "testdata/dict2.txt", 705 | "--scan-depth", 706 | "1", 707 | ) 708 | assert.NoError(t, err) 709 | 710 | assert.Equal(t, 0, serverAssertion.Len()) 711 | 712 | assert.Contains(t, loggerBuffer.String(), "certificate") 713 | } 714 | 715 | func TestScanShouldBeAbleToSkipSSLCertificatesCheck(t *testing.T) { 716 | logger, loggerBuffer := test.NewLogger() 717 | 718 | c := createCommand(logger) 719 | assert.NotNil(t, c) 720 | 721 | testServer, serverAssertion := test.NewTSLServerWithAssertion( 722 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 723 | w.WriteHeader(http.StatusOK) 724 | }), 725 | ) 726 | defer testServer.Close() 727 | 728 | err := executeCommand( 729 | c, 730 | "scan", 731 | testServer.URL, 732 | "--dictionary", 733 | "testdata/dict2.txt", 734 | "--scan-depth", 735 | "1", 736 | "--no-check-certificate", 737 | ) 738 | assert.NoError(t, err) 739 | 740 | assert.Equal(t, 16, serverAssertion.Len()) 741 | 742 | assert.NotContains(t, loggerBuffer.String(), "certificate signed by unknown authority") 743 | } 744 | 745 | func startSocks5TestServer(t *testing.T) net.Listener { 746 | conf := &socks5.Config{} 747 | 748 | server, err := socks5.New(conf) 749 | if err != nil { 750 | t.Fatalf("failed to create socks5: %s", err.Error()) 751 | } 752 | 753 | listener, err := net.Listen("tcp", socks5TestServerHost) 754 | if err != nil { 755 | t.Fatalf("failed to create listener: %s", err.Error()) 756 | } 757 | 758 | go func() { 759 | // Create SOCKS5 proxy on localhost port 8000 760 | if err := server.Serve(listener); err != nil { 761 | t.Logf("socks5 stopped serving: %s", err.Error()) 762 | } 763 | }() 764 | 765 | return listener 766 | } 767 | 768 | func TestScanShouldFailIfDictionaryFetchExceedTimeout(t *testing.T) { 769 | logger, _ := test.NewLogger() 770 | 771 | c := createCommand(logger) 772 | assert.NotNil(t, c) 773 | 774 | testServer, serverAssertion := test.NewServerWithAssertion( 775 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 776 | }), 777 | ) 778 | defer testServer.Close() 779 | 780 | dictionaryTestServer, _ := test.NewServerWithAssertion( 781 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 782 | time.Sleep(time.Second) 783 | w.Write([]byte("/dictionary/entry")) //nolint 784 | }), 785 | ) 786 | defer dictionaryTestServer.Close() 787 | 788 | err := executeCommand( 789 | c, 790 | "scan", 791 | testServer.URL, 792 | "--dictionary", 793 | dictionaryTestServer.URL, 794 | "--dictionary-get-timeout", 795 | "5", 796 | ) 797 | assert.Error(t, err) 798 | 799 | assert.Contains(t, err.Error(), "dictionary: failed to get") 800 | 801 | assert.Equal(t, 0, serverAssertion.Len()) 802 | } 803 | 804 | func TestScanShouldBeAbleToFetchRemoteDictionary(t *testing.T) { 805 | logger, _ := test.NewLogger() 806 | 807 | c := createCommand(logger) 808 | assert.NotNil(t, c) 809 | 810 | testServer, serverAssertion := test.NewServerWithAssertion( 811 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 812 | w.WriteHeader(http.StatusNotFound) 813 | }), 814 | ) 815 | defer testServer.Close() 816 | 817 | dictionaryTestServer, dictionaryTestServerAssertion := test.NewServerWithAssertion( 818 | http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { 819 | w.Write([]byte("/dictionary/entry")) //nolint 820 | }), 821 | ) 822 | defer dictionaryTestServer.Close() 823 | 824 | err := executeCommand( 825 | c, 826 | "scan", 827 | testServer.URL, 828 | "--dictionary", 829 | dictionaryTestServer.URL, 830 | "--dictionary-get-timeout", 831 | "500", 832 | ) 833 | assert.NoError(t, err) 834 | 835 | assert.Equal(t, 1, dictionaryTestServerAssertion.Len()) 836 | assert.Equal(t, 1, serverAssertion.Len()) 837 | 838 | serverAssertion.At(0, func(r http.Request) { 839 | assert.Equal(t, "/dictionary/entry", r.URL.Path) 840 | }) 841 | } 842 | --------------------------------------------------------------------------------