├── .gitignore ├── go.mod ├── banner.png ├── mkpath.png ├── .github └── ISSUE_TEMPLATE │ ├── bug.md │ ├── epic.md │ └── story.md ├── Dockerfile ├── round └── round.go ├── LICENSE ├── README.md └── mkpath.go /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | domains.txt 3 | wordlist.txt 4 | out.txt -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/trickest/mkpath 2 | 3 | go 1.18 4 | -------------------------------------------------------------------------------- /banner.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trickest/mkpath/HEAD/banner.png -------------------------------------------------------------------------------- /mkpath.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trickest/mkpath/HEAD/mkpath.png -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug 3 | about: Issue for bugs 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | #### Problem description 11 | 12 | #### Steps to reproduce 13 | 14 | #### Expected behaviour 15 | 16 | #### Screenshots if applicable 17 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/epic.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Epic 3 | about: Issue for epics 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | #### Description 11 | 12 | #### Epic specification link 13 | 14 | #### Issues in this epic: 15 | 16 | - [ ] Issue 1 17 | - [ ] Issue 2 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/story.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Story 3 | about: Issue for stories 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | #### Description 11 | 12 | #### Epic specification link 13 | 14 | #### Story specification link 15 | 16 | #### Issues in this story: 17 | 18 | - [ ] Issue 1 19 | - [ ] Issue 2 20 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.17.2-alpine AS build-env 2 | 3 | RUN apk add git 4 | ADD . /go/src/mkpath 5 | WORKDIR /go/src/mkpath 6 | RUN git checkout 1fe6937 && go build -o mkpath 7 | 8 | FROM alpine:3.14 9 | LABEL licenses.mkpath.name="MIT" \ 10 | licenses.mkpath.url="https://github.com/trickest/mkpath/blob/1fe6937da4346340b514759e83a40ba231bba5e2/LICENSE" \ 11 | licenses.golang.name="bsd-3-clause" \ 12 | licenses.golang.url="https://go.dev/LICENSE?m=text" 13 | 14 | COPY --from=build-env /go/src/mkpath/mkpath /bin/mkpath 15 | 16 | RUN mkdir -p /hive/in /hive/out 17 | 18 | WORKDIR /app 19 | RUN apk add bash 20 | 21 | ENTRYPOINT [ "mkpath" ] 22 | -------------------------------------------------------------------------------- /round/round.go: -------------------------------------------------------------------------------- 1 | package roundChan 2 | 3 | import ( 4 | "sync/atomic" 5 | ) 6 | 7 | // RoundRobin is an interface for representing round-robin balancing. 8 | type RoundRobin interface { 9 | Next() *chan string 10 | Add(*chan string) 11 | } 12 | 13 | type roundRobin struct { 14 | chs []*chan string 15 | next uint32 16 | } 17 | 18 | // New returns RoundRobin implementation(*roundRobin). 19 | func New(chs ...*chan string) RoundRobin { 20 | return &roundRobin{ 21 | chs: chs, 22 | } 23 | } 24 | 25 | // Next returns next channel 26 | func (r *roundRobin) Next() *chan string { 27 | n := atomic.AddUint32(&r.next, 1) 28 | return r.chs[(int(n)-1)%len(r.chs)] 29 | } 30 | 31 | // Add adds a channel 32 | func (r *roundRobin) Add(ch *chan string) { 33 | r.chs = append(r.chs, ch) 34 | } 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2022 Trickest 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

mkpath Tweet

2 |

Make URL path combinations using a wordlist

3 | 4 | ![mkpath](mkpath.png "mkpath") 5 | 6 | Read a wordlist file and generate path combinations for given domain or list of domains. Input from wordlist file is lowercased and unique words are processed. Additionally, wordlist can be filtered using regex. 7 | 8 | When you use mkpath's `-l` parameter, it will generate all path combinations up to the specified level, including all lower levels, using words from the wordlist. For instance, with `-l 2`, it will generate `len(permutation_list)^2 + len(permutation_list)` results, which is: 9 | - 30 combinations for a 5-word wordlist. 10 | - 10100 combinations for a 100-word wordlist. 11 | - 250500 combinations for a 500-word wordlist. 12 | 13 | 14 | # Installation 15 | ## Binary 16 | Binaries are available in the [latest release](https://github.com/trickest/mkpath/releases/latest). 17 | 18 | ## Docker 19 | ``` 20 | docker run quay.io/trickest/mkpath 21 | ``` 22 | 23 | ## From source 24 | ``` 25 | go install github.com/trickest/mkpath@latest 26 | ``` 27 | 28 | # Usage 29 | ``` 30 | -d string 31 | Input domain 32 | -df string 33 | Input domain file, one domain per line 34 | -l int 35 | URL path depth to generate (default 1) (default 1) 36 | -lower 37 | Convert wordlist file content to lowercase (default false) 38 | -o string 39 | Output file (optional) 40 | -only-dirs 41 | Generate directories only, files are filtered out (default false) 42 | -only-files 43 | Generate files only, file names are appended to given domains (default false) 44 | -r string 45 | Regex to filter words from wordlist file 46 | -t int 47 | Number of threads for every path depth (default 100) 48 | -w string 49 | Wordlist file 50 | ``` 51 | 52 | ### Example 53 | ##### wordlist.txt 54 | ``` 55 | dev 56 | prod/ 57 | admin.py 58 | app/login.html 59 | ``` 60 | 61 | ```shell script 62 | $ mkpath -d example.com -l 2 -w wordlist.txt 63 | example.com/dev 64 | example.com/prod 65 | example.com/dev/dev 66 | example.com/prod/dev 67 | example.com/dev/prod 68 | example.com/prod/prod 69 | example.com/dev/admin.py 70 | example.com/dev/app/login.html 71 | example.com/prod/admin.py 72 | example.com/prod/app/login.html 73 | example.com/dev/dev/admin.py 74 | example.com/dev/dev/app/login.html 75 | example.com/prod/dev/admin.py 76 | example.com/prod/dev/app/login.html 77 | example.com/dev/prod/admin.py 78 | example.com/dev/prod/app/login.html 79 | example.com/prod/prod/admin.py 80 | example.com/prod/prod/app/login.html 81 | 82 | ``` 83 | 84 | # Report Bugs / Feedback 85 | We look forward to any feedback you want to share with us or if you're stuck with a problem you can contact us at [support@trickest.com](mailto:support@trickest.com). You can also create an [Issue](https://github.com/trickest/mkpath/issues/new) or pull request on the Github repository. 86 | 87 | # Where does this fit in your methodology? 88 | Mkpath is an integral part of many workflows in the Trickest store. Sign up on [trickest.com](https://trickest.com) to get access to these workflows or build your own from scratch! 89 | 90 | [](https://trickest.io/auth/register) 91 | -------------------------------------------------------------------------------- /mkpath.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "flag" 7 | "fmt" 8 | "os" 9 | "os/signal" 10 | "regexp" 11 | "strconv" 12 | "strings" 13 | "sync" 14 | "syscall" 15 | 16 | roundChan "github.com/trickest/mkpath/round" 17 | ) 18 | 19 | const ( 20 | fileRegex = "[^?*:;{}]+\\.[^/?*:;{}]+" 21 | 22 | bufferSizeMB = 100 23 | maxWorkingThreads = 100000 24 | numberOfFiles = 1 25 | ) 26 | 27 | var ( 28 | domain string 29 | inputDomains []string 30 | domainFile string 31 | 32 | wordlist string 33 | toLowercase bool 34 | regex string 35 | dirWordSet map[string]bool 36 | fileWordSet map[string]bool 37 | 38 | depth int 39 | onlyDirs bool 40 | onlyFiles bool 41 | outputFileName string 42 | silent bool 43 | 44 | workers int 45 | workerThreadMax = make(chan struct{}, maxWorkingThreads) 46 | done = make(chan struct{}) 47 | wg sync.WaitGroup 48 | wgWrite sync.WaitGroup 49 | robin roundChan.RoundRobin 50 | ) 51 | 52 | func readDomainFile() { 53 | inputFile, err := os.Open(domainFile) 54 | if err != nil { 55 | fmt.Println("Could not open file to read domains:", err) 56 | os.Exit(1) 57 | } 58 | defer inputFile.Close() 59 | 60 | scanner := bufio.NewScanner(inputFile) 61 | for scanner.Scan() { 62 | inputDomains = append(inputDomains, strings.TrimSpace(scanner.Text())) 63 | } 64 | } 65 | 66 | func prepareDomains() { 67 | if domain == "" && domainFile == "" { 68 | fmt.Println("No domain input provided!") 69 | os.Exit(1) 70 | } 71 | 72 | inputDomains = make([]string, 0) 73 | if domain != "" { 74 | inputDomains = append(inputDomains, domain) 75 | } else { 76 | if domainFile != "" { 77 | readDomainFile() 78 | } 79 | } 80 | } 81 | 82 | func readWordlistFile() { 83 | var reg *regexp.Regexp 84 | var err error 85 | if regex != "" { 86 | reg, err = regexp.Compile(regex) 87 | if err != nil { 88 | fmt.Println(err) 89 | os.Exit(1) 90 | } 91 | } 92 | 93 | wordlistFile, err := os.Open(wordlist) 94 | if err != nil { 95 | fmt.Println("Could not open file to read wordlist:", err) 96 | os.Exit(1) 97 | } 98 | defer wordlistFile.Close() 99 | 100 | fileReg, err := regexp.Compile(fileRegex) 101 | if err != nil { 102 | fmt.Println(err) 103 | os.Exit(1) 104 | } 105 | 106 | dirWordSet = make(map[string]bool) 107 | fileWordSet = make(map[string]bool) 108 | scanner := bufio.NewScanner(wordlistFile) 109 | 110 | for scanner.Scan() { 111 | word := scanner.Text() 112 | if toLowercase { 113 | word = strings.ToLower(word) 114 | } 115 | word = strings.Trim(word, "/") 116 | if word != "" { 117 | if reg != nil { 118 | if !reg.Match([]byte(word)) { 119 | continue 120 | } 121 | } 122 | if fileReg.Match([]byte(word)) { 123 | fileWordSet[word] = true 124 | } else { 125 | dirWordSet[word] = true 126 | } 127 | } 128 | } 129 | } 130 | 131 | func closeWriters(number int) { 132 | for i := 0; i < number; i++ { 133 | done <- struct{}{} 134 | } 135 | } 136 | 137 | func spawnWriters(number int) { 138 | for i := 0; i < number; i++ { 139 | var bf bytes.Buffer 140 | ch := make(chan string, 100000) 141 | 142 | fileName := outputFileName 143 | fileSplit := strings.Split(fileName, ".") 144 | if len(fileSplit) == 1 { 145 | fileName += ".txt" 146 | } 147 | if number > 1 { 148 | fileSplit = strings.Split(fileName, ".") 149 | extension := "." + fileSplit[len(fileSplit)-1] 150 | fileName = strings.TrimSuffix(fileName, extension) + "-" + strconv.Itoa(i) + extension 151 | } 152 | file, err := os.Create(fileName) 153 | if err != nil { 154 | fmt.Println("Couldn't open file to write output:", err) 155 | os.Exit(1) 156 | } 157 | 158 | wgWrite.Add(1) 159 | go write(file, &bf, &ch) 160 | 161 | if robin == nil { 162 | robin = roundChan.New(&ch) 163 | continue 164 | } 165 | robin.Add(&ch) 166 | } 167 | } 168 | 169 | func write(file *os.File, buffer *bytes.Buffer, ch *chan string) { 170 | mainLoop: 171 | for { 172 | select { 173 | case <-done: 174 | for { 175 | if !writeOut(file, buffer, ch) { 176 | break 177 | } 178 | } 179 | if buffer.Len() > 0 { 180 | if file != nil { 181 | _, _ = file.WriteString(buffer.String()) 182 | buffer.Reset() 183 | } 184 | } 185 | break mainLoop 186 | default: 187 | writeOut(file, buffer, ch) 188 | } 189 | } 190 | wgWrite.Done() 191 | } 192 | 193 | func writeOut(file *os.File, buffer *bytes.Buffer, outputChannel *chan string) bool { 194 | select { 195 | case s := <-*outputChannel: 196 | buffer.WriteString(s) 197 | if buffer.Len() >= bufferSizeMB*1024*1024 { 198 | _, _ = file.WriteString(buffer.String()) 199 | buffer.Reset() 200 | } 201 | return true 202 | default: 203 | return false 204 | } 205 | } 206 | 207 | func combo(_comb string, level int, wg *sync.WaitGroup, wt *chan struct{}) { 208 | defer wg.Done() 209 | workerThreadMax <- struct{}{} 210 | 211 | if strings.Count(_comb, "/") > 0 { 212 | processOutput(_comb, robin.Next()) 213 | } 214 | 215 | var nextLevelWaitGroup sync.WaitGroup 216 | if level > 1 { 217 | nextLevelWt := make(chan struct{}, workers) 218 | for dw := range dirWordSet { 219 | nextLevelWaitGroup.Add(1) 220 | nextLevelWt <- struct{}{} 221 | go combo(_comb+"/"+dw, level-1, &nextLevelWaitGroup, &nextLevelWt) 222 | } 223 | } else { 224 | for dw := range dirWordSet { 225 | processOutput(_comb+"/"+dw, robin.Next()) 226 | } 227 | } 228 | 229 | nextLevelWaitGroup.Wait() 230 | <-workerThreadMax 231 | <-*wt 232 | } 233 | 234 | func processOutput(out string, outChan *chan string) { 235 | if onlyDirs || onlyFiles == onlyDirs { 236 | if !silent { 237 | fmt.Print(out + "\n") 238 | } 239 | *outChan <- out + "\n" 240 | } 241 | 242 | if onlyFiles || onlyFiles == onlyDirs { 243 | for file := range fileWordSet { 244 | if !silent { 245 | fmt.Print(out + "/" + file + "\n") 246 | } 247 | *outChan <- out + "/" + file + "\n" 248 | } 249 | } 250 | } 251 | 252 | func main() { 253 | flag.StringVar(&domain, "d", "", "Input domain") 254 | flag.StringVar(&domainFile, "df", "", "Input domain file, one domain per line") 255 | flag.StringVar(&wordlist, "w", "", "Wordlist file") 256 | flag.BoolVar(&toLowercase, "lower", false, "Convert wordlist file content to lowercase (default false)") 257 | flag.StringVar(®ex, "r", "", "Regex to filter words from wordlist file") 258 | flag.IntVar(&depth, "l", 1, "URL path depth to generate") 259 | flag.StringVar(&outputFileName, "o", "", "Output file (optional)") 260 | flag.BoolVar(&onlyDirs, "only-dirs", false, "Generate directories only, files are filtered out (default false)") 261 | flag.BoolVar(&onlyFiles, "only-files", false, "Generate files only, file names are appended to given domains (default false)") 262 | flag.IntVar(&workers, "t", 100, "Number of threads for every path depth") 263 | flag.BoolVar(&silent, "silent", true, "Skip writing generated paths to stdout (faster)") 264 | flag.Parse() 265 | 266 | go func() { 267 | signalChannel := make(chan os.Signal, 1) 268 | signal.Notify(signalChannel, os.Interrupt, syscall.SIGTERM, syscall.SIGKILL) 269 | <-signalChannel 270 | 271 | fmt.Println("Program interrupted, exiting...") 272 | os.Exit(0) 273 | }() 274 | 275 | if depth <= 0 || workers <= 0 { 276 | fmt.Println("Path depth and number of threads must be positive integers!") 277 | os.Exit(0) 278 | } 279 | 280 | prepareDomains() 281 | readWordlistFile() 282 | spawnWriters(numberOfFiles) 283 | 284 | if outputFileName == "" { 285 | silent = false 286 | } 287 | 288 | for _, d := range inputDomains { 289 | wg.Add(1) 290 | wt := make(chan struct{}, 1) 291 | wt <- struct{}{} 292 | go combo(d, depth, &wg, &wt) 293 | } 294 | 295 | wg.Wait() 296 | closeWriters(numberOfFiles) 297 | wgWrite.Wait() 298 | } 299 | --------------------------------------------------------------------------------