├── .gitignore
├── README.md
└── main.go


/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | .idea
3 | 
4 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # wordlistgen
 2 | 
 3 | ## What and why?
 4 | wordlistgen is a tool to pass a list of URLs and get back a list of relevant words for your wordlists. Wordlists are much more
 5 | effective when you take the application's context into consideration. wordlistgen pulls out URL components, such as subdomain names,
 6 | paths, query strings, etc. and spits them back to stdout so you can easily add them to your wordlists
 7 | 
 8 | # Installation
 9 | If you don't have Go installed, "go" do that!
10 | 
11 | ```go get -u github.com/ameenmaali/wordlistgen```
12 | 
13 | ## Usage
14 | wordlistgen takes URLs and paths from stdin, of which you will most likely want in a file such as:
15 | ```
16 | $ cat file.txt
17 | https://google.com/home/?q=2&d=asd
18 | http://my.site
19 | /api/v2/auth/me?id=123
20 | ```
21 | 
22 | Get unique URL components from a file of URLs and/or paths:
23 | 
24 | `cat hosts.txt | wordlistgen`
25 | 
26 | Get unique URL components from a file of URLs and/or paths, including query string values, and save to a file:
27 | 
28 | `cat hosts.txt | wordlistgen -qv > urlComponents.txt`
29 | 
30 | wordlistgen works at it's best when chained with other tools, such as [@tonnomnom's](https://github.com/tomnomnom) [waybackurls](https://github.com/tomnomnom/waybackurls) :
31 | 
32 | `cat hosts.txt | waybackurls | wordlistgen`
33 | 
34 | ### Help
35 | ```
36 | $ wordlistgen -h
37 | Usage of wordlistgen:
38 |   -fq
39 |     	If enabled, filter out query strings (e.g. if enabled /?q=123 - q would NOT be included in results
40 |   -qv
41 |     	If enabled, include query string values (e.g. if enabled /?q=123 - 123 would be included in results
42 | ```
43 | 


--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
  1 | package main
  2 | 
  3 | import (
  4 | 	"bufio"
  5 | 	"flag"
  6 | 	"fmt"
  7 | 	"net/url"
  8 | 	"os"
  9 | 	"strings"
 10 | )
 11 | 
 12 | type cliOptions struct {
 13 | 	IncludeQsValues bool
 14 | 	FilterQs        bool
 15 | }
 16 | 
 17 | var options cliOptions
 18 | 
 19 | func main() {
 20 | 	flag.BoolVar(&options.IncludeQsValues, "qv", false, "If enabled, include query string values (e.g. if enabled /?q=123 - 123 would be included in results")
 21 | 	flag.BoolVar(&options.FilterQs, "fq", false, "If enabled, filter out query strings (e.g. if enabled /?q=123 - q would NOT be included in results")
 22 | 	flag.Parse()
 23 | 
 24 | 	allComponents := make(map[string]bool)
 25 | 
 26 | 	sc := bufio.NewScanner(os.Stdin)
 27 | 	for sc.Scan() {
 28 | 		host := strings.ToLower(sc.Text())
 29 | 		components := getUrlComponents(host)
 30 | 		for _, component := range components {
 31 | 			if allComponents[component] {
 32 | 				continue
 33 | 			}
 34 | 			allComponents[component] = true
 35 | 			fmt.Println(component)
 36 | 		}
 37 | 	}
 38 | 
 39 | 	if err := sc.Err(); err != nil {
 40 | 		fmt.Fprintf(os.Stderr, "Cannot read input: %s\n", err)
 41 | 	}
 42 | }
 43 | 
 44 | func getUrlComponents(host string) []string {
 45 | 	var components []string
 46 | 	if !strings.HasPrefix(host, "/") && !strings.HasPrefix(host, "http") {
 47 | 		host = "http://" + host
 48 | 	}
 49 | 
 50 | 	u, err := url.Parse(host)
 51 | 
 52 | 	// If URL can't be parsed, ignore and move on
 53 | 	if err != nil {
 54 | 		return components
 55 | 	}
 56 | 
 57 | 	path := u.Path
 58 | 	pathFragments := strings.Split(path, "/")
 59 | 
 60 | 	// Remove first item from the slice as it will be blank
 61 | 	if len(pathFragments) > 0 {
 62 | 		pathFragments = pathFragments[1:]
 63 | 	}
 64 | 
 65 | 	// If query strings can't be parsed, set query strings as empty
 66 | 	queryStrings, err := url.ParseQuery(u.RawQuery)
 67 | 	if err != nil {
 68 | 		queryStrings = nil
 69 | 	}
 70 | 
 71 | 	domain := u.Host
 72 | 	domainFragments := strings.Split(domain, ".")
 73 | 
 74 | 	// Remove last item from the slice as it will be the extension (.com, .net, .etc)
 75 | 	if len(domainFragments) > 0 {
 76 | 		domainFragments = domainFragments[:len(domainFragments)-1]
 77 | 	}
 78 | 
 79 | 	for _, fragment := range domainFragments {
 80 | 		components = append(components, fragment)
 81 | 	}
 82 | 
 83 | 	for _, pathFragment := range pathFragments {
 84 | 		components = append(components, pathFragment)
 85 | 	}
 86 | 
 87 | 	if queryStrings != nil {
 88 | 		for qs, values := range queryStrings {
 89 | 			if !options.FilterQs {
 90 | 				components = append(components, qs)
 91 | 			}
 92 | 			if options.IncludeQsValues {
 93 | 				for _, val := range values {
 94 | 					components = append(components, val)
 95 | 				}
 96 | 			}
 97 | 		}
 98 | 	}
 99 | 
100 | 	return components
101 | }
102 | 


--------------------------------------------------------------------------------