├── README.md
└── sprawl.py


/README.md:
--------------------------------------------------------------------------------
 1 | # sprawl
 2 | Expand urls into one url for each path depth
 3 | 
 4 | ```
 5 | $ echo /this/is/a/test | ./sprawl.py -s
 6 | /this
 7 | /is/a/test
 8 | /this/is/a
 9 | /a/test
10 | /this/is
11 | /a
12 | /this/is/a/test
13 | /
14 | /is
15 | /test
16 | 
17 | ```
18 | [![asciicast](https://asciinema.org/a/6h2IhU6XwbPBE62xNEh4z3Y8S.png)](https://asciinema.org/a/6h2IhU6XwbPBE62xNEh4z3Y8S)
19 | 


--------------------------------------------------------------------------------
/sprawl.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python3
 2 | 
 3 | import argparse, sys
 4 | 
 5 | parser = argparse.ArgumentParser(description = "Expand a list of paths into a much larger wordlist by creating an entry for each path depth.")
 6 | 
 7 | parser.add_argument('-s', '--slash', default=False, action="store_true", help="prefix each line with a /, this is left off by default.")
 8 | args = parser.parse_args()
 9 | 
10 | prefix = "/" if args.slash else ""
11 | 
12 | def read_in():
13 | 	# Read contents of stdin
14 | 	return [x.strip() for x in sys.stdin.readlines()]
15 | 
16 | def strip_extra(line):
17 | 	# Remove protocol handler, domain, querystring
18 | 	line = line[1:] if line[0] == "/" else line
19 | 	line = "/".join(line.split("/")[3:]) if "//" in line else line
20 | 	return line.split("?")[0] if "?" in line else line
21 | 
22 | def tokenize(line):
23 | 	return line.split("/")
24 | 
25 | def expand(input_line):
26 | 	# tokenize the line
27 | 	tokens = tokenize(input_line)
28 | 
29 | 	expansion = []
30 | 	for t in tokens:
31 | 		# add each token by itself as a line
32 | 		expansion.append(t)
33 | 	for index in range(len(tokens)):
34 | 		# truncate each path depth and add each result
35 | 		line = "/".join(tokens[0:index+1])
36 | 		expansion.append(line)
37 | 	for index in range(len(tokens)):
38 | 		line = "/".join(tokens[index+1:])
39 | 		expansion.append(line)
40 | 	return expansion
41 | 
42 | 
43 | input_data = read_in()
44 | 
45 | output_set = set()
46 | 
47 | for line in input_data:
48 | 
49 | 	line = strip_extra(line)
50 | 	expansion = expand(line)
51 | 	for entry in expansion:
52 | 		output_set.add(f"{prefix}{entry}")
53 | 
54 | for entry in output_set:
55 | 	print(entry)
56 | 
57 | 


--------------------------------------------------------------------------------