├── .gitignore
├── README.md
├── docs
    ├── LICENSE
    └── requirements.txt
└── httpgrep.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | .bash_history
 2 | build.sh
 3 | clean.sh
 4 | *.db
 5 | .DS_Store
 6 | .git
 7 | id_dsa
 8 | id_rsa
 9 | *.key
10 | *.log
11 | *.o
12 | passwd
13 | *.pyc
14 | __pycache__
15 | shadow
16 | *.swn
17 | *.swo
18 | *.swp
19 | tags
20 | .vim.session
21 | .zhistory
22 | .zsh_history
23 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Description
 2 | 
 3 | A python tool which scans for HTTP servers and finds given strings in HTTP body
 4 | and HTTP response headers.
 5 | 
 6 | # Usage
 7 | 
 8 | ```
 9 | $ httpgrep -H
10 |     __    __  __
11 |    / /_  / /_/ /_____  ____ _________  ____
12 |   / __ \/ __/ __/ __ \/ __ `/ ___/ _ \/ __ \
13 |  / / / / /_/ /_/ /_/ / /_/ / /  /  __/ /_/ /
14 | /_/ /_/\__/\__/ .___/\__, /_/   \___/ .___/
15 |              /_/    /____/         /_/
16 | 
17 |      --== [ by nullsecurity.net ] ==--
18 | 
19 | usage
20 | 
21 |   httpgrep -h <args> -s <arg> [opts] | <misc>
22 | 
23 | opts
24 | 
25 |   -h <hosts|file>   - single host/url or host-/cidr-range or file containing
26 |                       hosts or file containing URLs, e.g.: foobar.net,
27 |                       192.168.0.1-192.168.0.254, 192.168.0.0/24, /tmp/hosts.txt
28 |                       NOTE: hosts can also contain ':<port>' on cmdline or in
29 |                       file.
30 |   -p <port>         - port to connect to (default: 80 if hosts were given)
31 |   -t                - use TLS/SSL to connect to service
32 |   -u <URI>          - URI to search given strings in, e.g.: /foobar/, /foo.html
33 |                       (default: /)
34 |   -s <str|file>     - a single string/regex or multile strings/regex in a file
35 |                       to find in given URIs and HTTP response headers,
36 |                       e.g.: 'tomcat 8', '/tmp/igot0daysforthese.txt'
37 |   -S <where>        - search strings in given places (default: headers,body)
38 |   -X <method>       - specify HTTP request method to use (default: get).
39 |                       use '?' to list available methods.
40 |   -a <user:pass>    - http auth credentials (format: 'user:pass')
41 |   -U <UA>           - set custom User-Agent (default: firefox, rv84, windows)
42 |   -b <bytes>        - num bytes to read from response. offset == response[0].
43 |                       (default: 64)
44 |   -x <threads>      - num threads for concurrent scans and checks (default: 80)
45 |   -c <seconds>      - num seconds for socket timeout (default: 3.0)
46 |   -i                - use case-insensitive search
47 |   -r                - perform reverse dns lookup for given IPv4 addresses
48 |                       NOTE: this will slow down the scanz
49 |   -l <file>         - log found matches to file
50 |   -v                - verbose mode (default: quiet)
51 | 
52 | misc
53 | 
54 |   -H                - print help
55 |   -V                - print version information
56 | ```
57 | 
58 | # Author
59 | 
60 | noptrix
61 | 
62 | # Notes
63 | 
64 | - quick'n'dirty code
65 | - httpgrep is already packaged and available for [BlackArch Linux](https://www.blackarch.org/)
66 | - My master-branches are always stable; dev-branches are created for current work.
67 | - All of my public stuff you find are officially announced and published via [nullsecurity.net](https://www.nullsecurity.net).
68 | 
69 | # License
70 | 
71 | Check docs/LICENSE.
72 | 
73 | # Disclaimer
74 | 
75 | We hereby emphasize, that the hacking related stuff found on
76 | [nullsecurity.net](http://nullsecurity.net) are only for education purposes.
77 | We are not responsible for any damages. You are responsible for your own
78 | actions.
79 | 


--------------------------------------------------------------------------------
/docs/LICENSE:
--------------------------------------------------------------------------------
 1 | The MIT License (MIT)
 2 | 
 3 | Copyright (c) 2012-2020 noptrix
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 
23 | 


--------------------------------------------------------------------------------
/docs/requirements.txt:
--------------------------------------------------------------------------------
1 | requests
2 | 


--------------------------------------------------------------------------------
/httpgrep.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | # -*- coding: utf-8 -*- ########################################################
  3 | #               ____                     _ __                                  #
  4 | #    ___  __ __/ / /__ ___ ______ ______(_) /___ __                            #
  5 | #   / _ \/ // / / (_-</ -_) __/ // / __/ / __/ // /                            #
  6 | #  /_//_/\_,_/_/_/___/\__/\__/\_,_/_/ /_/\__/\_, /                             #
  7 | #                                           /___/ team                         #
  8 | #                                                                              #
  9 | # httpgrep                                                                     #
 10 | # Scans HTTP servers to find given strings in HTTP body and HTTP response      #
 11 | # headers.                                                                     #
 12 | #                                                                              #
 13 | # NOTES                                                                        #
 14 | # quick'n'dirty code                                                           #
 15 | #                                                                              #
 16 | # AUTHOR                                                                       #
 17 | # noptrix                                                                      #
 18 | #                                                                              #
 19 | ################################################################################
 20 | 
 21 | 
 22 | import re
 23 | import sys
 24 | import os
 25 | import socket
 26 | import ipaddress
 27 | import requests
 28 | import warnings
 29 | import getopt
 30 | from concurrent.futures import ThreadPoolExecutor, as_completed
 31 | 
 32 | 
 33 | __author__ = 'noptrix'
 34 | __version__ = '2.5'
 35 | __copyright__ = 'santa clause'
 36 | __license__ = 'MIT'
 37 | 
 38 | 
 39 | SUCCESS = 0
 40 | FAILURE = 1
 41 | 
 42 | NORM = '\033[0m'
 43 | BOLD = '\033[1;37;10m'
 44 | RED = '\033[1;31;10m'
 45 | GREEN = '\033[1;32;10m'
 46 | YELLOW = '\033[1;33;10m'
 47 | BLUE = '\033[1;34;10m'
 48 | MAGENTA = '\033[1;35;10m'
 49 | 
 50 | BANNER = BLUE + r'''
 51 |     __    __  __
 52 |    / /_  / /_/ /_____  ____ _________  ____
 53 |   / __ \/ __/ __/ __ \/ __ `/ ___/ _ \/ __ \
 54 |  / / / / /_/ /_/ /_/ / /_/ / /  /  __/ /_/ /
 55 | /_/ /_/\__/\__/ .___/\__, /_/   \___/ .___/
 56 |              /_/    /____/         /_/
 57 | ''' + NORM + '''
 58 |      --== [ by nullsecurity.net ] ==--'''
 59 | 
 60 | HELP = BOLD + '''usage''' + NORM + '''
 61 | 
 62 |   httpgrep -h <args> -s <arg> [opts] | <misc>
 63 | 
 64 | ''' + BOLD + '''opts''' + NORM + '''
 65 | 
 66 |   -h <hosts|file>   - single host/url or host-/cidr-range or file containing
 67 |                       hosts or file containing URLs, e.g.: foobar.net,
 68 |                       192.168.0.1-192.168.0.254, 192.168.0.0/24, /tmp/hosts.txt
 69 |                       NOTE: hosts can also contain ':<port>' on cmdline or in
 70 |                       file.
 71 |   -p <port>         - port to connect to (default: 80 if hosts were given)
 72 |   -t                - use TLS/SSL to connect to service
 73 |   -u <URI>          - URI to search given strings in, e.g.: /foobar/, /foo.html
 74 |                       (default: /)
 75 |   -s <str|file>     - a single string/regex or multile strings/regex in a file
 76 |                       to find in given URIs and HTTP response headers,
 77 |                       e.g.: 'tomcat 8', '/tmp/igot0daysforthese.txt'
 78 |   -S <where>        - search strings in given places (default: headers,body)
 79 |   -X <method>       - specify HTTP request method to use (default: get).
 80 |                       use '?' to list available methods.
 81 |   -a <user:pass>    - http auth credentials (format: 'user:pass')
 82 |   -U <UA>           - set custom User-Agent (default: firefox, rv84, windows)
 83 |   -b <bytes>        - num bytes to read from response. offset == response[0].
 84 |                       (default: 64)
 85 |   -x <threads>      - num threads for concurrent scans and checks (default: 80)
 86 |   -c <seconds>      - num seconds for socket timeout (default: 3.0)
 87 |   -i                - use case-insensitive search
 88 |   -r                - perform reverse dns lookup for given IPv4 addresses
 89 |                       NOTE: this will slow down the scanz
 90 |   -l <file>         - log found matches to file
 91 |   -v                - verbose mode (default: quiet)
 92 | 
 93 | ''' + BOLD + '''misc''' + NORM + '''
 94 | 
 95 |   -H                - print help
 96 |   -V                - print version information
 97 | '''
 98 | 
 99 | 
100 | opts = {
101 |   'hosts': None,
102 |   'port': 80,
103 |   'ssl': False,
104 |   'uri': '/',
105 |   'searchstr': '',
106 |   'where': ('headers', 'body'),
107 |   'method': 'get',
108 |   'auth': False,
109 |   'ua': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:87.0) Gecko/20100101 Firefox/87.0',
110 |   'bytes': 64,
111 |   'threads': 80,
112 |   'timeout': 3.0,
113 |   'case_in': False,
114 |   'rptr': False,
115 |   'logfile': False,
116 |   'verbose': False,
117 | }
118 | 
119 | 
120 | def log(msg='', _type='normal', esc='\n'):
121 |   iprefix = f'{BOLD}{BLUE}[+]{NORM}'
122 |   gprefix = f'{BOLD}{GREEN}[*]{NORM}'
123 |   wprefix = f'{BOLD}{YELLOW}[!]{NORM}'
124 |   eprefix = f'{BOLD}{RED}[-]{NORM}'
125 |   vprefix = f'{BOLD}    >{NORM}'
126 | 
127 |   if _type == 'normal':
128 |     sys.stdout.write(f'{msg}')
129 |   elif _type == 'verbose':
130 |     sys.stdout.write(f'{vprefix} {msg}{esc}')
131 |   elif _type == 'info':
132 |     sys.stderr.write(f'{iprefix} {msg}{esc}')
133 |   elif _type == 'good':
134 |     sys.stderr.write(f'{gprefix} {msg}{esc}')
135 |   elif _type == 'warn':
136 |     sys.stderr.write(f'{wprefix} {msg}{esc}')
137 |   elif _type == 'error':
138 |     sys.stderr.write(f'{eprefix} {msg}{esc}')
139 |     sys.exit(FAILURE)
140 |   elif _type == 'spin':
141 |     sys.stderr.flush()
142 |     for i in ('-', '\\', '|', '/'):
143 |       sys.stderr.write(f'\r{BOLD}{BLUE}[{i}] {NORM}{msg} ')
144 |       time.sleep(0.025)
145 |   elif _type == 'file':
146 |     try:
147 |       with open(opts['logfile'], 'a+', encoding='utf-8') as f:
148 |         print(msg, file=f)
149 |     except:
150 |       log('could not open or write to logfile', 'warn')
151 | 
152 |   return
153 | 
154 | 
155 | def rptr(ipaddr):
156 |   if opts['rptr']:
157 |     try:
158 |       return socket.gethostbyaddr(ipaddr)[0]
159 |     except:
160 |       return ipaddr
161 | 
162 |   return ipaddr
163 | 
164 | 
165 | def get_strings(strings):
166 |   if os.path.isfile(strings):
167 |     with open(strings, 'r', encoding='utf-8') as f:
168 |       for string in f:
169 |         yield string.rstrip()
170 |   else:
171 |     yield strings # single string
172 | 
173 |   return
174 | 
175 | 
176 | def http_req(url):
177 |   m = getattr(requests, opts['method'])
178 |   r = m(url, timeout=opts['timeout'], headers={'User-Agent': opts['ua']},
179 |     verify=False, auth=opts['auth'])
180 | 
181 |   return r
182 | 
183 | 
184 | def scan(url, string):
185 |   if opts['verbose']:
186 |     log(f'scanning {url}' + ' ' * 20, 'verbose', esc='\r')
187 |     sys.stdout.flush()
188 | 
189 |   r = http_req(url)
190 | 
191 |   if 'body' in opts['where']:
192 |     idx = re.search(string, r.text, opts['case_in']).regs[0][0]
193 |     if idx:
194 |       res = repr(r.text[idx:idx+opts['bytes']])
195 |       log(f'{url} | body   | {res}', 'good')
196 |       if opts['logfile']:
197 |         log(f'{url} | body   | {res}', 'file')
198 | 
199 |   if 'headers' in opts['where']:
200 |     for k, v in r.headers.items():
201 |       if re.search(string, k, opts['case_in']) or \
202 |         re.search(string, v, opts['case_in']):
203 |         log(f"{url} | header | {k}: {v}", 'good')
204 |         if opts['logfile']:
205 |           log(f"{url} | header | {k}: {v}", 'file')
206 | 
207 |   if opts['verbose']:
208 |     sys.stdout.flush()
209 |     log('\n')
210 | 
211 |   return
212 | 
213 | 
214 | def build_url(host):
215 |   scheme = 'http'
216 |   if opts['ssl']:
217 |     scheme = 'https'
218 | 
219 |   if ':' in host:
220 |     return f'{scheme}://{host}{opts["uri"]}'
221 | 
222 |   url = f'{scheme}://{host}:{opts["port"]}{opts["uri"]}'
223 | 
224 |   return url
225 | 
226 | 
227 | def get_hosts(hosts):
228 |   try:
229 |     if os.path.isfile(hosts):
230 |       with open(hosts, 'r', encoding='utf-8') as f:
231 |         for host in f:
232 |           yield host.rstrip()
233 |     else:
234 |       if '-' in hosts:
235 |         start = ipaddress.IPv4Address(hosts.split('-')[0])
236 |         end = ipaddress.IPv4Address(hosts.split('-')[1])
237 |         for i in range(int(start), int(end) + 1):
238 |           ipaddr = str(ipaddress.IPv4Address(i))
239 |           yield rptr(str(ipaddr))
240 |       elif '/' in hosts and 'http' not in hosts:
241 |         for ipaddr in ipaddress.IPv4Network(hosts).hosts():
242 |           yield rptr(str(ipaddr))
243 |       else:
244 |         yield hosts # single host or url
245 |   except Exception as err:
246 |     log(err.args[0].lower(), 'error')
247 | 
248 |   return
249 | 
250 | 
251 | def check_search_place():
252 |   if 'headers' not in opts['where'] and 'body' not in opts['where']:
253 |     log("nope, i only know 'body' and 'headers'", 'error')
254 | 
255 |   return
256 | 
257 | 
258 | def check_http_method():
259 |   allowed = ('head', 'get', 'post', 'put', 'delete', 'patch', 'options')
260 | 
261 |   if opts['method'] == '?':
262 |     log('supported http methods\n', 'info')
263 |     for i in allowed:
264 |       log(f'{i}', 'verbose')
265 |     sys.exit(SUCCESS)
266 |   if opts['method'] not in allowed:
267 |     log(f'unsupported http method: {opts["method"]}', 'error')
268 | 
269 |   return
270 | 
271 | 
272 | def check_auth():
273 |   if opts['auth']:
274 |     if len(opts['auth']) != 2:
275 |       log(f'wrong user:pass supplied', 'error')
276 | 
277 |   return
278 | 
279 | 
280 | def check_argv(cmdline):
281 |   needed = ['-h', '-s', '-V', '-H']
282 | 
283 |   if '-h' not in cmdline or '-s' not in cmdline or \
284 |     set(needed).isdisjoint(set(cmdline)):
285 |       log('WTF? mount /dev/brain!', 'error')
286 | 
287 |   return
288 | 
289 | 
290 | def parse_cmdline(cmdline):
291 |   global opts
292 | 
293 |   try:
294 |     _opts, _args = getopt.getopt(sys.argv[1:], 'h:p:tu:s:S:X:a:U:b:x:c:irl:vVH')
295 |     for o, a in _opts:
296 |       if o == '-h':
297 |         opts['hosts'] = a
298 |       if o == '-p':
299 |         opts['port'] = a
300 |       if o == '-t':
301 |         opts['ssl'] = True
302 |       if o == '-u':
303 |         opts['uri'] = a
304 |       if o == '-s':
305 |         opts['searchstr'] = a
306 |       if o == '-S':
307 |         opts['where'] = a.split(',')
308 |       if o == '-X':
309 |         opts['method'] = a
310 |       if o == '-a':
311 |         opts['auth'] = tuple(a.split(':', 1))
312 |       if o == '-U':
313 |         opts['ua'] = a
314 |       if o == '-b':
315 |         opts['bytes'] = int(a)
316 |       if o == '-x':
317 |         opts['threads'] = int(a)
318 |       if o == '-c':
319 |         opts['timeout'] = float(a)
320 |       if o == '-i':
321 |         opts['case_in'] = re.IGNORECASE
322 |       if o == '-r':
323 |         opts['rptr'] = True
324 |       if o == '-l':
325 |         opts['logfile'] = a
326 |       if o == '-v':
327 |         opts['verbose'] = True
328 |       if o == '-V':
329 |         log(f'httpgrep v{__version__}', _type='info')
330 |         sys.exit(SUCCESS)
331 |       if o == '-H':
332 |         log(HELP)
333 |         sys.exit(SUCCESS)
334 |   except (getopt.GetoptError, ValueError) as err:
335 |     log(err.args[0].lower(), 'error')
336 | 
337 |   return
338 | 
339 | 
340 | def check_argc(cmdline):
341 |   if len(cmdline) == 0:
342 |     log('use -H for help', 'error')
343 | 
344 |   return
345 | 
346 | 
347 | def main(cmdline):
348 |   sys.stderr.write(f'{BANNER}\n\n')
349 |   check_argc(cmdline)
350 |   parse_cmdline(cmdline)
351 |   check_argv(cmdline)
352 |   check_http_method()
353 |   check_search_place()
354 |   check_auth()
355 | 
356 |   with ThreadPoolExecutor(opts['threads']) as exe:
357 |     log('w00t w00t, game started', 'info')
358 |     log('wait bitch, scanning', 'info')
359 |     if opts['verbose']:
360 |       log('\n')
361 |     for host in get_hosts(opts['hosts']):
362 |       url = host
363 |       if 'http' not in host:
364 |         url = build_url(host)
365 |       for string in get_strings(opts['searchstr']):
366 |         exe.submit(scan, url, string)
367 | 
368 |   if opts['verbose']:
369 |     log('\n\n')
370 |   log('n00b n00b, game over', 'info')
371 | 
372 |   return
373 | 
374 | 
375 | if __name__ == '__main__':
376 |   warnings.filterwarnings('ignore')
377 |   try:
378 |     main(sys.argv[1:])
379 |   except KeyboardInterrupt:
380 |     log('\n')
381 |     log('you aborted me', 'warn')
382 |     os._exit(SUCCESS)
383 | 
384 | 


--------------------------------------------------------------------------------