├── db ├── 500_blacklist.txt ├── 400_blacklist.txt ├── 403_blacklist.txt └── user-agents.txt ├── lib ├── __init__.py ├── controller │ ├── __init__.py │ └── banner.txt ├── output │ └── __init__.py ├── connection │ ├── __init__.py │ ├── RequestException.py │ ├── Response.py │ └── Requester.py ├── utils │ ├── __init__.py │ ├── RandomUtils.py │ ├── DefaultConfigParser.py │ ├── TerminalSize.py │ └── FileUtils.py ├── reports │ ├── __init__.py │ ├── SimpleReport.py │ ├── PlainTextReport.py │ ├── JSONReport.py │ └── BaseReport.py └── core │ ├── __init__.py │ ├── Path.py │ ├── ReportManager.py │ ├── Scanner.py │ ├── Dictionary.py │ └── Fuzzer.py ├── logs └── DO_NOT_DELETE_THIS_FOLDER.txt ├── reports └── DO_NOT_DELETE_THIS_FOLDER.txt ├── thirdparty ├── requests │ ├── packages │ │ ├── urllib3 │ │ │ ├── contrib │ │ │ │ ├── __init__.py │ │ │ │ └── ntlmpool.py │ │ │ ├── packages │ │ │ │ ├── __init__.py │ │ │ │ └── ssl_match_hostname │ │ │ │ │ ├── __init__.py │ │ │ │ │ └── _implementation.py │ │ │ ├── util │ │ │ │ ├── __init__.py │ │ │ │ ├── response.py │ │ │ │ ├── request.py │ │ │ │ ├── connection.py │ │ │ │ └── url.py │ │ │ ├── __init__.py │ │ │ ├── filepost.py │ │ │ ├── exceptions.py │ │ │ ├── request.py │ │ │ └── fields.py │ │ ├── __init__.py │ │ ├── README.rst │ │ └── chardet │ │ │ ├── compat.py │ │ │ ├── __init__.py │ │ │ ├── constants.py │ │ │ ├── euctwprober.py │ │ │ ├── euckrprober.py │ │ │ ├── gb2312prober.py │ │ │ ├── big5prober.py │ │ │ ├── cp949prober.py │ │ │ ├── charsetprober.py │ │ │ ├── mbcsgroupprober.py │ │ │ ├── codingstatemachine.py │ │ │ ├── chardetect.py │ │ │ ├── utf8prober.py │ │ │ ├── escprober.py │ │ │ ├── sbcsgroupprober.py │ │ │ ├── mbcharsetprober.py │ │ │ ├── eucjpprober.py │ │ │ ├── sjisprober.py │ │ │ ├── charsetgroupprober.py │ │ │ ├── sbcharsetprober.py │ │ │ └── latin1prober.py │ ├── certs.py │ ├── hooks.py │ ├── compat.py │ ├── __init__.py │ ├── exceptions.py │ ├── structures.py │ ├── status_codes.py │ └── api.py ├── __init__.py ├── sqlmap │ ├── __init__.py │ └── DynamicContentParser.py ├── oset │ ├── __init__.py │ ├── tests.py │ └── pyoset.py └── colorama │ ├── __init__.py │ ├── initialise.py │ ├── ansi.py │ ├── win32.py │ └── winterm.py ├── .gitignore ├── default.conf ├── CHANGELOG.md ├── dirsearch.py └── README.md /db/500_blacklist.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- 1 | pass 2 | -------------------------------------------------------------------------------- /logs/DO_NOT_DELETE_THIS_FOLDER.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /reports/DO_NOT_DELETE_THIS_FOLDER.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lib/controller/__init__.py: -------------------------------------------------------------------------------- 1 | from .Controller import * -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/contrib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /db/400_blacklist.txt: -------------------------------------------------------------------------------- 1 | % 2 | Nueva Carpeta 3 | New Folder 4 | -------------------------------------------------------------------------------- /lib/output/__init__.py: -------------------------------------------------------------------------------- 1 | from .CLIOutput import * 2 | 3 | pass -------------------------------------------------------------------------------- /thirdparty/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | pass 4 | -------------------------------------------------------------------------------- /thirdparty/sqlmap/__init__.py: -------------------------------------------------------------------------------- 1 | from .DynamicContentParser import * -------------------------------------------------------------------------------- /thirdparty/oset/__init__.py: -------------------------------------------------------------------------------- 1 | """Main Ordered Set module """ 2 | 3 | from .pyoset import oset 4 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import urllib3 4 | -------------------------------------------------------------------------------- /lib/connection/__init__.py: -------------------------------------------------------------------------------- 1 | from .Requester import * 2 | from .RequestException import * 3 | from .Response import * 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | reports/ 2 | logs/ 3 | 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | -------------------------------------------------------------------------------- /lib/controller/banner.txt: -------------------------------------------------------------------------------- 1 | 2 | _|. _ _ _ _ _ _|_ v{MAYOR_VERSION}.{MINOR_VERSION}.{REVISION} 3 | (_||| _) (/_(_|| (_| ) 4 | -------------------------------------------------------------------------------- /lib/utils/__init__.py: -------------------------------------------------------------------------------- 1 | from .FileUtils import * 2 | from .RandomUtils import * 3 | from .DefaultConfigParser import * 4 | pass 5 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import ssl_match_hostname 4 | 5 | -------------------------------------------------------------------------------- /lib/reports/__init__.py: -------------------------------------------------------------------------------- 1 | from .BaseReport import * 2 | from .SimpleReport import * 3 | from .PlainTextReport import * 4 | from .JSONReport import * 5 | pass 6 | -------------------------------------------------------------------------------- /lib/core/__init__.py: -------------------------------------------------------------------------------- 1 | from .Dictionary import * 2 | from .Fuzzer import * 3 | from .ArgumentParser import * 4 | from .Path import * 5 | from .ReportManager import * 6 | -------------------------------------------------------------------------------- /db/403_blacklist.txt: -------------------------------------------------------------------------------- 1 | .htaccess 2 | .htaccess.bak 3 | .htaccess.inc 4 | .htaccess/ 5 | .htpasswd.bak 6 | .htpasswd 7 | .htpasswd.inc 8 | .htpa55wd 9 | .htpasswd/ 10 | .htpasswrd 11 | 12 | -------------------------------------------------------------------------------- /thirdparty/colorama/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. 2 | from .initialise import init, deinit, reinit, colorama_text 3 | from .ansi import Fore, Back, Style, Cursor 4 | from .ansitowin32 import AnsiToWin32 5 | 6 | __version__ = '0.3.3' 7 | 8 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/README.rst: -------------------------------------------------------------------------------- 1 | If you are planning to submit a pull request to requests with any changes in 2 | this library do not go any further. These are independent libraries which we 3 | vendor into requests. Any changes necessary to these libraries must be made in 4 | them and submitted as separate pull requests to those libraries. 5 | 6 | urllib3 pull requests go here: https://github.com/shazow/urllib3 7 | 8 | chardet pull requests go here: https://github.com/chardet/chardet 9 | -------------------------------------------------------------------------------- /default.conf: -------------------------------------------------------------------------------- 1 | [general] 2 | #threads = 10 3 | #follow-redirects = False 4 | #exclude-status = 200,301 5 | #recursive = False 6 | #scanner-fail-path = InvalidPath123123 7 | #save-logs-home = True 8 | 9 | [reports] 10 | autosave-report = True 11 | autosave-report-format = plain 12 | 13 | [dictionary] 14 | #wordlist = test.dicc 15 | #lowercase = False 16 | 17 | [connection] 18 | #useragent = MyUserAgent 19 | #timeout = 30 20 | #max-retries = 5 21 | #http-proxy = localhost:8080 22 | #random-user-agents = True 23 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | # Python 3.2+ 3 | from ssl import CertificateError, match_hostname 4 | except ImportError: 5 | try: 6 | # Backport of the function from a pypi module 7 | from backports.ssl_match_hostname import CertificateError, match_hostname 8 | except ImportError: 9 | # Our vendored copy 10 | from ._implementation import CertificateError, match_hostname 11 | 12 | # Not needed, but documenting what we provide. 13 | __all__ = ('CertificateError', 'match_hostname') 14 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/util/__init__.py: -------------------------------------------------------------------------------- 1 | # For backwards compatibility, provide imports that used to be here. 2 | from .connection import is_connection_dropped 3 | from .request import make_headers 4 | from .response import is_fp_closed 5 | from .ssl_ import ( 6 | SSLContext, 7 | HAS_SNI, 8 | assert_fingerprint, 9 | resolve_cert_reqs, 10 | resolve_ssl_version, 11 | ssl_wrap_socket, 12 | ) 13 | from .timeout import ( 14 | current_time, 15 | Timeout, 16 | ) 17 | 18 | from .retry import Retry 19 | from .url import ( 20 | get_host, 21 | parse_url, 22 | split_first, 23 | Url, 24 | ) 25 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/util/response.py: -------------------------------------------------------------------------------- 1 | def is_fp_closed(obj): 2 | """ 3 | Checks whether a given file-like object is closed. 4 | 5 | :param obj: 6 | The file-like object to check. 7 | """ 8 | 9 | try: 10 | # Check via the official file-like-object way. 11 | return obj.closed 12 | except AttributeError: 13 | pass 14 | 15 | try: 16 | # Check if the object is a container for another file-like object that 17 | # gets released on exhaustion (e.g. HTTPResponse). 18 | return obj.fp is None 19 | except AttributeError: 20 | pass 21 | 22 | raise ValueError("Unable to determine whether fp is closed.") 23 | -------------------------------------------------------------------------------- /thirdparty/requests/certs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | certs.py 6 | ~~~~~~~~ 7 | 8 | This module returns the preferred default CA certificate bundle. 9 | 10 | If you are packaging Requests, e.g., for a Linux distribution or a managed 11 | environment, you can change the definition of where() to return a separately 12 | packaged CA bundle. 13 | """ 14 | import os.path 15 | 16 | try: 17 | from certifi import where 18 | except ImportError: 19 | def where(): 20 | """Return the preferred certificate bundle.""" 21 | # vendored bundle inside Requests 22 | return os.path.join(os.path.dirname(__file__), 'cacert.pem') 23 | 24 | if __name__ == '__main__': 25 | print(where()) 26 | -------------------------------------------------------------------------------- /lib/connection/RequestException.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | class RequestException(Exception): 20 | pass 21 | -------------------------------------------------------------------------------- /thirdparty/requests/hooks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.hooks 5 | ~~~~~~~~~~~~~~ 6 | 7 | This module provides the capabilities for the Requests hooks system. 8 | 9 | Available hooks: 10 | 11 | ``response``: 12 | The response generated from a Request. 13 | 14 | """ 15 | HOOKS = ['response'] 16 | 17 | def default_hooks(): 18 | return dict((event, []) for event in HOOKS) 19 | 20 | # TODO: response is the only one 21 | 22 | 23 | def dispatch_hook(key, hooks, hook_data, **kwargs): 24 | """Dispatches a hook dictionary on a given piece of data.""" 25 | hooks = hooks or dict() 26 | hooks = hooks.get(key) 27 | if hooks: 28 | if hasattr(hooks, '__call__'): 29 | hooks = [hooks] 30 | for hook in hooks: 31 | _hook_data = hook(hook_data, **kwargs) 32 | if _hook_data is not None: 33 | hook_data = _hook_data 34 | return hook_data 35 | -------------------------------------------------------------------------------- /thirdparty/oset/tests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- mode:python; tab-width: 2; coding: utf-8 -*- 3 | 4 | """Partially backported python ABC classes""" 5 | 6 | 7 | 8 | import doctest 9 | import unittest 10 | 11 | optionflags = doctest.NORMALIZE_WHITESPACE | \ 12 | doctest.ELLIPSIS | \ 13 | doctest.REPORT_ONLY_FIRST_FAILURE 14 | 15 | TESTFILES = [ 16 | 'pyoset.txt', 17 | ] 18 | 19 | 20 | def test_suite(): 21 | """Simple tes suite""" 22 | 23 | globs = {} 24 | try: 25 | from pprint import pprint 26 | globs['pprint'] = pprint 27 | except Exception: 28 | pass 29 | try: 30 | from interlude import interact 31 | globs['interact'] = interact 32 | except Exception: 33 | pass 34 | 35 | return unittest.TestSuite([ 36 | doctest.DocFileSuite( 37 | file, 38 | optionflags=optionflags, 39 | globs=globs, 40 | ) for file in TESTFILES 41 | ]) 42 | 43 | if __name__ == '__main__': 44 | unittest.main(defaultTest='test_suite') 45 | -------------------------------------------------------------------------------- /lib/core/Path.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | class Path(object): 20 | def __init__(self, path=None, status=None, response=None): 21 | self.path = path 22 | self.status = status 23 | self.response = response 24 | 25 | def __str__(self): 26 | return self.path -------------------------------------------------------------------------------- /lib/utils/RandomUtils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import random 20 | import string 21 | 22 | 23 | class RandomUtils(object): 24 | @classmethod 25 | def randString(cls, n=12, omit=None): 26 | seq = string.ascii_lowercase + string.ascii_uppercase + string.digits 27 | if omit: 28 | seq = list(set(seq) - set(omit)) 29 | return ''.join(random.choice(seq) for _ in range(n)) -------------------------------------------------------------------------------- /lib/reports/SimpleReport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | from lib.reports import * 20 | 21 | 22 | class SimpleReport(BaseReport): 23 | def generate(self): 24 | result = '' 25 | for path, _, _ in self.pathList: 26 | result += '{0}://{1}:{2}/'.format(self.protocol, self.host, self.port) 27 | result += ('{0}\n'.format(path) if self.basePath is '' else '{0}/{1}\n'.format(self.basePath, path)) 28 | return result 29 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/compat.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # Contributor(s): 3 | # Ian Cordasco - port to Python 4 | # 5 | # This library is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU Lesser General Public 7 | # License as published by the Free Software Foundation; either 8 | # version 2.1 of the License, or (at your option) any later version. 9 | # 10 | # This library is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public 16 | # License along with this library; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 18 | # 02110-1301 USA 19 | ######################### END LICENSE BLOCK ######################### 20 | 21 | import sys 22 | 23 | 24 | if sys.version_info < (3, 0): 25 | base_str = (str, unicode) 26 | else: 27 | base_str = (bytes, str) 28 | 29 | 30 | def wrap_ord(a): 31 | if sys.version_info < (3, 0) and isinstance(a, base_str): 32 | return ord(a) 33 | else: 34 | return a 35 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/__init__.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # This library is free software; you can redistribute it and/or 3 | # modify it under the terms of the GNU Lesser General Public 4 | # License as published by the Free Software Foundation; either 5 | # version 2.1 of the License, or (at your option) any later version. 6 | # 7 | # This library is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | # Lesser General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU Lesser General Public 13 | # License along with this library; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 15 | # 02110-1301 USA 16 | ######################### END LICENSE BLOCK ######################### 17 | 18 | __version__ = "2.3.0" 19 | from sys import version_info 20 | 21 | 22 | def detect(aBuf): 23 | if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or 24 | (version_info >= (3, 0) and not isinstance(aBuf, bytes))): 25 | raise ValueError('Expected a bytes object, not a unicode object') 26 | 27 | from . import universaldetector 28 | u = universaldetector.UniversalDetector() 29 | u.reset() 30 | u.feed(aBuf) 31 | u.close() 32 | return u.result 33 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | Changelog 2 | --------- 3 | - 0.3.8 - 2017.07.25 Delay argument added. Request by hostname switch added. Suppress empty switch added. Added Force Extensions switch. Multiple fixes. 4 | - 0.3.7 - 2016.08.22 Force extensions switch added. 5 | - 0.3.6 - 2016.02.14 Bugfixes 6 | - 0.3.5 - 2016.01.29 Improved heuristic, replaced urllib3 for requests, error logs, batch reports, user agent randomization, bugfixes 7 | - 0.3.0 - 2015.02.05 Fixed issue3, fixed timeout exception, ported to Python3, other bugfixes 8 | - 0.2.7 - 2014.11.21 Added Url List feature (-L). Changed output. Minor Fixes 9 | - 0.2.6 - 2014.9.12 Fixed bug when dictionary size is greater than threads count. Fixed URL encoding bug (issue2). 10 | - 0.2.5 - 2014.9.2 Shows Content-Length in output and reports, added default.conf file (for setting defaults) and report auto save feature added. 11 | - 0.2.4 - 2014.7.17 Added Windows support, --scan-subdir|--scan-subdirs argument added, --exclude-subdir|--exclude-subdirs added, --header argument added, dirbuster dictionaries added, fixed some concurrency bugs, MVC refactoring 12 | - 0.2.3 - 2014.7.7 Fixed some bugs, minor refactorings, exclude status switch, "pause/next directory" feature, changed help structure, expaded default dictionary 13 | - 0.2.2 - 2014.7.2 Fixed some bugs, showing percentage of tested paths and added report generation feature 14 | - 0.2.1 - 2014.5.1 Fixed some bugs and added recursive option 15 | - 0.2.0 - 2014.1.31 Initial public release 16 | -------------------------------------------------------------------------------- /lib/reports/PlainTextReport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | from lib.reports import * 20 | from lib.utils.FileUtils import * 21 | 22 | 23 | class PlainTextReport(BaseReport): 24 | 25 | def generate(self): 26 | result = '' 27 | for path, status, contentLength in self.pathList: 28 | result += '{0} '.format(status) 29 | result += '{0} '.format(FileUtils.sizeHuman(contentLength).rjust(6, ' ')) 30 | result += '{0}://{1}:{2}/'.format(self.protocol, self.host, self.port) 31 | result += ('{0}\n'.format(path) if self.basePath is '' else '{0}/{1}\n'.format(self.basePath, path)) 32 | return result 33 | 34 | 35 | -------------------------------------------------------------------------------- /dirsearch.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # This program is free software; you can redistribute it and/or modify 4 | # it under the terms of the GNU General Public License as published by 5 | # the Free Software Foundation; either version 2 of the License, or 6 | # (at your option) any later version. 7 | # 8 | # This program is distributed in the hope that it will be useful, 9 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 10 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 | # GNU General Public License for more details. 12 | # 13 | # You should have received a copy of the GNU General Public License 14 | # along with this program; if not, write to the Free Software 15 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 16 | # MA 02110-1301, USA. 17 | # 18 | # Author: Mauro Soria 19 | 20 | import sys 21 | if sys.version_info < (3, 0): 22 | sys.stdout.write("Sorry, dirsearch requires Python 3.x\n") 23 | sys.exit(1) 24 | 25 | from lib.core import ArgumentParser 26 | from lib.controller import * 27 | from lib.output import * 28 | 29 | class Program(object): 30 | def __init__(self): 31 | self.script_path = (os.path.dirname(os.path.realpath(__file__))) 32 | self.arguments = ArgumentParser(self.script_path) 33 | self.output = CLIOutput(self.arguments) 34 | self.controller = Controller(self.script_path, self.arguments, self.output) 35 | 36 | 37 | if __name__ == '__main__': 38 | main = Program() 39 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/constants.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | _debug = 0 30 | 31 | eDetecting = 0 32 | eFoundIt = 1 33 | eNotMe = 2 34 | 35 | eStart = 0 36 | eError = 1 37 | eItsMe = 2 38 | 39 | SHORTCUT_THRESHOLD = 0.95 40 | -------------------------------------------------------------------------------- /lib/core/ReportManager.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import threading 20 | 21 | class ReportManager(object): 22 | 23 | def __init__(self): 24 | self.outputs = [] 25 | self.lock = threading.Lock() 26 | 27 | def addOutput(self, output): 28 | self.outputs.append(output) 29 | 30 | def addPath(self, path, status, response): 31 | with self.lock: 32 | for output in self.outputs: 33 | output.addPath(path, status, response) 34 | 35 | def save(self): 36 | with self.lock: 37 | for output in self.outputs: 38 | output.save() 39 | 40 | def close(self): 41 | for output in self.outputs: 42 | output.close() 43 | 44 | 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | dirsearch 2 | ========= 3 | 4 | Current Release: v0.3.8 (2017.07.25) 5 | 6 | 7 | Overview 8 | -------- 9 | dirsearch is a simple command line tool designed to brute force directories and files in websites. 10 | 11 | 12 | Operating Systems supported 13 | --------------------------- 14 | - Windows XP/7/8/10 15 | - GNU/Linux 16 | - MacOSX 17 | 18 | Features 19 | -------- 20 | - Multithreaded 21 | - Keep alive connections 22 | - Support for multiple extensions and no-extension (-e|--extensions asp,php,) 23 | - Reporting (plain text, JSON) 24 | - Heuristically detects invalid web pages 25 | - Recursive brute forcing 26 | - HTTP proxy support 27 | - User agent randomization 28 | - Batch processing 29 | - Request delaying 30 | - Filter by size 31 | 32 | About wordlists 33 | --------------- 34 | Dictionaries must be text files. Each line will be processed as such, except that the special word %EXT% is used, which will generate one entry for each extension (-e | --extension) passed as an argument. 35 | 36 | Example: 37 | - example/ 38 | - example.%EXT% 39 | 40 | Passing the extensions "asp" and "aspx" will generate the following dictionary: 41 | - example/ 42 | - example.asp 43 | - example.aspx 44 | 45 | You can also use -f | --force-extensions switch to append extensions to every word in the wordlists (like DirBuster). 46 | 47 | License 48 | ------- 49 | Copyright (C) Mauro Soria (maurosoria at gmail dot com) 50 | 51 | License: GNU General Public License, version 2 52 | 53 | 54 | Contributors 55 | --------- 56 | Special thanks for these people. 57 | 58 | - Bo0oM 59 | - liamosaur 60 | - redshark1802 61 | - SUHAR1K 62 | - FireFart 63 | - k2l8m11n2 64 | - vlohacks 65 | - r0p0s3c 66 | 67 | 68 | -------------------------------------------------------------------------------- /thirdparty/requests/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | """ 6 | 7 | from .packages import chardet 8 | 9 | import sys 10 | 11 | # ------- 12 | # Pythons 13 | # ------- 14 | 15 | # Syntax sugar. 16 | _ver = sys.version_info 17 | 18 | #: Python 2.x? 19 | is_py2 = (_ver[0] == 2) 20 | 21 | #: Python 3.x? 22 | is_py3 = (_ver[0] == 3) 23 | 24 | try: 25 | import simplejson as json 26 | except (ImportError, SyntaxError): 27 | # simplejson does not support Python 3.2, it throws a SyntaxError 28 | # because of u'...' Unicode literals. 29 | import json 30 | 31 | # --------- 32 | # Specifics 33 | # --------- 34 | 35 | if is_py2: 36 | from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass 37 | from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag 38 | from urllib2 import parse_http_list 39 | import cookielib 40 | from Cookie import Morsel 41 | from StringIO import StringIO 42 | from .packages.urllib3.packages.ordered_dict import OrderedDict 43 | 44 | builtin_str = str 45 | bytes = str 46 | str = unicode 47 | basestring = basestring 48 | numeric_types = (int, long, float) 49 | 50 | elif is_py3: 51 | from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag 52 | from urllib.request import parse_http_list, getproxies, proxy_bypass 53 | from http import cookiejar as cookielib 54 | from http.cookies import Morsel 55 | from io import StringIO 56 | from collections import OrderedDict 57 | 58 | builtin_str = str 59 | str = str 60 | bytes = bytes 61 | basestring = (str, bytes) 62 | numeric_types = (int, float) 63 | -------------------------------------------------------------------------------- /lib/reports/JSONReport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import json 20 | 21 | from lib.reports import * 22 | 23 | 24 | class JSONReport(BaseReport): 25 | 26 | def addPath(self, path, status, response): 27 | contentLength = None 28 | try: 29 | contentLength = int(response.headers['content-length']) 30 | except (KeyError, ValueError): 31 | contentLength = len(response.body) 32 | self.pathList.append((path, status, contentLength, response.redirect)) 33 | 34 | def generate(self): 35 | headerName = '{0}://{1}:{2}/{3}'.format(self.protocol, self.host, self.port, self.basePath) 36 | result = {headerName: []} 37 | for path, status, contentLength, redirect in self.pathList: 38 | entry = {'status': status, 'path': path, 'content-length': contentLength, 'redirect' : redirect} 39 | result[headerName].append(entry) 40 | return json.dumps(result, sort_keys=True, indent=4) 41 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/euctwprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCTWDistributionAnalysis 31 | from .mbcssm import EUCTWSMModel 32 | 33 | class EUCTWProber(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(EUCTWSMModel) 37 | self._mDistributionAnalyzer = EUCTWDistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "EUC-TW" 42 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/euckrprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import EUCKRSMModel 32 | 33 | 34 | class EUCKRProber(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(EUCKRSMModel) 38 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "EUC-KR" 43 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/gb2312prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import GB2312DistributionAnalysis 31 | from .mbcssm import GB2312SMModel 32 | 33 | class GB2312Prober(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(GB2312SMModel) 37 | self._mDistributionAnalyzer = GB2312DistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "GB2312" 42 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/big5prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import Big5DistributionAnalysis 31 | from .mbcssm import Big5SMModel 32 | 33 | 34 | class Big5Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(Big5SMModel) 38 | self._mDistributionAnalyzer = Big5DistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "Big5" 43 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/cp949prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import CP949SMModel 32 | 33 | 34 | class CP949Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(CP949SMModel) 38 | # NOTE: CP949 is a superset of EUC-KR, so the distribution should be 39 | # not different. 40 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 41 | self.reset() 42 | 43 | def get_charset_name(self): 44 | return "CP949" 45 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/charsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from . import constants 30 | import re 31 | 32 | 33 | class CharSetProber: 34 | def __init__(self): 35 | pass 36 | 37 | def reset(self): 38 | self._mState = constants.eDetecting 39 | 40 | def get_charset_name(self): 41 | return None 42 | 43 | def feed(self, aBuf): 44 | pass 45 | 46 | def get_state(self): 47 | return self._mState 48 | 49 | def get_confidence(self): 50 | return 0.0 51 | 52 | def filter_high_bit_only(self, aBuf): 53 | aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf) 54 | return aBuf 55 | 56 | def filter_without_english_letters(self, aBuf): 57 | aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf) 58 | return aBuf 59 | 60 | def filter_with_english_letters(self, aBuf): 61 | # TODO 62 | return aBuf 63 | -------------------------------------------------------------------------------- /lib/connection/Response.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | class Response(object): 20 | 21 | def __init__(self, status, reason, headers, body): 22 | self.status = status 23 | self.reason = reason 24 | self.headers = headers 25 | self.body = body 26 | 27 | def __str__(self): 28 | return self.body 29 | 30 | def __int__(self): 31 | return self.status 32 | 33 | def __eq__(self, other): 34 | return self.status == other.status and self.body == other.body 35 | 36 | def __cmp__(self, other): 37 | return (self.body > other) - (self.body < other) 38 | 39 | def __len__(self): 40 | return len(self.body) 41 | 42 | def __hash__(self): 43 | return hash(self.body) 44 | 45 | def __del__(self): 46 | del self.body 47 | del self.headers 48 | del self.status 49 | del self.reason 50 | 51 | @property 52 | def redirect(self): 53 | headers = dict((key.lower(), value) for key, value in self.headers.items()) 54 | return headers.get("location") 55 | 56 | @property 57 | def pretty(self): 58 | try: 59 | from BeautifulSoup import BeautifulSoup 60 | except ImportError: 61 | raise Exception('BeautifulSoup must be installed to get pretty HTML =(') 62 | html = BeautifulSoup(self.body) 63 | return html.prettify() -------------------------------------------------------------------------------- /thirdparty/colorama/initialise.py: -------------------------------------------------------------------------------- 1 | # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. 2 | import atexit 3 | import contextlib 4 | import sys 5 | 6 | from .ansitowin32 import AnsiToWin32 7 | 8 | 9 | orig_stdout = None 10 | orig_stderr = None 11 | 12 | wrapped_stdout = None 13 | wrapped_stderr = None 14 | 15 | atexit_done = False 16 | 17 | 18 | def reset_all(): 19 | AnsiToWin32(orig_stdout).reset_all() 20 | 21 | 22 | def init(autoreset=False, convert=None, strip=None, wrap=True): 23 | 24 | if not wrap and any([autoreset, convert, strip]): 25 | raise ValueError('wrap=False conflicts with any other arg=True') 26 | 27 | global wrapped_stdout, wrapped_stderr 28 | global orig_stdout, orig_stderr 29 | 30 | orig_stdout = sys.stdout 31 | orig_stderr = sys.stderr 32 | 33 | if sys.stdout is None: 34 | wrapped_stdout = None 35 | else: 36 | sys.stdout = wrapped_stdout = \ 37 | wrap_stream(orig_stdout, convert, strip, autoreset, wrap) 38 | if sys.stderr is None: 39 | wrapped_stderr = None 40 | else: 41 | sys.stderr = wrapped_stderr = \ 42 | wrap_stream(orig_stderr, convert, strip, autoreset, wrap) 43 | 44 | global atexit_done 45 | if not atexit_done: 46 | atexit.register(reset_all) 47 | atexit_done = True 48 | 49 | 50 | def deinit(): 51 | if orig_stdout is not None: 52 | sys.stdout = orig_stdout 53 | if orig_stderr is not None: 54 | sys.stderr = orig_stderr 55 | 56 | 57 | @contextlib.contextmanager 58 | def colorama_text(*args, **kwargs): 59 | init(*args, **kwargs) 60 | try: 61 | yield 62 | finally: 63 | deinit() 64 | 65 | 66 | def reinit(): 67 | if wrapped_stdout is not None: 68 | sys.stdout = wrapped_stdout 69 | if wrapped_stderr is not None: 70 | sys.stderr = wrapped_stderr 71 | 72 | 73 | def wrap_stream(stream, convert, strip, autoreset, wrap): 74 | if wrap: 75 | wrapper = AnsiToWin32(stream, 76 | convert=convert, strip=strip, autoreset=autoreset) 77 | if wrapper.should_wrap(): 78 | stream = wrapper.stream 79 | return stream 80 | 81 | 82 | -------------------------------------------------------------------------------- /thirdparty/requests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # __ 4 | # /__) _ _ _ _ _/ _ 5 | # / ( (- (/ (/ (- _) / _) 6 | # / 7 | 8 | """ 9 | Requests HTTP library 10 | ~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | Requests is an HTTP library, written in Python, for human beings. Basic GET 13 | usage: 14 | 15 | >>> import requests 16 | >>> r = requests.get('https://www.python.org') 17 | >>> r.status_code 18 | 200 19 | >>> 'Python is a programming language' in r.content 20 | True 21 | 22 | ... or POST: 23 | 24 | >>> payload = dict(key1='value1', key2='value2') 25 | >>> r = requests.post('http://httpbin.org/post', data=payload) 26 | >>> print(r.text) 27 | { 28 | ... 29 | "form": { 30 | "key2": "value2", 31 | "key1": "value1" 32 | }, 33 | ... 34 | } 35 | 36 | The other HTTP methods are supported - see `requests.api`. Full documentation 37 | is at . 38 | 39 | :copyright: (c) 2015 by Kenneth Reitz. 40 | :license: Apache 2.0, see LICENSE for more details. 41 | 42 | """ 43 | 44 | __title__ = 'requests' 45 | __version__ = '2.7.0' 46 | __build__ = 0x020700 47 | __author__ = 'Kenneth Reitz' 48 | __license__ = 'Apache 2.0' 49 | __copyright__ = 'Copyright 2015 Kenneth Reitz' 50 | 51 | # Attempt to enable urllib3's SNI support, if possible 52 | try: 53 | from .packages.urllib3.contrib import pyopenssl 54 | pyopenssl.inject_into_urllib3() 55 | except ImportError: 56 | pass 57 | 58 | from . import utils 59 | from .models import Request, Response, PreparedRequest 60 | from .api import request, get, head, post, patch, put, delete, options 61 | from .sessions import session, Session 62 | from .status_codes import codes 63 | from .exceptions import ( 64 | RequestException, Timeout, URLRequired, 65 | TooManyRedirects, HTTPError, ConnectionError 66 | ) 67 | 68 | # Set default logging handler to avoid "No handler found" warnings. 69 | import logging 70 | try: # Python 2.7+ 71 | from logging import NullHandler 72 | except ImportError: 73 | class NullHandler(logging.Handler): 74 | def emit(self, record): 75 | pass 76 | 77 | logging.getLogger(__name__).addHandler(NullHandler()) 78 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/mbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | from .charsetgroupprober import CharSetGroupProber 31 | from .utf8prober import UTF8Prober 32 | from .sjisprober import SJISProber 33 | from .eucjpprober import EUCJPProber 34 | from .gb2312prober import GB2312Prober 35 | from .euckrprober import EUCKRProber 36 | from .cp949prober import CP949Prober 37 | from .big5prober import Big5Prober 38 | from .euctwprober import EUCTWProber 39 | 40 | 41 | class MBCSGroupProber(CharSetGroupProber): 42 | def __init__(self): 43 | CharSetGroupProber.__init__(self) 44 | self._mProbers = [ 45 | UTF8Prober(), 46 | SJISProber(), 47 | EUCJPProber(), 48 | GB2312Prober(), 49 | EUCKRProber(), 50 | CP949Prober(), 51 | Big5Prober(), 52 | EUCTWProber() 53 | ] 54 | self.reset() 55 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | urllib3 - Thread-safe connection pooling and re-using. 3 | """ 4 | 5 | __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' 6 | __license__ = 'MIT' 7 | __version__ = '1.10.4' 8 | 9 | 10 | from .connectionpool import ( 11 | HTTPConnectionPool, 12 | HTTPSConnectionPool, 13 | connection_from_url 14 | ) 15 | 16 | from . import exceptions 17 | from .filepost import encode_multipart_formdata 18 | from .poolmanager import PoolManager, ProxyManager, proxy_from_url 19 | from .response import HTTPResponse 20 | from .util.request import make_headers 21 | from .util.url import get_host 22 | from .util.timeout import Timeout 23 | from .util.retry import Retry 24 | 25 | 26 | # Set default logging handler to avoid "No handler found" warnings. 27 | import logging 28 | try: # Python 2.7+ 29 | from logging import NullHandler 30 | except ImportError: 31 | class NullHandler(logging.Handler): 32 | def emit(self, record): 33 | pass 34 | 35 | logging.getLogger(__name__).addHandler(NullHandler()) 36 | 37 | def add_stderr_logger(level=logging.DEBUG): 38 | """ 39 | Helper for quickly adding a StreamHandler to the logger. Useful for 40 | debugging. 41 | 42 | Returns the handler after adding it. 43 | """ 44 | # This method needs to be in this __init__.py to get the __name__ correct 45 | # even if urllib3 is vendored within another package. 46 | logger = logging.getLogger(__name__) 47 | handler = logging.StreamHandler() 48 | handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) 49 | logger.addHandler(handler) 50 | logger.setLevel(level) 51 | logger.debug('Added a stderr logging handler to logger: %s' % __name__) 52 | return handler 53 | 54 | # ... Clean up. 55 | del NullHandler 56 | 57 | 58 | import warnings 59 | # SecurityWarning's always go off by default. 60 | warnings.simplefilter('always', exceptions.SecurityWarning, append=True) 61 | # InsecurePlatformWarning's don't vary between requests, so we keep it default. 62 | warnings.simplefilter('default', exceptions.InsecurePlatformWarning, 63 | append=True) 64 | 65 | def disable_warnings(category=exceptions.HTTPWarning): 66 | """ 67 | Helper for quickly disabling all urllib3 warnings. 68 | """ 69 | warnings.simplefilter('ignore', category) 70 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/util/request.py: -------------------------------------------------------------------------------- 1 | from base64 import b64encode 2 | 3 | from ..packages.six import b 4 | 5 | ACCEPT_ENCODING = 'gzip,deflate' 6 | 7 | 8 | def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, 9 | basic_auth=None, proxy_basic_auth=None, disable_cache=None): 10 | """ 11 | Shortcuts for generating request headers. 12 | 13 | :param keep_alive: 14 | If ``True``, adds 'connection: keep-alive' header. 15 | 16 | :param accept_encoding: 17 | Can be a boolean, list, or string. 18 | ``True`` translates to 'gzip,deflate'. 19 | List will get joined by comma. 20 | String will be used as provided. 21 | 22 | :param user_agent: 23 | String representing the user-agent you want, such as 24 | "python-urllib3/0.6" 25 | 26 | :param basic_auth: 27 | Colon-separated username:password string for 'authorization: basic ...' 28 | auth header. 29 | 30 | :param proxy_basic_auth: 31 | Colon-separated username:password string for 'proxy-authorization: basic ...' 32 | auth header. 33 | 34 | :param disable_cache: 35 | If ``True``, adds 'cache-control: no-cache' header. 36 | 37 | Example:: 38 | 39 | >>> make_headers(keep_alive=True, user_agent="Batman/1.0") 40 | {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} 41 | >>> make_headers(accept_encoding=True) 42 | {'accept-encoding': 'gzip,deflate'} 43 | """ 44 | headers = {} 45 | if accept_encoding: 46 | if isinstance(accept_encoding, str): 47 | pass 48 | elif isinstance(accept_encoding, list): 49 | accept_encoding = ','.join(accept_encoding) 50 | else: 51 | accept_encoding = ACCEPT_ENCODING 52 | headers['accept-encoding'] = accept_encoding 53 | 54 | if user_agent: 55 | headers['user-agent'] = user_agent 56 | 57 | if keep_alive: 58 | headers['connection'] = 'keep-alive' 59 | 60 | if basic_auth: 61 | headers['authorization'] = 'Basic ' + \ 62 | b64encode(b(basic_auth)).decode('utf-8') 63 | 64 | if proxy_basic_auth: 65 | headers['proxy-authorization'] = 'Basic ' + \ 66 | b64encode(b(proxy_basic_auth)).decode('utf-8') 67 | 68 | if disable_cache: 69 | headers['cache-control'] = 'no-cache' 70 | 71 | return headers 72 | -------------------------------------------------------------------------------- /lib/reports/BaseReport.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | class BaseReport(object): 20 | 21 | def __init__(self, host, port, protocol, basePath, output): 22 | self.output = output 23 | self.port = port 24 | self.host = host 25 | self.protocol = protocol 26 | self.basePath = basePath 27 | if self.basePath.endswith('/'): 28 | self.basePath = self.basePath[:-1] 29 | if self.basePath.startswith('/'): 30 | self.basePath = self.basePath[1:] 31 | self.pathList = [] 32 | self.open() 33 | 34 | def addPath(self, path, status, response): 35 | contentLength = None 36 | try: 37 | contentLength = int(response.headers['content-length']) 38 | except (KeyError, ValueError): 39 | contentLength = len(response.body) 40 | self.pathList.append((path, status, contentLength)) 41 | 42 | def open(self): 43 | from os import name as os_name 44 | if os_name == "nt": 45 | from os.path import normpath, dirname 46 | from os import makedirs 47 | output = normpath(self.output) 48 | makedirs(dirname(output), exist_ok=True) 49 | self.output = output 50 | self.file = open(self.output, 'w+') 51 | 52 | def save(self): 53 | self.file.seek(0) 54 | self.file.truncate(0) 55 | self.file.flush() 56 | self.file.writelines(self.generate()) 57 | self.file.flush() 58 | 59 | def close(self): 60 | self.file.close() 61 | 62 | def generate(self): 63 | raise NotImplementedError 64 | 65 | 66 | -------------------------------------------------------------------------------- /thirdparty/oset/pyoset.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- mode:python; tab-width: 2; coding: utf-8 -*- 3 | 4 | """Partially backported python ABC classes""" 5 | 6 | 7 | 8 | try: 9 | from collections import MutableSet 10 | except ImportError: 11 | # Running in Python <= 2.5 12 | from ._abc import MutableSet 13 | 14 | 15 | KEY, PREV, NEXT = list(range(3)) 16 | 17 | 18 | class OrderedSet(MutableSet): 19 | 20 | def __init__(self, iterable=None): 21 | self.end = end = [] 22 | end += [None, end, end] # sentinel node for doubly linked list 23 | self.map = {} # key --> [key, prev, next] 24 | if iterable is not None: 25 | self |= iterable 26 | 27 | def __len__(self): 28 | return len(self.map) 29 | 30 | def __contains__(self, key): 31 | return key in self.map 32 | 33 | def __getitem__(self, key): 34 | return list(self)[key] 35 | 36 | def add(self, key): 37 | if key not in self.map: 38 | end = self.end 39 | curr = end[PREV] 40 | curr[NEXT] = end[PREV] = self.map[key] = [key, curr, end] 41 | 42 | def discard(self, key): 43 | if key in self.map: 44 | key, prev, next = self.map.pop(key) 45 | prev[NEXT] = next 46 | next[PREV] = prev 47 | 48 | def __iter__(self): 49 | end = self.end 50 | curr = end[NEXT] 51 | while curr is not end: 52 | yield curr[KEY] 53 | curr = curr[NEXT] 54 | 55 | def __reversed__(self): 56 | end = self.end 57 | curr = end[PREV] 58 | while curr is not end: 59 | yield curr[KEY] 60 | curr = curr[PREV] 61 | 62 | def pop(self, last=True): 63 | if not self: 64 | raise KeyError('set is empty') 65 | key = next(reversed(self)) if last else next(iter(self)) 66 | self.discard(key) 67 | return key 68 | 69 | def __repr__(self): 70 | if not self: 71 | return '%s()' % (self.__class__.__name__,) 72 | return '%s(%r)' % (self.__class__.__name__, list(self)) 73 | 74 | def __eq__(self, other): 75 | if isinstance(other, OrderedSet): 76 | return len(self) == len(other) and list(self) == list(other) 77 | return set(self) == set(other) 78 | 79 | def __del__(self): 80 | self.clear() # remove circular references 81 | 82 | oset = OrderedSet 83 | -------------------------------------------------------------------------------- /lib/utils/DefaultConfigParser.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import configparser 20 | 21 | class DefaultConfigParser(configparser.ConfigParser): 22 | def __init__(self): 23 | configparser.ConfigParser.__init__(self) 24 | 25 | 26 | def safe_get(self, section, option, default, allowed=None): 27 | try: 28 | result = configparser.ConfigParser.get(self, section, option) 29 | if allowed is not None: 30 | return result if result in allowed else default 31 | else: 32 | return result 33 | except (configparser.NoSectionError, configparser.NoOptionError): 34 | return default 35 | 36 | def safe_getfloat(self, section, option, default, allowed=None): 37 | try: 38 | result = configparser.ConfigParser.getfloat(self, section, option) 39 | if allowed is not None: 40 | return result if result in allowed else default 41 | else: 42 | return result 43 | except (configparser.NoSectionError, configparser.NoOptionError): 44 | return default 45 | 46 | def safe_getboolean(self, section, option, default, allowed=None): 47 | try: 48 | result = configparser.ConfigParser.getboolean(self, section, option) 49 | if allowed is not None: 50 | return result if result in allowed else default 51 | else: 52 | return result 53 | except (configparser.NoSectionError, configparser.NoOptionError): 54 | return default 55 | 56 | def safe_getint(self, section, option, default, allowed=None): 57 | try: 58 | result = configparser.ConfigParser.getint(self, section, option) 59 | if allowed is not None: 60 | return result if result in allowed else default 61 | else: 62 | return result 63 | except (configparser.NoSectionError, configparser.NoOptionError): 64 | return default -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/codingstatemachine.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .constants import eStart 29 | from .compat import wrap_ord 30 | 31 | 32 | class CodingStateMachine: 33 | def __init__(self, sm): 34 | self._mModel = sm 35 | self._mCurrentBytePos = 0 36 | self._mCurrentCharLen = 0 37 | self.reset() 38 | 39 | def reset(self): 40 | self._mCurrentState = eStart 41 | 42 | def next_state(self, c): 43 | # for each byte we get its class 44 | # if it is first byte, we also get byte length 45 | # PY3K: aBuf is a byte stream, so c is an int, not a byte 46 | byteCls = self._mModel['classTable'][wrap_ord(c)] 47 | if self._mCurrentState == eStart: 48 | self._mCurrentBytePos = 0 49 | self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] 50 | # from byte's class and stateTable, we get its next state 51 | curr_state = (self._mCurrentState * self._mModel['classFactor'] 52 | + byteCls) 53 | self._mCurrentState = self._mModel['stateTable'][curr_state] 54 | self._mCurrentBytePos += 1 55 | return self._mCurrentState 56 | 57 | def get_current_charlen(self): 58 | return self._mCurrentCharLen 59 | 60 | def get_coding_state_machine(self): 61 | return self._mModel['name'] 62 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/filepost.py: -------------------------------------------------------------------------------- 1 | import codecs 2 | 3 | from uuid import uuid4 4 | from io import BytesIO 5 | 6 | from .packages import six 7 | from .packages.six import b 8 | from .fields import RequestField 9 | 10 | writer = codecs.lookup('utf-8')[3] 11 | 12 | 13 | def choose_boundary(): 14 | """ 15 | Our embarassingly-simple replacement for mimetools.choose_boundary. 16 | """ 17 | return uuid4().hex 18 | 19 | 20 | def iter_field_objects(fields): 21 | """ 22 | Iterate over fields. 23 | 24 | Supports list of (k, v) tuples and dicts, and lists of 25 | :class:`~urllib3.fields.RequestField`. 26 | 27 | """ 28 | if isinstance(fields, dict): 29 | i = six.iteritems(fields) 30 | else: 31 | i = iter(fields) 32 | 33 | for field in i: 34 | if isinstance(field, RequestField): 35 | yield field 36 | else: 37 | yield RequestField.from_tuples(*field) 38 | 39 | 40 | def iter_fields(fields): 41 | """ 42 | .. deprecated:: 1.6 43 | 44 | Iterate over fields. 45 | 46 | The addition of :class:`~urllib3.fields.RequestField` makes this function 47 | obsolete. Instead, use :func:`iter_field_objects`, which returns 48 | :class:`~urllib3.fields.RequestField` objects. 49 | 50 | Supports list of (k, v) tuples and dicts. 51 | """ 52 | if isinstance(fields, dict): 53 | return ((k, v) for k, v in six.iteritems(fields)) 54 | 55 | return ((k, v) for k, v in fields) 56 | 57 | 58 | def encode_multipart_formdata(fields, boundary=None): 59 | """ 60 | Encode a dictionary of ``fields`` using the multipart/form-data MIME format. 61 | 62 | :param fields: 63 | Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). 64 | 65 | :param boundary: 66 | If not specified, then a random boundary will be generated using 67 | :func:`mimetools.choose_boundary`. 68 | """ 69 | body = BytesIO() 70 | if boundary is None: 71 | boundary = choose_boundary() 72 | 73 | for field in iter_field_objects(fields): 74 | body.write(b('--%s\r\n' % (boundary))) 75 | 76 | writer(body).write(field.render_headers()) 77 | data = field.data 78 | 79 | if isinstance(data, int): 80 | data = str(data) # Backwards compatibility 81 | 82 | if isinstance(data, six.text_type): 83 | writer(body).write(data) 84 | else: 85 | body.write(data) 86 | 87 | body.write(b'\r\n') 88 | 89 | body.write(b('--%s--\r\n' % (boundary))) 90 | 91 | content_type = str('multipart/form-data; boundary=%s' % boundary) 92 | 93 | return body.getvalue(), content_type 94 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/chardetect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Script which takes one or more file paths and reports on their detected 4 | encodings 5 | 6 | Example:: 7 | 8 | % chardetect somefile someotherfile 9 | somefile: windows-1252 with confidence 0.5 10 | someotherfile: ascii with confidence 1.0 11 | 12 | If no paths are provided, it takes its input from stdin. 13 | 14 | """ 15 | 16 | from __future__ import absolute_import, print_function, unicode_literals 17 | 18 | import argparse 19 | import sys 20 | from io import open 21 | 22 | from chardet import __version__ 23 | from chardet.universaldetector import UniversalDetector 24 | 25 | 26 | def description_of(lines, name='stdin'): 27 | """ 28 | Return a string describing the probable encoding of a file or 29 | list of strings. 30 | 31 | :param lines: The lines to get the encoding of. 32 | :type lines: Iterable of bytes 33 | :param name: Name of file or collection of lines 34 | :type name: str 35 | """ 36 | u = UniversalDetector() 37 | for line in lines: 38 | u.feed(line) 39 | u.close() 40 | result = u.result 41 | if result['encoding']: 42 | return '{0}: {1} with confidence {2}'.format(name, result['encoding'], 43 | result['confidence']) 44 | else: 45 | return '{0}: no result'.format(name) 46 | 47 | 48 | def main(argv=None): 49 | ''' 50 | Handles command line arguments and gets things started. 51 | 52 | :param argv: List of arguments, as if specified on the command-line. 53 | If None, ``sys.argv[1:]`` is used instead. 54 | :type argv: list of str 55 | ''' 56 | # Get command line arguments 57 | parser = argparse.ArgumentParser( 58 | description="Takes one or more file paths and reports their detected \ 59 | encodings", 60 | formatter_class=argparse.ArgumentDefaultsHelpFormatter, 61 | conflict_handler='resolve') 62 | parser.add_argument('input', 63 | help='File whose encoding we would like to determine.', 64 | type=argparse.FileType('rb'), nargs='*', 65 | default=[sys.stdin]) 66 | parser.add_argument('--version', action='version', 67 | version='%(prog)s {0}'.format(__version__)) 68 | args = parser.parse_args(argv) 69 | 70 | for f in args.input: 71 | if f.isatty(): 72 | print("You are running chardetect interactively. Press " + 73 | "CTRL-D twice at the start of a blank line to signal the " + 74 | "end of your input. If you want help, run chardetect " + 75 | "--help\n", file=sys.stderr) 76 | print(description_of(f, f.name)) 77 | 78 | 79 | if __name__ == '__main__': 80 | main() 81 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/utf8prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .charsetprober import CharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .mbcssm import UTF8SMModel 32 | 33 | ONE_CHAR_PROB = 0.5 34 | 35 | 36 | class UTF8Prober(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = CodingStateMachine(UTF8SMModel) 40 | self.reset() 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | self._mCodingSM.reset() 45 | self._mNumOfMBChar = 0 46 | 47 | def get_charset_name(self): 48 | return "utf-8" 49 | 50 | def feed(self, aBuf): 51 | for c in aBuf: 52 | codingState = self._mCodingSM.next_state(c) 53 | if codingState == constants.eError: 54 | self._mState = constants.eNotMe 55 | break 56 | elif codingState == constants.eItsMe: 57 | self._mState = constants.eFoundIt 58 | break 59 | elif codingState == constants.eStart: 60 | if self._mCodingSM.get_current_charlen() >= 2: 61 | self._mNumOfMBChar += 1 62 | 63 | if self.get_state() == constants.eDetecting: 64 | if self.get_confidence() > constants.SHORTCUT_THRESHOLD: 65 | self._mState = constants.eFoundIt 66 | 67 | return self.get_state() 68 | 69 | def get_confidence(self): 70 | unlike = 0.99 71 | if self._mNumOfMBChar < 6: 72 | for i in range(0, self._mNumOfMBChar): 73 | unlike = unlike * ONE_CHAR_PROB 74 | return 1.0 - unlike 75 | else: 76 | return unlike 77 | -------------------------------------------------------------------------------- /thirdparty/requests/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.exceptions 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | This module contains the set of Requests' exceptions. 8 | 9 | """ 10 | from .packages.urllib3.exceptions import HTTPError as BaseHTTPError 11 | 12 | 13 | class RequestException(IOError): 14 | """There was an ambiguous exception that occurred while handling your 15 | request.""" 16 | 17 | def __init__(self, *args, **kwargs): 18 | """ 19 | Initialize RequestException with `request` and `response` objects. 20 | """ 21 | response = kwargs.pop('response', None) 22 | self.response = response 23 | self.request = kwargs.pop('request', None) 24 | if (response is not None and not self.request and 25 | hasattr(response, 'request')): 26 | self.request = self.response.request 27 | super(RequestException, self).__init__(*args, **kwargs) 28 | 29 | 30 | class HTTPError(RequestException): 31 | """An HTTP error occurred.""" 32 | 33 | 34 | class ConnectionError(RequestException): 35 | """A Connection error occurred.""" 36 | 37 | 38 | class ProxyError(ConnectionError): 39 | """A proxy error occurred.""" 40 | 41 | 42 | class SSLError(ConnectionError): 43 | """An SSL error occurred.""" 44 | 45 | 46 | class Timeout(RequestException): 47 | """The request timed out. 48 | 49 | Catching this error will catch both 50 | :exc:`~requests.exceptions.ConnectTimeout` and 51 | :exc:`~requests.exceptions.ReadTimeout` errors. 52 | """ 53 | 54 | 55 | class ConnectTimeout(ConnectionError, Timeout): 56 | """The request timed out while trying to connect to the remote server. 57 | 58 | Requests that produced this error are safe to retry. 59 | """ 60 | 61 | 62 | class ReadTimeout(Timeout): 63 | """The server did not send any data in the allotted amount of time.""" 64 | 65 | 66 | class URLRequired(RequestException): 67 | """A valid URL is required to make a request.""" 68 | 69 | 70 | class TooManyRedirects(RequestException): 71 | """Too many redirects.""" 72 | 73 | 74 | class MissingSchema(RequestException, ValueError): 75 | """The URL schema (e.g. http or https) is missing.""" 76 | 77 | 78 | class InvalidSchema(RequestException, ValueError): 79 | """See defaults.py for valid schemas.""" 80 | 81 | 82 | class InvalidURL(RequestException, ValueError): 83 | """ The URL provided was somehow invalid. """ 84 | 85 | 86 | class ChunkedEncodingError(RequestException): 87 | """The server declared chunked encoding but sent an invalid chunk.""" 88 | 89 | 90 | class ContentDecodingError(RequestException, BaseHTTPError): 91 | """Failed to decode response content""" 92 | 93 | 94 | class StreamConsumedError(RequestException, TypeError): 95 | """The content for this response was already consumed""" 96 | 97 | 98 | class RetryError(RequestException): 99 | """Custom retries logic failed""" 100 | -------------------------------------------------------------------------------- /thirdparty/colorama/ansi.py: -------------------------------------------------------------------------------- 1 | # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. 2 | ''' 3 | This module generates ANSI character codes to printing colors to terminals. 4 | See: http://en.wikipedia.org/wiki/ANSI_escape_code 5 | ''' 6 | 7 | CSI = '\033[' 8 | OSC = '\033]' 9 | BEL = '\007' 10 | 11 | 12 | def code_to_chars(code): 13 | return CSI + str(code) + 'm' 14 | 15 | def set_title(title): 16 | return OSC + '2;' + title + BEL 17 | 18 | def clear_screen(mode=2): 19 | return CSI + str(mode) + 'J' 20 | 21 | def clear_line(mode=2): 22 | return CSI + str(mode) + 'K' 23 | 24 | 25 | class AnsiCodes(object): 26 | def __init__(self): 27 | # the subclasses declare class attributes which are numbers. 28 | # Upon instantiation we define instance attributes, which are the same 29 | # as the class attributes but wrapped with the ANSI escape sequence 30 | for name in dir(self): 31 | if not name.startswith('_'): 32 | value = getattr(self, name) 33 | setattr(self, name, code_to_chars(value)) 34 | 35 | 36 | class AnsiCursor(object): 37 | def UP(self, n=1): 38 | return CSI + str(n) + 'A' 39 | def DOWN(self, n=1): 40 | return CSI + str(n) + 'B' 41 | def FORWARD(self, n=1): 42 | return CSI + str(n) + 'C' 43 | def BACK(self, n=1): 44 | return CSI + str(n) + 'D' 45 | def POS(self, x=1, y=1): 46 | return CSI + str(y) + ';' + str(x) + 'H' 47 | 48 | 49 | class AnsiFore(AnsiCodes): 50 | BLACK = 30 51 | RED = 31 52 | GREEN = 32 53 | YELLOW = 33 54 | BLUE = 34 55 | MAGENTA = 35 56 | CYAN = 36 57 | WHITE = 37 58 | RESET = 39 59 | 60 | # These are fairly well supported, but not part of the standard. 61 | LIGHTBLACK_EX = 90 62 | LIGHTRED_EX = 91 63 | LIGHTGREEN_EX = 92 64 | LIGHTYELLOW_EX = 93 65 | LIGHTBLUE_EX = 94 66 | LIGHTMAGENTA_EX = 95 67 | LIGHTCYAN_EX = 96 68 | LIGHTWHITE_EX = 97 69 | 70 | 71 | class AnsiBack(AnsiCodes): 72 | BLACK = 40 73 | RED = 41 74 | GREEN = 42 75 | YELLOW = 43 76 | BLUE = 44 77 | MAGENTA = 45 78 | CYAN = 46 79 | WHITE = 47 80 | RESET = 49 81 | 82 | # These are fairly well supported, but not part of the standard. 83 | LIGHTBLACK_EX = 100 84 | LIGHTRED_EX = 101 85 | LIGHTGREEN_EX = 102 86 | LIGHTYELLOW_EX = 103 87 | LIGHTBLUE_EX = 104 88 | LIGHTMAGENTA_EX = 105 89 | LIGHTCYAN_EX = 106 90 | LIGHTWHITE_EX = 107 91 | 92 | 93 | class AnsiStyle(AnsiCodes): 94 | BRIGHT = 1 95 | DIM = 2 96 | NORMAL = 22 97 | RESET_ALL = 0 98 | 99 | Fore = AnsiFore() 100 | Back = AnsiBack() 101 | Style = AnsiStyle() 102 | Cursor = AnsiCursor() 103 | -------------------------------------------------------------------------------- /thirdparty/requests/structures.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.structures 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | Data structures that power Requests. 8 | 9 | """ 10 | 11 | import collections 12 | 13 | 14 | class CaseInsensitiveDict(collections.MutableMapping): 15 | """ 16 | A case-insensitive ``dict``-like object. 17 | 18 | Implements all methods and operations of 19 | ``collections.MutableMapping`` as well as dict's ``copy``. Also 20 | provides ``lower_items``. 21 | 22 | All keys are expected to be strings. The structure remembers the 23 | case of the last key to be set, and ``iter(instance)``, 24 | ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` 25 | will contain case-sensitive keys. However, querying and contains 26 | testing is case insensitive:: 27 | 28 | cid = CaseInsensitiveDict() 29 | cid['Accept'] = 'application/json' 30 | cid['aCCEPT'] == 'application/json' # True 31 | list(cid) == ['Accept'] # True 32 | 33 | For example, ``headers['content-encoding']`` will return the 34 | value of a ``'Content-Encoding'`` response header, regardless 35 | of how the header name was originally stored. 36 | 37 | If the constructor, ``.update``, or equality comparison 38 | operations are given keys that have equal ``.lower()``s, the 39 | behavior is undefined. 40 | 41 | """ 42 | def __init__(self, data=None, **kwargs): 43 | self._store = dict() 44 | if data is None: 45 | data = {} 46 | self.update(data, **kwargs) 47 | 48 | def __setitem__(self, key, value): 49 | # Use the lowercased key for lookups, but store the actual 50 | # key alongside the value. 51 | self._store[key.lower()] = (key, value) 52 | 53 | def __getitem__(self, key): 54 | return self._store[key.lower()][1] 55 | 56 | def __delitem__(self, key): 57 | del self._store[key.lower()] 58 | 59 | def __iter__(self): 60 | return (casedkey for casedkey, mappedvalue in self._store.values()) 61 | 62 | def __len__(self): 63 | return len(self._store) 64 | 65 | def lower_items(self): 66 | """Like iteritems(), but with all lowercase keys.""" 67 | return ( 68 | (lowerkey, keyval[1]) 69 | for (lowerkey, keyval) 70 | in self._store.items() 71 | ) 72 | 73 | def __eq__(self, other): 74 | if isinstance(other, collections.Mapping): 75 | other = CaseInsensitiveDict(other) 76 | else: 77 | return NotImplemented 78 | # Compare insensitively 79 | return dict(self.lower_items()) == dict(other.lower_items()) 80 | 81 | # Copy is required 82 | def copy(self): 83 | return CaseInsensitiveDict(self._store.values()) 84 | 85 | def __repr__(self): 86 | return str(dict(self.items())) 87 | 88 | class LookupDict(dict): 89 | """Dictionary lookup object.""" 90 | 91 | def __init__(self, name=None): 92 | self.name = name 93 | super(LookupDict, self).__init__() 94 | 95 | def __repr__(self): 96 | return '' % (self.name) 97 | 98 | def __getitem__(self, key): 99 | # We allow fall-through here, so values default to None 100 | 101 | return self.__dict__.get(key, None) 102 | 103 | def get(self, key, default=None): 104 | return self.__dict__.get(key, default) 105 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/escprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, 30 | ISO2022KRSMModel) 31 | from .charsetprober import CharSetProber 32 | from .codingstatemachine import CodingStateMachine 33 | from .compat import wrap_ord 34 | 35 | 36 | class EscCharSetProber(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = [ 40 | CodingStateMachine(HZSMModel), 41 | CodingStateMachine(ISO2022CNSMModel), 42 | CodingStateMachine(ISO2022JPSMModel), 43 | CodingStateMachine(ISO2022KRSMModel) 44 | ] 45 | self.reset() 46 | 47 | def reset(self): 48 | CharSetProber.reset(self) 49 | for codingSM in self._mCodingSM: 50 | if not codingSM: 51 | continue 52 | codingSM.active = True 53 | codingSM.reset() 54 | self._mActiveSM = len(self._mCodingSM) 55 | self._mDetectedCharset = None 56 | 57 | def get_charset_name(self): 58 | return self._mDetectedCharset 59 | 60 | def get_confidence(self): 61 | if self._mDetectedCharset: 62 | return 0.99 63 | else: 64 | return 0.00 65 | 66 | def feed(self, aBuf): 67 | for c in aBuf: 68 | # PY3K: aBuf is a byte array, so c is an int, not a byte 69 | for codingSM in self._mCodingSM: 70 | if not codingSM: 71 | continue 72 | if not codingSM.active: 73 | continue 74 | codingState = codingSM.next_state(wrap_ord(c)) 75 | if codingState == constants.eError: 76 | codingSM.active = False 77 | self._mActiveSM -= 1 78 | if self._mActiveSM <= 0: 79 | self._mState = constants.eNotMe 80 | return self.get_state() 81 | elif codingState == constants.eItsMe: 82 | self._mState = constants.eFoundIt 83 | self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 84 | return self.get_state() 85 | 86 | return self.get_state() 87 | -------------------------------------------------------------------------------- /thirdparty/requests/status_codes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .structures import LookupDict 4 | 5 | _codes = { 6 | 7 | # Informational. 8 | 100: ('continue',), 9 | 101: ('switching_protocols',), 10 | 102: ('processing',), 11 | 103: ('checkpoint',), 12 | 122: ('uri_too_long', 'request_uri_too_long'), 13 | 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), 14 | 201: ('created',), 15 | 202: ('accepted',), 16 | 203: ('non_authoritative_info', 'non_authoritative_information'), 17 | 204: ('no_content',), 18 | 205: ('reset_content', 'reset'), 19 | 206: ('partial_content', 'partial'), 20 | 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), 21 | 208: ('already_reported',), 22 | 226: ('im_used',), 23 | 24 | # Redirection. 25 | 300: ('multiple_choices',), 26 | 301: ('moved_permanently', 'moved', '\\o-'), 27 | 302: ('found',), 28 | 303: ('see_other', 'other'), 29 | 304: ('not_modified',), 30 | 305: ('use_proxy',), 31 | 306: ('switch_proxy',), 32 | 307: ('temporary_redirect', 'temporary_moved', 'temporary'), 33 | 308: ('permanent_redirect', 34 | 'resume_incomplete', 'resume',), # These 2 to be removed in 3.0 35 | 36 | # Client Error. 37 | 400: ('bad_request', 'bad'), 38 | 401: ('unauthorized',), 39 | 402: ('payment_required', 'payment'), 40 | 403: ('forbidden',), 41 | 404: ('not_found', '-o-'), 42 | 405: ('method_not_allowed', 'not_allowed'), 43 | 406: ('not_acceptable',), 44 | 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), 45 | 408: ('request_timeout', 'timeout'), 46 | 409: ('conflict',), 47 | 410: ('gone',), 48 | 411: ('length_required',), 49 | 412: ('precondition_failed', 'precondition'), 50 | 413: ('request_entity_too_large',), 51 | 414: ('request_uri_too_large',), 52 | 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), 53 | 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 54 | 417: ('expectation_failed',), 55 | 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), 56 | 422: ('unprocessable_entity', 'unprocessable'), 57 | 423: ('locked',), 58 | 424: ('failed_dependency', 'dependency'), 59 | 425: ('unordered_collection', 'unordered'), 60 | 426: ('upgrade_required', 'upgrade'), 61 | 428: ('precondition_required', 'precondition'), 62 | 429: ('too_many_requests', 'too_many'), 63 | 431: ('header_fields_too_large', 'fields_too_large'), 64 | 444: ('no_response', 'none'), 65 | 449: ('retry_with', 'retry'), 66 | 450: ('blocked_by_windows_parental_controls', 'parental_controls'), 67 | 451: ('unavailable_for_legal_reasons', 'legal_reasons'), 68 | 499: ('client_closed_request',), 69 | 70 | # Server Error. 71 | 500: ('internal_server_error', 'server_error', '/o\\', '✗'), 72 | 501: ('not_implemented',), 73 | 502: ('bad_gateway',), 74 | 503: ('service_unavailable', 'unavailable'), 75 | 504: ('gateway_timeout',), 76 | 505: ('http_version_not_supported', 'http_version'), 77 | 506: ('variant_also_negotiates',), 78 | 507: ('insufficient_storage',), 79 | 509: ('bandwidth_limit_exceeded', 'bandwidth'), 80 | 510: ('not_extended',), 81 | } 82 | 83 | codes = LookupDict(name='status_codes') 84 | 85 | for code, titles in _codes.items(): 86 | for title in titles: 87 | setattr(codes, title, code) 88 | if not title.startswith('\\'): 89 | setattr(codes, title.upper(), code) 90 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/sbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetgroupprober import CharSetGroupProber 30 | from .sbcharsetprober import SingleByteCharSetProber 31 | from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel, 32 | Latin5CyrillicModel, MacCyrillicModel, 33 | Ibm866Model, Ibm855Model) 34 | from .langgreekmodel import Latin7GreekModel, Win1253GreekModel 35 | from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel 36 | from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel 37 | from .langthaimodel import TIS620ThaiModel 38 | from .langhebrewmodel import Win1255HebrewModel 39 | from .hebrewprober import HebrewProber 40 | 41 | 42 | class SBCSGroupProber(CharSetGroupProber): 43 | def __init__(self): 44 | CharSetGroupProber.__init__(self) 45 | self._mProbers = [ 46 | SingleByteCharSetProber(Win1251CyrillicModel), 47 | SingleByteCharSetProber(Koi8rModel), 48 | SingleByteCharSetProber(Latin5CyrillicModel), 49 | SingleByteCharSetProber(MacCyrillicModel), 50 | SingleByteCharSetProber(Ibm866Model), 51 | SingleByteCharSetProber(Ibm855Model), 52 | SingleByteCharSetProber(Latin7GreekModel), 53 | SingleByteCharSetProber(Win1253GreekModel), 54 | SingleByteCharSetProber(Latin5BulgarianModel), 55 | SingleByteCharSetProber(Win1251BulgarianModel), 56 | SingleByteCharSetProber(Latin2HungarianModel), 57 | SingleByteCharSetProber(Win1250HungarianModel), 58 | SingleByteCharSetProber(TIS620ThaiModel), 59 | ] 60 | hebrewProber = HebrewProber() 61 | logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, 62 | False, hebrewProber) 63 | visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True, 64 | hebrewProber) 65 | hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber) 66 | self._mProbers.extend([hebrewProber, logicalHebrewProber, 67 | visualHebrewProber]) 68 | 69 | self.reset() 70 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/mbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | import sys 31 | from . import constants 32 | from .charsetprober import CharSetProber 33 | 34 | 35 | class MultiByteCharSetProber(CharSetProber): 36 | def __init__(self): 37 | CharSetProber.__init__(self) 38 | self._mDistributionAnalyzer = None 39 | self._mCodingSM = None 40 | self._mLastChar = [0, 0] 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | if self._mCodingSM: 45 | self._mCodingSM.reset() 46 | if self._mDistributionAnalyzer: 47 | self._mDistributionAnalyzer.reset() 48 | self._mLastChar = [0, 0] 49 | 50 | def get_charset_name(self): 51 | pass 52 | 53 | def feed(self, aBuf): 54 | aLen = len(aBuf) 55 | for i in range(0, aLen): 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 72 | else: 73 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 74 | charLen) 75 | 76 | self._mLastChar[0] = aBuf[aLen - 1] 77 | 78 | if self.get_state() == constants.eDetecting: 79 | if (self._mDistributionAnalyzer.got_enough_data() and 80 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 81 | self._mState = constants.eFoundIt 82 | 83 | return self.get_state() 84 | 85 | def get_confidence(self): 86 | return self._mDistributionAnalyzer.get_confidence() 87 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/util/connection.py: -------------------------------------------------------------------------------- 1 | import socket 2 | try: 3 | from select import poll, POLLIN 4 | except ImportError: # `poll` doesn't exist on OSX and other platforms 5 | poll = False 6 | try: 7 | from select import select 8 | except ImportError: # `select` doesn't exist on AppEngine. 9 | select = False 10 | 11 | 12 | def is_connection_dropped(conn): # Platform-specific 13 | """ 14 | Returns True if the connection is dropped and should be closed. 15 | 16 | :param conn: 17 | :class:`httplib.HTTPConnection` object. 18 | 19 | Note: For platforms like AppEngine, this will always return ``False`` to 20 | let the platform handle connection recycling transparently for us. 21 | """ 22 | sock = getattr(conn, 'sock', False) 23 | if sock is False: # Platform-specific: AppEngine 24 | return False 25 | if sock is None: # Connection already closed (such as by httplib). 26 | return True 27 | 28 | if not poll: 29 | if not select: # Platform-specific: AppEngine 30 | return False 31 | 32 | try: 33 | return select([sock], [], [], 0.0)[0] 34 | except socket.error: 35 | return True 36 | 37 | # This version is better on platforms that support it. 38 | p = poll() 39 | p.register(sock, POLLIN) 40 | for (fno, ev) in p.poll(0.0): 41 | if fno == sock.fileno(): 42 | # Either data is buffered (bad), or the connection is dropped. 43 | return True 44 | 45 | 46 | # This function is copied from socket.py in the Python 2.7 standard 47 | # library scanner suite. Added to its signature is only `socket_options`. 48 | def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, 49 | source_address=None, socket_options=None): 50 | """Connect to *address* and return the socket object. 51 | 52 | Convenience function. Connect to *address* (a 2-tuple ``(host, 53 | port)``) and return the socket object. Passing the optional 54 | *timeout* parameter will set the timeout on the socket instance 55 | before attempting to connect. If no *timeout* is supplied, the 56 | global default timeout setting returned by :func:`getdefaulttimeout` 57 | is used. If *source_address* is set it must be a tuple of (host, port) 58 | for the socket to bind as a source address before making the connection. 59 | An host of '' or port 0 tells the OS to use the default. 60 | """ 61 | 62 | host, port = address 63 | err = None 64 | for res in socket.getaddrinfo(host, port, 0, socket.SOCK_STREAM): 65 | af, socktype, proto, canonname, sa = res 66 | sock = None 67 | try: 68 | sock = socket.socket(af, socktype, proto) 69 | 70 | # If provided, set socket level options before connecting. 71 | # This is the only addition urllib3 makes to this function. 72 | _set_socket_options(sock, socket_options) 73 | 74 | if timeout is not socket._GLOBAL_DEFAULT_TIMEOUT: 75 | sock.settimeout(timeout) 76 | if source_address: 77 | sock.bind(source_address) 78 | sock.connect(sa) 79 | return sock 80 | 81 | except socket.error as _: 82 | err = _ 83 | if sock is not None: 84 | sock.close() 85 | sock = None 86 | 87 | if err is not None: 88 | raise err 89 | else: 90 | raise socket.error("getaddrinfo returns an empty list") 91 | 92 | 93 | def _set_socket_options(sock, options): 94 | if options is None: 95 | return 96 | 97 | for opt in options: 98 | sock.setsockopt(*opt) 99 | -------------------------------------------------------------------------------- /lib/utils/TerminalSize.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import os 20 | import shlex 21 | import struct 22 | import platform 23 | import subprocess 24 | 25 | 26 | def get_terminal_size(): 27 | """ getTerminalSize() 28 | - get width and height of console 29 | - works on linux,os x,windows,cygwin(windows) 30 | originally retrieved from: 31 | http://stackoverflow.com/questions/566746/how-to-get-console-window-width-in-python 32 | """ 33 | current_os = platform.system() 34 | tuple_xy = None 35 | if current_os == 'Windows': 36 | tuple_xy = _get_terminal_size_windows() 37 | if tuple_xy is None: 38 | tuple_xy = _get_terminal_size_tput() 39 | # needed for window's python in cygwin's xterm! 40 | if current_os in ['Linux', 'Darwin', 'FreeBSD'] or current_os.startswith('CYGWIN'): 41 | tuple_xy = _get_terminal_size_linux() 42 | if tuple_xy is None: 43 | tuple_xy = (80, 25) # default value 44 | return tuple_xy 45 | 46 | 47 | def _get_terminal_size_windows(): 48 | try: 49 | from ctypes import windll, create_string_buffer 50 | # stdin handle is -10 51 | # stdout handle is -11 52 | # stderr handle is -12 53 | h = windll.kernel32.GetStdHandle(-12) 54 | csbi = create_string_buffer(22) 55 | res = windll.kernel32.GetConsoleScreenBufferInfo(h, csbi) 56 | if res: 57 | (bufx, bufy, curx, cury, wattr, 58 | left, top, right, bottom, 59 | maxx, maxy) = struct.unpack("hhhhHhhhhhh", csbi.raw) 60 | sizex = right - left + 1 61 | sizey = bottom - top + 1 62 | return sizex, sizey 63 | except: 64 | pass 65 | 66 | 67 | def _get_terminal_size_tput(): 68 | # get terminal width 69 | # src: http://stackoverflow.com/questions/263890/how-do-i-find-the-width-height-of-a-terminal-window 70 | try: 71 | cols = int(subprocess.check_call(shlex.split('tput cols'))) 72 | rows = int(subprocess.check_call(shlex.split('tput lines'))) 73 | return (cols, rows) 74 | except: 75 | pass 76 | 77 | 78 | def _get_terminal_size_linux(): 79 | def ioctl_GWINSZ(fd): 80 | try: 81 | import fcntl 82 | import termios 83 | cr = struct.unpack('hh', 84 | fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) 85 | return cr 86 | except: 87 | pass 88 | cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 89 | if not cr: 90 | try: 91 | fd = os.open(os.ctermid(), os.O_RDONLY) 92 | cr = ioctl_GWINSZ(fd) 93 | os.close(fd) 94 | except: 95 | pass 96 | if not cr: 97 | try: 98 | cr = (os.environ['LINES'], os.environ['COLUMNS']) 99 | except: 100 | return None 101 | return int(cr[1]), int(cr[0]) 102 | 103 | if __name__ == "__main__": 104 | sizex, sizey = get_terminal_size() 105 | print('width =', sizex, 'height =', sizey) 106 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/eucjpprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from . import constants 30 | from .mbcharsetprober import MultiByteCharSetProber 31 | from .codingstatemachine import CodingStateMachine 32 | from .chardistribution import EUCJPDistributionAnalysis 33 | from .jpcntx import EUCJPContextAnalysis 34 | from .mbcssm import EUCJPSMModel 35 | 36 | 37 | class EUCJPProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(EUCJPSMModel) 41 | self._mDistributionAnalyzer = EUCJPDistributionAnalysis() 42 | self._mContextAnalyzer = EUCJPContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "EUC-JP" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mContextAnalyzer.feed(self._mLastChar, charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) 75 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 76 | charLen) 77 | 78 | self._mLastChar[0] = aBuf[aLen - 1] 79 | 80 | if self.get_state() == constants.eDetecting: 81 | if (self._mContextAnalyzer.got_enough_data() and 82 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 83 | self._mState = constants.eFoundIt 84 | 85 | return self.get_state() 86 | 87 | def get_confidence(self): 88 | contxtCf = self._mContextAnalyzer.get_confidence() 89 | distribCf = self._mDistributionAnalyzer.get_confidence() 90 | return max(contxtCf, distribCf) 91 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/sjisprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from .mbcharsetprober import MultiByteCharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .chardistribution import SJISDistributionAnalysis 32 | from .jpcntx import SJISContextAnalysis 33 | from .mbcssm import SJISSMModel 34 | from . import constants 35 | 36 | 37 | class SJISProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(SJISSMModel) 41 | self._mDistributionAnalyzer = SJISDistributionAnalysis() 42 | self._mContextAnalyzer = SJISContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return self._mContextAnalyzer.get_charset_name() 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | codingState = self._mCodingSM.next_state(aBuf[i]) 56 | if codingState == constants.eError: 57 | if constants._debug: 58 | sys.stderr.write(self.get_charset_name() 59 | + ' prober hit error at byte ' + str(i) 60 | + '\n') 61 | self._mState = constants.eNotMe 62 | break 63 | elif codingState == constants.eItsMe: 64 | self._mState = constants.eFoundIt 65 | break 66 | elif codingState == constants.eStart: 67 | charLen = self._mCodingSM.get_current_charlen() 68 | if i == 0: 69 | self._mLastChar[1] = aBuf[0] 70 | self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:], 71 | charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3 75 | - charLen], charLen) 76 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 77 | charLen) 78 | 79 | self._mLastChar[0] = aBuf[aLen - 1] 80 | 81 | if self.get_state() == constants.eDetecting: 82 | if (self._mContextAnalyzer.got_enough_data() and 83 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 84 | self._mState = constants.eFoundIt 85 | 86 | return self.get_state() 87 | 88 | def get_confidence(self): 89 | contxtCf = self._mContextAnalyzer.get_confidence() 90 | distribCf = self._mDistributionAnalyzer.get_confidence() 91 | return max(contxtCf, distribCf) 92 | -------------------------------------------------------------------------------- /thirdparty/sqlmap/DynamicContentParser.py: -------------------------------------------------------------------------------- 1 | from difflib import SequenceMatcher 2 | import re 3 | 4 | 5 | class DynamicContentParser: 6 | def __init__(self, requester, path, firstPage, secondPage, comparisons=2): 7 | self.DYNAMICITY_MARK_LENGTH = 32 8 | self.UPPER_RATIO_BOUND = 0.98 9 | self.requester = requester 10 | self.keyCallback = path 11 | self.comparisons = comparisons 12 | self.dynamicMarks = [] 13 | self.seqMatcher = SequenceMatcher() 14 | self.generateDynamicMarks(firstPage, secondPage) 15 | 16 | def generateDynamicMarks(self, firstPage, secondPage): 17 | if any(page is None for page in (firstPage, secondPage)): 18 | # No content 19 | return 20 | 21 | self.seqMatcher.set_seq1(firstPage) 22 | self.seqMatcher.set_seq2(secondPage) 23 | ratio = self.seqMatcher.quick_ratio() 24 | # In case of an intolerable difference turn on dynamicity removal engine 25 | if ratio <= self.UPPER_RATIO_BOUND: 26 | self.dynamicMarks += self.findDynamicContent(firstPage, secondPage) 27 | for i in range(self.comparisons): 28 | response = self.requester.request(self.keyCallback) 29 | secondPage = response.body 30 | self.dynamicMarks += self.findDynamicContent(firstPage, secondPage) 31 | self.cleanPage = self.removeDynamicContent(firstPage, self.dynamicMarks) 32 | self.seqMatcher.set_seq1(self.cleanPage) 33 | self.seqMatcher.set_seq2(self.removeDynamicContent(secondPage, self.dynamicMarks)) 34 | ratio = self.seqMatcher.quick_ratio() 35 | else: 36 | self.cleanPage = firstPage 37 | self.comparisonRatio = ratio 38 | 39 | def compareTo(self, page): 40 | seqMatcher = SequenceMatcher() 41 | seqMatcher.set_seq1(self.cleanPage) 42 | seqMatcher.set_seq2(self.removeDynamicContent(page, self.dynamicMarks)) 43 | ratio = seqMatcher.quick_ratio() 44 | return ratio 45 | 46 | def findDynamicContent(self, firstPage, secondPage): 47 | dynamicMarks = [] 48 | 49 | blocks = list(SequenceMatcher(None, firstPage, secondPage).get_matching_blocks()) 50 | 51 | # Removing too small matching blocks 52 | for block in blocks[:]: 53 | (_, _, length) = block 54 | 55 | if length <= self.DYNAMICITY_MARK_LENGTH: 56 | blocks.remove(block) 57 | 58 | # Making of dynamic markings based on prefix/suffix principle 59 | if len(blocks) > 0: 60 | blocks.insert(0, None) 61 | blocks.append(None) 62 | 63 | for i in range(len(blocks) - 1): 64 | prefix = firstPage[blocks[i][0]:blocks[i][0] + blocks[i][2]] if blocks[i] else None 65 | suffix = firstPage[blocks[i + 1][0]:blocks[i + 1][0] + blocks[i + 1][2]] if blocks[i + 1] else None 66 | 67 | if prefix is None and blocks[i + 1][0] == 0: 68 | continue 69 | 70 | if suffix is None and (blocks[i][0] + blocks[i][2] >= len(firstPage)): 71 | continue 72 | 73 | dynamicMarks.append((re.escape(prefix[int(-self.DYNAMICITY_MARK_LENGTH / 2):]) if prefix else None, 74 | re.escape(suffix[:int(self.DYNAMICITY_MARK_LENGTH / 2)]) if suffix else None)) 75 | 76 | return dynamicMarks 77 | 78 | def removeDynamicContent(self, page, dynamicMarks): 79 | """ 80 | Removing dynamic content from supplied page basing removal on 81 | precalculated dynamic markings 82 | """ 83 | if page: 84 | for item in dynamicMarks: 85 | prefix, suffix = item 86 | 87 | if prefix is None and suffix is None: 88 | continue 89 | elif prefix is None: 90 | page = re.sub(r'(?s)^.+%s' % suffix, suffix, str(page)) 91 | elif suffix is None: 92 | page = re.sub(r'(?s)%s.+$' % prefix, prefix, str(page)) 93 | else: 94 | page = re.sub(r'(?s)%s.+%s' % (prefix, suffix), '%s%s' % (prefix, suffix), str(page)) 95 | 96 | return page 97 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/charsetgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | import sys 30 | from .charsetprober import CharSetProber 31 | 32 | 33 | class CharSetGroupProber(CharSetProber): 34 | def __init__(self): 35 | CharSetProber.__init__(self) 36 | self._mActiveNum = 0 37 | self._mProbers = [] 38 | self._mBestGuessProber = None 39 | 40 | def reset(self): 41 | CharSetProber.reset(self) 42 | self._mActiveNum = 0 43 | for prober in self._mProbers: 44 | if prober: 45 | prober.reset() 46 | prober.active = True 47 | self._mActiveNum += 1 48 | self._mBestGuessProber = None 49 | 50 | def get_charset_name(self): 51 | if not self._mBestGuessProber: 52 | self.get_confidence() 53 | if not self._mBestGuessProber: 54 | return None 55 | # self._mBestGuessProber = self._mProbers[0] 56 | return self._mBestGuessProber.get_charset_name() 57 | 58 | def feed(self, aBuf): 59 | for prober in self._mProbers: 60 | if not prober: 61 | continue 62 | if not prober.active: 63 | continue 64 | st = prober.feed(aBuf) 65 | if not st: 66 | continue 67 | if st == constants.eFoundIt: 68 | self._mBestGuessProber = prober 69 | return self.get_state() 70 | elif st == constants.eNotMe: 71 | prober.active = False 72 | self._mActiveNum -= 1 73 | if self._mActiveNum <= 0: 74 | self._mState = constants.eNotMe 75 | return self.get_state() 76 | return self.get_state() 77 | 78 | def get_confidence(self): 79 | st = self.get_state() 80 | if st == constants.eFoundIt: 81 | return 0.99 82 | elif st == constants.eNotMe: 83 | return 0.01 84 | bestConf = 0.0 85 | self._mBestGuessProber = None 86 | for prober in self._mProbers: 87 | if not prober: 88 | continue 89 | if not prober.active: 90 | if constants._debug: 91 | sys.stderr.write(prober.get_charset_name() 92 | + ' not active\n') 93 | continue 94 | cf = prober.get_confidence() 95 | if constants._debug: 96 | sys.stderr.write('%s confidence = %s\n' % 97 | (prober.get_charset_name(), cf)) 98 | if bestConf < cf: 99 | bestConf = cf 100 | self._mBestGuessProber = prober 101 | if not self._mBestGuessProber: 102 | return 0.0 103 | return bestConf 104 | # else: 105 | # self._mBestGuessProber = self._mProbers[0] 106 | # return self._mBestGuessProber.get_confidence() 107 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py: -------------------------------------------------------------------------------- 1 | """The match_hostname() function from Python 3.3.3, essential when using SSL.""" 2 | 3 | # Note: This file is under the PSF license as the code comes from the python 4 | # stdlib. http://docs.python.org/3/license.html 5 | 6 | import re 7 | 8 | __version__ = '3.4.0.2' 9 | 10 | class CertificateError(ValueError): 11 | pass 12 | 13 | 14 | def _dnsname_match(dn, hostname, max_wildcards=1): 15 | """Matching according to RFC 6125, section 6.4.3 16 | 17 | http://tools.ietf.org/html/rfc6125#section-6.4.3 18 | """ 19 | pats = [] 20 | if not dn: 21 | return False 22 | 23 | # Ported from python3-syntax: 24 | # leftmost, *remainder = dn.split(r'.') 25 | parts = dn.split(r'.') 26 | leftmost = parts[0] 27 | remainder = parts[1:] 28 | 29 | wildcards = leftmost.count('*') 30 | if wildcards > max_wildcards: 31 | # Issue #17980: avoid denials of service by refusing more 32 | # than one wildcard per fragment. A survey of established 33 | # policy among SSL implementations showed it to be a 34 | # reasonable choice. 35 | raise CertificateError( 36 | "too many wildcards in certificate DNS name: " + repr(dn)) 37 | 38 | # speed up common case w/o wildcards 39 | if not wildcards: 40 | return dn.lower() == hostname.lower() 41 | 42 | # RFC 6125, section 6.4.3, subitem 1. 43 | # The client SHOULD NOT attempt to match a presented identifier in which 44 | # the wildcard character comprises a label other than the left-most label. 45 | if leftmost == '*': 46 | # When '*' is a fragment by itself, it matches a non-empty dotless 47 | # fragment. 48 | pats.append('[^.]+') 49 | elif leftmost.startswith('xn--') or hostname.startswith('xn--'): 50 | # RFC 6125, section 6.4.3, subitem 3. 51 | # The client SHOULD NOT attempt to match a presented identifier 52 | # where the wildcard character is embedded within an A-label or 53 | # U-label of an internationalized domain name. 54 | pats.append(re.escape(leftmost)) 55 | else: 56 | # Otherwise, '*' matches any dotless string, e.g. www* 57 | pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) 58 | 59 | # add the remaining fragments, ignore any wildcards 60 | for frag in remainder: 61 | pats.append(re.escape(frag)) 62 | 63 | pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 64 | return pat.match(hostname) 65 | 66 | 67 | def match_hostname(cert, hostname): 68 | """Verify that *cert* (in decoded format as returned by 69 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 70 | rules are followed, but IP addresses are not accepted for *hostname*. 71 | 72 | CertificateError is raised on failure. On success, the function 73 | returns nothing. 74 | """ 75 | if not cert: 76 | raise ValueError("empty or no certificate") 77 | dnsnames = [] 78 | san = cert.get('subjectAltName', ()) 79 | for key, value in san: 80 | if key == 'DNS': 81 | if _dnsname_match(value, hostname): 82 | return 83 | dnsnames.append(value) 84 | if not dnsnames: 85 | # The subject is only checked when there is no dNSName entry 86 | # in subjectAltName 87 | for sub in cert.get('subject', ()): 88 | for key, value in sub: 89 | # XXX according to RFC 2818, the most specific Common Name 90 | # must be used. 91 | if key == 'commonName': 92 | if _dnsname_match(value, hostname): 93 | return 94 | dnsnames.append(value) 95 | if len(dnsnames) > 1: 96 | raise CertificateError("hostname %r " 97 | "doesn't match either of %s" 98 | % (hostname, ', '.join(map(repr, dnsnames)))) 99 | elif len(dnsnames) == 1: 100 | raise CertificateError("hostname %r " 101 | "doesn't match %r" 102 | % (hostname, dnsnames[0])) 103 | else: 104 | raise CertificateError("no appropriate commonName or " 105 | "subjectAltName fields were found") 106 | -------------------------------------------------------------------------------- /lib/core/Scanner.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import re 20 | from difflib import SequenceMatcher 21 | 22 | from lib.utils import RandomUtils 23 | from thirdparty.sqlmap import DynamicContentParser 24 | 25 | 26 | class ScannerException(Exception): 27 | pass 28 | 29 | 30 | class Scanner(object): 31 | def __init__(self, requester, testPath=None, suffix=None): 32 | if testPath is None or testPath is "": 33 | self.testPath = RandomUtils.randString() 34 | else: 35 | self.testPath = testPath 36 | self.suffix = suffix if suffix is not None else "" 37 | self.requester = requester 38 | self.tester = None 39 | self.redirectRegExp = None 40 | self.invalidStatus = None 41 | self.dynamicParser = None 42 | self.ratio = 0.98 43 | self.redirectStatusCodes = [301, 302, 307] 44 | self.setup() 45 | 46 | def setup(self): 47 | firstPath = self.testPath + self.suffix 48 | firstResponse = self.requester.request(firstPath) 49 | self.invalidStatus = firstResponse.status 50 | if self.invalidStatus == 404: 51 | # Using the response status code is enough :-} 52 | return 53 | 54 | # look for redirects 55 | secondPath = RandomUtils.randString(omit=self.testPath) + self.suffix 56 | secondResponse = self.requester.request(secondPath) 57 | if firstResponse.status in self.redirectStatusCodes and firstResponse.redirect and secondResponse.redirect: 58 | self.redirectRegExp = self.generateRedirectRegExp(firstResponse.redirect, secondResponse.redirect) 59 | 60 | # Analyze response bodies 61 | self.dynamicParser = DynamicContentParser(self.requester, firstPath, firstResponse.body, secondResponse.body) 62 | baseRatio = float("{0:.2f}".format(self.dynamicParser.comparisonRatio)) # Rounding to 2 decimals 63 | # If response length is small, adjust ratio 64 | if len(firstResponse) < 2000: 65 | baseRatio -= 0.1 66 | if baseRatio < self.ratio: 67 | self.ratio = baseRatio 68 | 69 | def generateRedirectRegExp(self, firstLocation, secondLocation): 70 | if firstLocation is None or secondLocation is None: 71 | return None 72 | sm = SequenceMatcher(None, firstLocation, secondLocation) 73 | marks = [] 74 | for blocks in sm.get_matching_blocks(): 75 | i = blocks[0] 76 | n = blocks[2] 77 | # empty block 78 | if n == 0: 79 | continue 80 | mark = firstLocation[i:i + n] 81 | marks.append(mark) 82 | regexp = "^.*{0}.*$".format(".*".join(map(re.escape, marks))) 83 | return regexp 84 | 85 | def scan(self, path, response): 86 | if self.invalidStatus == 404 and response.status == 404: 87 | return False 88 | if self.invalidStatus != response.status: 89 | return True 90 | 91 | redirectToInvalid = False 92 | if self.redirectRegExp is not None and response.redirect is not None: 93 | redirectToInvalid = re.match(self.redirectRegExp, response.redirect) is not None 94 | # If redirection doesn't match the rule, mark as found 95 | if not redirectToInvalid: 96 | return True 97 | 98 | ratio = self.dynamicParser.compareTo(response.body) 99 | if ratio >= self.ratio: 100 | return False 101 | elif redirectToInvalid and ratio >= (self.ratio - 0.15): 102 | return False 103 | return True 104 | -------------------------------------------------------------------------------- /lib/utils/FileUtils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import os 20 | import os.path 21 | 22 | 23 | class File(object): 24 | def __init__(self, *pathComponents): 25 | self._path = FileUtils.buildPath(*pathComponents) 26 | self.content = None 27 | 28 | @property 29 | def path(self): 30 | return self._path 31 | 32 | @path.setter 33 | def path(self, value): 34 | raise NotImplemented 35 | 36 | def isValid(self): 37 | return FileUtils.isFile(self.path) 38 | 39 | def exists(self): 40 | return FileUtils.exists(self.path) 41 | 42 | def canRead(self): 43 | return FileUtils.canRead(self.path) 44 | 45 | def canWrite(self): 46 | return FileUtils.canWrite(self.path) 47 | 48 | def read(self): 49 | return FileUtils.read(self.path) 50 | 51 | def update(self): 52 | self.content = self.read() 53 | 54 | def content(self): 55 | if not self.content: 56 | self.content = FileUtils.read() 57 | return self.content() 58 | 59 | def getLines(self): 60 | for line in FileUtils.getLines(self.path): 61 | yield line 62 | 63 | def __cmp__(self, other): 64 | if not isinstance(other, File): 65 | raise NotImplemented 66 | return cmp(self.content(), other.content()) 67 | 68 | def __enter__(self): 69 | return self 70 | 71 | def __exit__(self, type, value, tb): 72 | pass 73 | 74 | 75 | class FileUtils(object): 76 | @staticmethod 77 | def buildPath(*pathComponents): 78 | if pathComponents: 79 | path = os.path.join(*pathComponents) 80 | else: 81 | path = '' 82 | return path 83 | 84 | @staticmethod 85 | def exists(fileName): 86 | return os.access(fileName, os.F_OK) 87 | 88 | @staticmethod 89 | def canRead(fileName): 90 | if not os.access(fileName, os.R_OK): 91 | return False 92 | try: 93 | with open(fileName): 94 | pass 95 | except IOError: 96 | return False 97 | return True 98 | 99 | @staticmethod 100 | def canWrite(fileName): 101 | return os.access(fileName, os.W_OK) 102 | 103 | @staticmethod 104 | def read(fileName): 105 | result = '' 106 | with open(fileName, 'r') as fd: 107 | for line in fd.readlines(): 108 | result += line 109 | return result 110 | 111 | @staticmethod 112 | def getLines(fileName): 113 | with open(fileName, 'r', errors="replace") as fd: 114 | return fd.read().splitlines() 115 | 116 | @staticmethod 117 | def isDir(fileName): 118 | return os.path.isdir(fileName) 119 | 120 | @staticmethod 121 | def isFile(fileName): 122 | return os.path.isfile(fileName) 123 | 124 | @staticmethod 125 | def createDirectory(directory): 126 | if not FileUtils.exists(directory): 127 | os.makedirs(directory) 128 | 129 | @staticmethod 130 | def sizeHuman(num): 131 | base = 1024 132 | for x in ['B ', 'KB', 'MB', 'GB']: 133 | if num < base and num > -base: 134 | return "%3.0f%s" % (num, x) 135 | num /= base 136 | return "%3.0f %s" % (num, 'TB') 137 | 138 | @staticmethod 139 | def writeLines(fileName, lines): 140 | content = None 141 | if type(lines) is list: 142 | content = "\n".join(lines) 143 | else: 144 | content = lines 145 | with open(fileName, "w") as f: 146 | f.writelines(content) 147 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/contrib/ntlmpool.py: -------------------------------------------------------------------------------- 1 | """ 2 | NTLM authenticating pool, contributed by erikcederstran 3 | 4 | Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 5 | """ 6 | 7 | try: 8 | from http.client import HTTPSConnection 9 | except ImportError: 10 | from httplib import HTTPSConnection 11 | from logging import getLogger 12 | from ntlm import ntlm 13 | 14 | from urllib3 import HTTPSConnectionPool 15 | 16 | 17 | log = getLogger(__name__) 18 | 19 | 20 | class NTLMConnectionPool(HTTPSConnectionPool): 21 | """ 22 | Implements an NTLM authentication version of an urllib3 connection pool 23 | """ 24 | 25 | scheme = 'https' 26 | 27 | def __init__(self, user, pw, authurl, *args, **kwargs): 28 | """ 29 | authurl is a random URL on the server that is protected by NTLM. 30 | user is the Windows user, probably in the DOMAIN\\username format. 31 | pw is the password for the user. 32 | """ 33 | super(NTLMConnectionPool, self).__init__(*args, **kwargs) 34 | self.authurl = authurl 35 | self.rawuser = user 36 | user_parts = user.split('\\', 1) 37 | self.domain = user_parts[0].upper() 38 | self.user = user_parts[1] 39 | self.pw = pw 40 | 41 | def _new_conn(self): 42 | # Performs the NTLM handshake that secures the connection. The socket 43 | # must be kept open while requests are performed. 44 | self.num_connections += 1 45 | log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % 46 | (self.num_connections, self.host, self.authurl)) 47 | 48 | headers = {} 49 | headers['Connection'] = 'Keep-Alive' 50 | req_header = 'Authorization' 51 | resp_header = 'www-authenticate' 52 | 53 | conn = HTTPSConnection(host=self.host, port=self.port) 54 | 55 | # Send negotiation message 56 | headers[req_header] = ( 57 | 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) 58 | log.debug('Request headers: %s' % headers) 59 | conn.request('GET', self.authurl, None, headers) 60 | res = conn.getresponse() 61 | reshdr = dict(res.getheaders()) 62 | log.debug('Response status: %s %s' % (res.status, res.reason)) 63 | log.debug('Response headers: %s' % reshdr) 64 | log.debug('Response data: %s [...]' % res.read(100)) 65 | 66 | # Remove the reference to the socket, so that it can not be closed by 67 | # the response object (we want to keep the socket open) 68 | res.fp = None 69 | 70 | # Server should respond with a challenge message 71 | auth_header_values = reshdr[resp_header].split(', ') 72 | auth_header_value = None 73 | for s in auth_header_values: 74 | if s[:5] == 'NTLM ': 75 | auth_header_value = s[5:] 76 | if auth_header_value is None: 77 | raise Exception('Unexpected %s response header: %s' % 78 | (resp_header, reshdr[resp_header])) 79 | 80 | # Send authentication message 81 | ServerChallenge, NegotiateFlags = \ 82 | ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) 83 | auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, 84 | self.user, 85 | self.domain, 86 | self.pw, 87 | NegotiateFlags) 88 | headers[req_header] = 'NTLM %s' % auth_msg 89 | log.debug('Request headers: %s' % headers) 90 | conn.request('GET', self.authurl, None, headers) 91 | res = conn.getresponse() 92 | log.debug('Response status: %s %s' % (res.status, res.reason)) 93 | log.debug('Response headers: %s' % dict(res.getheaders())) 94 | log.debug('Response data: %s [...]' % res.read()[:100]) 95 | if res.status != 200: 96 | if res.status == 401: 97 | raise Exception('Server rejected request: wrong ' 98 | 'username or password') 99 | raise Exception('Wrong server response: %s %s' % 100 | (res.status, res.reason)) 101 | 102 | res.fp = None 103 | log.debug('Connection established') 104 | return conn 105 | 106 | def urlopen(self, method, url, body=None, headers=None, retries=3, 107 | redirect=True, assert_same_host=True): 108 | if headers is None: 109 | headers = {} 110 | headers['Connection'] = 'Keep-Alive' 111 | return super(NTLMConnectionPool, self).urlopen(method, url, body, 112 | headers, retries, 113 | redirect, 114 | assert_same_host) 115 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/exceptions.py: -------------------------------------------------------------------------------- 1 | 2 | ## Base Exceptions 3 | 4 | class HTTPError(Exception): 5 | "Base exception used by this module." 6 | pass 7 | 8 | class HTTPWarning(Warning): 9 | "Base warning used by this module." 10 | pass 11 | 12 | 13 | 14 | class PoolError(HTTPError): 15 | "Base exception for errors caused within a pool." 16 | def __init__(self, pool, message): 17 | self.pool = pool 18 | HTTPError.__init__(self, "%s: %s" % (pool, message)) 19 | 20 | def __reduce__(self): 21 | # For pickling purposes. 22 | return self.__class__, (None, None) 23 | 24 | 25 | class RequestError(PoolError): 26 | "Base exception for PoolErrors that have associated URLs." 27 | def __init__(self, pool, url, message): 28 | self.url = url 29 | PoolError.__init__(self, pool, message) 30 | 31 | def __reduce__(self): 32 | # For pickling purposes. 33 | return self.__class__, (None, self.url, None) 34 | 35 | 36 | class SSLError(HTTPError): 37 | "Raised when SSL certificate fails in an HTTPS connection." 38 | pass 39 | 40 | 41 | class ProxyError(HTTPError): 42 | "Raised when the connection to a proxy fails." 43 | pass 44 | 45 | 46 | class DecodeError(HTTPError): 47 | "Raised when automatic decoding based on Content-Type fails." 48 | pass 49 | 50 | 51 | class ProtocolError(HTTPError): 52 | "Raised when something unexpected happens mid-request/response." 53 | pass 54 | 55 | 56 | #: Renamed to ProtocolError but aliased for backwards compatibility. 57 | ConnectionError = ProtocolError 58 | 59 | 60 | ## Leaf Exceptions 61 | 62 | class MaxRetryError(RequestError): 63 | """Raised when the maximum number of retries is exceeded. 64 | 65 | :param pool: The connection pool 66 | :type pool: :class:`~urllib3.connectionpool.HTTPConnectionPool` 67 | :param string url: The requested Url 68 | :param exceptions.Exception reason: The underlying error 69 | 70 | """ 71 | 72 | def __init__(self, pool, url, reason=None): 73 | self.reason = reason 74 | 75 | message = "Max retries exceeded with url: %s (Caused by %r)" % ( 76 | url, reason) 77 | 78 | RequestError.__init__(self, pool, url, message) 79 | 80 | 81 | class HostChangedError(RequestError): 82 | "Raised when an existing pool gets a request for a foreign host." 83 | 84 | def __init__(self, pool, url, retries=3): 85 | message = "Tried to open a foreign host with url: %s" % url 86 | RequestError.__init__(self, pool, url, message) 87 | self.retries = retries 88 | 89 | 90 | class TimeoutStateError(HTTPError): 91 | """ Raised when passing an invalid state to a timeout """ 92 | pass 93 | 94 | 95 | class TimeoutError(HTTPError): 96 | """ Raised when a socket timeout error occurs. 97 | 98 | Catching this error will catch both :exc:`ReadTimeoutErrors 99 | ` and :exc:`ConnectTimeoutErrors `. 100 | """ 101 | pass 102 | 103 | 104 | class ReadTimeoutError(TimeoutError, RequestError): 105 | "Raised when a socket timeout occurs while receiving data from a server" 106 | pass 107 | 108 | 109 | # This timeout error does not have a URL attached and needs to inherit from the 110 | # base HTTPError 111 | class ConnectTimeoutError(TimeoutError): 112 | "Raised when a socket timeout occurs while connecting to a server" 113 | pass 114 | 115 | 116 | class EmptyPoolError(PoolError): 117 | "Raised when a pool runs out of connections and no more are allowed." 118 | pass 119 | 120 | 121 | class ClosedPoolError(PoolError): 122 | "Raised when a request enters a pool after the pool has been closed." 123 | pass 124 | 125 | 126 | class LocationValueError(ValueError, HTTPError): 127 | "Raised when there is something wrong with a given URL input." 128 | pass 129 | 130 | 131 | class LocationParseError(LocationValueError): 132 | "Raised when get_host or similar fails to parse the URL input." 133 | 134 | def __init__(self, location): 135 | message = "Failed to parse: %s" % location 136 | HTTPError.__init__(self, message) 137 | 138 | self.location = location 139 | 140 | 141 | class ResponseError(HTTPError): 142 | "Used as a container for an error reason supplied in a MaxRetryError." 143 | GENERIC_ERROR = 'too many error responses' 144 | SPECIFIC_ERROR = 'too many {status_code} error responses' 145 | 146 | 147 | class SecurityWarning(HTTPWarning): 148 | "Warned when perfoming security reducing actions" 149 | pass 150 | 151 | 152 | class InsecureRequestWarning(SecurityWarning): 153 | "Warned when making an unverified HTTPS request." 154 | pass 155 | 156 | 157 | class SystemTimeWarning(SecurityWarning): 158 | "Warned when system time is suspected to be wrong" 159 | pass 160 | 161 | 162 | class InsecurePlatformWarning(SecurityWarning): 163 | "Warned when certain SSL configuration is not available on a platform." 164 | pass 165 | 166 | 167 | class ResponseNotChunked(ProtocolError, ValueError): 168 | "Response needs to be chunked in order to read it as chunks." 169 | pass 170 | -------------------------------------------------------------------------------- /lib/core/Dictionary.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import threading 20 | import urllib.request, urllib.parse, urllib.error 21 | from lib.utils.FileUtils import File 22 | from thirdparty.oset import * 23 | 24 | 25 | class Dictionary(object): 26 | 27 | def __init__(self, path, extensions, lowercase=False, forcedExtensions=False): 28 | self.entries = [] 29 | self.currentIndex = 0 30 | self.condition = threading.Lock() 31 | self._extensions = extensions 32 | self._path = path 33 | self._forcedExtensions = forcedExtensions 34 | self.lowercase = lowercase 35 | self.dictionaryFile = File(self.path) 36 | self.generate() 37 | 38 | @property 39 | def extensions(self): 40 | return self._extensions 41 | 42 | @extensions.setter 43 | def extensions(self, value): 44 | self._extensions = value 45 | 46 | @property 47 | def path(self): 48 | return self._path 49 | 50 | @path.setter 51 | def path(self, path): 52 | self._path = path 53 | 54 | @classmethod 55 | def quote(cls, string): 56 | return urllib.parse.quote(string, safe=":/~?%&+-=$") 57 | 58 | """ 59 | Dictionary.generate() behaviour 60 | 61 | Classic dirsearch wordlist: 62 | 1. If %EXT% keyword is present, append one with each extension REPLACED. 63 | 2. If the special word is no present, append line unmodified. 64 | 65 | Forced extensions wordlist (NEW): 66 | This type of wordlist processing is a mix between classic processing 67 | and DirBuster processing. 68 | 1. If %EXT% keyword is present in the line, immediately process as "classic dirsearch" (1). 69 | 2. If the line does not include the special word AND is NOT terminated by a slash, 70 | append one with each extension APPENDED (line.ext) and ONLYE ONE with a slash. 71 | 3. If the line does not include the special word and IS ALREADY terminated by slash, 72 | append line unmodified. 73 | """ 74 | def generate(self): 75 | result = [] 76 | for line in self.dictionaryFile.getLines(): 77 | # Skip comments 78 | if line.lstrip().startswith("#"): continue 79 | # Classic dirsearch wordlist processing (with %EXT% keyword) 80 | if '%EXT%' in line: 81 | for extension in self._extensions: 82 | quote = self.quote(line.replace('%EXT%', extension)) 83 | result.append(quote) 84 | # If forced extensions is used and the path is not a directory ... (terminated by /) 85 | # process line like a forced extension. 86 | elif self._forcedExtensions and not line.rstrip().endswith("/"): 87 | quoted = self.quote(line) 88 | for extension in self._extensions: 89 | if extension == "": 90 | result.append(quoted) 91 | else: 92 | result.append(quoted + '.' + extension) 93 | if quoted.strip() not in ['']: 94 | result.append(quoted + "/") 95 | # Append line unmodified. 96 | else: 97 | result.append(self.quote(line)) 98 | # oset library provides inserted ordered and unique collection. 99 | if self.lowercase: 100 | self.entries = list(oset(map(lambda l: l.lower(), result))) 101 | else: 102 | self.entries = list(oset(result)) 103 | del(result) 104 | 105 | def regenerate(self): 106 | self.generate(lowercase=self.lowercase) 107 | self.reset() 108 | 109 | def nextWithIndex(self, basePath=None): 110 | self.condition.acquire() 111 | try: 112 | result = self.entries[self.currentIndex] 113 | except IndexError: 114 | self.condition.release() 115 | raise StopIteration 116 | self.currentIndex = self.currentIndex + 1 117 | currentIndex = self.currentIndex 118 | self.condition.release() 119 | return currentIndex, result 120 | 121 | def __next__(self, basePath=None): 122 | _, path = self.nextWithIndex(basePath) 123 | return path 124 | 125 | def reset(self): 126 | self.condition.acquire() 127 | self.currentIndex = 0 128 | self.condition.release() 129 | 130 | def __len__(self): 131 | return len(self.entries) 132 | 133 | 134 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/sbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | import sys 30 | from . import constants 31 | from .charsetprober import CharSetProber 32 | from .compat import wrap_ord 33 | 34 | SAMPLE_SIZE = 64 35 | SB_ENOUGH_REL_THRESHOLD = 1024 36 | POSITIVE_SHORTCUT_THRESHOLD = 0.95 37 | NEGATIVE_SHORTCUT_THRESHOLD = 0.05 38 | SYMBOL_CAT_ORDER = 250 39 | NUMBER_OF_SEQ_CAT = 4 40 | POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1 41 | #NEGATIVE_CAT = 0 42 | 43 | 44 | class SingleByteCharSetProber(CharSetProber): 45 | def __init__(self, model, reversed=False, nameProber=None): 46 | CharSetProber.__init__(self) 47 | self._mModel = model 48 | # TRUE if we need to reverse every pair in the model lookup 49 | self._mReversed = reversed 50 | # Optional auxiliary prober for name decision 51 | self._mNameProber = nameProber 52 | self.reset() 53 | 54 | def reset(self): 55 | CharSetProber.reset(self) 56 | # char order of last character 57 | self._mLastOrder = 255 58 | self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT 59 | self._mTotalSeqs = 0 60 | self._mTotalChar = 0 61 | # characters that fall in our sampling range 62 | self._mFreqChar = 0 63 | 64 | def get_charset_name(self): 65 | if self._mNameProber: 66 | return self._mNameProber.get_charset_name() 67 | else: 68 | return self._mModel['charsetName'] 69 | 70 | def feed(self, aBuf): 71 | if not self._mModel['keepEnglishLetter']: 72 | aBuf = self.filter_without_english_letters(aBuf) 73 | aLen = len(aBuf) 74 | if not aLen: 75 | return self.get_state() 76 | for c in aBuf: 77 | order = self._mModel['charToOrderMap'][wrap_ord(c)] 78 | if order < SYMBOL_CAT_ORDER: 79 | self._mTotalChar += 1 80 | if order < SAMPLE_SIZE: 81 | self._mFreqChar += 1 82 | if self._mLastOrder < SAMPLE_SIZE: 83 | self._mTotalSeqs += 1 84 | if not self._mReversed: 85 | i = (self._mLastOrder * SAMPLE_SIZE) + order 86 | model = self._mModel['precedenceMatrix'][i] 87 | else: # reverse the order of the letters in the lookup 88 | i = (order * SAMPLE_SIZE) + self._mLastOrder 89 | model = self._mModel['precedenceMatrix'][i] 90 | self._mSeqCounters[model] += 1 91 | self._mLastOrder = order 92 | 93 | if self.get_state() == constants.eDetecting: 94 | if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD: 95 | cf = self.get_confidence() 96 | if cf > POSITIVE_SHORTCUT_THRESHOLD: 97 | if constants._debug: 98 | sys.stderr.write('%s confidence = %s, we have a' 99 | 'winner\n' % 100 | (self._mModel['charsetName'], cf)) 101 | self._mState = constants.eFoundIt 102 | elif cf < NEGATIVE_SHORTCUT_THRESHOLD: 103 | if constants._debug: 104 | sys.stderr.write('%s confidence = %s, below negative' 105 | 'shortcut threshhold %s\n' % 106 | (self._mModel['charsetName'], cf, 107 | NEGATIVE_SHORTCUT_THRESHOLD)) 108 | self._mState = constants.eNotMe 109 | 110 | return self.get_state() 111 | 112 | def get_confidence(self): 113 | r = 0.01 114 | if self._mTotalSeqs > 0: 115 | r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs 116 | / self._mModel['mTypicalPositiveRatio']) 117 | r = r * self._mFreqChar / self._mTotalChar 118 | if r >= 1.0: 119 | r = 0.99 120 | return r 121 | -------------------------------------------------------------------------------- /db/user-agents.txt: -------------------------------------------------------------------------------- 1 | Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36 2 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36 3 | Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36 4 | Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36 5 | Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36 6 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1 7 | Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0 8 | Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0 9 | Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0 10 | Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0 11 | Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0 12 | Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko 13 | Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko 14 | Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0 15 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN) 16 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0) 17 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0) 18 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0) 19 | Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64) 20 | Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0) 21 | Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0) 22 | Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0) 23 | Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US)) 24 | Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US) 25 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0) 26 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7) 27 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7 28 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E) 29 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112) 30 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0) 31 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0) 32 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E) 33 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0 34 | Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8) 35 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US) 36 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322) 37 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US) 38 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3) 39 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320) 40 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322) 41 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 42 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727) 43 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322) 44 | Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30) 45 | Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS) 46 | Mozilla/4.0(compatible; MSIE 7.0b; Windows NT 6.0) 47 | Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 6.0) 48 | Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30) 49 | Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; Media Center PC 3.0; .NET CLR 1.0.3705; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.1) 50 | Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.1; FDM; .NET CLR 1.1.4322) 51 | Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; en-US) 52 | Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 6.0; el-GR) 53 | Mozilla/5.0 (Windows; U; MSIE 7.0; Windows NT 5.2) 54 | Mozilla/4.0 (compatible; MSIE 6.1; Windows XP; .NET CLR 1.1.4322; .NET CLR 2.0.50727) 55 | Mozilla/4.0 (compatible; MSIE 6.1; Windows XP) 56 | Mozilla/4.0 (compatible; MSIE 6.01; Windows NT 6.0) 57 | Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 5.1; DigExt) 58 | Mozilla/4.0 (compatible; MSIE 6.0b; Windows NT 5.1) -------------------------------------------------------------------------------- /thirdparty/requests/packages/chardet/latin1prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetprober import CharSetProber 30 | from .constants import eNotMe 31 | from .compat import wrap_ord 32 | 33 | FREQ_CAT_NUM = 4 34 | 35 | UDF = 0 # undefined 36 | OTH = 1 # other 37 | ASC = 2 # ascii capital letter 38 | ASS = 3 # ascii small letter 39 | ACV = 4 # accent capital vowel 40 | ACO = 5 # accent capital other 41 | ASV = 6 # accent small vowel 42 | ASO = 7 # accent small other 43 | CLASS_NUM = 8 # total classes 44 | 45 | Latin1_CharToClass = ( 46 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 47 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F 48 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 49 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F 50 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 51 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F 52 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 53 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F 54 | OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 55 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F 56 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 57 | ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F 58 | OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 59 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F 60 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 61 | ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F 62 | OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 63 | OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F 64 | UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 65 | OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F 66 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 67 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF 68 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 69 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF 70 | ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 71 | ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF 72 | ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 73 | ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF 74 | ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 75 | ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF 76 | ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 77 | ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF 78 | ) 79 | 80 | # 0 : illegal 81 | # 1 : very unlikely 82 | # 2 : normal 83 | # 3 : very likely 84 | Latin1ClassModel = ( 85 | # UDF OTH ASC ASS ACV ACO ASV ASO 86 | 0, 0, 0, 0, 0, 0, 0, 0, # UDF 87 | 0, 3, 3, 3, 3, 3, 3, 3, # OTH 88 | 0, 3, 3, 3, 3, 3, 3, 3, # ASC 89 | 0, 3, 3, 3, 1, 1, 3, 3, # ASS 90 | 0, 3, 3, 3, 1, 2, 1, 2, # ACV 91 | 0, 3, 3, 3, 3, 3, 3, 3, # ACO 92 | 0, 3, 1, 3, 1, 1, 1, 3, # ASV 93 | 0, 3, 1, 3, 1, 1, 3, 3, # ASO 94 | ) 95 | 96 | 97 | class Latin1Prober(CharSetProber): 98 | def __init__(self): 99 | CharSetProber.__init__(self) 100 | self.reset() 101 | 102 | def reset(self): 103 | self._mLastCharClass = OTH 104 | self._mFreqCounter = [0] * FREQ_CAT_NUM 105 | CharSetProber.reset(self) 106 | 107 | def get_charset_name(self): 108 | return "windows-1252" 109 | 110 | def feed(self, aBuf): 111 | aBuf = self.filter_with_english_letters(aBuf) 112 | for c in aBuf: 113 | charClass = Latin1_CharToClass[wrap_ord(c)] 114 | freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) 115 | + charClass] 116 | if freq == 0: 117 | self._mState = eNotMe 118 | break 119 | self._mFreqCounter[freq] += 1 120 | self._mLastCharClass = charClass 121 | 122 | return self.get_state() 123 | 124 | def get_confidence(self): 125 | if self.get_state() == eNotMe: 126 | return 0.01 127 | 128 | total = sum(self._mFreqCounter) 129 | if total < 0.01: 130 | confidence = 0.0 131 | else: 132 | confidence = ((self._mFreqCounter[3] - self._mFreqCounter[1] * 20.0) 133 | / total) 134 | if confidence < 0.0: 135 | confidence = 0.0 136 | # lower the confidence of latin1 so that other more accurate 137 | # detector can take priority. 138 | confidence = confidence * 0.73 139 | return confidence 140 | -------------------------------------------------------------------------------- /thirdparty/requests/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.api 5 | ~~~~~~~~~~~~ 6 | 7 | This module implements the Requests API. 8 | 9 | :copyright: (c) 2012 by Kenneth Reitz. 10 | :license: Apache2, see LICENSE for more details. 11 | 12 | """ 13 | 14 | from . import sessions 15 | 16 | 17 | def request(method, url, **kwargs): 18 | """Constructs and sends a :class:`Request `. 19 | 20 | :param method: method for the new :class:`Request` object. 21 | :param url: URL for the new :class:`Request` object. 22 | :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. 23 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 24 | :param json: (optional) json data to send in the body of the :class:`Request`. 25 | :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. 26 | :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. 27 | :param files: (optional) Dictionary of ``'name': file-like-objects`` (or ``{'name': ('filename', fileobj)}``) for multipart encoding upload. 28 | :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. 29 | :param timeout: (optional) How long to wait for the server to send data 30 | before giving up, as a float, or a :ref:`(connect timeout, read 31 | timeout) ` tuple. 32 | :type timeout: float or tuple 33 | :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. 34 | :type allow_redirects: bool 35 | :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. 36 | :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. 37 | :param stream: (optional) if ``False``, the response content will be immediately downloaded. 38 | :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. 39 | :return: :class:`Response ` object 40 | :rtype: requests.Response 41 | 42 | Usage:: 43 | 44 | >>> import requests 45 | >>> req = requests.request('GET', 'http://httpbin.org/get') 46 | 47 | """ 48 | 49 | session = sessions.Session() 50 | response = session.request(method=method, url=url, **kwargs) 51 | # By explicitly closing the session, we avoid leaving sockets open which 52 | # can trigger a ResourceWarning in some cases, and look like a memory leak 53 | # in others. 54 | session.close() 55 | return response 56 | 57 | 58 | def get(url, params=None, **kwargs): 59 | """Sends a GET request. 60 | 61 | :param url: URL for the new :class:`Request` object. 62 | :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. 63 | :param \*\*kwargs: Optional arguments that ``request`` takes. 64 | :return: :class:`Response ` object 65 | :rtype: requests.Response 66 | """ 67 | 68 | kwargs.setdefault('allow_redirects', True) 69 | return request('get', url, params=params, **kwargs) 70 | 71 | 72 | def options(url, **kwargs): 73 | """Sends a OPTIONS request. 74 | 75 | :param url: URL for the new :class:`Request` object. 76 | :param \*\*kwargs: Optional arguments that ``request`` takes. 77 | :return: :class:`Response ` object 78 | :rtype: requests.Response 79 | """ 80 | 81 | kwargs.setdefault('allow_redirects', True) 82 | return request('options', url, **kwargs) 83 | 84 | 85 | def head(url, **kwargs): 86 | """Sends a HEAD request. 87 | 88 | :param url: URL for the new :class:`Request` object. 89 | :param \*\*kwargs: Optional arguments that ``request`` takes. 90 | :return: :class:`Response ` object 91 | :rtype: requests.Response 92 | """ 93 | 94 | kwargs.setdefault('allow_redirects', False) 95 | return request('head', url, **kwargs) 96 | 97 | 98 | def post(url, data=None, json=None, **kwargs): 99 | """Sends a POST request. 100 | 101 | :param url: URL for the new :class:`Request` object. 102 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 103 | :param json: (optional) json data to send in the body of the :class:`Request`. 104 | :param \*\*kwargs: Optional arguments that ``request`` takes. 105 | :return: :class:`Response ` object 106 | :rtype: requests.Response 107 | """ 108 | 109 | return request('post', url, data=data, json=json, **kwargs) 110 | 111 | 112 | def put(url, data=None, **kwargs): 113 | """Sends a PUT request. 114 | 115 | :param url: URL for the new :class:`Request` object. 116 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 117 | :param \*\*kwargs: Optional arguments that ``request`` takes. 118 | :return: :class:`Response ` object 119 | :rtype: requests.Response 120 | """ 121 | 122 | return request('put', url, data=data, **kwargs) 123 | 124 | 125 | def patch(url, data=None, **kwargs): 126 | """Sends a PATCH request. 127 | 128 | :param url: URL for the new :class:`Request` object. 129 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 130 | :param \*\*kwargs: Optional arguments that ``request`` takes. 131 | :return: :class:`Response ` object 132 | :rtype: requests.Response 133 | """ 134 | 135 | return request('patch', url, data=data, **kwargs) 136 | 137 | 138 | def delete(url, **kwargs): 139 | """Sends a DELETE request. 140 | 141 | :param url: URL for the new :class:`Request` object. 142 | :param \*\*kwargs: Optional arguments that ``request`` takes. 143 | :return: :class:`Response ` object 144 | :rtype: requests.Response 145 | """ 146 | 147 | return request('delete', url, **kwargs) 148 | -------------------------------------------------------------------------------- /thirdparty/colorama/win32.py: -------------------------------------------------------------------------------- 1 | # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. 2 | 3 | # from winbase.h 4 | STDOUT = -11 5 | STDERR = -12 6 | 7 | try: 8 | import ctypes 9 | from ctypes import LibraryLoader 10 | windll = LibraryLoader(ctypes.WinDLL) 11 | from ctypes import wintypes 12 | except (AttributeError, ImportError): 13 | windll = None 14 | SetConsoleTextAttribute = lambda *_: None 15 | winapi_test = lambda *_: None 16 | else: 17 | from ctypes import byref, Structure, c_char, POINTER 18 | 19 | COORD = wintypes._COORD 20 | 21 | class CONSOLE_SCREEN_BUFFER_INFO(Structure): 22 | """struct in wincon.h.""" 23 | _fields_ = [ 24 | ("dwSize", COORD), 25 | ("dwCursorPosition", COORD), 26 | ("wAttributes", wintypes.WORD), 27 | ("srWindow", wintypes.SMALL_RECT), 28 | ("dwMaximumWindowSize", COORD), 29 | ] 30 | def __str__(self): 31 | return '(%d,%d,%d,%d,%d,%d,%d,%d,%d,%d,%d)' % ( 32 | self.dwSize.Y, self.dwSize.X 33 | , self.dwCursorPosition.Y, self.dwCursorPosition.X 34 | , self.wAttributes 35 | , self.srWindow.Top, self.srWindow.Left, self.srWindow.Bottom, self.srWindow.Right 36 | , self.dwMaximumWindowSize.Y, self.dwMaximumWindowSize.X 37 | ) 38 | 39 | _GetStdHandle = windll.kernel32.GetStdHandle 40 | _GetStdHandle.argtypes = [ 41 | wintypes.DWORD, 42 | ] 43 | _GetStdHandle.restype = wintypes.HANDLE 44 | 45 | _GetConsoleScreenBufferInfo = windll.kernel32.GetConsoleScreenBufferInfo 46 | _GetConsoleScreenBufferInfo.argtypes = [ 47 | wintypes.HANDLE, 48 | POINTER(CONSOLE_SCREEN_BUFFER_INFO), 49 | ] 50 | _GetConsoleScreenBufferInfo.restype = wintypes.BOOL 51 | 52 | _SetConsoleTextAttribute = windll.kernel32.SetConsoleTextAttribute 53 | _SetConsoleTextAttribute.argtypes = [ 54 | wintypes.HANDLE, 55 | wintypes.WORD, 56 | ] 57 | _SetConsoleTextAttribute.restype = wintypes.BOOL 58 | 59 | _SetConsoleCursorPosition = windll.kernel32.SetConsoleCursorPosition 60 | _SetConsoleCursorPosition.argtypes = [ 61 | wintypes.HANDLE, 62 | COORD, 63 | ] 64 | _SetConsoleCursorPosition.restype = wintypes.BOOL 65 | 66 | _FillConsoleOutputCharacterA = windll.kernel32.FillConsoleOutputCharacterA 67 | _FillConsoleOutputCharacterA.argtypes = [ 68 | wintypes.HANDLE, 69 | c_char, 70 | wintypes.DWORD, 71 | COORD, 72 | POINTER(wintypes.DWORD), 73 | ] 74 | _FillConsoleOutputCharacterA.restype = wintypes.BOOL 75 | 76 | _FillConsoleOutputAttribute = windll.kernel32.FillConsoleOutputAttribute 77 | _FillConsoleOutputAttribute.argtypes = [ 78 | wintypes.HANDLE, 79 | wintypes.WORD, 80 | wintypes.DWORD, 81 | COORD, 82 | POINTER(wintypes.DWORD), 83 | ] 84 | _FillConsoleOutputAttribute.restype = wintypes.BOOL 85 | 86 | _SetConsoleTitleW = windll.kernel32.SetConsoleTitleA 87 | _SetConsoleTitleW.argtypes = [ 88 | wintypes.LPCSTR 89 | ] 90 | _SetConsoleTitleW.restype = wintypes.BOOL 91 | 92 | handles = { 93 | STDOUT: _GetStdHandle(STDOUT), 94 | STDERR: _GetStdHandle(STDERR), 95 | } 96 | 97 | def winapi_test(): 98 | handle = handles[STDOUT] 99 | csbi = CONSOLE_SCREEN_BUFFER_INFO() 100 | success = _GetConsoleScreenBufferInfo( 101 | handle, byref(csbi)) 102 | return bool(success) 103 | 104 | def GetConsoleScreenBufferInfo(stream_id=STDOUT): 105 | handle = handles[stream_id] 106 | csbi = CONSOLE_SCREEN_BUFFER_INFO() 107 | success = _GetConsoleScreenBufferInfo( 108 | handle, byref(csbi)) 109 | return csbi 110 | 111 | def SetConsoleTextAttribute(stream_id, attrs): 112 | handle = handles[stream_id] 113 | return _SetConsoleTextAttribute(handle, attrs) 114 | 115 | def SetConsoleCursorPosition(stream_id, position, adjust=True): 116 | position = COORD(*position) 117 | # If the position is out of range, do nothing. 118 | if position.Y <= 0 or position.X <= 0: 119 | return 120 | # Adjust for Windows' SetConsoleCursorPosition: 121 | # 1. being 0-based, while ANSI is 1-based. 122 | # 2. expecting (x,y), while ANSI uses (y,x). 123 | adjusted_position = COORD(position.Y - 1, position.X - 1) 124 | if adjust: 125 | # Adjust for viewport's scroll position 126 | sr = GetConsoleScreenBufferInfo(STDOUT).srWindow 127 | adjusted_position.Y += sr.Top 128 | adjusted_position.X += sr.Left 129 | # Resume normal processing 130 | handle = handles[stream_id] 131 | return _SetConsoleCursorPosition(handle, adjusted_position) 132 | 133 | def FillConsoleOutputCharacter(stream_id, char, length, start): 134 | handle = handles[stream_id] 135 | char = c_char(char.encode()) 136 | length = wintypes.DWORD(length) 137 | num_written = wintypes.DWORD(0) 138 | # Note that this is hard-coded for ANSI (vs wide) bytes. 139 | success = _FillConsoleOutputCharacterA( 140 | handle, char, length, start, byref(num_written)) 141 | return num_written.value 142 | 143 | def FillConsoleOutputAttribute(stream_id, attr, length, start): 144 | ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' 145 | handle = handles[stream_id] 146 | attribute = wintypes.WORD(attr) 147 | length = wintypes.DWORD(length) 148 | num_written = wintypes.DWORD(0) 149 | # Note that this is hard-coded for ANSI (vs wide) bytes. 150 | return _FillConsoleOutputAttribute( 151 | handle, attribute, length, start, byref(num_written)) 152 | 153 | def SetConsoleTitle(title): 154 | return _SetConsoleTitleW(title) 155 | -------------------------------------------------------------------------------- /lib/core/Fuzzer.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | from queue import Queue 20 | import threading 21 | 22 | from lib.connection.RequestException import RequestException 23 | from .Path import * 24 | from .Scanner import * 25 | 26 | 27 | class Fuzzer(object): 28 | def __init__(self, requester, dictionary, testFailPath=None, threads=1, matchCallbacks=[], notFoundCallbacks=[], 29 | errorCallbacks=[]): 30 | 31 | self.requester = requester 32 | self.dictionary = dictionary 33 | self.testFailPath = testFailPath 34 | self.basePath = self.requester.basePath 35 | self.threads = [] 36 | self.threadsCount = threads if len(self.dictionary) >= threads else len(self.dictionary) 37 | self.running = False 38 | self.scanners = {} 39 | self.defaultScanner = None 40 | self.matchCallbacks = matchCallbacks 41 | self.notFoundCallbacks = notFoundCallbacks 42 | self.errorCallbacks = errorCallbacks 43 | self.matches = [] 44 | self.errors = [] 45 | 46 | def wait(self, timeout=None): 47 | for thread in self.threads: 48 | thread.join(timeout) 49 | if timeout is not None and thread.is_alive(): 50 | return False 51 | return True 52 | 53 | def setupScanners(self): 54 | if len(self.scanners) != 0: 55 | self.scanners = {} 56 | self.defaultScanner = Scanner(self.requester, self.testFailPath, "") 57 | self.scanners['/'] = Scanner(self.requester, self.testFailPath, "/") 58 | for extension in self.dictionary.extensions: 59 | self.scanners[extension] = Scanner(self.requester, self.testFailPath, "." + extension) 60 | 61 | def setupThreads(self): 62 | if len(self.threads) != 0: 63 | self.threads = [] 64 | for thread in range(self.threadsCount): 65 | newThread = threading.Thread(target=self.thread_proc) 66 | newThread.daemon = True 67 | self.threads.append(newThread) 68 | 69 | def getScannerFor(self, path): 70 | if path.endswith('/'): 71 | return self.scanners['/'] 72 | for extension in list(self.scanners.keys()): 73 | if path.endswith(extension): 74 | return self.scanners[extension] 75 | # By default, returns empty tester 76 | return self.defaultScanner 77 | 78 | def start(self): 79 | # Setting up testers 80 | self.setupScanners() 81 | # Setting up threads 82 | self.setupThreads() 83 | self.index = 0 84 | self.dictionary.reset() 85 | self.runningThreadsCount = len(self.threads) 86 | self.running = True 87 | self.playEvent = threading.Event() 88 | self.pausedSemaphore = threading.Semaphore(0) 89 | self.playEvent.clear() 90 | self.exit = False 91 | for thread in self.threads: 92 | thread.start() 93 | self.play() 94 | 95 | def play(self): 96 | self.playEvent.set() 97 | 98 | def pause(self): 99 | self.playEvent.clear() 100 | for thread in self.threads: 101 | if thread.is_alive(): 102 | self.pausedSemaphore.acquire() 103 | 104 | def stop(self): 105 | self.running = False 106 | self.play() 107 | 108 | def scan(self, path): 109 | response = self.requester.request(path) 110 | result = None 111 | if self.getScannerFor(path).scan(path, response): 112 | result = (None if response.status == 404 else response.status) 113 | return result, response 114 | 115 | def isRunning(self): 116 | return self.running 117 | 118 | def finishThreads(self): 119 | self.running = False 120 | self.finishedEvent.set() 121 | 122 | def isFinished(self): 123 | return self.runningThreadsCount == 0 124 | 125 | def stopThread(self): 126 | self.runningThreadsCount -= 1 127 | 128 | def thread_proc(self): 129 | self.playEvent.wait() 130 | try: 131 | path = next(self.dictionary) 132 | while path is not None: 133 | try: 134 | status, response = self.scan(path) 135 | result = Path(path=path, status=status, response=response) 136 | if status is not None: 137 | self.matches.append(result) 138 | for callback in self.matchCallbacks: 139 | callback(result) 140 | else: 141 | for callback in self.notFoundCallbacks: 142 | callback(result) 143 | del status 144 | del response 145 | except RequestException as e: 146 | for callback in self.errorCallbacks: 147 | callback(path, e.args[0]['message']) 148 | continue 149 | finally: 150 | if not self.playEvent.isSet(): 151 | self.pausedSemaphore.release() 152 | self.playEvent.wait() 153 | path = next(self.dictionary) # Raises StopIteration when finishes 154 | if not self.running: 155 | break 156 | except StopIteration: 157 | return 158 | finally: 159 | self.stopThread() -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/request.py: -------------------------------------------------------------------------------- 1 | try: 2 | from urllib.parse import urlencode 3 | except ImportError: 4 | from urllib import urlencode 5 | 6 | from .filepost import encode_multipart_formdata 7 | 8 | 9 | __all__ = ['RequestMethods'] 10 | 11 | 12 | class RequestMethods(object): 13 | """ 14 | Convenience mixin for classes who implement a :meth:`urlopen` method, such 15 | as :class:`~urllib3.connectionpool.HTTPConnectionPool` and 16 | :class:`~urllib3.poolmanager.PoolManager`. 17 | 18 | Provides behavior for making common types of HTTP request methods and 19 | decides which type of request field encoding to use. 20 | 21 | Specifically, 22 | 23 | :meth:`.request_encode_url` is for sending requests whose fields are 24 | encoded in the URL (such as GET, HEAD, DELETE). 25 | 26 | :meth:`.request_encode_body` is for sending requests whose fields are 27 | encoded in the *body* of the request using multipart or www-form-urlencoded 28 | (such as for POST, PUT, PATCH). 29 | 30 | :meth:`.request` is for making any kind of request, it will look up the 31 | appropriate encoding format and use one of the above two methods to make 32 | the request. 33 | 34 | Initializer parameters: 35 | 36 | :param headers: 37 | Headers to include with all requests, unless other headers are given 38 | explicitly. 39 | """ 40 | 41 | _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) 42 | 43 | def __init__(self, headers=None): 44 | self.headers = headers or {} 45 | 46 | def urlopen(self, method, url, body=None, headers=None, 47 | encode_multipart=True, multipart_boundary=None, 48 | **kw): # Abstract 49 | raise NotImplemented("Classes extending RequestMethods must implement " 50 | "their own ``urlopen`` method.") 51 | 52 | def request(self, method, url, fields=None, headers=None, **urlopen_kw): 53 | """ 54 | Make a request using :meth:`urlopen` with the appropriate encoding of 55 | ``fields`` based on the ``method`` used. 56 | 57 | This is a convenience method that requires the least amount of manual 58 | effort. It can be used in most situations, while still having the 59 | option to drop down to more specific methods when necessary, such as 60 | :meth:`request_encode_url`, :meth:`request_encode_body`, 61 | or even the lowest level :meth:`urlopen`. 62 | """ 63 | method = method.upper() 64 | 65 | if method in self._encode_url_methods: 66 | return self.request_encode_url(method, url, fields=fields, 67 | headers=headers, 68 | **urlopen_kw) 69 | else: 70 | return self.request_encode_body(method, url, fields=fields, 71 | headers=headers, 72 | **urlopen_kw) 73 | 74 | def request_encode_url(self, method, url, fields=None, **urlopen_kw): 75 | """ 76 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 77 | the url. This is useful for request methods like GET, HEAD, DELETE, etc. 78 | """ 79 | if fields: 80 | url += '?' + urlencode(fields) 81 | return self.urlopen(method, url, **urlopen_kw) 82 | 83 | def request_encode_body(self, method, url, fields=None, headers=None, 84 | encode_multipart=True, multipart_boundary=None, 85 | **urlopen_kw): 86 | """ 87 | Make a request using :meth:`urlopen` with the ``fields`` encoded in 88 | the body. This is useful for request methods like POST, PUT, PATCH, etc. 89 | 90 | When ``encode_multipart=True`` (default), then 91 | :meth:`urllib3.filepost.encode_multipart_formdata` is used to encode 92 | the payload with the appropriate content type. Otherwise 93 | :meth:`urllib.urlencode` is used with the 94 | 'application/x-www-form-urlencoded' content type. 95 | 96 | Multipart encoding must be used when posting files, and it's reasonably 97 | safe to use it in other times too. However, it may break request 98 | signing, such as with OAuth. 99 | 100 | Supports an optional ``fields`` parameter of key/value strings AND 101 | key/filetuple. A filetuple is a (filename, data, MIME type) tuple where 102 | the MIME type is optional. For example:: 103 | 104 | fields = { 105 | 'foo': 'bar', 106 | 'fakefile': ('foofile.txt', 'contents of foofile'), 107 | 'realfile': ('barfile.txt', open('realfile').read()), 108 | 'typedfile': ('bazfile.bin', open('bazfile').read(), 109 | 'image/jpeg'), 110 | 'nonamefile': 'contents of nonamefile field', 111 | } 112 | 113 | When uploading a file, providing a filename (the first parameter of the 114 | tuple) is optional but recommended to best mimick behavior of browsers. 115 | 116 | Note that if ``headers`` are supplied, the 'Content-Type' header will 117 | be overwritten because it depends on the dynamic random boundary string 118 | which is used to compose the body of the request. The random boundary 119 | string can be explicitly set with the ``multipart_boundary`` parameter. 120 | """ 121 | if headers is None: 122 | headers = self.headers 123 | 124 | extra_kw = {'headers': {}} 125 | 126 | if fields: 127 | if 'body' in urlopen_kw: 128 | raise TypeError('request got values for both \'fields\' and \'body\', can only specify one.') 129 | 130 | if encode_multipart: 131 | body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) 132 | else: 133 | body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' 134 | 135 | extra_kw['body'] = body 136 | extra_kw['headers'] = {'Content-Type': content_type} 137 | 138 | extra_kw['headers'].update(headers) 139 | extra_kw.update(urlopen_kw) 140 | 141 | return self.urlopen(method, url, **extra_kw) 142 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/fields.py: -------------------------------------------------------------------------------- 1 | import email.utils 2 | import mimetypes 3 | 4 | from .packages import six 5 | 6 | 7 | def guess_content_type(filename, default='application/octet-stream'): 8 | """ 9 | Guess the "Content-Type" of a file. 10 | 11 | :param filename: 12 | The filename to guess the "Content-Type" of using :mod:`mimetypes`. 13 | :param default: 14 | If no "Content-Type" can be guessed, default to `default`. 15 | """ 16 | if filename: 17 | return mimetypes.guess_type(filename)[0] or default 18 | return default 19 | 20 | 21 | def format_header_param(name, value): 22 | """ 23 | Helper function to format and quote a single header parameter. 24 | 25 | Particularly useful for header parameters which might contain 26 | non-ASCII values, like file names. This follows RFC 2231, as 27 | suggested by RFC 2388 Section 4.4. 28 | 29 | :param name: 30 | The name of the parameter, a string expected to be ASCII only. 31 | :param value: 32 | The value of the parameter, provided as a unicode string. 33 | """ 34 | if not any(ch in value for ch in '"\\\r\n'): 35 | result = '%s="%s"' % (name, value) 36 | try: 37 | result.encode('ascii') 38 | except UnicodeEncodeError: 39 | pass 40 | else: 41 | return result 42 | if not six.PY3: # Python 2: 43 | value = value.encode('utf-8') 44 | value = email.utils.encode_rfc2231(value, 'utf-8') 45 | value = '%s*=%s' % (name, value) 46 | return value 47 | 48 | 49 | class RequestField(object): 50 | """ 51 | A data container for request body parameters. 52 | 53 | :param name: 54 | The name of this request field. 55 | :param data: 56 | The data/value body. 57 | :param filename: 58 | An optional filename of the request field. 59 | :param headers: 60 | An optional dict-like object of headers to initially use for the field. 61 | """ 62 | def __init__(self, name, data, filename=None, headers=None): 63 | self._name = name 64 | self._filename = filename 65 | self.data = data 66 | self.headers = {} 67 | if headers: 68 | self.headers = dict(headers) 69 | 70 | @classmethod 71 | def from_tuples(cls, fieldname, value): 72 | """ 73 | A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. 74 | 75 | Supports constructing :class:`~urllib3.fields.RequestField` from 76 | parameter of key/value strings AND key/filetuple. A filetuple is a 77 | (filename, data, MIME type) tuple where the MIME type is optional. 78 | For example:: 79 | 80 | 'foo': 'bar', 81 | 'fakefile': ('foofile.txt', 'contents of foofile'), 82 | 'realfile': ('barfile.txt', open('realfile').read()), 83 | 'typedfile': ('bazfile.bin', open('bazfile').read(), 'image/jpeg'), 84 | 'nonamefile': 'contents of nonamefile field', 85 | 86 | Field names and filenames must be unicode. 87 | """ 88 | if isinstance(value, tuple): 89 | if len(value) == 3: 90 | filename, data, content_type = value 91 | else: 92 | filename, data = value 93 | content_type = guess_content_type(filename) 94 | else: 95 | filename = None 96 | content_type = None 97 | data = value 98 | 99 | request_param = cls(fieldname, data, filename=filename) 100 | request_param.make_multipart(content_type=content_type) 101 | 102 | return request_param 103 | 104 | def _render_part(self, name, value): 105 | """ 106 | Overridable helper function to format a single header parameter. 107 | 108 | :param name: 109 | The name of the parameter, a string expected to be ASCII only. 110 | :param value: 111 | The value of the parameter, provided as a unicode string. 112 | """ 113 | return format_header_param(name, value) 114 | 115 | def _render_parts(self, header_parts): 116 | """ 117 | Helper function to format and quote a single header. 118 | 119 | Useful for single headers that are composed of multiple items. E.g., 120 | 'Content-Disposition' fields. 121 | 122 | :param header_parts: 123 | A sequence of (k, v) typles or a :class:`dict` of (k, v) to format 124 | as `k1="v1"; k2="v2"; ...`. 125 | """ 126 | parts = [] 127 | iterable = header_parts 128 | if isinstance(header_parts, dict): 129 | iterable = header_parts.items() 130 | 131 | for name, value in iterable: 132 | if value: 133 | parts.append(self._render_part(name, value)) 134 | 135 | return '; '.join(parts) 136 | 137 | def render_headers(self): 138 | """ 139 | Renders the headers for this request field. 140 | """ 141 | lines = [] 142 | 143 | sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] 144 | for sort_key in sort_keys: 145 | if self.headers.get(sort_key, False): 146 | lines.append('%s: %s' % (sort_key, self.headers[sort_key])) 147 | 148 | for header_name, header_value in self.headers.items(): 149 | if header_name not in sort_keys: 150 | if header_value: 151 | lines.append('%s: %s' % (header_name, header_value)) 152 | 153 | lines.append('\r\n') 154 | return '\r\n'.join(lines) 155 | 156 | def make_multipart(self, content_disposition=None, content_type=None, 157 | content_location=None): 158 | """ 159 | Makes this request field into a multipart request field. 160 | 161 | This method overrides "Content-Disposition", "Content-Type" and 162 | "Content-Location" headers to the request parameter. 163 | 164 | :param content_type: 165 | The 'Content-Type' of the request body. 166 | :param content_location: 167 | The 'Content-Location' of the request body. 168 | 169 | """ 170 | self.headers['Content-Disposition'] = content_disposition or 'form-data' 171 | self.headers['Content-Disposition'] += '; '.join([ 172 | '', self._render_parts( 173 | (('name', self._name), ('filename', self._filename)) 174 | ) 175 | ]) 176 | self.headers['Content-Type'] = content_type 177 | self.headers['Content-Location'] = content_location 178 | -------------------------------------------------------------------------------- /lib/connection/Requester.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # This program is free software; you can redistribute it and/or modify 3 | # it under the terms of the GNU General Public License as published by 4 | # the Free Software Foundation; either version 2 of the License, or 5 | # (at your option) any later version. 6 | # 7 | # This program is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 | # GNU General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU General Public License 13 | # along with this program; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, 15 | # MA 02110-1301, USA. 16 | # 17 | # Author: Mauro Soria 18 | 19 | import random 20 | import urllib.parse 21 | import socket 22 | import http.client 23 | import urllib.request 24 | import urllib.parse 25 | import urllib.error 26 | import time 27 | 28 | import thirdparty.requests as requests 29 | from .Response import * 30 | from .RequestException import * 31 | 32 | 33 | class Requester(object): 34 | 35 | headers = { 36 | 'User-agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36', 37 | 'Accept-Language': 'en-us', 38 | 'Accept-Encoding': 'identity', 39 | 'Keep-Alive': '300', 40 | 'Connection': 'keep-alive', 41 | 'Cache-Control': 'max-age=0', 42 | } 43 | 44 | def __init__(self, url, cookie=None, useragent=None, maxPool=1, maxRetries=5, delay=0, timeout=30, ip=None, proxy=None, 45 | redirect=False, requestByHostname=False): 46 | # if no backslash, append one 47 | if not url.endswith('/'): 48 | url = url + '/' 49 | parsed = urllib.parse.urlparse(url) 50 | self.basePath = parsed.path 51 | 52 | # if not protocol specified, set http by default 53 | if parsed.scheme != 'http' and parsed.scheme != 'https': 54 | parsed = urllib.parse.urlparse('http://' + url) 55 | self.basePath = parsed.path 56 | self.protocol = parsed.scheme 57 | if self.protocol != 'http' and self.protocol != 'https': 58 | self.protocol = 'http' 59 | self.host = parsed.netloc.split(':')[0] 60 | 61 | # resolve DNS to decrease overhead 62 | if ip is not None: 63 | self.ip = ip 64 | else: 65 | try: 66 | self.ip = socket.gethostbyname(self.host) 67 | except socket.gaierror: 68 | raise RequestException({'message': "Couldn't resolve DNS"}) 69 | self.headers['Host'] = self.host 70 | 71 | # If no port specified, set default (80, 443) 72 | try: 73 | self.port = parsed.netloc.split(':')[1] 74 | except IndexError: 75 | self.port = (443 if self.protocol == 'https' else 80) 76 | 77 | # Set cookie and user-agent headers 78 | if cookie is not None: 79 | self.setHeader('Cookie', cookie) 80 | if useragent is not None: 81 | self.setHeader('User-agent', useragent) 82 | self.maxRetries = maxRetries 83 | self.maxPool = maxPool 84 | self.delay = delay 85 | self.timeout = timeout 86 | self.pool = None 87 | self.proxy = proxy 88 | self.redirect = redirect 89 | self.randomAgents = None 90 | self.requestByHostname = requestByHostname 91 | self.session = requests.Session() 92 | 93 | def setHeader(self, header, content): 94 | self.headers[header] = content 95 | 96 | def setRandomAgents(self, agents): 97 | self.randomAgents = list(agents) 98 | 99 | def unsetRandomAgents(self): 100 | self.randomAgents = None 101 | 102 | def request(self, path): 103 | i = 0 104 | proxy = None 105 | result = None 106 | while i <= self.maxRetries: 107 | try: 108 | if self.proxy is not None: 109 | proxy = {"https" : self.proxy, "http" : self.proxy} 110 | if self.requestByHostname: 111 | url = "{0}://{1}:{2}".format(self.protocol, self.host, self.port) 112 | else: 113 | url = "{0}://{1}:{2}".format(self.protocol, self.ip, self.port) 114 | url = urllib.parse.urljoin(url, self.basePath) 115 | 116 | # Joining with concatenation because a urljoin bug with "::" 117 | if not url.endswith('/'): 118 | url += "/" 119 | if path.startswith('/'): 120 | path = path[1:] 121 | url += path 122 | 123 | headers = dict(self.headers) 124 | if self.randomAgents is not None: 125 | headers["User-agent"] = random.choice(self.randomAgents) 126 | headers["Host"] = self.host 127 | # include port in Host header if it's non-standard 128 | if (self.protocol == "https" and self.port != 443) or (self.protocol == "http" and self.port != 80): 129 | headers["Host"]+=":{0}".format(self.port) 130 | 131 | response = self.session.get(url, proxies=proxy, verify=False, allow_redirects=self.redirect, \ 132 | headers=headers, timeout=self.timeout) 133 | result = Response(response.status_code, response.reason, response.headers, response.content) 134 | time.sleep(self.delay) 135 | del headers 136 | break 137 | except requests.exceptions.TooManyRedirects as e: 138 | raise RequestException({'message': 'Too many redirects: {0}'.format(e)}) 139 | except ConnectionResetError as e: 140 | raise RequestException({'message': 'ConnectionResetError: {0}'.format(e)}) 141 | except requests.exceptions.ConnectionError as e: 142 | if self.proxy is not None: 143 | raise RequestException({'message': 'Error with the proxy: {0}'.format(e)}) 144 | continue 145 | except (requests.exceptions.ConnectTimeout, requests.exceptions.ReadTimeout, requests.exceptions.Timeout, http.client.IncompleteRead, \ 146 | socket.timeout): 147 | continue 148 | finally: 149 | i = i + 1 150 | if i > self.maxRetries: 151 | raise RequestException(\ 152 | {'message': 'CONNECTION TIMEOUT: There was a problem in the request to: {0}'.format(path)} 153 | ) 154 | return result 155 | -------------------------------------------------------------------------------- /thirdparty/requests/packages/urllib3/util/url.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | from ..exceptions import LocationParseError 4 | 5 | 6 | url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] 7 | 8 | 9 | class Url(namedtuple('Url', url_attrs)): 10 | """ 11 | Datastructure for representing an HTTP URL. Used as a return value for 12 | :func:`parse_url`. 13 | """ 14 | slots = () 15 | 16 | def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, 17 | query=None, fragment=None): 18 | if path and not path.startswith('/'): 19 | path = '/' + path 20 | return super(Url, cls).__new__(cls, scheme, auth, host, port, path, 21 | query, fragment) 22 | 23 | @property 24 | def hostname(self): 25 | """For backwards-compatibility with urlparse. We're nice like that.""" 26 | return self.host 27 | 28 | @property 29 | def request_uri(self): 30 | """Absolute path including the query string.""" 31 | uri = self.path or '/' 32 | 33 | if self.query is not None: 34 | uri += '?' + self.query 35 | 36 | return uri 37 | 38 | @property 39 | def netloc(self): 40 | """Network location including host and port""" 41 | if self.port: 42 | return '%s:%d' % (self.host, self.port) 43 | return self.host 44 | 45 | @property 46 | def url(self): 47 | """ 48 | Convert self into a url 49 | 50 | This function should more or less round-trip with :func:`.parse_url`. The 51 | returned url may not be exactly the same as the url inputted to 52 | :func:`.parse_url`, but it should be equivalent by the RFC (e.g., urls 53 | with a blank port will have : removed). 54 | 55 | Example: :: 56 | 57 | >>> U = parse_url('http://google.com/mail/') 58 | >>> U.url 59 | 'http://google.com/mail/' 60 | >>> Url('http', 'username:password', 'host.com', 80, 61 | ... '/path', 'query', 'fragment').url 62 | 'http://username:password@host.com:80/path?query#fragment' 63 | """ 64 | scheme, auth, host, port, path, query, fragment = self 65 | url = '' 66 | 67 | # We use "is not None" we want things to happen with empty strings (or 0 port) 68 | if scheme is not None: 69 | url += scheme + '://' 70 | if auth is not None: 71 | url += auth + '@' 72 | if host is not None: 73 | url += host 74 | if port is not None: 75 | url += ':' + str(port) 76 | if path is not None: 77 | url += path 78 | if query is not None: 79 | url += '?' + query 80 | if fragment is not None: 81 | url += '#' + fragment 82 | 83 | return url 84 | 85 | def __str__(self): 86 | return self.url 87 | 88 | def split_first(s, delims): 89 | """ 90 | Given a string and an iterable of delimiters, split on the first found 91 | delimiter. Return two split parts and the matched delimiter. 92 | 93 | If not found, then the first part is the full input string. 94 | 95 | Example:: 96 | 97 | >>> split_first('foo/bar?baz', '?/=') 98 | ('foo', 'bar?baz', '/') 99 | >>> split_first('foo/bar?baz', '123') 100 | ('foo/bar?baz', '', None) 101 | 102 | Scales linearly with number of delims. Not ideal for large number of delims. 103 | """ 104 | min_idx = None 105 | min_delim = None 106 | for d in delims: 107 | idx = s.find(d) 108 | if idx < 0: 109 | continue 110 | 111 | if min_idx is None or idx < min_idx: 112 | min_idx = idx 113 | min_delim = d 114 | 115 | if min_idx is None or min_idx < 0: 116 | return s, '', None 117 | 118 | return s[:min_idx], s[min_idx+1:], min_delim 119 | 120 | 121 | def parse_url(url): 122 | """ 123 | Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is 124 | performed to parse incomplete urls. Fields not provided will be None. 125 | 126 | Partly backwards-compatible with :mod:`urlparse`. 127 | 128 | Example:: 129 | 130 | >>> parse_url('http://google.com/mail/') 131 | Url(scheme='http', host='google.com', port=None, path='/mail/', ...) 132 | >>> parse_url('google.com:80') 133 | Url(scheme=None, host='google.com', port=80, path=None, ...) 134 | >>> parse_url('/foo?bar') 135 | Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) 136 | """ 137 | 138 | # While this code has overlap with stdlib's urlparse, it is much 139 | # simplified for our needs and less annoying. 140 | # Additionally, this implementations does silly things to be optimal 141 | # on CPython. 142 | 143 | if not url: 144 | # Empty 145 | return Url() 146 | 147 | scheme = None 148 | auth = None 149 | host = None 150 | port = None 151 | path = None 152 | fragment = None 153 | query = None 154 | 155 | # Scheme 156 | if '://' in url: 157 | scheme, url = url.split('://', 1) 158 | 159 | # Find the earliest Authority Terminator 160 | # (http://tools.ietf.org/html/rfc3986#section-3.2) 161 | url, path_, delim = split_first(url, ['/', '?', '#']) 162 | 163 | if delim: 164 | # Reassemble the path 165 | path = delim + path_ 166 | 167 | # Auth 168 | if '@' in url: 169 | # Last '@' denotes end of auth part 170 | auth, url = url.rsplit('@', 1) 171 | 172 | # IPv6 173 | if url and url[0] == '[': 174 | host, url = url.split(']', 1) 175 | host += ']' 176 | 177 | # Port 178 | if ':' in url: 179 | _host, port = url.split(':', 1) 180 | 181 | if not host: 182 | host = _host 183 | 184 | if port: 185 | # If given, ports must be integers. 186 | if not port.isdigit(): 187 | raise LocationParseError(url) 188 | port = int(port) 189 | else: 190 | # Blank ports are cool, too. (rfc3986#section-3.2.3) 191 | port = None 192 | 193 | elif not host and url: 194 | host = url 195 | 196 | if not path: 197 | return Url(scheme, auth, host, port, path, query, fragment) 198 | 199 | # Fragment 200 | if '#' in path: 201 | path, fragment = path.split('#', 1) 202 | 203 | # Query 204 | if '?' in path: 205 | path, query = path.split('?', 1) 206 | 207 | return Url(scheme, auth, host, port, path, query, fragment) 208 | 209 | def get_host(url): 210 | """ 211 | Deprecated. Use :func:`.parse_url` instead. 212 | """ 213 | p = parse_url(url) 214 | return p.scheme or 'http', p.hostname, p.port 215 | -------------------------------------------------------------------------------- /thirdparty/colorama/winterm.py: -------------------------------------------------------------------------------- 1 | # Copyright Jonathan Hartley 2013. BSD 3-Clause license, see LICENSE file. 2 | from . import win32 3 | 4 | 5 | # from wincon.h 6 | class WinColor(object): 7 | BLACK = 0 8 | BLUE = 1 9 | GREEN = 2 10 | CYAN = 3 11 | RED = 4 12 | MAGENTA = 5 13 | YELLOW = 6 14 | GREY = 7 15 | 16 | # from wincon.h 17 | class WinStyle(object): 18 | NORMAL = 0x00 # dim text, dim background 19 | BRIGHT = 0x08 # bright text, dim background 20 | BRIGHT_BACKGROUND = 0x80 # dim text, bright background 21 | 22 | class WinTerm(object): 23 | 24 | def __init__(self): 25 | self._default = win32.GetConsoleScreenBufferInfo(win32.STDOUT).wAttributes 26 | self.set_attrs(self._default) 27 | self._default_fore = self._fore 28 | self._default_back = self._back 29 | self._default_style = self._style 30 | # In order to emulate LIGHT_EX in windows, we borrow the BRIGHT style. 31 | # So that LIGHT_EX colors and BRIGHT style do not clobber each other, 32 | # we track them separately, since LIGHT_EX is overwritten by Fore/Back 33 | # and BRIGHT is overwritten by Style codes. 34 | self._light = 0 35 | 36 | def get_attrs(self): 37 | return self._fore + self._back * 16 + (self._style | self._light) 38 | 39 | def set_attrs(self, value): 40 | self._fore = value & 7 41 | self._back = (value >> 4) & 7 42 | self._style = value & (WinStyle.BRIGHT | WinStyle.BRIGHT_BACKGROUND) 43 | 44 | def reset_all(self, on_stderr=None): 45 | self.set_attrs(self._default) 46 | self.set_console(attrs=self._default) 47 | 48 | def fore(self, fore=None, light=False, on_stderr=False): 49 | if fore is None: 50 | fore = self._default_fore 51 | self._fore = fore 52 | # Emulate LIGHT_EX with BRIGHT Style 53 | if light: 54 | self._light |= WinStyle.BRIGHT 55 | else: 56 | self._light &= ~WinStyle.BRIGHT 57 | self.set_console(on_stderr=on_stderr) 58 | 59 | def back(self, back=None, light=False, on_stderr=False): 60 | if back is None: 61 | back = self._default_back 62 | self._back = back 63 | # Emulate LIGHT_EX with BRIGHT_BACKGROUND Style 64 | if light: 65 | self._light |= WinStyle.BRIGHT_BACKGROUND 66 | else: 67 | self._light &= ~WinStyle.BRIGHT_BACKGROUND 68 | self.set_console(on_stderr=on_stderr) 69 | 70 | def style(self, style=None, on_stderr=False): 71 | if style is None: 72 | style = self._default_style 73 | self._style = style 74 | self.set_console(on_stderr=on_stderr) 75 | 76 | def set_console(self, attrs=None, on_stderr=False): 77 | if attrs is None: 78 | attrs = self.get_attrs() 79 | handle = win32.STDOUT 80 | if on_stderr: 81 | handle = win32.STDERR 82 | win32.SetConsoleTextAttribute(handle, attrs) 83 | 84 | def get_position(self, handle): 85 | position = win32.GetConsoleScreenBufferInfo(handle).dwCursorPosition 86 | # Because Windows coordinates are 0-based, 87 | # and win32.SetConsoleCursorPosition expects 1-based. 88 | position.X += 1 89 | position.Y += 1 90 | return position 91 | 92 | def set_cursor_position(self, position=None, on_stderr=False): 93 | if position is None: 94 | # I'm not currently tracking the position, so there is no default. 95 | # position = self.get_position() 96 | return 97 | handle = win32.STDOUT 98 | if on_stderr: 99 | handle = win32.STDERR 100 | win32.SetConsoleCursorPosition(handle, position) 101 | 102 | def cursor_adjust(self, x, y, on_stderr=False): 103 | handle = win32.STDOUT 104 | if on_stderr: 105 | handle = win32.STDERR 106 | position = self.get_position(handle) 107 | adjusted_position = (position.Y + y, position.X + x) 108 | win32.SetConsoleCursorPosition(handle, adjusted_position, adjust=False) 109 | 110 | def erase_screen(self, mode=0, on_stderr=False): 111 | # 0 should clear from the cursor to the end of the screen. 112 | # 1 should clear from the cursor to the beginning of the screen. 113 | # 2 should clear the entire screen, and move cursor to (1,1) 114 | handle = win32.STDOUT 115 | if on_stderr: 116 | handle = win32.STDERR 117 | csbi = win32.GetConsoleScreenBufferInfo(handle) 118 | # get the number of character cells in the current buffer 119 | cells_in_screen = csbi.dwSize.X * csbi.dwSize.Y 120 | # get number of character cells before current cursor position 121 | cells_before_cursor = csbi.dwSize.X * csbi.dwCursorPosition.Y + csbi.dwCursorPosition.X 122 | if mode == 0: 123 | from_coord = csbi.dwCursorPosition 124 | cells_to_erase = cells_in_screen - cells_before_cursor 125 | if mode == 1: 126 | from_coord = win32.COORD(0, 0) 127 | cells_to_erase = cells_before_cursor 128 | elif mode == 2: 129 | from_coord = win32.COORD(0, 0) 130 | cells_to_erase = cells_in_screen 131 | # fill the entire screen with blanks 132 | win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) 133 | # now set the buffer's attributes accordingly 134 | win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) 135 | if mode == 2: 136 | # put the cursor where needed 137 | win32.SetConsoleCursorPosition(handle, (1, 1)) 138 | 139 | def erase_line(self, mode=0, on_stderr=False): 140 | # 0 should clear from the cursor to the end of the line. 141 | # 1 should clear from the cursor to the beginning of the line. 142 | # 2 should clear the entire line. 143 | handle = win32.STDOUT 144 | if on_stderr: 145 | handle = win32.STDERR 146 | csbi = win32.GetConsoleScreenBufferInfo(handle) 147 | if mode == 0: 148 | from_coord = csbi.dwCursorPosition 149 | cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X 150 | if mode == 1: 151 | from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) 152 | cells_to_erase = csbi.dwCursorPosition.X 153 | elif mode == 2: 154 | from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) 155 | cells_to_erase = csbi.dwSize.X 156 | # fill the entire screen with blanks 157 | win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) 158 | # now set the buffer's attributes accordingly 159 | win32.FillConsoleOutputAttribute(handle, self.get_attrs(), cells_to_erase, from_coord) 160 | 161 | def set_title(self, title): 162 | win32.SetConsoleTitle(title) 163 | --------------------------------------------------------------------------------