├── .gitignore ├── README.md ├── scanner ├── client.py ├── lib │ ├── __init__.py │ ├── core │ │ ├── __init__.py │ │ ├── common.py │ │ ├── crawler.py │ │ ├── data.py │ │ ├── engine.py │ │ ├── envinit.py │ │ ├── failure.py │ │ ├── log.py │ │ ├── option.py │ │ ├── requests.py │ │ └── settings.py │ └── util │ │ ├── __init__.py │ │ └── torndb.py ├── scripts │ ├── README.txt │ ├── __init__.py │ ├── dic │ │ └── web_shell.dic │ ├── file_upload.py │ ├── inter_ip_leak.py │ ├── phpmyadmin_leak.py │ ├── robots_leak.py │ ├── sql_inject.py │ ├── webshell_check.py │ └── xss.py ├── sql │ ├── category.sql │ └── rule.sql ├── tests │ ├── __init__.py │ └── test_request.py ├── thirdparty │ ├── __init__.py │ └── requests │ │ ├── __init__.py │ │ ├── adapters.py │ │ ├── api.py │ │ ├── auth.py │ │ ├── cacert.pem │ │ ├── certs.py │ │ ├── compat.py │ │ ├── cookies.py │ │ ├── exceptions.py │ │ ├── hooks.py │ │ ├── models.py │ │ ├── packages │ │ ├── README.rst │ │ ├── __init__.py │ │ ├── chardet │ │ │ ├── __init__.py │ │ │ ├── big5freq.py │ │ │ ├── big5prober.py │ │ │ ├── chardetect.py │ │ │ ├── chardistribution.py │ │ │ ├── charsetgroupprober.py │ │ │ ├── charsetprober.py │ │ │ ├── codingstatemachine.py │ │ │ ├── compat.py │ │ │ ├── constants.py │ │ │ ├── cp949prober.py │ │ │ ├── escprober.py │ │ │ ├── escsm.py │ │ │ ├── eucjpprober.py │ │ │ ├── euckrfreq.py │ │ │ ├── euckrprober.py │ │ │ ├── euctwfreq.py │ │ │ ├── euctwprober.py │ │ │ ├── gb2312freq.py │ │ │ ├── gb2312prober.py │ │ │ ├── hebrewprober.py │ │ │ ├── jisfreq.py │ │ │ ├── jpcntx.py │ │ │ ├── langbulgarianmodel.py │ │ │ ├── langcyrillicmodel.py │ │ │ ├── langgreekmodel.py │ │ │ ├── langhebrewmodel.py │ │ │ ├── langhungarianmodel.py │ │ │ ├── langthaimodel.py │ │ │ ├── latin1prober.py │ │ │ ├── mbcharsetprober.py │ │ │ ├── mbcsgroupprober.py │ │ │ ├── mbcssm.py │ │ │ ├── sbcharsetprober.py │ │ │ ├── sbcsgroupprober.py │ │ │ ├── sjisprober.py │ │ │ ├── universaldetector.py │ │ │ └── utf8prober.py │ │ └── urllib3 │ │ │ ├── __init__.py │ │ │ ├── _collections.py │ │ │ ├── connection.py │ │ │ ├── connectionpool.py │ │ │ ├── contrib │ │ │ ├── __init__.py │ │ │ ├── ntlmpool.py │ │ │ └── pyopenssl.py │ │ │ ├── exceptions.py │ │ │ ├── fields.py │ │ │ ├── filepost.py │ │ │ ├── packages │ │ │ ├── __init__.py │ │ │ ├── ordered_dict.py │ │ │ ├── six.py │ │ │ └── ssl_match_hostname │ │ │ │ ├── __init__.py │ │ │ │ └── _implementation.py │ │ │ ├── poolmanager.py │ │ │ ├── request.py │ │ │ ├── response.py │ │ │ └── util │ │ │ ├── __init__.py │ │ │ ├── connection.py │ │ │ ├── request.py │ │ │ ├── response.py │ │ │ ├── ssl_.py │ │ │ ├── timeout.py │ │ │ └── url.py │ │ ├── sessions.py │ │ ├── status_codes.py │ │ ├── structures.py │ │ └── utils.py ├── topmgr.py └── topscan.py ├── screenshot ├── create.jpg ├── detail.png ├── task.png ├── tree.png └── vul.png ├── topweb ├── .gitignore ├── app_site │ ├── __init__.py │ ├── context_processor.py │ ├── models.py │ ├── static │ │ ├── bootstrap │ │ │ ├── css │ │ │ │ ├── bootstrap-theme.css │ │ │ │ ├── bootstrap-theme.min.css │ │ │ │ ├── bootstrap.css │ │ │ │ └── bootstrap.min.css │ │ │ ├── fonts │ │ │ │ ├── glyphicons-halflings-regular.eot │ │ │ │ ├── glyphicons-halflings-regular.svg │ │ │ │ ├── glyphicons-halflings-regular.ttf │ │ │ │ └── glyphicons-halflings-regular.woff │ │ │ └── js │ │ │ │ ├── bootstrap.js │ │ │ │ └── bootstrap.min.js │ │ ├── css │ │ │ └── base.css │ │ ├── img │ │ │ ├── H.png │ │ │ ├── L.png │ │ │ ├── M.png │ │ │ ├── complete.png │ │ │ ├── complete00.png │ │ │ ├── complete01.png │ │ │ ├── complete11.png │ │ │ ├── complete2.png │ │ │ ├── complete3.png │ │ │ ├── domain.png │ │ │ ├── file_sprite.png │ │ │ ├── running.gif │ │ │ ├── stop.png │ │ │ ├── stop1.png │ │ │ ├── wait.png │ │ │ ├── wait1.png │ │ │ └── wait2.png │ │ └── js │ │ │ ├── base.js │ │ │ ├── base2.js │ │ │ ├── bootbox.js │ │ │ ├── csrf.js │ │ │ ├── jquery-1.11.1.min.js │ │ │ └── jstree │ │ │ ├── jstree.js │ │ │ ├── jstree.min.js │ │ │ └── themes │ │ │ └── default │ │ │ ├── 32px.png │ │ │ ├── 40px.png │ │ │ ├── style.css │ │ │ ├── style.min.css │ │ │ └── throbber.gif │ ├── templatetags │ │ ├── __init__.py │ │ └── registration_bootstrap.py │ ├── util.py │ └── views.py ├── db.sqlite3 ├── manage.py ├── project_scan │ ├── __init__.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── rule.sql └── templates │ ├── base.html │ ├── footer.html │ ├── header.html │ ├── home │ ├── detail.html │ ├── dirstructure.html │ ├── home.html │ ├── home2.html │ ├── message.html │ ├── task.html │ └── task_template.html │ ├── nav.html │ └── registration │ ├── form_field.html │ ├── logged_out.html │ ├── login.html │ ├── password_change_done.html │ ├── password_change_form.html │ ├── password_reset_complete.html │ ├── password_reset_confirm.html │ ├── password_reset_done.html │ └── password_reset_form.html ├── 漏扫使用说明.doc └── 详细说明.doc /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # PyInstaller 26 | # Usually these files are written by a python script from a template 27 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 28 | *.manifest 29 | *.spec 30 | 31 | # Installer logs 32 | pip-log.txt 33 | pip-delete-this-directory.txt 34 | 35 | # Unit test / coverage reports 36 | htmlcov/ 37 | .tox/ 38 | .coverage 39 | .cache 40 | nosetests.xml 41 | coverage.xml 42 | 43 | # Translations 44 | *.mo 45 | *.pot 46 | 47 | # Django stuff: 48 | *.log 49 | 50 | # Sphinx documentation 51 | docs/_build/ 52 | 53 | # PyBuilder 54 | target/ 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | leakScan 2 | ======== 3 | 4 | 在线漏洞扫描 5 | 6 | 任务列表: 7 | 8 | ![image](https://github.com/Skycrab/leakScan/blob/master/screenshot/task.png) 9 | 10 | 11 | 扫描任务创建: 12 | 13 | ![image](https://github.com/Skycrab/leakScan/blob/master/screenshot/create.jpg) 14 | 15 | 扫描基本信息: 16 | 17 | ![image](https://github.com/Skycrab/leakScan/blob/master/screenshot/detail.png) 18 | 19 | 20 | 目录结构: 21 | 22 | ![image](https://github.com/Skycrab/leakScan/blob/master/screenshot/tree.png) 23 | 24 | 25 | 漏洞详情: 26 | 27 | ![image](https://github.com/Skycrab/leakScan/blob/master/screenshot/vul.png) 28 | 29 | 30 | 详细可看使用说明 31 | -------------------------------------------------------------------------------- /scanner/client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import hmac 5 | import gevent 6 | from gevent import monkey 7 | monkey.patch_socket() 8 | 9 | addr = ('localhost', 6667) 10 | 11 | def send_cmd(): 12 | socket = gevent.socket.socket() 13 | socket.connect(addr) 14 | s="module:SCAN_MODULE\naction:start\n\n" 15 | print s 16 | socket.send(s) 17 | print 'send over' 18 | print socket.recv(1024) 19 | socket.close() 20 | 21 | #send_cmd() 22 | 23 | 24 | def send_request(module_name,request_headers): 25 | SECRE_KEY = "_TOP-SEC_**_BEIJING-ANFU_" 26 | socket = gevent.socket.socket() 27 | socket.connect(addr) 28 | request_headers['module'] = module_name 29 | request_headers['signature'] = hmac.new(SECRE_KEY, module_name).hexdigest() 30 | h = ["%s:%s" %(k, v) for k,v in request_headers.iteritems()] 31 | h.append('\n') 32 | request = '\n'.join(h) 33 | socket.send(request) 34 | print socket.recv(8192) 35 | socket.close() 36 | 37 | if __name__ =="__main__": 38 | import sys 39 | if sys.argv[1] == '1': 40 | print 'start' 41 | send_request('SCAN_MODULE',{'action':'start','task_id':1}) 42 | else: 43 | print 'stop' 44 | send_request('SCAN_MODULE',{'action':'stop','task_id':1}) 45 | 46 | 47 | -------------------------------------------------------------------------------- /scanner/lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/lib/__init__.py -------------------------------------------------------------------------------- /scanner/lib/core/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/lib/core/__init__.py -------------------------------------------------------------------------------- /scanner/lib/core/common.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | import os 4 | import shutil 5 | from urlparse import urlsplit, urlunsplit, urljoin as _urljoin 6 | from posixpath import normpath 7 | from datetime import datetime 8 | 9 | from lib.core.log import ERROR, DEBUG, INFO 10 | from lib.core.data import paths, conf 11 | from lib.core.settings import IGNORE_DEFAULT_FILE_SUFFIX 12 | from lib.util import db 13 | 14 | 15 | def urljoin(base, url, allow_fragments=True): 16 | """ 17 | >>>urljoin("http://www.baidu.com","../../../dd") 18 | 'http://www.baidu.com/dd' 19 | >>>urljoin("http://www.baidu.com","/dd/./././") 20 | 'http://www.baidu.com/dd 21 | """ 22 | _ = _urljoin(base, url, allow_fragments=True) 23 | p = urlsplit(_) 24 | path = p.path + '/' if p.path.endswith('/') else p.path 25 | return urlunsplit((p.scheme,p.netloc,normpath(p.path),p.query,p.fragment)) 26 | 27 | 28 | def banner(): 29 | pass 30 | 31 | def showpaths(): 32 | """ 33 | print paths for convenient debugging 34 | """ 35 | print paths 36 | 37 | def mkdir(path,remove=True): 38 | if os.path.isdir(path): 39 | if remove: 40 | try: 41 | shutil.rmtree(path) 42 | os.mkdir(path) 43 | except Exception: 44 | ERROR("rmtree except,path"+path) 45 | else: 46 | os.mkdir(path) 47 | 48 | def discard(url): 49 | index = url.rfind('.') 50 | if index != -1 and url[index+1:] in IGNORE_DEFAULT_FILE_SUFFIX: 51 | return True 52 | return False 53 | 54 | 55 | def set_unreachable_flag(task_id): 56 | sql = "UPDATE task SET `reachable`=0 WHERE id=%s" % task_id 57 | try: 58 | db.execute(sql) 59 | except Exception: 60 | ERROR('set_unreachable failed,task_id:%s,please check' % task_id) 61 | 62 | def update_task_status(task_id): 63 | sql = "UPDATE task SET `status`=3 WHERE id=%s" % task_id 64 | try: 65 | db.execute(sql) 66 | except Exception: 67 | ERROR('update_task_status failed,task_id:%s,please check' % task_id) 68 | 69 | def update_end_time(task_id): 70 | sql = "UPDATE task SET `end_time`=%s WHERE id=%s" 71 | try: 72 | db.execute(sql, datetime.now(), task_id) 73 | except Exception: 74 | ERROR('update_end_time failed,task_id:%s,please check' % task_id) 75 | 76 | 77 | def task_finsh_clean(task_id=None): 78 | if task_id is None: 79 | task_id = conf.taskid 80 | 81 | update_task_status(task_id) 82 | update_end_time(task_id) 83 | 84 | 85 | 86 | -------------------------------------------------------------------------------- /scanner/lib/core/data.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import re 5 | from collections import namedtuple 6 | 7 | GET, POST = "GET", "POST" 8 | DEFAULT_METHOD = "GET" 9 | PARAMS_PATTERN = re.compile(r"(?P[^&=]+)(?:=(?P[^&=]*))?") 10 | TITLE_PATTERN = re.compile(r"(?P<title>[^<]+)", re.I) 11 | 12 | SITETYPES = {'PHP':'.php', 'JSP':'.jsp', 'ASP.NET':'.asp', 'ASP':'.asp'} 13 | IPADDRESS_PATTERN = re.compile(r"") 14 | 15 | 16 | class ObjectDict(dict): 17 | """Makes a dictionary behave like an object, with attribute-style access. 18 | """ 19 | def __getattr__(self, name): 20 | try: 21 | return self[name] 22 | except KeyError: 23 | raise AttributeError(name) 24 | 25 | def __setattr__(self, name, value): 26 | self[name] = value 27 | 28 | #store path info,e.g. "script path" 29 | paths = ObjectDict() 30 | 31 | #store cmdline options 32 | cmdLineOptions = ObjectDict() 33 | 34 | #store basic configuration,e.g. default timeout:10s 35 | conf = ObjectDict() 36 | 37 | 38 | class Url(object): 39 | def __init__(self,url,method,params,referer): 40 | self.url = url 41 | self.method = method.upper() 42 | self.params = params 43 | self.referer = referer 44 | 45 | @classmethod 46 | def fromUrl(cls,url,referer=''): 47 | """ 48 | provide http://www.example.com/?q=.. 49 | """ 50 | url, params = url.split('?',1) if url.find('?') != -1 else (url,'') 51 | return cls(url,DEFAULT_METHOD,params,referer) 52 | 53 | @property 54 | def name(self): 55 | return self.__class__.__name__ 56 | 57 | def __str__(self): 58 | return "<%s(%s,%s,%s,%s)>" %(self.name,self.url,self.method,self.params,self.referer) 59 | __repr__ = __str__ 60 | 61 | 62 | Result = namedtuple('Result','response details') 63 | 64 | 65 | 66 | -------------------------------------------------------------------------------- /scanner/lib/core/envinit.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import os 5 | import sys 6 | from lib.core.settings import SCRIPTS_NAME, SCRIPTS_DIC_NAME, TEMP_NAME, LOG_NAME 7 | from lib.core.data import paths 8 | 9 | def rootPath(path=__file__): 10 | #reference: http://www.py2exe.org/index.cgi/WhereAmI 11 | return os.path.dirname(unicode(sys.executable if hasattr(sys,"frozen") else path,sys.getfilesystemencoding())) 12 | 13 | def setPaths(): 14 | _ = paths.ROOT_PATH 15 | paths.SCRIPTS = os.path.join(_, SCRIPTS_NAME) 16 | paths.FIlELOG = os.path.join(_, LOG_NAME) 17 | paths.TEMP = os.path.join(_, TEMP_NAME) 18 | paths.DIC = os.path.join(paths.SCRIPTS, SCRIPTS_DIC_NAME) 19 | 20 | def envinit(path): 21 | paths.ROOT_PATH = rootPath(path) 22 | setPaths() 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /scanner/lib/core/failure.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | class TopException(Exception): 5 | pass 6 | 7 | class DestinationUnReachable(TopException): 8 | def __init__(self,dest): 9 | self.args = dest 10 | self.dest = dest 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /scanner/lib/core/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import logging 5 | import sys 6 | import traceback 7 | 8 | from lib.core.data import paths 9 | 10 | FORMATTER = logging.Formatter("\r[%(asctime)s] [%(levelname)s] %(message)s", "%Y-%m-%d %H:%M:%S") 11 | 12 | LOGGER = logging.getLogger("TopLog") 13 | 14 | FILE_HANDLER = logging.FileHandler(paths.FIlELOG) 15 | FILE_HANDLER.setFormatter(FORMATTER) 16 | 17 | STDOUT_HANDLER = logging.StreamHandler(sys.stdout) 18 | STDOUT_HANDLER.setFormatter(FORMATTER) 19 | 20 | LOGGER.addHandler(FILE_HANDLER) 21 | LOGGER.addHandler(STDOUT_HANDLER) 22 | LOGGER.setLevel(logging.DEBUG) 23 | 24 | def _error(msg): 25 | if any(sys.exc_info()): 26 | LOGGER.error("\n".join((msg,traceback.format_exc()))) 27 | sys.exc_clear() 28 | else: 29 | LOGGER.error(msg+",But no exception detected,please check") 30 | 31 | 32 | ERROR = _error 33 | DEBUG = LOGGER.debug 34 | INFO = LOGGER.info 35 | WARN = LOGGER.warn 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /scanner/lib/core/option.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import os 5 | import sys 6 | import urlparse 7 | import gevent 8 | from gevent.monkey import patch_all 9 | 10 | from optparse import OptParseError 11 | from optparse import OptionParser 12 | 13 | from lib.core.data import cmdLineOptions ,conf ,paths , SITETYPES 14 | from lib.core.settings import CONNECTION_TIMEOUT, NETWORK_TIMEOUT, TASK_TABLE 15 | from lib.core.log import ERROR, DEBUG, INFO 16 | from lib.core.common import mkdir, set_unreachable_flag 17 | from lib.core.crawler import CrawlEngine 18 | from lib.core.requests import request 19 | from lib.core.failure import TopException, DestinationUnReachable 20 | from lib.util import db 21 | 22 | def get_target(task_id): 23 | sql = 'SELECT * FROM %s WHERE `ID`=%s' %(TASK_TABLE, task_id) 24 | task = db.get(sql) 25 | if not conf.url: 26 | conf.url = task.start_url 27 | conf.base = task.base 28 | conf.count = task.url_count 29 | conf.finished_progress = task.progress.split('|') if conf.goon else [] 30 | conf.robots_parsed = task.robots_parsed if conf.goon else False 31 | conf.sitemap_parsed = task.sitemap_parsed if conf.goon else False 32 | conf.spider_finish = True if conf.goon and task.spider_flag == 3 else False 33 | 34 | 35 | def _confsetting(): 36 | conf.update(cmdLineOptions) 37 | 38 | if not conf.connect_timeout: 39 | conf.connect_timeout = CONNECTION_TIMEOUT 40 | if not conf.timeout: 41 | conf.timeout = NETWORK_TIMEOUT 42 | 43 | get_target(conf.taskid) 44 | 45 | parser = urlparse.urlsplit(conf.url) 46 | conf.host = parser.netloc 47 | conf.scheme = parser.scheme 48 | conf.domain = "%s://%s%s" % (parser.scheme, parser.netloc,conf.base) 49 | conf.requestCache = os.path.join(paths.TEMP,conf.host) 50 | conf.site_type = None 51 | print conf 52 | 53 | def _geventpatch(): 54 | """ 55 | do monkey.patch_all carefully 56 | """ 57 | patch_all(socket=True, dns=True, time=True, select=True, thread=False, os=True, ssl=True, httplib=False, 58 | subprocess=False, sys=False, aggressive=True, Event=False) 59 | 60 | _dnscache = {} 61 | def _setDnsCache(): 62 | """ 63 | set dns cache for socket.getaddrinfo and gevent to avoid subsequent DNS requests 64 | """ 65 | def _getaddrinfo(*args, **kwargs): 66 | if args in _dnscache: 67 | #DEBUG(str(args)+' in cache') 68 | return _dnscache[args] 69 | 70 | else: 71 | #DEBUG(str(args)+' not in cache') 72 | _dnscache[args] = gevent.socket._getaddrinfo(*args, **kwargs) 73 | return _dnscache[args] 74 | 75 | if not hasattr(gevent.socket, '_getaddrinfo'): 76 | gevent.socket._getaddrinfo = gevent.socket.getaddrinfo 77 | gevent.socket.getaddrinfo = _getaddrinfo 78 | 79 | def _mkcachedir(): 80 | mkdir(conf.requestCache) 81 | 82 | def changesysEncoding(encodeing='utf-8'): 83 | import sys 84 | reload(sys) 85 | sys.setdefaultencoding(encodeing) 86 | 87 | def destReachable(dest=None): 88 | if not dest: 89 | dest = conf.url 90 | 91 | response = request(dest,timeout=conf.connect_timeout) 92 | if response is None: 93 | set_unreachable_flag(conf.taskid) 94 | raise DestinationUnReachable(dest) 95 | else: 96 | conf.site_type = sitetype_check(response) 97 | 98 | def check_type(msg): 99 | for k in SITETYPES.iterkeys(): 100 | if msg.find(k) != -1: 101 | return k 102 | 103 | def sitetype_check(response): 104 | site_type = None 105 | if 'x-powered-by' in response.headers: 106 | site_type = check_type(response.headers['x-powered-by'].upper()) 107 | if site_type is None: 108 | if 'server' in response.headers: 109 | site_type = check_type(response.headers['server'].upper()) 110 | return site_type 111 | 112 | def init(): 113 | _confsetting() 114 | _mkcachedir() 115 | _geventpatch() 116 | _setDnsCache() 117 | changesysEncoding() 118 | destReachable() 119 | 120 | def run(): 121 | CrawlEngine.start() 122 | 123 | 124 | def parseCmdline(): 125 | """ 126 | parse command line parameters and arguments and store in cmdLineOptions 127 | """ 128 | usage = "%s %s [options]" %("python",sys.argv[0]) 129 | parser = OptionParser(usage=usage) 130 | parser.add_option("-t","--task",dest="taskid",help="task id") 131 | parser.add_option("-u","--url",dest="url",help="target url") 132 | parser.add_option("-b","--base",dest="base",default="/",help="the base directory of the domain") 133 | parser.add_option("-d","--depth",dest="depth",type="int",default=0,help="crawl depth") 134 | parser.add_option("-c","--count",dest="count",type="int",default=0,help="crawl url max count") 135 | parser.add_option("--cookie",dest="cookie",help="http cookie header") 136 | parser.add_option("--connect_timeout",dest="connect_timeout",help="set connect timeout") 137 | parser.add_option("--timeout",dest="timeout",help="network timeout") 138 | parser.add_option("--continue",action="store_true",dest="goon",help="task continue run") 139 | try: 140 | args,_ = parser.parse_args(sys.argv[1:]) 141 | cmdLineOptions.update(args.__dict__) 142 | print cmdLineOptions 143 | except OptParseError: 144 | print parser.error() 145 | 146 | 147 | -------------------------------------------------------------------------------- /scanner/lib/core/settings.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | ################## mysql setting ################## 5 | MYSQL_HOST = "127.0.0.1" 6 | MYSQL_PORT = 3306 7 | MYSQL_USERNAME = "root" 8 | MYSQL_PASSWD = "" 9 | MYSQL_DATABASE = "kehan" 10 | 11 | ################## rule setting ################## 12 | TASK_TABLE = 'task' #任务表名 13 | RULE_TABLE = "rule" #规则表名 14 | URL_TABLE = "url" 15 | RESULT_TABLE = "result" 16 | RUN_URL_DEFAULT_FUN = "run_url" 17 | RUN_DOMAIN_DEFAULT_FUN = "run_domain" 18 | 19 | ################## path setting ################## 20 | SCRIPTS_NAME = "scripts" #规则目录 21 | SCRIPTS_DIC_NAME = "dic" #字典目录 22 | TEMP_NAME = "temp" #缓存文件 23 | LOG_NAME = "toplog.log" 24 | 25 | ################## gloabl default setting ################## 26 | DEFAULT_PAGE_ENCODING = "utf8" 27 | HEADER_BODY_BOUNDRY = "" 28 | 29 | ################## cmdline default setting ################## 30 | # may change by cmdline 31 | CONNECTION_TIMEOUT = 10 32 | NETWORK_TIMEOUT = 10 33 | 34 | ################## crawler default setting ################## 35 | IGNORE_DEFAULT_FILE_SUFFIX = ('jpg','png','jpeg','png','jpg','gif','bmp','svg', 36 | 'exe','rar','zip', 37 | 'js','css') 38 | 39 | 40 | ################## dic name setting ################## 41 | WEBSHELL_DIC_NAME = "web_shell.dic" 42 | 43 | -------------------------------------------------------------------------------- /scanner/lib/util/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | from lib.core.settings import MYSQL_HOST,MYSQL_PORT,MYSQL_USERNAME,MYSQL_PASSWD,MYSQL_DATABASE 5 | 6 | import torndb 7 | 8 | _host = "%s:%s" % (MYSQL_HOST,MYSQL_PORT) 9 | db = torndb.Connection(_host,MYSQL_DATABASE,MYSQL_USERNAME,MYSQL_PASSWD) -------------------------------------------------------------------------------- /scanner/scripts/README.txt: -------------------------------------------------------------------------------- 1 | plugins--规则脚本 2 | 规则脚本命名规范： 3 | 都用小写，用下划线隔开，只能是字母数字下划线 4 | 5 | 规则脚本分为两类 6 | 1.基于域名扫描的 7 | 不需要爬虫爬取的链接，如：cms指纹识别规则 8 | 如:inter_ip_leak.py, robots_leak.py 9 | 10 | 2.基于url扫描的 11 | 需要爬虫爬取的链接以及详细参数 12 | 如：sql_inject 13 | -------------------------------------------------------------------------------- /scanner/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/scripts/__init__.py -------------------------------------------------------------------------------- /scanner/scripts/dic/web_shell.dic: -------------------------------------------------------------------------------- 1 | 1 2 | 2 3 | 3 4 | 4 5 | 5 6 | 6 7 | 7 8 | 8 9 | 9 10 | 10 11 | 404 12 | data 13 | tools 14 | index0 15 | sh3ll 16 | shell 17 | shel 18 | she 19 | shell1 20 | shell99 21 | root 22 | rootshell 23 | bypass 24 | anonym0us 25 | anonymous 26 | shellnymous 27 | fuck 28 | system 29 | a 30 | b 31 | c 32 | abc 33 | d 34 | e 35 | f 36 | g 37 | h 38 | i 39 | j 40 | k 41 | l 42 | m 43 | n 44 | o 45 | p 46 | y 47 | z 48 | webshell 49 | hack 50 | h4ck -------------------------------------------------------------------------------- /scanner/scripts/file_upload.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import re 5 | 6 | from lib.core.data import Result 7 | from lib.core.requests import requestUrl 8 | 9 | _FILE_UPLOAD = re.compile(r"]+?type\s*=\s*(([\'\"])file\2|file)",re.I|re.M) 10 | def run_url(req,rule): 11 | req = requestUrl(req) 12 | if req and _FILE_UPLOAD.search(req.text): 13 | #print '--------------_FILE_UPLOAD------------------' 14 | return Result(req,'') 15 | -------------------------------------------------------------------------------- /scanner/scripts/inter_ip_leak.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import re 5 | 6 | from lib.core.data import Result 7 | from lib.core.requests import requestUrl 8 | 9 | _INNER_IPADDR = re.compile(r"\b((?:10\.\d|172\.(1[6-9]|2\d|3[0-1])|192\.168)(?:\.\d+){2})\b") 10 | def run_url(req,rule): 11 | """ 12 | req: 13 | url: http://www.example.com 14 | method: get/post 15 | params: name=skycrab&age24 16 | referer: http://www.example.com/refer.html 17 | rule: 18 | domain: http://www.example.com/ (scheme+host+basepath) 19 | """ 20 | req = requestUrl(req) 21 | if req: 22 | details = [match.group(1) for line in req.iter_lines() for match in _INNER_IPADDR.finditer(line) if all(0<=int(x)<=255 for x in match.group(1).split('.'))] 23 | if details: 24 | return Result(req, details) 25 | 26 | 27 | 28 | 29 | 30 | 31 | -------------------------------------------------------------------------------- /scanner/scripts/phpmyadmin_leak.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import re 5 | from lib.core.data import Result 6 | from lib.core.requests import request 7 | 8 | _PHPMYADMIN_VERSION = re.compile(r"

.*?phpMyAdmin(.+?)

",re.I) 9 | def run_domain(rule): 10 | admin = ('phpmyadmin','phpMyAdmin','db') 11 | keys = ('
', 12 | '|||<[^>]+>|\s+") 20 | 21 | DBMS_ERRORS = { 22 | "MySQL": (r"SQL syntax.*MySQL", r"Warning.*mysql_.*", r"valid MySQL result", r"MySqlClient\.", r"com\.mysql\.jdbc\.exceptions"), 23 | "PostgreSQL": (r"PostgreSQL.*ERROR", r"Warning.*\Wpg_.*", r"valid PostgreSQL result", r"Npgsql\.",r"org\.postgresql\.util\.PSQLException"), 24 | "Microsoft SQL Server": (r"Driver.* SQL[\-\_\ ]*Server", r"OLE DB.* SQL Server", r"(\W|\A)SQL Server.*Driver", r"Warning.*mssql_.*", r"(\W|\A)SQL Server.*[0-9a-fA-F]{8}", r"(?s)Exception.*\WSystem\.Data\.SqlClient\.", r"(?s)Exception.*\WRoadhouse\.Cms\."), 25 | "Microsoft Access": (r"Microsoft Access (\d+ )?Driver", r"JET Database Engine", r"Access Database Engine"), 26 | "Oracle": (r"ORA-[0-9][0-9][0-9][0-9]", r"Oracle error", r"Oracle.*Driver", r"Warning.*\Woci_.*", r"Warning.*\Wora_.*"), 27 | "IBM DB2": (r"CLI Driver.*DB2", r"DB2 SQL error", r"db2_\w+\("), 28 | "Informix": (r"Exception.*Informix",), 29 | "Firebird": (r"Dynamic SQL Error", r"Warning.*ibase_.*"), 30 | "SQLite": (r"SQLite/JDBCDriver", r"SQLite.Exception", r"System.Data.SQLite.SQLiteException", r"Warning.*sqlite_.*", r"Warning.*SQLite3::", r"\[SQLITE_ERROR\]"), 31 | "SAP MaxDB": (r"SQL error.*POS([0-9]+).*", r"Warning.*maxdb.*"), 32 | "Sybase": (r"(?i)Warning.*sybase.*", r"Sybase message", r"Sybase.*Server message.*"), 33 | "Ingres": (r"Warning.*ingres_", r"Ingres SQLSTATE", r"Ingres\W.*Driver"), 34 | "Frontbase": (r"Exception (condition )?\d+. Transaction rollback.",), 35 | "HSQLDB": (r"org\.hsqldb\.jdbc",) 36 | } 37 | 38 | def retrieve_content(req, payloads=None, **kwargs): 39 | retval = None 40 | res = requestUrl(req, payloads, **kwargs) 41 | if res: 42 | retval = {} 43 | retval[RESPONSE] = res 44 | retval[HTTPCODE] = res.status_code 45 | retval[HTML] = res.text 46 | match = TITLE_PATTERN.search(retval[HTML]) 47 | retval[TITLE] = match.group('title') if match else None 48 | retval[TEXT] = TEXT_PATTERN.sub(" ", retval[HTML]) 49 | return retval 50 | 51 | def sql_error_check(content): 52 | for (dbms, regex) in ((dbms, regex) for dbms in DBMS_ERRORS for regex in DBMS_ERRORS[dbms]): 53 | if re.search(regex, content, re.I): 54 | # print '***********' 55 | # print regex,dbms 56 | # print '***************' 57 | return dbms 58 | 59 | def run_url(req,rule): 60 | vulnerable = False 61 | details = [] 62 | response = None 63 | params = req.params 64 | for match in PARAMS_PATTERN.finditer(params): 65 | # sql error 66 | tampered = params.replace(match.group('value'), "%s%s" % (match.group('value'), "".join(random.sample(TAMPER_SQL_CHAR_POOL, len(TAMPER_SQL_CHAR_POOL))))) 67 | content = retrieve_content(req,tampered) 68 | if content is not None: 69 | dbms = sql_error_check(content[HTML]) 70 | if dbms: 71 | details.append(u"错误模式注入，数据库类型：%s，注入参数：%s" % (dbms, match.group('key'))) 72 | if response is None: 73 | response = content[RESPONSE] 74 | continue 75 | 76 | # cookie inject 77 | 78 | # referer inject 79 | 80 | # blind sql inject 81 | original = retrieve_content(req) 82 | if original is None: 83 | continue 84 | left, right = random.sample(xrange(256), 2) 85 | vulnerable = False 86 | for prefix, boolean, suffix in itertools.product(PREFIXES, BOOLEAN_TESTS, SUFFIXES): 87 | if not vulnerable: 88 | template = "%s%s%s" % (prefix, boolean, suffix) 89 | payloads = dict((x, params.replace(match.group('value'), "%s%s" % (match.group('value'), (template % (left, left if x else right))))) for x in (True, False)) 90 | contents = dict((x, retrieve_content(req, payloads[x])) for x in (True, False)) 91 | 92 | if any(contents[x] is None for x in (True, False)): 93 | continue 94 | 95 | if any(original[x] == contents[True][x] != contents[False][x] for x in (HTTPCODE, TITLE)) or len(original[TEXT]) == len(contents[True][TEXT]) != len(contents[False][TEXT]): 96 | vulnerable = True 97 | else: 98 | ratios = dict((x, difflib.SequenceMatcher(None, original[TEXT], contents[x][TEXT]).quick_ratio()) for x in (True, False)) 99 | vulnerable = ratios[True] > FUZZY_THRESHOLD and ratios[False] < FUZZY_THRESHOLD 100 | if vulnerable: 101 | details.append(u"盲注,注入参数：%s" % match.group('key')) 102 | if response is None: 103 | response = contents[False][RESPONSE] 104 | #end for 105 | #end for 106 | 107 | if response is not None: 108 | return Result(response,details) 109 | 110 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /scanner/scripts/webshell_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | import os 5 | import re 6 | from lib.core.data import Result, SITETYPES 7 | from lib.core.requests import request 8 | from lib.core.data import paths, conf 9 | from lib.core.settings import WEBSHELL_DIC_NAME 10 | 11 | WEBSHELL_FEATURE = { 12 | 'PHP': 13 | { 14 | '404.php':("404 Not Found", "Password'), 17 | 'Ani-Shell.php':('',"--Ani Shell-") 18 | 19 | }, 20 | 'JSP': 21 | { 22 | 23 | }, 24 | 'ASP': 25 | { 26 | 27 | }, 28 | 'ASP.NET': 29 | { 30 | 31 | } 32 | } 33 | 34 | 35 | _INPUT_TYPE = re.compile(r"]+?(>|/>)",re.I|re.M) 36 | _IP_PATTERN = re.compile(r"\D(\d{1,3}(?:.\d{1,3}){3})\D") 37 | _INPUT_NAMR = re.compile(r"]+?name\s*=\s*[\'\"\w]",re.I) 38 | _DIR_PATH = re.compile(r"[CDEFG]:|/\w",re.I) 39 | 40 | def run_url(req, rule): 41 | print '**********webshell*******' 42 | print rule.site_type 43 | if req.params != '' or not req.url.endswith('/'): 44 | return None 45 | 46 | suffix = SITETYPES[rule.site_type] if rule.site_type is not None else None 47 | sure = [] 48 | doubt = [] 49 | response = None 50 | for line in open(os.path.join(paths.DIC, WEBSHELL_DIC_NAME)): 51 | line = line.strip() # remove \n 52 | if suffix is None: 53 | urls = ["%s%s" % (line, t) for t in SITETYPES.itervalues()] 54 | else: 55 | urls = ["%s%s" % (line, suffix)] 56 | 57 | for u in urls: 58 | url = "%s%s" %(req.url, u) 59 | print url 60 | res = request(url) 61 | if res and res.status_code == 200: 62 | text = res.text 63 | ## feature 64 | if rule.site_type is not None and rule.site_type in WEBSHELL_FEATURE: 65 | features = WEBSHELL_FEATURE[rule.site_type] 66 | if all(text.find(key) != -1 for keys in features.itervalues() for key in keys): 67 | if response is None: 68 | response = res 69 | sure.append(url) 70 | continue 71 | 72 | 73 | ## one input 74 | match = _INPUT_TYPE.findall(text) 75 | if match and len(match) == 2: 76 | if response is None: 77 | response = res 78 | doubt.append(url) 79 | continue 80 | 81 | ## match regur expression 82 | if (match.group(1) for match in _IP_PATTERN.finditer(text) if all(0<=int(x)<=255 for x in match.group(1).split('.'))) \ 83 | and _INPUT_NAMR.search(text) \ 84 | and _DIR_PATH.search(text) : 85 | if response is None: 86 | response = res 87 | doubt.append(url) 88 | 89 | 90 | if response is not None: 91 | if sure: 92 | sure.insert(0,u"下列url是WebShell，请及时清理") 93 | if doubt: 94 | doubt.insert(0,u"下列url疑是为WebShell，请仔细检查!") 95 | return Result(response,sure+doubt) 96 | 97 | -------------------------------------------------------------------------------- /scanner/scripts/xss.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | import re 4 | import string 5 | import random 6 | 7 | from lib.core.data import PARAMS_PATTERN, Result, TITLE_PATTERN, GET, POST 8 | from lib.core.requests import requestUrl 9 | from lib.core.log import ERROR, DEBUG, INFO 10 | 11 | 12 | SMALLER_CHAR_POOL = ('<', '>') # characters used for XSS tampering of parameter values (smaller set - for avoiding possible SQLi errors) 13 | LARGER_CHAR_POOL = ('\'', '"', '>', '<') # characters used for XSS tampering of parameter values (larger set) 14 | PREFIX_SUFFIX_LENGTH = 5 # length of random prefix/suffix used in XSS tampering 15 | CONTEXT_DISPLAY_OFFSET = 10 # offset outside the affected context for displaying in vulnerability report 16 | REGEX_SPECIAL_CHARS = ('\\', '*', '.', '+', '[', ']', ')', '(') # characters reserved for regular expressions 17 | 18 | XSS_PATTERNS = ( # each (pattern) item consists of ((context regex), (prerequisite unfiltered characters), "info text") 19 | (r'\A[^<>]*%(chars)s[^<>]*\Z', ('<', '>'), "\"...\", pure text response, %(filtering)s filtering"), 20 | (r"]*>(?!.*']*%(chars)s[^>']*'.*", ('\''), "\"\", enclosed by script tags, inside single-quotes, %(filtering)s filtering"), 21 | (r']*>(?!.*"]*%(chars)s[^>"]*".*', ('"'), "'', enclosed by script tags, inside double-quotes, %(filtering)s filtering"), 22 | (r']*>(?!.*', (), "\"\", enclosed by script tags, %s"), 23 | (r'>[^<]*%(chars)s[^<]*(<|\Z)', ('<', '>'), "\">...<\", outside tags, %(filtering)s filtering"), 24 | (r"<[^>]*'[^>']*%(chars)s[^>']*'[^>]*>", ('\'',), "\"<.'...'.>\", inside tag, inside single-quotes, %(filtering)s filtering"), 25 | (r'<[^>]*"[^>"]*%(chars)s[^>"]*"[^>]*>', ('"',), "'<.\"...\".>', inside tag, inside double-quotes, %(filtering)s filtering"), 26 | (r'<[^>]*%(chars)s[^>]*>', (), "\"<...>\", inside tag, %(filtering)s filtering") 27 | ) 28 | 29 | 30 | def run_url(req, rule): 31 | def _contains(content, chars): 32 | content = re.sub(r"\\[%s]" % "".join(chars), "", content, re.S) if chars else content 33 | return all(char in content for char in chars) 34 | 35 | details = [] 36 | response = None 37 | params = req.params 38 | for match in PARAMS_PATTERN.finditer(params): 39 | found = False 40 | prefix, suffix = ["".join(random.sample(string.ascii_lowercase, PREFIX_SUFFIX_LENGTH)) for i in xrange(2)] 41 | for pool in (LARGER_CHAR_POOL, SMALLER_CHAR_POOL): 42 | if not found: 43 | tampered = params.replace(match.group('value'), "%s%s%s%s" % (match.group('value'), prefix, "".join(random.sample(pool, len(pool))), suffix)) 44 | res = requestUrl(req,tampered) 45 | if not res: 46 | continue 47 | content = res.text 48 | for sample in re.finditer("%s(.+?)%s" % (prefix, suffix), content, re.I|re.S): 49 | for regex, condition, info in XSS_PATTERNS: 50 | context = re.search(regex % dict((("chars",reduce(lambda filtered, char: filtered.replace(char, "\\%s" % char), REGEX_SPECIAL_CHARS, sample.group(0))),)), content, re.I|re.S) 51 | if context and not found and sample.group(1).strip(): 52 | #print sample.group(1),condition 53 | if _contains(sample.group(1), condition): 54 | msg = info % dict((("filtering", "no" if all(char in sample.group(1) for char in LARGER_CHAR_POOL) else "some"),)) 55 | DEBUG(msg) 56 | found = True 57 | if response is None: 58 | response = res 59 | details.append(u"漏洞参数：%s" % match.group('key')) 60 | break 61 | #end for 62 | #end for 63 | #end for 64 | if response is not None: 65 | return Result(response,details) 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /scanner/sql/category.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : 127.0.0.1 5 | Source Server Version : 50045 6 | Source Host : 127.0.0.1:3306 7 | Source Database : topscan 8 | 9 | Target Server Type : MYSQL 10 | Target Server Version : 50045 11 | File Encoding : 65001 12 | 13 | Date: 2014-06-11 18:58:08 14 | */ 15 | 16 | SET FOREIGN_KEY_CHECKS=0; 17 | 18 | -- ---------------------------- 19 | -- Table structure for category 20 | -- ---------------------------- 21 | DROP TABLE IF EXISTS `category`; 22 | CREATE TABLE `category` ( 23 | `id` tinyint(4) NOT NULL auto_increment, 24 | `category_id` tinyint(4) default NULL, 25 | `category_name` char(255) default NULL, 26 | PRIMARY KEY (`id`) 27 | ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; 28 | 29 | -- ---------------------------- 30 | -- Records of category 31 | -- ---------------------------- 32 | INSERT INTO `category` VALUES ('1', '1', '信息泄露'); 33 | -------------------------------------------------------------------------------- /scanner/sql/rule.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : 127.0.0.1 5 | Source Server Version : 50045 6 | Source Host : 127.0.0.1:3306 7 | Source Database : kehan 8 | 9 | Target Server Type : MYSQL 10 | Target Server Version : 50045 11 | File Encoding : 65001 12 | 13 | Date: 2014-07-03 13:58:05 14 | */ 15 | 16 | SET FOREIGN_KEY_CHECKS=0; 17 | 18 | -- ---------------------------- 19 | -- Table structure for rule 20 | -- ---------------------------- 21 | DROP TABLE IF EXISTS `rule`; 22 | CREATE TABLE `rule` ( 23 | `id` int(11) NOT NULL auto_increment, 24 | `rule_id` int(11) NOT NULL, 25 | `rule_name` varchar(128) NOT NULL, 26 | `run_type` int(11) NOT NULL, 27 | `risk` varchar(8) NOT NULL, 28 | `priority` int(11) NOT NULL, 29 | `file_name` varchar(128) NOT NULL, 30 | `category_id` int(11) NOT NULL, 31 | `description` longtext NOT NULL, 32 | `solution` longtext NOT NULL, 33 | PRIMARY KEY (`id`) 34 | ) ENGINE=InnoDB AUTO_INCREMENT=8 DEFAULT CHARSET=utf8; 35 | 36 | -- ---------------------------- 37 | -- Records of rule 38 | -- ---------------------------- 39 | INSERT INTO `rule` VALUES ('1', '1', 'SQL注入', '1', 'high', '1', 'sql_inject', '1', '', ''); 40 | INSERT INTO `rule` VALUES ('2', '2', '跨站脚本攻击', '1', 'middle', '1', 'xss', '2', '', ''); 41 | INSERT INTO `rule` VALUES ('3', '3', '网页木马检测', '1', 'high', '2', 'webshell_check', '3', '', ''); 42 | INSERT INTO `rule` VALUES ('4', '4', '文件上传', '1', 'high', '1', 'file_upload', '4', '', ''); 43 | INSERT INTO `rule` VALUES ('5', '5', 'robots.txt站点结构泄露', '2', 'low', '3', 'robots_leak', '4', '', ''); 44 | INSERT INTO `rule` VALUES ('6', '6', '发现PhpMyadmin', '2', 'low', '4', 'phpmyadmin_leak', '4', '', ''); 45 | INSERT INTO `rule` VALUES ('7', '7', '内部IP地址泄露', '1', 'low', '8', 'inter_ip_leak', '4', '', ''); 46 | -------------------------------------------------------------------------------- /scanner/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/tests/__init__.py -------------------------------------------------------------------------------- /scanner/tests/test_request.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | 4 | from lib.core.request import request 5 | import unittest 6 | from test import test_support 7 | 8 | class RequestTest(unittest.TestCase): 9 | def test_basic(self): 10 | r = request("http://www.baidu.com/") 11 | self.assertEqual(r.status_code,200) 12 | self.assertEqual(r.encoding,'utf-8') 13 | r = request("http://www.baidu.com/12") 14 | self.assertEqual(r.encoding,'gbk') 15 | 16 | def test_follow_redirect(self): 17 | r = request("http://www.sina.com/",allow_redirects=False) 18 | self.assertEqual(r.status_code,301) 19 | r = request("http://www.sina.com/") 20 | self.assertEqual(r.status_code,200) 21 | 22 | def test_unicode(self): 23 | r = request("http://www.baidu.com/") 24 | self.assertEqual(isinstance(r.text,unicode),True) 25 | self.assertEqual(r.text.find(u"百度")>0,True) 26 | r = request("http://www.baidu.com/12") 27 | self.assertEqual(r.text.find(u"访问出错")>0,True) 28 | 29 | 30 | def test_main(): 31 | test_support.run_unittest(RequestTest) 32 | 33 | test_main() -------------------------------------------------------------------------------- /scanner/thirdparty/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/thirdparty/__init__.py -------------------------------------------------------------------------------- /scanner/thirdparty/requests/__init__.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # __ 4 | # /__) _ _ _ _ _/ _ 5 | # / ( (- (/ (/ (- _) / _) 6 | # / 7 | 8 | """ 9 | requests HTTP library 10 | ~~~~~~~~~~~~~~~~~~~~~ 11 | 12 | Requests is an HTTP library, written in Python, for human beings. Basic GET 13 | usage: 14 | 15 | >>> import requests 16 | >>> r = requests.get('http://python.org') 17 | >>> r.status_code 18 | 200 19 | >>> 'Python is a programming language' in r.content 20 | True 21 | 22 | ... or POST: 23 | 24 | >>> payload = dict(key1='value1', key2='value2') 25 | >>> r = requests.post("http://httpbin.org/post", data=payload) 26 | >>> print(r.text) 27 | { 28 | ... 29 | "form": { 30 | "key2": "value2", 31 | "key1": "value1" 32 | }, 33 | ... 34 | } 35 | 36 | The other HTTP methods are supported - see `requests.api`. Full documentation 37 | is at . 38 | 39 | :copyright: (c) 2014 by Kenneth Reitz. 40 | :license: Apache 2.0, see LICENSE for more details. 41 | 42 | """ 43 | 44 | __title__ = 'requests' 45 | __version__ = '2.3.0' 46 | __build__ = 0x020300 47 | __author__ = 'Kenneth Reitz' 48 | __license__ = 'Apache 2.0' 49 | __copyright__ = 'Copyright 2014 Kenneth Reitz' 50 | 51 | # Attempt to enable urllib3's SNI support, if possible 52 | try: 53 | from .packages.urllib3.contrib import pyopenssl 54 | pyopenssl.inject_into_urllib3() 55 | except ImportError: 56 | pass 57 | 58 | from . import utils 59 | from .models import Request, Response, PreparedRequest 60 | from .api import request, get, head, post, patch, put, delete, options 61 | from .sessions import session, Session 62 | from .status_codes import codes 63 | from .exceptions import ( 64 | RequestException, Timeout, URLRequired, 65 | TooManyRedirects, HTTPError, ConnectionError 66 | ) 67 | 68 | # Set default logging handler to avoid "No handler found" warnings. 69 | import logging 70 | try: # Python 2.7+ 71 | from logging import NullHandler 72 | except ImportError: 73 | class NullHandler(logging.Handler): 74 | def emit(self, record): 75 | pass 76 | 77 | logging.getLogger(__name__).addHandler(NullHandler()) 78 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/api.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.api 5 | ~~~~~~~~~~~~ 6 | 7 | This module implements the Requests API. 8 | 9 | :copyright: (c) 2012 by Kenneth Reitz. 10 | :license: Apache2, see LICENSE for more details. 11 | 12 | """ 13 | 14 | from . import sessions 15 | 16 | 17 | def request(method, url, **kwargs): 18 | """Constructs and sends a :class:`Request `. 19 | Returns :class:`Response ` object. 20 | 21 | :param method: method for the new :class:`Request` object. 22 | :param url: URL for the new :class:`Request` object. 23 | :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. 24 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 25 | :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. 26 | :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. 27 | :param files: (optional) Dictionary of 'name': file-like-objects (or {'name': ('filename', fileobj)}) for multipart encoding upload. 28 | :param auth: (optional) Auth tuple to enable Basic/Digest/Custom HTTP Auth. 29 | :param timeout: (optional) Float describing the timeout of the request in seconds. 30 | :param allow_redirects: (optional) Boolean. Set to True if POST/PUT/DELETE redirect following is allowed. 31 | :param proxies: (optional) Dictionary mapping protocol to the URL of the proxy. 32 | :param verify: (optional) if ``True``, the SSL cert will be verified. A CA_BUNDLE path can also be provided. 33 | :param stream: (optional) if ``False``, the response content will be immediately downloaded. 34 | :param cert: (optional) if String, path to ssl client cert file (.pem). If Tuple, ('cert', 'key') pair. 35 | 36 | Usage:: 37 | 38 | >>> import requests 39 | >>> req = requests.request('GET', 'http://httpbin.org/get') 40 | 41 | """ 42 | 43 | session = sessions.Session() 44 | return session.request(method=method, url=url, **kwargs) 45 | 46 | 47 | def get(url, **kwargs): 48 | """Sends a GET request. Returns :class:`Response` object. 49 | 50 | :param url: URL for the new :class:`Request` object. 51 | :param \*\*kwargs: Optional arguments that ``request`` takes. 52 | """ 53 | 54 | kwargs.setdefault('allow_redirects', True) 55 | return request('get', url, **kwargs) 56 | 57 | 58 | def options(url, **kwargs): 59 | """Sends a OPTIONS request. Returns :class:`Response` object. 60 | 61 | :param url: URL for the new :class:`Request` object. 62 | :param \*\*kwargs: Optional arguments that ``request`` takes. 63 | """ 64 | 65 | kwargs.setdefault('allow_redirects', True) 66 | return request('options', url, **kwargs) 67 | 68 | 69 | def head(url, **kwargs): 70 | """Sends a HEAD request. Returns :class:`Response` object. 71 | 72 | :param url: URL for the new :class:`Request` object. 73 | :param \*\*kwargs: Optional arguments that ``request`` takes. 74 | """ 75 | 76 | kwargs.setdefault('allow_redirects', False) 77 | return request('head', url, **kwargs) 78 | 79 | 80 | def post(url, data=None, **kwargs): 81 | """Sends a POST request. Returns :class:`Response` object. 82 | 83 | :param url: URL for the new :class:`Request` object. 84 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 85 | :param \*\*kwargs: Optional arguments that ``request`` takes. 86 | """ 87 | 88 | return request('post', url, data=data, **kwargs) 89 | 90 | 91 | def put(url, data=None, **kwargs): 92 | """Sends a PUT request. Returns :class:`Response` object. 93 | 94 | :param url: URL for the new :class:`Request` object. 95 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 96 | :param \*\*kwargs: Optional arguments that ``request`` takes. 97 | """ 98 | 99 | return request('put', url, data=data, **kwargs) 100 | 101 | 102 | def patch(url, data=None, **kwargs): 103 | """Sends a PATCH request. Returns :class:`Response` object. 104 | 105 | :param url: URL for the new :class:`Request` object. 106 | :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. 107 | :param \*\*kwargs: Optional arguments that ``request`` takes. 108 | """ 109 | 110 | return request('patch', url, data=data, **kwargs) 111 | 112 | 113 | def delete(url, **kwargs): 114 | """Sends a DELETE request. Returns :class:`Response` object. 115 | 116 | :param url: URL for the new :class:`Request` object. 117 | :param \*\*kwargs: Optional arguments that ``request`` takes. 118 | """ 119 | 120 | return request('delete', url, **kwargs) 121 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/certs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | """ 5 | certs.py 6 | ~~~~~~~~ 7 | 8 | This module returns the preferred default CA certificate bundle. 9 | 10 | If you are packaging Requests, e.g., for a Linux distribution or a managed 11 | environment, you can change the definition of where() to return a separately 12 | packaged CA bundle. 13 | """ 14 | 15 | import os.path 16 | 17 | 18 | def where(): 19 | """Return the preferred certificate bundle.""" 20 | # vendored bundle inside Requests 21 | return os.path.join(os.path.dirname(__file__), 'cacert.pem') 22 | 23 | if __name__ == '__main__': 24 | print(where()) 25 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/compat.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | pythoncompat 5 | """ 6 | 7 | from .packages import chardet 8 | 9 | import sys 10 | 11 | # ------- 12 | # Pythons 13 | # ------- 14 | 15 | # Syntax sugar. 16 | _ver = sys.version_info 17 | 18 | #: Python 2.x? 19 | is_py2 = (_ver[0] == 2) 20 | 21 | #: Python 3.x? 22 | is_py3 = (_ver[0] == 3) 23 | 24 | #: Python 3.0.x 25 | is_py30 = (is_py3 and _ver[1] == 0) 26 | 27 | #: Python 3.1.x 28 | is_py31 = (is_py3 and _ver[1] == 1) 29 | 30 | #: Python 3.2.x 31 | is_py32 = (is_py3 and _ver[1] == 2) 32 | 33 | #: Python 3.3.x 34 | is_py33 = (is_py3 and _ver[1] == 3) 35 | 36 | #: Python 3.4.x 37 | is_py34 = (is_py3 and _ver[1] == 4) 38 | 39 | #: Python 2.7.x 40 | is_py27 = (is_py2 and _ver[1] == 7) 41 | 42 | #: Python 2.6.x 43 | is_py26 = (is_py2 and _ver[1] == 6) 44 | 45 | #: Python 2.5.x 46 | is_py25 = (is_py2 and _ver[1] == 5) 47 | 48 | #: Python 2.4.x 49 | is_py24 = (is_py2 and _ver[1] == 4) # I'm assuming this is not by choice. 50 | 51 | 52 | # --------- 53 | # Platforms 54 | # --------- 55 | 56 | 57 | # Syntax sugar. 58 | _ver = sys.version.lower() 59 | 60 | is_pypy = ('pypy' in _ver) 61 | is_jython = ('jython' in _ver) 62 | is_ironpython = ('iron' in _ver) 63 | 64 | # Assume CPython, if nothing else. 65 | is_cpython = not any((is_pypy, is_jython, is_ironpython)) 66 | 67 | # Windows-based system. 68 | is_windows = 'win32' in str(sys.platform).lower() 69 | 70 | # Standard Linux 2+ system. 71 | is_linux = ('linux' in str(sys.platform).lower()) 72 | is_osx = ('darwin' in str(sys.platform).lower()) 73 | is_hpux = ('hpux' in str(sys.platform).lower()) # Complete guess. 74 | is_solaris = ('solar==' in str(sys.platform).lower()) # Complete guess. 75 | 76 | try: 77 | import simplejson as json 78 | except (ImportError, SyntaxError): 79 | # simplejson does not support Python 3.2, it thows a SyntaxError 80 | # because of u'...' Unicode literals. 81 | import json 82 | 83 | # --------- 84 | # Specifics 85 | # --------- 86 | 87 | if is_py2: 88 | from urllib import quote, unquote, quote_plus, unquote_plus, urlencode, getproxies, proxy_bypass 89 | from urlparse import urlparse, urlunparse, urljoin, urlsplit, urldefrag 90 | from urllib2 import parse_http_list 91 | import cookielib 92 | from Cookie import Morsel 93 | from StringIO import StringIO 94 | from .packages.urllib3.packages.ordered_dict import OrderedDict 95 | from httplib import IncompleteRead 96 | 97 | builtin_str = str 98 | bytes = str 99 | str = unicode 100 | basestring = basestring 101 | numeric_types = (int, long, float) 102 | 103 | 104 | elif is_py3: 105 | from urllib.parse import urlparse, urlunparse, urljoin, urlsplit, urlencode, quote, unquote, quote_plus, unquote_plus, urldefrag 106 | from urllib.request import parse_http_list, getproxies, proxy_bypass 107 | from http import cookiejar as cookielib 108 | from http.cookies import Morsel 109 | from io import StringIO 110 | from collections import OrderedDict 111 | from http.client import IncompleteRead 112 | 113 | builtin_str = str 114 | str = str 115 | bytes = bytes 116 | basestring = (str, bytes) 117 | numeric_types = (int, float) 118 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/exceptions.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.exceptions 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | This module contains the set of Requests' exceptions. 8 | 9 | """ 10 | from .packages.urllib3.exceptions import HTTPError as BaseHTTPError 11 | 12 | 13 | class RequestException(IOError): 14 | """There was an ambiguous exception that occurred while handling your 15 | request.""" 16 | 17 | def __init__(self, *args, **kwargs): 18 | """ 19 | Initialize RequestException with `request` and `response` objects. 20 | """ 21 | response = kwargs.pop('response', None) 22 | self.response = response 23 | self.request = kwargs.pop('request', None) 24 | if (response is not None and not self.request and 25 | hasattr(response, 'request')): 26 | self.request = self.response.request 27 | super(RequestException, self).__init__(*args, **kwargs) 28 | 29 | 30 | class HTTPError(RequestException): 31 | """An HTTP error occurred.""" 32 | 33 | 34 | class ConnectionError(RequestException): 35 | """A Connection error occurred.""" 36 | 37 | 38 | class ProxyError(ConnectionError): 39 | """A proxy error occurred.""" 40 | 41 | 42 | class SSLError(ConnectionError): 43 | """An SSL error occurred.""" 44 | 45 | 46 | class Timeout(RequestException): 47 | """The request timed out.""" 48 | 49 | 50 | class URLRequired(RequestException): 51 | """A valid URL is required to make a request.""" 52 | 53 | 54 | class TooManyRedirects(RequestException): 55 | """Too many redirects.""" 56 | 57 | 58 | class MissingSchema(RequestException, ValueError): 59 | """The URL schema (e.g. http or https) is missing.""" 60 | 61 | 62 | class InvalidSchema(RequestException, ValueError): 63 | """See defaults.py for valid schemas.""" 64 | 65 | 66 | class InvalidURL(RequestException, ValueError): 67 | """ The URL provided was somehow invalid. """ 68 | 69 | 70 | class ChunkedEncodingError(RequestException): 71 | """The server declared chunked encoding but sent an invalid chunk.""" 72 | 73 | 74 | class ContentDecodingError(RequestException, BaseHTTPError): 75 | """Failed to decode response content""" 76 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/hooks.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.hooks 5 | ~~~~~~~~~~~~~~ 6 | 7 | This module provides the capabilities for the Requests hooks system. 8 | 9 | Available hooks: 10 | 11 | ``response``: 12 | The response generated from a Request. 13 | 14 | """ 15 | 16 | 17 | HOOKS = ['response'] 18 | 19 | 20 | def default_hooks(): 21 | hooks = {} 22 | for event in HOOKS: 23 | hooks[event] = [] 24 | return hooks 25 | 26 | # TODO: response is the only one 27 | 28 | 29 | def dispatch_hook(key, hooks, hook_data, **kwargs): 30 | """Dispatches a hook dictionary on a given piece of data.""" 31 | 32 | hooks = hooks or dict() 33 | 34 | if key in hooks: 35 | hooks = hooks.get(key) 36 | 37 | if hasattr(hooks, '__call__'): 38 | hooks = [hooks] 39 | 40 | for hook in hooks: 41 | _hook_data = hook(hook_data, **kwargs) 42 | if _hook_data is not None: 43 | hook_data = _hook_data 44 | 45 | return hook_data 46 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/README.rst: -------------------------------------------------------------------------------- 1 | If you are planning to submit a pull request to requests with any changes in 2 | this library do not go any further. These are independent libraries which we 3 | vendor into requests. Any changes necessary to these libraries must be made in 4 | them and submitted as separate pull requests to those libraries. 5 | 6 | urllib3 pull requests go here: https://github.com/shazow/urllib3 7 | 8 | chardet pull requests go here: https://github.com/chardet/chardet 9 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import urllib3 4 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/__init__.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # This library is free software; you can redistribute it and/or 3 | # modify it under the terms of the GNU Lesser General Public 4 | # License as published by the Free Software Foundation; either 5 | # version 2.1 of the License, or (at your option) any later version. 6 | # 7 | # This library is distributed in the hope that it will be useful, 8 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 9 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 10 | # Lesser General Public License for more details. 11 | # 12 | # You should have received a copy of the GNU Lesser General Public 13 | # License along with this library; if not, write to the Free Software 14 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 15 | # 02110-1301 USA 16 | ######################### END LICENSE BLOCK ######################### 17 | 18 | __version__ = "2.2.1" 19 | from sys import version_info 20 | 21 | 22 | def detect(aBuf): 23 | if ((version_info < (3, 0) and isinstance(aBuf, unicode)) or 24 | (version_info >= (3, 0) and not isinstance(aBuf, bytes))): 25 | raise ValueError('Expected a bytes object, not a unicode object') 26 | 27 | from . import universaldetector 28 | u = universaldetector.UniversalDetector() 29 | u.reset() 30 | u.feed(aBuf) 31 | u.close() 32 | return u.result 33 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/big5prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import Big5DistributionAnalysis 31 | from .mbcssm import Big5SMModel 32 | 33 | 34 | class Big5Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(Big5SMModel) 38 | self._mDistributionAnalyzer = Big5DistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "Big5" 43 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/chardetect.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | Script which takes one or more file paths and reports on their detected 4 | encodings 5 | 6 | Example:: 7 | 8 | % chardetect somefile someotherfile 9 | somefile: windows-1252 with confidence 0.5 10 | someotherfile: ascii with confidence 1.0 11 | 12 | If no paths are provided, it takes its input from stdin. 13 | 14 | """ 15 | from io import open 16 | from sys import argv, stdin 17 | 18 | from chardet.universaldetector import UniversalDetector 19 | 20 | 21 | def description_of(file, name='stdin'): 22 | """Return a string describing the probable encoding of a file.""" 23 | u = UniversalDetector() 24 | for line in file: 25 | u.feed(line) 26 | u.close() 27 | result = u.result 28 | if result['encoding']: 29 | return '%s: %s with confidence %s' % (name, 30 | result['encoding'], 31 | result['confidence']) 32 | else: 33 | return '%s: no result' % name 34 | 35 | 36 | def main(): 37 | if len(argv) <= 1: 38 | print(description_of(stdin)) 39 | else: 40 | for path in argv[1:]: 41 | with open(path, 'rb') as f: 42 | print(description_of(f, path)) 43 | 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/charsetgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Communicator client code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | import sys 30 | from .charsetprober import CharSetProber 31 | 32 | 33 | class CharSetGroupProber(CharSetProber): 34 | def __init__(self): 35 | CharSetProber.__init__(self) 36 | self._mActiveNum = 0 37 | self._mProbers = [] 38 | self._mBestGuessProber = None 39 | 40 | def reset(self): 41 | CharSetProber.reset(self) 42 | self._mActiveNum = 0 43 | for prober in self._mProbers: 44 | if prober: 45 | prober.reset() 46 | prober.active = True 47 | self._mActiveNum += 1 48 | self._mBestGuessProber = None 49 | 50 | def get_charset_name(self): 51 | if not self._mBestGuessProber: 52 | self.get_confidence() 53 | if not self._mBestGuessProber: 54 | return None 55 | # self._mBestGuessProber = self._mProbers[0] 56 | return self._mBestGuessProber.get_charset_name() 57 | 58 | def feed(self, aBuf): 59 | for prober in self._mProbers: 60 | if not prober: 61 | continue 62 | if not prober.active: 63 | continue 64 | st = prober.feed(aBuf) 65 | if not st: 66 | continue 67 | if st == constants.eFoundIt: 68 | self._mBestGuessProber = prober 69 | return self.get_state() 70 | elif st == constants.eNotMe: 71 | prober.active = False 72 | self._mActiveNum -= 1 73 | if self._mActiveNum <= 0: 74 | self._mState = constants.eNotMe 75 | return self.get_state() 76 | return self.get_state() 77 | 78 | def get_confidence(self): 79 | st = self.get_state() 80 | if st == constants.eFoundIt: 81 | return 0.99 82 | elif st == constants.eNotMe: 83 | return 0.01 84 | bestConf = 0.0 85 | self._mBestGuessProber = None 86 | for prober in self._mProbers: 87 | if not prober: 88 | continue 89 | if not prober.active: 90 | if constants._debug: 91 | sys.stderr.write(prober.get_charset_name() 92 | + ' not active\n') 93 | continue 94 | cf = prober.get_confidence() 95 | if constants._debug: 96 | sys.stderr.write('%s confidence = %s\n' % 97 | (prober.get_charset_name(), cf)) 98 | if bestConf < cf: 99 | bestConf = cf 100 | self._mBestGuessProber = prober 101 | if not self._mBestGuessProber: 102 | return 0.0 103 | return bestConf 104 | # else: 105 | # self._mBestGuessProber = self._mProbers[0] 106 | # return self._mBestGuessProber.get_confidence() 107 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/charsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from . import constants 30 | import re 31 | 32 | 33 | class CharSetProber: 34 | def __init__(self): 35 | pass 36 | 37 | def reset(self): 38 | self._mState = constants.eDetecting 39 | 40 | def get_charset_name(self): 41 | return None 42 | 43 | def feed(self, aBuf): 44 | pass 45 | 46 | def get_state(self): 47 | return self._mState 48 | 49 | def get_confidence(self): 50 | return 0.0 51 | 52 | def filter_high_bit_only(self, aBuf): 53 | aBuf = re.sub(b'([\x00-\x7F])+', b' ', aBuf) 54 | return aBuf 55 | 56 | def filter_without_english_letters(self, aBuf): 57 | aBuf = re.sub(b'([A-Za-z])+', b' ', aBuf) 58 | return aBuf 59 | 60 | def filter_with_english_letters(self, aBuf): 61 | # TODO 62 | return aBuf 63 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/codingstatemachine.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .constants import eStart 29 | from .compat import wrap_ord 30 | 31 | 32 | class CodingStateMachine: 33 | def __init__(self, sm): 34 | self._mModel = sm 35 | self._mCurrentBytePos = 0 36 | self._mCurrentCharLen = 0 37 | self.reset() 38 | 39 | def reset(self): 40 | self._mCurrentState = eStart 41 | 42 | def next_state(self, c): 43 | # for each byte we get its class 44 | # if it is first byte, we also get byte length 45 | # PY3K: aBuf is a byte stream, so c is an int, not a byte 46 | byteCls = self._mModel['classTable'][wrap_ord(c)] 47 | if self._mCurrentState == eStart: 48 | self._mCurrentBytePos = 0 49 | self._mCurrentCharLen = self._mModel['charLenTable'][byteCls] 50 | # from byte's class and stateTable, we get its next state 51 | curr_state = (self._mCurrentState * self._mModel['classFactor'] 52 | + byteCls) 53 | self._mCurrentState = self._mModel['stateTable'][curr_state] 54 | self._mCurrentBytePos += 1 55 | return self._mCurrentState 56 | 57 | def get_current_charlen(self): 58 | return self._mCurrentCharLen 59 | 60 | def get_coding_state_machine(self): 61 | return self._mModel['name'] 62 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/compat.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # Contributor(s): 3 | # Ian Cordasco - port to Python 4 | # 5 | # This library is free software; you can redistribute it and/or 6 | # modify it under the terms of the GNU Lesser General Public 7 | # License as published by the Free Software Foundation; either 8 | # version 2.1 of the License, or (at your option) any later version. 9 | # 10 | # This library is distributed in the hope that it will be useful, 11 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 12 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 13 | # Lesser General Public License for more details. 14 | # 15 | # You should have received a copy of the GNU Lesser General Public 16 | # License along with this library; if not, write to the Free Software 17 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 18 | # 02110-1301 USA 19 | ######################### END LICENSE BLOCK ######################### 20 | 21 | import sys 22 | 23 | 24 | if sys.version_info < (3, 0): 25 | base_str = (str, unicode) 26 | else: 27 | base_str = (bytes, str) 28 | 29 | 30 | def wrap_ord(a): 31 | if sys.version_info < (3, 0) and isinstance(a, base_str): 32 | return ord(a) 33 | else: 34 | return a 35 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/constants.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | _debug = 0 30 | 31 | eDetecting = 0 32 | eFoundIt = 1 33 | eNotMe = 2 34 | 35 | eStart = 0 36 | eError = 1 37 | eItsMe = 2 38 | 39 | SHORTCUT_THRESHOLD = 0.95 40 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/cp949prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import CP949SMModel 32 | 33 | 34 | class CP949Prober(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(CP949SMModel) 38 | # NOTE: CP949 is a superset of EUC-KR, so the distribution should be 39 | # not different. 40 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 41 | self.reset() 42 | 43 | def get_charset_name(self): 44 | return "CP949" 45 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/escprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .escsm import (HZSMModel, ISO2022CNSMModel, ISO2022JPSMModel, 30 | ISO2022KRSMModel) 31 | from .charsetprober import CharSetProber 32 | from .codingstatemachine import CodingStateMachine 33 | from .compat import wrap_ord 34 | 35 | 36 | class EscCharSetProber(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = [ 40 | CodingStateMachine(HZSMModel), 41 | CodingStateMachine(ISO2022CNSMModel), 42 | CodingStateMachine(ISO2022JPSMModel), 43 | CodingStateMachine(ISO2022KRSMModel) 44 | ] 45 | self.reset() 46 | 47 | def reset(self): 48 | CharSetProber.reset(self) 49 | for codingSM in self._mCodingSM: 50 | if not codingSM: 51 | continue 52 | codingSM.active = True 53 | codingSM.reset() 54 | self._mActiveSM = len(self._mCodingSM) 55 | self._mDetectedCharset = None 56 | 57 | def get_charset_name(self): 58 | return self._mDetectedCharset 59 | 60 | def get_confidence(self): 61 | if self._mDetectedCharset: 62 | return 0.99 63 | else: 64 | return 0.00 65 | 66 | def feed(self, aBuf): 67 | for c in aBuf: 68 | # PY3K: aBuf is a byte array, so c is an int, not a byte 69 | for codingSM in self._mCodingSM: 70 | if not codingSM: 71 | continue 72 | if not codingSM.active: 73 | continue 74 | codingState = codingSM.next_state(wrap_ord(c)) 75 | if codingState == constants.eError: 76 | codingSM.active = False 77 | self._mActiveSM -= 1 78 | if self._mActiveSM <= 0: 79 | self._mState = constants.eNotMe 80 | return self.get_state() 81 | elif codingState == constants.eItsMe: 82 | self._mState = constants.eFoundIt 83 | self._mDetectedCharset = codingSM.get_coding_state_machine() # nopep8 84 | return self.get_state() 85 | 86 | return self.get_state() 87 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/eucjpprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from . import constants 30 | from .mbcharsetprober import MultiByteCharSetProber 31 | from .codingstatemachine import CodingStateMachine 32 | from .chardistribution import EUCJPDistributionAnalysis 33 | from .jpcntx import EUCJPContextAnalysis 34 | from .mbcssm import EUCJPSMModel 35 | 36 | 37 | class EUCJPProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(EUCJPSMModel) 41 | self._mDistributionAnalyzer = EUCJPDistributionAnalysis() 42 | self._mContextAnalyzer = EUCJPContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "EUC-JP" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | # PY3K: aBuf is a byte array, so aBuf[i] is an int, not a byte 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mContextAnalyzer.feed(self._mLastChar, charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i - 1:i + 1], charLen) 75 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 76 | charLen) 77 | 78 | self._mLastChar[0] = aBuf[aLen - 1] 79 | 80 | if self.get_state() == constants.eDetecting: 81 | if (self._mContextAnalyzer.got_enough_data() and 82 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 83 | self._mState = constants.eFoundIt 84 | 85 | return self.get_state() 86 | 87 | def get_confidence(self): 88 | contxtCf = self._mContextAnalyzer.get_confidence() 89 | distribCf = self._mDistributionAnalyzer.get_confidence() 90 | return max(contxtCf, distribCf) 91 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/euckrprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCKRDistributionAnalysis 31 | from .mbcssm import EUCKRSMModel 32 | 33 | 34 | class EUCKRProber(MultiByteCharSetProber): 35 | def __init__(self): 36 | MultiByteCharSetProber.__init__(self) 37 | self._mCodingSM = CodingStateMachine(EUCKRSMModel) 38 | self._mDistributionAnalyzer = EUCKRDistributionAnalysis() 39 | self.reset() 40 | 41 | def get_charset_name(self): 42 | return "EUC-KR" 43 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/euctwprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import EUCTWDistributionAnalysis 31 | from .mbcssm import EUCTWSMModel 32 | 33 | class EUCTWProber(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(EUCTWSMModel) 37 | self._mDistributionAnalyzer = EUCTWDistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "EUC-TW" 42 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/gb2312prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from .mbcharsetprober import MultiByteCharSetProber 29 | from .codingstatemachine import CodingStateMachine 30 | from .chardistribution import GB2312DistributionAnalysis 31 | from .mbcssm import GB2312SMModel 32 | 33 | class GB2312Prober(MultiByteCharSetProber): 34 | def __init__(self): 35 | MultiByteCharSetProber.__init__(self) 36 | self._mCodingSM = CodingStateMachine(GB2312SMModel) 37 | self._mDistributionAnalyzer = GB2312DistributionAnalysis() 38 | self.reset() 39 | 40 | def get_charset_name(self): 41 | return "GB2312" 42 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/latin1prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetprober import CharSetProber 30 | from .constants import eNotMe 31 | from .compat import wrap_ord 32 | 33 | FREQ_CAT_NUM = 4 34 | 35 | UDF = 0 # undefined 36 | OTH = 1 # other 37 | ASC = 2 # ascii capital letter 38 | ASS = 3 # ascii small letter 39 | ACV = 4 # accent capital vowel 40 | ACO = 5 # accent capital other 41 | ASV = 6 # accent small vowel 42 | ASO = 7 # accent small other 43 | CLASS_NUM = 8 # total classes 44 | 45 | Latin1_CharToClass = ( 46 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 00 - 07 47 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 08 - 0F 48 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 10 - 17 49 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 18 - 1F 50 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 20 - 27 51 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 28 - 2F 52 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 30 - 37 53 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 38 - 3F 54 | OTH, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 40 - 47 55 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 48 - 4F 56 | ASC, ASC, ASC, ASC, ASC, ASC, ASC, ASC, # 50 - 57 57 | ASC, ASC, ASC, OTH, OTH, OTH, OTH, OTH, # 58 - 5F 58 | OTH, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 60 - 67 59 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 68 - 6F 60 | ASS, ASS, ASS, ASS, ASS, ASS, ASS, ASS, # 70 - 77 61 | ASS, ASS, ASS, OTH, OTH, OTH, OTH, OTH, # 78 - 7F 62 | OTH, UDF, OTH, ASO, OTH, OTH, OTH, OTH, # 80 - 87 63 | OTH, OTH, ACO, OTH, ACO, UDF, ACO, UDF, # 88 - 8F 64 | UDF, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # 90 - 97 65 | OTH, OTH, ASO, OTH, ASO, UDF, ASO, ACO, # 98 - 9F 66 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A0 - A7 67 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # A8 - AF 68 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B0 - B7 69 | OTH, OTH, OTH, OTH, OTH, OTH, OTH, OTH, # B8 - BF 70 | ACV, ACV, ACV, ACV, ACV, ACV, ACO, ACO, # C0 - C7 71 | ACV, ACV, ACV, ACV, ACV, ACV, ACV, ACV, # C8 - CF 72 | ACO, ACO, ACV, ACV, ACV, ACV, ACV, OTH, # D0 - D7 73 | ACV, ACV, ACV, ACV, ACV, ACO, ACO, ACO, # D8 - DF 74 | ASV, ASV, ASV, ASV, ASV, ASV, ASO, ASO, # E0 - E7 75 | ASV, ASV, ASV, ASV, ASV, ASV, ASV, ASV, # E8 - EF 76 | ASO, ASO, ASV, ASV, ASV, ASV, ASV, OTH, # F0 - F7 77 | ASV, ASV, ASV, ASV, ASV, ASO, ASO, ASO, # F8 - FF 78 | ) 79 | 80 | # 0 : illegal 81 | # 1 : very unlikely 82 | # 2 : normal 83 | # 3 : very likely 84 | Latin1ClassModel = ( 85 | # UDF OTH ASC ASS ACV ACO ASV ASO 86 | 0, 0, 0, 0, 0, 0, 0, 0, # UDF 87 | 0, 3, 3, 3, 3, 3, 3, 3, # OTH 88 | 0, 3, 3, 3, 3, 3, 3, 3, # ASC 89 | 0, 3, 3, 3, 1, 1, 3, 3, # ASS 90 | 0, 3, 3, 3, 1, 2, 1, 2, # ACV 91 | 0, 3, 3, 3, 3, 3, 3, 3, # ACO 92 | 0, 3, 1, 3, 1, 1, 1, 3, # ASV 93 | 0, 3, 1, 3, 1, 1, 3, 3, # ASO 94 | ) 95 | 96 | 97 | class Latin1Prober(CharSetProber): 98 | def __init__(self): 99 | CharSetProber.__init__(self) 100 | self.reset() 101 | 102 | def reset(self): 103 | self._mLastCharClass = OTH 104 | self._mFreqCounter = [0] * FREQ_CAT_NUM 105 | CharSetProber.reset(self) 106 | 107 | def get_charset_name(self): 108 | return "windows-1252" 109 | 110 | def feed(self, aBuf): 111 | aBuf = self.filter_with_english_letters(aBuf) 112 | for c in aBuf: 113 | charClass = Latin1_CharToClass[wrap_ord(c)] 114 | freq = Latin1ClassModel[(self._mLastCharClass * CLASS_NUM) 115 | + charClass] 116 | if freq == 0: 117 | self._mState = eNotMe 118 | break 119 | self._mFreqCounter[freq] += 1 120 | self._mLastCharClass = charClass 121 | 122 | return self.get_state() 123 | 124 | def get_confidence(self): 125 | if self.get_state() == eNotMe: 126 | return 0.01 127 | 128 | total = sum(self._mFreqCounter) 129 | if total < 0.01: 130 | confidence = 0.0 131 | else: 132 | confidence = ((self._mFreqCounter[3] / total) 133 | - (self._mFreqCounter[1] * 20.0 / total)) 134 | if confidence < 0.0: 135 | confidence = 0.0 136 | # lower the confidence of latin1 so that other more accurate 137 | # detector can take priority. 138 | confidence = confidence * 0.5 139 | return confidence 140 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/mbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | import sys 31 | from . import constants 32 | from .charsetprober import CharSetProber 33 | 34 | 35 | class MultiByteCharSetProber(CharSetProber): 36 | def __init__(self): 37 | CharSetProber.__init__(self) 38 | self._mDistributionAnalyzer = None 39 | self._mCodingSM = None 40 | self._mLastChar = [0, 0] 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | if self._mCodingSM: 45 | self._mCodingSM.reset() 46 | if self._mDistributionAnalyzer: 47 | self._mDistributionAnalyzer.reset() 48 | self._mLastChar = [0, 0] 49 | 50 | def get_charset_name(self): 51 | pass 52 | 53 | def feed(self, aBuf): 54 | aLen = len(aBuf) 55 | for i in range(0, aLen): 56 | codingState = self._mCodingSM.next_state(aBuf[i]) 57 | if codingState == constants.eError: 58 | if constants._debug: 59 | sys.stderr.write(self.get_charset_name() 60 | + ' prober hit error at byte ' + str(i) 61 | + '\n') 62 | self._mState = constants.eNotMe 63 | break 64 | elif codingState == constants.eItsMe: 65 | self._mState = constants.eFoundIt 66 | break 67 | elif codingState == constants.eStart: 68 | charLen = self._mCodingSM.get_current_charlen() 69 | if i == 0: 70 | self._mLastChar[1] = aBuf[0] 71 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 72 | else: 73 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 74 | charLen) 75 | 76 | self._mLastChar[0] = aBuf[aLen - 1] 77 | 78 | if self.get_state() == constants.eDetecting: 79 | if (self._mDistributionAnalyzer.got_enough_data() and 80 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 81 | self._mState = constants.eFoundIt 82 | 83 | return self.get_state() 84 | 85 | def get_confidence(self): 86 | return self._mDistributionAnalyzer.get_confidence() 87 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/mbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # Proofpoint, Inc. 13 | # 14 | # This library is free software; you can redistribute it and/or 15 | # modify it under the terms of the GNU Lesser General Public 16 | # License as published by the Free Software Foundation; either 17 | # version 2.1 of the License, or (at your option) any later version. 18 | # 19 | # This library is distributed in the hope that it will be useful, 20 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 21 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 22 | # Lesser General Public License for more details. 23 | # 24 | # You should have received a copy of the GNU Lesser General Public 25 | # License along with this library; if not, write to the Free Software 26 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 27 | # 02110-1301 USA 28 | ######################### END LICENSE BLOCK ######################### 29 | 30 | from .charsetgroupprober import CharSetGroupProber 31 | from .utf8prober import UTF8Prober 32 | from .sjisprober import SJISProber 33 | from .eucjpprober import EUCJPProber 34 | from .gb2312prober import GB2312Prober 35 | from .euckrprober import EUCKRProber 36 | from .cp949prober import CP949Prober 37 | from .big5prober import Big5Prober 38 | from .euctwprober import EUCTWProber 39 | 40 | 41 | class MBCSGroupProber(CharSetGroupProber): 42 | def __init__(self): 43 | CharSetGroupProber.__init__(self) 44 | self._mProbers = [ 45 | UTF8Prober(), 46 | SJISProber(), 47 | EUCJPProber(), 48 | GB2312Prober(), 49 | EUCKRProber(), 50 | CP949Prober(), 51 | Big5Prober(), 52 | EUCTWProber() 53 | ] 54 | self.reset() 55 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/sbcharsetprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | import sys 30 | from . import constants 31 | from .charsetprober import CharSetProber 32 | from .compat import wrap_ord 33 | 34 | SAMPLE_SIZE = 64 35 | SB_ENOUGH_REL_THRESHOLD = 1024 36 | POSITIVE_SHORTCUT_THRESHOLD = 0.95 37 | NEGATIVE_SHORTCUT_THRESHOLD = 0.05 38 | SYMBOL_CAT_ORDER = 250 39 | NUMBER_OF_SEQ_CAT = 4 40 | POSITIVE_CAT = NUMBER_OF_SEQ_CAT - 1 41 | #NEGATIVE_CAT = 0 42 | 43 | 44 | class SingleByteCharSetProber(CharSetProber): 45 | def __init__(self, model, reversed=False, nameProber=None): 46 | CharSetProber.__init__(self) 47 | self._mModel = model 48 | # TRUE if we need to reverse every pair in the model lookup 49 | self._mReversed = reversed 50 | # Optional auxiliary prober for name decision 51 | self._mNameProber = nameProber 52 | self.reset() 53 | 54 | def reset(self): 55 | CharSetProber.reset(self) 56 | # char order of last character 57 | self._mLastOrder = 255 58 | self._mSeqCounters = [0] * NUMBER_OF_SEQ_CAT 59 | self._mTotalSeqs = 0 60 | self._mTotalChar = 0 61 | # characters that fall in our sampling range 62 | self._mFreqChar = 0 63 | 64 | def get_charset_name(self): 65 | if self._mNameProber: 66 | return self._mNameProber.get_charset_name() 67 | else: 68 | return self._mModel['charsetName'] 69 | 70 | def feed(self, aBuf): 71 | if not self._mModel['keepEnglishLetter']: 72 | aBuf = self.filter_without_english_letters(aBuf) 73 | aLen = len(aBuf) 74 | if not aLen: 75 | return self.get_state() 76 | for c in aBuf: 77 | order = self._mModel['charToOrderMap'][wrap_ord(c)] 78 | if order < SYMBOL_CAT_ORDER: 79 | self._mTotalChar += 1 80 | if order < SAMPLE_SIZE: 81 | self._mFreqChar += 1 82 | if self._mLastOrder < SAMPLE_SIZE: 83 | self._mTotalSeqs += 1 84 | if not self._mReversed: 85 | i = (self._mLastOrder * SAMPLE_SIZE) + order 86 | model = self._mModel['precedenceMatrix'][i] 87 | else: # reverse the order of the letters in the lookup 88 | i = (order * SAMPLE_SIZE) + self._mLastOrder 89 | model = self._mModel['precedenceMatrix'][i] 90 | self._mSeqCounters[model] += 1 91 | self._mLastOrder = order 92 | 93 | if self.get_state() == constants.eDetecting: 94 | if self._mTotalSeqs > SB_ENOUGH_REL_THRESHOLD: 95 | cf = self.get_confidence() 96 | if cf > POSITIVE_SHORTCUT_THRESHOLD: 97 | if constants._debug: 98 | sys.stderr.write('%s confidence = %s, we have a' 99 | 'winner\n' % 100 | (self._mModel['charsetName'], cf)) 101 | self._mState = constants.eFoundIt 102 | elif cf < NEGATIVE_SHORTCUT_THRESHOLD: 103 | if constants._debug: 104 | sys.stderr.write('%s confidence = %s, below negative' 105 | 'shortcut threshhold %s\n' % 106 | (self._mModel['charsetName'], cf, 107 | NEGATIVE_SHORTCUT_THRESHOLD)) 108 | self._mState = constants.eNotMe 109 | 110 | return self.get_state() 111 | 112 | def get_confidence(self): 113 | r = 0.01 114 | if self._mTotalSeqs > 0: 115 | r = ((1.0 * self._mSeqCounters[POSITIVE_CAT]) / self._mTotalSeqs 116 | / self._mModel['mTypicalPositiveRatio']) 117 | r = r * self._mFreqChar / self._mTotalChar 118 | if r >= 1.0: 119 | r = 0.99 120 | return r 121 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/sbcsgroupprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is Mozilla Universal charset detector code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 2001 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # Shy Shalom - original C code 12 | # 13 | # This library is free software; you can redistribute it and/or 14 | # modify it under the terms of the GNU Lesser General Public 15 | # License as published by the Free Software Foundation; either 16 | # version 2.1 of the License, or (at your option) any later version. 17 | # 18 | # This library is distributed in the hope that it will be useful, 19 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 20 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 21 | # Lesser General Public License for more details. 22 | # 23 | # You should have received a copy of the GNU Lesser General Public 24 | # License along with this library; if not, write to the Free Software 25 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 26 | # 02110-1301 USA 27 | ######################### END LICENSE BLOCK ######################### 28 | 29 | from .charsetgroupprober import CharSetGroupProber 30 | from .sbcharsetprober import SingleByteCharSetProber 31 | from .langcyrillicmodel import (Win1251CyrillicModel, Koi8rModel, 32 | Latin5CyrillicModel, MacCyrillicModel, 33 | Ibm866Model, Ibm855Model) 34 | from .langgreekmodel import Latin7GreekModel, Win1253GreekModel 35 | from .langbulgarianmodel import Latin5BulgarianModel, Win1251BulgarianModel 36 | from .langhungarianmodel import Latin2HungarianModel, Win1250HungarianModel 37 | from .langthaimodel import TIS620ThaiModel 38 | from .langhebrewmodel import Win1255HebrewModel 39 | from .hebrewprober import HebrewProber 40 | 41 | 42 | class SBCSGroupProber(CharSetGroupProber): 43 | def __init__(self): 44 | CharSetGroupProber.__init__(self) 45 | self._mProbers = [ 46 | SingleByteCharSetProber(Win1251CyrillicModel), 47 | SingleByteCharSetProber(Koi8rModel), 48 | SingleByteCharSetProber(Latin5CyrillicModel), 49 | SingleByteCharSetProber(MacCyrillicModel), 50 | SingleByteCharSetProber(Ibm866Model), 51 | SingleByteCharSetProber(Ibm855Model), 52 | SingleByteCharSetProber(Latin7GreekModel), 53 | SingleByteCharSetProber(Win1253GreekModel), 54 | SingleByteCharSetProber(Latin5BulgarianModel), 55 | SingleByteCharSetProber(Win1251BulgarianModel), 56 | SingleByteCharSetProber(Latin2HungarianModel), 57 | SingleByteCharSetProber(Win1250HungarianModel), 58 | SingleByteCharSetProber(TIS620ThaiModel), 59 | ] 60 | hebrewProber = HebrewProber() 61 | logicalHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, 62 | False, hebrewProber) 63 | visualHebrewProber = SingleByteCharSetProber(Win1255HebrewModel, True, 64 | hebrewProber) 65 | hebrewProber.set_model_probers(logicalHebrewProber, visualHebrewProber) 66 | self._mProbers.extend([hebrewProber, logicalHebrewProber, 67 | visualHebrewProber]) 68 | 69 | self.reset() 70 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/sjisprober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | import sys 29 | from .mbcharsetprober import MultiByteCharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .chardistribution import SJISDistributionAnalysis 32 | from .jpcntx import SJISContextAnalysis 33 | from .mbcssm import SJISSMModel 34 | from . import constants 35 | 36 | 37 | class SJISProber(MultiByteCharSetProber): 38 | def __init__(self): 39 | MultiByteCharSetProber.__init__(self) 40 | self._mCodingSM = CodingStateMachine(SJISSMModel) 41 | self._mDistributionAnalyzer = SJISDistributionAnalysis() 42 | self._mContextAnalyzer = SJISContextAnalysis() 43 | self.reset() 44 | 45 | def reset(self): 46 | MultiByteCharSetProber.reset(self) 47 | self._mContextAnalyzer.reset() 48 | 49 | def get_charset_name(self): 50 | return "SHIFT_JIS" 51 | 52 | def feed(self, aBuf): 53 | aLen = len(aBuf) 54 | for i in range(0, aLen): 55 | codingState = self._mCodingSM.next_state(aBuf[i]) 56 | if codingState == constants.eError: 57 | if constants._debug: 58 | sys.stderr.write(self.get_charset_name() 59 | + ' prober hit error at byte ' + str(i) 60 | + '\n') 61 | self._mState = constants.eNotMe 62 | break 63 | elif codingState == constants.eItsMe: 64 | self._mState = constants.eFoundIt 65 | break 66 | elif codingState == constants.eStart: 67 | charLen = self._mCodingSM.get_current_charlen() 68 | if i == 0: 69 | self._mLastChar[1] = aBuf[0] 70 | self._mContextAnalyzer.feed(self._mLastChar[2 - charLen:], 71 | charLen) 72 | self._mDistributionAnalyzer.feed(self._mLastChar, charLen) 73 | else: 74 | self._mContextAnalyzer.feed(aBuf[i + 1 - charLen:i + 3 75 | - charLen], charLen) 76 | self._mDistributionAnalyzer.feed(aBuf[i - 1:i + 1], 77 | charLen) 78 | 79 | self._mLastChar[0] = aBuf[aLen - 1] 80 | 81 | if self.get_state() == constants.eDetecting: 82 | if (self._mContextAnalyzer.got_enough_data() and 83 | (self.get_confidence() > constants.SHORTCUT_THRESHOLD)): 84 | self._mState = constants.eFoundIt 85 | 86 | return self.get_state() 87 | 88 | def get_confidence(self): 89 | contxtCf = self._mContextAnalyzer.get_confidence() 90 | distribCf = self._mDistributionAnalyzer.get_confidence() 91 | return max(contxtCf, distribCf) 92 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/chardet/utf8prober.py: -------------------------------------------------------------------------------- 1 | ######################## BEGIN LICENSE BLOCK ######################## 2 | # The Original Code is mozilla.org code. 3 | # 4 | # The Initial Developer of the Original Code is 5 | # Netscape Communications Corporation. 6 | # Portions created by the Initial Developer are Copyright (C) 1998 7 | # the Initial Developer. All Rights Reserved. 8 | # 9 | # Contributor(s): 10 | # Mark Pilgrim - port to Python 11 | # 12 | # This library is free software; you can redistribute it and/or 13 | # modify it under the terms of the GNU Lesser General Public 14 | # License as published by the Free Software Foundation; either 15 | # version 2.1 of the License, or (at your option) any later version. 16 | # 17 | # This library is distributed in the hope that it will be useful, 18 | # but WITHOUT ANY WARRANTY; without even the implied warranty of 19 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 20 | # Lesser General Public License for more details. 21 | # 22 | # You should have received a copy of the GNU Lesser General Public 23 | # License along with this library; if not, write to the Free Software 24 | # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 25 | # 02110-1301 USA 26 | ######################### END LICENSE BLOCK ######################### 27 | 28 | from . import constants 29 | from .charsetprober import CharSetProber 30 | from .codingstatemachine import CodingStateMachine 31 | from .mbcssm import UTF8SMModel 32 | 33 | ONE_CHAR_PROB = 0.5 34 | 35 | 36 | class UTF8Prober(CharSetProber): 37 | def __init__(self): 38 | CharSetProber.__init__(self) 39 | self._mCodingSM = CodingStateMachine(UTF8SMModel) 40 | self.reset() 41 | 42 | def reset(self): 43 | CharSetProber.reset(self) 44 | self._mCodingSM.reset() 45 | self._mNumOfMBChar = 0 46 | 47 | def get_charset_name(self): 48 | return "utf-8" 49 | 50 | def feed(self, aBuf): 51 | for c in aBuf: 52 | codingState = self._mCodingSM.next_state(c) 53 | if codingState == constants.eError: 54 | self._mState = constants.eNotMe 55 | break 56 | elif codingState == constants.eItsMe: 57 | self._mState = constants.eFoundIt 58 | break 59 | elif codingState == constants.eStart: 60 | if self._mCodingSM.get_current_charlen() >= 2: 61 | self._mNumOfMBChar += 1 62 | 63 | if self.get_state() == constants.eDetecting: 64 | if self.get_confidence() > constants.SHORTCUT_THRESHOLD: 65 | self._mState = constants.eFoundIt 66 | 67 | return self.get_state() 68 | 69 | def get_confidence(self): 70 | unlike = 0.99 71 | if self._mNumOfMBChar < 6: 72 | for i in range(0, self._mNumOfMBChar): 73 | unlike = unlike * ONE_CHAR_PROB 74 | return 1.0 - unlike 75 | else: 76 | return unlike 77 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/__init__.py: -------------------------------------------------------------------------------- 1 | # urllib3/__init__.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | urllib3 - Thread-safe connection pooling and re-using. 9 | """ 10 | 11 | __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' 12 | __license__ = 'MIT' 13 | __version__ = 'dev' 14 | 15 | 16 | from .connectionpool import ( 17 | HTTPConnectionPool, 18 | HTTPSConnectionPool, 19 | connection_from_url 20 | ) 21 | 22 | from . import exceptions 23 | from .filepost import encode_multipart_formdata 24 | from .poolmanager import PoolManager, ProxyManager, proxy_from_url 25 | from .response import HTTPResponse 26 | from .util import make_headers, get_host, Timeout 27 | 28 | 29 | # Set default logging handler to avoid "No handler found" warnings. 30 | import logging 31 | try: # Python 2.7+ 32 | from logging import NullHandler 33 | except ImportError: 34 | class NullHandler(logging.Handler): 35 | def emit(self, record): 36 | pass 37 | 38 | logging.getLogger(__name__).addHandler(NullHandler()) 39 | 40 | def add_stderr_logger(level=logging.DEBUG): 41 | """ 42 | Helper for quickly adding a StreamHandler to the logger. Useful for 43 | debugging. 44 | 45 | Returns the handler after adding it. 46 | """ 47 | # This method needs to be in this __init__.py to get the __name__ correct 48 | # even if urllib3 is vendored within another package. 49 | logger = logging.getLogger(__name__) 50 | handler = logging.StreamHandler() 51 | handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) 52 | logger.addHandler(handler) 53 | logger.setLevel(level) 54 | logger.debug('Added an stderr logging handler to logger: %s' % __name__) 55 | return handler 56 | 57 | # ... Clean up. 58 | del NullHandler 59 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/contrib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/scanner/thirdparty/requests/packages/urllib3/contrib/__init__.py -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/contrib/ntlmpool.py: -------------------------------------------------------------------------------- 1 | # urllib3/contrib/ntlmpool.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | """ 8 | NTLM authenticating pool, contributed by erikcederstran 9 | 10 | Issue #10, see: http://code.google.com/p/urllib3/issues/detail?id=10 11 | """ 12 | 13 | try: 14 | from http.client import HTTPSConnection 15 | except ImportError: 16 | from httplib import HTTPSConnection 17 | from logging import getLogger 18 | from ntlm import ntlm 19 | 20 | from urllib3 import HTTPSConnectionPool 21 | 22 | 23 | log = getLogger(__name__) 24 | 25 | 26 | class NTLMConnectionPool(HTTPSConnectionPool): 27 | """ 28 | Implements an NTLM authentication version of an urllib3 connection pool 29 | """ 30 | 31 | scheme = 'https' 32 | 33 | def __init__(self, user, pw, authurl, *args, **kwargs): 34 | """ 35 | authurl is a random URL on the server that is protected by NTLM. 36 | user is the Windows user, probably in the DOMAIN\\username format. 37 | pw is the password for the user. 38 | """ 39 | super(NTLMConnectionPool, self).__init__(*args, **kwargs) 40 | self.authurl = authurl 41 | self.rawuser = user 42 | user_parts = user.split('\\', 1) 43 | self.domain = user_parts[0].upper() 44 | self.user = user_parts[1] 45 | self.pw = pw 46 | 47 | def _new_conn(self): 48 | # Performs the NTLM handshake that secures the connection. The socket 49 | # must be kept open while requests are performed. 50 | self.num_connections += 1 51 | log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s' % 52 | (self.num_connections, self.host, self.authurl)) 53 | 54 | headers = {} 55 | headers['Connection'] = 'Keep-Alive' 56 | req_header = 'Authorization' 57 | resp_header = 'www-authenticate' 58 | 59 | conn = HTTPSConnection(host=self.host, port=self.port) 60 | 61 | # Send negotiation message 62 | headers[req_header] = ( 63 | 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) 64 | log.debug('Request headers: %s' % headers) 65 | conn.request('GET', self.authurl, None, headers) 66 | res = conn.getresponse() 67 | reshdr = dict(res.getheaders()) 68 | log.debug('Response status: %s %s' % (res.status, res.reason)) 69 | log.debug('Response headers: %s' % reshdr) 70 | log.debug('Response data: %s [...]' % res.read(100)) 71 | 72 | # Remove the reference to the socket, so that it can not be closed by 73 | # the response object (we want to keep the socket open) 74 | res.fp = None 75 | 76 | # Server should respond with a challenge message 77 | auth_header_values = reshdr[resp_header].split(', ') 78 | auth_header_value = None 79 | for s in auth_header_values: 80 | if s[:5] == 'NTLM ': 81 | auth_header_value = s[5:] 82 | if auth_header_value is None: 83 | raise Exception('Unexpected %s response header: %s' % 84 | (resp_header, reshdr[resp_header])) 85 | 86 | # Send authentication message 87 | ServerChallenge, NegotiateFlags = \ 88 | ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) 89 | auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, 90 | self.user, 91 | self.domain, 92 | self.pw, 93 | NegotiateFlags) 94 | headers[req_header] = 'NTLM %s' % auth_msg 95 | log.debug('Request headers: %s' % headers) 96 | conn.request('GET', self.authurl, None, headers) 97 | res = conn.getresponse() 98 | log.debug('Response status: %s %s' % (res.status, res.reason)) 99 | log.debug('Response headers: %s' % dict(res.getheaders())) 100 | log.debug('Response data: %s [...]' % res.read()[:100]) 101 | if res.status != 200: 102 | if res.status == 401: 103 | raise Exception('Server rejected request: wrong ' 104 | 'username or password') 105 | raise Exception('Wrong server response: %s %s' % 106 | (res.status, res.reason)) 107 | 108 | res.fp = None 109 | log.debug('Connection established') 110 | return conn 111 | 112 | def urlopen(self, method, url, body=None, headers=None, retries=3, 113 | redirect=True, assert_same_host=True): 114 | if headers is None: 115 | headers = {} 116 | headers['Connection'] = 'Keep-Alive' 117 | return super(NTLMConnectionPool, self).urlopen(method, url, body, 118 | headers, retries, 119 | redirect, 120 | assert_same_host) 121 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/exceptions.py: -------------------------------------------------------------------------------- 1 | # urllib3/exceptions.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | 8 | ## Base Exceptions 9 | 10 | class HTTPError(Exception): 11 | "Base exception used by this module." 12 | pass 13 | 14 | 15 | class PoolError(HTTPError): 16 | "Base exception for errors caused within a pool." 17 | def __init__(self, pool, message): 18 | self.pool = pool 19 | HTTPError.__init__(self, "%s: %s" % (pool, message)) 20 | 21 | def __reduce__(self): 22 | # For pickling purposes. 23 | return self.__class__, (None, None) 24 | 25 | 26 | class RequestError(PoolError): 27 | "Base exception for PoolErrors that have associated URLs." 28 | def __init__(self, pool, url, message): 29 | self.url = url 30 | PoolError.__init__(self, pool, message) 31 | 32 | def __reduce__(self): 33 | # For pickling purposes. 34 | return self.__class__, (None, self.url, None) 35 | 36 | 37 | class SSLError(HTTPError): 38 | "Raised when SSL certificate fails in an HTTPS connection." 39 | pass 40 | 41 | 42 | class ProxyError(HTTPError): 43 | "Raised when the connection to a proxy fails." 44 | pass 45 | 46 | 47 | class ConnectionError(HTTPError): 48 | "Raised when a normal connection fails." 49 | pass 50 | 51 | 52 | class DecodeError(HTTPError): 53 | "Raised when automatic decoding based on Content-Type fails." 54 | pass 55 | 56 | 57 | ## Leaf Exceptions 58 | 59 | class MaxRetryError(RequestError): 60 | "Raised when the maximum number of retries is exceeded." 61 | 62 | def __init__(self, pool, url, reason=None): 63 | self.reason = reason 64 | 65 | message = "Max retries exceeded with url: %s" % url 66 | if reason: 67 | message += " (Caused by %s: %s)" % (type(reason), reason) 68 | else: 69 | message += " (Caused by redirect)" 70 | 71 | RequestError.__init__(self, pool, url, message) 72 | 73 | 74 | class HostChangedError(RequestError): 75 | "Raised when an existing pool gets a request for a foreign host." 76 | 77 | def __init__(self, pool, url, retries=3): 78 | message = "Tried to open a foreign host with url: %s" % url 79 | RequestError.__init__(self, pool, url, message) 80 | self.retries = retries 81 | 82 | 83 | class TimeoutStateError(HTTPError): 84 | """ Raised when passing an invalid state to a timeout """ 85 | pass 86 | 87 | 88 | class TimeoutError(HTTPError): 89 | """ Raised when a socket timeout error occurs. 90 | 91 | Catching this error will catch both :exc:`ReadTimeoutErrors 92 | ` and :exc:`ConnectTimeoutErrors `. 93 | """ 94 | pass 95 | 96 | 97 | class ReadTimeoutError(TimeoutError, RequestError): 98 | "Raised when a socket timeout occurs while receiving data from a server" 99 | pass 100 | 101 | 102 | # This timeout error does not have a URL attached and needs to inherit from the 103 | # base HTTPError 104 | class ConnectTimeoutError(TimeoutError): 105 | "Raised when a socket timeout occurs while connecting to a server" 106 | pass 107 | 108 | 109 | class EmptyPoolError(PoolError): 110 | "Raised when a pool runs out of connections and no more are allowed." 111 | pass 112 | 113 | 114 | class ClosedPoolError(PoolError): 115 | "Raised when a request enters a pool after the pool has been closed." 116 | pass 117 | 118 | 119 | class LocationParseError(ValueError, HTTPError): 120 | "Raised when get_host or similar fails to parse the URL input." 121 | 122 | def __init__(self, location): 123 | message = "Failed to parse: %s" % location 124 | HTTPError.__init__(self, message) 125 | 126 | self.location = location 127 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/filepost.py: -------------------------------------------------------------------------------- 1 | # urllib3/filepost.py 2 | # Copyright 2008-2013 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | import codecs 8 | import mimetypes 9 | 10 | from uuid import uuid4 11 | from io import BytesIO 12 | 13 | from .packages import six 14 | from .packages.six import b 15 | from .fields import RequestField 16 | 17 | writer = codecs.lookup('utf-8')[3] 18 | 19 | 20 | def choose_boundary(): 21 | """ 22 | Our embarassingly-simple replacement for mimetools.choose_boundary. 23 | """ 24 | return uuid4().hex 25 | 26 | 27 | def iter_field_objects(fields): 28 | """ 29 | Iterate over fields. 30 | 31 | Supports list of (k, v) tuples and dicts, and lists of 32 | :class:`~urllib3.fields.RequestField`. 33 | 34 | """ 35 | if isinstance(fields, dict): 36 | i = six.iteritems(fields) 37 | else: 38 | i = iter(fields) 39 | 40 | for field in i: 41 | if isinstance(field, RequestField): 42 | yield field 43 | else: 44 | yield RequestField.from_tuples(*field) 45 | 46 | 47 | def iter_fields(fields): 48 | """ 49 | .. deprecated:: 1.6 50 | 51 | Iterate over fields. 52 | 53 | The addition of :class:`~urllib3.fields.RequestField` makes this function 54 | obsolete. Instead, use :func:`iter_field_objects`, which returns 55 | :class:`~urllib3.fields.RequestField` objects. 56 | 57 | Supports list of (k, v) tuples and dicts. 58 | """ 59 | if isinstance(fields, dict): 60 | return ((k, v) for k, v in six.iteritems(fields)) 61 | 62 | return ((k, v) for k, v in fields) 63 | 64 | 65 | def encode_multipart_formdata(fields, boundary=None): 66 | """ 67 | Encode a dictionary of ``fields`` using the multipart/form-data MIME format. 68 | 69 | :param fields: 70 | Dictionary of fields or list of (key, :class:`~urllib3.fields.RequestField`). 71 | 72 | :param boundary: 73 | If not specified, then a random boundary will be generated using 74 | :func:`mimetools.choose_boundary`. 75 | """ 76 | body = BytesIO() 77 | if boundary is None: 78 | boundary = choose_boundary() 79 | 80 | for field in iter_field_objects(fields): 81 | body.write(b('--%s\r\n' % (boundary))) 82 | 83 | writer(body).write(field.render_headers()) 84 | data = field.data 85 | 86 | if isinstance(data, int): 87 | data = str(data) # Backwards compatibility 88 | 89 | if isinstance(data, six.text_type): 90 | writer(body).write(data) 91 | else: 92 | body.write(data) 93 | 94 | body.write(b'\r\n') 95 | 96 | body.write(b('--%s--\r\n' % (boundary))) 97 | 98 | content_type = str('multipart/form-data; boundary=%s' % boundary) 99 | 100 | return body.getvalue(), content_type 101 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/packages/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import ssl_match_hostname 4 | 5 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/packages/ssl_match_hostname/__init__.py: -------------------------------------------------------------------------------- 1 | try: 2 | # Python 3.2+ 3 | from ssl import CertificateError, match_hostname 4 | except ImportError: 5 | try: 6 | # Backport of the function from a pypi module 7 | from backports.ssl_match_hostname import CertificateError, match_hostname 8 | except ImportError: 9 | # Our vendored copy 10 | from ._implementation import CertificateError, match_hostname 11 | 12 | # Not needed, but documenting what we provide. 13 | __all__ = ('CertificateError', 'match_hostname') 14 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.py: -------------------------------------------------------------------------------- 1 | """The match_hostname() function from Python 3.3.3, essential when using SSL.""" 2 | 3 | # Note: This file is under the PSF license as the code comes from the python 4 | # stdlib. http://docs.python.org/3/license.html 5 | 6 | import re 7 | 8 | __version__ = '3.4.0.2' 9 | 10 | class CertificateError(ValueError): 11 | pass 12 | 13 | 14 | def _dnsname_match(dn, hostname, max_wildcards=1): 15 | """Matching according to RFC 6125, section 6.4.3 16 | 17 | http://tools.ietf.org/html/rfc6125#section-6.4.3 18 | """ 19 | pats = [] 20 | if not dn: 21 | return False 22 | 23 | # Ported from python3-syntax: 24 | # leftmost, *remainder = dn.split(r'.') 25 | parts = dn.split(r'.') 26 | leftmost = parts[0] 27 | remainder = parts[1:] 28 | 29 | wildcards = leftmost.count('*') 30 | if wildcards > max_wildcards: 31 | # Issue #17980: avoid denials of service by refusing more 32 | # than one wildcard per fragment. A survey of established 33 | # policy among SSL implementations showed it to be a 34 | # reasonable choice. 35 | raise CertificateError( 36 | "too many wildcards in certificate DNS name: " + repr(dn)) 37 | 38 | # speed up common case w/o wildcards 39 | if not wildcards: 40 | return dn.lower() == hostname.lower() 41 | 42 | # RFC 6125, section 6.4.3, subitem 1. 43 | # The client SHOULD NOT attempt to match a presented identifier in which 44 | # the wildcard character comprises a label other than the left-most label. 45 | if leftmost == '*': 46 | # When '*' is a fragment by itself, it matches a non-empty dotless 47 | # fragment. 48 | pats.append('[^.]+') 49 | elif leftmost.startswith('xn--') or hostname.startswith('xn--'): 50 | # RFC 6125, section 6.4.3, subitem 3. 51 | # The client SHOULD NOT attempt to match a presented identifier 52 | # where the wildcard character is embedded within an A-label or 53 | # U-label of an internationalized domain name. 54 | pats.append(re.escape(leftmost)) 55 | else: 56 | # Otherwise, '*' matches any dotless string, e.g. www* 57 | pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) 58 | 59 | # add the remaining fragments, ignore any wildcards 60 | for frag in remainder: 61 | pats.append(re.escape(frag)) 62 | 63 | pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) 64 | return pat.match(hostname) 65 | 66 | 67 | def match_hostname(cert, hostname): 68 | """Verify that *cert* (in decoded format as returned by 69 | SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 70 | rules are followed, but IP addresses are not accepted for *hostname*. 71 | 72 | CertificateError is raised on failure. On success, the function 73 | returns nothing. 74 | """ 75 | if not cert: 76 | raise ValueError("empty or no certificate") 77 | dnsnames = [] 78 | san = cert.get('subjectAltName', ()) 79 | for key, value in san: 80 | if key == 'DNS': 81 | if _dnsname_match(value, hostname): 82 | return 83 | dnsnames.append(value) 84 | if not dnsnames: 85 | # The subject is only checked when there is no dNSName entry 86 | # in subjectAltName 87 | for sub in cert.get('subject', ()): 88 | for key, value in sub: 89 | # XXX according to RFC 2818, the most specific Common Name 90 | # must be used. 91 | if key == 'commonName': 92 | if _dnsname_match(value, hostname): 93 | return 94 | dnsnames.append(value) 95 | if len(dnsnames) > 1: 96 | raise CertificateError("hostname %r " 97 | "doesn't match either of %s" 98 | % (hostname, ', '.join(map(repr, dnsnames)))) 99 | elif len(dnsnames) == 1: 100 | raise CertificateError("hostname %r " 101 | "doesn't match %r" 102 | % (hostname, dnsnames[0])) 103 | else: 104 | raise CertificateError("no appropriate commonName or " 105 | "subjectAltName fields were found") 106 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/__init__.py: -------------------------------------------------------------------------------- 1 | # urllib3/util/__init__.py 2 | # Copyright 2008-2014 Andrey Petrov and contributors (see CONTRIBUTORS.txt) 3 | # 4 | # This module is part of urllib3 and is released under 5 | # the MIT License: http://www.opensource.org/licenses/mit-license.php 6 | 7 | from .connection import is_connection_dropped 8 | from .request import make_headers 9 | from .response import is_fp_closed 10 | from .ssl_ import ( 11 | SSLContext, 12 | HAS_SNI, 13 | assert_fingerprint, 14 | resolve_cert_reqs, 15 | resolve_ssl_version, 16 | ssl_wrap_socket, 17 | ) 18 | from .timeout import ( 19 | current_time, 20 | Timeout, 21 | ) 22 | from .url import ( 23 | get_host, 24 | parse_url, 25 | split_first, 26 | Url, 27 | ) 28 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/connection.py: -------------------------------------------------------------------------------- 1 | from socket import error as SocketError 2 | try: 3 | from select import poll, POLLIN 4 | except ImportError: # `poll` doesn't exist on OSX and other platforms 5 | poll = False 6 | try: 7 | from select import select 8 | except ImportError: # `select` doesn't exist on AppEngine. 9 | select = False 10 | 11 | def is_connection_dropped(conn): # Platform-specific 12 | """ 13 | Returns True if the connection is dropped and should be closed. 14 | 15 | :param conn: 16 | :class:`httplib.HTTPConnection` object. 17 | 18 | Note: For platforms like AppEngine, this will always return ``False`` to 19 | let the platform handle connection recycling transparently for us. 20 | """ 21 | sock = getattr(conn, 'sock', False) 22 | if sock is False: # Platform-specific: AppEngine 23 | return False 24 | if sock is None: # Connection already closed (such as by httplib). 25 | return False 26 | 27 | if not poll: 28 | if not select: # Platform-specific: AppEngine 29 | return False 30 | 31 | try: 32 | return select([sock], [], [], 0.0)[0] 33 | except SocketError: 34 | return True 35 | 36 | # This version is better on platforms that support it. 37 | p = poll() 38 | p.register(sock, POLLIN) 39 | for (fno, ev) in p.poll(0.0): 40 | if fno == sock.fileno(): 41 | # Either data is buffered (bad), or the connection is dropped. 42 | return True 43 | 44 | 45 | 46 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/request.py: -------------------------------------------------------------------------------- 1 | from base64 import b64encode 2 | 3 | from ..packages import six 4 | 5 | 6 | ACCEPT_ENCODING = 'gzip,deflate' 7 | 8 | 9 | def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, 10 | basic_auth=None, proxy_basic_auth=None): 11 | """ 12 | Shortcuts for generating request headers. 13 | 14 | :param keep_alive: 15 | If ``True``, adds 'connection: keep-alive' header. 16 | 17 | :param accept_encoding: 18 | Can be a boolean, list, or string. 19 | ``True`` translates to 'gzip,deflate'. 20 | List will get joined by comma. 21 | String will be used as provided. 22 | 23 | :param user_agent: 24 | String representing the user-agent you want, such as 25 | "python-urllib3/0.6" 26 | 27 | :param basic_auth: 28 | Colon-separated username:password string for 'authorization: basic ...' 29 | auth header. 30 | 31 | :param proxy_basic_auth: 32 | Colon-separated username:password string for 'proxy-authorization: basic ...' 33 | auth header. 34 | 35 | Example: :: 36 | 37 | >>> make_headers(keep_alive=True, user_agent="Batman/1.0") 38 | {'connection': 'keep-alive', 'user-agent': 'Batman/1.0'} 39 | >>> make_headers(accept_encoding=True) 40 | {'accept-encoding': 'gzip,deflate'} 41 | """ 42 | headers = {} 43 | if accept_encoding: 44 | if isinstance(accept_encoding, str): 45 | pass 46 | elif isinstance(accept_encoding, list): 47 | accept_encoding = ','.join(accept_encoding) 48 | else: 49 | accept_encoding = ACCEPT_ENCODING 50 | headers['accept-encoding'] = accept_encoding 51 | 52 | if user_agent: 53 | headers['user-agent'] = user_agent 54 | 55 | if keep_alive: 56 | headers['connection'] = 'keep-alive' 57 | 58 | if basic_auth: 59 | headers['authorization'] = 'Basic ' + \ 60 | b64encode(six.b(basic_auth)).decode('utf-8') 61 | 62 | if proxy_basic_auth: 63 | headers['proxy-authorization'] = 'Basic ' + \ 64 | b64encode(six.b(proxy_basic_auth)).decode('utf-8') 65 | 66 | return headers 67 | 68 | 69 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/response.py: -------------------------------------------------------------------------------- 1 | def is_fp_closed(obj): 2 | """ 3 | Checks whether a given file-like object is closed. 4 | 5 | :param obj: 6 | The file-like object to check. 7 | """ 8 | if hasattr(obj, 'fp'): 9 | # Object is a container for another file-like object that gets released 10 | # on exhaustion (e.g. HTTPResponse) 11 | return obj.fp is None 12 | 13 | return obj.closed 14 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/ssl_.py: -------------------------------------------------------------------------------- 1 | from binascii import hexlify, unhexlify 2 | from hashlib import md5, sha1 3 | 4 | from ..exceptions import SSLError 5 | 6 | 7 | try: # Test for SSL features 8 | SSLContext = None 9 | HAS_SNI = False 10 | 11 | import ssl 12 | from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 13 | from ssl import SSLContext # Modern SSL? 14 | from ssl import HAS_SNI # Has SNI? 15 | except ImportError: 16 | pass 17 | 18 | 19 | def assert_fingerprint(cert, fingerprint): 20 | """ 21 | Checks if given fingerprint matches the supplied certificate. 22 | 23 | :param cert: 24 | Certificate as bytes object. 25 | :param fingerprint: 26 | Fingerprint as string of hexdigits, can be interspersed by colons. 27 | """ 28 | 29 | # Maps the length of a digest to a possible hash function producing 30 | # this digest. 31 | hashfunc_map = { 32 | 16: md5, 33 | 20: sha1 34 | } 35 | 36 | fingerprint = fingerprint.replace(':', '').lower() 37 | 38 | digest_length, rest = divmod(len(fingerprint), 2) 39 | 40 | if rest or digest_length not in hashfunc_map: 41 | raise SSLError('Fingerprint is of invalid length.') 42 | 43 | # We need encode() here for py32; works on py2 and p33. 44 | fingerprint_bytes = unhexlify(fingerprint.encode()) 45 | 46 | hashfunc = hashfunc_map[digest_length] 47 | 48 | cert_digest = hashfunc(cert).digest() 49 | 50 | if not cert_digest == fingerprint_bytes: 51 | raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' 52 | .format(hexlify(fingerprint_bytes), 53 | hexlify(cert_digest))) 54 | 55 | 56 | def resolve_cert_reqs(candidate): 57 | """ 58 | Resolves the argument to a numeric constant, which can be passed to 59 | the wrap_socket function/method from the ssl module. 60 | Defaults to :data:`ssl.CERT_NONE`. 61 | If given a string it is assumed to be the name of the constant in the 62 | :mod:`ssl` module or its abbrevation. 63 | (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. 64 | If it's neither `None` nor a string we assume it is already the numeric 65 | constant which can directly be passed to wrap_socket. 66 | """ 67 | if candidate is None: 68 | return CERT_NONE 69 | 70 | if isinstance(candidate, str): 71 | res = getattr(ssl, candidate, None) 72 | if res is None: 73 | res = getattr(ssl, 'CERT_' + candidate) 74 | return res 75 | 76 | return candidate 77 | 78 | 79 | def resolve_ssl_version(candidate): 80 | """ 81 | like resolve_cert_reqs 82 | """ 83 | if candidate is None: 84 | return PROTOCOL_SSLv23 85 | 86 | if isinstance(candidate, str): 87 | res = getattr(ssl, candidate, None) 88 | if res is None: 89 | res = getattr(ssl, 'PROTOCOL_' + candidate) 90 | return res 91 | 92 | return candidate 93 | 94 | 95 | if SSLContext is not None: # Python 3.2+ 96 | def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 97 | ca_certs=None, server_hostname=None, 98 | ssl_version=None): 99 | """ 100 | All arguments except `server_hostname` have the same meaning as for 101 | :func:`ssl.wrap_socket` 102 | 103 | :param server_hostname: 104 | Hostname of the expected certificate 105 | """ 106 | context = SSLContext(ssl_version) 107 | context.verify_mode = cert_reqs 108 | 109 | # Disable TLS compression to migitate CRIME attack (issue #309) 110 | OP_NO_COMPRESSION = 0x20000 111 | context.options |= OP_NO_COMPRESSION 112 | 113 | if ca_certs: 114 | try: 115 | context.load_verify_locations(ca_certs) 116 | # Py32 raises IOError 117 | # Py33 raises FileNotFoundError 118 | except Exception as e: # Reraise as SSLError 119 | raise SSLError(e) 120 | if certfile: 121 | # FIXME: This block needs a test. 122 | context.load_cert_chain(certfile, keyfile) 123 | if HAS_SNI: # Platform-specific: OpenSSL with enabled SNI 124 | return context.wrap_socket(sock, server_hostname=server_hostname) 125 | return context.wrap_socket(sock) 126 | 127 | else: # Python 3.1 and earlier 128 | def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, 129 | ca_certs=None, server_hostname=None, 130 | ssl_version=None): 131 | return wrap_socket(sock, keyfile=keyfile, certfile=certfile, 132 | ca_certs=ca_certs, cert_reqs=cert_reqs, 133 | ssl_version=ssl_version) 134 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/packages/urllib3/util/url.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | 3 | from ..exceptions import LocationParseError 4 | 5 | 6 | class Url(namedtuple('Url', ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'])): 7 | """ 8 | Datastructure for representing an HTTP URL. Used as a return value for 9 | :func:`parse_url`. 10 | """ 11 | slots = () 12 | 13 | def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, query=None, fragment=None): 14 | return super(Url, cls).__new__(cls, scheme, auth, host, port, path, query, fragment) 15 | 16 | @property 17 | def hostname(self): 18 | """For backwards-compatibility with urlparse. We're nice like that.""" 19 | return self.host 20 | 21 | @property 22 | def request_uri(self): 23 | """Absolute path including the query string.""" 24 | uri = self.path or '/' 25 | 26 | if self.query is not None: 27 | uri += '?' + self.query 28 | 29 | return uri 30 | 31 | @property 32 | def netloc(self): 33 | """Network location including host and port""" 34 | if self.port: 35 | return '%s:%d' % (self.host, self.port) 36 | return self.host 37 | 38 | 39 | def split_first(s, delims): 40 | """ 41 | Given a string and an iterable of delimiters, split on the first found 42 | delimiter. Return two split parts and the matched delimiter. 43 | 44 | If not found, then the first part is the full input string. 45 | 46 | Example: :: 47 | 48 | >>> split_first('foo/bar?baz', '?/=') 49 | ('foo', 'bar?baz', '/') 50 | >>> split_first('foo/bar?baz', '123') 51 | ('foo/bar?baz', '', None) 52 | 53 | Scales linearly with number of delims. Not ideal for large number of delims. 54 | """ 55 | min_idx = None 56 | min_delim = None 57 | for d in delims: 58 | idx = s.find(d) 59 | if idx < 0: 60 | continue 61 | 62 | if min_idx is None or idx < min_idx: 63 | min_idx = idx 64 | min_delim = d 65 | 66 | if min_idx is None or min_idx < 0: 67 | return s, '', None 68 | 69 | return s[:min_idx], s[min_idx+1:], min_delim 70 | 71 | 72 | def parse_url(url): 73 | """ 74 | Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is 75 | performed to parse incomplete urls. Fields not provided will be None. 76 | 77 | Partly backwards-compatible with :mod:`urlparse`. 78 | 79 | Example: :: 80 | 81 | >>> parse_url('http://google.com/mail/') 82 | Url(scheme='http', host='google.com', port=None, path='/', ...) 83 | >>> parse_url('google.com:80') 84 | Url(scheme=None, host='google.com', port=80, path=None, ...) 85 | >>> parse_url('/foo?bar') 86 | Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) 87 | """ 88 | 89 | # While this code has overlap with stdlib's urlparse, it is much 90 | # simplified for our needs and less annoying. 91 | # Additionally, this implementations does silly things to be optimal 92 | # on CPython. 93 | 94 | scheme = None 95 | auth = None 96 | host = None 97 | port = None 98 | path = None 99 | fragment = None 100 | query = None 101 | 102 | # Scheme 103 | if '://' in url: 104 | scheme, url = url.split('://', 1) 105 | 106 | # Find the earliest Authority Terminator 107 | # (http://tools.ietf.org/html/rfc3986#section-3.2) 108 | url, path_, delim = split_first(url, ['/', '?', '#']) 109 | 110 | if delim: 111 | # Reassemble the path 112 | path = delim + path_ 113 | 114 | # Auth 115 | if '@' in url: 116 | # Last '@' denotes end of auth part 117 | auth, url = url.rsplit('@', 1) 118 | 119 | # IPv6 120 | if url and url[0] == '[': 121 | host, url = url.split(']', 1) 122 | host += ']' 123 | 124 | # Port 125 | if ':' in url: 126 | _host, port = url.split(':', 1) 127 | 128 | if not host: 129 | host = _host 130 | 131 | if port: 132 | # If given, ports must be integers. 133 | if not port.isdigit(): 134 | raise LocationParseError(url) 135 | port = int(port) 136 | else: 137 | # Blank ports are cool, too. (rfc3986#section-3.2.3) 138 | port = None 139 | 140 | elif not host and url: 141 | host = url 142 | 143 | if not path: 144 | return Url(scheme, auth, host, port, path, query, fragment) 145 | 146 | # Fragment 147 | if '#' in path: 148 | path, fragment = path.split('#', 1) 149 | 150 | # Query 151 | if '?' in path: 152 | path, query = path.split('?', 1) 153 | 154 | return Url(scheme, auth, host, port, path, query, fragment) 155 | 156 | 157 | def get_host(url): 158 | """ 159 | Deprecated. Use :func:`.parse_url` instead. 160 | """ 161 | p = parse_url(url) 162 | return p.scheme or 'http', p.hostname, p.port 163 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/status_codes.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | from .structures import LookupDict 4 | 5 | _codes = { 6 | 7 | # Informational. 8 | 100: ('continue',), 9 | 101: ('switching_protocols',), 10 | 102: ('processing',), 11 | 103: ('checkpoint',), 12 | 122: ('uri_too_long', 'request_uri_too_long'), 13 | 200: ('ok', 'okay', 'all_ok', 'all_okay', 'all_good', '\\o/', '✓'), 14 | 201: ('created',), 15 | 202: ('accepted',), 16 | 203: ('non_authoritative_info', 'non_authoritative_information'), 17 | 204: ('no_content',), 18 | 205: ('reset_content', 'reset'), 19 | 206: ('partial_content', 'partial'), 20 | 207: ('multi_status', 'multiple_status', 'multi_stati', 'multiple_stati'), 21 | 208: ('already_reported',), 22 | 226: ('im_used',), 23 | 24 | # Redirection. 25 | 300: ('multiple_choices',), 26 | 301: ('moved_permanently', 'moved', '\\o-'), 27 | 302: ('found',), 28 | 303: ('see_other', 'other'), 29 | 304: ('not_modified',), 30 | 305: ('use_proxy',), 31 | 306: ('switch_proxy',), 32 | 307: ('temporary_redirect', 'temporary_moved', 'temporary'), 33 | 308: ('resume_incomplete', 'resume'), 34 | 35 | # Client Error. 36 | 400: ('bad_request', 'bad'), 37 | 401: ('unauthorized',), 38 | 402: ('payment_required', 'payment'), 39 | 403: ('forbidden',), 40 | 404: ('not_found', '-o-'), 41 | 405: ('method_not_allowed', 'not_allowed'), 42 | 406: ('not_acceptable',), 43 | 407: ('proxy_authentication_required', 'proxy_auth', 'proxy_authentication'), 44 | 408: ('request_timeout', 'timeout'), 45 | 409: ('conflict',), 46 | 410: ('gone',), 47 | 411: ('length_required',), 48 | 412: ('precondition_failed', 'precondition'), 49 | 413: ('request_entity_too_large',), 50 | 414: ('request_uri_too_large',), 51 | 415: ('unsupported_media_type', 'unsupported_media', 'media_type'), 52 | 416: ('requested_range_not_satisfiable', 'requested_range', 'range_not_satisfiable'), 53 | 417: ('expectation_failed',), 54 | 418: ('im_a_teapot', 'teapot', 'i_am_a_teapot'), 55 | 422: ('unprocessable_entity', 'unprocessable'), 56 | 423: ('locked',), 57 | 424: ('failed_dependency', 'dependency'), 58 | 425: ('unordered_collection', 'unordered'), 59 | 426: ('upgrade_required', 'upgrade'), 60 | 428: ('precondition_required', 'precondition'), 61 | 429: ('too_many_requests', 'too_many'), 62 | 431: ('header_fields_too_large', 'fields_too_large'), 63 | 444: ('no_response', 'none'), 64 | 449: ('retry_with', 'retry'), 65 | 450: ('blocked_by_windows_parental_controls', 'parental_controls'), 66 | 451: ('unavailable_for_legal_reasons', 'legal_reasons'), 67 | 499: ('client_closed_request',), 68 | 69 | # Server Error. 70 | 500: ('internal_server_error', 'server_error', '/o\\', '✗'), 71 | 501: ('not_implemented',), 72 | 502: ('bad_gateway',), 73 | 503: ('service_unavailable', 'unavailable'), 74 | 504: ('gateway_timeout',), 75 | 505: ('http_version_not_supported', 'http_version'), 76 | 506: ('variant_also_negotiates',), 77 | 507: ('insufficient_storage',), 78 | 509: ('bandwidth_limit_exceeded', 'bandwidth'), 79 | 510: ('not_extended',), 80 | } 81 | 82 | codes = LookupDict(name='status_codes') 83 | 84 | for (code, titles) in list(_codes.items()): 85 | for title in titles: 86 | setattr(codes, title, code) 87 | if not title.startswith('\\'): 88 | setattr(codes, title.upper(), code) 89 | -------------------------------------------------------------------------------- /scanner/thirdparty/requests/structures.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | """ 4 | requests.structures 5 | ~~~~~~~~~~~~~~~~~~~ 6 | 7 | Data structures that power Requests. 8 | 9 | """ 10 | 11 | import collections 12 | 13 | 14 | class CaseInsensitiveDict(collections.MutableMapping): 15 | """ 16 | A case-insensitive ``dict``-like object. 17 | 18 | Implements all methods and operations of 19 | ``collections.MutableMapping`` as well as dict's ``copy``. Also 20 | provides ``lower_items``. 21 | 22 | All keys are expected to be strings. The structure remembers the 23 | case of the last key to be set, and ``iter(instance)``, 24 | ``keys()``, ``items()``, ``iterkeys()``, and ``iteritems()`` 25 | will contain case-sensitive keys. However, querying and contains 26 | testing is case insensitive: 27 | 28 | cid = CaseInsensitiveDict() 29 | cid['Accept'] = 'application/json' 30 | cid['aCCEPT'] == 'application/json' # True 31 | list(cid) == ['Accept'] # True 32 | 33 | For example, ``headers['content-encoding']`` will return the 34 | value of a ``'Content-Encoding'`` response header, regardless 35 | of how the header name was originally stored. 36 | 37 | If the constructor, ``.update``, or equality comparison 38 | operations are given keys that have equal ``.lower()``s, the 39 | behavior is undefined. 40 | 41 | """ 42 | def __init__(self, data=None, **kwargs): 43 | self._store = dict() 44 | if data is None: 45 | data = {} 46 | self.update(data, **kwargs) 47 | 48 | def __setitem__(self, key, value): 49 | # Use the lowercased key for lookups, but store the actual 50 | # key alongside the value. 51 | self._store[key.lower()] = (key, value) 52 | 53 | def __getitem__(self, key): 54 | return self._store[key.lower()][1] 55 | 56 | def __delitem__(self, key): 57 | del self._store[key.lower()] 58 | 59 | def __iter__(self): 60 | return (casedkey for casedkey, mappedvalue in self._store.values()) 61 | 62 | def __len__(self): 63 | return len(self._store) 64 | 65 | def lower_items(self): 66 | """Like iteritems(), but with all lowercase keys.""" 67 | return ( 68 | (lowerkey, keyval[1]) 69 | for (lowerkey, keyval) 70 | in self._store.items() 71 | ) 72 | 73 | def __eq__(self, other): 74 | if isinstance(other, collections.Mapping): 75 | other = CaseInsensitiveDict(other) 76 | else: 77 | return NotImplemented 78 | # Compare insensitively 79 | return dict(self.lower_items()) == dict(other.lower_items()) 80 | 81 | # Copy is required 82 | def copy(self): 83 | return CaseInsensitiveDict(self._store.values()) 84 | 85 | def __repr__(self): 86 | return str(dict(self.items())) 87 | 88 | class LookupDict(dict): 89 | """Dictionary lookup object.""" 90 | 91 | def __init__(self, name=None): 92 | self.name = name 93 | super(LookupDict, self).__init__() 94 | 95 | def __repr__(self): 96 | return '' % (self.name) 97 | 98 | def __getitem__(self, key): 99 | # We allow fall-through here, so values default to None 100 | 101 | return self.__dict__.get(key, None) 102 | 103 | def get(self, key, default=None): 104 | return self.__dict__.get(key, default) 105 | -------------------------------------------------------------------------------- /scanner/topscan.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding:UTF-8-*- 3 | import os 4 | import sys 5 | 6 | from lib.core.data import paths 7 | from lib.core.envinit import envinit 8 | envinit(__file__) 9 | 10 | from lib.core.log import ERROR,DEBUG,INFO,WARN 11 | from lib.core.option import parseCmdline,init 12 | from lib.core.failure import TopException,DestinationUnReachable 13 | from lib.core.engine import run 14 | from lib.core.common import task_finsh_clean 15 | 16 | def main(): 17 | try: 18 | parseCmdline() 19 | init() 20 | run() 21 | #user_test() 22 | except KeyboardInterrupt: 23 | INFO("User aborted,scan stop") 24 | except DestinationUnReachable as e: 25 | WARN("Destination:%s not reachable,please check" % e.dest) 26 | except TopException: 27 | ERROR("User define exception") 28 | except Exception as e: 29 | ERROR("Exception occur,scan stop") 30 | finally: 31 | task_finsh_clean() 32 | INFO("Scan finished!") 33 | 34 | def test(): 35 | sys.argv = ['python','topscan.py','-t','1','-u','http://127.0.0.1/vul/file_upload.php','-b','/vul/'] 36 | main() 37 | 38 | def user_test(): 39 | from tests import test_request 40 | 41 | if __name__ == "__main__": 42 | main() 43 | 44 | # else: 45 | # if 1: 46 | # print 'test' 47 | # test() -------------------------------------------------------------------------------- /screenshot/create.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/screenshot/create.jpg -------------------------------------------------------------------------------- /screenshot/detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/screenshot/detail.png -------------------------------------------------------------------------------- /screenshot/task.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/screenshot/task.png -------------------------------------------------------------------------------- /screenshot/tree.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/screenshot/tree.png -------------------------------------------------------------------------------- /screenshot/vul.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/screenshot/vul.png -------------------------------------------------------------------------------- /topweb/.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | *.pyc 3 | *.pyo 4 | *.bd -------------------------------------------------------------------------------- /topweb/app_site/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/__init__.py -------------------------------------------------------------------------------- /topweb/app_site/context_processor.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from django.contrib.sites.models import Site 3 | 4 | def site(request): 5 | return { 6 | 'site' : Site.objects.get_current(), 7 | } -------------------------------------------------------------------------------- /topweb/app_site/models.py: -------------------------------------------------------------------------------- 1 | from django.db import models 2 | from django.contrib import admin 3 | 4 | # Create your models here. 5 | 6 | class Task(models.Model): 7 | """ 8 | status: 0 create(wait) 1 running 2 stop 3 finish 9 | spider_flag: 0 create(wait) 1 running 2 stop 3 finish 10 | """ 11 | class Meta: 12 | db_table = 'task' 13 | 14 | name = models.CharField(max_length=200) 15 | status = models.IntegerField() 16 | start_url = models.URLField() 17 | base = models.CharField(max_length=40) 18 | url_count = models.IntegerField() 19 | progress = models.TextField() 20 | spider_flag = models.IntegerField(1) 21 | robots_parsed = models.BooleanField(default=False) 22 | sitemap_parsed = models.BooleanField(default=False) 23 | reachable = models.BooleanField(default=True) 24 | start_time = models.DateTimeField(auto_now=True) 25 | end_time = models.DateTimeField(blank=True, null=True) 26 | 27 | 28 | class Url(models.Model): 29 | 30 | class Meta: 31 | db_table = 'url' 32 | 33 | task_id = models.IntegerField() 34 | url = models.URLField() 35 | method = models.CharField(max_length=10) 36 | params = models.CharField(max_length=200,blank=True) 37 | referer = models.CharField(max_length=200,blank=True) 38 | start_time = models.DateTimeField(blank=True, null=True) 39 | end_time = models.DateTimeField(blank=True, null=True) 40 | 41 | class Result(models.Model): 42 | 43 | class Meta: 44 | db_table = 'result' 45 | 46 | task_id = models.IntegerField() 47 | rule_id = models.IntegerField() 48 | risk = models.IntegerField(1) # 1 low 2 middle 3 high 49 | url = models.URLField() 50 | detail = models.TextField(blank=True) 51 | request = models.TextField(blank=True) 52 | response = models.TextField(blank=True) 53 | 54 | 55 | class Rule(models.Model): 56 | 57 | class Meta: 58 | db_table = 'rule' 59 | 60 | rule_id = models.IntegerField() 61 | rule_name = models.CharField(max_length=128) 62 | run_type = models.IntegerField(1) 63 | risk = models.CharField(max_length=4) 64 | priority = models.IntegerField(1) 65 | file_name = models.CharField(max_length=128) 66 | category_id = models.IntegerField() 67 | description = models.TextField(blank=True) 68 | solution = models.TextField(blank=True) 69 | 70 | 71 | 72 | 73 | admin.site.register(Task) -------------------------------------------------------------------------------- /topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/bootstrap/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /topweb/app_site/static/css/base.css: -------------------------------------------------------------------------------- 1 | body{ 2 | padding-top: 60px; 3 | padding-bottom: 40px; 4 | } 5 | 6 | 7 | #loginform { 8 | margin: auto; 9 | width: 500px; 10 | } 11 | 12 | #footer { 13 | margin: auto; 14 | width: 235px; 15 | } 16 | 17 | #resetform { 18 | margin: auto; 19 | width: 600px; 20 | } 21 | 22 | #resetdone { 23 | margin: auto; 24 | width: 410px; 25 | } 26 | 27 | #resetpasswd { 28 | margin: auto; 29 | width: 800px; 30 | } 31 | 32 | 33 | /*扫描任务*/ 34 | .content .margin_ten { 35 | margin-top: 10px; 36 | margin-bottom: 5px; 37 | } 38 | .main_panel { 39 | min-height: 400px; 40 | } 41 | .tab_margin { 42 | margin-top: 15px; 43 | margin-left: 20px; 44 | margin-bottom: 40px; 45 | } 46 | .ellipsis { 47 | text-overflow:ellipsis;white-space:nowrap;overflow: hidden; 48 | } 49 | 50 | .break_word { 51 | word-wrap:break-word; 52 | } 53 | 54 | .wait { 55 | background:url('/static/img/wait.png') 0 0 no-repeat; 56 | } 57 | 58 | .task_record .running { 59 | background:url('/static/img/running.gif'); 60 | } 61 | .stop { 62 | background:url('/static/img/stop.png') 0 0 no-repeat; 63 | } 64 | .complete { 65 | background:url('/static/img/complete.png') 0 0 no-repeat; 66 | } 67 | 68 | #dir_tree .folder { background:url('/static/img/file_sprite.png') right bottom no-repeat; } 69 | #dir_tree .file { background:url('/static/img/file_sprite.png') 0 0 no-repeat; } 70 | #dir_tree .file-pdf { background-position: -32px 0 } 71 | #dir_tree .file-as { background-position: -36px 0 } 72 | #dir_tree .file-c { background-position: -72px -0px } 73 | #dir_tree .file-iso { background-position: -108px -0px } 74 | #dir_tree .file-htm, #dir_tree .file-html, #dir_tree .file-xml, #dir_tree .file-xsl { background-position: -126px -0px } 75 | #dir_tree .file-cf { background-position: -162px -0px } 76 | #dir_tree .file-cpp { background-position: -216px -0px } 77 | #dir_tree .file-cs { background-position: -236px -0px } 78 | #dir_tree .file-sql { background-position: -272px -0px } 79 | #dir_tree .file-xls, #dir_tree .file-xlsx { background-position: -362px -0px } 80 | #dir_tree .file-h { background-position: -488px -0px } 81 | #dir_tree .file-crt, #dir_tree .file-pem, #dir_tree .file-cer { background-position: -452px -18px } 82 | #dir_tree .file-php { background-position: -108px -18px } 83 | #dir_tree .file-jpg, #dir_tree .file-jpeg, #dir_tree .file-png, #dir_tree .file-gif, #dir_tree .file-bmp { background-position: -126px -18px } 84 | #dir_tree .file-ppt, #dir_tree .file-pptx { background-position: -144px -18px } 85 | #dir_tree .file-rb { background-position: -180px -18px } 86 | #dir_tree .file-text, #dir_tree .file-txt, #dir_tree .file-md, #dir_tree .file-log, #dir_tree .file-htaccess { background-position: -254px -18px } 87 | #dir_tree .file-doc, #dir_tree .file-docx { background-position: -362px -18px } 88 | #dir_tree .file-zip, #dir_tree .file-gz, #dir_tree .file-tar, #dir_tree .file-rar { background-position: -416px -18px } 89 | #dir_tree .file-js { background-position: -434px -18px } 90 | #dir_tree .file-css { background-position: -144px -0px } 91 | #dir_tree .file-fla { background-position: -398px -0px } 92 | 93 | /* detail */ 94 | #vul_detail_tree .domain { background:url('/static/img/domain.png') 0 0 no-repeat; } 95 | #vul_detail_tree .high { background:url('/static/img/H.png') 0 0 no-repeat; } 96 | #vul_detail_tree .middle { background:url('/static/img/M.png') 0 0 no-repeat; } 97 | #vul_detail_tree .low { background:url('/static/img/L.png') 0 0 no-repeat; } 98 | 99 | #vul_detail .detail_text { font-size: 13px;} 100 | 101 | .tab-content .row { 102 | margin-top:10px; 103 | } 104 | -------------------------------------------------------------------------------- /topweb/app_site/static/img/H.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/H.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/L.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/L.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/M.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/M.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete00.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete00.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete01.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete01.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete11.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete11.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete2.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/complete3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/complete3.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/domain.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/domain.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/file_sprite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/file_sprite.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/running.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/running.gif -------------------------------------------------------------------------------- /topweb/app_site/static/img/stop.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/stop.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/stop1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/stop1.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/wait.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/wait.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/wait1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/wait1.png -------------------------------------------------------------------------------- /topweb/app_site/static/img/wait2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/img/wait2.png -------------------------------------------------------------------------------- /topweb/app_site/static/js/csrf.js: -------------------------------------------------------------------------------- 1 | function getCookie(name) { 2 | var cookieValue = null; 3 | if (document.cookie && document.cookie != '') { 4 | var cookies = document.cookie.split(';'); 5 | for (var i = 0; i < cookies.length; i++) { 6 | var cookie = jQuery.trim(cookies[i]); 7 | // Does this cookie string begin with the name we want? 8 | if (cookie.substring(0, name.length + 1) == (name + '=')) { 9 | cookieValue = decodeURIComponent(cookie.substring(name.length + 1)); 10 | break; 11 | } 12 | } 13 | } 14 | return cookieValue; 15 | } 16 | var csrftoken = getCookie('csrftoken'); 17 | 18 | function csrfSafeMethod(method) { 19 | // these HTTP methods do not require CSRF protection 20 | return (/^(GET|HEAD|OPTIONS|TRACE)$/.test(method)); 21 | } 22 | function sameOrigin(url) { 23 | // test that a given url is a same-origin URL 24 | // url could be relative or scheme relative or absolute 25 | var host = document.location.host; // host + port 26 | var protocol = document.location.protocol; 27 | var sr_origin = '//' + host; 28 | var origin = protocol + sr_origin; 29 | // Allow absolute or scheme relative URLs to same origin 30 | return (url == origin || url.slice(0, origin.length + 1) == origin + '/') || 31 | (url == sr_origin || url.slice(0, sr_origin.length + 1) == sr_origin + '/') || 32 | // or any other URL that isn't scheme relative or absolute i.e relative. 33 | !(/^(\/\/|http:|https:).*/.test(url)); 34 | } 35 | $.ajaxSetup({ 36 | beforeSend: function(xhr, settings) { 37 | if (!csrfSafeMethod(settings.type) && sameOrigin(settings.url)) { 38 | // Send the token to same-origin, relative URLs only. 39 | // Send the token only if the method warrants CSRF protection 40 | // Using the CSRFToken value acquired earlier 41 | xhr.setRequestHeader("X-CSRFToken", csrftoken); 42 | } 43 | } 44 | }); 45 | -------------------------------------------------------------------------------- /topweb/app_site/static/js/jstree/themes/default/32px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/js/jstree/themes/default/32px.png -------------------------------------------------------------------------------- /topweb/app_site/static/js/jstree/themes/default/40px.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/js/jstree/themes/default/40px.png -------------------------------------------------------------------------------- /topweb/app_site/static/js/jstree/themes/default/throbber.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/app_site/static/js/jstree/themes/default/throbber.gif -------------------------------------------------------------------------------- /topweb/app_site/templatetags/__init__.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 -------------------------------------------------------------------------------- /topweb/app_site/templatetags/registration_bootstrap.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | from django import template 3 | register = template.Library() 4 | 5 | @register.filter() 6 | def add_class(field, css): 7 | return field.as_widget(attrs={"class":css}) 8 | -------------------------------------------------------------------------------- /topweb/app_site/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | #-*-encoding=UTF-8-*- 3 | 4 | import hmac 5 | import json 6 | import urlparse 7 | 8 | import gevent 9 | from gevent import monkey 10 | monkey.patch_socket() 11 | 12 | from app_site.models import Task, Url, Result, Rule 13 | 14 | ADDR = ('localhost', 6667) 15 | 16 | def get_domain(task_id): 17 | task = Task.objects.get(id=task_id) 18 | _ = urlparse.urlsplit(task.start_url) 19 | domain = "%s://%s%s" % ( _.scheme, _.netloc, task.base) 20 | return domain 21 | 22 | def json_success(msg=''): 23 | data = {'success':True,'msg':msg} 24 | return json.dumps(data) 25 | 26 | def json_error(msg): 27 | data = {'success':False,'msg':msg} 28 | return json.dumps(data) 29 | 30 | 31 | def enum(*sequential, **named): 32 | start = named.pop('start',0) 33 | end = len(sequential) + start 34 | enums = dict(zip(sequential, range(start, end)), **named) 35 | return type('Enum', (), enums) 36 | 37 | 38 | SECRE_KEY = { 39 | "SCAN_MODULE" : "_TOP-SEC_**_BEIJING-ANFU_" 40 | } 41 | 42 | def send_request(module_name,request_headers): 43 | try: 44 | key = SECRE_KEY.get(module_name,'UNKNOWN_KEY') 45 | socket = gevent.socket.socket() 46 | socket.connect(ADDR) 47 | request_headers['module'] = module_name 48 | request_headers['signature'] = hmac.new(key, module_name).hexdigest() 49 | h = ["%s:%s" %(k, v) for k,v in request_headers.iteritems()] 50 | h.append('\n') 51 | request = '\n'.join(h) 52 | socket.send(request) 53 | content = socket.recv(8192) 54 | socket.close() 55 | return json.loads(content) 56 | except Exception: 57 | return {'success':False,'msg':'send_request error'} 58 | 59 | 60 | 61 | #send_request('SCAN_MODULE',{'action':'start','task_ids':1}) -------------------------------------------------------------------------------- /topweb/db.sqlite3: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/db.sqlite3 -------------------------------------------------------------------------------- /topweb/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import sys 4 | 5 | if __name__ == "__main__": 6 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "project_scan.settings") 7 | 8 | from django.core.management import execute_from_command_line 9 | 10 | execute_from_command_line(sys.argv) 11 | -------------------------------------------------------------------------------- /topweb/project_scan/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/topweb/project_scan/__init__.py -------------------------------------------------------------------------------- /topweb/project_scan/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for django-registration-bootstrap project. 3 | 4 | For more information on this file, see 5 | https://docs.djangoproject.com/en/1.6/topics/settings/ 6 | 7 | For the full list of settings and their values, see 8 | https://docs.djangoproject.com/en/1.6/ref/settings/ 9 | """ 10 | 11 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 12 | import os 13 | BASE_DIR = os.path.dirname(os.path.dirname(__file__)) 14 | 15 | 16 | # Quick-start development settings - unsuitable for production 17 | # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/ 18 | 19 | # SECURITY WARNING: keep the secret key used in production secret! 20 | SECRET_KEY = 'rh*k!m+ef-)l$naa-xnj2z0v^4!4hved4rj(48(^u1cz3co' 21 | 22 | # SECURITY WARNING: don't run with debug turned on in production! 23 | DEBUG = True 24 | 25 | TEMPLATE_DEBUG = True 26 | 27 | ALLOWED_HOSTS = [] 28 | 29 | 30 | # Application definition 31 | 32 | INSTALLED_APPS = ( 33 | 'django.contrib.admin', 34 | 'django.contrib.auth', 35 | 'django.contrib.contenttypes', 36 | 'django.contrib.sessions', 37 | 'django.contrib.messages', 38 | 'django.contrib.staticfiles', 39 | 'django.contrib.sites', 40 | 'app_site', 41 | ) 42 | 43 | MIDDLEWARE_CLASSES = ( 44 | 'django.contrib.sessions.middleware.SessionMiddleware', 45 | 'django.middleware.common.CommonMiddleware', 46 | 'django.middleware.csrf.CsrfViewMiddleware', 47 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 48 | 'django.contrib.messages.middleware.MessageMiddleware', 49 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 50 | ) 51 | 52 | TEMPLATE_CONTEXT_PROCESSORS = ( 53 | "django.contrib.auth.context_processors.auth", 54 | "django.core.context_processors.debug", 55 | "django.core.context_processors.i18n", 56 | "django.core.context_processors.media", 57 | "django.core.context_processors.static", 58 | "django.core.context_processors.tz", 59 | "django.contrib.messages.context_processors.messages", 60 | 61 | "app_site.context_processor.site" 62 | ) 63 | 64 | ROOT_URLCONF = 'project_scan.urls' 65 | 66 | WSGI_APPLICATION = 'project_scan.wsgi.application' 67 | 68 | 69 | # Database 70 | # https://docs.djangoproject.com/en/1.6/ref/settings/#databases 71 | 72 | # DATABASES = { 73 | # 'default': { 74 | # 'ENGINE': 'django.db.backends.sqlite3', 75 | # 'NAME': os.path.join(BASE_DIR, 'db.sqlite3'), 76 | # } 77 | # } 78 | 79 | DATABASES = { 80 | 'default': { 81 | 'ENGINE': 'django.db.backends.mysql', 82 | 'NAME': 'kehan', 83 | 'USER': 'root', 84 | 'PASSWORD': '', 85 | 'HOST': '127.0.0.1', 86 | 'PORT':'' 87 | } 88 | } 89 | 90 | # Internationalization 91 | # https://docs.djangoproject.com/en/1.6/topics/i18n/ 92 | 93 | LANGUAGE_CODE = 'zh_CN' 94 | 95 | TIME_ZONE = 'Asia/Shanghai' 96 | 97 | USE_I18N = True 98 | 99 | USE_L10N = True 100 | 101 | USE_TZ = False 102 | 103 | 104 | # Static files (CSS, JavaScript, Images) 105 | # https://docs.djangoproject.com/en/1.6/howto/static-files/ 106 | SITE_ID = 1 107 | 108 | STATIC_URL = '/static/' 109 | LOGIN_REDIRECT_URL = '/' 110 | 111 | TEMPLATE_DIRS = ( 112 | os.path.join(BASE_DIR, 'templates') 113 | ) 114 | -------------------------------------------------------------------------------- /topweb/project_scan/urls.py: -------------------------------------------------------------------------------- 1 | from django.conf.urls import patterns, include, url 2 | from django.conf.urls.static import static 3 | from django.conf import settings 4 | from django.contrib import admin 5 | from django.views.generic.base import TemplateView 6 | from app_site import views 7 | 8 | admin.autodiscover() 9 | 10 | #url(r'^$', TemplateView.as_view(template_name="base.html"), name='index'), 11 | 12 | urlpatterns = patterns('', 13 | url(r'^$', views.index,name='index'), 14 | url(r'', include('django.contrib.auth.urls')), 15 | url(r'task$', views.task), 16 | url(r'detail$', views.detail), 17 | url(r'policy$', views.policy,name='policy'), 18 | url(r'^admin/', include(admin.site.urls)), 19 | ) #+ static(settings.STATIC_URL, document_root=settings.STATIC_ROOT) 20 | -------------------------------------------------------------------------------- /topweb/project_scan/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for novo project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/1.6/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "novo.settings") 12 | 13 | from django.core.wsgi import get_wsgi_application 14 | application = get_wsgi_application() 15 | -------------------------------------------------------------------------------- /topweb/rule.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : 127.0.0.1 5 | Source Server Version : 50045 6 | Source Host : 127.0.0.1:3306 7 | Source Database : topscan 8 | 9 | Target Server Type : MYSQL 10 | Target Server Version : 50045 11 | File Encoding : 65001 12 | 13 | Date: 2014-06-23 12:28:40 14 | */ 15 | 16 | SET FOREIGN_KEY_CHECKS=0; 17 | 18 | -- ---------------------------- 19 | -- Table structure for rule 20 | -- ---------------------------- 21 | DROP TABLE IF EXISTS `rule`; 22 | CREATE TABLE `rule` ( 23 | `id` int(11) NOT NULL auto_increment, 24 | `rule_id` int(11) NOT NULL, 25 | `rule_name` char(255) NOT NULL, 26 | `run_type` tinyint(2) NOT NULL, 27 | `risk` char(8) default NULL, 28 | `priority` tinyint(2) NOT NULL, 29 | `filename` char(255) NOT NULL, 30 | `category_id` int(11) NOT NULL, 31 | `description` mediumtext NOT NULL, 32 | `solution` mediumtext NOT NULL, 33 | PRIMARY KEY (`id`) 34 | ) ENGINE=MyISAM AUTO_INCREMENT=12 DEFAULT CHARSET=utf8; 35 | 36 | -- ---------------------------- 37 | -- Records of rule 38 | -- ---------------------------- 39 | INSERT INTO `rule` VALUES ('1', '1', 'robots.txt站点结构泄露', '2', 'low', '1', 'robots_leak', '1', '', ''); 40 | INSERT INTO `rule` VALUES ('2', '2', 'Web应用指纹识别', '2', 'low', '1', 'app_fingure', '1', '', ''); 41 | INSERT INTO `rule` VALUES ('3', '3', '内部IP地址泄露', '1', 'low', '10', 'inter_ip_leak', '1', '', ''); 42 | INSERT INTO `rule` VALUES ('7', '7', 'SQL注入', '1', 'high', '1', 'sql_inject', '2', '', ''); 43 | INSERT INTO `rule` VALUES ('4', '4', '服务器错误', '1', 'middle', '1', 'server_error', '1', '', ''); 44 | INSERT INTO `rule` VALUES ('5', '5', '发现死链接', '1', 'low', '9', 'deak_link', '1', '', ''); 45 | INSERT INTO `rule` VALUES ('6', '6', '发现后台登陆页面', '2', 'low', '1', 'adminpage_leak', '1', '', ''); 46 | INSERT INTO `rule` VALUES ('8', '8', '跨站脚本攻击', '1', 'middle', '1', 'xss', '0', '', ''); 47 | INSERT INTO `rule` VALUES ('9', '9', '检测到文件上传', '1', 'high', '1', 'file_upload', '0', '', ''); 48 | INSERT INTO `rule` VALUES ('10', '10', '检测到phpMyAdmin', '2', 'low', '1', 'phpmyadmin_leak', '0', '', ''); 49 | INSERT INTO `rule` VALUES ('11', '11', '检测到WebShell', '1', 'high', '8', 'webshell_check', '0', '', ''); 50 | -------------------------------------------------------------------------------- /topweb/templates/base.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | 6 | 在线漏洞检测平台 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% block nav %} 21 | 46 | {% endblock %} 47 | 48 | 49 | -------------------------------------------------------------------------------- /topweb/templates/footer.html: -------------------------------------------------------------------------------- 1 | 4 | -------------------------------------------------------------------------------- /topweb/templates/header.html: -------------------------------------------------------------------------------- 1 | 2 | 在线漏洞检测平台 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /topweb/templates/home/detail.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

  扫描详细信息

5 |
6 |
7 |
8 | 10 |
11 |
12 |
13 |
14 | 20 | 21 |
22 |
23 |
24 |
扫描进度:
25 |
26 |
27 |
28 | 10% 29 |
30 |
31 |
32 |
33 |
34 |
正在扫描:
35 |
sql注入
36 |
37 | 38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
漏洞名称:
54 |
55 |
56 |
57 |
描述信息:
58 |
59 |
60 |
61 |
解决方案:
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
链接:
72 |
73 |
74 |
75 |
详细:
76 |
77 |
78 |
79 |
请求:
80 |
81 |
82 |
83 |
响应:
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 | 95 | 96 |
97 | 98 |
99 |
100 |
101 |
102 | -------------------------------------------------------------------------------- /topweb/templates/home/dirstructure.html: -------------------------------------------------------------------------------- 1 |
2 |
-------------------------------------------------------------------------------- /topweb/templates/home/home.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 | 9 | 10 | {% include 'nav.html' %} 11 |
12 |
13 |
14 |
15 |

  扫描任务管理

16 |
17 |
18 |
19 | 21 | 25 | 27 | 29 |
30 | {% include 'home/task.html' %} 31 |
32 |
33 |
34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | {% include 'home/task_template.html' %} 48 | 49 |
选择序号任务名状态漏洞统计开始时间结束时间
50 |
51 |
52 | 53 |
54 |
55 |
56 | {% include 'footer.html' %} 57 | 58 | 59 | -------------------------------------------------------------------------------- /topweb/templates/home/home2.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 | 9 | 10 | {% include 'nav.html' %} 11 |
12 |
13 | 14 | 19 |
20 |
21 | 22 |
23 |
24 |
25 | 27 | 31 | 33 |
34 | {% include 'home/task.html' %} 35 |
36 |
37 |
38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | {% include 'home/task_template.html' %} 51 | 52 |
选择序号任务名状态起始时间结束时间
53 |
54 |
55 | 56 |
57 |
58 | 59 |
60 | 61 |
62 |
63 | {% include 'footer.html' %} 64 | 65 | 66 | -------------------------------------------------------------------------------- /topweb/templates/home/message.html: -------------------------------------------------------------------------------- 1 | 2 |
3 |
4 |
5 | 6 |

Oh snap! You got an error!

7 |

Change this and that and try again. Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec elit. Cras mattis consectetur purus sit amet fermentum.

8 |

9 | 10 | 11 |

12 |
13 |
14 |
15 | 16 | -------------------------------------------------------------------------------- /topweb/templates/home/task.html: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /topweb/templates/home/task_template.html: -------------------------------------------------------------------------------- 1 | {% for task in tasks %} 2 | 3 | 4 | {{ task.id }} 5 | {{ forloop.counter }} 6 | {{ task.name }} 7 | 8 | {% ifequal task.status 0 %} 9 | 等待 10 | 11 | {% endifequal %} 12 | {% ifequal task.status 1 %} 13 | 扫描 14 | 15 | {% endifequal %} 16 | {% ifequal task.status 2 %} 17 | 暂停 18 | 19 | {% endifequal %} 20 | {% ifequal task.status 3 %} 21 | 结束 22 | 23 | {% endifequal %} 24 | 25 | 26 |
27 |
28 | {{ task.h_c }}(高) 29 |
30 |
31 | {{ task.m_c }}(中) 32 |
33 |
34 | {{ task.l_c }}(低) 35 |
36 |
37 | {{ task.s_c }}(漏洞) 38 |
39 |
40 | 41 | 42 | {{ task.start_time|date:"Y-m-d H:i:s" }} 43 | {% if task.end_time %} 44 | {{ task.end_time|date:"Y-m-d H:i:s" }} 45 | {% else %} 46 | 未结束 47 | {% endif %} 48 | 49 |
50 | 51 | 55 | 70 |
71 | 72 | 73 | {% endfor %} -------------------------------------------------------------------------------- /topweb/templates/nav.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 |
3 | 42 |
-------------------------------------------------------------------------------- /topweb/templates/registration/form_field.html: -------------------------------------------------------------------------------- 1 | {% load registration_bootstrap %} 2 |
3 | 4 |
5 | {{ field|add_class:'form-control input-lg' }} 6 | {% if field.errors %} 7 | {{ field.errors|join:'
' }} 8 | {% endif %} 9 | {% if field.help_text %} 10 | {{ field.help_text }} 11 | {% endif %} 12 |
13 |
-------------------------------------------------------------------------------- /topweb/templates/registration/logged_out.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /topweb/templates/registration/login.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 |
9 |
10 | {% if form.errors %} 11 |
12 |

用户名，密码有误！

13 |
14 | {% endif %} 15 |
{% csrf_token %} 16 | 登录 17 | {% for field in form %} 18 | {% include 'registration/form_field.html' %} 19 | {% endfor %} 20 |
21 |
22 | 23 |   24 |
25 |
26 |
27 |

重置密码

28 | 37 |
38 |
39 | {% include 'footer.html' %} 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /topweb/templates/registration/password_change_done.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 |
9 |
10 |

{% trans 'Password change successful' %}

11 |

{% trans 'Back' %}

12 |
13 |
14 | {% include 'footer.html' %} 15 | 16 | 17 | 18 | -------------------------------------------------------------------------------- /topweb/templates/registration/password_change_form.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 | {% include 'nav.html' %} 9 |
10 |
11 | {% if form.errors %} 12 |
13 |

请修改以下错误！

14 |
15 | {% endif %} 16 |
17 |
{% csrf_token %} 18 | {% trans 'Password change' %} 19 |
20 |
21 | {% trans "Please enter your old password, for security's sake, and then enter your new password twice so we can verify you typed it in correctly." %} 22 |
23 |
24 | {% for field in form %} 25 | {% include 'registration/form_field.html' %} 26 | {% endfor %} 27 |
28 |
29 | 30 |   31 |
32 |
33 | 42 |
43 |
44 |
45 |
46 | 47 | -------------------------------------------------------------------------------- /topweb/templates/registration/password_reset_complete.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load i18n %} 3 | 4 | {% block container %} 5 |
6 |

{% trans 'Password reset complete' %}

7 |

{% trans "Your password has been set. You may go ahead and log in now." %}

8 |
9 | {% endblock %} -------------------------------------------------------------------------------- /topweb/templates/registration/password_reset_confirm.html: -------------------------------------------------------------------------------- 1 | {% extends 'base.html' %} 2 | {% load i18n %} 3 | 4 | {% block container %} 5 |
6 | {% if validlink %} 7 | {% if form.errors %} 8 |
9 | × 10 |

{% blocktrans count form.errors.items|length as counter %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %}

11 | 16 |
17 | {% endif %} 18 |
{% csrf_token %} 19 |
20 | {% trans 'Password reset unsuccessful' %} 21 |
22 |
23 | {% trans "Please enter your new password twice so we can verify you typed it in correctly." %} 24 |
25 |
26 | {% for field in form %} 27 | {% include 'registration/form_field.html' %} 28 | {% endfor %} 29 |
30 | 31 |
32 |
33 | 36 |
37 | {% else %} 38 |

{% trans 'Password reset unsuccessful' %}

39 |

40 | {% trans "The password reset link was invalid, possibly because it has already been used. Please request a new password reset." %} 41 |

42 | {% endif %} 43 |
44 | {% endblock %} -------------------------------------------------------------------------------- /topweb/templates/registration/password_reset_done.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 |
9 |
10 |

密码重置成功

11 |

我们已经发送了一封包含重置密码的电子邮件，请您注意查收！

12 |
13 |
14 | {% include 'footer.html' %} 15 | 16 | -------------------------------------------------------------------------------- /topweb/templates/registration/password_reset_form.html: -------------------------------------------------------------------------------- 1 | {% load i18n %} 2 | 3 | 4 | 5 | {% include 'header.html' %} 6 | 7 | 8 |
9 |
10 | {% if form.errors %} 11 |
12 |

{% blocktrans count form.errors.items|length as counter %}Please correct the error below.{% plural %}Please correct the errors below.{% endblocktrans %}

13 |
14 | {% endif %} 15 |
{% csrf_token %} 16 |
17 | 密码重置 18 |
19 |
20 | 请输入你的Email地址。 21 |
22 |
23 | {% for field in form %} 24 | {% include 'registration/form_field.html' %} 25 | {% endfor %} 26 |
27 |
28 | 29 |
30 |
31 |
32 | 41 |
42 |
43 |
44 | {% include 'footer.html' %} 45 | 46 | 47 | -------------------------------------------------------------------------------- /漏扫使用说明.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/漏扫使用说明.doc -------------------------------------------------------------------------------- /详细说明.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/trysec/leakScan/5ab4f4162a060ed5954e8272291eb05201f8fd5f/详细说明.doc --------------------------------------------------------------------------------