├── vrequest ├── template │ ├── v │ │ ├── __init__.py │ │ ├── spiders │ │ │ └── __init__.py │ │ ├── items.py │ │ ├── settings.py │ │ ├── middlewares.py │ │ └── pipelines.py │ └── scrapy.cfg ├── ico.ico ├── __init__.py ├── pyascii_deviation.py ├── menu.py ├── pycaesar.py ├── pygif.py ├── combinekey.py ├── pyrc4.py ├── pypng2gif.py ├── pyrots.py ├── pyset_pypi_src.py ├── pywindowproxy.py ├── pyrail_fence.py ├── pywiener_attack.py ├── py_mini_requests.py ├── pycompress.py ├── pycv2.py ├── root.py ├── pybacon.py ├── pymorse.py ├── pyfinddesktopfile.py ├── pyprime.py ├── pybrainfuck.py ├── pyevpkdf.py ├── pyplus.py ├── py_vthread.py ├── main.py ├── pymultialgo.py ├── pymd2.py ├── ast_hook_inject.js ├── pypyzbar.py ├── pysimplejs2python.py ├── pybase.py ├── pyscreenshot.py ├── pyjspack.py ├── main.js └── pymini_yolo.py ├── .gitignore ├── test ├── show.png ├── show2.png ├── show3.png ├── show4.png ├── show5.png ├── show6.png ├── show7.png ├── show8.png ├── _test_cmdline.py └── _create_al_frame_mode.py ├── LICENSE ├── setup.py ├── MYLOG.md └── README.md /vrequest/template/v/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vrequest.egg-info/ 2 | __pycache__/ 3 | dist/ 4 | build/ -------------------------------------------------------------------------------- /test/show.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show.png -------------------------------------------------------------------------------- /test/show2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show2.png -------------------------------------------------------------------------------- /test/show3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show3.png -------------------------------------------------------------------------------- /test/show4.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show4.png -------------------------------------------------------------------------------- /test/show5.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show5.png -------------------------------------------------------------------------------- /test/show6.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show6.png -------------------------------------------------------------------------------- /test/show7.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show7.png -------------------------------------------------------------------------------- /test/show8.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/test/show8.png -------------------------------------------------------------------------------- /vrequest/ico.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cilame/vrequest/HEAD/vrequest/ico.ico -------------------------------------------------------------------------------- /vrequest/__init__.py: -------------------------------------------------------------------------------- 1 | __author__ = 'cilame' 2 | __version__ = '1.9.16' 3 | __email__ = 'opaquism@hotmail.com' 4 | __github__ = 'https://github.com/cilame/vrequest' -------------------------------------------------------------------------------- /vrequest/template/v/spiders/__init__.py: -------------------------------------------------------------------------------- 1 | # This package will contain the spiders of your Scrapy project 2 | # 3 | # Please refer to the documentation for information on how to create and manage 4 | # your spiders. 5 | -------------------------------------------------------------------------------- /vrequest/template/scrapy.cfg: -------------------------------------------------------------------------------- 1 | # Automatically created by: scrapy startproject 2 | # 3 | # For more information about the [deploy] section see: 4 | # https://scrapyd.readthedocs.io/en/latest/deploy.html 5 | 6 | [settings] 7 | default = v.settings 8 | 9 | [deploy] 10 | #url = http://localhost:6800/ 11 | project = v 12 | -------------------------------------------------------------------------------- /vrequest/template/v/items.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your scraped items 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/items.html 7 | 8 | import scrapy 9 | 10 | 11 | class VItem(scrapy.Item): 12 | # define the fields for your item here like: 13 | # name = scrapy.Field() 14 | pass 15 | -------------------------------------------------------------------------------- /test/_test_cmdline.py: -------------------------------------------------------------------------------- 1 | # 增加环境变量，仅测试使用 2 | import os 3 | import sys 4 | p = os.path.split(os.getcwd())[0] 5 | sys.path = [p] + sys.path 6 | import sys;print(sys.stdout.encoding) 7 | 8 | 9 | try: 10 | # 处理 sublime 执行时输出乱码 11 | import io 12 | import sys 13 | sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8') 14 | sys.stdout._CHUNK_SIZE = 1 15 | except: 16 | pass 17 | 18 | 19 | from vrequest import main 20 | main.execute() -------------------------------------------------------------------------------- /vrequest/pyascii_deviation.py: -------------------------------------------------------------------------------- 1 | def ascii_deviation(string, n): 2 | r = [] 3 | for i in string.encode(): 4 | r.append(i + n) 5 | try: 6 | return bytes(r).decode() 7 | except: 8 | return 'error decoding. {}'.format(bytes(r)) 9 | 10 | 11 | if __name__ == '__main__': 12 | s = 'YXNkZg==' 13 | s = 'cbXudqGG' 14 | for i in range(-20,20): 15 | v = '{:>3} --- {}'.format(i, ascii_deviation(s, i)) 16 | print(v) -------------------------------------------------------------------------------- /vrequest/menu.py: -------------------------------------------------------------------------------- 1 | import tkinter 2 | import types 3 | 4 | from .root import root 5 | 6 | menu = tkinter.Menu(root, tearoff=0) 7 | 8 | def bind_menu(func, name=None): 9 | if not isinstance(func, types.FunctionType): 10 | raise TypeError('{} must be a FunctionType.'.format(str(func))) 11 | labelname = name if name is not None else func.__name__ 12 | menu.add_command(label=labelname, command=func) 13 | root.bind("",lambda e:menu.post(e.x_root,e.y_root)) 14 | -------------------------------------------------------------------------------- /vrequest/pycaesar.py: -------------------------------------------------------------------------------- 1 | # 凯撒密码 2 | def caesar(t, n, keys='abcdefghijklmnopqrstuvwxyz'): 3 | s = keys 4 | u = s.upper() 5 | r = '' 6 | for i in t: 7 | if i in s: 8 | r += s[(s.index(i) + n)% len(keys)] 9 | elif i in u: 10 | r += u[(u.index(i) + n)% len(keys)] 11 | else: 12 | r += i 13 | return r 14 | 15 | if __name__ == '__main__': 16 | s = 'qianshanniaofeijue1234ASDF' 17 | for i in range(0, 26): 18 | v = caesar(s, i) 19 | print('deviation: {:>2} --- result: {} '.format(i, v)) -------------------------------------------------------------------------------- /vrequest/pygif.py: -------------------------------------------------------------------------------- 1 | # 该脚本切分gif无需依赖 2 | from tkinter import * 3 | 4 | def mk_phlist(filename): 5 | global phlist # 这个 global 是必须的，PhotoImage 加载多个图片的机制有点迷。 6 | phlist = [] 7 | for i in range(100): 8 | try: 9 | photo = PhotoImage(file=filename, format='gif -index {}'.format(i)) 10 | phlist.append(photo) 11 | except: 12 | print(i) 13 | break 14 | return phlist 15 | 16 | if __name__ == '__main__': 17 | root = Tk() 18 | 19 | filename = 'eee.gif' 20 | phlist = mk_phlist(filename) # 先有 tk 对象之后再创建才能成功 21 | text = Text(root) 22 | for i in phlist: 23 | text.image_create(END, image=i) 24 | 25 | text.pack(fill=BOTH,expand=True) 26 | root.mainloop() 27 | 28 | 29 | -------------------------------------------------------------------------------- /vrequest/combinekey.py: -------------------------------------------------------------------------------- 1 | import types 2 | 3 | from .root import root 4 | 5 | def bind_ctl_key(func, key=None, shift=False): 6 | if key is None: 7 | raise TypeError('{} must be a lowercase.'.format(key)) 8 | if not isinstance(func, types.FunctionType): 9 | raise TypeError('{} must be a FunctionType.'.format(str(func))) 10 | key = key.upper() if shift else key 11 | root.bind("".format(key),lambda e:func()) 12 | 13 | 14 | def bind_alt_key(func, key=None, shift=False): 15 | if key is None: 16 | raise TypeError('{} must be a lowercase.'.format(key)) 17 | if not isinstance(func, types.FunctionType): 18 | raise TypeError('{} must be a FunctionType.'.format(str(func))) 19 | key = key.upper() if shift else key 20 | root.bind("".format(key),lambda e:func()) 21 | 22 | 23 | -------------------------------------------------------------------------------- /vrequest/pyrc4.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | def rc4(data, 4 | key = b'default-key', 5 | mode = "encode", 6 | enfunc=base64.b64encode, 7 | defunc=base64.b64decode): 8 | if mode == "decode": data = defunc(data) 9 | S = list(range(256)) 10 | j = 0 11 | for i in range(256): 12 | j = (j + S[i] + key[i%len(key)]) % 256 13 | S[i], S[j] = S[j], S[i] 14 | i, j = 0, 0 15 | R = [] 16 | for c in data: 17 | i = (i + 1) % 256 18 | j = (j + S[i]) % 256 19 | S[i], S[j] = S[j], S[i] 20 | t = c ^ (S[(S[i] + S[j]) % 256]) 21 | R.append(t) 22 | if mode == "encode": return enfunc(bytes(R)) 23 | return bytes(R) 24 | 25 | if __name__ == '__main__': 26 | data = '123' 27 | key = '123' 28 | 29 | print('key:', key) 30 | print('data:', data) 31 | 32 | key = key.encode() 33 | data = data.encode() 34 | encd = rc4(data,key,mode='encode');print('rc4 encode:',encd.decode()) 35 | decd = rc4(encd,key,mode='decode');print('rc4 decode:',decd.decode()) -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 cilame 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /vrequest/pypng2gif.py: -------------------------------------------------------------------------------- 1 | # 通过生成的png生成gif图片 2 | 3 | import os 4 | from PIL import Image 5 | 6 | def create_gif(filepathname,filedir,size=None,scale=None,step=1): 7 | d = [] 8 | for i in os.listdir(filedir): 9 | if i.endswith('.png'): 10 | d.append(os.path.join(filedir,i)) 11 | func = lambda i: int(os.path.split(i)[-1].rsplit('.')[0]) 12 | r = [] 13 | for i in sorted(d,key=func): 14 | r.append(Image.open(i)) 15 | q = [] 16 | for i in r[::step]: 17 | if size is not None: 18 | q.append(i.resize(size)) 19 | elif scale is not None: 20 | width = int(i.width/scale) 21 | height = int(i.height/scale) 22 | q.append(i.resize((width,height))) 23 | else: 24 | q.append(i) 25 | q[0].save( 26 | filepathname, 27 | save_all=True, 28 | append_images=q[1:], 29 | loop=0, 30 | duration=120, 31 | comment=b"aaabb" 32 | ) 33 | return q[0].size 34 | 35 | if __name__ == '__main__': 36 | filedir = r'C:\Users\Administrator\Desktop\_temp' 37 | filepathname = r'C:\Users\Administrator\Desktop\gif.gif' 38 | create_gif(filepathname,filedir,step=2) 39 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # python setup.py bdist_wheel upload 2 | 3 | from setuptools import setup 4 | import vrequest 5 | 6 | setup( 7 | name = "vrequest", 8 | version = vrequest.__version__, 9 | keywords = "vrequest", 10 | author = "cilame", 11 | author_email = "opaquism@hotmail.com", 12 | url="https://github.com/cilame/vrequest", 13 | license = "MIT", 14 | description = "", 15 | long_description = '', 16 | long_description_content_type="text/markdown", 17 | classifiers = [ 18 | 'License :: OSI Approved :: MIT License', 19 | 'Programming Language :: Python', 20 | 'Intended Audience :: Developers', 21 | ], 22 | 23 | packages = [ 24 | "vrequest", 25 | ], 26 | package_data ={ 27 | "vrequest":[ 28 | 'template/*', 29 | 'template/v/*', 30 | 'template/v/spiders/*', 31 | 'ico.ico', 32 | 'astjs.zip', 33 | 'main.js', 34 | 'ast_hook_inject.js', 35 | ] 36 | }, 37 | python_requires=">=3.0", 38 | install_requires=[ 39 | 'requests', 40 | 'lxml', 41 | ], 42 | entry_points={ 43 | 'gui_scripts': [ 44 | 'vv = vrequest.main:execute', 45 | 'vrequest = vrequest.main:execute', 46 | 'ee = vrequest.main:algo', 47 | ] 48 | }, 49 | ) -------------------------------------------------------------------------------- /vrequest/pyrots.py: -------------------------------------------------------------------------------- 1 | def rot5(string): 2 | s = '0123456789' 3 | r = [] 4 | for i in string: 5 | if i in s: 6 | v = s[(s.index(i)+len(s)//2)% len(s)] 7 | else: 8 | v = i 9 | r.append(v) 10 | return ''.join(r) 11 | 12 | def rot13(string): 13 | s = 'abcdefghijklmnopqrstuvwxyz' 14 | u = s.upper() 15 | r = [] 16 | for i in string: 17 | if i in s: 18 | v = s[(s.index(i)+len(s)//2)% len(s)] 19 | elif i in u: 20 | v = u[(u.index(i)+len(u)//2)% len(u)] 21 | else: 22 | v = i 23 | r.append(v) 24 | return ''.join(r) 25 | 26 | def rot18(string): 27 | s = '0123456789abcdefghijklmnopqrstuvwxyz' 28 | u = s.upper() 29 | r = [] 30 | for i in string: 31 | if i in s: 32 | v = s[(s.index(i)+len(s)//2)% len(s)] 33 | elif i in u: 34 | v = u[(u.index(i)+len(u)//2)% len(u)] 35 | else: 36 | v = i 37 | r.append(v) 38 | return ''.join(r) 39 | 40 | def rot47(string): 41 | s = list(range(33,127)) 42 | r = [] 43 | for i in string.encode(): 44 | if i in s: 45 | v = s[(s.index(i)+len(s)//2)% len(s)] 46 | else: 47 | v = i 48 | r.append(v) 49 | return bytes(r).decode() 50 | 51 | 52 | if __name__ == '__main__': 53 | s = 'nihaoaxiongdi1234567890_ABCDEFGHIJKLMNOPQRSTUVWXYZ' 54 | s = rot5(s); print('enc>',s) 55 | s = rot5(s); print('dec=',s) 56 | s = rot13(s); print('enc>',s) 57 | s = rot13(s); print('dec=',s) 58 | s = rot18(s); print('enc>',s) 59 | s = rot18(s); print('dec=',s) # 注意，加密数据若存在数字，则逆运算的大小将无法全部还原 60 | s = rot47(s); print('enc>',s) 61 | s = rot47(s); print('dec=',s) # 注意，加密数据若存在数字，则逆运算的大小将无法全部还原 62 | print() -------------------------------------------------------------------------------- /vrequest/pyset_pypi_src.py: -------------------------------------------------------------------------------- 1 | import re, os 2 | import urllib.parse 3 | 4 | try: 5 | appdatapath = os.environ['APPDATA'] or os.path.join(os.path.expanduser('~'), 'AppData', 'Roaming') 6 | pippath = os.path.join(appdatapath, 'pip') 7 | pipfile = os.path.join(pippath, 'pip.ini') 8 | if not os.path.isdir(pippath): 9 | os.mkdir(pippath) 10 | except: 11 | import traceback 12 | traceback.print_exc() 13 | 14 | diclist = [ 15 | ['http://pypi.douban.com/simple/', '豆瓣'], 16 | ['http://mirrors.aliyun.com/pypi/simple/', '阿里云'], 17 | ['http://pypi.tuna.tsinghua.edu.cn/simple/', '清华大学'], 18 | ['http://pypi.mirrors.ustc.edu.cn/simple/', '中国科学技术大学'], 19 | ] 20 | dic_url2name = {} 21 | dic_name2url = {} 22 | for k,v in diclist: 23 | dic_url2name[k] = v 24 | dic_name2url[v] = k 25 | 26 | def read_setting(): 27 | if not os.path.isfile(pipfile): 28 | return None 29 | else: 30 | with open(pipfile) as f: 31 | setstr = f.read() 32 | mirrors = re.findall('\nmirrors = ([^\n]+)', setstr)[0] 33 | return dic_url2name.get(mirrors) 34 | 35 | def write_setting(name=None): 36 | setting = '''[global]\nindex-url = {}\n[install]\nuse-mirrors = true\nmirrors = {}\ntrusted-host = {}'''.strip() 37 | if name is None: 38 | if os.path.isfile(pipfile): 39 | os.remove(pipfile) 40 | return 41 | if name not in dic_name2url: 42 | raise Exception("{} must in {}".format(name, list(dic_name2url))) 43 | mirrors = dic_name2url.get(name) 44 | index_url = mirrors.strip(' /') 45 | trusted_host = urllib.parse.urlsplit(index_url).netloc 46 | with open(pipfile, 'w') as f: 47 | f.write(setting.format(index_url, mirrors, trusted_host)) 48 | 49 | if __name__ == '__main__': 50 | write_setting('豆瓣') 51 | v = read_setting() 52 | print(v) -------------------------------------------------------------------------------- /vrequest/pywindowproxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding:UTF-8 -*- 2 | import ctypes 3 | import winreg 4 | class WindowProxySetting: 5 | def __init__(self, proxy="127.0.0.1:8888"): 6 | proxy_path = r'Software\\Microsoft\\Windows\\CurrentVersion\\Internet Settings' 7 | self.proxy = proxy 8 | self.root = winreg.HKEY_CURRENT_USER 9 | self.hKey = winreg.OpenKey(winreg.HKEY_CURRENT_USER, proxy_path) 10 | self.set_proxy = [ 11 | [proxy_path, "ProxyEnable", winreg.REG_DWORD, 1], 12 | [proxy_path, "ProxyServer", winreg.REG_SZ, self.proxy], 13 | ] 14 | def _flash(self): 15 | # 让修改立即生效 16 | try: 17 | internet_set_option = ctypes.windll.Wininet.InternetSetOptionW 18 | internet_set_option(0, 39, 0, 0) 19 | internet_set_option(0, 37, 0, 0) 20 | except:pass 21 | def open(self): 22 | self.set_proxy[0][3] = 1 23 | for keypath, value_name, value_type, value in self.set_proxy: 24 | self.hKey = winreg.CreateKey(self.root, keypath) 25 | winreg.SetValueEx(self.hKey, value_name, 0, value_type, value) 26 | self._flash() 27 | def close(self): 28 | self.set_proxy[0][3] = 0 29 | for keypath, value_name, value_type, value in self.set_proxy: 30 | self.hKey = winreg.CreateKey(self.root, keypath) 31 | winreg.SetValueEx(self.hKey, value_name, 0, value_type, value) 32 | self._flash() 33 | def get_state(self): 34 | value, type = winreg.QueryValueEx(self.hKey, "ProxyEnable") 35 | return bool(value) 36 | def get_server(self): 37 | value, type = winreg.QueryValueEx(self.hKey, "ProxyServer") 38 | return value 39 | 40 | if __name__ == '__main__': 41 | wproxy = WindowProxySetting() 42 | v = wproxy.get_state() 43 | print(v) 44 | v = wproxy.get_server() 45 | print(v) 46 | 47 | # wproxy.open() 48 | # wproxy.close() -------------------------------------------------------------------------------- /vrequest/pyrail_fence.py: -------------------------------------------------------------------------------- 1 | # 栅栏密码 2 | def rail_fence_enc(string, n, padding=None): 3 | # 增加padding主要是为了加密的鲁棒性 4 | b = [] 5 | q = [] 6 | for idx,i in enumerate(string, 1): 7 | q.append(i) 8 | if idx % n == 0: 9 | b.append(q) 10 | q = [] 11 | r = [] 12 | if q: 13 | for i in '@~!#$%^&*': # 自动选一个padding进行处理 14 | if i not in string: 15 | padding = i 16 | break 17 | if padding is None: 18 | raise 'cannot find a padding, cos @~!#$%^&* all in string.' 19 | q += [padding] * (n - len(q)) 20 | b.append(q) 21 | for i in zip(*b): 22 | r.extend(i) 23 | return ''.join(r), padding 24 | 25 | def rail_fence_dec(string, n, padding=None, return_matrix=False): 26 | # 这里的输入必须可以生成矩阵，否则会有异常情况 27 | a = len(string)/n 28 | b = len(string)//n 29 | n = b if a == b else b+1 30 | b = [] 31 | for i in range(0,n): 32 | b.append(string[i::int(n)]) 33 | r = [] 34 | for i in zip(b): 35 | r.extend(i) 36 | if return_matrix: 37 | return r 38 | return ''.join(r).rstrip(padding) 39 | 40 | 41 | def get_primes(_int): 42 | r = [] 43 | for i in range(2, _int): 44 | if _int % i == 0: 45 | r.append(i) 46 | return r 47 | 48 | def rail_fence_enum(string, return_matrix=False): 49 | prs = get_primes(len(string)) 50 | r = [] 51 | for i in prs: 52 | v = rail_fence_dec(string, i, return_matrix=return_matrix) 53 | a, b = i, len(string)//i 54 | r.append((a,b,v)) 55 | return r 56 | 57 | 58 | if __name__ == '__main__': 59 | # 栅栏加密，默认使用带填充模式 60 | # 也可以通过 padding 来判断加密是否存在自动填充情况 61 | s, padding = rail_fence_enc('asdfasdfasdff', 3) 62 | print(s) 63 | print(padding) 64 | s = rail_fence_dec(s, 3, padding=padding) 65 | print(s) 66 | print() 67 | 68 | 69 | s, padding = rail_fence_enc('abcdefghijklmn', 7) 70 | print(s) 71 | for i in rail_fence_enum(s): 72 | print(i) -------------------------------------------------------------------------------- /MYLOG.md: -------------------------------------------------------------------------------- 1 | ``` 2 | # 最近想要实现的一个小功能 3 | # 就是想通过简单的 tkinter 来实现一个非常简单的url分析工具 4 | # 目的就是想要实现，快速测试，快色生成代码的一些功能， 5 | # 其中也包含一些简单的智能分析功能，让简单的工作更高效的完成。 6 | 7 | # 初期的功能大概就是： 8 | # 0 简洁的GUI 9 | # 1 智能分析列表路径（这个比较麻烦） 10 | # 2 生成简单的测试代码，支持一键执行。这里不能用python自带的IDLE，因为非常卡。 11 | # 3 支持默认配置、记录历史配置、增删配置的功能。 12 | # 4 简单的命令行打开。 13 | 14 | #20190115 15 | 今天应该会是经可能将整个项目的框架稍微处理好一点 16 | 将一些初始化的内容尽可能写更好一些，先暂时不考虑测试，将整体功能实现先。 17 | 其中主要就是输入框的设计会稍微有点麻烦。 18 | # 目前的主要问题就在于怎么设计一个请求页的数据结构 19 | # 方便存储，也方便使用。 20 | 21 | #20190117 22 | 整理了整个框架的结构，现在使用变得更加方便了。 23 | 快捷键的设置已经处理完毕，就差对接结构，让数据使用更加方便一些。 24 | 现在突然有种想要把 pyinstaller 也集成在里面的冲动 25 | 直接生成一个 exe 工具的感觉其实还是不错喂。 26 | 27 | #20190118 28 | 关于请求回复的对接想要考虑使用一个新窗口来接收 response ，同样是一个标签窗口 29 | response 窗口的标签名字对应请求的标签名字 30 | 补充：后续发现新窗口在便利性上并不好，并且技术难度稍微有点大，目前就先针对功能开发即可。 31 | 所以放弃新生成窗口的方式来实现 response 部分的处理，现在用新的 tab 来装 response。 32 | # 目前需要解决的问题： 33 | # 1 怎么才能做到大量文本粘贴而只粘贴少部分数据 34 | # 并且还要在要在内部接收到全部的数据。 35 | # 目前需要做的事儿： 36 | # 1 请求的处理 37 | # 2 解析的处理 38 | # 3 生成代码的处理 39 | 40 | #20190119 41 | **还需要考虑标签颜色的处理 42 | 43 | #20190120 44 | 后续发现没有必要魔改粘贴的方法，只需要忽略自动回车就能有效的缓解问题。 45 | 生成的代码的处理在功能上还是稍微有点少了些，不过现在还是没有复杂的功能。 46 | 并且现在在 post 的理解上面还是不太够。后续需要参考一些现已有的工具的处理方式。 47 | 48 | #20190122 49 | 感觉还是需要考虑怎么来处理json数据的内容以及输出内容 50 | 目前 post 就简单传入 data 一个字典来处理，后续如果出现有问题的地方 51 | 也会考虑怎么修改自动生成代码的正确性。就酱紫了。 52 | 53 | #20190123 54 | 修复生成url代码中的小问题，在比较苛刻情况下才会出现的结构解析也能生成正确的代码了。 55 | 比如说知乎的超长接口 url，现在能正确格式化成正确的代码。解决 post 中 body 传输的问题， 56 | 算是一种暂时通配解决方式。现在body能接收的格式有两种：一种用冒号分割，一种用等号分割 57 | eg.1 58 | courseId: 1001553026 59 | pageIndex: 1 60 | pageSize: 20 61 | orderBy: 3 62 | eg.2 63 | callCount=1 64 | scriptSessionId=${scriptSessionId}190 65 | httpSessionId=11e660f244a04251ab8e8420f4cc28b3 66 | c0-scriptName=CommonBean 67 | c0-methodName=obtain 68 | c0-id=0 69 | c0-param0=string:ActivitySetting 70 | batchId=1548220950032 71 | 72 | #20190412 73 | 解决了scrapy代码生成以及执行的功能，后续可能就是简单的整理生成代码的框架上面的小问题了。 74 | 整理修复了一下的几个小的BUG。 75 | 76 | #20190423 77 | 将xpath的自动解析功能增强到和json列表解析几乎相同水平。这应该会是对网页解析的最后一个功能了。 78 | 后续的处理可能更多会在其他功能的补充或是debug上面。框架基本定型了。 79 | 80 | #20190501 81 | 在处理一些极端的编码时会出现很抽象的问题， 82 | 主要原因是类似有一些 utf-8 编码网页里面参入了几个 utf-8 无法解析的字符时候 83 | utf8 就完全不能解析了，所以考虑到这点并且考虑到我的工具主要是用在处理utf-8和gbk上面 84 | 那么就稍微修改一点点，至少方便使用一些。 85 | 86 | #20200305 87 | 一个非常真实且操蛋的问题： SimpleDialog 和 threading 不兼容，有种日狗的错觉。 88 | （已经解决） 89 | 90 | #20200329 91 | 一个非常操蛋的问题，就是 pyzbar 曾测试于某些 win7 系统不能默认直接使用。 92 | 所以将 pyzbar 硬集成在该工具里面增加非常大的包重量感觉没有特别大的必要。 93 | 所以现在考虑直接使用库，后续会考虑将该功能直接用第三方实现。 94 | 并且，现在想要将某个依赖于 js 的对 js 进行解混肴的工具集成进来，所以需要考虑库的压缩。 95 | 96 | #20220505 97 | # python setup.py bdist_wheel upload <=== 这里的提交方式已经过时了，会出现其他问题，使用下面的方式提交。 98 | # python setup.py sdist; twine upload dist/… 99 | ``` -------------------------------------------------------------------------------- /vrequest/pywiener_attack.py: -------------------------------------------------------------------------------- 1 | import decimal 2 | 3 | # 使用自带函数库实现高精度计算，十进制1000位已经够用 4 | decimal.getcontext().prec = 1000 5 | Decimal = decimal.Decimal 6 | 7 | def trans(x, y): 8 | res = [] 9 | while y: 10 | res.append(x //y) 11 | x, y = y, x%y 12 | return res 13 | 14 | def next_fract(sub_res): 15 | num, denom = 1, 0 16 | for i in sub_res[::-1]: 17 | denom, num = num, i*num+denom 18 | return denom, num 19 | 20 | def ex_gcd(a,b): 21 | if b == 0: 22 | return (1,0,a) 23 | (x, y, r) = ex_gcd(b,a%b) 24 | t = x 25 | x = y 26 | y = t - a//b*y 27 | return (x,y,r) 28 | 29 | def sub_fract(e, n): 30 | res = trans(e, n) 31 | res = list(map(next_fract,(res[0:i] for i in range(1,len(res))))) 32 | return res 33 | 34 | def get_pq(a,b,c): 35 | par = (Decimal(str(b)) * Decimal(str(b)) \ 36 | - Decimal('4') * Decimal(str(a)) * Decimal(str(c))) \ 37 | ** Decimal(str('0.5')) 38 | par = int(par) # 将 Decimal 对象转换回 int 对象，以免在后续出现异常 39 | return (-b+par)//(2*a), (-b-par)//(2*a) 40 | 41 | def wiener_attack(e, n): 42 | for d, k in sub_fract(e, n): #用一个for循环来注意试探e/n的连续函数的渐进分数，直到找到一个满足条件的渐进分数 43 | if k==0: continue 44 | if (e*d-1) % k!=0: continue 45 | try: 46 | px, qy = get_pq(1, n-((e*d-1)//k)+1, n) 47 | except: 48 | return 'Error, Wiener_attack algorithm does not work!' 49 | if px*qy == n: 50 | p, q = abs(px), abs(qy) #可能会得到两个负数，负负得正未尝不会出现 51 | a, b, r = ex_gcd((p-1)*(q-1), e) 52 | return b 53 | 54 | if __name__ == '__main__': 55 | e = 14058695417015334071588010346586749790539913287499707802938898719199384604316115908373997739604466972535533733290829894940306314501336291780396644520926473 56 | n = 33608051123287760315508423639768587307044110783252538766412788814888567164438282747809126528707329215122915093543085008547092423658991866313471837522758159 57 | d = wiener_attack(e, n) 58 | print("d =",d) 59 | 60 | e = 354611102441307572056572181827925899198345350228753730931089393275463916544456626894245415096107834465778409532373187125318554614722599301791528916212839368121066035541008808261534500586023652767712271625785204280964688004680328300124849680477105302519377370092578107827116821391826210972320377614967547827619 61 | n = 460657813884289609896372056585544172485318117026246263899744329237492701820627219556007788200590119136173895989001382151536006853823326382892363143604314518686388786002989248800814861248595075326277099645338694977097459168530898776007293695728101976069423971696524237755227187061418202849911479124793990722597 62 | d = wiener_attack(e, n) 63 | print("d =",d) -------------------------------------------------------------------------------- /vrequest/py_mini_requests.py: -------------------------------------------------------------------------------- 1 | # mini requests @author: vilame 2 | def mini_requests(defaultencode='utf-8'): 3 | def requests(method): 4 | import ssl, json 5 | from urllib import request, error 6 | from urllib.parse import quote 7 | class resper: 8 | def __init__(self, url, res, req, encode, errormsg=None): 9 | self.url = url 10 | self.resb = res.read() 11 | self.status_code = res.status 12 | self.dectype = encode 13 | self.errormsg = errormsg 14 | self.request = req 15 | self.headers = dict(res.getheaders()) 16 | def __getattr__(self, name): 17 | if name == 'content': return self._content() 18 | if name == 'text': return self._text() 19 | def __str__(self): return ''.format(self.status_code) 20 | def _content(self): return self.resb 21 | def _text(self): return self.resb.decode(self.dectype, errors='ignore') 22 | def json(self): return json.loads(self.resb) 23 | def func(url, headers=None, data=None, verify=True, proxies=None, encode=defaultencode): 24 | req = request.Request(url, method=method) 25 | if type(data) == dict: data = '&'.join(['{}={}'.format(quote(k), quote(v)) for k, v in data.items()]).encode() 26 | if type(data) == str: data = data.encode() 27 | req.body = data or b'' 28 | headers = headers or {} 29 | for k, v in list(headers.items()): 30 | if k.lower() == 'accept-encoding': 31 | headers.pop(k); continue 32 | req.add_header(k, v) 33 | lis = [request.ProxyHandler(proxies)] 34 | if not verify: 35 | ctx = ssl.create_default_context() 36 | ctx.check_hostname = False 37 | ctx.verify_mode = ssl.CERT_NONE 38 | lis.append(request.HTTPSHandler(context = ctx)) 39 | opener = request.build_opener(*lis) 40 | errormsg = None 41 | try: 42 | ret = opener.open(req) if method == 'GET' else opener.open(req, data=data) 43 | except error.HTTPError as e: 44 | class err: 45 | def __init__(self, status, headers, body=b''): 46 | self.status = status 47 | self.read = lambda:body 48 | self.getheaders = lambda: headers 49 | ret = err(e.getcode(), e.headers.items(), e.file.read()); errormsg = e.msg 50 | return resper(url, ret, req, encode, errormsg) 51 | return func 52 | requests.get = requests('GET') 53 | requests.post = requests('POST') 54 | return requests 55 | requests = mini_requests() -------------------------------------------------------------------------------- /vrequest/pycompress.py: -------------------------------------------------------------------------------- 1 | import zlib 2 | import lzma 3 | import gzip 4 | import base64 5 | 6 | 7 | def format_compress_string(data, base, compress, width, encode): 8 | return format_compress(data, base, compress, width, encode, is_file=False) 9 | 10 | def format_compress_file(base, compress, width, encode): 11 | return format_compress(None, base, compress, width, encode, is_file=True) 12 | 13 | def format_compress(data, base, compress, width, encode, is_file=False): 14 | if not is_file: 15 | data = data.encode(encode) 16 | else: 17 | import tkinter.filedialog 18 | file = tkinter.filedialog.askopenfiles() 19 | if file: 20 | file = file[0].name 21 | with open(file, 'rb') as f: 22 | data = f.read() 23 | else: 24 | return '' 25 | 26 | if base == 'base64': _encode, _decode = base64.b64encode, base64.b64decode 27 | if base == 'base85': _encode, _decode = base64.b85encode, base64.b85decode 28 | 29 | if compress == 'None': zcompress, zdecompress = lambda i:i, lambda i:i 30 | if compress == 'zlib': zcompress, zdecompress = lambda i:zlib.compress(i)[2:-4], lambda i:zlib.decompress(i,-15) 31 | if compress == 'lzma': zcompress, zdecompress = lambda i:lzma.compress(i), lambda i:lzma.decompress(i) 32 | if compress == 'gzip': zcompress, zdecompress = lambda i:gzip.compress(i), lambda i:gzip.decompress(i) 33 | 34 | if base == 'base64': fbstring = 'zstring = base64.b64decode(zstring)' 35 | if base == 'base85': fbstring = 'zstring = base64.b85decode(zstring)' 36 | if compress == 'None': importstring = 'import base64'; fzstring = '' 37 | if compress == 'zlib': importstring = 'import base64, zlib'; fzstring = 'zstring = zlib.decompress(zstring,-15)' 38 | if compress == 'lzma': importstring = 'import base64, lzma'; fzstring = 'zstring = lzma.decompress(zstring)' 39 | if compress == 'gzip': importstring = 'import base64, gzip'; fzstring = 'zstring = gzip.decompress(zstring)' 40 | 41 | if is_file: 42 | datastring = 'bitdata = zstring\nprint(bitdata[:100])' 43 | else: 44 | datastring = 'string = zstring.decode("{}")\nprint(string)'.format(encode) 45 | 46 | zdata = zcompress(data) 47 | edata = _encode(zdata).decode() 48 | pack = [] 49 | for idx,i in enumerate(edata,1): 50 | pack.append(i) 51 | if idx % width == 0: 52 | pack.append("'\n'") 53 | packdata = "'{}'".format(''.join(pack)) 54 | leninfo = '# len(zstring): {}'.format(len(edata)) 55 | 56 | return r''' 57 | zstring = ( 58 | {} 59 | ) 60 | 61 | {} 62 | {} 63 | {}{} 64 | {} 65 | '''.format(packdata, leninfo, importstring, fbstring, '\n' + fzstring if fzstring else fzstring, datastring).strip() 66 | 67 | if __name__ == '__main__': 68 | string = r'''testcode''' 69 | v = format_compress_string(string, 'base64', 'None', 10) 70 | print(v) 71 | 72 | v = format_compress_file('base64', 'None', 10) 73 | print(v) -------------------------------------------------------------------------------- /vrequest/pycv2.py: -------------------------------------------------------------------------------- 1 | canread = ['.bmp', '.dib', '.jpeg', '.jpg', '.jpe', '.jp2', '.png', '.webp', '.pbm', '.pgm', '.ppm', '.pxm', '.pnm', '.sr', '.ras', '.tiff', '.tif', '.exr', '.hdr', '.pic'] 2 | 3 | import cv2 4 | 5 | def canny(filepathname, left=70, right=140): 6 | v = cv2.imread(filepathname) 7 | s = cv2.cvtColor(v, cv2.COLOR_BGR2GRAY) 8 | s = cv2.Canny(s, left, right) 9 | cv2.imshow('nier',s) 10 | return s 11 | 12 | # 圈出最小方矩形框，这里Canny算法后都是白色线条，所以取色范围 127-255 即可。 13 | # ret, binary = cv2.threshold(s,127,255,cv2.THRESH_BINARY) 14 | # contours, hierarchy = cv2.findContours(binary,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 15 | # for c in contours: 16 | # x,y,w,h = cv2.boundingRect(c) 17 | # if w>5 and h>10: # 有约束的画框 18 | # cv2.rectangle(v,(x,y),(x+w,y+h),(155,155,0),1) 19 | # # cv2.drawContours(s,contours,-1,(0,0,255),3) # 画所有框 20 | # cv2.imshow('nier2',v) 21 | 22 | # cv2.waitKey() 23 | # cv2.destroyAllWindows() 24 | 25 | 26 | def laplacian(filepathname): 27 | v = cv2.imread(filepathname) 28 | s = cv2.cvtColor(v, cv2.COLOR_BGR2GRAY) 29 | s = cv2.Laplacian(s, cv2.CV_16S, ksize=3) 30 | s = cv2.convertScaleAbs(s) 31 | cv2.imshow('nier',s) 32 | return s 33 | 34 | # ret, binary = cv2.threshold(s,40,255,cv2.THRESH_BINARY) 35 | # contours, hierarchy = cv2.findContours(binary,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 36 | # for c in contours: 37 | # x,y,w,h = cv2.boundingRect(c) 38 | # if w>5 and h>10: 39 | # cv2.rectangle(v,(x,y),(x+w,y+h),(155,155,0),1) 40 | # cv2.imshow('nier2',v) 41 | 42 | # cv2.waitKey() 43 | # cv2.destroyAllWindows() 44 | 45 | 46 | def sobel(filepathname): 47 | v = cv2.imread(filepathname) 48 | s = cv2.cvtColor(v,cv2.COLOR_BGR2GRAY) 49 | x, y = cv2.Sobel(s,cv2.CV_16S,1,0), cv2.Sobel(s,cv2.CV_16S,0,1) 50 | s = cv2.convertScaleAbs(cv2.subtract(x,y)) 51 | s = cv2.blur(s,(9,9)) 52 | cv2.imshow('nier',s) 53 | return s 54 | 55 | # ret, binary = cv2.threshold(s,40,255,cv2.THRESH_BINARY) 56 | # contours, hierarchy = cv2.findContours(binary,cv2.RETR_TREE,cv2.CHAIN_APPROX_SIMPLE) 57 | # for c in contours: 58 | # x,y,w,h = cv2.boundingRect(c) 59 | # if w>5 and h>10: 60 | # cv2.rectangle(v,(x,y),(x+w,y+h),(155,155,0),1) 61 | # cv2.imshow('nier2',v) 62 | 63 | # cv2.waitKey() 64 | # cv2.destroyAllWindows() 65 | 66 | 67 | def findmatchtemplate(filepathname, befindimage): 68 | # 从 befindimage 中找到 filepathname，（befindimage 是大图，filepathname 是小图） 69 | img1 = cv2.imread(filepathname) 70 | img2 = cv2.imread(befindimage) 71 | w, h = img1.shape[:2] 72 | v = cv2.matchTemplate(img2,img1,cv2.TM_CCOEFF) 73 | a, b, c, top_left = cv2.minMaxLoc(v) 74 | bot_right = top_left[0]+h, top_left[1]+w 75 | img3 = cv2.rectangle(img2, top_left, bot_right, (155,155,0), 1) 76 | cv2.imshow('nier', img3) 77 | # cv2.waitKey() 78 | # cv2.destroyAllWindows() 79 | return top_left[0], top_left[1], w, h -------------------------------------------------------------------------------- /vrequest/root.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tkinter 3 | import json 4 | import traceback 5 | 6 | root = tkinter.Tk() 7 | 8 | # 配置文件名字 9 | DEFAULTS_NAME = '.vrequest' 10 | DEFAULTS_HEADERS = ''' 11 | accept-encoding: gzip, deflate 12 | accept-language: zh-CN,zh;q=0.9 13 | accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8 14 | user-agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36 15 | ''' 16 | # TODO, 一般新建请求页的时候需要的默认值，方便使用。 17 | 18 | ''' 19 | #20190117 20 | # 1 边框大小位置 21 | # 2 需要的配置数据 22 | # 初步定下下面的数据结构 23 | { 24 | siz:'500x300' 25 | set:{ 26 | tab_name1:{setting} 27 | tab_name2:{setting} 28 | tab_name3:{setting} 29 | } 30 | } 31 | 关于 setting的数据结构 32 | { 33 | type:request # 这里要有多种类型的配置，为了方便处理保存和恢复选择用的函数 34 | set:{ 35 | # 不同类型的配置结构不一样 36 | # 以 request为例， 37 | method: GET/POST/... 38 | url: url 39 | headers: headers 40 | body: body 41 | # 也有帮助标签 42 | } 43 | } 44 | ''' 45 | 46 | 47 | # 默认的数据结构 48 | config = { 49 | 'siz':'680x850+100+100', 50 | 'set':{}, 51 | 'focus':None, 52 | } 53 | 54 | 55 | home = None 56 | 57 | # 用来配置一些需要持久化的配置 58 | def get_config_from_homepath(): 59 | global config, home 60 | defaults_conf = config.copy() 61 | try: 62 | home = os.environ.get('HOME') 63 | home = home if home else os.environ.get('HOMEDRIVE') + os.environ.get('HOMEPATH') 64 | configfile = os.path.join(home,DEFAULTS_NAME) 65 | if os.path.exists(configfile): 66 | with open(configfile,encoding='utf-8') as f: 67 | defaults_conf = json.load(f) 68 | config = defaults_conf 69 | except: 70 | print('unlocal homepath.') 71 | traceback.print_exc() 72 | 73 | 74 | # 修改本地的默认配置，让其变成一个全局可用参数 75 | get_config_from_homepath() 76 | 77 | 78 | # 用来持久化当前配置的情况，简单的快照，便于使用 79 | def set_config_from_homepath(): 80 | global config, home 81 | defaults_conf = config 82 | try: 83 | home = os.environ.get('HOME') 84 | home = home if home else os.environ.get('HOMEDRIVE') + os.environ.get('HOMEPATH') 85 | configfile = os.path.join(home,DEFAULTS_NAME) 86 | with open(configfile,'w',encoding='utf-8') as f: 87 | f.write(json.dumps(defaults_conf,indent=4)) 88 | except: 89 | print('unlocal homepath.') 90 | traceback.print_exc() 91 | 92 | 93 | # 装饰器 94 | # 装饰函数在执行时候进行持久化当前配置的操作（通常在请求 url 时候进行保存） 95 | def save(func): 96 | def _save(*a,**kw): 97 | v = func(*a,**kw) 98 | set_config_from_homepath() 99 | return v 100 | return _save 101 | 102 | 103 | 104 | # 绑定窗口大小变化，让配置也能记录窗口状态。 105 | def change_siz(): 106 | global config 107 | config['siz'] = '{}x{}+{}+{}'.format( 108 | root.winfo_width(), 109 | root.winfo_height(), 110 | root.winfo_x(), 111 | root.winfo_y(), ) 112 | 113 | 114 | # 用于处理收尾的函数。 115 | tails = [] -------------------------------------------------------------------------------- /vrequest/pybacon.py: -------------------------------------------------------------------------------- 1 | def bacon_enc(string,a='a',b='b',ver='v1'): 2 | if ver == 'v1': 3 | maps = { 4 | 'A': 'aaaaa', 'B': 'aaaab', 'C': 'aaaba', 'D': 'aaabb', 'E': 'aabaa', 'F': 'aabab', 5 | 'G': 'aabba', 'H': 'aabbb', 'I': 'abaaa', 'J': 'abaab', 'K': 'ababa', 'L': 'ababb', 6 | 'M': 'abbaa', 'N': 'abbab', 'O': 'abbba', 'P': 'abbbb', 'Q': 'baaaa', 'R': 'baaab', 7 | 'S': 'baaba', 'T': 'baabb', 'U': 'babaa', 'V': 'babab', 'W': 'babba', 'X': 'babbb', 8 | 'Y': 'bbaaa', 'Z': 'bbaab' 9 | } 10 | elif ver == 'v2': 11 | maps = { 12 | "A":"aaaaa", "G":"aabba", "N":"abbaa", "T":"baaba", "B":"aaaab", "H":"aabbb", 13 | "O":"abbab", "C":"aaaba", "P":"abbba", "W":"babaa", 14 | "D":"aaabb", "K":"abaab", "Q":"abbbb", "X":"babab", "E":"aabaa", "L":"ababa", 15 | "R":"baaaa", "Y":"babba", "F":"aabab", "M":"ababb", "S":"baaab", "Z":"babbb", 16 | "U":"baabb", "V":"baabb", # "U-V":"baabb" 17 | "I":"abaaa", "J":"abaaa", # "I-J":"abaaa" 18 | } 19 | r = [] 20 | for i in string.upper(): 21 | if i in maps: 22 | r.append(maps.get(i).replace('a', a).replace('b', b)) 23 | else: 24 | r.append('[unfind:{}]'.format(i)) 25 | return ' '.join(r) 26 | 27 | def bacon_dec(string,a='a',b='b',ver='v1'): 28 | if ver == 'v1': 29 | maps = { 30 | 'aaaaa': 'A', 'aaaab': 'B', 'aaaba': 'C', 'aaabb': 'D', 'aabaa': 'E', 'aabab': 'F', 31 | 'aabba': 'G', 'aabbb': 'H', 'abaaa': 'I', 'abaab': 'J', 'ababa': 'K', 'ababb': 'L', 32 | 'abbaa': 'M', 'abbab': 'N', 'abbba': 'O', 'abbbb': 'P', 'baaaa': 'Q', 'baaab': 'R', 33 | 'baaba': 'S', 'baabb': 'T', 'babaa': 'U', 'babab': 'V', 'babba': 'W', 'babbb': 'X', 34 | 'bbaaa': 'Y', 'bbaab': 'Z' 35 | } 36 | elif ver == 'v2': 37 | maps = { 38 | "aaaaa":"A", "aabba":"G", "abbaa":"N", "baaba":"T", "aaaab":"B", 39 | "aabbb":"H", "abbab":"O", "baabb":"U-V", "aaaba":"C", "abaaa":"I-J", 40 | "abbba":"P", "babaa":"W", "aaabb":"D", "abaab":"K", "abbbb":"Q", 41 | "babab":"X", "aabaa":"E", "ababa":"L", "baaaa":"R", "babba":"Y", 42 | "aabab":"F", "ababb":"M", "baaab":"S", "babbb":"Z", 43 | } 44 | r = [] 45 | _string = string.replace(a, 'a').replace(b, 'b') 46 | for i in _string.split(): 47 | if i.lower() in maps: 48 | r.append(maps.get(i.lower())[-1]) # v2 模式统统使用最后一个字符解密 49 | else: 50 | r.append('[unfind:{}]'.format(i)) 51 | return ''.join(r) 52 | 53 | def bacon_v1_enc(string,a='a',b='b'): return bacon_enc(string,a=a,b=b,ver='v1') 54 | def bacon_v2_enc(string,a='a',b='b'): return bacon_enc(string,a=a,b=b,ver='v2') 55 | def bacon_v1_dec(string,a='a',b='b'): return bacon_dec(string,a=a,b=b,ver='v1') 56 | def bacon_v2_dec(string,a='a',b='b'): return bacon_dec(string,a=a,b=b,ver='v2') 57 | 58 | if __name__ == '__main__': 59 | s = 'lkasjdklfjalsdf' 60 | s = bacon_v1_enc(s) 61 | print(s) 62 | s = bacon_v1_dec(s) 63 | print(s) 64 | s = 'lkasjdklfjalsdf' 65 | s = bacon_v2_enc(s) 66 | print(s) 67 | s = bacon_v2_dec(s) 68 | print(s) -------------------------------------------------------------------------------- /vrequest/pymorse.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | # 莫斯密码 4 | def morse_dec(string, a='.', b='-', p=None): 5 | morse = { 6 | '.-' :'A', '-...':'B', '-.-.':'C', '-..' :'D', '.' :'E', 7 | '..-.':'F', '--.' :'G', '....':'H', '..' :'I', '.---':'J', 8 | '-.-':'K', '.-..' : 'L', '--' :'M', '-.' :'N', '---':'O', 9 | '.--.' : 'P', '--.-' : 'Q', '.-.':'R', '...':'S', '-' :'T', 10 | '..-':'U', '...-' : 'V', '.--':'W', '-..-' : 'X', '-.--' : 'Y', 11 | '--..' : 'Z', '.----' : '1', '..---' : '2', '...--' : '3', 12 | '....-' : '4', '.....' : '5', '-....' : '6', '--...' : '7', 13 | '---..' : '8','----.' : '9','-----' : '0', 14 | '-...-' : '=', '.-.-':'~', '.-...' :'', '.-.-.' : '', '...-.-' : '', 15 | '-.--.' : '', '..-.-' : '', '....--' : '', '...-.' : '', 16 | '.-..-.' : '\\', '.----.' : '\'', '...-..-' : '$', '-.--.' : '(', '-.--.-' : ')', 17 | '--..--' : ',', '-....-' : '-', '.-.-.-' : '.', '-..-.' : '/', '---...' : ':', 18 | '-.-.-.' : ';', '..--..' : '?', '..--.-' : '_', '.--.-.' : '@', '-.-.--' : '!' 19 | } 20 | _a, _b = '.', '-' 21 | string = re.sub(r'\[undefined:( *)\]', r'\1', string) 22 | _names = string.split(' ') if p is None else string.split(p) 23 | r = [] 24 | for ps in _names: 25 | ps = ps.replace(a, _a).replace(b, _b) 26 | ge = morse.get(ps) 27 | v = re.findall(r'\[undefined:([^\]]*)\]', ps) 28 | if ge: 29 | r.append(ge) 30 | elif v: 31 | r.append(v[0]) 32 | else: 33 | r.append(' ') 34 | return ''.join(r) 35 | def morse_enc(string, a='.', b='-', p=None): 36 | morse = { 37 | 'A': '.-', 'B': '-...', 'C': '-.-.', 'D': '-..', 'E': '.', 38 | 'F': '..-.', 'G': '--.', 'H': '....', 'I': '..', 'J': '.---', 39 | 'K': '-.-', 'L': '.-..', 'M': '--', 'N': '-.', 'O': '---', 'P': 40 | '.--.', 'Q': '--.-', 'R': '.-.', 'S': '...', 'T': '-', 'U': '..-', 41 | 'V': '...-', 'W': '.--', 'X': '-..-', 'Y': '-.--', 'Z': '--..', 42 | '1': '.----', '2': '..---', '3': '...--', '4': '....-', '5': '.....', 43 | '6': '-....', '7': '--...', '8': '---..', '9': '----.', '0': '-----', 44 | '=': '-...-', '~': '.-.-', '': '.-...', '': '.-.-.', 45 | '': '...-.-', '(': '-.--.', '': '..-.-', '': '....--', 46 | '': '...-.', '\\': '.-..-.', "'": '.----.', '$': '...-..-', 47 | ')': '-.--.-', ',': '--..--', '-': '-....-', '.': '.-.-.-', '/': '-..-.', 48 | ':': '---...', ';': '-.-.-.', '?': '..--..', '_': '..--.-', '@': '.--.-.', '!': '-.-.--' 49 | } 50 | _a, _b = '.', '-' 51 | r = [] 52 | for i in string: 53 | if i.upper() in morse: 54 | v = morse[i.upper()].replace(_a, a).replace(_b, b) 55 | else: 56 | v = '[undefined:{}]'.format(i) 57 | r.append(v) 58 | return ' '.join(r) if p is None else p.join(r) 59 | 60 | if __name__ == '__main__': 61 | s = morse_dec('-... -.- -.-. - ..-. -- .. ... -.-.') 62 | print(s) 63 | s = morse_enc('asdfasdfnj()@/!$asdnLKE???#') 64 | print(s) 65 | print(morse_dec(s)) 66 | print() 67 | 68 | s = 'asdfasdf asdfasdf asdfasdf*&!@%*$&^*&' 69 | s = morse_enc(s) 70 | print(s) 71 | print(morse_dec(s)) -------------------------------------------------------------------------------- /vrequest/pyfinddesktopfile.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tkinter 3 | from tkinter import * 4 | 5 | class SimpleDialog: 6 | 7 | def __init__(self, master, 8 | text='', buttons=[], default=None, cancel=None, 9 | title=None, class_=None): 10 | if class_: 11 | self.root = Toplevel(master, class_=class_) 12 | else: 13 | self.root = Toplevel(master) 14 | if title: 15 | self.root.title(title) 16 | self.root.iconname(title) 17 | self.message = Message(self.root, text=text, aspect=400) 18 | self.message.pack(expand=1, fill=BOTH) 19 | self.frame = Frame(self.root) 20 | self.frame.pack(fill=BOTH) 21 | self.num = default 22 | self.cancel = cancel 23 | self.default = default 24 | self.root.bind('', self.return_event) 25 | for num in range(len(buttons)): 26 | s = buttons[num] 27 | b = Button(self.frame, text=s, 28 | command=(lambda self=self, num=num: self.done(num))) 29 | b.config(relief=RIDGE, borderwidth=1) 30 | b.pack(side=TOP, fill=BOTH) 31 | self.root.protocol('WM_DELETE_WINDOW', self.wm_delete_window) 32 | self._set_transient(master) 33 | 34 | def _set_transient(self, master, relx=0.5, rely=0.3): 35 | widget = self.root 36 | widget.withdraw() # Remain invisible while we figure out the geometry 37 | widget.transient(master) 38 | widget.update_idletasks() # Actualize geometry information 39 | if master.winfo_ismapped(): 40 | m_width = master.winfo_width() 41 | m_height = master.winfo_height() 42 | m_x = master.winfo_rootx() 43 | m_y = master.winfo_rooty() 44 | else: 45 | m_width = master.winfo_screenwidth() 46 | m_height = master.winfo_screenheight() 47 | m_x = m_y = 0 48 | w_width = widget.winfo_reqwidth() 49 | w_height = widget.winfo_reqheight() 50 | x = m_x + (m_width - w_width) * relx 51 | y = m_y + (m_height - w_height) * rely 52 | if x+w_width > master.winfo_screenwidth(): 53 | x = master.winfo_screenwidth() - w_width 54 | elif x < 0: 55 | x = 0 56 | if y+w_height > master.winfo_screenheight(): 57 | y = master.winfo_screenheight() - w_height 58 | elif y < 0: 59 | y = 0 60 | widget.geometry("+%d+%d" % (x, y)) 61 | widget.deiconify() # Become visible at the desired location 62 | 63 | def go(self): 64 | self.root.wait_visibility() 65 | self.root.grab_set() 66 | self.root.mainloop() 67 | self.root.destroy() 68 | return self.num 69 | 70 | def return_event(self, event): 71 | if self.default is None: 72 | self.root.bell() 73 | else: 74 | self.done(self.default) 75 | 76 | def wm_delete_window(self): 77 | if self.cancel is None: 78 | self.root.bell() 79 | else: 80 | self.done(self.cancel) 81 | 82 | def done(self, num): 83 | self.num = num 84 | self.root.quit() 85 | 86 | def finddesktop(): 87 | return os.path.join(os.path.expanduser("~"),'Desktop') 88 | 89 | def findfile_desktop(end=''): 90 | dtop = os.path.join(os.path.expanduser("~"),'Desktop') 91 | r = [] 92 | for i in os.listdir(dtop): 93 | if i.lower().endswith(end.lower()): 94 | r.append(i) 95 | return r 96 | 97 | if __name__ == '__main__': 98 | root = Tk() 99 | gifs = findfile_desktop(end='') 100 | s = SimpleDialog(root,buttons=gifs) 101 | v = s.go() 102 | print(gifs[v]) 103 | root.mainloop() -------------------------------------------------------------------------------- /vrequest/template/v/settings.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Scrapy settings for v project 4 | # 5 | # For simplicity, this file contains only settings considered important or 6 | # commonly used. You can find more settings consulting the documentation: 7 | # 8 | # https://doc.scrapy.org/en/latest/topics/settings.html 9 | # https://doc.scrapy.org/en/latest/topics/downloader-middleware.html 10 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 11 | 12 | BOT_NAME = 'v' 13 | 14 | SPIDER_MODULES = ['v.spiders'] 15 | NEWSPIDER_MODULE = 'v.spiders' 16 | 17 | 18 | # Crawl responsibly by identifying yourself (and your website) on the user-agent 19 | #USER_AGENT = 'v (+http://www.yourdomain.com)' 20 | 21 | # Obey robots.txt rules 22 | ROBOTSTXT_OBEY = False 23 | 24 | # Configure maximum concurrent requests performed by Scrapy (default: 16) 25 | #CONCURRENT_REQUESTS = 32 26 | 27 | # Configure a delay for requests for the same website (default: 0) 28 | # See https://doc.scrapy.org/en/latest/topics/settings.html#download-delay 29 | # See also autothrottle settings and docs 30 | #DOWNLOAD_DELAY = 3 31 | # The download delay setting will honor only one of: 32 | #CONCURRENT_REQUESTS_PER_DOMAIN = 16 33 | #CONCURRENT_REQUESTS_PER_IP = 16 34 | 35 | # Disable cookies (enabled by default) 36 | #COOKIES_ENABLED = False 37 | 38 | # Disable Telnet Console (enabled by default) 39 | TELNETCONSOLE_ENABLED = False # 几乎用不到的功能默认关闭，提高任务执行效率 40 | 41 | # Override the default request headers: 42 | #DEFAULT_REQUEST_HEADERS = { 43 | # 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 44 | # 'Accept-Language': 'en', 45 | #} 46 | 47 | IMAGES_STORE = 'image' # 默认在该脚本路径下创建文件夹、下载图片(不解开 VImagePipeline 管道注释则该配置无效) 48 | FILES_STORE = 'file' # 默认在该脚本路径下创建文件夹、下载文件(不解开 VFilePipeline 管道注释则该配置无效) 49 | MEDIAS_STORE = 'media' # 默认在该脚本路径下创建文件夹、下载【媒体】(不解开 VVideoPipeline 管道注释则该配置无效) 50 | ITEM_PIPELINES = { 51 | # 'v.pipelines.VPipeline': 101, # 普通的中间件使用(解开即可测试，如需魔改，请在脚本顶部找对应的类进行自定义处理) 52 | # 'v.pipelines.VImagePipeline': 102, # 图片下载中间件，item 带有 src 字段则以此作为图片地址下载到 IMAGES_STORE 地址的文件夹内 53 | # 'v.pipelines.VFilePipeline': 103, # 文件下载中间件，item 带有 src 字段则以此作为文件地址下载到 FILES_STORE 地址的文件夹内 54 | # 'v.pipelines.VVideoPipeline': 104, # 视频下载中间件，同上，以 src 作为下载地址，下载到当前路径下的 video 文件夹内 55 | # 'v.pipelines.VMySQLPipeline': 105, # MySql 插入中间件，具体请看类的描述 56 | } 57 | SPIDER_MIDDLEWARES = { 58 | # 'v.middlewares.VSpiderMiddleware': 543, # 原版模板的单脚本插入方式 59 | } 60 | DOWNLOADER_MIDDLEWARES = { 61 | # 'v.middlewares.VDownloaderMiddleware': 543, # 原版模板的单脚本插入方式 62 | # 'v.middlewares.VSeleniumMiddleware': 544, # 单脚本 Selenium 中间件配置，解开自动使用 Selenium，详细请看 VSeleniumMiddleware 类中间件代码。 63 | } 64 | EXTENSIONS = { 65 | # 'scrapy.extensions.logstats.LogStats': None, 66 | # 关闭 scrapy EXTENSIONS默认中间件方式如上，程序执行时，日志的头部有当前任务都有哪些中间件加载，按需在对应管道中配置为 None 即可关闭 67 | # 同理 SPIDER_MIDDLEWARES / DOWNLOADER_MIDDLEWARES 这两个“中间件配置”字典也可以用相同的方式关掉 scrapy 默认组件 68 | # 【*】注意：不同分类的默认中间件需在对应分类的“中间件配置”字典中配置才能关闭， 69 | } 70 | 71 | # Enable and configure the AutoThrottle extension (disabled by default) 72 | # See https://doc.scrapy.org/en/latest/topics/autothrottle.html 73 | #AUTOTHROTTLE_ENABLED = True 74 | # The initial download delay 75 | #AUTOTHROTTLE_START_DELAY = 5 76 | # The maximum download delay to be set in case of high latencies 77 | #AUTOTHROTTLE_MAX_DELAY = 60 78 | # The average number of requests Scrapy should be sending in parallel to 79 | # each remote server 80 | #AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0 81 | # Enable showing throttling stats for every response received: 82 | #AUTOTHROTTLE_DEBUG = False 83 | 84 | # Enable and configure HTTP caching (disabled by default) 85 | # See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings 86 | #HTTPCACHE_ENABLED = True 87 | #HTTPCACHE_EXPIRATION_SECS = 0 88 | #HTTPCACHE_DIR = 'httpcache' 89 | #HTTPCACHE_IGNORE_HTTP_CODES = [] 90 | #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage' 91 | 92 | MEDIA_ALLOW_REDIRECTS = True # 图片下载时默认打开该开关，一定程度上能规避一些图片下载的问题。 -------------------------------------------------------------------------------- /vrequest/pyprime.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | # miller-rabin 算法素性检测 4 | def isprime_mr(a,b=None): 5 | if b is None: b = [2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59,61,67,71,73,79,83,89,97] 6 | if a == 2:return True 7 | if a%2==0 or a==1:return False 8 | for x in b: 9 | t = a - 1 10 | while t%2 == 0: 11 | v = pow(x,t,a)# python自带函数pow就带有快速幂取模功能 12 | if v not in [0,1,a-1]: 13 | return False 14 | else: 15 | if v in [0,a-1]: break 16 | t //= 2 17 | return True 18 | 19 | # Pollcard Rho 质因数分解，比普通质因分解更好的一种算法，能处理更大的数字 20 | def prime_list_rho(n,root=None): 21 | # 之所以不用 def prime_list_rho(n,root=[]): 的方式是因为这里有坑 22 | # 作为函数对象的内部参数的临时地址多次使用会一直引用一个地址， 23 | # 所以考虑到该参数最后会作为结果返回出去，所以需要考虑初始化处理 24 | if root is None: root = [] 25 | if not root and n<2: 26 | raise ValueError(n) 27 | if isprime_mr(n): 28 | root.append(n) 29 | return root 30 | from random import randint 31 | from math import gcd 32 | while True: 33 | x = randint(0,n-1) + 1 34 | c = randint(0,n-1) + 1 35 | y = x 36 | i = j = 2 37 | while True: 38 | x = ((x**2)+c)%n 39 | z = abs(x - y) 40 | d = gcd(z,n) 41 | if d>1 and dpip install vrequest 52 | # 通过git+pip进行安装 53 | C:\Users\zhoulin08>pip install git+https://github.com/cilame/vrequest.git 54 | # 下载安装该库会自动依赖安装 requests, lxml 这两个库 55 | # 其他的依赖不会主动安装，因为与请求功能没有强耦合要求。所以会在那些功能被使用的时候提示没有该库 56 | # 额外的依赖和功能： 57 | # pyexecjs 或 js2py 任选其一，用于测试执行 js 脚本 58 | # jsbeautifier 用于结构化 js 脚本 59 | # cryptography 用于一些比较小众且我很难找得到的算法加解密 60 | 61 | # 另外，如果你只想要使用加解密功能的话，你可以增加 --no-deps 的pip下载配置避免依赖下载，因为加解密部分的功能， 62 | # 只要工具内没有明说存在依赖，就代表那些解密的功能全部都由不需要依赖，都有单个脚本直接实现。 63 | # 甚至二维码的加解密都将依赖库都压缩着包含在了单个脚本内部。 64 | C:\Users\zhoulin08>pip install vrequest --no-deps 65 | 66 | # 如果你想体验全部的功能的话建议这么装，安装好之后直接体验所有功能 67 | C:\Users\zhoulin08>pip install vrequest scrapy js2py jsbeautifier cryptography 68 | ``` 69 | 70 | - ##### 打开方式 71 | 72 | ```bash 73 | # 在安装该函数库后就有一个命令行工具，直接在命令行输入 vv 或者 vrequest 都可以打开该GUI工具 74 | # eg. 75 | C:\Users\zhoulin08>vv 76 | # 并且功能描述都有写在帮助页，直接按照帮助页中的各种快捷键处理方式就可以 77 | # 0 该工具大部分都可以用快捷键操作 78 | # 1 可以将请求过的数据记录在本地，并且可以更新配置 79 | # 哪怕关掉工具也能很方便的恢复请求的配置状态，方便使用 80 | # 2 快速生成请求代码，让请求更加方便实现。 81 | # ps. 82 | C:\Users\zhoulin08>ee 83 | # 直接在命令行输入 ee 直接打开便捷加密窗口，不用再工具里右键打开， 84 | # 或者输入 vv e 也可以打开，这里重复功能是为了防止与其他工具冲突， 85 | ``` 86 | 87 | - ##### 一个简单的请求 88 | 89 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show2.png) 90 | 91 | - ##### 一个简单的列表分析 92 | 93 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show3.png) 94 | 95 | - ##### requests 代码生成 96 | 97 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show4.png) 98 | 99 | - ##### 列表分析代码 100 | 101 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show5.png) 102 | 103 | - ##### 直接执行 requests 的代码 104 | ``` 105 | 下面这里的代码执行和将生成的代码转到 IDE 里面执行是一样的，就不多赘述了。 106 | ``` 107 | 108 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show6.png) 109 | 110 | - ##### 同理直接执行 scrapy 的代码 111 | ``` 112 | 下面这里的的代码执行有两种， 113 | 一种是项目执行，就是简单开个shell： 114 | 在默认环境生成项目，将该脚本拷贝到项目的 spiders 路径下以项目路径执行 scrapy crwal v 的来启动爬虫 115 | 一种执行的方式是单脚本执行： 116 | 单脚本执行可以直接放在 IDE 里面当作是一般 python 脚本一样执行 scrapy 爬虫，比较方便开发 117 | ``` 118 | 119 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show7.png) 120 | 121 | - ##### 单脚本执行scrapy、简单的数据的收集 122 | 123 | ![image](https://raw.githubusercontent.com/cilame/vrequest/master/test/show8.png) -------------------------------------------------------------------------------- /vrequest/pybrainfuck.py: -------------------------------------------------------------------------------- 1 | # brainfuck 和 ook 的核心算法相同，共用一个脚本 2 | 3 | # brainfuck 4 | def evaluate(code:str): 5 | code = cleanup(list(code)) 6 | bmap = buildbmap(code) 7 | cells, ptr, cellptr, ret = [0], 0, 0, [] 8 | while ptr < len(code): 9 | cmd = code[ptr] 10 | if cmd == ">": 11 | cellptr += 1 12 | if cellptr == len(cells): cells.append(0) 13 | if cmd == "<": cellptr = 0 if cellptr <= 0 else cellptr - 1 14 | if cmd == "+": cells[cellptr] = cells[cellptr] + 1 if cells[cellptr] < 255 else 0 15 | if cmd == "-": cells[cellptr] = cells[cellptr] - 1 if cells[cellptr] > 0 else 255 16 | if cmd == "[" and cells[cellptr] == 0: ptr = bmap[ptr] 17 | if cmd == "]" and cells[cellptr] != 0: ptr = bmap[ptr] 18 | if cmd == ",": cells[cellptr] = b'\xff' 19 | if cmd == ".": ret.append(chr(cells[cellptr])) 20 | ptr += 1 21 | return ''.join(ret) 22 | 23 | def cleanup(code): 24 | return ''.join(filter(lambda x: x in '.,[]<>+-', code)) 25 | 26 | def buildbmap(code): 27 | _stack, bmap = [], {} 28 | for pos, cmd in enumerate(code): 29 | if cmd == "[": _stack.append(pos) 30 | if cmd == "]": 31 | start = _stack.pop() 32 | bmap[start] = pos 33 | bmap[pos] = start 34 | return bmap 35 | 36 | import re 37 | # ook! 38 | def parse_ook_to_brainfuckmap(string, abc=('!', '?', '.')): 39 | maps = { 40 | ('!', '?'): '[', 41 | ('?', '!'): ']', 42 | ('.', '.'): '+', 43 | ('!', '!'): '-', 44 | ('.', '?'): '>', 45 | ('?', '.'): '<', 46 | ('!', '.'): '.', 47 | ('.', '!'): ',', 48 | } 49 | a, b, c = [i if i not in r'$()*+.[]?\/^{}' else '\\'+i for i in abc] 50 | rexgep = '|'.join([a, b, c]) 51 | v = re.findall(rexgep, string) 52 | r = [] 53 | for i in zip(v[::2],v[1::2]): 54 | t = [j.replace(a[-1], '!')\ 55 | .replace(b[-1], '?')\ 56 | .replace(c[-1], '.') for j in i] 57 | t = tuple(t) 58 | r.append(maps.get(t)) 59 | return ''.join(r) 60 | 61 | 62 | if __name__ == '__main__': 63 | # 普通的 brainfuck 64 | s = '>+++++++++[<++++++++>-]<.>+++++++[<++++>-]<+.+++++++..+++.>>>++++++++[<++++>-]<.>>>++++++++++[<+++++++++>-]<---.<<<<.+++.------.--------.>>+.' 65 | print(evaluate(s)) 66 | print() 67 | 68 | # ook 加密实际上使用三个种类的字符以长度为2的组合来对应 brainfuck 里面的八个字符， 69 | # 实际上里面的算法还是使用的 brainfuck 的算法 70 | s = 'Ook. Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook? Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook? Ook! Ook! Ook? Ook! Ook? Ook. Ook! Ook. Ook. Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook? Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook? Ook! Ook! Ook? Ook! Ook? Ook. Ook. Ook. Ook! Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook. Ook! Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook. Ook. Ook? Ook. Ook? Ook. Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook? Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook? Ook! Ook! Ook? Ook! Ook? Ook. Ook! Ook. Ook. Ook? Ook. Ook? Ook. Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook? Ook? Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook? Ook! Ook! Ook? Ook! Ook? Ook. Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook. Ook? Ook. Ook? Ook. Ook? Ook. Ook? Ook. Ook! Ook. Ook. Ook. Ook. Ook. Ook. Ook. Ook! Ook. Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook. Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook! Ook. Ook. Ook? Ook. Ook? Ook. Ook. Ook! Ook.' 71 | s = parse_ook_to_brainfuckmap(s) 72 | print(s) 73 | print(evaluate(s)) 74 | s = '''..... ..... ..... ..... !?!!. ?.... ..... ..... ..... .?.?! .?... .!... 75 | ..... ..... !.?.. ..... !?!!. ?!!!! !!?.? !.?!! !!!.. ..... ..... .!.?. 76 | ..... ...!? !!.?. ..... ..?.? !.?.. ..... .!.?. ..... ..... !?!!. ?!!!! 77 | !!!!! !?.?! .?!.? ..... ....! ?!!.? ..... ...?. ?!.?. ..... !.?.. ..... 78 | !?!!. ?!!!! !!?.? !.?!! !!!!! !!!!. ..... ...!. ?.... ...!? !!.?. ..... 79 | ?.?!. ?..!. ?.... ..... !?!!. ?!!!! !!!!? .?!.? !!!!! !!!!! !!!.? ..... 80 | ..!?! !.?.. ....? .?!.? ....! .!!!. !!!!! !!!!! !!!!! !!.?. ..... .!?!! 81 | .?... ...?. ?!.?. ..... !.!!! !!!!! !.?.. ..... ..!?! !.?.. ..... .?.?! 82 | .?... ..... !.?.''' 83 | s = parse_ook_to_brainfuckmap(s) 84 | print(s) 85 | print(evaluate(s)) 86 | print() -------------------------------------------------------------------------------- /vrequest/pyevpkdf.py: -------------------------------------------------------------------------------- 1 | import os 2 | import base64 3 | from hashlib import new 4 | 5 | 6 | 7 | 8 | # cryptojs 的默认加密方式每次加密都不一样，并且都有 U2FsdGVkX1 这个头标志 9 | # 虽然每次加密的数据都不一样，但是解密都是一样的，这是因为他使用了一个随机数，并且这个随机数是放在加密数据里面 10 | # 用这个随机数进行散列算法，算出 key 以及 iv 使用固定的模式进行加密就可以每次加密不同了。 11 | # 将这个随机参数作为 salt 算出 key和iv 后进行加密 12 | # 通过 result = base64('Salted__' + salt + encodedata) 算出真实地加密后地数据结果， 13 | # （另外 EvpKDF 算法是从 cryptojs js代码中逆向出的 python 算法，不知道以后有没有可能成为什么标准。） 14 | 15 | def EvpKDF(hash_name, password, salt, iterations, dklen=None): 16 | dkey = b'' 17 | block = None 18 | hasher = new(hash_name) 19 | while len(dkey) < dklen: 20 | if block: 21 | hasher.update(block) 22 | hasher.update(password) 23 | hasher.update(salt) 24 | block = hasher.digest() 25 | hasher = new(hash_name) 26 | for i in range(1, iterations): 27 | hasher.update(block) 28 | block = hasher.digest() 29 | hasher = new(hash_name) 30 | dkey += block 31 | return dkey[:dklen] 32 | 33 | def make_cryptojs_from_default_params(key, block_size, iv_size, randomsalt=True): 34 | dklen = (block_size+iv_size)//8 35 | blocklen = block_size//8 36 | ivlen = iv_size//8 37 | salt = os.urandom(8) if randomsalt else b'\x00'*8 # 默认使用随即盐,否则使用八位0填充 38 | v = EvpKDF('md5', key, salt, 1, dklen=dklen) # 从 cryptojs 逆向出的默认 iterations 为 1 39 | key, iv = v[:blocklen], v[-ivlen:] 40 | return key, iv, salt 41 | 42 | def parse_cryptojs_from_default_params(key, block_size, iv_size, data): 43 | dklen = (block_size+iv_size)//8 44 | blocklen = block_size//8 45 | ivlen = iv_size//8 46 | salt = data[8:16] # 固定长度 8. 47 | v = EvpKDF('md5', key, salt, 1, dklen=dklen) # 从 cryptojs 逆向出的默认 iterations 为 1 48 | key, iv, salt, data = v[:blocklen], v[-ivlen:], salt, data[16:] # 16位之后是真实的加密数据，抛出去还需要正式的算法解密 49 | return key, iv, salt, data 50 | 51 | 52 | 53 | 54 | 55 | if __name__ == '__main__': 56 | 57 | # 这里的加密模式固定使用 cbc/pkcs7 58 | #（这里仅仅使用了aes示范，如果是des或其他的算法，key和iv的长度多少会有一定的变化） 59 | # 是为了示范怎么使用当前的加解密的参数， 60 | # 如果你需要用下面代码并使用自定义的各种参数，请注意下面四个注释部分的参数修改：算法，模式，block_size，iv_size。 61 | # 如果你掌握js逆向想要从 cryptojs 里面检查加密正确，请注意js里面的 EvpKDF 函数以及传入参数的各种正确性以便调整 62 | # aes/cbc/pkcs7 63 | from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 64 | from cryptography.hazmat.primitives import padding 65 | from cryptography.hazmat.backends import default_backend 66 | def get_encryptor(key, iv=None): 67 | algoer = algorithms.AES(key) #这里的AES算法（若要换成des算法，这里换成TripleDES，该加密库中，DES 事实上等于 TripleDES 的密钥长度为 64bit 时的加解密） 68 | cipher = Cipher(algoer, modes.CBC(iv), backend=default_backend()) #这里的CBC模式 69 | def enc(bitstring): 70 | padder = padding.PKCS7(algoer.block_size).padder() 71 | bitstring = padder.update(bitstring) + padder.finalize() 72 | encryptor = cipher.encryptor() 73 | return encryptor.update(bitstring) + encryptor.finalize() 74 | def dec(bitstring): 75 | decryptor = cipher.decryptor() 76 | ddata = decryptor.update(bitstring) + decryptor.finalize() 77 | unpadder = padding.PKCS7(algoer.block_size).unpadder() 78 | return unpadder.update(ddata) + unpadder.finalize() 79 | class f:pass 80 | f.encrypt = enc 81 | f.decrypt = dec 82 | return f 83 | def cryptojs_default_aes_enc(realkey, realdata): 84 | block_size, iv_size = 256, 128 # aes block_size 256 # iv_size 128（若要换成des，这里改成(64,64)） 85 | key,iv,salt = make_cryptojs_from_default_params(realkey, block_size, iv_size) 86 | encryptor = get_encryptor(key, iv) 87 | return base64.b64encode(b'Salted__' + salt + encryptor.encrypt(realdata)) 88 | def cryptojs_default_aes_dec(realkey, encdata): 89 | block_size, iv_size = 256, 128 # aes block_size 256 # iv_size 128（若要换成des，这里改成(64,64)） 90 | key,iv,salt,_data = parse_cryptojs_from_default_params(realkey, block_size, iv_size, base64.b64decode(encdata)) 91 | encryptor = get_encryptor(key, iv) 92 | return encryptor.decrypt(_data) 93 | 94 | # 加解密混合验证 95 | realkey = b'123' 96 | realdata = b'something' 97 | encdata = cryptojs_default_aes_enc(realkey, realdata) 98 | decdata = cryptojs_default_aes_dec(realkey, encdata) 99 | print(encdata) 100 | print(decdata) 101 | print() 102 | 103 | # 直接使用网站获取的加密结果进行验证 104 | realkey = b'123' 105 | encdata = b'U2FsdGVkX19e0LihgCXF3n6vPFLihWnI3CDRHrKwzAk=' 106 | decdata = cryptojs_default_aes_dec(realkey, encdata) 107 | print(decdata) -------------------------------------------------------------------------------- /vrequest/pyplus.py: -------------------------------------------------------------------------------- 1 | import html 2 | import urllib.parse 3 | 4 | import re 5 | 6 | 7 | def bitstring_to_base_8(bitstring, encoding='utf-8'): 8 | return ''.join([oct(i).replace('0o','\\o') for i in bitstring]) 9 | 10 | def base_8_to_bitstring(string, encoding='utf-8'): 11 | func = lambda i: int(i, 8) 12 | return bytes(list(map(func, re.findall('[0-7]{1,3}',string)))).decode(encoding) 13 | 14 | def bitstring_to_base_10(bitstring, encoding='utf-8'): 15 | return ''.join(['\\'+str(i) for i in bitstring]) 16 | 17 | def base_10_to_bitstring(string, encoding='utf-8'): 18 | return bytes(list(map(int, re.findall('[0-9]{1,3}',string)))).decode(encoding) 19 | 20 | def bitstring_to_base_16(bitstring, encoding='utf-8'): 21 | return ''.join([hex(i).replace('0x','\\x') for i in bitstring]) 22 | 23 | def base_16_to_bitstring(string, encoding='utf-8'): 24 | func = lambda i: int(i, 16) 25 | return bytes(list(map(func, re.findall('[0-9a-fA-F]{2}',string)))).decode(encoding) 26 | 27 | def quote_to_bitstring(string, encoding='utf-8'): 28 | return urllib.parse.quote(string.decode(encoding), encoding=encoding) 29 | 30 | def bitstring_to_quote(bitstring, encoding='utf-8'): 31 | return urllib.parse.unquote(bitstring, encoding=encoding) 32 | 33 | def urlquote_to_bitstring(string, encoding='utf-8'): 34 | def quote_val(url): 35 | for i in re.findall(r'[\?&][^=&]*=([^&]*)',url): 36 | url = url.replace(i,'{}'.format(urllib.parse.quote_plus(i, encoding=encoding))) 37 | return url 38 | return quote_val(string.decode(encoding)) 39 | 40 | def bitstring_to_urlquote(bitstring, encoding='utf-8'): 41 | def unquote_val(url): 42 | for i in re.findall(r'[\?&][^=&]*=([^&]*)',url): 43 | url = url.replace(i,'{}'.format(urllib.parse.unquote_plus(i, encoding=encoding))) 44 | return url 45 | return unquote_val(bitstring) 46 | 47 | def escape_to_bitstring(string, encoding='utf-8'): 48 | return html.escape(string.decode(encoding)) 49 | 50 | def bitstring_to_escape(bitstring, encoding='utf-8'): 51 | return html.unescape(bitstring) 52 | 53 | def bitstring_to_unicode(string, encoding='utf-8'): 54 | return string.encode(encoding).decode('unicode_escape') 55 | 56 | def unicode_to_bitstring(bitstring, encoding='utf-8'): 57 | return bitstring.decode(encoding).encode('unicode_escape').decode(encoding) 58 | 59 | 60 | def bitstring_to_binary(string, encoding='utf-8'): 61 | return bytes(list(map(lambda i:int(i, 2), re.findall('[0-1]{1,8}',string)))).decode(encoding) 62 | 63 | def binary_to_bitstring(bitstring, encoding='utf-8'): 64 | r = [] 65 | for i in bitstring: 66 | r.append('{:>08}'.format(bin(i)[2:])) 67 | return ' '.join(r) 68 | 69 | 70 | html_quote = { 71 | 'base_2': (binary_to_bitstring, bitstring_to_binary), 72 | 'base_8': (bitstring_to_base_8, base_8_to_bitstring), 73 | 'base_10': (bitstring_to_base_10, base_10_to_bitstring), 74 | 'base_16': (bitstring_to_base_16, base_16_to_bitstring), 75 | 'escape': (escape_to_bitstring, bitstring_to_escape), 76 | 'quote': (quote_to_bitstring, bitstring_to_quote), 77 | 'urlquote': (urlquote_to_bitstring, bitstring_to_urlquote), 78 | 'unicode': (unicode_to_bitstring, bitstring_to_unicode), 79 | } 80 | 81 | if __name__ == '__main__': 82 | try: 83 | # 处理 sublime 执行时输出乱码 84 | import io 85 | import sys 86 | sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8') 87 | sys.stdout._CHUNK_SIZE = 1 88 | except: 89 | pass 90 | 91 | 92 | k = b'123123123asdf891(*&^&%^' 93 | v = bitstring_to_base_8(k) 94 | print(v) 95 | v = base_8_to_bitstring(v) 96 | print(v) 97 | print() 98 | 99 | v = bitstring_to_base_10(k) 100 | print(v) 101 | v = base_10_to_bitstring(v) 102 | print(v) 103 | print() 104 | 105 | v = bitstring_to_base_16(k) 106 | print(v) 107 | v = base_16_to_bitstring(v) 108 | print(v) 109 | print() 110 | 111 | v = html.escape(k.decode()) 112 | print(v) 113 | v = html.unescape(v) 114 | print(v) 115 | print() 116 | 117 | v = urllib.parse.quote(k.decode()) 118 | print(v) 119 | v = urllib.parse.unquote(v) 120 | print(v) 121 | 122 | url = 'https://www.baidu.com/s?ie=UTF-8&wd=%E7%99%BE%E5%BA%A6' 123 | v = bitstring_to_urlquote(url) 124 | print(v) 125 | v = urlquote_to_bitstring(v.encode()) 126 | print(v) 127 | 128 | v = base_16_to_bitstring(r'\xe6\x9c\x89\xe7\x82\xb9\xe5\x8f\xaf\xe7\x88\xb1') 129 | print(v) 130 | v = base_16_to_bitstring(r'e69c89e782b9e58fafe788b1') 131 | print(v) 132 | 133 | v = '你好'.encode() 134 | v = unicode_to_bitstring(v) 135 | print(v) 136 | v = bitstring_to_unicode(v) 137 | print(v) 138 | 139 | v = '你好'.encode() 140 | v = binary_to_bitstring(v) 141 | print(v) 142 | v = bitstring_to_binary(v) 143 | print(v) 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /vrequest/py_vthread.py: -------------------------------------------------------------------------------- 1 | class pool: 2 | import time,queue,traceback,builtins,functools 3 | from threading import Thread,RLock,current_thread,main_thread 4 | orig_func = {} 5 | _org_print = print 6 | lock = RLock() 7 | class KillThreadParams(Exception): pass 8 | _monitor = None 9 | _monitor_run_num = {} 10 | _pool_queue = {} 11 | _pool_func_num = {} 12 | def __init__(self,pool_num=None,gqueue='v',monitor=True): 13 | if gqueue not in self._pool_queue: 14 | self._pool_queue[gqueue] = pool.queue.Queue() 15 | self._pool = self._pool_queue[gqueue] 16 | pool._patch_print() 17 | if monitor: self.main_monitor() 18 | if gqueue not in self._monitor_run_num: 19 | self._monitor_run_num[gqueue] = pool.queue.Queue() 20 | num = self._auto_pool_num(pool_num) 21 | if gqueue not in self._pool_func_num: 22 | self._pool_func_num[gqueue] = num 23 | self._run(num,gqueue) 24 | else: 25 | if pool_num is not None: 26 | self.change_thread_num(num,gqueue) 27 | def __call__(self,func): 28 | pool.orig_func[func.__name__] = func 29 | @pool.functools.wraps(func) 30 | def _run_threads(*args,**kw): self._pool.put((func,args,kw)) 31 | return _run_threads 32 | @classmethod 33 | def change_thread_num(self,num,gqueue='v'): 34 | if gqueue in self._pool_func_num: 35 | x = self._pool_func_num[gqueue] - num 36 | if x < 0: self._run(abs(x),gqueue) 37 | if x > 0: [self._pool_queue[gqueue].put(self.KillThreadParams) for _ in range(abs(x))] 38 | self._pool_func_num[gqueue] = num 39 | @classmethod 40 | def main_monitor(self): 41 | def _func(): 42 | while True: 43 | pool.time.sleep(.25) 44 | if not pool.main_thread().isAlive() and all(map(lambda i:i.empty(),self._monitor_run_num.values())): 45 | self.close_all() 46 | break 47 | if not self._monitor: 48 | self._monitor = pool.Thread(target=_func,name="MainMonitor") 49 | self._monitor.start() 50 | @classmethod 51 | def close_by_gqueue(self,gqueue='v'): self.change_thread_num(0,gqueue) 52 | @classmethod 53 | def close_all(self): 54 | for i in self._pool_func_num: self.change_thread_num(0,i) 55 | @classmethod 56 | def wait(self, gqueue='v'): 57 | while self.check_stop(gqueue): pool.time.sleep(.25) 58 | @classmethod 59 | def check_stop(self, gqueue='v'): 60 | return self._monitor_run_num[gqueue].qsize() or self._pool_queue[gqueue].qsize() 61 | @staticmethod 62 | def atom(func): 63 | def _atom(*arg,**kw): 64 | with pool.lock: return func(*arg,**kw) 65 | return _atom 66 | @staticmethod 67 | def _patch_print(): pool.builtins.print = pool._new_print 68 | @staticmethod 69 | def _new_print(*arg,**kw): 70 | with pool.lock: pool._org_print("[{}]".format(pool.current_thread().getName().center(13)),*arg,**kw) 71 | @staticmethod 72 | def _auto_pool_num(num): 73 | if not num: 74 | try: 75 | from multiprocessing import cpu_count 76 | num = cpu_count() 77 | except: 78 | print("cpu_count error. use default num 4.") 79 | num = 4 80 | return num 81 | @classmethod 82 | def _run(self,num,gqueue): 83 | def _pools_pull(): 84 | ct = pool.current_thread() 85 | ct.setName("{}_{}".format(ct.getName(), gqueue)) 86 | while True: 87 | v = self._pool_queue[gqueue].get() 88 | if v == self.KillThreadParams: return 89 | try: 90 | func,args,kw = v 91 | self._monitor_run_num[gqueue].put('V') 92 | func(*args,**kw) 93 | except BaseException as e: 94 | print(pool.traceback.format_exc()) 95 | finally: 96 | self._monitor_run_num[gqueue].get('V') 97 | for _ in range(num): pool.Thread(target=_pools_pull).start() 98 | if __name__ == '__main__': 99 | # 简化代码版本的 vthread 库（分组线程池装饰器），一行代码即可实现线程池操作 100 | # 以下为使用/测试 “装饰器线程池” 的代码，你可以在正式使用前熟悉一下使用方法 101 | import time, random 102 | # 被 pool 装饰器装饰的函数，正常执行会变成任务提交的功能， 103 | # 会将函数执行的任务提交给线程池进行执行，所以任务提交并不会卡住程序 104 | # 所以需要多线程操作的函数不要写 return 语句，以及尽量使用主线程中的 list 或 queue 来收集执行结果 105 | @pool(10) # 开启线程池组，默认名字为 'v'，线程数量为10 106 | def func1(a,b): 107 | rd = random.random(); time.sleep(rd) 108 | print(a+b, '{:.3f}'.format(rd)) 109 | @pool(3,gqueue='h') # 开启线程池组，指定名字为 'h'，线程数量为3 110 | def func2(a,b,c): 111 | rd = random.random(); time.sleep(rd) 112 | print(a*b*c, 'hhhhhhh', '{:.3f}'.format(rd)) 113 | for i in range(30): func1(i,i*i) # 随便丢30个任务查看多任务多线程池执行效果 114 | for i in range(30): func2(i,i+i,3) # 随便丢30个任务查看多任务多线程池执行效果 115 | print('start wait.') 116 | pool.wait() # 等待函数 func1 任务在默认的 gqueue='v' 的“线程池组”里面全部执行完 117 | pool.wait(gqueue='h') # 等待函数 func2 在 gqueue='h' 的“线程池组”里面全部执行完 118 | print('end wait.') 119 | # 另外 print 函数自动变成输入带有线程名字前缀的、带有锁的函数 -------------------------------------------------------------------------------- /vrequest/main.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | from . import __version__ 5 | 6 | from .root import ( 7 | root, 8 | config, 9 | change_siz, 10 | tails, 11 | ) 12 | from .menu import bind_menu 13 | from .tab import ( 14 | nb, 15 | nb_names, 16 | bind_frame, 17 | delete_curr_tab, 18 | cancel_delete, 19 | create_new_reqtab, 20 | create_new_rsptab, 21 | create_helper, 22 | change_tab_name, 23 | send_request, 24 | save_config, 25 | switch_response_log, 26 | create_test_code, 27 | create_scrapy_code, 28 | get_html_pure_text, 29 | get_xpath_elements, 30 | get_auto_xpath, 31 | get_auto_json, 32 | choice_auto_json, 33 | execute_code, 34 | execute_scrapy_code, 35 | create_js_parse, 36 | create_selenium_parse, 37 | create_temp_idle, 38 | create_cmd_idle, 39 | create_encoder, 40 | create_test_code_urllib, 41 | pyset_pypi_gui, 42 | ) 43 | from .combinekey import ( 44 | bind_ctl_key, 45 | bind_alt_key, 46 | ) 47 | 48 | # 这里的框架就是目前需要设计处理的图形内容 49 | from .frame import ( 50 | helper_window, 51 | request_window, 52 | ) 53 | 54 | 55 | # === 初始化 === 56 | settings = config['set'] 57 | if not settings: 58 | create_helper() 59 | else: 60 | for key,setting in settings.items(): 61 | if setting.get('type') == 'request': 62 | tab_id = bind_frame(request_window(setting),key) 63 | if key == config['focus']: 64 | nb.select(tab_id) # 保持最后执行成功时的 tab 焦点 65 | 66 | 67 | # === 创建/删除/帮助 === 68 | # 绑定右键菜单 69 | bind_menu(create_new_reqtab, '创建请求标签 [Ctrl+q]') 70 | bind_menu(delete_curr_tab, '删除当前标签 [Ctrl+w]') 71 | bind_menu(change_tab_name, '改当前标签名 [Ctrl+e]') 72 | bind_menu(save_config, '保存配置快照 [Ctrl+s]') 73 | bind_menu(create_js_parse, '创建 js解析页 [Ctrl+j]') 74 | bind_menu(create_helper, '帮助文档标签 [Ctrl+h]') 75 | bind_menu(create_selenium_parse, '浏览器执行窗 [Alt+w]*') 76 | bind_menu(create_encoder, '创建便捷加密编码窗口') 77 | bind_menu(pyset_pypi_gui, '设置全局 pypi 下载源') 78 | 79 | # 绑定 Ctrl + key 的组合键 80 | bind_ctl_key(create_new_reqtab, 'q') 81 | bind_ctl_key(delete_curr_tab, 'w') 82 | # 撤销 ctrl + shift + w （必须是保存过的配置，并且撤销队列在程序关闭后就清空） 83 | bind_ctl_key(cancel_delete, 'w',shift=True) 84 | bind_ctl_key(change_tab_name, 'e') 85 | bind_ctl_key(save_config, 's') 86 | bind_ctl_key(send_request, 'r') 87 | bind_ctl_key(create_helper, 'h') 88 | bind_ctl_key(create_js_parse, 'j') 89 | bind_ctl_key(create_cmd_idle, '`') 90 | 91 | def _scrapy_or_selenium(): 92 | _select = nb.select() 93 | cname = nb_names.get(_select)['name'] 94 | ctype = (nb_names.get(_select).get('setting') or {}).get('type') 95 | # 如果当前窗口是 scrapy 代码窗口则代表直接项目执行 scrapy 代码，否则创建 selenium 窗口。 96 | create_selenium_parse() if ctype != 'scrapy' else execute_scrapy_code() 97 | 98 | # 绑定 response 事件 99 | bind_alt_key(create_new_rsptab, 'r') 100 | bind_alt_key(create_test_code, 'c') # 生成代码 101 | bind_alt_key(get_html_pure_text, 'd') # 获取文本 102 | bind_alt_key(get_xpath_elements, 'x') # 获取xpath 103 | bind_alt_key(get_auto_xpath, 'f') # 解析路径xpath 104 | bind_alt_key(get_auto_json, 'z') # 分析json列表 105 | bind_alt_key(choice_auto_json, 'q') # 选则json列表 106 | bind_alt_key(execute_code, 'v') # 代码执行 107 | bind_alt_key(create_scrapy_code, 's') # 生成scrapy代码 108 | bind_alt_key(_scrapy_or_selenium, 'w') # 用自动生成的环境执行scrapy代码 109 | bind_alt_key(create_temp_idle, '`') # 使用临时的idle文本 110 | bind_alt_key(create_test_code_urllib, 'u') # 生成 urllib(py3) 请求的代碼 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | def algo(): 122 | from .frame import encode_window 123 | fr = encode_window() 124 | ico = os.path.join(os.path.split(__file__)[0],'ico.ico') 125 | fr.iconbitmap(ico) 126 | fr.title('命令行输入 ee 则可快速打开便捷加密窗口(为防冲突，输入vv e也可以打开), 组合快捷键 Alt+` 快速打开IDLE') 127 | fr.bind('',lambda *a:fr.master.quit()) 128 | fr.bind('',lambda *a:create_temp_idle()) 129 | fr.bind('',lambda *a:create_cmd_idle()) 130 | fr.protocol("WM_DELETE_WINDOW",lambda *a:fr.master.quit()) 131 | fr.master.withdraw() 132 | fr.mainloop() 133 | 134 | 135 | escodegen = None 136 | def execute(): 137 | argv = sys.argv 138 | if 'e' in argv: 139 | algo() 140 | return 141 | 142 | def preimport(): 143 | global escodegen 144 | import time 145 | time.sleep(.5) # 需要花点时间导包的部分，用别的线程进行预加载，增加工具顺滑度 146 | try: import js2py 147 | except: pass 148 | try: import execjs 149 | except: pass 150 | try: 151 | import js2py.py_node_modules.escodegen as escodegen 152 | except: pass 153 | import threading 154 | threading.Thread(target=preimport).start() 155 | 156 | root.title('vrequest [{}]'.format(__version__)) 157 | ico = os.path.join(os.path.split(__file__)[0],'ico.ico') 158 | root.iconbitmap(ico) 159 | root.geometry(config.get('siz') or '600x725+100+100') 160 | root.bind('',lambda e:change_siz()) 161 | root.bind('',lambda e:switch_response_log()) 162 | 163 | def quit_(): 164 | try: 165 | for tail in tails: 166 | try: 167 | tail() 168 | except: 169 | import traceback 170 | print(traceback.format_exc()) 171 | finally: 172 | root.destroy() 173 | 174 | root.protocol("WM_DELETE_WINDOW",lambda *a: quit_()) 175 | root.mainloop() 176 | 177 | if __name__ == '__main__': 178 | execute() -------------------------------------------------------------------------------- /vrequest/pymultialgo.py: -------------------------------------------------------------------------------- 1 | from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 2 | from cryptography.hazmat.primitives import padding 3 | from cryptography.hazmat.backends import default_backend 4 | 5 | # [*] AES len(key) [128, 192, 256, 512] len(iv) 128 6 | # [*] Camellia len(key) [128, 192, 256] len(iv) 128 7 | # [*] SEED len(key) [128] len(iv) 128 8 | # ChaCha20 len(key) [256] len(iv) 128 (nonce) 9 | # [*] Blowfish len(key) range(32, 449, 8) len(iv) 64 10 | # [*] CAST5 len(key) range(40, 129, 8) len(iv) 64 11 | # [*] IDEA len(key) [128] len(iv) 64 12 | # [*] TripleDES len(key) [64, 128, 192] len(iv) 64 13 | # [*] DES len(key) [64, 128, 192] len(iv) 64 14 | # ARC4 len(key) [40, 56, 64, 80, 128, 160, 192, 256] # 不使用iv 15 | # 带有 [*] 的可以有不同的加密模式 16 | 17 | algos = dict( 18 | AES = algorithms.AES, 19 | ARC4 = algorithms.ARC4, 20 | Blowfish = algorithms.Blowfish, 21 | CAST5 = algorithms.CAST5, 22 | Camellia = algorithms.Camellia, 23 | ChaCha20 = algorithms.ChaCha20, 24 | IDEA = algorithms.IDEA, 25 | SEED = algorithms.SEED, 26 | TripleDES = algorithms.TripleDES, 27 | DES = algorithms.TripleDES, # 注意，DES 事实上等于 TripleDES 的密钥长度为 64bit 时的加解密 28 | ) 29 | 30 | def get_encryptor(algoname, key, iv=None, mode='ecb', padd='pkcs7'): 31 | backend = default_backend() 32 | if algoname == 'ChaCha20': 33 | algoer = algorithms.ChaCha20(key, iv) 34 | cipher = Cipher(algoer, None, backend=backend) 35 | elif algoname == 'ARC4': 36 | algoer = algorithms.ARC4(key) 37 | cipher = Cipher(algoer, None, backend=backend) 38 | else: 39 | algoer = algos[algoname](key) 40 | if mode == 'ecb': mode = modes.ECB() 41 | if mode == 'cfb': mode = modes.CFB(iv) 42 | if mode == 'ofb': mode = modes.OFB(iv) 43 | if mode == 'cbc': mode = modes.CBC(iv) 44 | if mode == 'ctr': mode = modes.CTR(iv) 45 | cipher = Cipher(algoer, mode, backend=backend) 46 | 47 | def enc(bitstring): 48 | if algoname not in ['ARC4', 'ChaCha20']: 49 | if padd.upper() == 'PKCS7': 50 | padder = padding.PKCS7(algoer.block_size).padder() 51 | bitstring = padder.update(bitstring) + padder.finalize() 52 | elif padd.upper() == 'ANSIX923': 53 | padder = padding.ANSIX923(algoer.block_size).padder() 54 | bitstring = padder.update(bitstring) + padder.finalize() 55 | encryptor = cipher.encryptor() 56 | return encryptor.update(bitstring) + encryptor.finalize() 57 | 58 | def dec(bitstring): 59 | decryptor = cipher.decryptor() 60 | ddata = decryptor.update(bitstring) + decryptor.finalize() 61 | if algoname not in ['ARC4', 'ChaCha20']: 62 | if padd.upper() == 'PKCS7': 63 | unpadder = padding.PKCS7(algoer.block_size).unpadder() 64 | ddata = unpadder.update(ddata) + unpadder.finalize() 65 | elif padd.upper() == 'ANSIX923': 66 | unpadder = padding.ANSIX923(algoer.block_size).unpadder() 67 | ddata = unpadder.update(ddata) + unpadder.finalize() 68 | return ddata 69 | class f:pass 70 | f.encrypt = enc 71 | f.decrypt = dec 72 | return f 73 | 74 | 75 | if __name__ == '__main__': 76 | key = b'1234567812345678' # 密码 77 | iv = b'1234567812345678' # 某些加密模式需要的参数，（ARC4，ChaCha20 加密或者所有的 ecb 模式下，该参数无效！！） 78 | algoname = 'DES' # 加密名字 79 | mode = 'ecb' # ARC4，ChaCha20 模式下该参数无效 80 | padd = 'pkcs7' # 默认使用该 padding 方式。 81 | encryptor = get_encryptor(algoname, key, iv, mode, padd) 82 | 83 | # key, iv 的长度请阅读头部注释注意 84 | bitstring = b'1234567812345678' 85 | v = encryptor.encrypt(bitstring) 86 | print(v) 87 | import base64; print(base64.b64encode(v)) 88 | v = encryptor.decrypt(v) 89 | print(v) 90 | 91 | 92 | 93 | 94 | # 这里是常用模式 aes/cbc/pkcs7 模式下的 get_encryptor 简化代码 95 | # 上面的代码是考虑完全的通用性以及在工具内部被使用的必要多封装，这里主要是为了更加方便使用的简写 96 | #import base64 97 | #from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes 98 | #from cryptography.hazmat.primitives import padding 99 | #from cryptography.hazmat.backends import default_backend 100 | #def get_encryptor(key, iv=None): 101 | # algoer = algorithms.AES(key) #若要使用DES这里改成TripleDES 102 | # mode = modes.CBC(iv) #模式若是ecb则为 modes.ECB(), 其余模式均为一个参数 mode.***(iv) 103 | # cipher = Cipher(algoer, mode, backend=default_backend()) 104 | # def enc(bitstring): 105 | # padder = padding.PKCS7(algoer.block_size).padder() 106 | # bitstring = padder.update(bitstring) + padder.finalize() 107 | # encryptor = cipher.encryptor() 108 | # return encryptor.update(bitstring) + encryptor.finalize() 109 | # def dec(bitstring): 110 | # decryptor = cipher.decryptor() 111 | # ddata = decryptor.update(bitstring) + decryptor.finalize() 112 | # unpadder = padding.PKCS7(algoer.block_size).unpadder() 113 | # return unpadder.update(ddata) + unpadder.finalize() 114 | # class f:pass 115 | # f.encrypt = enc 116 | # f.decrypt = dec 117 | # return f 118 | #if __name__ == '__main__': 119 | # key = b'1234567890123456' #密码 120 | # iv = b'1234567890123456' #某些加密模式需要的参数，（ARC4，ChaCha20 加密或者所有的 ecb 模式下，该参数无效！！） 121 | # data = '2LWYSMdnDJSym1TSN54uesXryeud7lOPCtlpWV16dAw='.encode() 122 | # db64 = base64.b64decode(data) 123 | # encryptor = get_encryptor(key, iv) 124 | # v = encryptor.decrypt(db64) 125 | # print(v) -------------------------------------------------------------------------------- /vrequest/pymd2.py: -------------------------------------------------------------------------------- 1 | # 2 | # The MD2 hash function. It is described in RFC 1319. 3 | # 4 | # Copyright (c) 2018 Project Nayuki. (MIT License) 5 | # https://www.nayuki.io/page/cryptographic-primitives-in-plain-python 6 | # 7 | # Permission is hereby granted, free of charge, to any person obtaining a copy of 8 | # this software and associated documentation files (the "Software"), to deal in 9 | # the Software without restriction, including without limitation the rights to 10 | # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 11 | # the Software, and to permit persons to whom the Software is furnished to do so, 12 | # subject to the following conditions: 13 | # - The above copyright notice and this permission notice shall be included in 14 | # all copies or substantial portions of the Software. 15 | # - The Software is provided "as is", without warranty of any kind, express or 16 | # implied, including but not limited to the warranties of merchantability, 17 | # fitness for a particular purpose and noninfringement. In no event shall the 18 | # authors or copyright holders be liable for any claim, damages or other 19 | # liability, whether in an action of contract, tort or otherwise, arising from, 20 | # out of or in connection with the Software or the use or other dealings in the 21 | # Software. 22 | # 23 | 24 | # ---- Public functions ---- 25 | 26 | # Computes the hash of the given bytelist message, returning a new 16-element bytelist. 27 | def _hash(message, printdebug=False): 28 | # Make a shallow copy of the list to prevent modifying the caller's list object 29 | assert isinstance(message, list) 30 | msg = list(message) 31 | if printdebug: print("md2.hash(message = {} bytes)".format(len(message))) 32 | 33 | # Append the termination padding 34 | padlen = _BLOCK_SIZE - (len(msg) % _BLOCK_SIZE) 35 | msg.extend([padlen] * padlen) 36 | 37 | # Initialize the hash state 38 | state = tuple([0] * 48) 39 | checksum = tuple([0] * 16) 40 | 41 | # Compress each block in the augmented message 42 | assert len(msg) % _BLOCK_SIZE == 0 43 | for i in range(len(msg) // _BLOCK_SIZE): 44 | block = tuple(msg[i * _BLOCK_SIZE : (i + 1) * _BLOCK_SIZE]) 45 | state, checksum = _compress(block, state, checksum, printdebug) 46 | 47 | # Compress the checksum as the final block 48 | state, checksum = _compress(checksum, state, checksum, printdebug) 49 | 50 | # Return a prefix of the final state as a bytelist 51 | return list(state[ : 16]) 52 | 53 | 54 | # ---- Private functions ---- 55 | 56 | def _compress(block, state, checksum, printdebug): 57 | # Check argument types and lengths 58 | assert isinstance(block, tuple) and len(block) == _BLOCK_SIZE 59 | assert isinstance(state, tuple) and len(state) == 48 60 | assert isinstance(checksum, tuple) and len(checksum) == 16 61 | 62 | # Copy the block into the state 63 | newstate = list(state) 64 | for i in range(16): 65 | b = block[i] 66 | assert 0 <= b <= 0xFF 67 | newstate[i + 16] = b 68 | newstate[i + 32] = b ^ newstate[i] 69 | 70 | # Perform 18 rounds of hashing 71 | t = 0 72 | for i in range(18): 73 | for j in range(len(newstate)): 74 | newstate[j] ^= _SBOX[t] 75 | t = newstate[j] 76 | t = (t + i) & 0xFF 77 | 78 | # Checksum the block 79 | newchecksum = list(checksum) 80 | l = newchecksum[-1] 81 | for i in range(16): 82 | l = newchecksum[i] ^ _SBOX[block[i] ^ l] 83 | newchecksum[i] = l 84 | 85 | # Return new state and checksum as a tuples 86 | return (tuple(newstate), tuple(newchecksum)) 87 | 88 | 89 | # ---- Numerical constants/tables ---- 90 | 91 | _BLOCK_SIZE = 16 # In bytes 92 | 93 | _SBOX = [ # A permutation of the 256 byte values, from 0x00 to 0xFF 94 | 0x29, 0x2E, 0x43, 0xC9, 0xA2, 0xD8, 0x7C, 0x01, 0x3D, 0x36, 0x54, 0xA1, 0xEC, 0xF0, 0x06, 0x13, 95 | 0x62, 0xA7, 0x05, 0xF3, 0xC0, 0xC7, 0x73, 0x8C, 0x98, 0x93, 0x2B, 0xD9, 0xBC, 0x4C, 0x82, 0xCA, 96 | 0x1E, 0x9B, 0x57, 0x3C, 0xFD, 0xD4, 0xE0, 0x16, 0x67, 0x42, 0x6F, 0x18, 0x8A, 0x17, 0xE5, 0x12, 97 | 0xBE, 0x4E, 0xC4, 0xD6, 0xDA, 0x9E, 0xDE, 0x49, 0xA0, 0xFB, 0xF5, 0x8E, 0xBB, 0x2F, 0xEE, 0x7A, 98 | 0xA9, 0x68, 0x79, 0x91, 0x15, 0xB2, 0x07, 0x3F, 0x94, 0xC2, 0x10, 0x89, 0x0B, 0x22, 0x5F, 0x21, 99 | 0x80, 0x7F, 0x5D, 0x9A, 0x5A, 0x90, 0x32, 0x27, 0x35, 0x3E, 0xCC, 0xE7, 0xBF, 0xF7, 0x97, 0x03, 100 | 0xFF, 0x19, 0x30, 0xB3, 0x48, 0xA5, 0xB5, 0xD1, 0xD7, 0x5E, 0x92, 0x2A, 0xAC, 0x56, 0xAA, 0xC6, 101 | 0x4F, 0xB8, 0x38, 0xD2, 0x96, 0xA4, 0x7D, 0xB6, 0x76, 0xFC, 0x6B, 0xE2, 0x9C, 0x74, 0x04, 0xF1, 102 | 0x45, 0x9D, 0x70, 0x59, 0x64, 0x71, 0x87, 0x20, 0x86, 0x5B, 0xCF, 0x65, 0xE6, 0x2D, 0xA8, 0x02, 103 | 0x1B, 0x60, 0x25, 0xAD, 0xAE, 0xB0, 0xB9, 0xF6, 0x1C, 0x46, 0x61, 0x69, 0x34, 0x40, 0x7E, 0x0F, 104 | 0x55, 0x47, 0xA3, 0x23, 0xDD, 0x51, 0xAF, 0x3A, 0xC3, 0x5C, 0xF9, 0xCE, 0xBA, 0xC5, 0xEA, 0x26, 105 | 0x2C, 0x53, 0x0D, 0x6E, 0x85, 0x28, 0x84, 0x09, 0xD3, 0xDF, 0xCD, 0xF4, 0x41, 0x81, 0x4D, 0x52, 106 | 0x6A, 0xDC, 0x37, 0xC8, 0x6C, 0xC1, 0xAB, 0xFA, 0x24, 0xE1, 0x7B, 0x08, 0x0C, 0xBD, 0xB1, 0x4A, 107 | 0x78, 0x88, 0x95, 0x8B, 0xE3, 0x63, 0xE8, 0x6D, 0xE9, 0xCB, 0xD5, 0xFE, 0x3B, 0x00, 0x1D, 0x39, 108 | 0xF2, 0xEF, 0xB7, 0x0E, 0x66, 0x58, 0xD0, 0xE4, 0xA6, 0x77, 0x72, 0xF8, 0xEB, 0x75, 0x4B, 0x0A, 109 | 0x31, 0x44, 0x50, 0xB4, 0x8F, 0xED, 0x1F, 0x1A, 0xDB, 0x99, 0x8D, 0x33, 0x9F, 0x11, 0x83, 0x14, 110 | ] 111 | 112 | def md2(message:bytes): 113 | s = _hash(list(message)) 114 | v = 0 115 | for idx,i in enumerate(s[::-1]): 116 | v += i << idx*8 117 | return hex(v)[2:] 118 | 119 | if __name__ == '__main__': 120 | message = b'111111231231111' 121 | print(md2(message)) -------------------------------------------------------------------------------- /vrequest/ast_hook_inject.js: -------------------------------------------------------------------------------- 1 | // cnpm install @babel/core @babel/types @babel/generator -g 2 | const fs = require("fs") 3 | const babel = require("@babel/core"); 4 | const types = require("@babel/types"); 5 | const generator = require("@babel/generator"); 6 | const hookFunctionName = "cc11"; 7 | 8 | function injectHook(jsCode) { 9 | const ast = babel.parse(jsCode); 10 | babel.traverse(ast, { 11 | // 变量声明 12 | VariableDeclaration(path) { 13 | const node = path.node; 14 | if (!(node.declarations && node.declarations.length)) { 15 | return; 16 | } 17 | for (let variableDeclarator of node.declarations) { 18 | if (!variableDeclarator.init) { 19 | continue; 20 | } 21 | if (types.isFunctionExpression(variableDeclarator.init)) { 22 | continue; 23 | } 24 | let varName = ""; 25 | if (types.isIdentifier(variableDeclarator.id) || types.isMemberExpression(variableDeclarator.id)) { 26 | varName = generator.default(variableDeclarator.id).code; 27 | } 28 | try { 29 | const hookFunctionArguments = [ 30 | types.stringLiteral(varName), 31 | variableDeclarator.init, 32 | types.stringLiteral("var-init") 33 | ]; 34 | variableDeclarator.init = types.callExpression(types.identifier(hookFunctionName), hookFunctionArguments) 35 | } catch (e) { 36 | console.error(e); 37 | } 38 | } 39 | }, 40 | AssignmentExpression(path) { 41 | const node = path.node; 42 | if (types.isFunctionExpression(node)) { 43 | return; 44 | } 45 | let varName = ""; 46 | if (types.isIdentifier(node.left) || types.isMemberExpression(node.left)) { 47 | varName = generator.default(node.left).code; 48 | } 49 | try { 50 | const hookFunctionArguments = [ 51 | types.stringLiteral(varName), 52 | node.right, 53 | types.stringLiteral("assign") 54 | ]; 55 | node.right = types.callExpression(types.identifier(hookFunctionName), hookFunctionArguments) 56 | } catch (e) { 57 | console.error(e); 58 | } 59 | }, 60 | // 对象表达式 61 | ObjectExpression(path) { 62 | const node = path.node; 63 | if (!(node.properties && node.properties.length)) { 64 | return; 65 | } 66 | for (let objectProperty of node.properties) { 67 | const propertyValue = objectProperty.value; 68 | if (types.isFunctionExpression(propertyValue)) { 69 | continue; 70 | } 71 | if (types.isObjectExpression(propertyValue)) { 72 | continue; 73 | } 74 | if (!propertyValue) { 75 | return; 76 | } 77 | let objectKey = objectProperty.key; 78 | if (types.isIdentifier(objectKey)) { 79 | objectKey = types.stringLiteral(objectKey.name); 80 | } 81 | try { 82 | const hookFunctionArguments = [ 83 | objectKey, 84 | propertyValue, 85 | types.stringLiteral("object-key-init") 86 | ]; 87 | objectProperty.value = types.callExpression(types.identifier(hookFunctionName), hookFunctionArguments); 88 | } catch (e) { 89 | console.error(e); 90 | } 91 | } 92 | }, 93 | // 函数的形参 94 | FunctionDeclaration(path) { 95 | const node = path.node; 96 | if (!(node.params && node.params.length)) { 97 | return; 98 | } 99 | const params = node.params; 100 | if (types.isBlockStatement(node.body)) { 101 | // 函数体是个代码块的，则在代码块最前面插入Hook，检查参数的值 102 | for (let i = params.length - 1; i >= 0; i--) { 103 | try { 104 | const paramName = params[i]; 105 | const hookFunctionArguments = [ 106 | types.stringLiteral(generator.default(paramName).code), 107 | paramName, 108 | types.stringLiteral("function-parameter") 109 | ]; 110 | const hookFunction = types.callExpression(types.identifier(hookFunctionName), hookFunctionArguments); 111 | node.body.body.unshift(types.expressionStatement(hookFunction)); 112 | } catch (e) { 113 | console.error(e); 114 | } 115 | } 116 | } 117 | } 118 | }) 119 | return generator.default(ast).code; 120 | } 121 | // module.exports.injectHook = injectHook; 122 | 123 | var hookfunc = ` 124 | if (!window.myDB){ 125 | window.myDB = [] 126 | }; 127 | function `+hookFunctionName+`(a, b, c){ 128 | var DB = window.myDB 129 | var tp = Object.prototype.toString.call(b) 130 | if (/String/.test(tp) && tp.length > 10){ 131 | DB.push([b, getCodeLocation()]) 132 | } 133 | return b 134 | }; 135 | function findv(str){ 136 | var DB = window.myDB 137 | for (var i = 0; i < DB.length; i++) { 138 | if (DB[i][0].indexOf(str) != -1){ 139 | console.log(DB[i][0], DB[i][1]) 140 | } 141 | } 142 | }; 143 | function getCodeLocation() { 144 | const c = new Error().stack.split("\\n"); 145 | while (c.length > 0 && c[0].indexOf("`+hookFunctionName+`") === -1) { 146 | c.shift(); 147 | } 148 | if (c.length < 2) { 149 | return null; 150 | } 151 | c.shift(); 152 | return c.shift(); 153 | }; 154 | ` 155 | 156 | // var jscode = fs.readFileSync("./test_hook.js", { encoding: "utf-8" }); 157 | // v = hookfunc + injectHook(jscode) 158 | // console.log(v) 159 | function make_inject_hook(jscode) { 160 | return hookfunc + injectHook(jscode) 161 | } -------------------------------------------------------------------------------- /vrequest/pypyzbar.py: -------------------------------------------------------------------------------- 1 | # 已经将二进制的 zbar 集成在脚本内。 2 | # 现在只需要本地一个脚本就能实现二维码解密 3 | 4 | import zlib 5 | import ctypes 6 | from struct import pack, calcsize, unpack 7 | GetWindowDC = ctypes.windll.user32.GetWindowDC 8 | GetSystemMetrics = ctypes.windll.user32.GetSystemMetrics 9 | SelectObject = ctypes.windll.gdi32.SelectObject 10 | DeleteObject = ctypes.windll.gdi32.DeleteObject 11 | BitBlt = ctypes.windll.gdi32.BitBlt 12 | GetDIBits = ctypes.windll.gdi32.GetDIBits 13 | CreateCompatibleDC = ctypes.windll.gdi32.CreateCompatibleDC 14 | CreateCompatibleBitmap = ctypes.windll.gdi32.CreateCompatibleBitmap 15 | 16 | def screenshot(shape:'left,top,width,height'=None): 17 | def png_bit(data, size, level=6): 18 | width, height = size 19 | line = width * 3 20 | png_filter = pack(">B", 0) 21 | scanlines = b"".join( 22 | [png_filter + data[y * line : y * line + line] for y in range(height)][::-1] 23 | ) 24 | magic = pack(">8B", 137, 80, 78, 71, 13, 10, 26, 10) 25 | ihdr = [b"", b"IHDR", b"", b""] 26 | ihdr[2] = pack(">2I5B", width, height, 8, 2, 0, 0, 0) 27 | ihdr[3] = pack(">I", zlib.crc32(b"".join(ihdr[1:3])) & 0xFFFFFFFF) 28 | ihdr[0] = pack(">I", len(ihdr[2])) 29 | idat = [b"", b"IDAT", zlib.compress(scanlines, level), b""] 30 | idat[3] = pack(">I", zlib.crc32(b"".join(idat[1:3])) & 0xFFFFFFFF) 31 | idat[0] = pack(">I", len(idat[2])) 32 | iend = [b"", b"IEND", b"", b""] 33 | iend[3] = pack(">I", zlib.crc32(iend[1]) & 0xFFFFFFFF) 34 | iend[0] = pack(">I", len(iend[2])) 35 | return magic + b"".join(ihdr + idat + iend) 36 | 37 | left, top, width, height = shape if shape else (0, 0, GetSystemMetrics(0), GetSystemMetrics(1)) 38 | bmi = pack('LHHHH', calcsize('LHHHH'), width, height, 1, 32) 39 | srcdc = GetWindowDC(0) 40 | memdc = CreateCompatibleDC(srcdc) 41 | svbmp = CreateCompatibleBitmap(srcdc, width, height) 42 | SelectObject(memdc, svbmp); BitBlt(memdc, 0, 0, width, height, srcdc, left, top, 13369376) 43 | _data = ctypes.create_string_buffer(height * width * 4) 44 | got_bits = GetDIBits(memdc, svbmp, 0, height, _data, bmi, 0) 45 | DeleteObject(memdc) 46 | data = bytes(_data) 47 | rgb = bytearray(width * height * 3) 48 | rgb[0::3],rgb[1::3],rgb[2::3] = data[2::4],data[1::4],data[0::4] 49 | size = (width, height) 50 | return png_bit(rgb, size) # 全屏截图 png bit 数据 51 | 52 | def create_png_pixel_tobytes(png_bit, shapelimit=None): 53 | left, top, width, height = [0,0,10000000,10000000] if shapelimit is None else shapelimit 54 | b = png_bit.find(b'IHDR') 55 | q = calcsize(">2I") 56 | w, h = unpack(">2I", png_bit[b+4:b+4+q]) 57 | b = png_bit.find(b'IDAT') 58 | q = calcsize(">I") 59 | v = unpack(">I", png_bit[b-q:b])[0] 60 | v = png_bit[b+4:b+4+v] 61 | z = zlib.decompress(v[2:-4], -15) 62 | l, p = w * 3 + 1, [] 63 | for i in range(h): 64 | if i >= top and i < top + height: 65 | li = z[i*l:(i+1)*l][1:] 66 | for j in range(w): 67 | if j >= left and j < left + width: 68 | p.append(sum(li[(j)*3 : (j+1)*3])//3) 69 | w, h = (w, h) if shapelimit is None else (width, height) 70 | p = [255 if i > 123 else 0 for i in p] # 人工二值化 71 | return bytes(p), w, h 72 | 73 | 74 | 75 | import tkinter 76 | # 主要的截图处理工具，用于快速截图或者鼠标框选部分进行定位处理的工具 77 | class PicCapture: 78 | def __init__(self, root, png): 79 | self.X = tkinter.IntVar(value=0) 80 | self.Y = tkinter.IntVar(value=0) 81 | sw = root.winfo_screenwidth() 82 | sh = root.winfo_screenheight() 83 | self.top = tkinter.Toplevel(root, width=sw, height=sh) 84 | self.top.overrideredirect(True) 85 | self.canvas = tkinter.Canvas(self.top, bg='white', width=sw, height=sh) 86 | self.image = tkinter.PhotoImage(file=png) 87 | self.canvas.create_image(sw//2, sh//2, image=self.image) 88 | self.fin_draw = None 89 | def btndown(event): 90 | self.X.set(event.x) 91 | self.Y.set(event.y) 92 | self.sel = True 93 | self.canvas.bind('', btndown) 94 | def btnmove(event): 95 | if not self.sel: 96 | return 97 | try: 98 | self.canvas.delete(self.fin_draw) 99 | except Exception as e: 100 | pass 101 | self.fin_draw = self.canvas.create_rectangle( 102 | self.X.get(), 103 | self.Y.get(), 104 | event.x, 105 | event.y, 106 | outline='red') 107 | 108 | self.canvas.bind('', btnmove) 109 | def btnup(event): 110 | self.sel = False 111 | try: 112 | self.canvas.delete(self.fin_draw) 113 | except Exception as e: 114 | pass 115 | left, right = sorted([self.X.get(), event.x]) 116 | top, bottom = sorted([self.Y.get(), event.y]) 117 | self.rect = (left, top, right, bottom) 118 | self.top.destroy() 119 | self.canvas.bind('', btnup) 120 | self.canvas.pack(fill=tkinter.BOTH, expand=tkinter.YES) 121 | 122 | import tempfile 123 | def screenshot_rect(root): 124 | filename = os.path.join(os.path.dirname(tempfile.mktemp()), 'temp.png') 125 | screenshot_bit = screenshot() 126 | with open(filename,'wb') as f: 127 | f.write(screenshot_bit) 128 | picshot = PicCapture(root, filename) 129 | root.wait_window(picshot.top) 130 | left, top, right, bottom = picshot.rect 131 | left, top, width, height = left, top, right-left, bottom-top 132 | os.remove(filename) 133 | return create_png_pixel_tobytes(screenshot_bit, (left, top, width, height)) 134 | 135 | import os 136 | import base64 137 | import pyzbar.pyzbar as pyzbar 138 | decode = pyzbar.decode 139 | 140 | if __name__ == '__main__': 141 | # 截屏处理全部二维码 142 | screenshot_bit = screenshot() 143 | pixbytes, w, h = create_png_pixel_tobytes(screenshot_bit) 144 | deco = decode((pixbytes, w, h)) 145 | for i in deco: 146 | print(i) 147 | 148 | # 手动截图处理二维码 149 | s = tkinter.Tk() 150 | def parse_rect_from_png_bytes(*a): 151 | pixbytes, w, h = screenshot_rect(s) 152 | deco = decode((pixbytes, w, h)) 153 | for i in deco: 154 | print(i) 155 | tkinter.Button(s,text='截图解析二维码',command=parse_rect_from_png_bytes,width=40).pack() 156 | s.mainloop() -------------------------------------------------------------------------------- /vrequest/pysimplejs2python.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | def simplejs2python(s): 4 | s = '\n' + s 5 | s = re.sub(r'(\n *)function', r'\1def', s) 6 | for _ in range(10): s = re.sub(r'(\n *\}\n)', r'\n', s) 7 | s = re.sub(r'(\n *\}\n)', r'\n', s) 8 | s = re.sub(r'(\n *)\} *([^\n]+\n)', r'\1\2', s) 9 | s = re.sub(r' *\{ *\n', r':\n', s) 10 | s = re.sub(r'(\n *)(if *\([^\(\)]+\)) *([^\n\{:]+\n)', r'\1\2:\1 \3', s) 11 | 12 | # 这里考虑处理简单的for循环条件的置换处理,不过这里有很大的变数要处理，后续再搞 13 | def deal_for(e): 14 | g = e.group(1) 15 | a,b,c = e.group(2),e.group(3),e.group(4) 16 | v = g + a.strip() + g + 'while ({}):'.format(b.strip()) + ' [{}]'.format(c.strip()) 17 | return v 18 | s = re.sub(r'(\n *)for *\(([^;]*?);([^;]*?);([^;]*?)\):?', deal_for, s) 19 | def deal_while_1(s): 20 | q = [] 21 | g = None 22 | k = None 23 | t = 0 24 | for i in s.splitlines(): 25 | if t == 1: 26 | v = re.findall(r'^( *)', i)[0] 27 | if i.strip() != '': 28 | if g is None: 29 | g = len(v) 30 | else: 31 | if len(v) < g: 32 | q.append(' '*g + k) 33 | t = 2 34 | else: 35 | v = re.findall(r'^( *)(while[^\n]+)\[([^\n]+)\] *$', i) 36 | if v: 37 | i = (v[0][0] + v[0][1]).rstrip() 38 | k = v[0][2].strip() 39 | t = 1 40 | q.append(i) 41 | q = '\n'.join(q) 42 | if re.findall(r'while[^\n]+\[([^\n]+)\]', q) and q != s: 43 | return deal_while_1(q) 44 | else: 45 | return q 46 | s = deal_while_1(s) 47 | # 处理自增应该在处理循环之后 48 | s = re.sub(r'(\n *)([^\n]*?)([a-zA-Z0-9_$]+)( *\+\+)([^\n]*)', r'\1\2\3\5; \3 += 1', s) 49 | s = re.sub(r'(\n *)([^\n]*?)(\+\+ *)([a-zA-Z0-9_$]+)([^\n]*)', r'\1\4 += 1;\2\4\5', s) 50 | def deal_if_1(e): 51 | if e.group(2).strip().endswith(':'): 52 | return e.group(0) 53 | else: 54 | return e.group(1) + e.group(2) + ':' 55 | s = re.sub(r'(\n *)(if *\([^\n]*\))', deal_if_1, s) 56 | s = re.sub(r'(\n *)else if', r'\1elif', s) 57 | s = re.sub(r'(\n *)else *', r'\1else:', s) 58 | s = re.sub(r'(\n *)var ', r'\1', s) 59 | s = re.sub(r'(\n *)//', r'\1#', s) 60 | def deal_comment_1(e): 61 | r = [] 62 | for i in e.group(0).splitlines(): 63 | i = i.strip('/ *') 64 | if i: r.append('# ' + i) 65 | return '\n'.join(r) 66 | s = re.sub(r'/\*.*?\*/', deal_comment_1, s, flags=re.S) 67 | s = re.sub(r'\n\n+', r'\n\n', s, flags=re.S) 68 | s = re.sub(r'\| *\n', r'| \\\n', s, flags=re.S) 69 | s = re.sub(r'; *;', r';', s) 70 | s = re.sub(r': *:', r':', s) 71 | s = re.sub(r'<<<', r'<<', s) 72 | s = re.sub(r'>>>', r'>>', s) 73 | s = re.sub(r'!==', r'!=', s) 74 | 75 | s = re.sub(r'((?:[a-zA-Z0-9_$]+(?:\[[a-zA-Z0-9_$]+\])* *\. *)*[a-zA-Z0-9_$]+(?:\[[a-zA-Z0-9_$]+\])*) *\. *length', r'len(\1)', s) # 处理 .length 函数为 len() 76 | def find_something(s, p='charAt', b='()'): 77 | o = re.findall(r'(?:[a-zA-Z0-9_$]+(?:\[[a-zA-Z0-9_$]+\])* *\. *)*[a-zA-Z0-9_$]+(?:\[[a-zA-Z0-9_$]+\])* *\. *' + p, s) 78 | if o: 79 | p = o[0] 80 | v = s.find(p) 81 | if v != -1: 82 | r = 0 83 | c = [] 84 | for i in s[v+len(p):]: 85 | t = False 86 | if i == b[0]: r += 1; t = True 87 | if i == b[1]: r -= 1; t = True 88 | c.append(i) 89 | if t and r == 0: 90 | break 91 | v = ''.join(c) 92 | return o[0], v 93 | def replace_something(s, p='charAt', b='()'): 94 | cont = 0 95 | while True: 96 | cont += 1 97 | r = find_something(s, p, b) 98 | if r and cont <= 500: 99 | o, v = r 100 | if p == 'charAt': s = s.replace('.' + o.rsplit('.', 1)[-1] + v, '[{}]'.format(v.strip()[1:-1])) 101 | elif p == 'charCodeAt': s = s.replace(o + v, 'ord({}[{}])'.format(o.rsplit('.', 1)[0], v.strip()[1:-1])) 102 | elif p == 'fromCharCode': s = s.replace(o + v, 'bytes({}).decode()'.format(v)) 103 | else: 104 | return s 105 | s = replace_something(s, p='charAt') 106 | s = replace_something(s, p='charCodeAt') 107 | s = replace_something(s, p='fromCharCode') 108 | 109 | # 这里考虑处理赋值的对齐,不过现在还是有点问题 110 | # s = re.sub(r'(\n *)([^\n=]+=[^\n=]+), *', r'\1\2', s) 111 | s = re.sub(r'_\$([a-zA-Z0-9_]{2})', r'_\1', s) # 文书相关的处理 112 | s = re.sub(r'; *\n', r'\n', s) # 去除尾部分号(非必要) 113 | s = '''# 该功能仅用于简单的函数算法转换， 114 | # 在一定程度上能方便更加青睐于纯 python 教徒对 js 翻译 115 | # 请勿对翻译后的代码抱有过度信赖，该功能仅依赖于正则替换，所以生成代码很可能需要一定的微调。 116 | ''' + s + ''' 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | # 备用函数注意 141 | # 当函数出现左移(<<)或者右移(>>)函数的情况下，由于js与python在这里结果不一样，所以可以尝试解开下面的函数 142 | # 用 python 函数实现js代码中的左移和右移，例如： s << 13 -> rotleft(s, 13) ，部分数字加减需要约束范围 143 | # 则使用 limitint 来保证函数范围。 144 | # 因为功能实现很急，又因为 python 的位移很奇怪，暂时用到了比较笨的方法，后续有空再改。 145 | # def _rot(num, rnum, side): 146 | # bnum = bin(((1 << 32) - 1) & num)[2:] 147 | # s = [None] * 32 148 | # for idx, i in enumerate('{:>032s}'.format(bnum[-32:])): 149 | # s[idx] = '0' if i == '0' else '1' 150 | # if side == 'left': 151 | # s.extend(['0'] * rnum) 152 | # s = s[-32:] 153 | # elif side == 'right': 154 | # s = ['0'] * rnum + s 155 | # s = s[:32] 156 | # if s[0] == '1': 157 | # for i in range(1,32): 158 | # s[i] = '1' if s[i] == '0' else '0' 159 | # v = -int(''.join(s[1:]), 2)-1 160 | # else: 161 | # v = int(''.join(s),2) 162 | # return v 163 | # def limitint(num): 164 | # bnum = bin(((1 << 32) - 1) & num)[2:] 165 | # s = [None] * 32 166 | # for idx, i in enumerate('{:>032s}'.format(bnum[-32:])): 167 | # s[idx] = '0' if i == '0' else '1' 168 | # if s[0] == '1': 169 | # for i in range(1,32): 170 | # s[i] = '1' if s[i] == '0' else '0' 171 | # v = -int(''.join(s[1:]), 2)-1 172 | # else: 173 | # v = int(''.join(s),2) 174 | # return v 175 | # def rotleft(num, rnum): return _rot(num, rnum, 'left') 176 | # def rotright(num, rnum): return _rot(num, rnum, 'right') 177 | ''' 178 | return s.rstrip(' }\n') 179 | 180 | if __name__ == '__main__': 181 | s = ''' 182 | // 123123 183 | function _$TM() { 184 | var _$fz = []; 185 | for (var _$xA = 0; _$xA < 256; ++_$xA) { 186 | var _$F1 = _$xA; 187 | for (var _$vR = 0; _$vR < 8; ++_$vR) { 188 | if ((_$F1 & 0x80) !== 0) 189 | _$F1 = (_$F1 << 1) ^ 7; 190 | else 191 | _$F1 <<= 1; 192 | } 193 | _$fz[_$xA] = _$F1 & 0xff; 194 | } 195 | return _$fz; 196 | ''' 197 | print(simplejs2python(s)) -------------------------------------------------------------------------------- /vrequest/pybase.py: -------------------------------------------------------------------------------- 1 | # b36 看网上的说明，貌似应用于将数字转化成字符串进行保存？ 2 | def b36encode(num): 3 | try: 4 | num = int(num) 5 | except: 6 | raise TypeError('num must be an integer') 7 | alpha = '0123456789abcdefghijklmnopqrstuvwxyz' 8 | if num < 0: 9 | return '-' + b36encode(-num) 10 | val = '' 11 | while num != 0: 12 | num, idx = divmod(num, len(alpha)) 13 | val = alpha[idx] + val 14 | return val or '0' 15 | 16 | def b36decode(val): 17 | return str(int(val, 36)).encode() 18 | 19 | # b62 看网上的说明，貌似应用于将数字转化成字符串进行保存？ 20 | def b62encode(num): 21 | try: 22 | num = int(num) 23 | except: 24 | raise TypeError('num must be an integer') 25 | alpha = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' 26 | if num < 0: 27 | return '-' + b36encode(-num) 28 | val = '' 29 | while num != 0: 30 | num, idx = divmod(num, len(alpha)) 31 | val = alpha[idx] + val 32 | return val or '0' 33 | 34 | def b62decode(val): 35 | val = val.decode() 36 | alpha = '0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' 37 | r = 0 38 | for idx,i in enumerate(val[::-1]): 39 | r += alpha.index(i) * (len(alpha) ** idx) 40 | return str(r).encode() 41 | 42 | # Base91 encode/decode for Python 2 and Python 3 43 | # 44 | # Copyright (c) 2012 Adrien Beraud 45 | # Copyright (c) 2015 Guillaume Jacquenot 46 | # All rights reserved. 47 | # 48 | # Redistribution and use in source and binary forms, with or without 49 | # modification, are permitted provided that the following conditions are met: 50 | # 51 | # * Redistributions of source code must retain the above copyright notice, 52 | # this list of conditions and the following disclaimer. 53 | # * Redistributions in binary form must reproduce the above copyright notice, 54 | # this list of conditions and the following disclaimer in the documentation 55 | # and/or other materials provided with the distribution. 56 | # * Neither the name of Adrien Beraud, Wisdom Vibes Pte. Ltd., nor the names 57 | # of its contributors may be used to endorse or promote products derived 58 | # from this software without specific prior written permission. 59 | # 60 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 61 | # AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 62 | # IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 63 | # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE 64 | # FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 65 | # DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 66 | # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 67 | # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 68 | # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 69 | # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 70 | 71 | 72 | # import string 73 | # a = string.ascii_uppercase 74 | # b = string.ascii_lowercase 75 | # c = string.digits 76 | # d = string.punctuation.replace('"','').replace('\\','').replace('-','').replace('\'','')+'"' 77 | # alphabet = (a + b + c + d).encode() 78 | # print(alphabet) 79 | 80 | import struct 81 | base91_alphabet = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 82 | 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 83 | 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 84 | 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 85 | '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '!', '#', '$', 86 | '%', '&', '(', ')', '*', '+', ',', '.', '/', ':', ';', '<', '=', 87 | '>', '?', '@', '[', ']', '^', '_', '`', '{', '|', '}', '~', '"'] 88 | 89 | decode_table = dict((v, k) for k, v in enumerate(base91_alphabet)) 90 | 91 | def b91decode(encoded_str): 92 | ''' Decode Base91 string to a bytearray ''' 93 | v = -1 94 | b = 0 95 | n = 0 96 | out = b'' 97 | for strletter in encoded_str.decode(): 98 | if not strletter in decode_table: 99 | continue 100 | c = decode_table[strletter] 101 | if (v < 0): 102 | v = c 103 | else: 104 | v += c * 91 105 | b |= v << n 106 | n += 13 if (v & 8191) > 88 else 14 107 | while True: 108 | out += struct.pack('B', b & 255) 109 | b >>= 8 110 | n -= 8 111 | if not n > 7: 112 | break 113 | v = -1 114 | if v + 1: 115 | out += struct.pack('B', (b | v << n) & 255) 116 | return out 117 | 118 | def b91encode(bindata): 119 | ''' Encode a bytearray to a Base91 string ''' 120 | b = 0 121 | n = 0 122 | out = '' 123 | for count in range(len(bindata)): 124 | byte = bindata[count:count + 1] 125 | b |= struct.unpack('B', byte)[0] << n 126 | n += 8 127 | if n > 13: 128 | v = b & 8191 129 | if v > 88: 130 | b >>= 13 131 | n -= 13 132 | else: 133 | v = b & 16383 134 | b >>= 14 135 | n -= 14 136 | out += base91_alphabet[v % 91] + base91_alphabet[v // 91] 137 | if n: 138 | out += base91_alphabet[b % 91] 139 | if n > 7 or b > 90: 140 | out += base91_alphabet[b // 91] 141 | return out.encode() 142 | 143 | # b58 144 | # 没有0，大写的O，大写的I，以及小写的l 145 | alphabet = b'123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz' 146 | alphalen = len(alphabet) 147 | 148 | def b58encode_int(i, default_one=True): 149 | '''Encode an integer using Base58''' 150 | if not i and default_one: 151 | return alphabet[0:1] 152 | string = b"" 153 | while i: 154 | i, idx = divmod(i, alphalen) 155 | string = alphabet[idx:idx+1] + string 156 | return string 157 | 158 | def b58encode(v): 159 | nPad = len(v) 160 | v = v.lstrip(b'\0') 161 | nPad -= len(v) 162 | p, acc = 1, 0 163 | for c in reversed(v): 164 | acc += p * c 165 | p = p << 8 166 | result = b58encode_int(acc, default_one=False) 167 | return (alphabet[0:1] * nPad + result) 168 | 169 | def b58decode_int(v): 170 | decimal = 0 171 | for char in v: 172 | decimal = decimal * alphalen + alphabet.index(char) 173 | return decimal 174 | 175 | def b58decode(v): 176 | origlen = len(v) 177 | v = v.lstrip(alphabet[0:1]) 178 | newlen = len(v) 179 | acc = b58decode_int(v) 180 | result = [] 181 | while acc > 0: 182 | acc, mod = divmod(acc, 256) 183 | result.append(mod) 184 | return (b'\0' * (origlen - newlen) + bytes(reversed(result))) 185 | 186 | from base64 import * 187 | base_algos = { 188 | 'base36':(b36encode, b36decode), 189 | 'base62':(b62encode, b62decode), 190 | 'base16':(b16encode, b16decode), 191 | 'base32':(b32encode, b32decode), 192 | 'base58':(b58encode, b58decode), 193 | 'base64':(b64encode, b64decode), 194 | 'urlsafe_b64':(urlsafe_b64encode, urlsafe_b64decode), 195 | 'base85':(b85encode, b85decode), 196 | 'base91':(b91encode, b91decode), 197 | } 198 | 199 | if __name__ == '__main__': 200 | s = b36encode(1111111111111111) 201 | print(s) 202 | s = b36decode(s) 203 | print(s) 204 | print() 205 | 206 | s = b62encode(1111111111111111) 207 | print(s) 208 | s = b62decode(s) 209 | print(s) 210 | print() 211 | 212 | s = b91encode(b'asdfasdf') 213 | print(s) 214 | s = b91decode(s) 215 | print(s) 216 | print() 217 | 218 | s = b58encode(b'asdfasdf') 219 | print(s) 220 | s = b58decode(s) 221 | print(s) 222 | print() -------------------------------------------------------------------------------- /test/_create_al_frame_mode.py: -------------------------------------------------------------------------------- 1 | try: 2 | # 处理 sublime 执行时输出乱码 3 | import io 4 | import sys 5 | sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8') 6 | sys.stdout._CHUNK_SIZE = 1 7 | except: 8 | pass 9 | 10 | 11 | def create_new_al(flag, alname): 12 | s = r''' 13 | # 这里后续需要考虑增加各种各样的加密解密以及代码的记录 14 | # 光是aes就有5种加解密方式 15 | def $change_cbit_1(*content): 16 | if content: 17 | encd = $line1_fent1.get().strip() 18 | blen = len(content[0].encode(encd))*8 19 | $line1_cbit1['text'] = str(blen)+'bit' 20 | return True 21 | def $change_cbit_2(*content): 22 | if content: 23 | encd = $line1_fent1.get().strip() 24 | blen = len(content[0].encode(encd))*8 25 | $line2_cbit2['text'] = str(blen)+'bit' 26 | return True 27 | 28 | $change_cbit1 = root.register($change_cbit_1) 29 | $change_cbit2 = root.register($change_cbit_2) 30 | $title = Frame(ff0) 31 | $title.pack(side=tkinter.TOP,fill=tkinter.X) 32 | Label($title, text=$titlestring).pack(fill=tkinter.X,expand=True) 33 | $line1 = Frame(ff0) 34 | $line1.pack(side=tkinter.TOP,fill=tkinter.X) 35 | Label($line1, text='密码',width=4).pack(side=tkinter.LEFT,padx=2) 36 | $line1_ent1 = Entry($line1,width=17,validate='key',validatecommand=($change_cbit1, '%P')) 37 | $line1_ent1.pack(side=tkinter.LEFT) 38 | $line1_ent1.bind('', $change_cbit1) 39 | $line1_cbit1 = Label($line1, text='0bit',width=6) 40 | $line1_cbit1.pack(side=tkinter.LEFT,padx=6) 41 | $line1_mode1 = Combobox($line1,width=4,state='readonly') 42 | $line1_mode1['values'] = ['b16','b32','b64','b85'] 43 | $line1_mode1.current(2) 44 | $line1_mode1.pack(side=tkinter.RIGHT) 45 | Label($line1, text='编码',width=4).pack(side=tkinter.RIGHT,padx=5) 46 | def _$swich_encd1(*a): 47 | s = $line1_fent1.get().strip() 48 | if s == 'utf-8': 49 | $line1_fent1.delete(0,tkinter.END) 50 | $line1_fent1.insert(0,'gbk') 51 | elif s == 'gbk': 52 | $line1_fent1.delete(0,tkinter.END) 53 | $line1_fent1.insert(0,'utf-8') 54 | else: 55 | $line1_fent1.delete(0,tkinter.END) 56 | $line1_fent1.insert(0,'utf-8') 57 | $change_cbit_1($line1_ent1.get().strip()) 58 | $change_cbit_2($line2_ent2.get().strip()) 59 | $line1_fent1 = Entry($line1,width=5) 60 | $line1_fent1.insert(0,'utf-8') 61 | $line1_fent1.pack(side=tkinter.RIGHT) 62 | Button($line1, text='密码/iv/数据编码格式',command=_$swich_encd1).pack(side=tkinter.RIGHT) 63 | $line1_mode2 = Combobox($line1,width=4,state='readonly') 64 | $line1_mode2['values'] = ['cbc','cfb','ofb','ctr','ecb',] 65 | $line1_mode2.current(0) 66 | $line1_mode2.pack(side=tkinter.RIGHT) 67 | Label($line1, text='模式',width=4).pack(side=tkinter.RIGHT,padx=5) 68 | $line2 = Frame(ff0) 69 | $line2.pack(side=tkinter.TOP,fill=tkinter.X) 70 | Label($line2, text='iv',width=4).pack(side=tkinter.LEFT,padx=2) 71 | $line2_ent2 = Entry($line2,width=17,validate='key',validatecommand=($change_cbit2, '%P')) 72 | $line2_ent2.pack(side=tkinter.LEFT) 73 | 74 | $line2_cbit2 = Label($line2, text='128:bit',width=6) 75 | $line2_cbit2.pack(side=tkinter.LEFT,padx=6) 76 | $line2_ent2.insert(0,'1234567890123456') 77 | Label($line2, text='ecb模式：iv无效；ctr模式：iv长度不限制',).pack(side=tkinter.LEFT,padx=6) 78 | 79 | def _$alname_encode(*a): 80 | encd = $line1_fent1.get().strip() 81 | mode = $line1_mode2.get().strip() 82 | eout = $line1_mode1.get().strip() 83 | key = $line1_ent1.get().strip().encode(encd) 84 | iv = $line2_ent2.get().strip().encode(encd) 85 | data = ftxt.get(0.,tkinter.END).strip('\n').encode(encd) 86 | limitnum = int(entlimit.get().strip()) 87 | ftxt.delete(0.,tkinter.END) 88 | try: 89 | from . import pyaes 90 | except: 91 | # 请勿在本脚本测试时安装了 pyaes，pyaes的源码部分有问题 92 | import pyaes 93 | if eout == 'b16':_encode = base64.b16encode; _decode = base64.b16decode 94 | if eout == 'b32':_encode = base64.b32encode; _decode = base64.b32decode 95 | if eout == 'b64':_encode = base64.b64encode; _decode = base64.b64decode 96 | if eout == 'b85':_encode = base64.b85encode; _decode = base64.b85decode 97 | Encrypter = pyaes.Encrypter 98 | Counter = pyaes.Counter 99 | AESModesOfOperation = pyaes.AESModesOfOperation 100 | 101 | try: 102 | if mode in 'ctr': 103 | enc = Encrypter(AESModesOfOperation[mode](key, Counter(int.from_bytes(iv, 'big')))) 104 | elif mode == 'ecb': 105 | enc = Encrypter(AESModesOfOperation[mode](key)) 106 | else: 107 | enc = Encrypter(AESModesOfOperation[mode](key, iv)) 108 | en = _encode(enc.feed(data)).decode(encd) 109 | if len(en) > limitnum: 110 | print('警告！') 111 | print('加密数据长度({})过长（超过{}字符，超过的部分不显示）'.format(len(en),limitnum)) 112 | print('因为 tkinter 性能瓶颈，不宜在 tkinter 窗口展示，请使用算法在别的IDE内实现') 113 | print('---------------------------------------------------') 114 | print(en[:limitnum]) 115 | else: 116 | print(en) 117 | except: 118 | print(traceback.format_exc()) 119 | 120 | def _$alname_decode(*a): 121 | encd = $line1_fent1.get().strip() 122 | mode = $line1_mode2.get().strip() 123 | eout = $line1_mode1.get().strip() 124 | key = $line1_ent1.get().strip().encode(encd) 125 | iv = $line2_ent2.get().strip().encode(encd) 126 | data = ftxt.get(0.,tkinter.END).strip('\n').encode(encd) 127 | ftxt.delete(0.,tkinter.END) 128 | try: 129 | from . import pyaes 130 | except: 131 | # 请勿在本脚本测试时安装了 pyaes，pyaes的源码部分有问题 132 | import pyaes 133 | if eout == 'b16':_encode = base64.b16encode; _decode = base64.b16decode 134 | if eout == 'b32':_encode = base64.b32encode; _decode = base64.b32decode 135 | if eout == 'b64':_encode = base64.b64encode; _decode = base64.b64decode 136 | if eout == 'b85':_encode = base64.b85encode; _decode = base64.b85decode 137 | Decrypter = pyaes.Decrypter 138 | Counter = pyaes.Counter 139 | AESModesOfOperation = pyaes.AESModesOfOperation 140 | 141 | try: 142 | if mode in 'ctr': 143 | dec = Decrypter(AESModesOfOperation[mode](key, Counter(int.from_bytes(iv, 'big')))) 144 | elif mode == 'ecb': 145 | dec = Decrypter(AESModesOfOperation[mode](key)) 146 | else: 147 | dec = Decrypter(AESModesOfOperation[mode](key, iv)) 148 | dc = dec.feed(_decode(data)).decode(encd) 149 | print(dc) 150 | except: 151 | print(traceback.format_exc()) 152 | 153 | def _$alname_code(*a): 154 | try: 155 | from . import pyaes 156 | except: 157 | import pyaes 158 | ftxt.delete(0.,tkinter.END) 159 | with open(pyaes.__file__, encoding='utf-8') as f: 160 | data = f.read().strip('\n') 161 | print(data) 162 | 163 | Button($line2, text='[算法]',command=_$alname_code,width=5).pack(side=tkinter.RIGHT) 164 | Button($line2, text='解密',command=_$alname_decode,width=5).pack(side=tkinter.RIGHT) 165 | Button($line2, text='加密',command=_$alname_encode,width=5).pack(side=tkinter.RIGHT) 166 | ''' 167 | 168 | s = s.replace('$alname', alname) 169 | s = s.replace('$line1_cbit1', flag + '1_cbit1') 170 | s = s.replace('$line1_ent1', flag + '1_ent1') 171 | s = s.replace('$line1_fent1', flag + '1_fent1') 172 | s = s.replace('$line1_mode1', flag + '1_mode1') 173 | s = s.replace('$line1_mode2', flag + '1_mode2') 174 | s = s.replace('$line2_cbit2', flag + '2_cbit2') 175 | s = s.replace('$line2_ent2', flag + '2_ent2') 176 | s = s.replace('$titlestring', repr(' 以下算法为 AES 加解密算法 [密码长度需注意:128bit,192bit,256bit] [iv长度需注意:128bit]。')) 177 | s = s.replace('$line2', flag + '2') 178 | s = s.replace('$line1', flag + '1') 179 | s = s.replace('$title', flag + '0') 180 | s = s.replace('$change_cbit1', flag + '_change_cbit1') 181 | s = s.replace('$change_cbit2', flag + '_change_cbit2') 182 | s = s.replace('$change_cbit_1', flag + '_change_cbit_1') 183 | s = s.replace('$change_cbit_2', flag + '_change_cbit_2') 184 | s = s.replace('$swich_encd1', flag + 'swich_encd1') 185 | return s 186 | 187 | alname = 'serpent' 188 | s = create_new_al('f200', alname) 189 | print(s) -------------------------------------------------------------------------------- /vrequest/pyscreenshot.py: -------------------------------------------------------------------------------- 1 | # 处理简单的截图以及截图录制的功能 2 | 3 | import zlib 4 | import ctypes 5 | from struct import pack, calcsize, unpack 6 | GetWindowDC = ctypes.windll.user32.GetWindowDC 7 | GetSystemMetrics = ctypes.windll.user32.GetSystemMetrics 8 | SelectObject = ctypes.windll.gdi32.SelectObject 9 | DeleteObject = ctypes.windll.gdi32.DeleteObject 10 | BitBlt = ctypes.windll.gdi32.BitBlt 11 | GetDIBits = ctypes.windll.gdi32.GetDIBits 12 | CreateCompatibleDC = ctypes.windll.gdi32.CreateCompatibleDC 13 | CreateCompatibleBitmap = ctypes.windll.gdi32.CreateCompatibleBitmap 14 | 15 | def desktop_ltwh(): 16 | return (0, 0, GetSystemMetrics(0), GetSystemMetrics(1)) 17 | 18 | def screenshot(shape:'left,top,width,height'=None): 19 | def png_bit(data, size, level=6): 20 | width, height = size 21 | line = width * 3 22 | png_filter = pack(">B", 0) 23 | scanlines = b"".join( 24 | [png_filter + data[y * line : y * line + line] for y in range(height)][::-1] 25 | ) 26 | magic = pack(">8B", 137, 80, 78, 71, 13, 10, 26, 10) 27 | ihdr = [b"", b"IHDR", b"", b""] 28 | ihdr[2] = pack(">2I5B", width, height, 8, 2, 0, 0, 0) 29 | ihdr[3] = pack(">I", zlib.crc32(b"".join(ihdr[1:3])) & 0xFFFFFFFF) 30 | ihdr[0] = pack(">I", len(ihdr[2])) 31 | idat = [b"", b"IDAT", zlib.compress(scanlines, level), b""] 32 | idat[3] = pack(">I", zlib.crc32(b"".join(idat[1:3])) & 0xFFFFFFFF) 33 | idat[0] = pack(">I", len(idat[2])) 34 | iend = [b"", b"IEND", b"", b""] 35 | iend[3] = pack(">I", zlib.crc32(iend[1]) & 0xFFFFFFFF) 36 | iend[0] = pack(">I", len(iend[2])) 37 | return magic + b"".join(ihdr + idat + iend) 38 | 39 | left, top, width, height = shape if shape else desktop_ltwh() 40 | bmi = pack('LHHHH', calcsize('LHHHH'), width, height, 1, 32) 41 | srcdc = GetWindowDC(0) 42 | memdc = CreateCompatibleDC(srcdc) 43 | svbmp = CreateCompatibleBitmap(srcdc, width, height) 44 | SelectObject(memdc, svbmp); BitBlt(memdc, 0, 0, width, height, srcdc, left, top, 13369376) 45 | _data = ctypes.create_string_buffer(height * width * 4) 46 | got_bits = GetDIBits(memdc, svbmp, 0, height, _data, bmi, 0) 47 | DeleteObject(memdc) 48 | data = bytes(_data) 49 | rgb = bytearray(width * height * 3) 50 | rgb[0::3],rgb[1::3],rgb[2::3] = data[2::4],data[1::4],data[0::4] 51 | size = (width, height) 52 | return png_bit(rgb, size) # 全屏截图 png bit 数据 53 | 54 | def create_png_pixel_tobytes(png_bit, shapelimit=None): 55 | left, top, width, height = [0,0,10000000,10000000] if shapelimit is None else shapelimit 56 | b = png_bit.find(b'IHDR') 57 | q = calcsize(">2I") 58 | w, h = unpack(">2I", png_bit[b+4:b+4+q]) 59 | b = png_bit.find(b'IDAT') 60 | q = calcsize(">I") 61 | v = unpack(">I", png_bit[b-q:b])[0] 62 | v = png_bit[b+4:b+4+v] 63 | z = zlib.decompress(v[2:-4], -15) 64 | l, p = w * 3 + 1, [] 65 | for i in range(h): 66 | if i >= top and i < top + height: 67 | li = z[i*l:(i+1)*l][1:] 68 | for j in range(w): 69 | if j >= left and j < left + width: 70 | p.append(sum(li[(j)*3 : (j+1)*3])//3) 71 | w, h = (w, h) if shapelimit is None else (width, height) 72 | p = [255 if i > 123 else 0 for i in p] # 人工二值化 73 | return bytes(p), w, h 74 | 75 | 76 | 77 | import tkinter 78 | # 主要的截图处理工具，用于快速截图或者鼠标框选部分进行定位处理的工具 79 | class PicCapture: 80 | def __init__(self, root, png): 81 | self.X = tkinter.IntVar(value=0) 82 | self.Y = tkinter.IntVar(value=0) 83 | sw = root.winfo_screenwidth() 84 | sh = root.winfo_screenheight() 85 | self.top = tkinter.Toplevel(root, width=sw, height=sh) 86 | self.top.overrideredirect(True) 87 | self.canvas = tkinter.Canvas(self.top, bg='white', width=sw, height=sh) 88 | self.image = tkinter.PhotoImage(file=png) 89 | self.canvas.create_image(sw//2, sh//2, image=self.image) 90 | self.fin_draw = None 91 | def btndown(event): 92 | self.X.set(event.x) 93 | self.Y.set(event.y) 94 | self.sel = True 95 | self.canvas.bind('', btndown) 96 | def btnmove(event): 97 | if not self.sel: 98 | return 99 | try: 100 | self.canvas.delete(self.fin_draw) 101 | except Exception as e: 102 | pass 103 | self.fin_draw = self.canvas.create_rectangle( 104 | self.X.get(), 105 | self.Y.get(), 106 | event.x, 107 | event.y, 108 | outline='red') 109 | 110 | self.canvas.bind('', btnmove) 111 | def btnup(event): 112 | self.sel = False 113 | try: 114 | self.canvas.delete(self.fin_draw) 115 | except Exception as e: 116 | pass 117 | left, right = sorted([self.X.get(), event.x]) 118 | top, bottom = sorted([self.Y.get(), event.y]) 119 | self.rect = (left, top, right, bottom) 120 | self.top.destroy() 121 | self.canvas.bind('', btnup) 122 | self.canvas.pack(fill=tkinter.BOTH, expand=tkinter.YES) 123 | 124 | import os 125 | import tempfile 126 | def screenshot_rect(root): 127 | filename = os.path.join(os.path.dirname(tempfile.mktemp()), 'temp.png') 128 | screenshot_bit = screenshot() 129 | with open(filename,'wb') as f: 130 | f.write(screenshot_bit) 131 | picshot = PicCapture(root, filename) 132 | root.wait_window(picshot.top) 133 | left, top, right, bottom = picshot.rect 134 | left, top, width, height = left, top, right-left, bottom-top 135 | os.remove(filename) 136 | return screenshot((left, top, width, height)) 137 | 138 | def screenshot_rect_int(root): 139 | filename = os.path.join(os.path.dirname(tempfile.mktemp()), 'temp.png') 140 | screenshot_bit = screenshot() 141 | with open(filename,'wb') as f: 142 | f.write(screenshot_bit) 143 | picshot = PicCapture(root, filename) 144 | root.wait_window(picshot.top) 145 | left, top, right, bottom = picshot.rect 146 | left, top, width, height = left, top, right-left, bottom-top 147 | os.remove(filename) 148 | return left, top, width, height 149 | 150 | import time 151 | import threading 152 | import shutil 153 | class screenvideo: 154 | def __init__(self, filepath, rect, dirname='_temp', print=print): 155 | self.filepath = os.path.join(filepath, dirname) 156 | self.rect = rect 157 | self.cencel = False 158 | self.print = print 159 | if not os.path.isdir(self.filepath): 160 | if os.path.isfile(self.filepath): 161 | os.remove(self.filepath) 162 | os.mkdir(self.filepath) 163 | else: 164 | shutil.rmtree(self.filepath) 165 | os.mkdir(self.filepath) 166 | def start_video(self): 167 | self.print('start.') 168 | def _(): 169 | idx = 0 170 | while True: 171 | time.sleep(1/24) 172 | filepathname = os.path.join(self.filepath, '{}.png'.format(idx)) 173 | self.print('write in:{}'.format(filepathname)) 174 | idx += 1 175 | with open(filepathname, 'wb') as f: 176 | f.write(screenshot(self.rect)) 177 | if self.cencel: 178 | self.print('stop.') 179 | break 180 | threading.Thread(target=_).start() 181 | def stop_video(self): 182 | self.cencel = True 183 | 184 | sv = None 185 | def _start_video(desktop, rect, dirname, print=print): 186 | global sv 187 | sv = screenvideo(desktop, rect, dirname, print) 188 | sv.start_video() 189 | def _stop_video(): 190 | global sv 191 | if sv is not None: 192 | sv.stop_video() 193 | 194 | if __name__ == '__main__': 195 | # 截图存放到桌面 196 | s = tkinter.Tk() 197 | def start(*a): 198 | desktop = os.path.join(os.path.expanduser("~"),'Desktop') 199 | rect = screenshot_rect_int(s) 200 | _start_video(desktop, rect, '_temp') 201 | def stop(*a): 202 | _stop_video() 203 | 204 | def parse_rect_from_png_bytes(*a): 205 | dtop = os.path.join(os.path.expanduser("~"),'Desktop') 206 | dfile = os.path.join(dtop, '_temp.png') 207 | bitpng = screenshot_rect(s) 208 | with open(dfile, 'wb') as f: 209 | f.write(bitpng) 210 | 211 | tkinter.Button(s,text='截图获取坐标开始录制',command=start,width=40).pack() 212 | tkinter.Button(s,text='停止录制',command=stop,width=40).pack() 213 | tkinter.Button(s,text='普通的截图存放到桌面',command=parse_rect_from_png_bytes,width=40).pack() 214 | s.mainloop() -------------------------------------------------------------------------------- /vrequest/template/v/middlewares.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define here the models for your spider middleware 4 | # 5 | # See documentation in: 6 | # https://doc.scrapy.org/en/latest/topics/spider-middleware.html 7 | 8 | from scrapy import signals 9 | 10 | 11 | class VSpiderMiddleware(object): 12 | # Not all methods need to be defined. If a method is not defined, 13 | # scrapy acts as if the spider middleware does not modify the 14 | # passed objects. 15 | 16 | @classmethod 17 | def from_crawler(cls, crawler): 18 | # This method is used by Scrapy to create your spiders. 19 | s = cls() 20 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 21 | return s 22 | 23 | def process_spider_input(self, response, spider): 24 | # Called for each response that goes through the spider 25 | # middleware and into the spider. 26 | 27 | # Should return None or raise an exception. 28 | return None 29 | 30 | def process_spider_output(self, response, result, spider): 31 | # Called with the results returned from the Spider, after 32 | # it has processed the response. 33 | 34 | # Must return an iterable of Request, dict or Item objects. 35 | for i in result: 36 | yield i 37 | 38 | def process_spider_exception(self, response, exception, spider): 39 | # Called when a spider or process_spider_input() method 40 | # (from other spider middleware) raises an exception. 41 | 42 | # Should return either None or an iterable of Response, dict 43 | # or Item objects. 44 | pass 45 | 46 | def process_start_requests(self, start_requests, spider): 47 | # Called with the start requests of the spider, and works 48 | # similarly to the process_spider_output() method, except 49 | # that it doesn’t have a response associated. 50 | 51 | # Must return only requests (not items). 52 | for r in start_requests: 53 | yield r 54 | 55 | def spider_opened(self, spider): 56 | spider.logger.info('Spider opened: %s' % spider.name) 57 | 58 | 59 | class VDownloaderMiddleware(object): 60 | # Not all methods need to be defined. If a method is not defined, 61 | # scrapy acts as if the downloader middleware does not modify the 62 | # passed objects. 63 | 64 | @classmethod 65 | def from_crawler(cls, crawler): 66 | # This method is used by Scrapy to create your spiders. 67 | s = cls() 68 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 69 | return s 70 | 71 | def process_request(self, request, spider): 72 | # Called for each request that goes through the downloader 73 | # middleware. 74 | 75 | # Must either: 76 | # - return None: continue processing this request 77 | # - or return a Response object 78 | # - or return a Request object 79 | # - or raise IgnoreRequest: process_exception() methods of 80 | # installed downloader middleware will be called 81 | return None 82 | 83 | def process_response(self, request, response, spider): 84 | # Called with the response returned from the downloader. 85 | 86 | # Must either; 87 | # - return a Response object 88 | # - return a Request object 89 | # - or raise IgnoreRequest 90 | return response 91 | 92 | def process_exception(self, request, exception, spider): 93 | # Called when a download handler or a process_request() 94 | # (from other downloader middleware) raises an exception. 95 | 96 | # Must either: 97 | # - return None: continue processing this exception 98 | # - return a Response object: stops process_exception() chain 99 | # - return a Request object: stops process_exception() chain 100 | pass 101 | 102 | def spider_opened(self, spider): 103 | spider.logger.info('Spider opened: %s' % spider.name) 104 | 105 | 106 | 107 | 108 | 109 | 110 | # 配置 selenium 的使用方式 111 | import time 112 | from scrapy.http import HtmlResponse 113 | class VSeleniumMiddleware(object): 114 | @classmethod 115 | def from_crawler(cls, crawler): 116 | s = cls() 117 | crawler.signals.connect(s.spider_opened, signal=signals.spider_opened) 118 | crawler.signals.connect(s.spider_closed, signal=signals.spider_closed) 119 | return s 120 | def process_request(self, request, spider): 121 | try: 122 | self.webdriver.get(url=request.url) 123 | time.sleep(2) 124 | # 部分智能等待的代码，提高浏览器效率的处理 125 | # from selenium.webdriver.common.by import By 126 | # from selenium.webdriver.support import expected_conditions as EC 127 | # from selenium.webdriver.support.wait import WebDriverWait as wbw 128 | # locator = (By.XPATH, '//img[@class="focus-item-img"]') 129 | # # wbw(self.webdriver,10).until(EC.presence_of_element_located(locator)) # 判断某个元素是否被加到了dom树里 130 | # wbw(self.webdriver,10).until(EC.visibility_of_element_located(locator)) # 判断某个元素是否被添加到了dom里并且可见，即宽和高都大于0 131 | current_url = self.webdriver.current_url 132 | page_source = self.webdriver.page_source 133 | except Exception as e: 134 | return self._parse_selenium_temp_exceptions(request, spider, e) 135 | # 若是出现请求异常(验证码，或者重新登陆之类的处理)，请在这里判断 page_source 是否是异常情况，并在这里处理重新进行登录或其他 136 | h = HtmlResponse( 137 | url = current_url, 138 | headers = {'Selenium':'Selenium cannot get a certain headers, This is the information created automatically by middleware.'}, 139 | body = page_source, 140 | encoding = 'utf-8', 141 | request = request 142 | ) 143 | return h 144 | def process_response(self, request, response, spider): 145 | return response 146 | def spider_opened(self, spider): 147 | spider.logger.info('Spider %s opened: %s' % (self.__class__.__name__, spider.name)) 148 | self._open_webdriver() 149 | self._login() 150 | def spider_closed(self): 151 | if getattr(self, 'webdriver', None): self.webdriver.quit() 152 | def _parse_selenium_temp_exceptions(self, request, spider, e): 153 | stats = spider.crawler.stats 154 | if 'Failed to establish a new connection' in str(e): # 仅仅捕捉浏览器异常关闭的异常，尝试重启，并重新将请求发送到队列 155 | if getattr(self, 'restart_show_toggle', None) is None: 156 | self.restart_show_toggle = True 157 | if self.restart_show_toggle: 158 | self.restart_show_toggle = False # 让 Catch webdriver 仅显示一次 159 | spider.logger.info('Catch webdriver exception:{}, try to restart webdriver.'.format(e.__class__)) 160 | self._open_webdriver() 161 | retries = request.meta.get('selenium_retry_times', 0) + 1 # 在 selenium 异常无法重启处理情况下一个请求最多尝试共3次请求 162 | if retries <= 3: 163 | retryreq = request.copy() 164 | retryreq.meta['selenium_retry_times'] = retries 165 | retryreq.dont_filter = True 166 | stats.inc_value('selenium_retry/count') 167 | return retryreq 168 | else: 169 | stats.inc_value('selenium_retry/max_reached') 170 | spider.logger.info("Gave up selenium_retrying %(request)s (failed %(retries)d times)", 171 | {'request': request, 'retries': retries}) 172 | else: 173 | stats.inc_value('selenium_unknow_error/count') 174 | stats.inc_value('selenium_unknow_error/reason_count/%s' % e.__class__.__name__) 175 | import traceback 176 | spider.logger.info('\n'+traceback.format_exc().strip()) 177 | def _open_webdriver(self): # 该函数同时作为重启 webdriver 功能使用 178 | try: self.spider_closed() 179 | except: pass 180 | from selenium import webdriver 181 | option = webdriver.ChromeOptions() 182 | extset = ['enable-automation', 'ignore-certificate-errors'] 183 | ignimg = "profile.managed_default_content_settings.images" 184 | mobile = {'deviceName':'Galaxy S5'} 185 | option.add_argument("--disable-infobars") # 旧版本关闭“chrome正受到自动测试软件的控制”信息 186 | option.add_experimental_option("excludeSwitches", extset) # 新版本关闭“chrome正受到自动测试软件的控制”信息 187 | option.add_experimental_option("useAutomationExtension", False) # 新版本关闭“请停用以开发者模式运行的扩展程序”信息 188 | # option.add_experimental_option('mobileEmulation', mobile) # 是否使用手机模式打开浏览器 189 | # option.add_experimental_option("prefs", {ignore_image: 2}) # 开启浏览器时不加载图片(headless模式该配置无效) 190 | # option.add_argument('--start-maximized') # 开启浏览器时是否最大化(headless模式该配置无效) 191 | # option.add_argument('--headless') # 无界面打开浏览器 192 | # option.add_argument('--window-size=1920,1080') # 无界面打开浏览器时候只能用这种方式实现最大化 193 | # option.add_argument('--disable-gpu') # 禁用 gpu 硬件加速 194 | # option.add_argument("--auto-open-devtools-for-tabs") # 开启浏览器时候是否打开开发者工具(F12) 195 | # option.add_argument("--user-agent=Mozilla/5.0 HELL") # 修改 UA 信息 196 | # option.add_argument('--proxy-server=http://127.0.0.1:8888') # 增加代理 197 | self.webdriver = webdriver.Chrome(chrome_options=option) 198 | def _login(self): 199 | # 如果有登录处理，则写在这里 200 | pass -------------------------------------------------------------------------------- /vrequest/template/v/pipelines.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | # Define your item pipelines here 4 | # 5 | # Don't forget to add your pipeline to the ITEM_PIPELINES setting 6 | # See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html 7 | 8 | import scrapy 9 | from scrapy import Request, Selector 10 | from lxml import etree 11 | import re 12 | import json 13 | from urllib.parse import unquote, quote 14 | 15 | # 基础 item 中间件模板 16 | class VPipeline(object): 17 | def process_item(self, item, spider): 18 | print('\n==== 这里是动态增加的“下载中间件”部分 ====\n') 19 | return item 20 | 21 | # 图片下载 item 中间件 22 | import logging, hashlib 23 | from scrapy.pipelines.images import ImagesPipeline 24 | from scrapy.exceptions import DropItem 25 | class VImagePipeline(ImagesPipeline): 26 | def get_media_requests(self, item, info): 27 | yield Request(item['src'], meta=item) 28 | def file_path(self, request, response=None, info=None): 29 | url = request if not isinstance(request, Request) else request.url 30 | image_name = request.meta.get('image_name') # 使用 item中的 image_name 字段作为文件名进行存储，没有该字段则使用 url的 md5作为文件名存储 31 | image_name = re.sub(r'[/\\:\*"<>\|\?]', '_', image_name).strip()[:80] if image_name else hashlib.md5(url.encode()).hexdigest() 32 | return '%s.jpg' % image_name # 生成的图片文件名字，此处可用/符号增加多级分类路径（路径不存在则自动创建），使用 image_name 请注意重名可能性。 33 | def item_completed(self, results, item, info): # 判断下载是否成功 34 | k, v = results[0] 35 | item['image_download_stat'] = 'success' if k else 'fail' 36 | item['image_path'] = v['path'] if k else None # 保留文件名地址 37 | if not k: logging.info('download fail {}'.format(item)) 38 | else: logging.info('download success {}'.format(item)) 39 | return item 40 | 41 | # 文件下载 item 中间件 42 | import logging, hashlib 43 | from scrapy.pipelines.files import FilesPipeline 44 | class VFilePipeline(FilesPipeline): 45 | FILES_STORE = None 46 | def __init__(self, store_uri, download_func=None, settings=None): 47 | super().__init__(store_uri, download_func, settings) 48 | VFilePipeline.FILES_STORE = settings.get('FILES_STORE') 49 | def get_media_requests(self, item, info): 50 | headers = { 51 | "accept-encoding": "gzip, deflate", # auto delete br encoding. cos requests and scrapy can not decode it. 52 | "accept-language": "zh-CN,zh;q=0.9", 53 | "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8", 54 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.75 Safari/537.36" 55 | } 56 | item = item.copy() 57 | item['download_timeout'] = 180 # 下载单条文件的时间限制 58 | yield Request(item['src'], headers=headers, meta=item) 59 | def file_path(self, request, response=None, info=None): 60 | url = request if not isinstance(request, Request) else request.url 61 | file_name = request.meta.get('file_name') 62 | file_type = request.meta.get('file_type') 63 | file_name = re.sub(r'[/\\:\*"<>\|\?]', '_', file_name).strip()[:80] if file_name else hashlib.md5(url.encode()).hexdigest() 64 | if not file_type: 65 | file_type = request.url.rsplit('.', 1)[-1] 66 | file_type = file_type if '/' not in file_type else 'unknown' 67 | return '{}.{}'.format(file_name, file_type) 68 | def item_completed(self, results, item, info): # 判断下载是否成功 69 | k, v = results[0] 70 | item['file_download_stat'] = 'success' if k else 'fail' 71 | item['file_path'] = os.path.join(VFilePipeline.FILES_STORE, v['path']).replace('\\', '/') if k else None # 保留文件名地址 72 | if not k: logging.info('download fail {}'.format(item)) 73 | else: logging.info('download success {}'.format(item)) 74 | return item 75 | 76 | # 视频下载 item 中间件 77 | import os, sys 78 | import logging, hashlib, traceback 79 | from scrapy.exceptions import NotConfigured 80 | class VVideoPipeline(object): 81 | MEDIAS_STORE = None 82 | @classmethod 83 | def from_crawler(cls, crawler): 84 | s = cls(crawler.settings) 85 | return s 86 | def __init__(self, settings=None): 87 | VVideoPipeline.MEDIAS_STORE = settings.get('MEDIAS_STORE') 88 | if not VVideoPipeline.MEDIAS_STORE: 89 | err = 'before use VVideoPipeline. pls set MEDIAS_STORE first !!!' 90 | logging.error('\n--------------\n{}\n--------------'.format(err)) 91 | raise NotConfigured 92 | def process_item(self, item, spider): 93 | url = item['src'] 94 | localpage_ = os.path.dirname(os.path.realpath(sys.argv[0])) # 确保下载路径为“该脚本”或“被pytinstaller打包时候的工具”路径下的video文件夹 95 | localpage = os.path.join(localpage_, VVideoPipeline.MEDIAS_STORE) 96 | try: 97 | ### 【you-get】 98 | # import you_get.common 99 | # you_get.common.skip_existing_file_size_check = True # 防止发现重复视频时会强制要求输入“是否覆盖”，卡住程序，默认不覆盖 100 | # you_get.common.any_download(url, output_dir=localpage, merge=True, info_only=False) 101 | ### 【youtube-dl】 （推荐使用这个，因为这个在存储的文件名字的自定义存储上会更强） 102 | from youtube_dl import YoutubeDL 103 | file_name, file_type = item.get('file_name'), item.get('file_type') 104 | fpath = '{}/%(title)s.%(ext)s'.format(item.get('file_path').strip('/\\')) if item.get('file_path') else '%(title)s.%(ext)s' 105 | fpath = os.path.join(localpage, fpath).replace('\\', '/') 106 | fpath = fpath.replace('%(title)s', file_name) if file_name else fpath 107 | fpath = fpath.replace('%(ext)s', file_type) if file_type else fpath 108 | ytdl = YoutubeDL({'outtmpl': fpath, 'ffmpeg_location':None}) # 如果已配置ffmpeg环境则不用修改 109 | info = ytdl.extract_info(url, download=True) 110 | dpath = {} 111 | if '%(title)s' in fpath: dpath['title'] = info['title'] 112 | if '%(ext)s' in fpath: dpath['ext'] = info['ext'] 113 | path = fpath % dpath 114 | 115 | item['media_download_stat'] = 'success' 116 | item['media_path'] = path.replace(localpage_.replace('\\', '/'), '.') # 保留文件名地址 117 | logging.info('download success {}'.format(item)) 118 | except: 119 | item['media_download_stat'] = 'fail' 120 | item['media_path'] = None 121 | logging.info('download fail {}'.format(item)) 122 | logging.info('download reason {}'.format(traceback.format_exc())) 123 | return item 124 | 125 | # 数据库上传 item 中间件(不考虑字段类型处理，每个字段统统使用 MEDIUMTEXT 类型存储 json.dumps 后的 value) 126 | # 如果有数据库字段类型的个性化处理，请非常注意的修改 insert_item 和 init_database 两个函数中对于字段类型的初始化、插入的处理，process_item无需修改。 127 | import hmac, logging, traceback 128 | from twisted.enterprise import adbapi 129 | class VMySQLPipeline(object): 130 | dbn = {} 131 | def process_item(self, item, spider): 132 | mysql_config = item.pop('__mysql__', None) # 存储时自动删除配置 133 | if mysql_config and item: 134 | if type(mysql_config) is dict: 135 | table = mysql_config.pop('table', None) 136 | db = mysql_config.get('db', None) or 'vrequest' 137 | mysql_config.setdefault('charset','utf8mb4') 138 | mysql_config.setdefault('db', db) 139 | dbk = hmac.new(b'',json.dumps(mysql_config, sort_keys=True).encode(),'md5').hexdigest() 140 | if dbk not in self.dbn: 141 | self.dbn[dbk] = adbapi.ConnectionPool('pymysql', **mysql_config) 142 | self.init_database(self.dbn[dbk], mysql_config, db, table, item) 143 | self.dbn[dbk].runInteraction(self.insert_item, db, table, item) 144 | return item 145 | else: 146 | raise TypeError('Unable Parse mysql_config type:{}'.format(type(mysql_config))) 147 | else: 148 | return item 149 | def insert_item(self, conn, db, table, item): 150 | table_sql = ''.join(["'{}',".format(json.dumps(v, ensure_ascii=False).replace("'","\\'")) for k,v in item.items()]) 151 | insert_sql = 'INSERT INTO `{}`.`{}` VALUES({})'.format(db, table, table_sql.strip(',')) 152 | try: 153 | conn.execute(insert_sql) 154 | logging.info('insert sql success') 155 | except Exception as e: 156 | logging.info('insert sql fail: {}'.format(insert_sql)) 157 | logging.error(traceback.format_exc()) 158 | def init_database(self, pool, mysql_config, db, table, item): 159 | # 需要注意的是，在一些非常老的版本的mysql 里面并不支持 utf8mb4。这是 mysql 的设计缺陷，赶紧使用大于 5.5 版本的 mysql ! 160 | # 创建db，创建表名，所有字段都以 MEDIUMTEXT 存储，用 json.dumps 保证了数据类型也能存储，后续取出时只需要每个值 json.loads 这样就能获取数据类型 161 | # 例如一个数字类型 123 -> json.dumps -> '123' -> json.loads -> 123，统一类型存储，取出时又能保证数据类型，这种处理会很方便 162 | # MEDIUMTEXT 最大能使用16M 的长度，所以对于一般的 html 文本也非常足够。如有自定义字段类型的需求，请注意修改该处。 163 | db, charset = mysql_config.pop('db'), mysql_config.get('charset') 164 | try: 165 | conn = pool.dbapi.connect(**mysql_config) 166 | cursor = conn.cursor() 167 | table_sql = ''.join(['`{}` MEDIUMTEXT NULL,'.format(str(k)) for k,v in item.items()]) 168 | cursor.execute('Create Database If Not Exists {} Character Set {}'.format(db, charset)) 169 | cursor.execute('Create Table If Not Exists `{}`.`{}` ({})'.format(db, table, table_sql.strip(','))) 170 | conn.commit(); cursor.close(); conn.close() 171 | except Exception as e: 172 | traceback.print_exc() 173 | 174 | # 阿里 Oss 文件上传中间件模板 175 | # 依赖 pip install oss2 176 | class VOssPipeline: 177 | BUCKET_STORE = None 178 | @classmethod 179 | def from_crawler(cls, crawler): 180 | s = cls() 181 | import oss2 182 | aid = 'kkkkkkkkkkkkkkkkkkkkkkkk' 183 | ack = 'vvvvvvvvvvvvvvvvvvvvvvvvvvvvvv' 184 | enp = 'http://oss-cn-hangzhou.aliyuncs.com' 185 | _bucket = '' 186 | VOssPipeline.BUCKET_STORE = oss2.Bucket(oss2.Auth(aid,ack), enp, _bucket) 187 | return s 188 | def process_item(self, item, spider): 189 | # 示例: 用于将下载到的图片上传到Oss的代码如下 190 | # ipath = item.get('image_path') 191 | # if ipath and os.path.isfile(ipath): self.update_data(ipath, ipath) 192 | return item 193 | def update_data(self, object_name, localfile_name): 194 | VOssPipeline.BUCKET_STORE.put_object_from_file(object_name, localfile_name) -------------------------------------------------------------------------------- /vrequest/pyjspack.py: -------------------------------------------------------------------------------- 1 | 2 | comment = r''' 3 | // 功能： 4 | // 将需要的文件压缩打包成 es5 语法的，可以打包库文件，这样使用起来就方便很多了。 5 | 6 | // 安装： 7 | // 选择或创建一个空文件夹，然后在文件夹路径打开命令行使用下面的命令安装环境和打包js脚本。 8 | // npm install -g cnpm --registry=https://registry.npm.taobao.org 9 | // npm init -y 10 | // cnpm install webpack-cli webpack -S 11 | // cnpm install babel-cli babel-preset-env -S 12 | // 一行安装 13 | // npm install -g cnpm --registry=https://registry.npm.taobao.org && npm init -y && cnpm install webpack-cli webpack -S && cnpm install babel-cli babel-preset-env -S 14 | 15 | // 打包： 16 | // npx webpack --entry="./index.js" 17 | // npx babel dist/main.js -d es5 --presets=babel-preset-env 18 | // 一行打包 19 | // npx webpack --entry="./index.js" && npx babel dist/main.js -d es5 --presets=babel-preset-env 20 | 21 | // 在 es5/dist 文件夹下找 main.js 就是目标文件了。 22 | // 可以新建一个文件夹命名为 index.js 输入下面两行内容。用上面的方式打包，后续可直接作单脚本使用。 23 | // const CryptoJS = require('crypto-js') 24 | // window.CryptoJS = CryptoJS 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | // 额外的常用的一些工具代码 34 | // sublime 工具配置 35 | { 36 | "shell": true, 37 | "encoding": "utf8", 38 | "cmd": ["taskkill", "/F", "/IM", "node.exe", "&", "node", "$file"], 39 | "variants":[ 40 | { "name": "node inspect", 41 | "shell_cmd":"taskkill /F /IM \"node.exe\" & node --inspect-brk \"$file\"", 42 | }, 43 | ] 44 | } 45 | 46 | // 快捷挂钩 js 代码 47 | _window = typeof global=='undefined'?window:global 48 | _window._vPxy = _window._vPxy?_window._vPxy:function(G, M, F, prefix){ 49 | var util = typeof global=='undefined'?undefined:require('util'); 50 | _window._vLine = _window._vLine?_vLine:function _vLine(o, lines){ 51 | if (util){ 52 | var lines = lines || 5 53 | var v = util.inspect(o) 54 | var c = ` ... ` 55 | function make_space_gap(num){ 56 | return _spacearr[num] 57 | } 58 | function weak_output(v, num){ 59 | return ('\n' + v + '\n').split('\n').slice(0, num).concat(c).map(function(e, i){return make_space_gap(_global_gap)+e}).join('\n') 60 | } 61 | function count_lines(v){ 62 | return (v.match(/\n/g)||'').length 63 | } 64 | if (count_lines(v) > lines){ 65 | return weak_output(v, lines) 66 | }else{ 67 | return v 68 | } 69 | }else{ 70 | if (typeof o == 'symbol'){ 71 | return o.toString() 72 | } 73 | return o 74 | } 75 | } 76 | _window._global_gap = _window._global_gap?_window._global_gap:80 77 | _window._print = _window._print?_window._print:console.log 78 | _window.start = _window.start?_window.start:1 79 | _window.ostart = _window.ostart?_window.ostart:0 80 | _window.ret_detail = _window.ret_detail?_window.ret_detail:1 81 | _window._isArray = _window._isArray?_window._isArray:Array.isArray 82 | _window._stringify = _window._stringify?_window._stringify:JSON.stringify 83 | _window._spacearr = _window._spacearr?_window._spacearr:Array(100).fill(0).map(function(e,i){return Array(i).fill(' ').join('')}) 84 | _window._ignore_log = _window._ignore_log?_window._ignore_log:[] 85 | _window._vLog = _window._vLog?_window._vLog:function _vLog(F, O, P){ 86 | if (start){ 87 | for (var i = 0; i < _ignore_log.length; i++) { 88 | var ig = _ignore_log[i] 89 | if (ig[0] == F && ig[1] == O && (ig[2]===undefined || (ig[2]===P && P !== undefined))){ 90 | return 91 | } 92 | } 93 | _print.apply(_print, [].slice.call(arguments, 3)) 94 | } 95 | } 96 | _window.myparselog = _window.myparselog?_window.myparselog:function myparselog(V){ 97 | ostart = start 98 | start = 0 99 | var r = typeof V=='string'? 100 | _stringify(V.length > 200?V.slice(0,200) + '... ':V) 101 | : 102 | typeof V=='number'?V: 103 | typeof V=='function'?V: 104 | typeof V=='undefined'?undefined: 105 | typeof V=='boolean'?V: 106 | V===null?null: 107 | ret_detail?_vLine(V): 108 | _isArray(V)?_stringify(V): 109 | `` 110 | start = ostart 111 | return r 112 | } 113 | _window._v_hidden_Inject = _window._v_hidden_Inject?_window._v_hidden_Inject:function(){return _vDoDefault} 114 | _window._vDoDefault = _window._vDoDefault?_window._vDoDefault:Symbol('undefined') 115 | _window._vInject = _window._vInject?_window._vInject:function(){return _vDoDefault} 116 | var prefix = prefix?_spacearr[prefix]:'' 117 | var _vLog = (typeof global=='undefined'?window:global)._vLog || _print 118 | function LS(T, M, F, L){ 119 | var pr = prefix + `[Proxy] ${M}[${T.constructor.name}].(Prxoy)${F} ==>> ${L?L:''}` 120 | var taillen = _global_gap - pr.length 121 | pr += _spacearr[(taillen > 0)?taillen:0] 122 | return pr 123 | } 124 | return new Proxy(G, { 125 | get: function(T, P, R){ 126 | var Rt; 127 | var In = _v_hidden_Inject(F, 'get', arguments); 128 | try{ 129 | if (In !== _vDoDefault){ 130 | Rt = In 131 | }else{ 132 | Rt = Reflect.get(T, P, R) 133 | } 134 | }catch(e){ 135 | if (P !== Symbol.unscopables){ _vLog(F, 'get', P, LS(G, M, 'get', P), '[GET ERROR]'); } // get 获取出错时，需要输出究竟是因为哪个参数报错的 136 | throw e; 137 | } 138 | if (P !== Symbol.unscopables){ _vLog(F, 'get', P, LS(G, M, 'get', myparselog(P)), myparselog(Rt)); } 139 | return Rt; 140 | }, 141 | has: function(T, P){ 142 | var Rt; 143 | var In = _v_hidden_Inject(F, 'has', arguments); 144 | if (In !== _vDoDefault){ 145 | Rt = In 146 | }else{ 147 | Rt = Reflect.has(T, P) 148 | } 149 | if (T !== _interceptor){ _vLog(F, 'has', P, LS(G, M, 'has', P), Rt); } 150 | return Rt; 151 | }, 152 | getPrototypeOf: function(T){ 153 | var Rt; 154 | var In = _v_hidden_Inject(F, 'getPrototypeOf', undefined, arguments); 155 | if (In !== _vDoDefault){ 156 | Rt = In 157 | }else{ 158 | Rt = Reflect.getPrototypeOf(T) 159 | } 160 | _vLog(F, 'getPrototypeOf', undefined, LS(G, M, 'getPrototypeOf'), myparselog(T) ); 161 | return Rt; 162 | }, 163 | set: function(T, P, V, R){ 164 | var Rt; 165 | var In = _v_hidden_Inject(F, 'set', arguments); 166 | if (In !== _vDoDefault){ 167 | Rt = In 168 | }else{ 169 | Rt = Reflect.set(T, P, V, R) 170 | } 171 | if (P != '__cilame__'){_vLog(F, 'set', P, LS(G, M, 'set', myparselog(P)), myparselog(V) )} 172 | return Rt; 173 | }, 174 | apply: function(T, A, L){ 175 | var Rt; 176 | var In = _v_hidden_Inject(F, 'apply', arguments); 177 | if (In !== _vDoDefault){ 178 | Rt = In 179 | } else{ 180 | Rt = Reflect.apply(T, A, L) 181 | } 182 | _vLog(F, 'apply', undefined, LS(G, M, 'apply', myparselog(L)), myparselog(Rt) ); 183 | return Rt; 184 | }, 185 | deleteProperty: function(T, P){ 186 | var Rt; 187 | var In = _v_hidden_Inject(F, 'deleteProperty', arguments); 188 | if (In !== _vDoDefault){ 189 | Rt = In 190 | }else{ 191 | Rt = Reflect.deleteProperty(T, P) 192 | } 193 | _vLog(F, 'deleteProperty', P, LS(G, M, 'deleteProperty', P), Rt); 194 | return Rt; 195 | }, 196 | setPrototypeOf: function(T, P){ 197 | var Rt; 198 | var In = _v_hidden_Inject(F, 'setPrototypeOf', arguments); 199 | if (In !== _vDoDefault){ 200 | Rt = In 201 | }else{ 202 | Rt = Reflect.setPrototypeOf(T, P) 203 | } 204 | _vLog(F, 'setPrototypeOf', P, LS(G, M, 'setPrototypeOf'), T, P, Rt); 205 | return Rt; 206 | }, 207 | ownKeys: function(T){ 208 | var Rt; 209 | var In = _v_hidden_Inject(F, 'ownKeys', arguments); 210 | if (In !== _vDoDefault){ 211 | Rt = In 212 | }else{ 213 | Rt = Reflect.ownKeys(T) 214 | } 215 | _vLog(F, 'ownKeys', undefined, LS(G, M, 'ownKeys'), myparselog(T), myparselog(Rt) ); 216 | return Rt; 217 | }, 218 | construct: function(T, L, N){ 219 | var Rt; 220 | var In = _v_hidden_Inject(F, 'construct', arguments); 221 | if (In !== _vDoDefault){ 222 | Rt = In 223 | } else{ 224 | Rt = Reflect.construct(T, L, N) 225 | } 226 | _vLog(F, 'construct', undefined, LS(G, M, 'construct', myparselog(L)), myparselog(Rt) ); 227 | return Rt; 228 | }, 229 | isExtensible: function(T){ 230 | var Rt; 231 | var In = _v_hidden_Inject(F, 'isExtensible', arguments); 232 | if (In !== _vDoDefault){ 233 | Rt = In 234 | }else{ 235 | Rt = Reflect.isExtensible(T) 236 | } 237 | _vLog(F, 'isExtensible', undefined, LS(G, M, 'isExtensible'), Rt); 238 | return Rt; 239 | }, 240 | defineProperty: function(T, P, A){ 241 | var Rt; 242 | var In = _v_hidden_Inject(F, 'defineProperty', arguments); 243 | if (In !== _vDoDefault){ 244 | Rt = In 245 | }else{ 246 | Rt = Reflect.defineProperty(T, P, A) 247 | } 248 | _vLog(F, 'defineProperty', P, LS(G, M, 'defineProperty', P), T, A, Rt); 249 | return Rt; 250 | }, 251 | preventExtensions: function(T){ 252 | var Rt; 253 | var In = _v_hidden_Inject(F, 'preventExtensions', arguments); 254 | if (In !== _vDoDefault){ 255 | Rt = In 256 | }else{ 257 | Rt = Reflect.preventExtensions(T) 258 | } 259 | _vLog(F, 'preventExtensions', undefined, LS(G, M, 'preventExtensions'), Rt); 260 | return Rt; 261 | }, 262 | getOwnPropertyDescriptor: function(T, P){ 263 | var Rt; 264 | var In = _v_hidden_Inject(F, 'getOwnPropertyDescriptor', arguments); 265 | if (In !== _vDoDefault){ 266 | Rt = In 267 | }else{ 268 | Rt = Reflect.getOwnPropertyDescriptor(T, P) 269 | } 270 | _vLog(F, 'getOwnPropertyDescriptor', P, LS(G, M, 'getOwnPropertyDescriptor', myparselog(P)), myparselog(T), Rt ); 271 | return Rt; 272 | }, 273 | }) 274 | } 275 | _window._v_hidden_Inject = _window._v_hidden_Inject?_window._v_hidden_Inject:function(F, O, args){ 276 | var r = _window._vInject(F, O, args) 277 | return (r !== _vDoDefault)?r:_vDoDefault 278 | } 279 | _vInject = function(F, O, args){ 280 | if (F == 'XXX'){ 281 | if (O == 'get'){ 282 | // 被 _vPxy 代理的所有对象都会走这里，_vPxy 函数传入的第三个参数就是这里的 F ，操作就是 O ，操作所用的参数就是 args 283 | // 你可以在这个函数内尽情的修改获取某些对象的操作 284 | _print('==========') 285 | // return undefined 286 | } 287 | } 288 | return _vDoDefault // 返回默认操作，不返回这个值，所有的操作都将会变成 undefined，with 内的所有操作都将被拦截。 289 | } 290 | 291 | x = _vPxy({}, "XX", "XXX", 4) 292 | x.asdf 293 | 294 | 295 | 296 | 297 | ''' -------------------------------------------------------------------------------- /vrequest/main.js: -------------------------------------------------------------------------------- 1 | function FormatMember(path) { 2 | // _0x19882c['removeCookie']['toString']() 3 | // | 4 | // | 5 | // | 6 | // v 7 | // _0x19882c.removeCookie.toString() 8 | var curNode = path.node; 9 | if(!t.isStringLiteral(curNode.property)) 10 | return; 11 | if(curNode.computed === undefined || !curNode.computed === true) 12 | return; 13 | if (!/[a-zA-Z_$][0-9a-zA-Z_$]*/.test(curNode.property.value)) 14 | return; 15 | curNode.property = t.identifier(curNode.property.value); 16 | curNode.computed = false; 17 | } 18 | 19 | function TransCondition(path) { 20 | // a = m?11:22; 21 | // | 22 | // | 23 | // | 24 | // v 25 | // m ? a = 11 : a = 22; 26 | let {test, consequent, alternate} = path.node; 27 | const ParentPath = path.parentPath; 28 | if (ParentPath.isAssignmentExpression()) { 29 | let {operator, left} = ParentPath.node; 30 | if (operator === "=") { 31 | consequent = t.AssignmentExpression("=", left, consequent) 32 | alternate = t.AssignmentExpression("=", left, alternate) 33 | ParentPath.replaceWith(t.conditionalExpression(test, consequent, alternate)) 34 | } 35 | } 36 | } 37 | function ConditionToIf(path) { 38 | // m ? a = 11 : a = 22; 39 | // | 40 | // | 41 | // | 42 | // v 43 | // if (m) { 44 | // a = 11; 45 | // } else { 46 | // a = 22; 47 | // } 48 | let {expression} = path.node; 49 | if(!t.isConditionalExpression(expression)) return; 50 | let {test, consequent, alternate} = expression; 51 | path.replaceWith(t.ifStatement( 52 | test, 53 | t.blockStatement([t.expressionStatement(consequent),]), 54 | t.blockStatement([t.expressionStatement(alternate),]) 55 | )); 56 | } 57 | 58 | function ConditionVarToIf(path) { 59 | // var m ? a = 11 : a = 22; 60 | // | 61 | // | 62 | // | 63 | // v 64 | // if (m) { 65 | // var a = 11; 66 | // } else { 67 | // var a = 22; 68 | // } 69 | let {id, init} = path.node; 70 | if (!t.isConditionalExpression(init)) return; 71 | const ParentPath = path.parentPath; 72 | const ParentNode = path.parent; 73 | if (!t.isVariableDeclaration(ParentNode)) return; 74 | if (t.isForStatement(ParentPath.parentPath)) return; 75 | let kind = ParentNode.kind; 76 | let {test, consequent, alternate} = init; 77 | ParentPath.replaceWith(t.ifStatement( 78 | test, 79 | t.blockStatement([t.variableDeclaration(kind, [t.variableDeclarator(id, consequent)]),]), 80 | t.blockStatement([t.variableDeclaration(kind, [t.variableDeclarator(id, alternate)]),]) 81 | )); 82 | } 83 | 84 | function RemoveComma(path) { 85 | // a = 1, b = ddd(), c = null; 86 | // | 87 | // | 88 | // | 89 | // v 90 | // a = 1; 91 | // b = ddd(); 92 | // c = null; 93 | let {expression} = path.node 94 | if (!t.isSequenceExpression(expression)) 95 | return; 96 | let body = [] 97 | expression.expressions.forEach( 98 | express => { 99 | body.push(t.expressionStatement(express)) 100 | } 101 | ) 102 | path.replaceInline(body) 103 | } 104 | 105 | function RemoveVarComma(path) { 106 | // var a = 1, b = ddd(), c = null; 107 | // | 108 | // | 109 | // | 110 | // v 111 | // var a = 1; 112 | // var b = ddd(); 113 | // var c = null; 114 | let {kind, declarations} = path.node; 115 | if (declarations.length < 2) return; 116 | if (t.isForStatement(path.parentPath)) return; 117 | temp = []; 118 | declarations.forEach( 119 | VariableDeclarator => { 120 | temp.push(t.variableDeclaration(kind, [VariableDeclarator])) 121 | } 122 | ) 123 | path.replaceInline(temp); 124 | } 125 | 126 | function MergeObj(path) { 127 | // var _0xb28de8 = {}; 128 | // _0xb28de8["abcd"] = function(_0x22293f, _0x5a165e) { 129 | // return _0x22293f == _0x5a165e; 130 | // }; 131 | // _0xb28de8.dbca = function(_0xfbac1e, _0x23462f, _0x556555) { 132 | // return _0xfbac1e(_0x23462f, _0x556555); 133 | // }; 134 | // _0xb28de8.aaa = function(_0x57e640) { 135 | // return _0x57e640(); 136 | // }; 137 | // _0xb28de8["bbb"] = "eee"; 138 | // var _0x15e145 = _0xb28de8; 139 | // | 140 | // | 141 | // | 142 | // v 143 | // var _0xb28de8 = { 144 | // "abcd": function (_0x22293f, _0x5a165e) { 145 | // return _0x22293f == _0x5a165e; 146 | // }, 147 | // "dbca": function (_0xfbac1e, _0x23462f, _0x556555) { 148 | // return _0xfbac1e(_0x23462f, _0x556555); 149 | // }, 150 | // "aaa": function (_0x57e640) { 151 | // return _0x57e640(); 152 | // }, 153 | // "bbb": "eee" 154 | // }; 155 | const {id, init} = path.node; 156 | if (!t.isObjectExpression(init)) // 判断是否是定义对象 157 | return; 158 | let name = id.name; 159 | let properties = init.properties; 160 | let scope = path.scope; 161 | let binding = scope.getBinding(name); 162 | if (!binding || binding.constantViolations.length > 0) { // 确认该对象没有被多次定义 163 | return; 164 | } 165 | let paths = binding.referencePaths; 166 | scope.traverse(scope.block, { 167 | AssignmentExpression: function(_path) { 168 | const left = _path.get("left"); 169 | const right = _path.get("right"); 170 | if (!left.isMemberExpression()) 171 | return; 172 | const object = left.get("object"); 173 | const property = left.get("property"); 174 | if (object.isIdentifier({name: name}) && property.isStringLiteral() && _path.scope == scope) { 175 | properties.push(t.ObjectProperty(t.valueToNode(property.node.value), right.node)); 176 | _path.remove(); 177 | } 178 | if (object.isIdentifier({name: name}) && property.isIdentifier() && _path.scope == scope) { 179 | properties.push(t.ObjectProperty(t.valueToNode(property.node.name), right.node)); 180 | _path.remove(); 181 | } 182 | } 183 | }) 184 | paths.map(function(refer_path) { 185 | let bindpath = refer_path.parentPath; 186 | if (!t.isVariableDeclarator(bindpath.node)) return; 187 | let bindname = bindpath.node.id.name; 188 | bindpath.scope.rename(bindname, name, bindpath.scope.block); 189 | bindpath.remove(); 190 | }); 191 | } 192 | 193 | function CallToStr(path) { 194 | // var _0xb28de8 = { 195 | // "abcd": function(_0x22293f, _0x5a165e) { 196 | // return _0x22293f == _0x5a165e; 197 | // }, 198 | // "dbca": function(_0xfbac1e, _0x23462f, _0x556555) { 199 | // return _0xfbac1e(_0x23462f, _0x556555); 200 | // }, 201 | // "aaa": function(_0x57e640) { 202 | // return _0x57e640(); 203 | // }, 204 | // "bbb": "eee" 205 | // }; 206 | // var aa = _0xb28de8["abcd"](123, 456); 207 | // var bb = _0xb28de8["dbca"](bcd, 11, 22); 208 | // var cc = _0xb28de8["aaa"](dcb); 209 | // var dd = _0xb28de8["bbb"]; 210 | // | 211 | // | 212 | // | 213 | // v 214 | // var aa = 123 == 456; 215 | // var bb = bcd(11, 22); 216 | // var cc = dcb(); 217 | // var dd = "eee"; 218 | var node = path.node; 219 | if (!t.isObjectExpression(node.init)) // 判断是否使用对象 220 | return; 221 | var objPropertiesList = node.init.properties; 222 | if (objPropertiesList.length == 0) 223 | return; 224 | var objName = node.id.name; 225 | // 是否可删除该对象：发生替换时可删除，否则不删除 226 | var del_flag = false 227 | objPropertiesList.forEach(prop => { 228 | var key = prop.key.value; 229 | if(t.isFunctionExpression(prop.value)) { 230 | var retStmt = prop.value.body.body[0]; 231 | var fnPath = path.getFunctionParent() || path.scope.path; 232 | fnPath.traverse({ 233 | CallExpression: function (_path) { 234 | var _node = _path.node.callee; 235 | if (!t.isMemberExpression(_path.node.callee)) 236 | return; 237 | if (!t.isIdentifier(_node.object) || _node.object.name !== objName) 238 | return; 239 | if (!(t.isStringLiteral(_node.property) || t.isIdentifier(_node.property))) 240 | return; 241 | if (!(_node.property.value == key || _node.property.name == key)) 242 | return; 243 | var args = _path.node.arguments; 244 | // 二元运算, 逻辑运算, 函数调用 245 | if (t.isBinaryExpression(retStmt.argument) && args.length===2) { 246 | _path.replaceWith(t.binaryExpression(retStmt.argument.operator, args[0], args[1])); 247 | } 248 | else if(t.isLogicalExpression(retStmt.argument) && args.length==2) { 249 | _path.replaceWith(t.logicalExpression(retStmt.argument.operator, args[0], args[1])); 250 | } 251 | else if(t.isCallExpression(retStmt.argument) && t.isIdentifier(retStmt.argument.callee)) { 252 | _path.replaceWith(t.callExpression(args[0], args.slice(1))) 253 | } 254 | del_flag = true; 255 | } 256 | }) 257 | } 258 | else if (t.isStringLiteral(prop.value)){ 259 | var retStmt = prop.value.value; 260 | var fnPath = path.getFunctionParent() || path.scope.path; 261 | fnPath.traverse({ 262 | MemberExpression:function (_path) { 263 | var _node = _path.node; 264 | if (!t.isIdentifier(_node.object) || _node.object.name !== objName) 265 | return; 266 | if (!(t.isStringLiteral(_node.property) || t.isIdentifier(_node.property))) 267 | return; 268 | if (!(_node.property.value == key || _node.property.name == key)) 269 | return; 270 | _path.replaceWith(t.stringLiteral(retStmt)) 271 | del_flag = true; 272 | } 273 | }) 274 | } 275 | }); 276 | if (del_flag) { 277 | // 如果发生替换，则删除该对象, 该处可能出问题，因为字典的内容未必会饱和使用 278 | path.remove(); 279 | } 280 | } 281 | 282 | function delExtra(path) { 283 | // ['\x49\x63\x4b\x72\x77\x70\x2f\x44\x6c\x67\x3d\x3d',0x123]; 284 | // | 285 | // | 286 | // | 287 | // v 288 | // ["IcKrwp/Dlg==", 291]; 289 | delete path.node.extra; 290 | } 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | function get_ob_enc(ast) { 305 | var first_idx = 0 306 | for (var i = 0; i < ast.program.body.length; i++) { 307 | if (ast.program.body[i].type != 'EmptyStatement'){ 308 | first_idx = i; 309 | break 310 | } 311 | } 312 | var decrypt_code = ast.program.body.slice(first_idx, first_idx+3) 313 | var rest_code = ast.program.body.slice(first_idx+3) 314 | ast.program.body = decrypt_code 315 | var {code} = generator(ast, { 316 | compact: true 317 | }) 318 | global_code = code 319 | decryptStr = decrypt_code[2].declarations[0].id.name 320 | var flag = true 321 | const visitor = { 322 | "ExpressionStatement"(path) { 323 | path.traverse({ 324 | "StringLiteral"(_path) { 325 | delete _path.node.extra 326 | }, 327 | MemberExpression: FormatMember 328 | }) 329 | var code = path.toString() 330 | if (flag && code.indexOf("atob") != -1) { 331 | atob_node = path.node 332 | flag = false 333 | } 334 | } 335 | } 336 | traverse(ast, visitor) 337 | ast.program.body = [atob_node] 338 | var {code} = generator(ast, { 339 | jsescOption: { 340 | minimal: true, 341 | } 342 | }) 343 | const comment = "// atob函数，后面可能会判断其是否存在，勿删！" 344 | atob_code = comment + "\n!" + code + "\n" 345 | ast.program.body = rest_code 346 | return ast 347 | } 348 | 349 | function pas_ob_enc(ast) { 350 | eval(global_code) 351 | traverse(ast, { 352 | CallExpression: funToStr, 353 | StringLiteral: delExtra, 354 | NumericLiteral: delExtra, 355 | }) 356 | return ast; 357 | function funToStr(path) { 358 | var node = path.node; 359 | if (!t.isIdentifier(node.callee, {name: decryptStr})) 360 | return; 361 | let value = eval(path.toString()) 362 | // console.log("还原前：" + path.toString(), "还原后：" + value); 363 | path.replaceWith(t.valueToNode(value)); 364 | } 365 | function delExtra(path) { 366 | delete path.node.extra; 367 | } 368 | } 369 | 370 | function ReplaceWhile(path) { 371 | var node = path.node; 372 | if (!(t.isBooleanLiteral(node.test) || t.isUnaryExpression(node.test))) 373 | return; 374 | if (!(node.test.prefix || node.test.value)) 375 | return; 376 | if (!t.isBlockStatement(node.body)) 377 | return; 378 | var body = node.body.body; 379 | if (!t.isSwitchStatement(body[0]) || !t.isMemberExpression(body[0].discriminant) || !t.isBreakStatement(body[1])) 380 | return; 381 | var swithStm = body[0]; 382 | var arrName = swithStm.discriminant.object.name; 383 | var argName = swithStm.discriminant.property.argument.name 384 | let arr = []; 385 | let all_presibling = path.getAllPrevSiblings(); 386 | all_presibling.forEach(pre_path => { 387 | const {declarations} = pre_path.node; 388 | let {id, init} = declarations[0] 389 | if (arrName == id.name) { 390 | arr = init.callee.object.value.split('|'); 391 | pre_path.remove() 392 | } 393 | if (argName == id.name) { 394 | pre_path.remove() 395 | } 396 | }) 397 | var caseList = swithStm.cases; 398 | var resultBody = []; 399 | arr.map(targetIdx => { 400 | var targetBody = caseList[targetIdx].consequent; 401 | if (t.isContinueStatement(targetBody[targetBody.length - 1])) 402 | targetBody.pop(); 403 | resultBody = resultBody.concat(targetBody) 404 | }); 405 | path.replaceInline(resultBody); 406 | } 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | 419 | 420 | 421 | 422 | 423 | 424 | 425 | 426 | 427 | 428 | 429 | function muti_process_defusion(jscode){ 430 | var ast = parser.parse(jscode); 431 | 432 | // ob 解混淆处理部分 433 | // ast = get_ob_enc(ast) 434 | // ast = pas_ob_enc(ast) 435 | // traverse(ast, {VariableDeclarator: {exit: MergeObj},}); // 可能出问题（不可通用） 436 | // traverse(ast, {VariableDeclarator: {exit: CallToStr},}); // 可能出问题（不可通用） 437 | // traverse(ast, {WhileStatement: {exit: [ReplaceWhile]},}); // 反控制流平坦化 438 | 439 | // 通用解混淆部分 440 | traverse(ast, {StringLiteral: delExtra,}) // 清理二进制显示内容 441 | traverse(ast, {NumericLiteral: delExtra,}) // 清理二进制显示内容 442 | traverse(ast, {ConditionalExpression: TransCondition,}); // 三元表达式 443 | traverse(ast, {ExpressionStatement: ConditionToIf,}); // 三元表达式转换成if 444 | traverse(ast, {VariableDeclarator: ConditionVarToIf,}); // 赋值语句的 三元表达式转换成if 445 | traverse(ast, {ExpressionStatement: RemoveComma,}); // 逗号表达式转换 446 | traverse(ast, {VariableDeclaration: RemoveVarComma,}); // 赋值语句的 逗号表达式转换 447 | traverse(ast, {MemberExpression: FormatMember,}); // obj['func1']['func2']() --> obj.func1.func2() 448 | var { code } = generator(ast, { jsescOption: { minimal: true, } }); 449 | return code; 450 | } 451 | // const fs = require('fs'); 452 | // var jscode = fs.readFileSync("./source.js", { 453 | // encoding: "utf-8" 454 | // }); 455 | // code = muti_process_defusion(jscode); 456 | // console.log(code); 457 | // fs.writeFileSync('./code.js', code, { 458 | // encoding: "utf-8" 459 | // }) -------------------------------------------------------------------------------- /vrequest/pymini_yolo.py: -------------------------------------------------------------------------------- 1 | # 开发于 python3，仅需要下面两个第三方依赖，训练的数据为 labelimg 标注型的数据。 2 | # 依赖 pytorch：（官网找安装方式）开发使用版本为 torch-1.4.0-cp36-cp36m-win_amd64.whl 3 | # 依赖 opencv： （pip install opencv-contrib-python==3.4.1.15） 4 | # 其实这里的 opencv 版本不重要，py3能用就行，只是个人喜欢这个版本，因为能用sift图像检测，稳。 5 | 6 | 7 | import cv2 8 | import numpy as np 9 | import torch 10 | 11 | import os 12 | import math 13 | import xml.dom.minidom 14 | 15 | # 读取voc格式文件 16 | def read_voc_xml(file, islist=True): 17 | d = xml.dom.minidom.parse(file) 18 | v = d.getElementsByTagName('annotation')[0] 19 | f = v.getElementsByTagName('path')[0].firstChild.data 20 | if not os.path.isfile(f): 21 | # 如果读取 xml 内的图片文件地址失败，则会在 xml 地址寻对应名字的图片文件再试一次 22 | # 所以打标的图片文件应该尽量和 voc 格式的xml文件地址放在一起，增加便利 23 | imgname = os.path.split(f)[-1] 24 | xmlpath = os.path.split(file)[0] 25 | f = os.path.join(xmlpath, imgname) 26 | if not os.path.isfile(f): 27 | raise 'fail load img: {}'.format(f) 28 | size = v.getElementsByTagName('size')[0] 29 | npimg = cv2.imdecode(np.fromfile(f, dtype=np.uint8), -1) 30 | npimg = cv2.cvtColor(npimg, cv2.COLOR_BGR2RGB) # [y,x,c] 31 | npimg = cv2.resize(npimg, (416, 416)) 32 | npimg_ = np.transpose(npimg, (2,1,0)) # [c,x,y] 33 | def readobj(obj): 34 | d = {} 35 | bbox = obj.getElementsByTagName('bndbox')[0] 36 | d['width'] = int(size.getElementsByTagName('width')[0].firstChild.data) 37 | d['height'] = int(size.getElementsByTagName('height')[0].firstChild.data) 38 | d['ratew'] = rw = d['width']/416 39 | d['rateh'] = rh = d['height']/416 40 | d['depth'] = int(size.getElementsByTagName('depth')[0].firstChild.data) 41 | d['cate'] = obj.getElementsByTagName('name')[0].firstChild.data 42 | d['xmin'] = int(bbox.getElementsByTagName('xmin')[0].firstChild.data)/rw 43 | d['ymin'] = int(bbox.getElementsByTagName('ymin')[0].firstChild.data)/rh 44 | d['xmax'] = int(bbox.getElementsByTagName('xmax')[0].firstChild.data)/rw 45 | d['ymax'] = int(bbox.getElementsByTagName('ymax')[0].firstChild.data)/rh 46 | d['w'] = d['xmax'] - d['xmin'] 47 | d['h'] = d['ymax'] - d['ymin'] 48 | d['rect'] = d['xmin'],d['ymin'],d['xmax'],d['ymax'] 49 | d['centerx'] = (d['xmin'] + d['xmax'])/2. 50 | d['centery'] = (d['ymin'] + d['ymax'])/2. 51 | d['numpy'] = npimg_ 52 | d['file'] = f 53 | return d 54 | if islist: r = [readobj(obj) for obj in v.getElementsByTagName('object')] 55 | else: r = readobj(v.getElementsByTagName('object')[0]) 56 | return r 57 | 58 | # 生成 y_true 用于误差计算 59 | def make_y_true(imginfo, S, anchors, class_types): 60 | def get_max_match_anchor_idx(anchors, bw, bh): 61 | ious = [] 62 | for aw, ah in anchors: 63 | mi = min(aw,bw)*min(ah,bh) 64 | ma = max(aw,bw)*max(ah,bh) 65 | ious.append(mi/(aw*ah + bw*bh - mi)) 66 | return ious.index(max(ious)) 67 | cx = imginfo['centerx'] 68 | cy = imginfo['centery'] 69 | bw = imginfo['w'] 70 | bh = imginfo['h'] 71 | gap = int(416/S) 72 | ww = list(range(416))[::int(gap)] 73 | for wi in range(len(ww)): 74 | if ww[wi] > cx: 75 | break 76 | hh = list(range(416))[::int(gap)] 77 | for hi in range(len(hh)): 78 | if hh[hi] > cy: 79 | break 80 | wi, hi = wi - 1, hi - 1 81 | sx, sy = (cx-ww[wi])/gap, (cy-hh[hi])/gap # 用ceil左上角做坐标并进行归一化 82 | ceillen = (5+len(class_types)) 83 | log = math.log 84 | z = torch.zeros((S, S, len(anchors)*ceillen)) 85 | indx = get_max_match_anchor_idx(anchors, bw, bh) 86 | for i, (aw, ah) in enumerate(anchors): 87 | if i == indx: 88 | left = i*ceillen 89 | clz = [0.]*len(class_types) 90 | clz[class_types.get(imginfo['cate'])] = 1. 91 | v = torch.FloatTensor([sx, sy, log(bw/aw), log(bh/ah), 1.] + clz) 92 | z[wi, hi, left:left+ceillen] = v 93 | return z 94 | 95 | # 将经过 backbone 的矩阵数据转换成坐标和分类名字 96 | def parse_y_pred(ypred, anchors, class_types, islist=False, threshold=0.2, nms_threshold=0): 97 | ceillen = 5+len(class_types) 98 | sigmoid = lambda x:1/(1+math.exp(-x)) 99 | infos = [] 100 | for idx in range(len(anchors)): 101 | if USE_CUDA: 102 | a = ypred[:,:,:,4+idx*ceillen].cpu().detach().numpy() 103 | else: 104 | a = ypred[:,:,:,4+idx*ceillen].detach().numpy() 105 | for ii,i in enumerate(a[0]): 106 | for jj,j in enumerate(i): 107 | infos.append((ii,jj,idx,sigmoid(j))) 108 | infos = sorted(infos, key=lambda i:-i[3]) 109 | def get_xyxy_clz_con(info): 110 | gap = 416/ypred.shape[1] 111 | x,y,idx,con = info 112 | gp = idx*ceillen 113 | contain = torch.sigmoid(ypred[0,x,y,gp+4]) 114 | pred_xy = torch.sigmoid(ypred[0,x,y,gp+0:gp+2]) 115 | pred_wh = ypred[0,x,y,gp+2:gp+4] 116 | pred_clz = ypred[0,x,y,gp+5:gp+5+len(class_types)] 117 | if USE_CUDA: 118 | pred_xy = pred_xy.cpu().detach().numpy() 119 | pred_wh = pred_wh.cpu().detach().numpy() 120 | pred_clz = pred_clz.cpu().detach().numpy() 121 | else: 122 | pred_xy = pred_xy.detach().numpy() 123 | pred_wh = pred_wh.detach().numpy() 124 | pred_clz = pred_clz.detach().numpy() 125 | exp = math.exp 126 | cx, cy = map(float, pred_xy) 127 | rx, ry = (cx + x)*gap, (cy + y)*gap 128 | rw, rh = map(float, pred_wh) 129 | rw, rh = exp(rw)*anchors[idx][0], exp(rh)*anchors[idx][1] 130 | clz_ = list(map(float, pred_clz)) 131 | xx = rx - rw/2 132 | _x = rx + rw/2 133 | yy = ry - rh/2 134 | _y = ry + rh/2 135 | np.set_printoptions(precision=2, linewidth=200, suppress=True) 136 | if USE_CUDA: 137 | log_cons = torch.sigmoid(ypred[:,:,:,gp+4]).cpu().detach().numpy() 138 | else: 139 | log_cons = torch.sigmoid(ypred[:,:,:,gp+4]).detach().numpy() 140 | log_cons = np.transpose(log_cons, (0, 2, 1)) 141 | for key in class_types: 142 | if clz_.index(max(clz_)) == class_types[key]: 143 | clz = key 144 | break 145 | return [xx, yy, _x, _y], clz, con, log_cons 146 | def nms(infos): 147 | if not infos: return infos 148 | def iou(xyxyA,xyxyB): 149 | ax1,ay1,ax2,ay2 = xyxyA 150 | bx1,by1,bx2,by2 = xyxyB 151 | minx, miny = max(ax1,bx1), max(ay1, by1) 152 | maxx, maxy = min(ax2,bx2), min(ay2, by2) 153 | intw, inth = max(maxx-minx, 0), max(maxy-miny, 0) 154 | areaA = (ax2-ax1)*(ay2-ay1) 155 | areaB = (bx2-bx1)*(by2-by1) 156 | areaI = intw*inth 157 | return areaI/(areaA+areaB-areaI) 158 | rets = [] 159 | infos = infos[::-1] 160 | while infos: 161 | curr = infos.pop() 162 | if rets and any([iou(r[0], curr[0]) > nms_threshold for r in rets]): 163 | continue 164 | rets.append(curr) 165 | return rets 166 | if islist: 167 | v = [get_xyxy_clz_con(i) for i in infos if i[3] > threshold] 168 | if nms_threshold: 169 | return nms(v) 170 | else: 171 | return v 172 | else: 173 | return get_xyxy_clz_con(infos[0]) 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | import torch 191 | import torch.nn as nn 192 | import torch.nn.functional as F 193 | import torch.utils.data as Data 194 | from torch.autograd import Variable 195 | from collections import OrderedDict 196 | 197 | USE_CUDA = True if torch.cuda.is_available() else False 198 | DEVICE = 'cuda' if USE_CUDA else 'cpu' 199 | torch.set_printoptions(precision=2, sci_mode=False, linewidth=120, profile='full') 200 | 201 | class Mini(nn.Module): 202 | class ConvBN(nn.Module): 203 | def __init__(self, cin, cout, kernel_size=3, stride=1, padding=None): 204 | super().__init__() 205 | padding = (kernel_size - 1) // 2 if not padding else padding 206 | self.conv = nn.Conv2d(cin, cout, kernel_size, stride, padding, bias=False) 207 | self.bn = nn.BatchNorm2d(cout, momentum=0.01) 208 | self.relu = nn.LeakyReLU(0.1, inplace=True) 209 | def forward(self, x): 210 | return self.relu(self.bn(self.conv(x))) 211 | def __init__(self, anchors, class_types, inchennel=3): 212 | super().__init__() 213 | self.oceil = len(anchors)*(5+len(class_types)) 214 | self.model = nn.Sequential( 215 | OrderedDict([ 216 | ('ConvBN_0', self.ConvBN(inchennel, 32)), 217 | ('Pool_0', nn.MaxPool2d(2, 2)), 218 | ('ConvBN_1', self.ConvBN(32, 48)), 219 | ('Pool_1', nn.MaxPool2d(2, 2)), 220 | ('ConvBN_2', self.ConvBN(48, 64)), 221 | ('Pool_2', nn.MaxPool2d(2, 2)), 222 | ('ConvBN_3', self.ConvBN(64, 80)), 223 | ('Pool_3', nn.MaxPool2d(2, 2)), 224 | ('ConvBN_4', self.ConvBN(80, 96)), 225 | ('Pool_4', nn.MaxPool2d(2, 2)), 226 | ('ConvBN_5', self.ConvBN(96, 102)), 227 | ('ConvEND', nn.Conv2d(102, self.oceil, 1)), 228 | ]) 229 | ) 230 | def forward(self, x): 231 | return self.model(x).permute(0,2,3,1) 232 | 233 | class yoloLoss(nn.Module): 234 | def __init__(self, S, anchors, class_types): 235 | super(yoloLoss,self).__init__() 236 | self.S = S 237 | self.B = len(anchors) 238 | self.clazlen = len(class_types) 239 | self.ceillen = (5+self.clazlen) 240 | self.anchors = torch.FloatTensor(anchors).to(DEVICE) 241 | 242 | def get_iou(self,box_pred,box_targ,anchor_idx): 243 | rate = 416/self.S 244 | pre_xy = box_pred[...,:2] * rate 245 | pre_wh_half = torch.exp(box_pred[...,2:4])*self.anchors[anchor_idx]/2 246 | pre_mins = pre_xy - pre_wh_half 247 | pre_maxs = pre_xy + pre_wh_half 248 | true_xy = box_targ[...,:2] * rate 249 | true_wh_half = torch.exp(box_targ[...,2:4])*self.anchors[anchor_idx]/2 250 | true_mins = true_xy - true_wh_half 251 | true_maxs = true_xy + true_wh_half 252 | 253 | inter_mins = torch.max(true_mins, pre_mins) 254 | inter_maxs = torch.min(true_maxs, pre_maxs) 255 | inter_wh = torch.max(inter_maxs - inter_mins, torch.FloatTensor([0.]).to(DEVICE)) 256 | inter_area = inter_wh[...,0] * inter_wh[...,1] 257 | ture_area = torch.exp(box_pred[...,2])*self.anchors[anchor_idx][0] * torch.exp(box_pred[...,3])*self.anchors[anchor_idx][1] 258 | pred_area = torch.exp(box_targ[...,2])*self.anchors[anchor_idx][0] * torch.exp(box_targ[...,3])*self.anchors[anchor_idx][1] 259 | ious = inter_area/(ture_area+pred_area-inter_area) 260 | return ious 261 | 262 | def forward(self,predict_tensor,target_tensor,callback=None): 263 | N = predict_tensor.size()[0] 264 | box_contain_loss = 0 265 | noo_contain_loss = 0 266 | locxy_loss = 0 267 | locwh_loss = 0 268 | loc_loss = 0 269 | class_loss = 0 270 | for idx in range(self.B): 271 | targ_tensor = target_tensor [:,:,:,idx*self.ceillen:(idx+1)*self.ceillen] 272 | pred_tensor = predict_tensor[:,:,:,idx*self.ceillen:(idx+1)*self.ceillen] 273 | coo_mask = (targ_tensor[:,:,:,4] > 0).unsqueeze(-1).expand_as(targ_tensor) 274 | noo_mask = (targ_tensor[:,:,:,4] == 0).unsqueeze(-1).expand_as(targ_tensor) 275 | if not torch.any(coo_mask): 276 | noo_pred = pred_tensor[noo_mask].view(-1,self.ceillen) 277 | noo_targ = targ_tensor[noo_mask].view(-1,self.ceillen) 278 | noo_contain_loss += F.mse_loss(torch.sigmoid(noo_pred[...,4]), noo_targ[...,4],reduction='sum')*.1 279 | else: 280 | coo_pred = pred_tensor[coo_mask].view(-1,self.ceillen) 281 | coo_targ = targ_tensor[coo_mask].view(-1,self.ceillen) 282 | noo_pred = pred_tensor[noo_mask].view(-1,self.ceillen) 283 | noo_targ = targ_tensor[noo_mask].view(-1,self.ceillen) 284 | 285 | box_pred = coo_pred[...,0:5].contiguous().view(-1,5) 286 | box_targ = coo_targ[...,0:5].contiguous().view(-1,5) 287 | class_pred = coo_pred[...,5:5+self.clazlen] 288 | class_targ = coo_targ[...,5:5+self.clazlen] 289 | 290 | box_pred[...,:2] = torch.sigmoid(box_pred[...,:2]) 291 | ious = self.get_iou(box_pred,box_targ,idx) 292 | box_contain_loss += F.mse_loss(torch.sigmoid(box_pred[...,4])*ious, box_targ[...,4],reduction='sum') 293 | noo_contain_loss += F.mse_loss(torch.sigmoid(noo_pred[...,4]), noo_targ[...,4],reduction='sum')*.1 294 | locxy_loss += F.mse_loss(box_pred[...,0:2], box_targ[...,0:2],reduction='sum') 295 | locwh_loss += F.mse_loss(box_pred[...,2:4], box_targ[...,2:4],reduction='sum') 296 | loc_loss += locxy_loss + locwh_loss 297 | class_loss += F.mse_loss(class_pred,class_targ,reduction='sum') 298 | # print('[ ious ] :', ious) 299 | all_loss = (box_contain_loss + noo_contain_loss + loc_loss + class_loss)/N/self.B 300 | global print 301 | print = callback if callback else print 302 | print( 303 | '[ loss ] (con|non){:>.3f}|{:>.3f},(xy|wh){:>.3f}|{:>.3f},(class){:>.3f},(all){:>.3f}.'.format( 304 | box_contain_loss.item(), noo_contain_loss.item(), locxy_loss.item(), 305 | locwh_loss.item(), class_loss.item(), all_loss.item(), 306 | ) 307 | ) 308 | return all_loss 309 | 310 | def train(train_data, anchors, class_types): 311 | EPOCH = 1000 312 | BATCH_SIZE = 4 313 | LR = 0.001 314 | train_loader = Data.DataLoader( 315 | dataset = train_data, 316 | batch_size = BATCH_SIZE, 317 | shuffle = True, 318 | ) 319 | try: 320 | state = torch.load('net.pkl') 321 | net = Mini(anchors, class_types) 322 | net.load_state_dict(state['net']) 323 | net.to(DEVICE) 324 | optimizer = state['optimizer'] 325 | epoch = state['epoch'] 326 | print('load train.') 327 | except: 328 | import traceback 329 | excp = traceback.format_exc() 330 | if 'FileNotFoundError' not in excp: 331 | print(traceback.format_exc()) 332 | net = Mini(anchors, class_types) 333 | net.to(DEVICE) 334 | optimizer = torch.optim.Adam(net.parameters(), lr=LR) 335 | epoch = 0 336 | print('new train.') 337 | yloss = yoloLoss(13, anchors=anchors, class_types=class_types, ) 338 | net.train() 339 | for epoch in range(epoch, epoch+EPOCH): 340 | print('epoch', epoch) 341 | for step, (x_true_, y_true_) in enumerate(train_loader): 342 | print('[{:<3}]'.format(step), end='') 343 | x_true = Variable(x_true_).to(DEVICE) 344 | y_true = Variable(y_true_).to(DEVICE) 345 | output = net(x_true) 346 | loss = yloss(output, y_true) 347 | optimizer.zero_grad() 348 | loss.backward() 349 | optimizer.step() 350 | state = {'net':net.state_dict(), 'optimizer':optimizer, 'epoch':epoch+1, 351 | 'anchors':anchors, 'class_types':class_types} 352 | torch.save(state, 'net.pkl') 353 | print('save.') 354 | print('end.') 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | def drawrect(img, rect, text): 372 | cv2.rectangle(img, tuple(rect[:2]), tuple(rect[2:]), (10,250,10), 2, 1) 373 | x, y = rect[:2] 374 | def cv2ImgAddText(img, text, left, top, textColor=(0, 255, 0), textSize=20): 375 | from PIL import Image, ImageDraw, ImageFont 376 | img = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) 377 | draw = ImageDraw.Draw(img) 378 | fontText = ImageFont.truetype( "font/simsun.ttc", textSize, encoding="utf-8") 379 | draw.text((left, top), text, textColor, font=fontText) 380 | return cv2.cvtColor(np.asarray(img), cv2.COLOR_RGB2BGR) 381 | import re 382 | if re.findall('[\u4e00-\u9fa5]', text): 383 | img = cv2ImgAddText(img, text, x, y-12, (10,10,250), 12) # 如果存在中文则使用这种方式绘制文字 384 | else: 385 | cv2.putText(img, text, (x,y), cv2.FONT_HERSHEY_SIMPLEX, 0.4, (10,10,250), 1) 386 | return img 387 | def get_all_draw_rects(filename, state): 388 | net = state['net'] 389 | anchors = state['anchors'] 390 | class_types = state['class_types'] 391 | npimg = cv2.imdecode(np.fromfile(filename, dtype=np.uint8), -1) 392 | height, width = npimg.shape[:2] 393 | npimg = cv2.cvtColor(npimg, cv2.COLOR_BGR2RGB) # [y,x,c] 394 | npimg = cv2.resize(npimg, (416, 416)) 395 | npimg_ = np.transpose(npimg, (2,1,0)) # [c,x,y] 396 | y_pred = net(torch.FloatTensor(npimg_).unsqueeze(0).to(DEVICE)) 397 | v = parse_y_pred(y_pred, anchors, class_types, islist=True, threshold=0.2, nms_threshold=0.4) 398 | r = [] 399 | for i in v: 400 | rect, clz, con, log_cons = i 401 | rw, rh = width/416, height/416 402 | rect[0],rect[2] = int(rect[0]*rw),int(rect[2]*rw) 403 | rect[1],rect[3] = int(rect[1]*rh),int(rect[3]*rh) 404 | r.append([rect, clz, con, log_cons]) 405 | # 绘制所有定位的框 406 | img = cv2.imdecode(np.fromfile(filename, dtype=np.uint8), -1) 407 | for i in r: 408 | rect, clz, con, log_cons = i 409 | img = drawrect(img, rect, '{}|{:<.2f}'.format(clz,con)) 410 | cv2.imshow('test', img) 411 | cv2.waitKey(0) 412 | cv2.destroyAllWindows() 413 | 414 | def load_net(filename): 415 | state = torch.load(filename) 416 | anchors = state['anchors'] 417 | class_types = state['class_types'] 418 | net = Mini(anchors, class_types) 419 | net.load_state_dict(state['net']) 420 | net.to(DEVICE) 421 | net.eval() 422 | state['net'] = net 423 | return state 424 | 425 | 426 | 427 | 428 | 429 | 430 | 431 | def load_voc_data(xmlpath, anchors): 432 | files = [os.path.join(xmlpath, path) for path in os.listdir(xmlpath) if path.endswith('.xml')] 433 | imginfos = [] 434 | print('use anchors:', anchors) 435 | print('load xml file number:{}, start.'.format(len(files))) 436 | for idx, file in enumerate(files): 437 | if idx % 1000 == 0: print('loading {}/{}'.format(idx, len(files))) 438 | imginfos.extend(read_voc_xml(file, islist=True)) 439 | print('load all file. ok.') 440 | # 注意这里加载数据的方式是小批量加载处理，所以自动生成 class_types 441 | # 如果有大量数据想要进行多批次训练，那么就需要注意 class_types 的生成。 442 | class_types = [imginfo.get('cate') for imginfo in imginfos] 443 | print('load class types. start.') 444 | class_types = {typ:idx for idx,typ in enumerate(sorted(list(set(class_types))))} 445 | print('load class types. ok.') 446 | print('class_types:', class_types) 447 | train_data = [] 448 | print('make x_true,y_true. start.') 449 | for idx, imginfo in enumerate(imginfos): 450 | if idx % 1000 == 0: print('makeing x_true,y_true. {}/{}'.format(idx, len(files))) 451 | x_true = torch.FloatTensor(imginfo['numpy']) 452 | y_true = make_y_true(imginfo, 13, anchors, class_types) 453 | train_data.append([x_true, y_true]) 454 | print('make x_true,y_true. ok.') 455 | return train_data, imginfos, class_types 456 | 457 | 458 | 459 | 460 | 461 | 462 | 463 | 464 | 465 | 466 | 467 | # 加载数据，生成训练数据的结构，主要需要的三个数据 anchors，class_types，train_data 468 | # 训练结束后会将 anchors, class_types 信息一并存放，所以预测时无需重新加载数据获取这两项信息 469 | # 如果存在之前的训练文件，会自动加载进行继续训练，并且保存时会覆盖之前的模型 470 | # 另外这里的 anchor 数量可以自由调整，如果所定位的形状没有太大变化，设置成一个 [[60, 60]] 会节约计算资源 471 | if __name__ == '__main__': 472 | xmlpath = './train_img' 473 | anchors = [[60, 60]] 474 | train_data, imginfos, class_types = load_voc_data(xmlpath, anchors) 475 | train(train_data, anchors, class_types) 476 | 477 | # testpath = './train_img' 478 | # state = load_net('net.pkl') 479 | # v = [os.path.join(testpath, i) for i in os.listdir(testpath) if i.lower().endswith('.jpg') or i.lower().endswith('.png')] 480 | # v = v[::-1] 481 | # for i in v: 482 | # get_all_draw_rects(i, state) 483 | 484 | --------------------------------------------------------------------------------