├── .gitignore ├── README.md ├── app ├── config.ini ├── config_douban.json └── douban_house.py └── core ├── auto ├── auto_request.py └── config_process.py ├── exception └── exceptions.py ├── notice └── email_notice.py ├── session └── platform_session.py └── util ├── config_util.py ├── data_util.py ├── file_util.py ├── log_util.py └── time_util.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Created by .ignore support plugin (hsz.mobi) 2 | ### JetBrains template 3 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm 4 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 5 | 6 | # User-specific stuff 7 | .idea/**/workspace.xml 8 | .idea/**/tasks.xml 9 | .idea/**/dictionaries 10 | .idea/**/shelf 11 | 12 | # Sensitive or high-churn files 13 | .idea/**/dataSources/ 14 | .idea/**/dataSources.ids 15 | .idea/**/dataSources.local.xml 16 | .idea/**/sqlDataSources.xml 17 | .idea/**/dynamic.xml 18 | .idea/**/uiDesigner.xml 19 | .idea/**/dbnavigator.xml 20 | 21 | # Gradle 22 | .idea/**/gradle.xml 23 | .idea/**/libraries 24 | 25 | # CMake 26 | cmake-build-debug/ 27 | cmake-build-release/ 28 | 29 | # Mongo Explorer plugin 30 | .idea/**/mongoSettings.xml 31 | 32 | # File-based project format 33 | *.iws 34 | 35 | # IntelliJ 36 | out/ 37 | 38 | # mpeltonen/sbt-idea plugin 39 | .idea_modules/ 40 | 41 | # JIRA plugin 42 | atlassian-ide-plugin.xml 43 | 44 | # Cursive Clojure plugin 45 | .idea/replstate.xml 46 | 47 | # Crashlytics plugin (for Android Studio and IntelliJ) 48 | com_crashlytics_export_strings.xml 49 | crashlytics.properties 50 | crashlytics-build.properties 51 | fabric.properties 52 | 53 | # Editor-based Rest Client 54 | .idea/httpRequests 55 | 56 | test/* 57 | *test.* 58 | IMP_* 59 | *.html 60 | *.jpg 61 | 62 | /.idea/ 63 | /app/my_config.ini 64 | 65 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # infomation_monitor_platform 2 | 这是一个通用的脚本请求的平台代码，可以在这个平台上面写各种网络请求的脚本，例如刷课、抢票、刷信息等 3 | ## 一、项目说明 4 | 本项目的出发点是减少重复制造轮子，在日常生活中，很多时候我们需要写个脚本去帮我们刷课，抢票等等，因为我多次写这种脚本，我就想着为啥不去写个通用的平台代码，这样以后每次写脚本都直接配置，不需要写代码。 5 | 因为一般的需求主要就是两种： 6 | 1. 登陆某个网站，按时刷新某个网页，看我们希望出现的内容有没有出现，例如在豆瓣上面看房子，我们刷新特定页面，看有没有特定的房源出现，如果出现了就通知我们或者直接发送模拟请求锁定房源（课程等）。 7 | 2. 登陆某个网站，定时做某种请求，比如典型的就是高校抢课，在某天中午12点把我们的选课请求发送出去 8 | 9 | 分析以上两种行为，我们能够抽取大量的同逻辑代码，各种需求唯一的不同就是发送的数据和页面不一样，然后各自的请求链接不一样，所以本项目就是致力于把逻辑相同的代码都统一维护，方便后面快速产生对应业务的请求代码。 10 | ## 二、项目终极目标 11 | 最后只需要配置文件，不需要写代码，配置好跳转数据发送抽取等。 12 | 13 | **一口吃不成一个胖子，第一步先完成基本的框架设计，核心代码的完成，这样在写具体请求的时候能够更快完成。** 14 | ## 三、模块介绍 15 | 目前主要有四个模块： 16 | 1. 异常模块，主要定义项目的各种异常 17 | 2. 通知模块，主要用于各类通知的代码，比如邮件，包括以后的短信、电话、微信等等 18 | 3. 核心平台模块，目前主要用于维护核心session和做session持久化等操作 19 | 4. 工具模块，现在主要有四大工具： 20 | 1. 配置处理工具，主要有ini文件和json文件的配置处理 21 | 2. 时间处理，包含定时器等，时间字符串处理 22 | 3. 文件处理工具 23 | 4. 数据预处理 24 | ## 四、demo介绍 25 | 目前这个demo比较简单，主要是请求豆瓣的group界面，刷我们需要的房源的demo 26 | 27 | ## 五、设计说明 28 | 整个项目以 request queue 为核心，每个request对应一个operation，每个operation可以往request queue里面增加一个request，也可以同时做其他的事情，比如：email通知。当整个的request queue为空的时候平台退出。 -------------------------------------------------------------------------------- /app/config.ini: -------------------------------------------------------------------------------- 1 | [CONFIG] 2 | form_email = ****@m*ail.win 3 | form_password = z***********4 4 | 5 | -------------------------------------------------------------------------------- /app/config_douban.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "", 3 | "main_request_tag": "main", 4 | "request_list": [ 5 | { 6 | "request_tag": "", 7 | "url": "", 8 | "request_type": "get", 9 | "data": { 10 | "username": "", 11 | "password": "" 12 | }, 13 | "return_type": "html", 14 | "requestOperation": { 15 | "type": "monitor,simulation", 16 | "condition": { 17 | "targetElement": "", 18 | "dataType": "str,num", 19 | "symbol": "in,=,>,<,>=,<=", 20 | "aimData": "", 21 | "true": { 22 | "type": "goto,notice", 23 | "gotoSetting": { 24 | "gotoTag": "" 25 | }, 26 | "noticeSetting": { 27 | "noticeType": "email", 28 | "to": "lin@mmail.win", 29 | "content": "notice content" 30 | }, 31 | "false": {} 32 | } 33 | }, 34 | "monitorSetting": {}, 35 | "simulationSetting": {} 36 | }, 37 | "exceptionOperation": { 38 | "exceptionType": "no_login", 39 | "no_login": {}, 40 | "default": { 41 | "type": [ 42 | "goto", 43 | "notice" 44 | ], 45 | "gotoSetting": { 46 | "gotoTag": "main" 47 | }, 48 | "noticeSetting": { 49 | "noticeType": "email", 50 | "to": "lin@mmail.win", 51 | "content": "notice content" 52 | } 53 | } 54 | } 55 | } 56 | ], 57 | "config_file_name": "", 58 | "config_type": "", 59 | "main_request_name": "main", 60 | "resultOperation": { 61 | "type": [ 62 | "goto", 63 | "notice" 64 | ], 65 | "gotoSetting": { 66 | "gotoTag": "" 67 | }, 68 | "noticeSetting": { 69 | "noticeType": "email", 70 | "to": "lin@mmail.win", 71 | "content": "notice content" 72 | } 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /app/douban_house.py: -------------------------------------------------------------------------------- 1 | from core.util import config_util 2 | from core.util import data_util 3 | from core.util import time_util 4 | from core.util import file_util 5 | from core.session import platform_session 6 | from core.notice import email_notice 7 | import re 8 | 9 | if __name__ == '__main__': 10 | # 内容如config.ini，用户名和密码 11 | cu = config_util.ConfigUtil('my_config.ini') 12 | pfs = platform_session.load_ps() 13 | if pfs is None: 14 | pfs = platform_session.IMP_Session() 15 | pfs.add_value('source', 'index_nav') 16 | pfs.add_value('form_email', cu.get('form_email')) 17 | pfs.add_value('form_password', cu.get('form_password')) 18 | login_response = pfs.post('https://accounts.douban.com/login') 19 | file_util.write(login_response.text, 'douban.html') 20 | login_data = data_util.pre_process(login_response, data_util.ResponseType.html) 21 | 22 | img_tag = login_data.find('img', id='captcha_image') 23 | if img_tag == None: 24 | print('登陆成功') 25 | else: 26 | file_util.write(pfs.get(img_tag.get('src')).content, 'captcha_image.jpg', mode='wb') 27 | print(img_tag.get('src')) 28 | p = re.compile('=.+&') 29 | img_id = p.search(img_tag.get('src')).group(0)[1:-1] 30 | print(img_id) 31 | print(login_data.find_all('input', attrs={'type': 'hidden'}, value=True)) 32 | ci = input('请输入验证码：') 33 | ''' 34 | ck: dZjd 35 | source: None 36 | redir: https://www.douban.com 37 | form_email: reg@****.win 38 | form_password: z******4 39 | captcha-solution: respect 40 | captcha-id: 9qfsFkUFSQ6rXPtZfbH0Gqhk:en 41 | login: 登录 42 | ''' 43 | pfs.add_value('source', 'index_nav') 44 | pfs.add_value('form_email', cu.get('form_email')) 45 | pfs.add_value('form_password', cu.get('form_password')) 46 | # pfs.add_value('ck', login_data.find('input', name='ck')['value']) 47 | pfs.add_value('captcha-solution', ci) 48 | pfs.add_value('captcha-id', img_id) 49 | pfs.add_value('login', '登录') 50 | login_response = pfs.post('https://accounts.douban.com/login') 51 | login_data = data_util.pre_process(login_response, data_util.ResponseType.html) 52 | 53 | html = pfs.get('https://www.douban.com/group/').text 54 | data = data_util.pre_process(html, data_util.ResponseType.html) 55 | tags = data.find_all('a', class_='title') 56 | 57 | print(tags) 58 | 59 | pfs.persistence() 60 | -------------------------------------------------------------------------------- /core/auto/auto_request.py: -------------------------------------------------------------------------------- 1 | from core.auto import config_process 2 | from core.util import log_util 3 | from core.util import data_util 4 | from core.util import config_util 5 | from core.session.platform_session import IMP_Session 6 | 7 | log = log_util.get_logger() 8 | 9 | 10 | def main_process(main_config_file): 11 | reu = config_process.RequestConfigUtil(main_config_file) 12 | request_queue = [reu.mrn] 13 | session = IMP_Session(reu.ju.get('name')) 14 | # todo should support mutil-threads 15 | while request_queue: 16 | request_name = request_queue.pop() 17 | request = reu.get_request(request_name) 18 | # todo check request 19 | 20 | # do a request 21 | url = request.get('url') 22 | data = request.get('data') 23 | if data: 24 | for key in data: 25 | session.add_value(key, data[key]) 26 | # todo exception process 27 | if request.get('request_type') == 'get': 28 | response = session.get(url) 29 | else: 30 | response = session.post(url) 31 | 32 | # begin condition process 33 | if not request.get('requestOperation.condition'): 34 | print(f'cannot find condition {request_name}') 35 | continue 36 | condition = config_util.DictWrapper(request.get('requestOperation.condition')) 37 | response_data = None 38 | if request.get('return_type') == 'html': 39 | response_type = data_util.ResponseType.html 40 | html = data_util.pre_process(response, response_type) 41 | tag = html.find(name=condition.get('targetElement.name'), attrs=condition.get('targetElement.attrs')) 42 | if tag is None: 43 | print(f'cannot find a tag {condition.get("targetElement")}') 44 | else: 45 | response_data = tag.string 46 | else: 47 | response_type = data_util.ResponseType.json 48 | json = data_util.pre_process(response, response_type) 49 | response_data = json.get(condition.get('targetElement')) 50 | 51 | # begin operation 52 | if response_data: 53 | operation_name = condition_execute(response_data, condition) 54 | operation = reu.get_operation(operation_name) 55 | if 'goto' in operation.get('type'): 56 | request_queue.append(operation.get('gotoSetting.gotoTag')) 57 | if 'notice' in operation.get('type'): 58 | notice_process(operation.get('noticeSetting')) 59 | 60 | 61 | def notice_process(noticeSetting=None): 62 | # todo notice process 63 | pass 64 | 65 | 66 | def condition_execute(data, condition): 67 | aimData = condition.get('aimData') 68 | if condition.get('dataType') == 'int': 69 | data = int(data) 70 | aimData = int(aimData) 71 | elif condition.get('dataType') == 'float': 72 | data = float(data) 73 | aimData = float(aimData) 74 | else: 75 | aimData = str(aimData) 76 | 77 | c = False 78 | symbol = condition.get('symbol') 79 | 80 | if symbol == 'in': 81 | c = aimData in data 82 | elif symbol == '=': 83 | c = aimData == data 84 | elif symbol == '>': 85 | c = data > aimData 86 | elif symbol == '<': 87 | c = data < aimData 88 | elif symbol == '>=': 89 | c = data >= aimData 90 | elif symbol == '<=': 91 | c = data <= aimData 92 | if c: 93 | return condition.get('true') 94 | else: 95 | return condition.get('false') 96 | -------------------------------------------------------------------------------- /core/auto/config_process.py: -------------------------------------------------------------------------------- 1 | from core.util import log_util 2 | from core.util import config_util 3 | from core.exception import exceptions 4 | import json 5 | 6 | 7 | class RequestConfigUtil(): 8 | 9 | def __init__(self, config_file): 10 | json_util = config_util.JsonConfigUtil(config_file) 11 | self.ju = json_util 12 | self.request_list = json_util.get('request_list') 13 | self.operation_list = json_util.get('operation_list') 14 | if json_util.get('main_request_name') is not None: 15 | self.mrn = json_util.get('main_request_name') 16 | if self.mrn == None: 17 | raise exceptions.ConfigError(f'cannot find main request name') 18 | 19 | def get_request(self, name): 20 | for request in self.request_list: 21 | if name is not None and 'name' in request and request['name'] == name: 22 | # todo should write a cache 23 | return config_util.DictWrapper(request) 24 | raise exceptions.ConfigError(f'cannot find a request named "{name}"') 25 | 26 | def get_operation(self, name): 27 | for operation in self.operation_list: 28 | if name is not None and 'name' in operation and operation['name'] == name: 29 | return config_util.DictWrapper(operation) 30 | raise exceptions.ConfigError(f'cannot find a operation named "{name}"') 31 | 32 | def config_check(config_file: object) -> object: 33 | code = 0 34 | msg = '' 35 | # todo 完成config检查，有误可以raise Exception 36 | return code, msg 37 | # with open(config_file, encoding='utf-8') as cf: 38 | # config = json.load(cf) 39 | # 40 | # print(config.keys()) 41 | # if config.request_list != None: 42 | # print(config.request_list) 43 | 44 | def request_check(request_list, main_request_name): 45 | # for request in request_list: 46 | pass 47 | -------------------------------------------------------------------------------- /core/exception/exceptions.py: -------------------------------------------------------------------------------- 1 | class UnknownException(Exception): 2 | pass 3 | 4 | 5 | class LoginErrorException(Exception): 6 | pass 7 | 8 | 9 | class DataProcessException(Exception): 10 | pass 11 | 12 | 13 | class VerificationException(Exception): 14 | pass 15 | 16 | 17 | class FormatInvalidException(Exception): 18 | pass 19 | 20 | 21 | class ParameterException(Exception): 22 | pass 23 | 24 | class ConfigError(Exception): 25 | pass 26 | -------------------------------------------------------------------------------- /core/notice/email_notice.py: -------------------------------------------------------------------------------- 1 | from email.mime.text import MIMEText 2 | import smtplib 3 | from email.header import Header 4 | 5 | 6 | class Email(object): 7 | def __init__(self, smtp_server, user_email_address, password, port=25): 8 | self.smtp_server = smtp_server 9 | self.user_email_address = user_email_address 10 | self.password = password 11 | self.port = port 12 | 13 | def sent_email(self, content, to_addr): 14 | message = MIMEText(content, 'plain', 'utf-8') 15 | message['From'] = 'IMP<%s>' % self.user_email_address 16 | message['To'] = 'IMP_rec<%s>' % to_addr 17 | 18 | subject = '信监平台：' + content 19 | message['Subject'] = Header(subject, 'utf-8') 20 | 21 | try: 22 | smtpObj = smtplib.SMTP() 23 | smtpObj.connect(self.smtp_server, self.port) # 25 为 SMTP 端口号 24 | smtpObj.login(self.user_email_address, self.password) 25 | smtpObj.sendmail('<%s>' % self.user_email_address, '<%s>' % to_addr, message.as_string()) 26 | print("邮件发送成功") 27 | except Exception as e: 28 | print('>' * 40 + str(e)) 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /core/session/platform_session.py: -------------------------------------------------------------------------------- 1 | import requests 2 | from core.util import file_util 3 | from core.exception import exceptions 4 | import pickle 5 | import time 6 | import os 7 | import sys 8 | 9 | 10 | class IMP_Session(object): 11 | DEFAULT_USER_AGENT = 'Mozilla/5.0 (Windows NT 6.2; WOW64; rv:17.0) Gecko/20100101 Firefox/17.0' 12 | 13 | def __init__(self, name='%d' % int(time.time()), user_agent=DEFAULT_USER_AGENT): 14 | self.create_time = time.time() 15 | self.session = requests.session() 16 | self.headers = self.session.headers 17 | self.headers['User-Agent'] = user_agent 18 | self.preserve_data = {} 19 | self.preserve_data_times = {} 20 | self.name = 'IMP_%s' % name 21 | pass 22 | 23 | def get(self, url, **kwargs): 24 | if 'params' in kwargs: 25 | self.session.get(url=url, **kwargs) 26 | response = self.session.get(url=url, params=self.__generate_data(), **kwargs) 27 | return response 28 | 29 | def post(self, url, json=None, **kwargs): 30 | if 'data' in kwargs: 31 | self.session.post(url, json=json, **kwargs) 32 | response = self.session.post(url, data=self.__generate_data(), json=json, **kwargs) 33 | return response 34 | 35 | def add_value(self, key, value, preserve_times=1): 36 | ''' 37 | :param key:数据的key 38 | :param value:数据值 39 | :param preserve_times:这个数据是否需要多次用到，这里就是用大的次数，-1表示永久用到，默认是1 40 | :return:无 41 | ''' 42 | if preserve_times == -1: 43 | preserve_times = sys.maxsize 44 | self.preserve_data[key] = value 45 | self.preserve_data_times[key] = preserve_times 46 | 47 | def __generate_data(self): 48 | data = {} 49 | delete_list = [] 50 | for key in self.preserve_data_times: 51 | if self.preserve_data_times[key] > 1: 52 | self.preserve_data_times[key] -= 1 53 | data[key] = self.preserve_data[key] 54 | elif self.preserve_data_times[key] == 1: 55 | data[key] = self.preserve_data[key] 56 | delete_list.append(key) 57 | for key in delete_list: 58 | del self.preserve_data[key] 59 | del self.preserve_data_times[key] 60 | return data 61 | 62 | def persistence(self): 63 | with open(self.name, mode='wb') as psp: 64 | pickle.dump(self, psp) 65 | 66 | def get_survival_time(self): 67 | return time.time() - self.create_time 68 | 69 | 70 | def load_ps(name=None): 71 | if name is None: 72 | files = os.listdir('.') 73 | imps = [] 74 | for file in files: 75 | if 'IMP_' in file: 76 | imps.append(file) 77 | imps.sort(reverse=True) 78 | if len(imps) > 0: 79 | name = imps[0] 80 | if name is not None: 81 | try: 82 | with open(name, 'rb') as rpkl: 83 | return pickle.load(rpkl) 84 | except Exception as e: 85 | print('打开名字为%s的会话错误：%s' % (name, str(e))) 86 | return None 87 | # raise exceptions.ParameterException('打开名字为%s的会话错误：%s' % (name, str(e))) 88 | else: 89 | # raise exceptions.ParameterException('没有找到会话') 90 | return None 91 | -------------------------------------------------------------------------------- /core/util/config_util.py: -------------------------------------------------------------------------------- 1 | import configparser 2 | import os 3 | import json 4 | from core.exception.exceptions import * 5 | 6 | 7 | class ConfigUtil(object): 8 | __DEFAULT_SECTION = 'CONFIG' 9 | __DEFAULT_FILE = 'config.ini' 10 | 11 | def __init__(self, config_file_path=__DEFAULT_FILE): 12 | self.cp = configparser.ConfigParser() 13 | self.config_file_path = config_file_path 14 | if os.path.exists(config_file_path): 15 | self.cp.read(config_file_path, 'utf-8') 16 | 17 | def put(self, key, value, section=__DEFAULT_SECTION): 18 | if not self.cp.has_section(section): 19 | self.cp.add_section(section) 20 | self.cp.set(section, key, value) 21 | self.cp.write(open(self.config_file_path, mode='w', encoding='utf-8')) 22 | 23 | def get(self, key, section=__DEFAULT_SECTION): 24 | if self.cp.has_option(section, key): 25 | return self.cp.get(section, key) 26 | return None 27 | 28 | 29 | class JsonConfigUtil(object): 30 | __DEFAULT_FILE = 'config.json' 31 | 32 | def __init__(self, config_file_path=__DEFAULT_FILE, write2file=True): 33 | self.config_file_path = config_file_path 34 | self.write2file = write2file 35 | if os.path.exists(config_file_path): 36 | with open(config_file_path, encoding='utf-8') as cf: 37 | self.data = json.load(cf) 38 | else: 39 | self.data = {} 40 | self.dw = DictWrapper(self.data) 41 | 42 | def get_json_object(self): 43 | return self.data 44 | 45 | def put(self, key, value): 46 | self.dw.put(key, value) 47 | if self.write2file: 48 | with open(self.config_file_path, mode='w', encoding='utf-8') as cf: 49 | json.dump(self.data, cf, indent=4) 50 | 51 | def get(self, key): 52 | return self.dw.get(key) 53 | 54 | 55 | class DictWrapper(): 56 | def __init__(self, data): 57 | self.data = data 58 | 59 | def get_dict(self): 60 | return self.data 61 | 62 | def put(self, key, value): 63 | if value == self.get(key): 64 | return 65 | data = self.data 66 | if isinstance(key, str): 67 | ks = key.split('.') 68 | for i in range(len(ks) - 1): 69 | if isinstance(data, dict): 70 | if ks[i] not in data: 71 | data[ks[i]] = {} 72 | data = data[ks[i]] 73 | else: 74 | raise ParameterException('key: %s is not a dict' % ks[i - 1]) 75 | data[ks[len(ks) - 1]] = value 76 | 77 | def get(self, key): 78 | data = self.data 79 | if isinstance(key, str): 80 | ks = key.split('.') 81 | for i in range(len(ks)): 82 | if ks[i] in data: 83 | data = data[ks[i]] 84 | else: 85 | return None 86 | return data 87 | return None 88 | -------------------------------------------------------------------------------- /core/util/data_util.py: -------------------------------------------------------------------------------- 1 | from bs4 import BeautifulSoup 2 | import json 3 | from enum import Enum 4 | import requests 5 | from core.util.config_util import DictWrapper 6 | 7 | 8 | class ResponseType(Enum): 9 | json = 0, 10 | html = 1 11 | 12 | 13 | def pre_process(data, type=ResponseType.json, return_dw=True): 14 | if isinstance(data, requests.models.Response): 15 | data = data.text 16 | if type == ResponseType.json: 17 | if return_dw: 18 | return DictWrapper(json.loads(data, encoding='utf-8')) 19 | else: 20 | return json.loads(data, encoding='utf-8') 21 | elif type == ResponseType.html: 22 | return BeautifulSoup(data, 'lxml') 23 | -------------------------------------------------------------------------------- /core/util/file_util.py: -------------------------------------------------------------------------------- 1 | from core.util import time_util 2 | 3 | 4 | def write(data, file_name=None, mode='w', encoding='utf-8'): 5 | if file_name is None: 6 | file_name = time_util.get_time_str().replace(' ', '-').replace(':', '-') 7 | if mode in ['wb', 'rb']: 8 | file = open(file_name, mode=mode) 9 | else: 10 | file = open(file_name, mode=mode, encoding=encoding) 11 | file.write(data) 12 | file.flush() 13 | file.close() 14 | -------------------------------------------------------------------------------- /core/util/log_util.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | def get_logger(filename='STD_LOG.log', logger_name='STD_LOG', level=logging.DEBUG, file_level=logging.INFO, 5 | fstr='%(asctime)s - %(name)s - %(levelname)s - %(message)s'): 6 | # 创建一个logger 7 | logger = logging.getLogger(logger_name) 8 | logger.setLevel(level) 9 | # 定义handler的输出格式 10 | formatter = logging.Formatter(fstr) 11 | 12 | if filename is not None: 13 | # 创建一个handler，用于写入日志文件 14 | fh = logging.FileHandler(filename) 15 | fh.setLevel(file_level) 16 | fh.setFormatter(formatter) 17 | logger.addHandler(fh) 18 | 19 | # 再创建一个handler，用于输出到控制台 20 | ch = logging.StreamHandler() 21 | ch.setLevel(level) 22 | ch.setFormatter(formatter) 23 | # 给logger添加handler 24 | logger.addHandler(ch) 25 | return logger 26 | -------------------------------------------------------------------------------- /core/util/time_util.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | import random 4 | from enum import Enum 5 | import re 6 | from core.exception.exceptions import * 7 | import threading 8 | 9 | 10 | class UTC(datetime.tzinfo): 11 | """UTC""" 12 | 13 | def __init__(self, offset=0): 14 | self._offset = offset 15 | 16 | def utcoffset(self, dt): 17 | return datetime.timedelta(hours=self._offset) 18 | 19 | def tzname(self, dt): 20 | return "UTC +%s" % self._offset 21 | 22 | def dst(self, dt): 23 | return datetime.timedelta(hours=self._offset) 24 | 25 | 26 | class Mul_type(Enum): 27 | linear = 0, 28 | exponential = 1 29 | 30 | 31 | class ExponentialSleep(object): 32 | def __init__(self, aim_str, m=0.7, r=0.3, mul_type=Mul_type.exponential, base_mum=2): 33 | self.continue_num = 0 34 | self.aim_str = aim_str 35 | self.m = m 36 | self.r = r 37 | self.base_num = base_mum 38 | self.mul_type = mul_type 39 | 40 | def judge_sleep(self, e): 41 | if self.aim_str in str(e): 42 | self.continue_num += 1 43 | else: 44 | self.continue_num = 0 45 | 46 | if self.mul_type == Mul_type.exponential: 47 | random_sleep(self.m, self.r, self.base_num ** self.continue_num) 48 | else: 49 | random_sleep(self.m, self.r, self.base_num * self.continue_num) 50 | 51 | 52 | def random_sleep(m=0.7, r=0.3, mul=1): 53 | # 随机休眠时间 54 | time.sleep(mul * random.randint((m - r) * 100, (m + r) * 100) / 100) 55 | 56 | 57 | def get_timestamp(time_str, tzinfo_num=8): 58 | p = re.compile('\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}') 59 | if p.fullmatch(time_str) is not None: 60 | dd = time_str.split(' ') 61 | dds = dd[0].split('-') 62 | ts = dd[1].split(':') 63 | return datetime.datetime(int(dds[0]), int(dds[1]), int(dds[2]), int(ts[0]), 64 | int(ts[1]), int(ts[2]), 0, 65 | tzinfo=UTC(tzinfo_num)).timestamp() 66 | else: 67 | raise FormatInvalidException('格式必须为：xxxx-xx-xx HH:mm:ss') 68 | 69 | 70 | def timer(start_time, operation=None, login_operation=None): 71 | times = 0 72 | while 1: 73 | timeDelay = start_time - time.time() 74 | if timeDelay > 0: 75 | print('操作时间未到:%d,还有%d小时%d分钟%d秒' % (times, timeDelay / 3600, timeDelay / 60 % 60, timeDelay % 60)) 76 | if timeDelay > 10 * 60: 77 | timeSleep = 5 * 60 78 | elif timeDelay > 2 * 60: 79 | if timeDelay > 4 * 60: 80 | login_operation() 81 | timeSleep = 60 82 | elif timeDelay > 13: 83 | timeSleep = 10 84 | else: 85 | timeSleep = 1 86 | print("休眠%d秒" % timeSleep) 87 | time.sleep(timeSleep) 88 | else: 89 | operation() 90 | break 91 | 92 | 93 | def loop(operation=None, judge_operation=None, m=5, r=1): 94 | while judge_operation(): 95 | operation() 96 | random_sleep(m, r) 97 | 98 | 99 | def thread_timer(start_time, login_operation, operation): 100 | t = ThreadTimer(start_time, login_operation, operation) 101 | t.start() 102 | return t 103 | 104 | 105 | class ThreadTimer(threading.Thread): 106 | def __init__(self, start_time, login_operation, operation): 107 | threading.Thread.__init__(self) 108 | self.start_time = start_time 109 | self.login_operation = login_operation 110 | self.operation = operation 111 | 112 | def run(self): 113 | timer(self.start_time, self.login_operation, self.operation) 114 | 115 | 116 | def get_time_str(p_tuple=time.localtime(), format='%Y-%m-%d %H:%M:%S'): 117 | ''' 118 | %Y Year with century as a decimal number. 119 | %m Month as a decimal number [01,12]. 120 | %d Day of the month as a decimal number [01,31]. 121 | %H Hour (24-hour clock) as a decimal number [00,23]. 122 | %M Minute as a decimal number [00,59]. 123 | %S Second as a decimal number [00,61]. 124 | %z Time zone offset from UTC. 125 | %a Locale's abbreviated weekday name. 126 | %A Locale's full weekday name. 127 | %b Locale's abbreviated month name. 128 | %B Locale's full month name. 129 | %c Locale's appropriate date and time representation. 130 | %I Hour (12-hour clock) as a decimal number [01,12]. 131 | %p Locale's equivalent of either AM or PM. 132 | ''' 133 | return time.strftime(format, p_tuple) 134 | --------------------------------------------------------------------------------