├── README.md ├── config.ini ├── libs ├── asyncSpider.py └── downloader.py ├── main.py ├── requirements.txt └── utils ├── decorators.py ├── github.py ├── logger.py ├── readSetting.py └── sqlite.py /README.md: -------------------------------------------------------------------------------- 1 | ## Introduce 2 | 3 | you can watch video by global cdn network through the project 4 | 5 | ## Environment 6 | 7 | - Python3.6+ 8 | There is a tutorial on how to install python3.9 in ubuntu 16 and ubuntu 18 --> [click me](https://segmentfault.com/a/1190000021967408) 9 | - pip 10 | ```shell 11 | wget https://bootstrap.pypa.io/get-pip.py 12 | python3 get-pip.py 13 | ``` 14 | 15 | ## How to use 16 | 17 | 1. edit the config file 18 | 19 | edit the `config.ini` file and modify the github information 20 | 21 | 2. run the script 22 | ```shell 23 | python3 main.py 24 | ``` 25 | 26 | ## Function 27 | 28 | - scanning the whole 91porn website videoInfo 29 | 30 | - download the video media 31 | 32 | - upload the video media to github and generate cdn url 33 | 34 | ## ToDo List 35 | 36 | - support onedrive 37 | 38 | - watch the video in website -------------------------------------------------------------------------------- /config.ini: -------------------------------------------------------------------------------- 1 | [sqlite] 2 | # 数据库文件存放地，请勿修改 3 | path = 91porn.db 4 | 5 | [github] 6 | # github的账号，仓库名以及Token 7 | # 如果你不知道token在哪生成，请查看该文档：https://docs.github.com/cn/github/authenticating-to-github/creating-a-personal-access-token 8 | UserName = yaourts 9 | Repository = tests 10 | AccessToken = xxx 11 | 12 | [91porn] 13 | # 默认的91porn链接，其有好几个页面，可按需修改 14 | # host请勿修改 15 | url = https://91porn.com/v.php?next=watch 16 | host = 91porn.com 17 | ;url = https://a1016.91p01.com/v.php?next=watch 18 | ;host = a1016.91p01.com 19 | 20 | [onedrive] -------------------------------------------------------------------------------- /libs/asyncSpider.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : asyncSpider 5 | Description : 爬取91的视频信息 6 | Author : jccia 7 | date : 2021/5/9 8 | ------------------------------------------------- 9 | """ 10 | import asyncio 11 | import random 12 | import re 13 | from ipaddress import IPv4Address 14 | from urllib.parse import unquote 15 | 16 | import aiohttp 17 | import requests as requests 18 | from fake_useragent import UserAgent 19 | from lxml import etree 20 | from concurrent.futures import ThreadPoolExecutor, wait 21 | 22 | from libs.downloader import DownLoader 23 | from utils.logger import Logger 24 | from utils.readSetting import Config 25 | 26 | ua = UserAgent() 27 | downloader = DownLoader() 28 | log = Logger(filename="log/asyncSpider.log") 29 | 30 | 31 | class Spider(Config): 32 | 33 | def __init__(self): 34 | super().__init__() 35 | self.max_threads = 10 36 | self.payload = { 37 | "session_language": "cn_CN" 38 | } 39 | 40 | @staticmethod 41 | def __parse_results(html): 42 | 43 | try: 44 | html = etree.HTML(html, etree.HTMLParser()) 45 | return html 46 | except Exception as e: 47 | raise e 48 | 49 | def getPageNum(self, url): 50 | """ 51 | 获取网页数量 52 | :param url: 53 | :return: 54 | """ 55 | headers = { 56 | "User-Agent": ua.random, 57 | "Host": self.pornHost, 58 | "referer": url, 59 | "X-Forwarded-For": str(IPv4Address(random.getrandbits(32))) 60 | } 61 | response = requests.request("POST", url, headers=headers, timeout=30) 62 | log.logger.debug(response.status_code) 63 | html = self.__parse_results(response.text) 64 | tmpPages = html.xpath("//div[contains(@class,'pagingnav')]/form/a/text()") 65 | pages = [] 66 | for page in tmpPages: 67 | if page.isdigit(): 68 | pages.append(int(page)) 69 | log.logger.debug("总共页数:{}".format(max(pages))) 70 | return max(pages) 71 | 72 | def getVideoUrlList(self, pageUrl): 73 | """ 74 | 获取每页中的视频信息 75 | :param pageUrl: 76 | :return: 77 | """ 78 | try: 79 | headers = { 80 | "User-Agent": ua.random, 81 | "Host": self.pornHost, 82 | "referer": pageUrl, 83 | "X-Forwarded-For": str(IPv4Address(random.getrandbits(32))) 84 | } 85 | response = requests.request("POST", pageUrl, headers=headers, data=self.payload, timeout=30) 86 | html = self.__parse_results(response.text) 87 | videoUrls = html.xpath("//div[contains(@class,'well well-sm videos-text-align')]/a/@href") 88 | videoThumbs = html.xpath("//div[contains(@class,'well well-sm videos-text-align')]/a/div/img/@src") 89 | videoTitles = html.xpath("//div[contains(@class,'well well-sm videos-text-align')]/a/span[contains(@class,'video-title title-truncate m-t-5')]/text()") 90 | videoDurations = html.xpath("//div[contains(@class,'well well-sm videos-text-align')]/a/div/span/text()") 91 | log.logger.info("videoTitle:{}, videoDuration:{}, videoUrl:{}, videoThumb:{}".format(videoTitles, videoDurations, videoUrls, videoThumbs)) 92 | return videoTitles, videoDurations, videoUrls, videoThumbs 93 | except Exception as e: 94 | log.logger.exception(e) 95 | 96 | async def downloadHtml(self, session, url): 97 | """ 98 | 下载网页内容 99 | :param url: 100 | :return: 101 | """ 102 | headers = { 103 | "User-Agent": ua.random, 104 | "Host": self.pornHost, 105 | "referer": url, 106 | "X-Forwarded-For": str(IPv4Address(random.getrandbits(32))) 107 | } 108 | async with session.get(url, headers=headers, timeout=60) as response: 109 | assert response.status == 200 110 | html = await response.read() 111 | return html 112 | 113 | async def getM38UUrl(self, session, url): 114 | text = await self.downloadHtml(session, url) 115 | try: 116 | videoEncodeUrl = re.compile(r'document.write\(strencode2\(([^)]+)').search(text.decode('utf-8')).group(1) 117 | log.logger.debug(unquote(videoEncodeUrl, encoding='utf-8', errors='replace')) 118 | videoDecodeUrl = re.compile(r'src=\'(http[a-z\.:0-9\/]+)').search(unquote(videoEncodeUrl, encoding='utf-8', errors='replace')).group(1) 119 | return videoDecodeUrl 120 | except Exception as e: 121 | log.logger.exception("获取视频链接:{}出错，抛出异常:{}".format(url, e)) 122 | log.logger.error("视频链接:{}, 获取视频链接异常的网页信息:{}".format(url, text)) 123 | return None 124 | 125 | async def handleTasks(self, task_id, work_queue): 126 | """ 127 | 异步获取m3u8地址 128 | :param task_id: 129 | :param work_queue: 130 | :return: 131 | """ 132 | while not work_queue.empty(): 133 | videoTitle, videoDuration, videoUrl, videoThumbUrl = await work_queue.get() 134 | try: 135 | async with aiohttp.ClientSession() as session: 136 | videoDecodeUrl = await self.getM38UUrl(session, videoUrl) 137 | if videoDecodeUrl: 138 | downloader.run(videoDecodeUrl, videoThumbUrl, videoTitle=videoTitle, videoDuration=videoDuration) 139 | else: 140 | log.logger.exception(videoDecodeUrl) 141 | log.logger.info( 142 | "videoTitle:{}, videoDuration:{}, videoUrl:{}, videoThumb:{}".format(videoTitle, videoDuration, videoDecodeUrl, videoThumbUrl)) 143 | except Exception as e: 144 | log.logger.exception("videoTitle:{}, videoDuration:{}, videoUrl:{}, videoThumb:{}, exception:{}".format(videoTitle, videoDuration, videoUrl, videoThumbUrl, e)) 145 | 146 | def eventLoop(self, pageUrl): 147 | videoTitles, videoDurations, videoUrls, videoThumbs = self.getVideoUrlList(pageUrl) 148 | q = asyncio.Queue() 149 | [q.put_nowait((videoTitles[i], videoDurations[i], videoUrls[i], videoThumbs[i])) for i in range(len(videoTitles))] 150 | # loop = asyncio.new_event_loop() 151 | loop = asyncio.new_event_loop() 152 | asyncio.set_event_loop(loop) 153 | try: 154 | tasks = [self.handleTasks(task_id, q, ) for task_id in range(self.max_threads)] 155 | loop.run_until_complete(asyncio.wait(tasks)) 156 | except RuntimeError as e: 157 | pass 158 | finally: 159 | loop.close() 160 | 161 | # def aa(self, loop, start, end): 162 | # if start == 0: 163 | # start += 1 164 | # 165 | # while start < end: 166 | # url = "{}&page={}".format(self.pornUrl, start) 167 | # self.eventLoop(loop, url) 168 | # start += 1 169 | 170 | def run(self): 171 | pageNum = self.getPageNum(self.pornUrl) 172 | 173 | # part = pageNum // 24 174 | # pool = ThreadPoolExecutor(max_workers=6) 175 | # futures = [] 176 | # loop = asyncio.new_event_loop() 177 | # for i in range(24): 178 | # start = part * i 179 | # if i == 24 -1: 180 | # end = pageNum 181 | # else: 182 | # end = start + part - 1 183 | # log.logger.debug("startPageNum:{}, endPageNum:{}".format(start, end)) 184 | # futures.append(pool.submit(self.aa, loop, start, end)) 185 | # wait(futures) 186 | for i in range(1, int(pageNum)): 187 | url = "{}&page={}".format(self.pornUrl, i) 188 | self.eventLoop(url) 189 | -------------------------------------------------------------------------------- /libs/downloader.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : downloader 5 | Description : 91视频与封面下载，通篇采用了低端的retry机制保证下载和上传的成功率 6 | Author : jccia 7 | date : 2021/5/9 8 | ------------------------------------------------- 9 | """ 10 | import base64 11 | import os 12 | import random 13 | import re 14 | import time 15 | from concurrent.futures import ThreadPoolExecutor, wait 16 | from ipaddress import IPv4Address 17 | 18 | import requests 19 | from fake_useragent import UserAgent 20 | 21 | from utils.github import Github 22 | from utils.logger import Logger 23 | from utils.sqlite import Database 24 | 25 | db = Database() 26 | ua = UserAgent() 27 | github = Github() 28 | log = Logger(filename="log/downloader.log") 29 | 30 | 31 | class DownLoader(object): 32 | 33 | def __init__(self): 34 | if not os.path.exists("video"): 35 | os.mkdir("video") 36 | 37 | def m3u8s(self, url, **kwargs): 38 | """ 39 | 从m3u8文件中提取数据 40 | :param url: 41 | :return: 42 | """ 43 | headers = { 44 | "User-Agent": ua.random, 45 | "Host": "91porn.com" 46 | } 47 | baseUrl = re.compile(r'(.*)/[0-9]+\.m3u8').search(url).group(1) 48 | fileName = re.compile(r'/([0-9]+)\.m3u8').search(url).group(1) 49 | if not os.path.exists("video/{}".format(fileName)): 50 | os.mkdir("video/{}".format(fileName)) 51 | 52 | i = 0 53 | while i < 3: 54 | headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 55 | r = requests.request("GET", url, headers=headers) 56 | if r.status_code != 400: 57 | with open("video/{}/{}.m38u".format(fileName, fileName), "wb") as fp: 58 | fp.write(r.content) 59 | 60 | with open("video/{}/{}.m38u".format(fileName, fileName), "rb+") as file: 61 | urls = [] 62 | files = [] 63 | lines = file.readlines() 64 | for line in lines: 65 | if line.endswith(b".ts\n"): 66 | urls.append(baseUrl + "/" + str(line.strip(b"\n")).replace("\'", "").replace("b", "")) 67 | files.append(str(line.strip(b"\n")).replace("\'", "").replace("b", "")) 68 | 69 | uploadUrl = github.uploadFile(fileName, "{}.m3u8".format(fileName), base64.b64encode(r.content)) 70 | if uploadUrl: 71 | log.logger.info("videoTitle:{}, videoDuration:{}, videoUrl:{}".format(kwargs.get("videoTitle"), 72 | kwargs.get("videoDuration"), 73 | uploadUrl)) 74 | sql = '''INSERT INTO defaultVideo (videoId, videoTitle, videoUrl, videoDuration) VALUES ("%s", "%s", "%s", "%s")''' % ( 75 | fileName, kwargs.get("videoTitle"), uploadUrl, kwargs.get("videoDuration")) 76 | log.logger.info("插入视频基本信息sql:{}".format(sql)) 77 | db.insert(sql) 78 | return status, fileName, urls, files 79 | time.sleep(random.randint(2, 5)) 80 | i += 1 81 | log.logger.warning("url:{}, 下载失败, 返回状态码结果:{}, 返回结果:{}, 重试次数:{}".format(url, r.status_code, r.text, i)) 82 | 83 | def downVideo(self, tsFileName, tsUrl, file): 84 | """ 85 | 下载视频文件和缩略图 86 | :param tsFileName: 87 | :param tsUrl: 88 | :param file: 89 | :param thumbUrl: 90 | :return: 91 | """ 92 | headers = { 93 | "User-Agent": ua.random, 94 | "Host": "91porn.com" 95 | } 96 | i = 0 97 | while i < 3: 98 | headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 99 | r = requests.request("GET", tsUrl, headers=headers) 100 | if r.status_code == 200: 101 | log.logger.info("下载ts文件:{}成功, 返回状态码:{}".format(file, r.status_code)) 102 | github.uploadFile(tsFileName, file, base64.b64encode(r.content)) 103 | 104 | break 105 | time.sleep(random.randint(2, 5)) 106 | i += 1 107 | log.logger.warning("File:{}, 下载失败, 返回状态码结果:{}, 返回结果:{}, 重试次数:{}".format(file, r.status_code, r.text, i)) 108 | 109 | def downThumb(self, tsFileName, thumbUrl, **kwargs): 110 | """ 111 | 下载封面 112 | :param tsFileName: 113 | :param thumbUrl: 114 | :param kwargs: 115 | :return: 116 | """ 117 | headers = { 118 | "User-Agent": ua.random, 119 | "Host": "91porn.com" 120 | } 121 | thumbFileName = re.compile(r'/([0-9]+\.[a-z]+)').search(thumbUrl).group(1) 122 | 123 | i = 0 124 | while i < 3: 125 | headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 126 | r = requests.request("GET", thumbUrl, headers=headers) 127 | if r.status_code == 200: 128 | uploadUrl = github.uploadFile(tsFileName, thumbFileName, base64.b64encode(r.content)) 129 | if uploadUrl: 130 | log.logger.info("videoTitle:{}, videoDuration:{}, videoThumbUrl:{}".format(kwargs.get("videoTitle"), 131 | kwargs.get( 132 | "videoDuration"), 133 | uploadUrl)) 134 | sql = '''update defaultVideo videoPic set value="%s" where videoId=%s''' % (uploadUrl, tsFileName) 135 | log.logger.info("插入图片封面链接sql:{}".format(sql)) 136 | db.insert(sql) 137 | time.sleep(random.randint(2, 5)) 138 | i += 1 139 | log.logger.warning("thumbUrl:{}, 下载失败, 返回状态码结果:{}, 返回结果:{}, 重试次数:{}".format(thumbUrl, r.status_code, r.text, i)) 140 | 141 | def run(self, m3u8Url, thumbUrl, **kwargs): 142 | """ 143 | 下载器主函数 144 | :param m3u8Url: 145 | :param thumbUrl: 146 | :return: 147 | """ 148 | pool = ThreadPoolExecutor(max_workers=20) 149 | futures = [] 150 | tsFileName, tsUrls, tsFiles = self.m3u8s(m3u8Url, videoTitle=kwargs.get("videoTitle"), videoDuration=kwargs.get("videoDuration")) 151 | if tsFileName: 152 | self.downThumb(tsFileName, thumbUrl) 153 | for i in range(len(tsUrls)): 154 | tsUrl = tsUrls[i] 155 | tsFile = tsFiles[i] 156 | futures.append(pool.submit(self.downVideo, tsFileName, tsUrl, tsFile)) 157 | time.sleep(random.random()) 158 | wait(futures) 159 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : main 5 | Description : 主函数入口 6 | Author : jccia 7 | date : 2021/5/11 8 | ------------------------------------------------- 9 | """ 10 | import os 11 | from utils.sqlite import Database 12 | from libs.asyncSpider import Spider 13 | 14 | 15 | def main(): 16 | if not os.path.exists("91porn.db"): 17 | db = Database() 18 | db.initDb() 19 | 20 | spider = Spider() 21 | spider.run() 22 | 23 | 24 | if __name__ == '__main__': 25 | main() 26 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aiohttp==3.7.4.post0 2 | aiohttp-socks==0.6.0 3 | async-timeout==3.0.1 4 | attrs==21.2.0 5 | beautifulsoup4==4.9.3 6 | certifi==2020.12.5 7 | chardet==4.0.0 8 | configparser==5.0.2 9 | fake-useragent==0.1.11 10 | idna==2.10 11 | iso8601==0.1.14 12 | lxml==4.6.3 13 | m3u8==0.8.0 14 | multidict==5.1.0 15 | PySocks==1.7.1 16 | python-socks==1.2.4 17 | requests==2.25.1 18 | soupsieve==2.2.1 19 | typing-extensions==3.10.0.0 20 | urllib3==1.26.4 21 | yarl==1.6.3 22 | -------------------------------------------------------------------------------- /utils/decorators.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : decorators 5 | Description : 6 | Author : jccia 7 | date : 2021/5/11 8 | ------------------------------------------------- 9 | """ 10 | from functools import wraps 11 | from utils.sqlite import Database 12 | 13 | db = Database() 14 | 15 | 16 | def insertDataToDb(func): 17 | 18 | @wraps(func) 19 | def decorator(self, url): 20 | fileName, urls, files = func(self, url) 21 | 22 | return decorator 23 | 24 | -------------------------------------------------------------------------------- /utils/github.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : github 5 | Description : github接口操作 6 | Author : jccia 7 | date : 2021/5/11 8 | ------------------------------------------------- 9 | """ 10 | import requests 11 | import random 12 | import time 13 | import json 14 | from ipaddress import IPv4Address 15 | from utils.logger import Logger 16 | from utils.readSetting import Config 17 | 18 | log = Logger("log/github.log") 19 | 20 | 21 | class Github(Config): 22 | 23 | def __init__(self): 24 | super().__init__() 25 | # self.username = UserName 26 | # self.repository = Repository 27 | self.params = { 28 | "access_token": self.githubAccessToken 29 | } 30 | self.headers = { 31 | 'Content-Type': 'application/json', 32 | "Authorization": "token {}".format(self.githubAccessToken) 33 | } 34 | 35 | def checkRepoSize(self): 36 | """ 37 | 检查仓库大小，考虑到后续超1G后切换仓库 38 | :return: 39 | """ 40 | url = "https://api.github.com/repos/{}/{}".format(self.githubUserName, self.githubRepository) 41 | self.headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 42 | try: 43 | response = requests.request("GET", url, headers=self.headers) 44 | assert response.status_code == 200 45 | return int(response.json()["size"]) 46 | except Exception as e: 47 | log.logger.exception(e) 48 | return None 49 | 50 | def createRepo(self): 51 | """ 52 | 新建github仓库 53 | :return: 54 | """ 55 | url = "https://api.github.com/{}/repos".format(self.githubUserName) 56 | self.headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 57 | payload = { 58 | "name": "{}{}".format(self.githubRepository, random.randint(1, 10)) 59 | } 60 | 61 | try: 62 | response = requests.request("GET", url, headers=self.headers, data=json.dumps(payload)) 63 | assert response.status_code == 200 64 | log.logger.debug(response.text) 65 | except Exception as e: 66 | log.logger.exception(e) 67 | return None 68 | 69 | def uploadFile(self, FileName, File, Content): 70 | """ 71 | 上传文件至github 72 | :param FileName: 73 | :param File: 74 | :param Content: 75 | :return: 76 | """ 77 | url = "https://api.github.com/repos/{}/{}/contents/{}/{}".format(self.githubUserName, self.githubRepository, FileName, File) 78 | payload = { 79 | "message": "upload by spider", 80 | "committer": { 81 | "name": self.githubUserName, 82 | "email": "915617545@qq.com" 83 | }, 84 | "content": bytes.decode(Content) 85 | } 86 | self.headers["X-Forwarded-For"] = str(IPv4Address(random.getrandbits(32))) 87 | # socks.set_default_proxy(socks.SOCKS5, "127.0.0.1", 1080) 88 | # socket.socket = socks.socksocket 89 | i = 0 90 | while i < 5: 91 | response = requests.request("PUT", url, headers=self.headers, data=json.dumps(payload)) 92 | if response.status_code == 201 or response.status_code == 200: 93 | log.logger.info("File:{}, 上传成功".format(File)) 94 | url = "https://cdn.jsdelivr.net/gh/{}/{}/{}/{}".format(self.githubUserName, self.githubRepository, FileName, File) 95 | log.logger.info(url) 96 | return url 97 | 98 | time.sleep(random.randint(2, 5)) 99 | i += 1 100 | log.logger.warning("File:{}, 上传失败, 返回状态码结果:{}, 返回结果:{}, 重试次数:{}".format(File, response.status_code, response.text, i)) -------------------------------------------------------------------------------- /utils/logger.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : logger 5 | Description : 6 | Author : jccia 7 | date : 2021/5/10 8 | ------------------------------------------------- 9 | """ 10 | import logging 11 | import os 12 | from logging import handlers 13 | 14 | if not os.path.exists("log"): 15 | os.mkdir("log") 16 | 17 | 18 | class Logger(object): 19 | """ 20 | 即在终端打印日志也在文件中打印日志 21 | """ 22 | level_relations = { 23 | 'debug': logging.DEBUG, 24 | 'info': logging.INFO, 25 | 'warning': logging.WARNING, 26 | 'error': logging.ERROR, 27 | 'crit': logging.CRITICAL 28 | } 29 | 30 | def __init__(self, filename, level='debug', when='D', backCount=3, 31 | fmt='[%(asctime)s] [%(levelname)s] [%(filename)s:%(module)s.%(funcName)s:%(lineno)d] [%(process)d] %(message)s'): 32 | self.logger = logging.getLogger(filename) 33 | format_str = logging.Formatter(fmt) 34 | self.logger.setLevel(self.level_relations.get(level)) 35 | sh = logging.StreamHandler() 36 | sh.setFormatter(format_str) 37 | th = handlers.TimedRotatingFileHandler(filename=filename, when=when, backupCount=backCount, 38 | encoding='utf-8') 39 | th.setFormatter(format_str) 40 | self.logger.addHandler(sh) 41 | self.logger.addHandler(th) 42 | -------------------------------------------------------------------------------- /utils/readSetting.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : readSetting 5 | Description : 读取配置文件 6 | Author : jccia 7 | date : 2021/5/11 8 | ------------------------------------------------- 9 | """ 10 | import configparser 11 | 12 | 13 | class Config(object): 14 | 15 | def __init__(self): 16 | self.conf = configparser.ConfigParser() 17 | cfg = "config.ini" 18 | self.conf.read(cfg, encoding="utf-8") 19 | 20 | self.githubUserName = self.conf.get("github", "UserName") 21 | self.githubRepository = self.conf.get("github", "Repository") 22 | self.githubAccessToken = self.conf.get("github", "AccessToken") 23 | 24 | self.pornUrl = self.conf.get("91porn", "url") 25 | self.pornHost = self.conf.get("91porn", "host") 26 | 27 | self.dbPath = self.conf.get("sqlite", "path") 28 | 29 | 30 | if __name__ == '__main__': 31 | a = Config() 32 | -------------------------------------------------------------------------------- /utils/sqlite.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | ------------------------------------------------- 4 | File Name : sqlite 5 | Description : 6 | Author : jccia 7 | date : 2021/5/11 8 | ------------------------------------------------- 9 | """ 10 | import sqlite3 11 | import threading 12 | from utils.readSetting import Config 13 | from utils.logger import Logger 14 | 15 | sqliteMutex = threading.Lock() 16 | 17 | log = Logger(filename="log/sqlite.log") 18 | 19 | 20 | class Database(Config): 21 | 22 | def __init__(self): 23 | super(Database, self).__init__() 24 | 25 | def initDb(self): 26 | con = sqlite3.connect(self.dbPath) 27 | dbTable = "defaultVideo" 28 | videoInfo = '''CREATE TABLE IF NOT EXISTS %s 29 | ( 30 | id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, 31 | videoId VARCHAR(20), 32 | videoTitle VARCHAR(20), 33 | videoUrl VARCHAR(20), 34 | videoDuration VARCHAR(20), 35 | videoPic VARCHAR(20), 36 | date TIMESTAMP NOT NULL DEFAULT (datetime('now','localtime')) 37 | ); 38 | ''' % dbTable 39 | 40 | cur = con.cursor() 41 | cur.execute(videoInfo) 42 | con.commit() 43 | con.close() 44 | 45 | def select(self, sql): 46 | """ 47 | select 操作，加锁防止deadlock 48 | :param sql: 49 | :return: 50 | """ 51 | with sqliteMutex: 52 | con = sqlite3.connect(self.dbPath) 53 | cur = con.cursor() 54 | cur.execute(sql) 55 | con.close() 56 | return cur 57 | 58 | def insert(self, sql): 59 | """ 60 | insert 操作，加锁防止deadlock 61 | :param sql: 62 | :return: 63 | """ 64 | with sqliteMutex: 65 | con = sqlite3.connect(self.dbPath) 66 | cur = con.cursor() 67 | try: 68 | cur.execute(sql) 69 | con.commit() 70 | except Exception as e: 71 | log.logger.exception("sql:{}, 运行异常:{}".format(sql, e)) 72 | con.rollback() 73 | finally: 74 | con.close() 75 | --------------------------------------------------------------------------------