├── .gitattributes ├── .gitignore ├── LICENSE ├── README.md ├── helper ├── __init__.py ├── config.py ├── constants.py ├── extractor.py ├── login.py ├── main.py ├── node.py ├── problems.py ├── templates.py └── utils.py ├── imgs ├── example_cn.png ├── example_en.png ├── leetcode-logo.png ├── problem.png └── run.png ├── run.py └── tests ├── __init__.py └── test_config.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | db/ 2 | .vscode 3 | config.json 4 | **/__pycache__ 5 | **/.ipynb_checkpoints/ 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 KivenChen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 6 | # LeetCode_Helper 7 | 8 | ## 概述 9 | 10 | Python 实现的 LeetCode 仓库美化程序。爬取 LeetCode-cn AC 的题目描述和提交的代码，并整理至相应的文件夹，生成相应的 README 文件。 11 | 12 | 本项目参考了: 13 | 14 | - [leetcode-spider](https://github.com/zhantong/leetcode-spider) 15 | - [LeetCodeCrawler](https://github.com/ZhaoxiZhang/LeetCodeCrawler) 16 | 17 | ## 特点 18 | 19 | - 支持爬取题目列表（中英文），保存为指定目录下的 README 和 README_EN 文件 20 | - 支持爬取题目描述（中英文），保存为对应 title 文件夹下的 README 和 README_EN 文件 21 | - 支持爬取用户提交的代码，保存为对应 title 文件夹下的 AC 源码（可以是任意语言） 22 | - 支持修改导出数据的模板 23 | - 异步下载题目描述，高速并发导出文件 24 | - 支持增量更新，当 LeetCode-cn 有新内容（题目/提交的代码）时，可以选择增量形式更新 25 | 26 | ## 使用 27 | 28 | 使用 `git clone` 或直接下载本仓库代码至本地 29 | 30 | 本项目需要用到第三方库 `requests` 和 `aiohttp`，可通过 `pip` 命令安装。 31 | 32 | 运行 `python run.py` 33 | 34 | ![](imgs/run.png) 35 | 36 | ## 效果 37 | 38 | 具体效果以及爬取的具体数据可参看我的 repo: [LeetCode](https://github.com/KivenCkl/LeetCode) 39 | 40 | ![](imgs/example_cn.png) 41 | 42 | ![](imgs/example_en.png) 43 | 44 | ![](imgs/problem.png) 45 | 46 | 你可以根据你自己的需求爱好修改 `templates.py` 其中的模板 47 | 48 | 可以修改其根目录下的 `config.json` 文件: 49 | 50 | ``` json 51 | { 52 | "username": "leetcode-cn@leetcode", 53 | "password": "leetcode", 54 | "outputDir": "../LeetCode", 55 | "timeInterval": 0.1 56 | } 57 | ``` 58 | 59 | - `username` 和 `password` 对应你的 LeetCode-cn 账号和密码 60 | - `outputDir` 对应你希望存放源码文件的目录 61 | - `timeInterval` 为访问 LeetCode-cn 的时间间隔，默认为 0.1s 62 | -------------------------------------------------------------------------------- /helper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/helper/__init__.py -------------------------------------------------------------------------------- /helper/config.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-22 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | import json 7 | import os 8 | 9 | 10 | class Config: 11 | ''' 12 | 获取配置信息并储存至 `config.json` 13 | ''' 14 | def __init__(self): 15 | self.data = self.__getConfig() 16 | 17 | def __getConfig(self): 18 | path = os.path.join(os.path.abspath(os.path.join(__file__, "../..")), 19 | 'config.json') 20 | if not os.path.exists(path): 21 | username = input('请输入您的用户名: ') 22 | password = input('请输入您的密码: ') 23 | outputDir = input('请选择您要输出的目录: ') 24 | data = dict(username=username, 25 | password=password, 26 | outputDir=outputDir, 27 | timeInterval=0.1) 28 | with open(path, 'w') as f: 29 | json.dump(data, f) 30 | return data 31 | else: 32 | with open(path, 'r') as f: 33 | return json.load(f) 34 | 35 | def __getData(self, item): 36 | return self.data.get(item) if self.data else None 37 | 38 | @property 39 | def username(self): 40 | return self.__getData('username') 41 | 42 | @property 43 | def password(self): 44 | return self.__getData('password') 45 | 46 | @property 47 | def outputDir(self): 48 | return self.__getData('outputDir') 49 | 50 | @property 51 | def timeInterval(self): 52 | return self.__getData('timeInterval') or 0.1 53 | 54 | 55 | config = Config() 56 | -------------------------------------------------------------------------------- /helper/constants.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-22 4 | @LastEditTime: 2019-05-23 5 | ''' 6 | 7 | # LeetCode 相关链接 8 | LEETCODE = "https://leetcode-cn.com" 9 | GRAPHQL = LEETCODE + "/graphql" 10 | LOGIN = LEETCODE + "/accounts/login/" 11 | PROBLEMS = LEETCODE + "/api/problems/all/" 12 | SUBMISSIONS_FORMAT = LEETCODE + "/api/submissions/?offset={}&limit=20" 13 | CODE_FORMAT = LEETCODE + "/submissions/latest/?qid={}&lang={}" 14 | TAG_FORMAT = LEETCODE + "/tag/{}" 15 | PROBLEM_FORMAT = LEETCODE + "/problems/{}/" 16 | 17 | HEADERS = { 18 | 'Origin': 19 | LEETCODE, 20 | 'User-Agent': 21 | 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134' 22 | } 23 | 24 | DIFFICULTY_CN = ['', '简单', '中等', '困难'] 25 | DIFFICULTY_EN = ['', 'easy', 'medium', 'hard'] 26 | 27 | LANGS = { 28 | 'bash': { 29 | 'lang': 'Shell', 30 | 'ext': 'sh', 31 | 'style': '#' 32 | }, 33 | 'c': { 34 | 'lang': 'C', 35 | 'ext': 'c', 36 | 'style': '//' 37 | }, 38 | 'cpp': { 39 | 'lang': 'C++', 40 | 'ext': 'cpp', 41 | 'style': '//' 42 | }, 43 | 'csharp': { 44 | 'lang': 'C#', 45 | 'ext': 'cs', 46 | 'style': '//' 47 | }, 48 | 'golang': { 49 | 'lang': 'Go', 50 | 'ext': 'go', 51 | 'style': '//' 52 | }, 53 | 'java': { 54 | 'lang': 'Java', 55 | 'ext': 'java', 56 | 'style': '//' 57 | }, 58 | 'javascript': { 59 | 'lang': 'JavaScript', 60 | 'ext': 'js', 61 | 'style': '//' 62 | }, 63 | 'kotlin': { 64 | 'lang': 'Kotlin', 65 | 'ext': 'kt', 66 | 'style': '//' 67 | }, 68 | 'mysql': { 69 | 'lang': 'SQL', 70 | 'ext': 'sql', 71 | 'style': '--' 72 | }, 73 | 'python': { 74 | 'lang': 'Python', 75 | 'ext': 'py', 76 | 'style': '#' 77 | }, 78 | 'python3': { 79 | 'lang': 'Python3', 80 | 'ext': 'py', 81 | 'style': '#' 82 | }, 83 | 'ruby': { 84 | 'lang': 'Ruby', 85 | 'ext': 'rb', 86 | 'style': '#' 87 | }, 88 | 'rust': { 89 | 'lang': 'Rust', 90 | 'ext': 'rs', 91 | 'style': '//' 92 | }, 93 | 'scala': { 94 | 'lang': 'Scala', 95 | 'ext': 'scala', 96 | 'style': '//' 97 | }, 98 | 'swift': { 99 | 'lang': 'Swift', 100 | 'ext': 'swift', 101 | 'style': '//' 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /helper/extractor.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-26 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | import os 7 | import time 8 | import concurrent.futures 9 | from .templates import * 10 | from .constants import * 11 | 12 | 13 | class Extractor: 14 | ''' 15 | 高速并发导出数据类 16 | ''' 17 | def __init__(self, output_dir, author): 18 | self.output_dir = output_dir 19 | self.author = author 20 | 21 | def extractInfo(self, info, datas): 22 | '''导出 LeetCode README 文件''' 23 | # 当前时间 24 | cur_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) 25 | readme_cn_path = os.path.join(self.output_dir, 'README.md') 26 | with open(readme_cn_path, 'w', encoding='utf-8') as f: 27 | f.write( 28 | TEMPLATE_README_CN.format( 29 | user_name=info.user_name, 30 | num_solved=info.num_solved, 31 | num_total=info.num_total, 32 | ac_easy=info.ac_easy, 33 | ac_medium=info.ac_medium, 34 | ac_hard=info.ac_hard, 35 | time=cur_time)) 36 | solutions = [] 37 | for i, data in enumerate(datas): 38 | solutions.append('[{0}](Problemset/{1}/{1}.{2})'.format( 39 | data["language"], data["title_slug"], 40 | LANGS[data["lang"]]["ext"])) 41 | title = '[{}](Problemset/{}/README.md)'.format( 42 | data["title_cn"], data["title_slug"]) 43 | # 判断同一问题是否有多个解 44 | if i == len(datas) - 1 or datas[i]['title_en'] != datas[ 45 | i + 1]['title_en']: 46 | f.write( 47 | TEMPLATE_README_APPEND.format( 48 | frontend_id=data['frontend_id'], 49 | title=title, 50 | paid_only=data['paid_only'], 51 | is_favor=data['is_favor'], 52 | solutions='
'.join(solutions), 53 | ac_rate=data['ac_rate'], 54 | difficulty=DIFFICULTY_CN[data['difficulty']], 55 | tags=data['tags_cn'].replace('- ', '').replace( 56 | '\n', '
'))) 57 | solutions = [] 58 | print(f'{os.path.abspath(readme_cn_path)} done!') 59 | readme_en_path = os.path.join(self.output_dir, 'README_EN.md') 60 | with open(readme_en_path, 'w', encoding='utf-8') as f: 61 | f.write( 62 | TEMPLATE_README_EN.format( 63 | user_name=info.user_name, 64 | num_solved=info.num_solved, 65 | num_total=info.num_total, 66 | ac_easy=info.ac_easy, 67 | ac_medium=info.ac_medium, 68 | ac_hard=info.ac_hard, 69 | time=cur_time)) 70 | solutions = [] 71 | for i, data in enumerate(datas): 72 | solutions.append('[{0}](Problemset/{1}/{1}.{2})'.format( 73 | data["language"], data["title_slug"], 74 | LANGS[data["lang"]]["ext"])) 75 | title = '[{}](Problemset/{}/README_EN.md)'.format( 76 | data["title_en"], data["title_slug"]) 77 | if i == len(datas) - 1 or datas[i]['title_en'] != datas[ 78 | i + 1]['title_en']: 79 | f.write( 80 | TEMPLATE_README_APPEND.format( 81 | frontend_id=data['frontend_id'], 82 | title=title, 83 | paid_only=data['paid_only'], 84 | is_favor=data['is_favor'], 85 | solutions='
'.join(solutions), 86 | ac_rate=data['ac_rate'], 87 | difficulty=DIFFICULTY_EN[data['difficulty']], 88 | tags=data['tags_en'].replace('- ', '').replace( 89 | '\n', '
'))) 90 | solutions = [] 91 | print(f'{os.path.abspath(readme_en_path)} done!') 92 | 93 | def __extractDesc(self, data): 94 | if data['d_stored'] == 1: 95 | return 96 | folder_path = os.path.join(self.output_dir, 'Problemset', 97 | data['title_slug']) 98 | if not os.path.exists(folder_path): 99 | # 创建问题文件夹 100 | os.makedirs(folder_path, exist_ok=True) 101 | readme_cn_path = os.path.join(folder_path, 'README.md') 102 | title_cn = '[{}. {}]({})'.format( 103 | data['frontend_id'], data['title_cn'], 104 | PROBLEM_FORMAT.format(data['title_slug'])) 105 | with open(readme_cn_path, 'w', encoding='utf-8') as f: 106 | f.write( 107 | TEMPLATE_DESC_CN.format( 108 | title_cn=title_cn, 109 | content_cn=data['content_cn'], 110 | tags_cn=data['tags_cn'], 111 | similar_questions_cn=data['similar_questions_cn'])) 112 | print(f'{os.path.abspath(readme_cn_path)} done!') 113 | 114 | readme_en_path = os.path.join(folder_path, 'README_EN.md') 115 | title_en = '[{}. {}]({})'.format( 116 | data['frontend_id'], data['title_en'], 117 | PROBLEM_FORMAT.format(data['title_slug'])) 118 | with open(readme_en_path, 'w', encoding='utf-8') as f: 119 | f.write( 120 | TEMPLATE_DESC_EN.format( 121 | title_en=title_en, 122 | content_en=data['content_en'], 123 | tags_en=data['tags_en'], 124 | similar_questions_en=data['similar_questions_en'])) 125 | print(f'{os.path.abspath(readme_en_path)} done!') 126 | 127 | def extractDesc(self, datas): 128 | '''导出问题描述''' 129 | with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: 130 | # futures = {executor.submit(self.__extractDesc, data): data for data in datas} 131 | # for future in concurrent.futures.as_completed(futures): 132 | # data = futures[future] 133 | for _ in executor.map(self.__extractDesc, datas): 134 | pass 135 | 136 | def __extractCode(self, data): 137 | if data['s_stored'] == 1: 138 | return 139 | folder_path = os.path.join(self.output_dir, 'Problemset', 140 | data['title_slug']) 141 | if not os.path.exists(folder_path): 142 | os.makedirs(folder_path) 143 | code_path = os.path.join( 144 | folder_path, f'{data["title_slug"]}.{LANGS[data["lang"]]["ext"]}') 145 | with open(code_path, 'w', encoding='utf-8') as f: 146 | f.write( 147 | TEMPLATE_CODE.format( 148 | style=LANGS[data['lang']]['style'], 149 | title_cn=data['title_cn'], 150 | title_en=data['title_en'], 151 | author=self.author, 152 | timestamp=data['timestamp'], 153 | runtime=data['runtime'], 154 | memory=data['memory'], 155 | code=data['code'])) 156 | print(f'{os.path.abspath(code_path)} done!') 157 | 158 | def extractCode(self, datas): 159 | '''导出代码''' 160 | with concurrent.futures.ThreadPoolExecutor(max_workers=20) as executor: 161 | for _ in executor.map(self.__extractCode, datas): 162 | pass 163 | -------------------------------------------------------------------------------- /helper/login.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-22 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | import requests 7 | from .constants import LEETCODE, LOGIN, HEADERS 8 | 9 | 10 | class Login: 11 | ''' 12 | 登录 LeetCode-cn, 获取 cookies 值 13 | ''' 14 | def __init__(self, username, password): 15 | self.username = username 16 | self.password = password 17 | self.__cookies = '' 18 | self.status = False 19 | 20 | def doLogin(self): 21 | resp = requests.get(LEETCODE, headers=HEADERS) 22 | # token = resp.cookies['csrftoken'] 23 | token = "" 24 | headers = HEADERS.copy() 25 | headers.update({ 26 | 'referer': LOGIN, 27 | 'x-csrftoken': token, 28 | 'x-requested-with': 'XMLHttpRequest' 29 | }) 30 | payload = { 31 | 'login': self.username, 32 | 'password': self.password, 33 | 'csrfmiddlewaretoken': token 34 | } 35 | cookies = {'csrftoken': token} 36 | resp = requests.post( 37 | LOGIN, data=payload, headers=headers, cookies=cookies) 38 | if resp.status_code == 200: 39 | self.status = True 40 | self.__cookies = resp.cookies 41 | # user = resp.json()['form']['fields']['login']['value'] 42 | if self.status: 43 | print(f'{self.username} 登录成功！') 44 | else: 45 | print('登录失败！') 46 | print('请检查用户名和密码！') 47 | 48 | @property 49 | def cookies(self): 50 | if not self.status: 51 | self.doLogin() 52 | return self.__cookies 53 | -------------------------------------------------------------------------------- /helper/main.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-23 4 | ''' 5 | from .problems import Problems 6 | import asyncio 7 | 8 | 9 | class Main: 10 | '''主程序''' 11 | def __init__(self): 12 | self.problems = Problems() 13 | 14 | def __info(self): 15 | print(self.problems.info) 16 | 17 | def update(self): 18 | '''更新数据''' 19 | self.__info() 20 | loop = asyncio.get_event_loop() 21 | loop.run_until_complete(self.problems.update()) 22 | 23 | def rebuild(self): 24 | '''重建数据''' 25 | self.__info() 26 | self.problems.clearDB() 27 | self.update() 28 | -------------------------------------------------------------------------------- /helper/node.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-24 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | import json 7 | import re 8 | import time 9 | from .constants import LANGS, TAG_FORMAT 10 | 11 | 12 | class ProblemInfoNode: 13 | '''解析问题基本信息''' 14 | def __init__(self, json_data): 15 | self.difficulty = json_data['difficulty']['level'] 16 | self.is_favor = self.__formFavor(json_data['is_favor']) 17 | self.paid_only = self.__formPaid(json_data['paid_only']) 18 | self.status = json_data['status'] 19 | self.id = json_data['stat']['question_id'] 20 | # self.frontend_id = self.__formId( 21 | # json_data['stat']['frontend_question_id']) 22 | self.frontend_id = json_data['stat']['frontend_question_id'] 23 | self.title_en = json_data['stat']['question__title'] 24 | self.title_slug = json_data['stat']['question__title_slug'] 25 | self.total_acs = json_data['stat']['total_acs'] 26 | self.total_submitted = json_data['stat']['total_submitted'] 27 | 28 | @property 29 | def ac_rate(self): 30 | return '{:.1%}'.format(self.total_acs / (self.total_submitted + 1)) 31 | 32 | def __formFavor(self, is_favor): 33 | return '❤️' if is_favor else '' 34 | 35 | def __formPaid(self, paid_only): 36 | return '🔒' if paid_only else '' 37 | 38 | def __formId(self, id): 39 | return '{:0>4d}'.format(id) 40 | 41 | 42 | class ProblemDescNode: 43 | '''解析问题描述信息''' 44 | def __init__(self, json_data): 45 | self.id = json_data['data']['question']['questionId'] 46 | self.content_en = self.__formContentEN( 47 | json_data['data']['question']['content']) 48 | self.title_cn = json_data['data']['question']['translatedTitle'] 49 | self.content_cn = self.__formContentCN( 50 | json_data['data']['question']['translatedContent']) 51 | self.similar_questions_cn, self.similar_questions_en = self.__formSimilarQuestions( 52 | json_data['data']['question']['similarQuestions']) 53 | self.tags_cn, self.tags_en = self.__formTags( 54 | json_data['data']['question']['topicTags']) 55 | # self.hints = '\n'.join(json_data['data']['question']['hints']) 56 | 57 | def __formSimilarQuestions(self, similar_questions): 58 | question_list = re.findall(r'{.*?}', similar_questions) 59 | similar_questions_cn, similar_questions_en = [], [] 60 | if question_list: 61 | for q in question_list: 62 | data = json.loads(q) 63 | similar_questions_cn.append('- [{}](../{}/README.md)'.format( 64 | data['translatedTitle'], data['titleSlug'])) 65 | similar_questions_en.append( 66 | '- [{}](../{}/README_EN.md)'.format( 67 | data['title'], data['titleSlug'])) 68 | return '\n'.join(similar_questions_cn), '\n'.join(similar_questions_en) 69 | 70 | def __formTags(self, tags): 71 | tags_cn, tags_en = [], [] 72 | for tag in tags: 73 | tags_cn.append( 74 | f'- [{tag["translatedName"]}]({TAG_FORMAT.format(tag["slug"])})' 75 | ) 76 | tags_en.append( 77 | f'- [{tag["name"]}]({TAG_FORMAT.format(tag["slug"])})') 78 | return '\n'.join(tags_cn), '\n'.join(tags_en) 79 | 80 | def __formContentCN(self, content): 81 | if not content: 82 | return 83 | return content.replace('↵↵', '').replace('↵', '\n') 84 | 85 | def __formContentEN(self, content): 86 | if not content: 87 | return 88 | return content.replace('↵', '').replace('\r\n', '\n') 89 | 90 | 91 | class SubmissionNode: 92 | '''解析提交的代码信息''' 93 | def __init__(self, dic): 94 | self.submission_id = dic['id'] 95 | self.lang = dic['lang'] 96 | self.memory = dic['memory'] 97 | self.runtime = dic['runtime'] 98 | self.timestamp = self.__formTime(int(dic['timestamp'])) 99 | self.title_slug = dic['title_slug'] 100 | 101 | @property 102 | def language(self): 103 | return LANGS[self.lang]['lang'] 104 | 105 | def __formTime(self, timeStamp): 106 | return time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(timeStamp)) 107 | 108 | def __formCode(self, code): 109 | return code.replace('↵', '\n') 110 | 111 | 112 | class InfoNode: 113 | '''解析用户基本信息''' 114 | def __init__(self, json_data): 115 | self.user_name = json_data.get('user_name') 116 | self.ac_easy = json_data.get('ac_easy') 117 | self.ac_medium = json_data.get('ac_medium') 118 | self.ac_hard = json_data.get('ac_hard') 119 | self.num_solved = json_data.get('num_solved') 120 | self.num_total = json_data.get('num_total') 121 | 122 | def __repr__(self): 123 | return ( 124 | f'user_name: {self.user_name}\nac_easy: {self.ac_easy}\nac_medium: {self.ac_medium}\nac_hard: {self.ac_hard}\nnum_solved: {self.num_solved}\nnum_total: {self.num_total}' 125 | ) 126 | -------------------------------------------------------------------------------- /helper/problems.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-23 4 | @LastEditTime: 2019-05-23 5 | ''' 6 | import os 7 | import sqlite3 8 | import requests 9 | import asyncio 10 | import aiohttp 11 | from .config import config 12 | from .login import Login 13 | from .extractor import Extractor 14 | from .utils import handle_tasks 15 | from .node import InfoNode, ProblemInfoNode, ProblemDescNode, SubmissionNode 16 | from .constants import PROBLEMS, HEADERS, GRAPHQL, CODE_FORMAT 17 | 18 | 19 | class Problems: 20 | '''核心逻辑''' 21 | def __init__(self): 22 | self.login = Login(config.username, config.password) 23 | self.__db_dir = os.path.abspath(os.path.join(__file__, "../..", "db")) 24 | if not os.path.exists(self.__db_dir): 25 | os.makedirs(self.__db_dir) 26 | self.db_path = os.path.join(self.__db_dir, "leetcode.db") 27 | self.__cookies = self.login.cookies 28 | self.problems_json = self.__getProblemsJson() 29 | 30 | def __getProblemsJson(self): 31 | resp = requests.get(PROBLEMS, headers=HEADERS, cookies=self.__cookies) 32 | if resp.status_code == 200: 33 | return resp.json() 34 | 35 | @property 36 | def info(self): 37 | '''获取用户基本信息''' 38 | return InfoNode(self.problems_json) 39 | 40 | def __dict_factory(self, cursor, row): 41 | '''修改 SQLite 数据呈现方式''' 42 | d = {} 43 | for idx, col in enumerate(cursor.description): 44 | d[col[0]] = row[idx] 45 | return d 46 | 47 | def updateProblemsInfo(self): 48 | '''更新问题基本信息''' 49 | problems_list = self.problems_json.get('stat_status_pairs') 50 | conn = sqlite3.connect(self.db_path) 51 | c = conn.cursor() 52 | c.execute(''' 53 | CREATE TABLE IF NOT EXISTS problem ( 54 | id INTEGER, 55 | frontend_id TEXT, 56 | title_en TEXT, 57 | title_slug TEXT, 58 | difficulty INTEGER, 59 | paid_only INTEGER, 60 | is_favor INTEGER, 61 | status TEXT, 62 | total_acs INTEGER, 63 | total_submitted INTEGER, 64 | ac_rate TEXT, 65 | PRIMARY KEY(id) 66 | ) 67 | ''') 68 | c.execute('DELETE FROM problem') 69 | for problem in problems_list: 70 | p = ProblemInfoNode(problem) 71 | c.execute( 72 | ''' 73 | INSERT INTO problem ( 74 | id, frontend_id, title_en, title_slug, difficulty, paid_only, is_favor, status, total_acs, total_submitted, ac_rate 75 | ) 76 | VALUES ( 77 | ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? 78 | ) 79 | ''', (p.id, p.frontend_id, p.title_en, p.title_slug, 80 | p.difficulty, p.paid_only, p.is_favor, p.status, 81 | p.total_acs, p.total_submitted, p.ac_rate)) 82 | conn.commit() 83 | conn.close() 84 | 85 | async def __getProblemDesc(self, title_slug): 86 | payload = { 87 | 'query': ''' 88 | query questionData($titleSlug: String!) { 89 | question(titleSlug: $titleSlug) { 90 | questionId 91 | content 92 | translatedTitle 93 | translatedContent 94 | similarQuestions 95 | topicTags { 96 | name 97 | slug 98 | translatedName 99 | } 100 | hints 101 | } 102 | } 103 | ''', 104 | 'operationName': 'questionData', 105 | 'variables': { 106 | 'titleSlug': title_slug 107 | } 108 | } 109 | async with aiohttp.ClientSession(cookies=self.__cookies) as session: 110 | async with session.post(GRAPHQL, json=payload, 111 | headers=HEADERS) as resp: 112 | return await resp.json() 113 | 114 | async def storeProblemsDesc(self): 115 | '''存储 AC 问题描述信息''' 116 | conn = sqlite3.connect(self.db_path) 117 | c = conn.cursor() 118 | c.execute("SELECT title_slug FROM problem WHERE status == 'ac'") 119 | res = c.fetchall() 120 | if not res: 121 | return 122 | loop = asyncio.get_event_loop() 123 | problems_list = await handle_tasks( 124 | loop, self.__getProblemDesc, [dict(title_slug=t[0]) for t in res]) 125 | c.execute(''' 126 | CREATE TABLE IF NOT EXISTS description ( 127 | id INTEGER, 128 | content_en TEXT, 129 | title_cn TEXT, 130 | content_cn TEXT, 131 | similar_questions_cn TEXT, 132 | similar_questions_en TEXT, 133 | tags_cn TEXT, 134 | tags_en TEXT, 135 | d_stored INTEGER DEFAULT 0, 136 | PRIMARY KEY(id) 137 | ) 138 | ''') 139 | for problem in problems_list: 140 | p = ProblemDescNode(problem) 141 | c.execute( 142 | ''' 143 | INSERT OR IGNORE INTO description ( 144 | id, content_en, title_cn, content_cn, similar_questions_cn, similar_questions_en, tags_cn, tags_en 145 | ) 146 | VALUES ( 147 | ?, ?, ?, ?, ?, ?, ?, ? 148 | ) 149 | ''', (p.id, p.content_en, p.title_cn, p.content_cn, 150 | p.similar_questions_cn, p.similar_questions_en, 151 | p.tags_cn, p.tags_en)) 152 | conn.commit() 153 | conn.close() 154 | 155 | def updateProblemsDesc(self): 156 | '''更新处理后的 description 数据库''' 157 | conn = sqlite3.connect(self.db_path) 158 | c = conn.cursor() 159 | c.execute('UPDATE description SET d_stored=1') 160 | conn.commit() 161 | conn.close() 162 | 163 | async def __getSubmissions(self, title_slug, offset=0, limit=500): 164 | payload = { 165 | 'query': ''' 166 | query submissions($offset: Int!, $limit: Int!, $lastKey: String, $questionSlug: String!) { 167 | submissionList(offset: $offset, limit: $limit, lastKey: $lastKey, questionSlug: $questionSlug) { 168 | lastKey 169 | hasNext 170 | submissions { 171 | id 172 | statusDisplay 173 | lang 174 | runtime 175 | timestamp 176 | url 177 | isPending 178 | memory 179 | __typename 180 | } 181 | __typename 182 | } 183 | } 184 | ''', 185 | 'operationName': 'submissions', 186 | 'variables': { 187 | 'limit': limit, 188 | 'offset': offset, 189 | 'questionSlug': title_slug 190 | } 191 | } 192 | async with aiohttp.ClientSession(cookies=self.__cookies) as session: 193 | async with session.post(GRAPHQL, json=payload, 194 | headers=HEADERS) as resp: 195 | return await resp.json(), title_slug 196 | 197 | async def __getCode(self, qid, lang): 198 | url = CODE_FORMAT.format(qid, lang) 199 | async with aiohttp.ClientSession(cookies=self.__cookies) as session: 200 | async with session.get(url, headers=HEADERS) as resp: 201 | return await resp.json(), qid, lang 202 | 203 | async def storeSubmissions(self): 204 | '''存储提交的代码信息''' 205 | conn = sqlite3.connect(self.db_path) 206 | c = conn.cursor() 207 | c.execute("SELECT title_slug FROM problem WHERE status == 'ac'") 208 | res = c.fetchall() 209 | if not res: 210 | return 211 | loop = asyncio.get_event_loop() 212 | submissions_list = await handle_tasks( 213 | loop, self.__getSubmissions, [dict(title_slug=t[0]) for t in res]) 214 | data = [] 215 | for submissions, title_slug in submissions_list: 216 | dic = set() 217 | for submission in submissions['data']['submissionList'][ 218 | 'submissions']: 219 | status = submission['statusDisplay'] 220 | key = submission['lang'] 221 | if status == 'Accepted' and key not in dic: 222 | data.append(submission) 223 | data[-1]['title_slug'] = title_slug 224 | dic.add(key) 225 | c.execute(''' 226 | CREATE TABLE IF NOT EXISTS submission ( 227 | submission_id INTEGER, 228 | lang TEXT, 229 | language TEXT, 230 | memory TEXT, 231 | runtime TEXT, 232 | timestamp TEXT, 233 | title_slug TEXT, 234 | s_stored INTEGER DEFAULT 0, 235 | PRIMARY KEY(submission_id) 236 | ) 237 | ''') 238 | for submission in data: 239 | s = SubmissionNode(submission) 240 | c.execute( 241 | ''' 242 | INSERT OR IGNORE INTO submission ( 243 | submission_id, lang, language, memory, runtime, timestamp, title_slug 244 | ) 245 | VALUES ( 246 | ?, ?, ?, ?, ?, ?, ? 247 | ) 248 | ''', (s.submission_id, s.lang, s.language, s.memory, s.runtime, 249 | s.timestamp, s.title_slug)) 250 | conn.commit() 251 | conn.close() 252 | 253 | async def storeCodes(self): 254 | '''存储提交的代码''' 255 | conn = sqlite3.connect(self.db_path) 256 | c = conn.cursor() 257 | c.execute( 258 | "SELECT p.id, lang FROM submission s LEFT JOIN problem p ON s.title_slug=p.title_slug" 259 | ) 260 | res = c.fetchall() 261 | if not res: 262 | return 263 | loop = asyncio.get_event_loop() 264 | codes_list = await handle_tasks( 265 | loop, self.__getCode, [dict(qid=t[0], lang=t[1]) for t in res]) 266 | try: 267 | c.execute("ALTER TABLE submission ADD COLUMN code TEXT") 268 | except sqlite3.OperationalError: 269 | pass 270 | for code, qid, lang in codes_list: 271 | c.execute( 272 | """ 273 | UPDATE submission SET code = ? 274 | WHERE title_slug=(SELECT title_slug 275 | FROM problem 276 | WHERE id=?) 277 | AND lang=?""", (code.get("code", ""), qid, lang)) 278 | conn.commit() 279 | conn.close() 280 | 281 | def updateSubmissions(self): 282 | '''更新处理后的 submission 数据库''' 283 | conn = sqlite3.connect(self.db_path) 284 | c = conn.cursor() 285 | c.execute('UPDATE submission SET s_stored=1') 286 | conn.commit() 287 | conn.close() 288 | 289 | async def update(self): 290 | '''增量式更新数据''' 291 | output_dir = config.outputDir 292 | if not os.path.exists(output_dir): 293 | os.makedirs(output_dir) 294 | extractor = Extractor(output_dir, config.username) 295 | self.updateProblemsInfo() 296 | await self.storeProblemsDesc() 297 | await self.storeSubmissions() 298 | await self.storeCodes() 299 | conn = sqlite3.connect(self.db_path) 300 | conn.row_factory = self.__dict_factory 301 | c = conn.cursor() 302 | # 获取新的数据 303 | c.execute(''' 304 | SELECT * 305 | FROM description d 306 | JOIN problem p 307 | ON p.id=d.id 308 | JOIN submission s 309 | ON p.title_slug=s.title_slug 310 | WHERE (d.d_stored=0 OR s.s_stored=0) 311 | ORDER BY p.id DESC 312 | ''') 313 | datas = c.fetchall() 314 | if datas: 315 | extractor.extractDesc(datas) 316 | self.updateProblemsDesc() 317 | extractor.extractCode(datas) 318 | self.updateSubmissions() 319 | await self.storeCodes() 320 | c.execute(''' 321 | SELECT * 322 | FROM description d 323 | JOIN problem p 324 | ON p.id=d.id 325 | JOIN submission s 326 | ON p.title_slug=s.title_slug 327 | ORDER BY p.id DESC 328 | ''') 329 | datas = c.fetchall() 330 | extractor.extractInfo(self.info, datas) 331 | print('数据已更新！') 332 | conn.close() 333 | 334 | def clearDB(self): 335 | '''重建数据''' 336 | conn = sqlite3.connect(self.db_path) 337 | c = conn.cursor() 338 | c.execute(''' 339 | SELECT name 340 | FROM sqlite_master 341 | WHERE type="table" AND name="problem" 342 | ''') 343 | if c.fetchone(): 344 | # 删除 problem 表 345 | c.execute('DROP TABLE problem') 346 | c.execute(''' 347 | SELECT name 348 | FROM sqlite_master 349 | WHERE type="table" AND name="description" 350 | ''') 351 | if c.fetchone(): 352 | # 删除 description 表 353 | c.execute('DROP TABLE description') 354 | c.execute(''' 355 | SELECT name 356 | FROM sqlite_master 357 | WHERE type="table" AND name="submission" 358 | ''') 359 | if c.fetchone(): 360 | # 删除 submission 表 361 | c.execute('DROP TABLE submission') 362 | conn.commit() 363 | conn.close() 364 | -------------------------------------------------------------------------------- /helper/templates.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-27 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | 7 | # 根目录下的 README 英文模板 8 | TEMPLATE_README_EN = ''' 9 | | English | [简体中文](README.md) | 10 | 11 |

12 |

13 | 14 | 15 | 16 | 17 | 18 |

19 |

My LeetCode Solutions

20 | 21 |

22 |
23 | Last updated: {time} 24 |
25 |

26 | 27 |

The source code is fetched using the tool LeetCode_Helper.

28 | 29 | | # | Title | Solutions | Acceptance | Difficulty | Tags | 30 | |:--:|:-----|:---------:|:----:|:----:|:----:| 31 | ''' 32 | 33 | # 根目录下的 README 中文模板 34 | TEMPLATE_README_CN = ''' 35 | | [English](README_EN.md) | 简体中文 | 36 | 37 |

38 |

39 | 40 | 41 | 42 | 43 | 44 |

45 |

LeetCode 的解答

46 | 47 |

48 |
49 | 最近一次更新: {time} 50 |
51 |

52 | 53 |

The source code is fetched using the tool LeetCode_Helper.

54 | 55 | | # | 题名 | 解答 | 通过率 | 难度 | 标签 | 56 | |:--:|:-----|:---------:|:----:|:----:|:----:| 57 | ''' 58 | 59 | # 根目录下 README 中的题目概要信息 60 | TEMPLATE_README_APPEND = '|{frontend_id}|{title}{paid_only}{is_favor}|{solutions}|{ac_rate}|{difficulty}|{tags}|\n' 61 | 62 | # 题目描述 README 英文模板 63 | TEMPLATE_DESC_EN = ''' 64 | | English | [简体中文](README.md) | 65 | 66 | # {title_en} 67 | 68 | ## Description 69 | 70 | {content_en} 71 | 72 | ## Related Topics 73 | 74 | {tags_en} 75 | 76 | ## Similar Questions 77 | 78 | {similar_questions_en} 79 | ''' 80 | 81 | # 题目描述 README 中文模板 82 | TEMPLATE_DESC_CN = ''' 83 | | [English](README_EN.md) | 简体中文 | 84 | 85 | # {title_cn} 86 | 87 | ## 题目描述 88 | 89 | {content_cn} 90 | 91 | ## 相关话题 92 | 93 | {tags_cn} 94 | 95 | ## 相似题目 96 | 97 | {similar_questions_cn} 98 | ''' 99 | 100 | # 题目代码模板 101 | TEMPLATE_CODE = ''' 102 | {style} @Title: {title_cn} ({title_en}) 103 | {style} @Author: {author} 104 | {style} @Date: {timestamp} 105 | {style} @Runtime: {runtime} 106 | {style} @Memory: {memory} 107 | 108 | {code} 109 | ''' 110 | -------------------------------------------------------------------------------- /helper/utils.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-23 4 | @LastEditTime: 2019-05-05 5 | ''' 6 | import asyncio 7 | import aiohttp 8 | import os 9 | import time 10 | from functools import partial 11 | from .config import config 12 | 13 | 14 | def mkdir(path): 15 | if not os.path.exists(path): 16 | os.makedirs(path) 17 | print(f'{path} 已创建！') 18 | else: 19 | print(f'{path} 已存在！') 20 | return os.path.abspath(path) 21 | 22 | 23 | # 异步 request 24 | async def request(url='', method='get', cookies='', headers='', **kwargs): 25 | async with aiohttp.ClientSession(cookies=cookies) as session: 26 | async with session.request(method=method, 27 | url=url, 28 | headers=headers, 29 | **kwargs) as resp: 30 | return await resp.read() 31 | 32 | 33 | # 异步调度函数 34 | async def handle_tasks(loop, func, args): 35 | if isinstance(args, list): 36 | tasks = { 37 | asyncio.ensure_future(func(**arg)): partial(func, **arg) 38 | for arg in args 39 | } 40 | # loop.run_until_complete(asyncio.wait(tasks)) 41 | pending = set(tasks.keys()) 42 | res = [] 43 | while pending: 44 | finished, pending = await asyncio.wait( 45 | pending, return_when=asyncio.FIRST_COMPLETED) 46 | time.sleep(config.timeInterval) 47 | for task in finished: 48 | if task.exception(): 49 | coro = tasks[task] 50 | # print(f"{coro} retry...") 51 | new_task = asyncio.ensure_future(coro()) 52 | tasks[new_task] = coro 53 | pending.add(new_task) 54 | else: 55 | res.append(task.result()) 56 | return res 57 | elif isinstance(args, str): 58 | task = asyncio.ensure_future(func(args)) 59 | loop.run_until_complete(task) 60 | return [task.result()] 61 | -------------------------------------------------------------------------------- /imgs/example_cn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/imgs/example_cn.png -------------------------------------------------------------------------------- /imgs/example_en.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/imgs/example_en.png -------------------------------------------------------------------------------- /imgs/leetcode-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/imgs/leetcode-logo.png -------------------------------------------------------------------------------- /imgs/problem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/imgs/problem.png -------------------------------------------------------------------------------- /imgs/run.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/imgs/run.png -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-23 4 | @LastEditTime: 2019-05-02 5 | ''' 6 | from helper.main import Main 7 | 8 | if __name__ == "__main__": 9 | while True: 10 | print('欢迎使用 LeetCode_Helper, 请选择: ') 11 | print('1. 更新') 12 | print('2. 重建') 13 | print('q. 退出') 14 | key = input() 15 | if key == 'q': 16 | break 17 | elif key == '1': 18 | Main().update() 19 | break 20 | elif key == '2': 21 | Main().rebuild() 22 | break 23 | else: 24 | print('请重新选择！') 25 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KivenCkl/LeetCode_Helper/08c4fe612047ebb34ddaf2dfee463db845042486/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_config.py: -------------------------------------------------------------------------------- 1 | ''' 2 | @Author: KivenChen 3 | @Date: 2019-04-22 4 | @LastEditTime: 2019-04-30 5 | ''' 6 | import unittest 7 | import os 8 | import sys 9 | sys.path.append(os.path.abspath(os.path.join(__file__, "../.."))) 10 | from helper.config import Config 11 | 12 | 13 | class TestConfig(unittest.TestCase): 14 | def test_init(self): 15 | pass 16 | 17 | def test_getUsername(self): 18 | config = Config() 19 | self.assertIsNotNone(config.username) 20 | 21 | def test_getPassword(self): 22 | config = Config() 23 | self.assertIsNotNone(config.password) 24 | 25 | def test_getOutputDir(self): 26 | config = Config() 27 | self.assertIsNotNone(config.outputDir) 28 | 29 | 30 | if __name__ == "__main__": 31 | unittest.main() 32 | --------------------------------------------------------------------------------