├── .gitignore ├── LICENSE ├── cache_handler.py ├── cert_handler.py ├── client_handler.py ├── configs.py ├── crl_server.py ├── downloader.py ├── gradle_handler.py ├── http_handler.py ├── init.py ├── log_handler.py ├── main.py ├── mfc_handler.py ├── readme.md ├── requirements.txt ├── socks_handler.py └── utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | 3 | server.crt 4 | server.key 5 | ca_server.crt 6 | ca_server.key 7 | crl.pem 8 | 9 | .cache/* 10 | 11 | cacerts 12 | truststore.jks 13 | 14 | log/* 15 | 16 | .vscode/* 17 | 18 | mfc.yaml -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | 294 | Copyright (C) 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | , 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /cache_handler.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from pathlib import Path 3 | import threading 4 | import time 5 | import traceback 6 | import shutil 7 | 8 | from filelock import FileLock 9 | 10 | from configs import * 11 | import configs 12 | from utils import log 13 | from enum import Enum 14 | 15 | # cache structure: 16 | # 17 | # .cache/{cache_key}/.meta 18 | # {id in hex} {file type id} {file name} {last hit timestamp} {size in bytes} 19 | # 20 | # .cache/{cache_key}/{file id in hex} 21 | 22 | # cache init 23 | Path(CACHE_DIR).mkdir(exist_ok=True) 24 | 25 | class CacheType(Enum): 26 | WEB_FILE = 1 27 | CERT = 2 28 | 29 | def _parse_cache_meta_line(line: str): 30 | """解析元数据行""" 31 | line_parts = line.strip().split('\t') 32 | if len(line_parts) != 5: 33 | raise ValueError("Invalid meta data line") 34 | return { 35 | 'id': line_parts[0], 36 | 'type': CacheType(int(line_parts[1])), 37 | 'name': line_parts[2], 38 | 'last_hit': float(line_parts[3]), 39 | 'size': int(line_parts[4]) 40 | } 41 | 42 | def _parse_cache_meta(meta_str: str): 43 | """解析元数据""" 44 | l = [] 45 | for line in meta_str.split('\n'): 46 | if line.strip() == "": 47 | continue 48 | l.append(_parse_cache_meta_line(line)) 49 | return l 50 | 51 | def _save_cache_meta(meta: list): 52 | """保存元数据""" 53 | return '\n'.join(f"{m['id']}\t{m['type'].value}\t{m['name']}\t{m['last_hit']}\t{m['size']}" for m in meta) 54 | 55 | def _get_available_cache_id(meta: list): 56 | """获取可用缓存ID""" 57 | used_ids = set(m['id'] for m in meta) 58 | for i in range(16**16): 59 | if str(i.to_bytes(2, 'big').hex()) not in used_ids: 60 | return str(i.to_bytes(2, 'big').hex()) 61 | raise ValueError("No available cache id") 62 | 63 | def _get_cache_key(type: CacheType, name: str): 64 | """生成缓存键""" 65 | return hashlib.sha256((type.name + "#" + name).encode('utf-8')).hexdigest() 66 | 67 | def _check_disk_space(): 68 | """检查磁盘空间是否充足""" 69 | if not Path(CACHE_DIR).exists(): 70 | return True 71 | used = sum(f.stat().st_size for f in Path(CACHE_DIR).glob('*') if f.is_file()) 72 | return used < DISK_CACHE_MAX_SIZE 73 | 74 | 75 | def save_to_cache(type: CacheType, name: str, data: bytes): 76 | """保存数据到缓存系统""" 77 | if (not configs.with_cache) and type == CacheType.WEB_FILE: 78 | return False 79 | 80 | data_size = len(data) 81 | if data_size > DISK_CACHE_MAX_FILE_SIZE: 82 | log(f"Jummping cache for file {name}: too large ({data_size / 1024 / 1024:.2f} MB)") 83 | return False 84 | 85 | if data_size < DISK_CACHE_MIN_FILE_SIZE and type == CacheType.WEB_FILE: 86 | log(f"Jummping cache for file {name}: too small ({data_size / 1024 / 1024:.2f} MB)") 87 | return False 88 | 89 | if not _check_disk_space(): 90 | log(f"Jummping cache for file {name}: no space left") 91 | return False 92 | 93 | cache_key = _get_cache_key(type, name) 94 | cache_dir = CACHE_DIR + "/" + cache_key 95 | Path(cache_dir).mkdir(exist_ok=True) 96 | 97 | meta_file = CACHE_DIR + "/" + cache_key + "/.meta" 98 | if not Path(meta_file).exists(): 99 | with open(meta_file, 'w') as f: 100 | f.write("") 101 | 102 | locker = FileLock(meta_file + ".lock") 103 | try: 104 | with locker.acquire(timeout=10): 105 | meta = None 106 | with open(meta_file) as f: 107 | meta = _parse_cache_meta(f.read()) 108 | for m in meta: 109 | if m['name'] == name and m['type'] == type: 110 | log(f"Jummping cache for file {type.name}#{name}: already exist") 111 | return False 112 | 113 | cache_id = _get_available_cache_id(meta) 114 | meta.append({ 115 | 'id': cache_id, 116 | 'type': type, 117 | 'name': name, 118 | 'last_hit': time.time(), 119 | 'size': data_size 120 | }) 121 | with open(meta_file, 'w') as f: 122 | f.write(_save_cache_meta(meta)) 123 | 124 | cache_file = cache_dir + "/" + cache_id 125 | with open(cache_file, 'wb') as f: 126 | f.write(data) 127 | return True 128 | except Exception as e: 129 | log(f"Failed to check cache: {e}") 130 | traceback.print_exc() 131 | return False 132 | 133 | def get_path_from_cache(type: CacheType, name: str): 134 | """从缓存中获取数据路径""" 135 | cache_key = _get_cache_key(type, name) 136 | cache_dir = CACHE_DIR + "/" + cache_key 137 | if not Path(cache_dir).exists(): 138 | return None 139 | 140 | meta_file = CACHE_DIR + "/" + cache_key + "/.meta" 141 | if not Path(meta_file).exists(): 142 | return None 143 | 144 | locker = FileLock(meta_file + ".lock") 145 | try: 146 | with locker.acquire(timeout=10): 147 | meta = None 148 | with open(meta_file) as f: 149 | meta = _parse_cache_meta(f.read()) 150 | for m in meta: 151 | if m['name'] == name and m['type'] == type: 152 | cache_file = cache_dir + "/" + m['id'] 153 | if not Path(cache_file).exists(): 154 | raise ValueError("Cache file not found, but meta data exists") 155 | m['last_hit'] = time.time() 156 | with open(meta_file, 'w') as f: 157 | f.write(_save_cache_meta(meta)) 158 | 159 | log(f"Cache hit for file {type.name}#{name}: {m['size'] / 1024 / 1024:.2f} MB") 160 | return cache_file 161 | return None 162 | except Exception as e: 163 | log(f"Failed to get cache path: {e}") 164 | traceback.print_exc() 165 | return None 166 | 167 | def get_from_cache(type: CacheType, name: str): 168 | """从缓存中获取数据""" 169 | if not configs.with_cache: 170 | return None 171 | 172 | path = get_path_from_cache(type, name) 173 | if path is None: 174 | return None 175 | 176 | locker = FileLock(path + ".lock") 177 | try: 178 | with locker.acquire(timeout=10): 179 | with open(path, 'rb') as f: 180 | return f.read() 181 | except Exception as e: 182 | log(f"Failed to get cache: {e}") 183 | traceback.print_exc() 184 | return None 185 | 186 | def _clean_cache(): 187 | """定期清理过期缓存""" 188 | while True: 189 | now = time.time() 190 | log("Cleaning cache...") 191 | for cache_key in os.listdir(CACHE_DIR): 192 | meta_file = CACHE_DIR + "/" + cache_key + "/.meta" 193 | if not Path(meta_file).exists(): 194 | shutil.rmtree(CACHE_DIR + "/" + cache_key, ignore_errors=True) # ignore errors 195 | continue 196 | 197 | locker = FileLock(meta_file + ".lock") 198 | try: 199 | with locker.acquire(timeout=10): 200 | meta = None 201 | with open(meta_file) as f: 202 | meta = _parse_cache_meta(f.read()) 203 | 204 | expired_ids = [m['id'] for m in meta if m['last_hit'] + CACHE_EXPIRE_SECONDS < now] 205 | for expired_id in expired_ids: 206 | cache_file = CACHE_DIR + "/" + cache_key + "/" + expired_id 207 | if Path(cache_file).exists(): 208 | os.remove(cache_file) # ignore errors 209 | log(f"Cleaned cache file {cache_file}") 210 | meta = [m for m in meta if m['id'] != expired_id] 211 | 212 | if len(meta) == 0: 213 | os.remove(meta_file) # ignore errors 214 | shutil.rmtree(CACHE_DIR + "/" + cache_key, ignore_errors=True) # ignore errors 215 | log(f"Cleaned cache directory {CACHE_DIR + '/' + cache_key}") 216 | else: 217 | with open(meta_file, 'w') as f: 218 | f.write(_save_cache_meta(meta)) 219 | except Exception as e: 220 | log(f"Failed to clean cache: {e}") 221 | traceback.print_exc() 222 | log("Cleaning cache done") 223 | time.sleep(CACHE_EXPIRE_SECONDS) # 每24小时清理一次 224 | 225 | # 启动清理线程 226 | threading.Thread(target=_clean_cache, daemon=True).start() -------------------------------------------------------------------------------- /cert_handler.py: -------------------------------------------------------------------------------- 1 | import os 2 | import threading 3 | from cryptography import x509 4 | from cryptography.hazmat.primitives import hashes, serialization 5 | from cryptography.hazmat.primitives.asymmetric import rsa 6 | from cryptography.x509.oid import NameOID 7 | from cryptography.hazmat.backends import default_backend 8 | 9 | from datetime import datetime, timedelta, timezone 10 | from cache_handler import CacheType, get_path_from_cache, save_to_cache 11 | from configs import ALWAYS_APPEND_DOMAIN_NAMES, CERT_FILE, CRL_SERVER_HOST, CRL_SERVER_PORT, KEY_FILE, CRL_FILE 12 | from utils import log 13 | 14 | # Module-level cache with thread-local storage 15 | _ca_cache = threading.local() 16 | 17 | # 添加线程锁 18 | _ca_lock = threading.Lock() 19 | 20 | def _init_ca(): 21 | """Initialize CA certificate and key in cache""" 22 | with _ca_lock: 23 | if not hasattr(_ca_cache, 'ca_cert') or not hasattr(_ca_cache, 'ca_key'): 24 | if os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE): 25 | _load_ca() 26 | else: 27 | raise RuntimeError("CA certificate not found") 28 | 29 | def _load_ca(): 30 | """Load CA certificate and key from files into cache""" 31 | with open(KEY_FILE, "rb") as f: 32 | _ca_cache.ca_key = serialization.load_pem_private_key( 33 | f.read(), 34 | password=None, 35 | backend=default_backend() 36 | ) 37 | 38 | with open(CERT_FILE, "rb") as f: 39 | _ca_cache.ca_cert = x509.load_pem_x509_certificate( 40 | f.read(), 41 | default_backend() 42 | ) 43 | 44 | def _generate_crl(): 45 | """Generate Certificate Revocation List (CRL)""" 46 | _init_ca() 47 | 48 | if not hasattr(_ca_cache, 'ca_cert') or not hasattr(_ca_cache, 'ca_key'): 49 | raise RuntimeError("CA certificate not initialized") 50 | 51 | builder = x509.CertificateRevocationListBuilder() 52 | builder = builder.issuer_name(_ca_cache.ca_cert.subject) 53 | builder = builder.last_update(datetime.now(timezone.utc)) 54 | builder = builder.next_update(datetime.now(timezone.utc) + timedelta(days=365)) 55 | 56 | # Currently empty CRL (no revoked certificates) 57 | crl = builder.sign( 58 | private_key=_ca_cache.ca_key, 59 | algorithm=hashes.SHA256(), 60 | backend=default_backend() 61 | ) 62 | 63 | with open(CRL_FILE, "wb") as f: 64 | f.write(crl.public_bytes(serialization.Encoding.PEM)) 65 | 66 | log(f"Generated CRL: {CRL_FILE}") 67 | 68 | def generate_ca(): 69 | """Generate new CA certificate and save to files""" 70 | # check if CA already exists 71 | if os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE): 72 | raise RuntimeError("CA certificate already exists") 73 | 74 | # Generate private key 75 | key = rsa.generate_private_key( 76 | public_exponent=65537, 77 | key_size=2048, 78 | backend=default_backend() 79 | ) 80 | 81 | # Create self-signed certificate 82 | subject = issuer = x509.Name([ 83 | x509.NameAttribute(NameOID.COMMON_NAME, "DO NOT TRUST multithread_downloading_proxy"), 84 | ]) 85 | 86 | cert = x509.CertificateBuilder().subject_name( 87 | subject 88 | ).issuer_name( 89 | issuer 90 | ).public_key( 91 | key.public_key() 92 | ).serial_number( 93 | x509.random_serial_number() 94 | ).not_valid_before( 95 | datetime.now(timezone.utc) 96 | ).not_valid_after( 97 | datetime.now(timezone.utc) + timedelta(days=365) 98 | ).add_extension( 99 | x509.BasicConstraints(ca=True, path_length=None), 100 | critical=True, 101 | ).add_extension( 102 | x509.KeyUsage( 103 | digital_signature=False, 104 | content_commitment=False, 105 | key_encipherment=False, 106 | data_encipherment=False, 107 | key_agreement=False, 108 | key_cert_sign=True, 109 | crl_sign=True, # 允许CRL签名 110 | encipher_only=False, 111 | decipher_only=False 112 | ), 113 | critical=True, 114 | ).sign(key, hashes.SHA256(), default_backend()) 115 | 116 | # Save to files 117 | with open(KEY_FILE, "wb") as f: 118 | f.write(key.private_bytes( 119 | encoding=serialization.Encoding.PEM, 120 | format=serialization.PrivateFormat.TraditionalOpenSSL, 121 | encryption_algorithm=serialization.NoEncryption(), 122 | )) 123 | 124 | with open(CERT_FILE, "wb") as f: 125 | f.write(cert.public_bytes(serialization.Encoding.PEM)) 126 | 127 | # Update cache 128 | with _ca_lock: 129 | _ca_cache.ca_key = key 130 | _ca_cache.ca_cert = cert 131 | 132 | # 在CA证书生成后再生成CRL 133 | _generate_crl() 134 | 135 | def _issue_certificate(base_domain: str, domains: list[str]): 136 | """Issue a certificate for the given domain using cached CA""" 137 | _init_ca() 138 | 139 | if not hasattr(_ca_cache, 'ca_cert') or not hasattr(_ca_cache, 'ca_key'): 140 | raise RuntimeError("CA certificate not initialized") 141 | 142 | # Build certificate 143 | subject = x509.Name([ 144 | x509.NameAttribute(NameOID.COMMON_NAME, base_domain), 145 | ]) 146 | 147 | # 创建证书构建器 148 | builder = x509.CertificateBuilder().subject_name( 149 | subject 150 | ).issuer_name( 151 | _ca_cache.ca_cert.subject 152 | ).public_key( 153 | _ca_cache.ca_key.public_key() 154 | ).serial_number( 155 | x509.random_serial_number() 156 | ).not_valid_before( 157 | datetime.now(timezone.utc) 158 | ).not_valid_after( 159 | datetime.now(timezone.utc) + timedelta(days=90) 160 | ) 161 | 162 | # 添加主题备用名称扩展 163 | builder = builder.add_extension( 164 | x509.SubjectAlternativeName([ 165 | x509.DNSName(domain) for domain in domains 166 | ]), 167 | critical=False 168 | ) 169 | 170 | builder = builder.add_extension( 171 | x509.CRLDistributionPoints([ 172 | x509.DistributionPoint( 173 | full_name=[x509.UniformResourceIdentifier(f"http://{CRL_SERVER_HOST}:{CRL_SERVER_PORT}/crl.pem")], 174 | relative_name=None, 175 | reasons=None, 176 | crl_issuer=None 177 | ) 178 | ]), 179 | critical=False 180 | ) 181 | 182 | # 添加基本约束扩展 183 | builder = builder.add_extension( 184 | x509.BasicConstraints(ca=False, path_length=None), 185 | critical=True 186 | ) 187 | 188 | # 添加密钥用法扩展 189 | builder = builder.add_extension( 190 | x509.KeyUsage( 191 | digital_signature=True, 192 | content_commitment=False, 193 | key_encipherment=True, 194 | data_encipherment=False, 195 | key_agreement=False, 196 | key_cert_sign=False, 197 | crl_sign=False, 198 | encipher_only=False, 199 | decipher_only=False 200 | ), 201 | critical=True 202 | ) 203 | 204 | # 添加扩展密钥用法扩展 205 | builder = builder.add_extension( 206 | x509.ExtendedKeyUsage([ 207 | x509.ExtendedKeyUsageOID.SERVER_AUTH, 208 | x509.ExtendedKeyUsageOID.CLIENT_AUTH 209 | ]), 210 | critical=False 211 | ) 212 | 213 | # 签名并创建证书 214 | cert = builder.sign(_ca_cache.ca_key, hashes.SHA256(), default_backend()) 215 | 216 | return cert.public_bytes(serialization.Encoding.PEM) 217 | 218 | def get_certificate(base_domain: str, domains: list[str]): 219 | """Get certificate for the given domain, or issue a new one if not found in cache""" 220 | for domain in ALWAYS_APPEND_DOMAIN_NAMES: 221 | domains.append(domain) 222 | key = base_domain + ":" + ",".join(domains) 223 | # First check cache 224 | cache_path = get_path_from_cache(CacheType.CERT, key) 225 | if cache_path: 226 | return cache_path 227 | 228 | # Not in cache, issue new certificate 229 | try: 230 | cert_data = _issue_certificate(base_domain, domains) 231 | if not save_to_cache(CacheType.CERT, key, cert_data): 232 | raise RuntimeError("Failed to save certificate to cache") 233 | 234 | return get_path_from_cache(CacheType.CERT, key) 235 | except Exception as e: 236 | raise RuntimeError(f"Failed to get certificate: {str(e)}") 237 | 238 | -------------------------------------------------------------------------------- /client_handler.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import ssl 3 | import traceback 4 | 5 | from configs import * 6 | 7 | from http_handler import handle_http 8 | from cert_handler import get_certificate 9 | 10 | from utils import decode_header, log, get_base_domain, logger 11 | 12 | def handle_ssl_client(client_socket: socket.socket, domain: str): 13 | """Handle SSL client connection with optional domain-specific certificate""" 14 | 15 | try: 16 | base_domain = get_base_domain(domain) 17 | cert_path = get_certificate(base_domain, [domain] if len(domain.split(".")) > 2 else [base_domain, "*." + base_domain]) 18 | context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) 19 | context.load_cert_chain(cert_path, KEY_FILE) 20 | context.check_hostname = False 21 | client_ssl_socket = context.wrap_socket(client_socket, server_side=True) 22 | 23 | handle_client(client_ssl_socket, with_https=True) 24 | except Exception as e: 25 | logger.error(f"SSL handshake failed: {e}") 26 | client_socket.close() 27 | 28 | def handle_client(client_socket: socket.socket, with_https=False, existing_buf=b''): 29 | try: 30 | request_raw = existing_buf 31 | while buf := client_socket.recv(4096): 32 | request_raw += buf 33 | if len(buf) < 4096: 34 | break 35 | 36 | # Try UTF-8 first, fallback to ISO-8859-1 if fails 37 | try: 38 | request = request_raw.decode('utf-8') 39 | except UnicodeDecodeError: 40 | request = request_raw.decode('iso-8859-1') 41 | 42 | if not request: 43 | log("Received empty request, closing socket.") 44 | client_socket.close() 45 | return 46 | 47 | method, url, headers = decode_header(request_raw, with_https) 48 | 49 | if method.upper() == "CONNECT": 50 | host = headers["Host"] 51 | if not host: 52 | raise ValueError("No Host header in CONNECT request") 53 | 54 | client_socket.send(b"HTTP/1.1 200 Connection Established\r\n\r\n") 55 | 56 | handle_ssl_client(client_socket, host.split(":")[0]) 57 | return 58 | 59 | handle_http(client_socket, url, headers, method, with_https, request_raw) 60 | except ssl.SSLError as e: 61 | if 'shutdown while in init' in str(e).lower(): 62 | logger.warning("SSL handshake interrupted by client") 63 | else: 64 | logger.error(f"SSL Error: {e}") 65 | except Exception as e: 66 | logger.error(f"Error handling client: {e}") 67 | log(traceback.format_exc()) # 记录堆栈跟踪 -------------------------------------------------------------------------------- /configs.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | # 配置参数 / Configuration parameters 4 | PROXY_HOST = '127.0.0.1' 5 | PROXY_PORT = 27579 6 | 7 | # 自签名证书路径 / Self-signed certificate paths 8 | CERT_FILE = "ca_server.crt" 9 | KEY_FILE = "ca_server.key" 10 | CRL_FILE = "crl.pem" # 证书吊销列表文件 / Certificate Revocation List file 11 | CRL_SERVER_HOST = "127.0.0.1" # CRL分发服务器主机 / CRL distribution server host 12 | CRL_SERVER_PORT = 27580 # CRL分发服务器端口 没事别瞎改 要不然你就得删缓存了 / CRL distribution server port (Don't change randomly or you'll need to clear cache) 13 | ALWAYS_APPEND_DOMAIN_NAMES = ["*.honkaiimpact3.com", "hoyoverse.com", "*.hoyoverse.com"] # 证书强制附加域名 / Force append domain names to certificate 14 | 15 | # 下载器阈值 / Downloader thresholds 16 | DOWNLOADER_MAX_THREADS = 32 17 | DOWNLOADER_MULTIPART_THRESHOLD = 1 * 1024 * 1024 # 1MB 18 | DOWNLOADER_PROXIES = {"http": None, "https": None} # deprecated 19 | DOWNLOADER_TRUST_ENV = False # deprecated 20 | DOWNLOADER_MAX_CHUNK_SIZE = 512 * 1024 # 0.5MB 21 | 22 | # 代理地址 / Proxy URLs 23 | HTTP_PROXY = f"http://{PROXY_HOST}:{PROXY_PORT}" 24 | HTTPS_PROXY = f"http://{PROXY_HOST}:{PROXY_PORT}" 25 | SOCKS5_PORT = 27581 # SOCKS5代理端口 / SOCKS5 proxy port 26 | 27 | # 获取 Gradle 用户目录 / Gradle user home directory 28 | GRADLE_USER_HOME = os.getenv("GRADLE_USER_HOME", os.path.expanduser("~/.gradle")) 29 | GRADLE_PROPERTIES_PATH = os.path.join(GRADLE_USER_HOME, "gradle.properties") 30 | 31 | # 缓存配置 / Cache configuration 32 | CACHE_DIR = ".cache" # Cache directory 33 | DISK_CACHE_MAX_SIZE = 10 * 1024 * 1024 * 1024 # 10GB磁盘缓存 / 10GB disk cache max size 34 | DISK_CACHE_MIN_FILE_SIZE = 1024 * 1024 # 缓存区间起点 / Minimum file size to cache 35 | DISK_CACHE_MAX_FILE_SIZE = 256 * 1024 * 1024 # 缓存区间终点 / Maximum file size to cache 36 | CACHE_EXPIRE_SECONDS = 24 * 60 * 60 # 缓存有效期 / Cache expiration time in seconds 37 | 38 | with_cache = False # 是否使用缓存 / Whether to use cache 39 | def set_with_cache(value: bool): 40 | global with_cache 41 | with_cache = value 42 | 43 | with_history = False # 是否使用历史记录 / Whether to use history 44 | def set_with_history(value: bool): 45 | global with_history 46 | with_history = value 47 | 48 | HISTORY_DIR = "log" # 历史记录目录 / History directory 49 | HISTORY_DIVIDER_H1 = "##=============##" 50 | HISTORY_DIVIDER_H2 = "===========" 51 | 52 | # socket配置 / Socket configuration 53 | CLIENT_SOCKET_MAX_CACHE_SIZE = 64 * 1024 # 客户端缓存区最大值 / Maximum size of client socket cache 54 | 55 | # tunnel配置 / Tunnel configuration 56 | TUNNEL_RECV_SIZE = 4096 # 隧道接收一级缓存区大小 / Tunnel receive level 1 buffer size 57 | TUNNEL_RECV_BUFFER_SIZE = 1024 * 1024 # 隧道接收二级缓存区大小 / Tunnel receive level 2 buffer size 58 | 59 | # mfc配置 / MFC configuration (手动文件缓存 / Manual File Cache) 60 | MFC_CONFIG_FILE = "mfc.yaml" -------------------------------------------------------------------------------- /crl_server.py: -------------------------------------------------------------------------------- 1 | from flask import Flask, send_file 2 | from configs import CRL_FILE, CRL_SERVER_HOST, CRL_SERVER_PORT 3 | import waitress 4 | 5 | app = Flask(__name__) 6 | 7 | @app.route('/crl.pem') 8 | def serve_crl(): 9 | """Serve the Certificate Revocation List file""" 10 | return send_file(CRL_FILE, mimetype='application/x-pem-file') 11 | 12 | def start_crl_server(): 13 | """Start the CRL server in a separate thread""" 14 | waitress.serve( 15 | app, 16 | host=CRL_SERVER_HOST, 17 | port=CRL_SERVER_PORT, 18 | threads=4 19 | ) 20 | 21 | if __name__ == '__main__': 22 | start_crl_server() 23 | -------------------------------------------------------------------------------- /downloader.py: -------------------------------------------------------------------------------- 1 | import multiprocessing 2 | import requests 3 | from concurrent.futures import ThreadPoolExecutor, as_completed 4 | import threading 5 | import traceback 6 | import time 7 | 8 | import urllib3 9 | 10 | from configs import * 11 | from utils import log, progress_bar, logger 12 | from cache_handler import CacheType, get_from_cache, save_to_cache 13 | 14 | def generate_schedule(l_range: int, r_range: int): 15 | file_size = r_range - l_range + 1 16 | # decide the chunk size based on the file size 17 | if file_size <= 10 * 1024 * 1024: # 10MB 18 | chunk_size = file_size // DOWNLOADER_MAX_THREADS 19 | elif file_size <= 500 * 1024 * 1024: # 500MB 20 | chunk_size = file_size // DOWNLOADER_MAX_THREADS // 3 21 | else: 22 | chunk_size = DOWNLOADER_MAX_CHUNK_SIZE 23 | 24 | if chunk_size > DOWNLOADER_MAX_CHUNK_SIZE: 25 | chunk_size = DOWNLOADER_MAX_CHUNK_SIZE 26 | 27 | schedule = [] 28 | 29 | # generate the schedule 30 | total_chunks = (file_size + chunk_size - 1) // chunk_size 31 | for i in range(total_chunks): 32 | start = i * chunk_size 33 | end = min(start + chunk_size - 1, file_size - 1) 34 | schedule.append({ 35 | "start": start + l_range, 36 | "end": end + l_range, 37 | "chunk_id": i, 38 | "chunk_data": None, 39 | "consumed": False, 40 | "downloaded": False, 41 | }) 42 | 43 | return schedule 44 | 45 | def download_file_with_schedule(url: str, headers: dict, file_size: int, schedule: list, lock: threading.Lock): 46 | """下载文件, 如果击中缓存就返回bytes形式, 否则通过callback实时更新下载进度""" 47 | try: 48 | cached_data = get_from_cache(CacheType.WEB_FILE, url + "#" + str(headers) + "#" + str(file_size)) 49 | if cached_data is not None: 50 | return cached_data 51 | except Exception as e: 52 | logger.error(f"获取缓存失败: {str(e)}") 53 | traceback.print_exc() 54 | raise 55 | 56 | old_headers = headers 57 | new_headers = {} 58 | for k, v in headers.items(): 59 | new_headers[k] = v 60 | headers = new_headers 61 | 62 | progress_task = progress_bar.create_task(f"downloading {url}", total=file_size) 63 | 64 | try: 65 | log(f"开始多线程下载 (总大小: {file_size/1024/1024:.2f}MB)") 66 | 67 | exceptions = [] 68 | max_retries = 3 # 最大重试次数 69 | 70 | def download_chunk(schedule_item: dict, on_success_callback: callable): 71 | retries = 0 72 | while retries <= max_retries: 73 | try: 74 | start = schedule_item["start"] 75 | end = schedule_item["end"] 76 | headers["Range"] = f"bytes={start}-{end}" 77 | 78 | session = requests.Session() 79 | session.trust_env = DOWNLOADER_TRUST_ENV 80 | 81 | # http = urllib3.PoolManager() 82 | 83 | # 设置连接超时和读取超时 84 | with session.get(url, headers=headers, stream=False, timeout=(5, 30), proxies=DOWNLOADER_PROXIES, allow_redirects=False) as r: 85 | # with http.request('GET', url, headers=headers, preload_content=False, timeout=urllib3.Timeout(connect=5, read=30), retries=urllib3.Retry(total=3)) as r: 86 | # r.decode_content = False 87 | if r.status_code >= 300 or r.status_code < 200: 88 | raise requests.exceptions.HTTPError(f"HTTP {r.status_code} {r.reason}") 89 | chunk_data = r.content 90 | 91 | if len(chunk_data) != (end - start + 1): 92 | raise Exception(f"分片大小不匹配: {len(chunk_data)}!= {(end - start + 1)} for {schedule_item['chunk_id']}") 93 | 94 | # log(f"response headers: {r.headers}") #!TEST 95 | 96 | with lock: 97 | schedule_item["chunk_data"] = chunk_data 98 | schedule_item["downloaded"] = True 99 | progress_bar.update(progress_task, len(chunk_data)) 100 | on_success_callback() 101 | 102 | break # 下载成功则退出循环 103 | 104 | except Exception as e: 105 | retries += 1 106 | if retries > max_retries: 107 | with lock: 108 | exceptions.append(e) 109 | logger.error(f"分片 {schedule_item['chunk_id']} 下载失败: {str(e)}") 110 | traceback.print_exc() 111 | break 112 | time.sleep(2 ** retries) # 指数退避重试 113 | 114 | # 使用线程池动态分配任务 115 | with ThreadPoolExecutor(max_workers=DOWNLOADER_MAX_THREADS) as executor: 116 | def on_success_callback(): 117 | pass 118 | 119 | futures = [executor.submit(download_chunk, schedule_item, on_success_callback) for schedule_item in schedule] 120 | 121 | # 实时监控任务状态 122 | for future in as_completed(futures): 123 | if exceptions: 124 | executor.shutdown(wait=False) 125 | raise exceptions[0] 126 | 127 | 128 | 129 | result = b''.join([schedule_item["chunk_data"] for schedule_item in schedule if schedule_item["chunk_data"] is not None]) 130 | save_to_cache(CacheType.WEB_FILE, url + "#" + str(old_headers) + "#" + str(file_size), result) 131 | log("下载完成并已缓存") 132 | return 133 | 134 | except Exception as e: 135 | logger.error(f"下载失败: {str(e)}") 136 | traceback.print_exc() 137 | finally: 138 | progress_bar.remove_task(progress_task) -------------------------------------------------------------------------------- /gradle_handler.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from utils import log 4 | from configs import * 5 | 6 | def set_gradle_proxies(gradle_properties_path): 7 | """ 8 | 修改 Gradle 的 gradle.properties 文件以设置代理和信任证书。 9 | :param gradle_properties_path: gradle.properties 文件的路径 10 | """ 11 | # 定义需要写入的代理配置 12 | proxy_config = { 13 | "systemProp.http.proxyHost": PROXY_HOST, 14 | "systemProp.http.proxyPort": PROXY_PORT, 15 | "systemProp.https.proxyHost": PROXY_HOST, 16 | "systemProp.https.proxyPort": PROXY_PORT, 17 | "systemProp.javax.net.ssl.trustStore": os.path.abspath("truststore.jks").replace("\\", "/"), # 信任存储文件路径 18 | "systemProp.javax.net.ssl.trustStorePassword": "changeit", # 信任存储密码 19 | "systemProp.javax.net.ssl.trustStoreType": "JKS", # 信任存储类型 20 | } 21 | 22 | # 读取现有内容并准备更新 23 | updated_lines = [] 24 | if os.path.exists(gradle_properties_path): 25 | with open(gradle_properties_path, "r", encoding="utf-8") as f: 26 | lines = f.readlines() 27 | 28 | # 遍历现有行，更新或保留非代理相关的配置 29 | for line in lines: 30 | key = line.split("=")[0].strip() 31 | if key in proxy_config: 32 | # 如果已存在相关配置，则用新值覆盖 33 | updated_lines.append(f"{key}={proxy_config[key]}\n") 34 | del proxy_config[key] # 移除已处理的配置项 35 | else: 36 | # 保留其他配置 37 | updated_lines.append(line) 38 | 39 | # 添加未写入的新代理配置 40 | for key, value in proxy_config.items(): 41 | updated_lines.append(f"{key}={value}\n") 42 | 43 | # 将更新后的内容写回文件 44 | with open(gradle_properties_path, "w", encoding="utf-8") as f: 45 | f.writelines(updated_lines) 46 | 47 | log(f"Gradle proxies and truststore set in {gradle_properties_path}") 48 | 49 | def clear_gradle_proxies(gradle_properties_path): 50 | """ 51 | 清除 Gradle 的代理和信任证书配置。 52 | :param gradle_properties_path: gradle.properties 文件的路径 53 | """ 54 | proxy_config = { 55 | "systemProp.http.proxyHost", 56 | "systemProp.http.proxyPort", 57 | "systemProp.https.proxyHost", 58 | "systemProp.https.proxyPort", 59 | "systemProp.javax.net.ssl.trustStore", 60 | "systemProp.javax.net.ssl.trustStorePassword", 61 | "systemProp.javax.net.ssl.trustStoreType", 62 | } 63 | 64 | # 读取现有内容并准备更新 65 | updated_lines = [] 66 | if os.path.exists(gradle_properties_path): 67 | with open(gradle_properties_path, "r", encoding="utf-8") as f: 68 | lines = f.readlines() 69 | 70 | for line in lines: 71 | key = line.split("=")[0].strip() 72 | if key in proxy_config: 73 | # 移除相关配置 74 | continue 75 | else: 76 | # 保留其他配置 77 | updated_lines.append(line) 78 | 79 | # 将更新后的内容写回文件 80 | with open(gradle_properties_path, "w", encoding="utf-8") as f: 81 | f.writelines(updated_lines) 82 | 83 | log(f"Gradle proxies and truststore cleared in {gradle_properties_path}") -------------------------------------------------------------------------------- /http_handler.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import select 3 | import socket 4 | import ssl 5 | import threading 6 | import time 7 | import traceback 8 | from urllib.parse import urlparse 9 | import requests 10 | 11 | from configs import * 12 | import configs 13 | from mfc_handler import get_mfc_dir, handle_mfc_download, is_cache_disabled 14 | from utils import decode_header, filter_transfer_headers, log, logger 15 | from downloader import download_file_with_schedule, generate_schedule 16 | from log_handler import LoggingSocketDecorator, request_tracker 17 | 18 | def _handle_multithread_download(client_socket: socket.socket, target_url: str, headers: dict, content_length: int, response_headers: dict, response: requests.Response, range: str | None, full_length: int | None): 19 | l_range = 0 20 | r_range = None 21 | if range is not None: 22 | _range = range.split("=")[1] 23 | l_range = int(_range.split("-")[0]) 24 | if len(_range.split("-")) == 2 and _range.split("-")[1] != "": 25 | r_range = int(_range.split("-")[1]) 26 | 27 | try: 28 | def safe_send(data): 29 | try: 30 | client_socket.sendall(data) 31 | return True 32 | except (ConnectionResetError, BrokenPipeError, socket.timeout) as e: 33 | logger.error(f"Send failed: {type(e).__name__}") 34 | return False 35 | 36 | if not range or not r_range: 37 | r_range = l_range + content_length - 1 38 | 39 | if range is not None: 40 | response_headers["Content-Range"] = f"bytes {l_range}-{r_range}/{full_length}" 41 | response_headers["Accept-Ranges"] = "bytes" 42 | response_headers["Connection"] = "keep-alive" 43 | response_headers_raw = f"HTTP/1.1 {response.status_code} {response.reason}\r\n" 44 | for key, value in response_headers.items(): 45 | response_headers_raw += f"{key}: {value}\r\n" 46 | response_headers_raw += "\r\n" 47 | 48 | safe_send(response_headers_raw.encode()) 49 | 50 | schedule = generate_schedule(l_range, r_range) 51 | chunk_num = len(schedule) 52 | 53 | lock = threading.Lock() 54 | 55 | download_process = threading.Thread( 56 | target=download_file_with_schedule, 57 | args=(target_url, headers, r_range - l_range + 1, schedule, lock), 58 | ) 59 | download_process.start() 60 | 61 | # Main thread sending loop 62 | current_chunk_id = 0 63 | while True: 64 | with lock: 65 | if schedule[current_chunk_id]["downloaded"]: 66 | if not safe_send(schedule[current_chunk_id]["chunk_data"]): 67 | raise Exception("Send failed") 68 | schedule[current_chunk_id]["consumed"] = True 69 | if not configs.with_cache: 70 | schedule[current_chunk_id]["chunk_data"] = None 71 | 72 | current_chunk_id += 1 73 | if current_chunk_id == chunk_num: 74 | break 75 | 76 | if result := download_process.join(): 77 | if not safe_send(result): 78 | raise Exception("Send failed") 79 | 80 | safe_send(b"\r\n") 81 | 82 | except Exception as e: 83 | logger.error(f"Download failed: {e}") 84 | log(traceback.format_exc()) 85 | 86 | class InterceptStatus(Enum): 87 | PASS = 0 88 | CLOSE_DIRECTLY = 1 89 | NO_PASS = 2 90 | 91 | def _on_header(client_socket: socket.socket, header: bytes, is_ssl: bool): 92 | method, url, headers = decode_header(header, is_ssl) 93 | 94 | if method != "GET": 95 | return InterceptStatus.PASS 96 | 97 | range_h = headers.get("Range") 98 | # if range is too complex, we don't handle it 99 | if range_h is not None and "," in range_h: 100 | return InterceptStatus.PASS 101 | 102 | if is_cache_disabled(url): 103 | return InterceptStatus.PASS 104 | 105 | content_length = -1 106 | full_length = -1 107 | response_headers = {} 108 | response = None 109 | 110 | attempts = 1 111 | 112 | # Fetch HEAD 113 | for attempt in range(attempts): 114 | try: 115 | session = requests.Session() 116 | session.trust_env = DOWNLOADER_TRUST_ENV 117 | with session.request('HEAD', url, allow_redirects=False, timeout=10, headers=headers, proxies=DOWNLOADER_PROXIES) as head_response: 118 | content_length = int(head_response.headers.get('Content-Length', -1)) 119 | if head_response.headers.get('Content-Range') is not None: 120 | full_length = int(head_response.headers.get('Content-Range', None).split("/")[-1]) 121 | else: 122 | full_length = content_length # full file, no range 123 | response_headers = filter_transfer_headers(head_response.headers) 124 | response = head_response 125 | if content_length != -1: 126 | log(f"Content size: {content_length/1024/1024:.2f}MB") 127 | else: 128 | log("Content size: unknown") 129 | return InterceptStatus.PASS 130 | break 131 | except Exception as e: 132 | if attempt == attempts - 1: # 最后一次尝试失败 133 | logger.error(f"Head request failed after {attempts} attempts: {e}") 134 | return InterceptStatus.PASS 135 | 136 | if get_mfc_dir(url) is not None: 137 | log("Using manual cache for large file") 138 | handle_mfc_download(client_socket, url, headers, content_length, response_headers, response, range_h, full_length) 139 | return InterceptStatus.CLOSE_DIRECTLY 140 | 141 | if content_length >= DOWNLOADER_MULTIPART_THRESHOLD: 142 | log("Using multi-thread download for large file with chunked transfer") 143 | _handle_multithread_download(client_socket, url, headers, content_length, response_headers, response, range_h, full_length) 144 | return InterceptStatus.CLOSE_DIRECTLY 145 | 146 | return InterceptStatus.PASS 147 | 148 | def _extract_http_header(data: bytes): 149 | endpos = -1 150 | for marker in [b"\r\n\r\n", b"\n\n"]: 151 | pos = data.find(marker) 152 | if pos != -1: 153 | endpos = pos 154 | if marker == b"\r\n\r\n": 155 | endpos += 4 156 | else: 157 | endpos += 2 158 | 159 | break 160 | 161 | if endpos == -1: 162 | return None, None 163 | 164 | return data[:endpos], data[endpos:] 165 | 166 | def _tunnel(client: socket.socket, server: socket.socket, is_ssl: bool): 167 | sockets = [client, server] 168 | client_cache = b"" 169 | 170 | def flush_cache(no_send=False): 171 | nonlocal client_cache 172 | if client_cache and not no_send: 173 | server.sendall(client_cache) 174 | client_cache = b"" 175 | 176 | while True: 177 | try: 178 | time.sleep(0.1) 179 | r, _, _ = select.select(sockets, [], [], 5) 180 | for sock in r: 181 | data = b"" 182 | while d := sock.recv(TUNNEL_RECV_SIZE): 183 | data += d 184 | if len(d) < TUNNEL_RECV_SIZE or len(data) >= TUNNEL_RECV_BUFFER_SIZE: 185 | break 186 | 187 | if not data: 188 | return 189 | 190 | if sock is client: 191 | client_cache += data 192 | 193 | if not client_cache.startswith((b"GET", b"POST", b"HEAD", b"PUT", b"DELETE", b"OPTIONS", b"PATCH")): 194 | flush_cache() 195 | continue 196 | 197 | header, _ = _extract_http_header(client_cache) 198 | if header is None: 199 | continue 200 | 201 | status = _on_header(client, header, is_ssl) 202 | if status == InterceptStatus.PASS: 203 | flush_cache() 204 | continue 205 | elif status == InterceptStatus.CLOSE_DIRECTLY: 206 | return 207 | elif status == InterceptStatus.NO_PASS: 208 | flush_cache(no_send=True) 209 | continue 210 | else: 211 | client.sendall(data) 212 | 213 | except (socket.error, ConnectionResetError): 214 | logger.error("Socket error") 215 | log(traceback.format_exc()) 216 | return 217 | 218 | def handle_http(client_socket: socket.socket, url: str, headers: dict, method: str, is_ssl: bool, init_data: bytes): 219 | # 设置socket超时和缓冲区 220 | client_socket.settimeout(30) # 30秒操作超时 221 | 222 | # 记录请求信息 223 | client_ip, client_port = client_socket.getpeername() 224 | log(f"New HTTP request from {client_ip}:{client_port} for {url}") 225 | 226 | parsed_url = urlparse(url) 227 | 228 | port = parsed_url.port or (443 if parsed_url.scheme == "https" else 80) 229 | 230 | server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 231 | server_socket.settimeout(30) # 30秒操作超时 232 | server_socket.connect((parsed_url.hostname, port)) 233 | 234 | if is_ssl: 235 | server_socket = ssl.create_default_context().wrap_socket(server_socket, server_hostname=parsed_url.hostname) 236 | 237 | if configs.with_history: 238 | tracker = request_tracker.init_request(url) 239 | client_socket = LoggingSocketDecorator(client_socket, tracker) 240 | server_socket = LoggingSocketDecorator(server_socket, tracker) 241 | 242 | def close_all(): 243 | log(f"Closing sockets of {client_ip}:{client_port} for {url}") 244 | time.sleep(10) 245 | if client_socket.fileno() != -1: 246 | if is_ssl: 247 | try: 248 | if hasattr(client_socket, '_sslobj') and client_socket._sslobj is not None: 249 | try: 250 | client_socket.unwrap() 251 | except ssl.SSLError as e: 252 | # Handling situations where the SSL connection is not fully established 253 | logger.warning(f"SSL unwrap error: {e}") 254 | client_socket.close() 255 | except Exception as e: 256 | logger.error(f"Error closing SSL connection: {e}") 257 | client_socket.close() 258 | if server_socket.fileno() != -1: 259 | server_socket.close() 260 | 261 | status = None 262 | try: 263 | status = _on_header(client_socket, init_data, is_ssl) 264 | except Exception as e: 265 | logger.error(f"Header hook failed: {e}") 266 | traceback.print_exc() 267 | close_all() 268 | return 269 | 270 | if status == InterceptStatus.PASS: 271 | server_socket.sendall(init_data) 272 | elif status == InterceptStatus.CLOSE_DIRECTLY or status == InterceptStatus.NO_PASS: 273 | close_all() 274 | return 275 | 276 | try: 277 | log(f"Starting tunnel from {client_ip}:{client_port} to {parsed_url.hostname}:{port}") 278 | _tunnel(client_socket, server_socket, is_ssl) 279 | except Exception as e: 280 | logger.error(f"Tunnel failed: {e}") 281 | traceback.print_exc() 282 | finally: 283 | close_all() -------------------------------------------------------------------------------- /init.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import shutil 4 | from cert_handler import generate_ca 5 | from configs import CERT_FILE, KEY_FILE 6 | 7 | def check_ca_status(): 8 | """Check if CA certificate exists and is valid""" 9 | if os.path.exists(CERT_FILE) and os.path.exists(KEY_FILE): 10 | try: 11 | print("CA certificate exists and is valid") 12 | print(f"Certificate file: {CERT_FILE}") 13 | print(f"Private key file: {KEY_FILE}") 14 | print("\nSECURITY WARNING: This is a HTTPS proxy CA certificate") 15 | print("This CA is used to decrypt HTTPS traffic - DO NOT share it with anyone") 16 | print("REMOVE it from trusted certificates when not actively in use") 17 | except Exception as e: 18 | print(f"CA certificate exists but is invalid: {str(e)}") 19 | else: 20 | print("No CA certificate found") 21 | 22 | def clear_cache(): 23 | """Clear cache""" 24 | shutil.rmtree(os.path.join(os.getcwd(), ".cache")) 25 | 26 | def show_help(): 27 | """Display help information""" 28 | print("CA Certificate Management Tool") 29 | print("Usage:") 30 | print(" python init.py - Show current status and help") 31 | print(" python init.py --generate-ca - Generate new CA certificate") 32 | print(" python init.py --clear-cache - Clear proxy cache") 33 | print("\nImportant Notes:") 34 | print("1. CA Certificate Import (OPTIONAL):") 35 | print(" - Only required when using as system proxy") 36 | print(" - To trust this CA on Windows:") 37 | print(" - Doule-click the certificate file and select 'Install Certificate'") 38 | print(" - Import to 'Trusted Root Certification Authorities'") 39 | print("2. SECURITY CRITICAL: This is a HTTPS proxy CA") 40 | print(" - It decrypts HTTPS traffic - NEVER share the certificate") 41 | print(" - HIGH RISK: Importing makes ALL HTTPS traffic vulnerable") 42 | print(" - MUST REMOVE from trusted certificates after each use") 43 | print(" - Use win + R to run 'certmgr.msc' and delete the certificate") 44 | 45 | def main(): 46 | parser = argparse.ArgumentParser(description='CA Certificate Management') 47 | parser.add_argument('--generate-ca', action='store_true', help='Generate new CA certificate') 48 | parser.add_argument('--clear-cache', action='store_true', help='Clear proxy cache') 49 | 50 | args = parser.parse_args() 51 | 52 | if args.clear_cache: 53 | print("Clearing proxy cache...") 54 | clear_cache() 55 | print("Proxy cache cleared successfully") 56 | elif args.generate_ca: 57 | print("Generating new CA certificate...") 58 | generate_ca() 59 | print("CA certificate generated successfully") 60 | print("\nIMPORTANT NEXT STEPS:") 61 | print("1. Windows Trust (OPTIONAL):") 62 | print(" - Only required for system proxy usage") 63 | print(" - Manually trust this CA certificate in Windows") 64 | print("2. SECURITY ALERT: This is a HTTPS proxy CA") 65 | print(" - NEVER share these files with anyone") 66 | print(" - HIGH RISK: Makes ALL HTTPS traffic vulnerable") 67 | print(" - MUST REMOVE after each use session") 68 | print(" - Use win + R to run 'certmgr.msc' and delete the certificate") 69 | print(f" Certificate file: {CERT_FILE}") 70 | print(f" Private key file: {KEY_FILE}") 71 | else: 72 | check_ca_status() 73 | print("\n") 74 | show_help() 75 | 76 | if __name__ == "__main__": 77 | main() 78 | -------------------------------------------------------------------------------- /log_handler.py: -------------------------------------------------------------------------------- 1 | from enum import Enum 2 | import socket 3 | import time 4 | 5 | from configs import * 6 | 7 | class DataType(Enum): 8 | FROM_CLIENT = 0 9 | FROM_SERVER = 1 10 | 11 | class ConversationType(Enum): 12 | HEADER = 0 13 | DATA = 1 14 | 15 | class Conversation: 16 | def __init__(self, conversation_type: ConversationType, data: bytes, data_type: DataType): 17 | self.conversation_type = conversation_type 18 | if conversation_type == ConversationType.HEADER: 19 | self.data = data 20 | self.data_type = data_type 21 | self.length = len(data) 22 | self.time = time.time() 23 | 24 | class _Tracker: 25 | def __init__(self, id: int, url: str): 26 | self.id = id 27 | self.url = url 28 | self.init_time = time.time() 29 | self.conversation_history = [] 30 | self.client_buffer = b"" 31 | self.server_buffer = b"" 32 | self.client_data_flag = False 33 | self.server_data_flag = False 34 | 35 | def get_size(self): 36 | """ 37 | Get the total size of all data exchanged in this request. 38 | """ 39 | size = 0 40 | for conversation in self.conversation_history: 41 | size += conversation.length 42 | return size 43 | 44 | def on_data(self, data: bytes, data_type: DataType): 45 | """ 46 | Process incoming data chunks. 47 | Headers are buffered to ensure they are stored in a single conversation. 48 | Data is processed directly without buffering. 49 | """ 50 | # Get the appropriate buffer and flag 51 | buffer = self.client_buffer if data_type == DataType.FROM_CLIENT else self.server_buffer 52 | data_flag = self.client_data_flag if data_type == DataType.FROM_CLIENT else self.server_data_flag 53 | 54 | # HTTP method markers for detecting new requests 55 | start_markers = [b"GET ", b"POST ", b"PUT ", b"DELETE ", b"HEAD ", 56 | b"OPTIONS ", b"TRACE ", b"CONNECT ", b"PATCH ", b"HTTP/"] 57 | 58 | if not data_flag: # Processing headers 59 | buffer += data 60 | 61 | # Check for complete header 62 | header_end = buffer.find(b'\r\n\r\n') 63 | if header_end != -1: 64 | # Extract complete header 65 | header_data = buffer[:header_end+4] 66 | remaining_data = buffer[header_end+4:] 67 | 68 | # Store header conversation 69 | header_conv = Conversation( 70 | conversation_type=ConversationType.HEADER, 71 | data=header_data, 72 | data_type=data_type 73 | ) 74 | self.conversation_history.append(header_conv) 75 | 76 | # Process any remaining data as body 77 | if len(remaining_data) > 0: 78 | data_conv = Conversation( 79 | conversation_type=ConversationType.DATA, 80 | data=remaining_data, 81 | data_type=data_type 82 | ) 83 | self.conversation_history.append(data_conv) 84 | 85 | # Clear buffer and update flag 86 | buffer = b"" 87 | if data_type == DataType.FROM_CLIENT: 88 | self.client_data_flag = True 89 | self.client_buffer = buffer 90 | else: 91 | self.server_data_flag = True 92 | self.server_buffer = buffer 93 | else: # Processing data 94 | # Check for new request in current data 95 | new_request_pos = -1 96 | for marker in start_markers: 97 | pos = data.find(marker) 98 | if pos != -1 and (new_request_pos == -1 or pos < new_request_pos): 99 | new_request_pos = pos 100 | 101 | if new_request_pos != -1: 102 | # Found new request, process data before it 103 | if new_request_pos > 0: 104 | data_conv = Conversation( 105 | conversation_type=ConversationType.DATA, 106 | data=data[:new_request_pos], 107 | data_type=data_type 108 | ) 109 | self.conversation_history.append(data_conv) 110 | 111 | # Process the new request header 112 | remaining_data = data[new_request_pos:] 113 | if data_type == DataType.FROM_CLIENT: 114 | self.client_buffer = remaining_data 115 | self.client_data_flag = False 116 | else: 117 | self.server_buffer = remaining_data 118 | self.server_data_flag = False 119 | else: 120 | # No new request, process entire chunk as data 121 | data_conv = Conversation( 122 | conversation_type=ConversationType.DATA, 123 | data=data, 124 | data_type=data_type 125 | ) 126 | self.conversation_history.append(data_conv) 127 | 128 | class LoggingSocketDecorator(): 129 | def __init__(self, socket: socket.socket, tracker: _Tracker): 130 | self._socket = socket 131 | self._tracker = tracker 132 | 133 | def _wrapper(self, method, method_name): 134 | def inner(*args, **kwargs): 135 | if method.__name__ in ["send", "sendall"]: 136 | data = args[0] 137 | self._tracker.on_data(data, DataType.FROM_CLIENT) 138 | 139 | result = method(*args, **kwargs) 140 | 141 | if method.__name__ == "recv": 142 | self._tracker.on_data(result, DataType.FROM_SERVER) 143 | 144 | return result 145 | return inner 146 | 147 | def __getattr__(self, attr): 148 | method = getattr(self._socket, attr) 149 | 150 | if callable(method): 151 | return self._wrapper(method, attr) 152 | 153 | return method 154 | 155 | class RequestTracker: 156 | def __init__(self): 157 | self.request_list = [] 158 | self._count = 0 159 | 160 | def init_request(self, url: str): 161 | tracker = _Tracker(self._count, url) 162 | self._count += 1 163 | self.request_list.append(tracker) 164 | return tracker 165 | 166 | def dump(self, file_path: str, sort_lambda=lambda x: x.init_time): 167 | req_list = sorted(self.request_list, key=sort_lambda) 168 | with open(file_path, 'w') as f: 169 | for tracker in req_list: 170 | f.write(f"Request {tracker.id} - {tracker.url} - {tracker.init_time} - {tracker.get_size() / 1024 / 1024:.2f} MB\n") 171 | for conversation in tracker.conversation_history: 172 | f.write(f"{conversation.data_type.name} - {conversation.conversation_type.name} - {conversation.length} - {conversation.time}\n") 173 | f.write(HISTORY_DIVIDER_H2 + "\n") 174 | if conversation.conversation_type == ConversationType.HEADER: 175 | f.write(f"{conversation.data.decode()}") 176 | f.write(HISTORY_DIVIDER_H1 + "\n") 177 | 178 | 179 | request_tracker = RequestTracker() -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | from pathlib import Path 3 | import socket 4 | import threading 5 | import time 6 | from client_handler import handle_client 7 | import configs 8 | from crl_server import start_crl_server 9 | 10 | from configs import * 11 | 12 | from gradle_handler import set_gradle_proxies, clear_gradle_proxies 13 | from log_handler import request_tracker 14 | from utils import log 15 | 16 | 17 | def start_proxy(proxy_host, proxy_port, handler, server): 18 | server.bind((proxy_host, proxy_port)) 19 | server.listen(1000) 20 | log(f"Proxy listening on {proxy_host}:{proxy_port}") 21 | 22 | # 设置accept超时时间 23 | server.settimeout(1.0) # 设置超时时间为1秒 24 | 25 | while True: 26 | try: 27 | client_socket, addr = server.accept() 28 | log(f"Accepted connection from {addr}") 29 | threading.Thread(target=handler, args=(client_socket,), daemon=True).start() 30 | except socket.timeout: 31 | continue # 如果超时，继续循环 32 | 33 | if __name__ == '__main__': 34 | parser = argparse.ArgumentParser() 35 | parser.add_argument("--with-cache", action="store_true", help="Enable cache") 36 | parser.add_argument("--with-history", action="store_true", help="Enable history") 37 | parser.add_argument("--gradle", action="store_true", help="Set gradle proxies") 38 | parser.add_argument("--socks5", action="store_true", help="Enable SOCKS5 proxy") 39 | parser.add_argument("--print-env", action="store_true", help="Print proxy environment variables") 40 | args = parser.parse_args() 41 | 42 | set_with_cache(args.with_cache) 43 | set_with_history(args.with_history) 44 | 45 | if args.print_env: 46 | print(f"http_proxy={HTTP_PROXY}") 47 | print(f"https_proxy={HTTPS_PROXY}") 48 | 49 | try: 50 | if args.gradle: 51 | set_gradle_proxies(GRADLE_PROPERTIES_PATH) 52 | 53 | # Start CRL server in a separate thread 54 | crl_thread = threading.Thread( 55 | target=start_crl_server, 56 | daemon=True, 57 | name="CRL Server" 58 | ) 59 | crl_thread.start() 60 | 61 | http_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 62 | threading.Thread(target=start_proxy, args=(PROXY_HOST, PROXY_PORT, handle_client, http_server), daemon=True, name="HTTP Proxy").start() 63 | 64 | if args.socks5: 65 | from socks_handler import handle_socks5_client 66 | socks5_server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 67 | threading.Thread(target=start_proxy, args=(PROXY_HOST, SOCKS5_PORT, handle_socks5_client, socks5_server), daemon=True, name="SOCKS5 Proxy").start() 68 | while True: 69 | time.sleep(1) 70 | finally: 71 | if args.gradle: 72 | clear_gradle_proxies(GRADLE_PROPERTIES_PATH) 73 | if configs.with_history: 74 | Path(HISTORY_DIR).mkdir(exist_ok=True) 75 | request_tracker.dump(HISTORY_DIR + "/" + str(time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())) + "_sort_by_time.log") 76 | request_tracker.dump(HISTORY_DIR + "/" + str(time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime())) + "_sort_by_size.log", 77 | sort_lambda=lambda x: -x.get_size()) 78 | -------------------------------------------------------------------------------- /mfc_handler.py: -------------------------------------------------------------------------------- 1 | import ctypes 2 | import os 3 | from pathlib import Path 4 | import socket 5 | import traceback 6 | import requests 7 | import yaml 8 | 9 | from utils import logger 10 | from configs import * 11 | 12 | mfc_config = [] 13 | 14 | if os.path.exists(MFC_CONFIG_FILE): 15 | with open(MFC_CONFIG_FILE, "rb") as f: 16 | try: 17 | mfc_config = yaml.load(f, Loader=yaml.FullLoader) 18 | except yaml.YAMLError as e: 19 | logger.error(f"Failed to parse {MFC_CONFIG_FILE}: {e}") 20 | 21 | def check_mfc_config() -> bool: 22 | if not isinstance(mfc_config, list): 23 | logger.error(f"{MFC_CONFIG_FILE} should be a list") 24 | return False 25 | 26 | for item in mfc_config: 27 | if not isinstance(item, dict): 28 | logger.error(f"Each item in {MFC_CONFIG_FILE} should be a dictionary") 29 | return False 30 | 31 | if "url" not in item: 32 | logger.error(f"Each item in {MFC_CONFIG_FILE} should have a 'url' key") 33 | return False 34 | 35 | if "cache" not in item: 36 | logger.error(f"Each item in {MFC_CONFIG_FILE} should have a 'cache' key") 37 | return False 38 | 39 | if "cache" != "true" and (not os.path.exists(item["cache"]) or os.path.isdir(item["cache"])): 40 | logger.error(f"Cache directory {item['cache']} does not exist or is a directory") 41 | return False 42 | 43 | return True 44 | 45 | if not check_mfc_config(): 46 | raise Exception("MFC config file is invalid") 47 | 48 | def is_cache_disabled(url: str) -> bool: 49 | for item in mfc_config: 50 | if item["url"] == url: 51 | return item["cache"] == "false" 52 | return False 53 | 54 | def get_mfc_dir(url: str) -> Path | None: 55 | for item in mfc_config: 56 | if item["url"] == url: 57 | if os.path.exists(item["cache"]) and not os.path.isdir(item["cache"]): 58 | return Path(item["cache"]) 59 | return None 60 | 61 | def handle_mfc_download(client_socket: socket.socket, target_url: str, headers: dict, content_length: int, response_headers: dict, response: requests.Response, range: str | None, full_length: int | None): 62 | mfc_path = get_mfc_dir(target_url) 63 | if mfc_path is None: 64 | raise Exception("MFC cache directory not found, should not happen due to previous check") 65 | 66 | if mfc_path.stat().st_size != full_length: 67 | logger.error(f"MFC cache file size {os.path.getsize(mfc_path)} does not match full length {full_length}") 68 | return 69 | 70 | l_range = 0 71 | r_range = None 72 | if range is not None: 73 | _range = range.split("=")[1] 74 | l_range = int(_range.split("-")[0]) 75 | if len(_range.split("-")) == 2 and _range.split("-")[1] != "": 76 | r_range = int(_range.split("-")[1]) 77 | 78 | try: 79 | def safe_send(data): 80 | try: 81 | client_socket.sendall(data) 82 | return True 83 | except (ConnectionResetError, BrokenPipeError, socket.timeout) as e: 84 | logger.error(f"Send failed: {type(e).__name__}") 85 | return False 86 | 87 | if not range or not r_range: 88 | r_range = l_range + content_length - 1 89 | 90 | if range is not None: 91 | response_headers["Content-Range"] = f"bytes {l_range}-{r_range}/{full_length}" 92 | response_headers["Accept-Ranges"] = "bytes" 93 | response_headers["Connection"] = "keep-alive" 94 | response_headers_raw = f"HTTP/1.1 {response.status_code} {response.reason}\r\n" 95 | for key, value in response_headers.items(): 96 | response_headers_raw += f"{key}: {value}\r\n" 97 | response_headers_raw += "\r\n" 98 | 99 | safe_send(response_headers_raw.encode()) 100 | 101 | client_socket.sendfile(mfc_path.open("rb"), l_range, r_range - l_range + 1) 102 | 103 | safe_send(b"\r\n") 104 | 105 | except Exception as e: 106 | logger.error(f"MFC send failed: {e}") 107 | logger.log(traceback.format_exc()) -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | [English Version](#english-version) 2 | 3 | 不支持除了GET以外的方法使用多线程下载 4 | 不推荐用于日常浏览器使用, 有些功能可能不支持 5 | 它不是那么稳定, 可能会导致一些东西失效, 莫名其妙404, 500, SSL Handshake Error等错误等, 所以如果出事了, 先把这个关掉 6 | 7 | 通过 --with-cache 参数开启缓存, 默认会对一些特定文件上24小时缓存, 详情见configs.py 8 | 通过 --with-history 参数开启历史记录, 它会记录流量, 然后默认在关闭时dump到/log 9 | 通过 --gradle 参数为gradle开启代理, 详细配置见configs.py 10 | 通过 --socks5 参数开启socks5代理 11 | 通过 --print-env 参数来打印关于代理的环境变量 12 | 13 | 参考init.py来导入ca证书 14 | 注意, ca证书导入是可选项, 当且仅当你想要它作为系统代理的时候才需要使用, 而且它比较危险, 建议使用过后删除 15 | 16 | ## gradle (java) 证书导入 17 | 18 | cacerts是你从你的java home/lib/security目录下找到的证书文件，truststore.jks是你自己创建的信任库文件, 给gradle用的 19 | 记得用gradle对应的java home 20 | 记得重启你的IDE 21 | 如果不是gralde, 你可能需要手动把信任库文件放回对应目录 22 | 23 | ```bash 24 | keytool -importcert -alias do_not_trust_multithread_downloading_proxy_ca -file ca_server.crt -keystore truststore.jks -storepass changeit -noprompt 25 | keytool -importkeystore -srckeystore cacerts -destkeystore truststore.jks -srcstorepass changeit -deststorepass changeit -noprompt 26 | ``` 27 | 28 | ## 手动文件缓存 29 | 30 | 你可以手动指定直接从缓存返回的文件, 配置文件保存在configs.py指定的MFC_CONFIG_FILE里, 例子如下: 31 | 32 | ```yaml 33 | - url: https://example.com/file.zip # 对这个url跳过缓存(严格匹配), 优先级最高 34 | cache: false 35 | - url: https://example.com/file2.zip # 对这个url使用指定缓存路径 36 | cache: /path/to/cache/file2.zip 37 | ``` 38 | 39 | ## 相关推荐工具 40 | [netch](https://github.com/netchx/netch) 强制为特定软件使用socks5代理 41 | [dn](https://github.com/franticxx/dn) 多线程下载器(建议大于等于0.1.4版本) 42 | 43 | 44 | ## English Version 45 | 46 | The multi-thread downloading proxy only supports GET method. Other HTTP methods are not supported. 47 | Not recommended for daily browser use as some features may not work properly. 48 | It's quite unstable and may cause failures, random 404/500 errors, SSL handshake errors, etc. If any issue occurs, disable it immediately. 49 | 50 | Cache can be enabled with --with-cache parameter. By default it sets 24-hour cache for certain files, see configs.py for details. 51 | History can be enabled with --with-history parameter. It records traffic and dumps it to /log when closed. 52 | Gradle proxying can be enabled with --gradle parameter. See configs.py for details of configuration. 53 | Socks5 proxying can be enabled with --socks5 parameter. 54 | Print environment variables about proxying with --print-env parameter. 55 | 56 | Refer to init.py to import CA certificates. 57 | Note: CA certificate import is optional and only required when using as system proxy. It's potentially dangerous - recommended to remove after use. 58 | 59 | ## Gradle (Java) Certificate Import 60 | 61 | cacerts is the certificate file from your java home/lib/security directory. truststore.jks is the truststore file you created for gradle. 62 | Make sure to use the java home corresponding to your gradle installation. 63 | Don't forget to restart your IDE after configuration. 64 | If you are not using gradle, you may need to manually copy the truststore file back to the corresponding directory. 65 | 66 | ```bash 67 | keytool -importcert -alias do_not_trust_multithread_downloading_proxy_ca -file ca_server.crt -keystore truststore.jks -storepass changeit -noprompt 68 | keytool -importkeystore -srckeystore cacerts -destkeystore truststore.jks -srcstorepass changeit -deststorepass changeit -noprompt 69 | ``` 70 | 71 | ## Manual file cache 72 | 73 | You can specify files to be returned directly from cache, configuration is stored in MFC_CONFIG_FILE specified in configs.py, for example: 74 | 75 | ```yaml 76 | - url: https://example.com/file.zip # Skip cache for this url (strict match), highest priority 77 | cache: false 78 | - url: https://example.com/file2.zip # Use specified cache path for this url 79 | cache: /path/to/cache/file2.zip 80 | ``` 81 | 82 | ## Recommanded Related tools 83 | [netch](https://github.com/netchx/netch) Force proxy for specific software 84 | [dn](https://github.com/franticxx/dn) Multi-thread downloading tool (version >= 0.1.4 is recommended) 85 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cryptography 2 | requests>=2.23.2 3 | tqdm 4 | filelock 5 | waitress>=3.0.1 6 | rich 7 | pyyaml 8 | urllib3>=2.2.2 -------------------------------------------------------------------------------- /socks_handler.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import struct 3 | import select 4 | from typing import Tuple 5 | from configs import * 6 | from utils import log, logger 7 | from client_handler import handle_client, handle_ssl_client 8 | 9 | class Socks5Handler: 10 | def __init__(self, client_socket: socket.socket): 11 | self.client_socket = client_socket 12 | self.remote_socket = None 13 | self.is_http = False 14 | self.buffer = b'' 15 | 16 | def handle(self): 17 | try: 18 | self.client_socket.settimeout(10.0) # Set connection timeout 19 | 20 | # 1. Version and method negotiation 21 | version, nmethods = self._recv_initial_request() 22 | if version != 0x05: 23 | raise ValueError(f"Unsupported SOCKS version: {version}") 24 | # log(f"Received SOCKS5 version {version} and {nmethods} methods") 25 | 26 | # Send method selection (0x00 - no authentication) 27 | self.client_socket.sendall(bytes([0x05, 0x00])) 28 | 29 | # 2. Receive client request 30 | version, cmd, _, addr_type = self._recv_request() 31 | if version != 0x05: 32 | raise ValueError(f"Unsupported SOCKS version: {version}") 33 | # log(f"Received SOCKS5 request for {cmd} to {addr_type}") 34 | 35 | # Handle different address types 36 | if addr_type == 0x01: # IPv4 37 | addr = socket.inet_ntoa(self.client_socket.recv(4)) 38 | elif addr_type == 0x03: # Domain name 39 | domain_length = ord(self.client_socket.recv(1)) 40 | addr = self.client_socket.recv(domain_length).decode('utf-8') 41 | elif addr_type == 0x04: # IPv6 42 | ipv6_bytes = self.client_socket.recv(16) 43 | if len(ipv6_bytes) != 16: 44 | raise ValueError("Invalid IPv6 address length") 45 | addr = socket.inet_ntop(socket.AF_INET6, ipv6_bytes) 46 | else: 47 | raise ValueError(f"Unsupported address type: {addr_type}") 48 | 49 | port = struct.unpack('!H', self.client_socket.recv(2))[0] 50 | 51 | # log(f"Received request for {addr}:{port}") 52 | 53 | # 3. Handle command 54 | if cmd == 0x01: # CONNECT 55 | self._handle_connect(addr, port) 56 | elif cmd == 0x02: # BIND 57 | self._handle_bind(addr, port) 58 | elif cmd == 0x03: # UDP ASSOCIATE 59 | self._handle_udp_associate(addr, port) 60 | else: 61 | raise ValueError(f"Unsupported command: {cmd}") 62 | 63 | except Exception as e: 64 | logger.error(f"SOCKS5 error: {e}") 65 | self._send_reply(0x05, 0x01) # General failure 66 | if self.remote_socket: 67 | self.remote_socket.close() 68 | self.client_socket.close() 69 | 70 | def _recv_initial_request(self) -> Tuple[int, int]: 71 | try: 72 | 73 | # Receive version identifier/method selection message 74 | data = b'' 75 | while len(data) < 2: 76 | chunk = self.client_socket.recv(2 - len(data)) 77 | if not chunk: 78 | raise ValueError("Connection closed during negotiation") 79 | data += chunk 80 | 81 | # Verify protocol version (use bytes comparison) 82 | if data[0:1] != b'\x05': 83 | raise ValueError(f"Unsupported SOCKS version: {data[0]}") 84 | 85 | # Verify methods length 86 | nmethods = data[1] 87 | if len(data) < 2 + nmethods: 88 | remaining = 2 + nmethods - len(data) 89 | data += self.client_socket.recv(remaining) 90 | 91 | # Check for no-auth method 92 | methods = data[2:2+nmethods] 93 | if b'\x00' not in methods: 94 | raise ValueError("No acceptable authentication methods") 95 | 96 | return 5, 1 # SOCKS5 with no-auth 97 | except socket.timeout: 98 | raise ValueError("Negotiation timeout") 99 | except Exception as e: 100 | logger.error(f"Error during initial request: {e}") 101 | raise 102 | 103 | def _recv_request(self) -> Tuple[int, int, int, int]: 104 | # More lenient request handling for HTTPS 105 | data = b'' 106 | while len(data) < 4: 107 | chunk = self.client_socket.recv(4 - len(data)) 108 | if not chunk: 109 | raise ValueError("Connection closed prematurely") 110 | data += chunk 111 | return data[0], data[1], data[2], data[3] # version, cmd, rsv, addr_type 112 | 113 | def _handle_connect(self, addr: str, port: int): 114 | try: 115 | # Determine address family 116 | if ':' in addr: # IPv6 117 | family = socket.AF_INET6 118 | else: # IPv4 119 | family = socket.AF_INET 120 | 121 | self.remote_socket = socket.socket(family, socket.SOCK_STREAM) 122 | self.remote_socket.settimeout(10.0) # Set connection timeout 123 | # log(f"Connecting to {addr}:{port}") 124 | try: 125 | self.remote_socket.connect((addr, port)) 126 | if not self.client_socket._closed: # Check if client still connected 127 | self._send_reply(0x00, 0x00) # Success 128 | else: 129 | logger.error("Client disconnected before reply could be sent") 130 | except socket.timeout: 131 | logger.error(f"Connection timeout to {addr}:{port}") 132 | raise 133 | 134 | # Detect traffic type (HTTP or HTTPS) 135 | self._detect_traffic_type() 136 | 137 | if self.is_http: # Handle both HTTP and HTTPS 138 | # Check if it's HTTPS (TLS) or plain HTTP 139 | if len(self.buffer) >= 3 and self.buffer[0] == 0x16 and self.buffer[1] == 0x03: 140 | # log("Wrapping HTTPS connection with SSL") 141 | handle_ssl_client(self.client_socket, addr) 142 | else: 143 | # log("Handling as HTTP proxy") 144 | handle_client(self.client_socket, existing_buf=self.buffer[:]) 145 | self.buffer = b'' # Clear buffer after handling client 146 | else: 147 | # Direct forwarding for non-HTTP traffic 148 | self._transfer_data() 149 | except socket.gaierror as e: 150 | logger.error(f"Address resolution failed for {addr}:{port}: {e}") 151 | self._send_reply(0x05, 0x04) # Host unreachable 152 | raise 153 | except Exception as e: 154 | logger.error(f"Connection failed to {addr}:{port}: {e}") 155 | self._send_reply(0x05, 0x04) # Host unreachable 156 | raise 157 | 158 | def _detect_traffic_type(self): 159 | """Detect traffic type (HTTP/HTTPS) by peeking at the first few bytes""" 160 | try: 161 | # Peek at the first 16 bytes without consuming them 162 | data = self.client_socket.recv(16, socket.MSG_PEEK) 163 | self.buffer += data 164 | # log(f"Received {data}") 165 | 166 | # Check for TLS/SSL handshake (HTTPS) 167 | if len(data) >= 3 and data[0] == 0x16 and data[1] == 0x03: 168 | self.is_http = True # Actually HTTPS but we'll handle similarly 169 | return 170 | 171 | # Check for plain HTTP methods 172 | if len(data) >= 7: # Minimum for HTTP methods 173 | first_line = data.decode('utf-8', errors='ignore').split('\r\n')[0] 174 | if first_line.startswith(('GET ', 'POST ', 'PUT ', 'DELETE ', 'HEAD ', 'OPTIONS ', 'CONNECT ')): 175 | self.is_http = True 176 | return 177 | 178 | self.is_http = False 179 | except Exception as e: 180 | log(f"Traffic detection error: {e}") 181 | self.is_http = False 182 | 183 | def _send_reply(self, rep: int, _: int, bind_addr: str = '0.0.0.0', bind_port: int = 0): 184 | """Send SOCKS5 reply with optional bind address and port""" 185 | ver = 0x05 186 | rsv = 0x00 187 | 188 | # Determine address type 189 | if ':' in bind_addr: # IPv6 190 | addr_type = 0x04 191 | bnd_addr = socket.inet_pton(socket.AF_INET6, bind_addr) 192 | else: # IPv4 193 | addr_type = 0x01 194 | bnd_addr = socket.inet_aton(bind_addr) 195 | 196 | # Pack port number 197 | port_bytes = struct.pack('!H', bind_port) 198 | 199 | reply = bytes([ver, rep, rsv, addr_type]) + bnd_addr + port_bytes 200 | self.client_socket.sendall(reply) 201 | 202 | def _handle_bind(self, addr: str, port: int): 203 | """Handle BIND command (0x02) for reverse connections""" 204 | try: 205 | # Create listening socket 206 | bind_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 207 | bind_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 208 | bind_socket.bind(('0.0.0.0', 0)) # Bind to any available port 209 | bind_socket.listen(1) 210 | 211 | # Get bound address and port 212 | bind_addr, bind_port = bind_socket.getsockname() 213 | 214 | # Send first reply (server listening) 215 | self._send_reply(0x00, 0x00, bind_addr, bind_port) 216 | 217 | # Wait for incoming connection 218 | self.remote_socket, _ = bind_socket.accept() 219 | bind_socket.close() 220 | 221 | # Send second reply (connection established) 222 | remote_addr, remote_port = self.remote_socket.getpeername() 223 | self._send_reply(0x00, 0x00, remote_addr, remote_port) 224 | 225 | # Start data transfer 226 | self._transfer_data() 227 | 228 | except Exception as e: 229 | log(f"BIND command failed: {e}") 230 | self._send_reply(0x05, 0x01) # General failure 231 | raise 232 | 233 | def _handle_udp_associate(self, addr: str, port: int): 234 | """Handle UDP ASSOCIATE command (0x03) for UDP forwarding""" 235 | try: 236 | # Create UDP socket 237 | udp_socket = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 238 | udp_socket.bind(('0.0.0.0', 0)) # Bind to any available port 239 | 240 | # Get bound address and port 241 | udp_addr, udp_port = udp_socket.getsockname() 242 | 243 | # Send reply with UDP endpoint info 244 | self._send_reply(0x00, 0x00, udp_addr, udp_port) 245 | 246 | # Set client socket to non-blocking for UDP association 247 | self.client_socket.setblocking(False) 248 | 249 | # UDP association loop 250 | while True: 251 | r, _, _ = select.select([self.client_socket, udp_socket], [], [], 10.0) 252 | 253 | if self.client_socket in r: 254 | # Client sent data (should be empty in SOCKS5 UDP) 255 | data = self.client_socket.recv(4096) 256 | if not data: # Connection closed 257 | break 258 | 259 | if udp_socket in r: 260 | # Forward UDP datagram to client 261 | data, addr = udp_socket.recvfrom(4096) 262 | self.client_socket.sendall(data) 263 | 264 | except Exception as e: 265 | log(f"UDP ASSOCIATE failed: {e}") 266 | self._send_reply(0x05, 0x01) # General failure 267 | raise 268 | finally: 269 | udp_socket.close() 270 | 271 | def _transfer_data(self): 272 | """Direct forwarding for non-HTTP traffic""" 273 | while True: 274 | r, _, _ = select.select([self.client_socket, self.remote_socket], [], []) 275 | for sock in r: 276 | data = sock.recv(4096) 277 | if not data: 278 | return 279 | if sock is self.client_socket: 280 | self.remote_socket.sendall(data) 281 | else: 282 | self.client_socket.sendall(data) 283 | 284 | def handle_socks5_client(client_socket: socket.socket): 285 | handler = Socks5Handler(client_socket) 286 | handler.handle() 287 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | import threading 2 | import time 3 | from rich.progress import Progress, BarColumn, DownloadColumn 4 | from rich.console import Console 5 | 6 | console = Console() 7 | 8 | class Logger: 9 | """日志记录器""" 10 | def __init__(self): 11 | self._console = console 12 | 13 | def log(self, message: str): 14 | """记录普通日志""" 15 | self._console.print(f"[{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}] [{threading.current_thread().name}] [LOG] {message}") 16 | 17 | def error(self, message: str): 18 | """记录错误日志""" 19 | self._console.print(f"[{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}] [{threading.current_thread().name}] [ERROR] {message}", style="bold red") 20 | 21 | def warning(self, message: str): 22 | """记录警告日志""" 23 | self._console.print(f"[{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}] [{threading.current_thread().name}] [WARNING] {message}", style="bold yellow") 24 | 25 | class ProgressBar: 26 | """进度条管理类""" 27 | def __init__(self): 28 | self._console = console 29 | self._progress = Progress( 30 | "[progress.description] {task.description}", 31 | BarColumn(bar_width=None), 32 | DownloadColumn(), 33 | console=self._console, 34 | refresh_per_second=10 35 | ) 36 | self._lock = threading.Lock() 37 | self._count = 0 38 | 39 | def create_task(self, description: str, total: int): 40 | """创建进度条任务""" 41 | self._start() 42 | return self._progress.add_task(description, total=total) 43 | 44 | def update(self, task_id, advance: int): 45 | """更新进度""" 46 | self._progress.update(task_id, advance=advance) 47 | 48 | def remove_task(self, task_id): 49 | """移除任务""" 50 | self._progress.remove_task(task_id) 51 | self._stop() 52 | 53 | def _start(self): 54 | """启动进度条""" 55 | with self._lock: 56 | self._count += 1 57 | if self._count == 1: 58 | self._progress.start() 59 | 60 | def _stop(self): 61 | """停止进度条""" 62 | with self._lock: 63 | self._count -= 1 64 | if self._count == 0: 65 | self._progress.stop() 66 | 67 | def render(self): 68 | """手动渲染进度条""" 69 | with self._lock: 70 | for task_id, description in self._tasks.items(): 71 | task = self._progress.tasks[task_id] 72 | progress_text = self._progress.get_renderable(task, BarColumn(bar_width=None)).render(self._console, self._console.options) 73 | self._console.print(description) 74 | self._console.print(progress_text) 75 | self._console.print() # 确保每个任务之间换行 76 | 77 | # 全局进度条实例 78 | progress_bar = ProgressBar() 79 | 80 | # 全局日志记录器实例 81 | logger = Logger() 82 | 83 | def log(message: str): 84 | """兼容旧代码的日志函数""" 85 | logger.log(message) 86 | 87 | def get_current_thread_name(): 88 | return threading.current_thread().name 89 | 90 | def get_base_domain(domain: str): 91 | """Extract base domain (second-level domain) from given domain""" 92 | parts = domain.lower().split('.') 93 | if len(parts) > 2: 94 | return '.'.join(parts[-2:]) 95 | return domain 96 | 97 | def filter_transfer_headers(headers: dict): 98 | """ 99 | Filter out headers that are related to transfer encoding, which is python will handle automatically. 100 | """ 101 | transfer_related_headers = ['Transfer-Encoding', 'Content-Encoding'] 102 | filtered_headers = {k: v for k, v in headers.items() if k not in transfer_related_headers} 103 | return filtered_headers 104 | 105 | def decode_header(data: bytes, with_https: bool): 106 | """ 107 | Decode header data 108 | """ 109 | # Try UTF-8 first, fallback to ISO-8859-1 if fails 110 | try: 111 | header = data.decode('utf-8') 112 | except UnicodeDecodeError: 113 | header = data.decode('iso-8859-1') 114 | 115 | first_line = header.split('\r\n')[0].strip() 116 | parts = [p for p in first_line.split(' ') if p] # Remove empty strings from split 117 | if len(parts) < 3: 118 | raise ValueError(f"Invalid HTTP request line: {first_line}") 119 | 120 | method = parts[0] 121 | path_or_url = parts[1] 122 | version = ' '.join(parts[2:]) # Handle cases where version contains spaces 123 | 124 | headers = {} 125 | for line in header.split('\r\n')[1:]: 126 | if not line: 127 | break 128 | parts = line.split(':', 1) 129 | if len(parts)!= 2: 130 | raise ValueError("Invalid HTTP header line") 131 | key = parts[0].strip().lower().capitalize() 132 | headers[key] = parts[1].strip() 133 | 134 | url = path_or_url 135 | 136 | if not url.startswith(("http://", "https://")): 137 | # get from host header, ignoring https 138 | host = headers.get("Host") 139 | if not host: 140 | # Try alternative headers 141 | host = headers.get("X-Forwarded-Host") or headers.get("X-Host") 142 | if not host: 143 | raise ValueError("No Host, X-Forwarded-Host or X-Host header in request") 144 | 145 | url = f"{'https' if with_https else 'http'}://{host}{path_or_url}" 146 | 147 | return method, url, headers 148 | --------------------------------------------------------------------------------