├── .gitignore ├── Fgproxy.png ├── LICENSE ├── README.md ├── haproxy.png ├── log.png └── proxy ├── __init__.py ├── adb.py ├── api.py ├── bridge.py ├── command.py ├── deploy └── cli.py ├── header.py ├── network.py └── structure.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | MANIFEST 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | .pytest_cache/ 49 | 50 | # Translations 51 | *.mo 52 | *.pot 53 | 54 | # Django stuff: 55 | *.log 56 | local_settings.py 57 | db.sqlite3 58 | 59 | # Flask stuff: 60 | instance/ 61 | .webassets-cache 62 | 63 | # Scrapy stuff: 64 | .scrapy 65 | 66 | # Sphinx documentation 67 | docs/_build/ 68 | 69 | # PyBuilder 70 | target/ 71 | 72 | # Jupyter Notebook 73 | .ipynb_checkpoints 74 | 75 | # pyenv 76 | .python-version 77 | 78 | # celery beat schedule file 79 | celerybeat-schedule 80 | 81 | # SageMath parsed files 82 | *.sage.py 83 | 84 | # Environments 85 | .env 86 | .venv 87 | env/ 88 | venv/ 89 | ENV/ 90 | env.bak/ 91 | venv.bak/ 92 | 93 | # Spyder project settings 94 | .spyderproject 95 | .spyproject 96 | 97 | # Rope project settings 98 | .ropeproject 99 | 100 | # mkdocs documentation 101 | /site 102 | 103 | # mypy 104 | .mypy_cache/ 105 | 106 | # Mac os 107 | .DS_Store 108 | */.DS_Store 109 | 110 | # pycharm 111 | .idea 112 | .env 113 | -------------------------------------------------------------------------------- /Fgproxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kr1s77/FgSurfing/c1704c30476431bd36e0a6a240a91e32e6723dce/Fgproxy.png -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 FGSurfing 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

FGProxy

2 | 3 |

Master

5 | 6 | 7 | ##### FGProxy 是一个企业级 `4G` 代理程序。它是一个比树莓派实现的4g代理程序更稳定的代理程序。它摒弃了物联网卡的缺点，使用了真正的 `4G` 手机卡来实现，部署实现都非常的简单。 8 | 9 | #### 文档可能比较简洁，部署失败的可以邮件联系我，或者找我远程协助搭建，有啥问题也可以在 issues 里面提出。 10 | ##### 有兴趣的同学可以尝试使用 haproxy + prometheus + grafana 做个代理的监控，还是非常不错的！ 11 | 12 | #### ✅[bridge.py](https://github.com/Kr1s77/FgSurfing/blob/main/proxy/bridge.py) 可以单独运行在任何服务器上，那么运行 `bridge.py` 的服务器也可以作为代理使用，如果你想测试，可以通过这个方式 13 | 14 | #### Overall structure 15 | 16 | The following logic is then triggered： 17 |

Master

18 | 19 | 20 | ##### HAPROXY STATUS 21 |

Master

22 | 23 | --- 24 | 25 | #### Before You Begin 26 | 1. [Install lineageos](https://wiki.lineageos.org/devices/sailfish/install) at `Google Pixel` 27 | 2. The mobile phone must use a mobile phone that can execute `adb root`, I use [Google Pixel](https://en.wikipedia.org/wiki/Pixel_(1st_generation))，My system is [lineageos](https://www.lineageos.org/) 28 | 3. Before using this program, you need to confirm that your phone has been configured according to the link below [How-do-I-run-python-on-Android-devices](https://kr1s77.github.io/2021/7/12/How-do-I-run-python-on-Android-devices/) 29 | 4. Then you can execute the Fgproxy program. After the execution is completed, the port will be mapped to 30000, 30001, 30002... The number of ports is the number of devices 30 | 5. After that, configure haproxy for load balancing [Haproxy](https://github.com/haproxy/haproxy) 31 | 6. Since the machine is in our local area, we need to do intranet penetration to forward the local haproxy load balancing port, here we need [FRP](https://github.com/fatedier/frp) 32 | 33 | The above is the overall configuration process. After the configuration is completed, the frp exit is our proxy port, which can be used in the crawler. 34 | 35 | #### Create 36 | 37 | > ```shell 38 | > $ git clone https://github.com/Kr1s77/FgSurfing.git 39 | > $ cd FgSurfing/proxy 40 | > $ python3 api.py 41 | > >> [2021-07-15 14:22:32,522 INFO] -> Count: [1] Devices Found 42 | > >> [2021-07-15 14:22:32,522 INFO] -> Deploy device: FAXXXXXXX 1/1 43 | > ``` 44 | 45 | ##### Currently most of it has been completed 46 | > Anyone is welcome to participate and improve 47 | > one person can go fast, but a group of people can go further 48 | 49 | -------------------------------------------------------------------------------- /haproxy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kr1s77/FgSurfing/c1704c30476431bd36e0a6a240a91e32e6723dce/haproxy.png -------------------------------------------------------------------------------- /log.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kr1s77/FgSurfing/c1704c30476431bd36e0a6a240a91e32e6723dce/log.png -------------------------------------------------------------------------------- /proxy/__init__.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | __author__ = 'Kris ' 3 | __version__ = '1.0.0' 4 | __site__ = 'https://github.com/Kr1s77/FgSurfing' 5 | -------------------------------------------------------------------------------- /proxy/adb.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | """ 3 | FGSurfing.api 4 | ~~~~~~~~~~~~ 5 | Use this module connect your mobile by android debug bridge 6 | :copyright: (c) 2021 by Kris 7 | """ 8 | import os 9 | import re 10 | import time 11 | from network import Network 12 | # from func_timeout import exceptions 13 | # from func_timeout import func_set_timeout 14 | from command import CmdExecute, AdbCommand 15 | 16 | 17 | class AdbTools(object): 18 | """Can use adb cmd... 19 | """ 20 | 21 | def __init__(self, device_id): 22 | self.command = AdbCommand(device_id=device_id) 23 | self.cmd = CmdExecute(self.command) 24 | 25 | def check_airplane_status(self): 26 | # :Check airplane mode is open 27 | return True if int(self.cmd.execute('CHECK_AIRPLANE_MODE')) else False 28 | 29 | def forward_tcp_port(self, local_port, device_port): 30 | return self.cmd.execute('FORWARD_DEVICE_PORT', f'tcp:{local_port} tcp:{device_port}') 31 | 32 | def turn_on_airplane_mode(self): 33 | self.cmd.execute('TURN_ON_AIRPLANE_MODE') 34 | self.cmd.execute('SET_AIRPLANE_MODE_ON') 35 | time.sleep(3) 36 | return None 37 | 38 | def close_airplane_mode(self): 39 | self.cmd.execute('CLOSE_AIRPLANE_MODE') 40 | self.cmd.execute('SET_AIRPLANE_MODE_OFF') 41 | return None 42 | 43 | def push(self, filepath: str, remote_path: str) -> str: 44 | return self.cmd.execute('ADB_PUSH', filepath, remote_path) 45 | 46 | def decompress(self, remote_path: str, remote_file: str) -> str: 47 | return self.cmd.execute('GZ_DECOMPRESS', remote_path, '-C', remote_file) 48 | 49 | def delete_and_create_dir(self, remote_path: str) -> str: 50 | self.cmd.execute('RM_DIR', remote_path) 51 | return self.cmd.execute('MK_DIR', remote_path) 52 | 53 | def running_server(self, host, port): 54 | return self.cmd.execute('RUN_SERVER', host, str(port), '&>/dev/null', '&') 55 | 56 | def get_server_pid(self, port: int): 57 | return self.cmd.execute('PROCESS_ID', str(port), '| grep python') 58 | 59 | def check_remote_port(self, port: int) -> bool: 60 | return True if self.get_server_pid(port) else False 61 | 62 | def start_as_root(self): 63 | flag = self.cmd.execute('ROOT') 64 | time.sleep(2) 65 | return flag 66 | 67 | def bridge_process(self): 68 | return self.cmd.execute('GREP') 69 | 70 | def kill_port_process(self, port): 71 | # info = self.get_server_pid(port=port) 72 | # # 'ps -ef | grep {port} | grep -v grep | awk "{print $2}"| sed -e "s/^/kill -9 /g" | sh' 73 | # 74 | # pid = info.split(' ')[-1].strip().split('/')[0] 75 | pid = self.bridge_process() 76 | if not pid: 77 | return 78 | return self.cmd.execute('KILL_PROCESS', pid) 79 | 80 | def remove_forward(self, port): 81 | return self.cmd.execute('FORWARD_DEVICE_PORT', f'--remove tcp:{port}') 82 | 83 | @staticmethod 84 | def kill_master_process(port): 85 | pid = os.popen(f'lsof -t -i:{port}').read() 86 | if not pid: 87 | return 88 | command = f'kill -9 {pid}' 89 | print(f'[Command]: {command}') 90 | return os.popen(command).read() 91 | 92 | def ping_test(self): 93 | ping = self.cmd.execute('PING_TEST') 94 | if 'ping: unknown host baidu.com' in ping: 95 | # 不通杀死端口 96 | return False 97 | 98 | match = re.search(r'time=(.*?) ms', ping) 99 | if not match: 100 | return False 101 | 102 | return True 103 | 104 | @staticmethod 105 | def check_local_port(port: int): 106 | """检测本地端口""" 107 | result = os.popen(f'echo "" | telnet 127.0.0.1 {port}').read() 108 | if "Escape character is '^]'" not in result: 109 | return False 110 | return True 111 | 112 | 113 | class Device(object): 114 | """Android Device... 115 | """ 116 | 117 | def __init__( 118 | self, 119 | device_id: str, 120 | port: int = 8118, 121 | ip: str = '0.0.0.0', 122 | network: Network = Network() 123 | ): 124 | self.ip = ip 125 | self.port = port 126 | self.network = network 127 | self.device_id = device_id 128 | self.adb = AdbTools(device_id=device_id) # adb 129 | 130 | self.airplane_mode_is_open = False # airplane mode status 131 | self.transfer_port_is_open = False # device port status 132 | 133 | self.adb.start_as_root() # start device as root 134 | self.initialize_device() # init device message 135 | 136 | self.is_change_ip = False 137 | 138 | def initialize_device(self) -> tuple: 139 | self.airplane_mode_is_open = self.adb.check_airplane_status() 140 | self.transfer_port_is_open = self.adb.check_remote_port( 141 | port=self.port 142 | ) 143 | return self.airplane_mode_is_open, self.transfer_port_is_open 144 | -------------------------------------------------------------------------------- /proxy/api.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | """ 3 | Master Process 4 | 5 | 6 | Why cluster running at ip6-localhost:8899? 7 | /bin/bash$ adb shell cat /etc/hosts 8 | 127.0.0.1 localhost 9 | ::1 ip6-localhost 10 | 11 | It can also run in ::1:8899. 12 | 下一步计划：检查 4g 网络是否可用 13 | 14 | # 设备检测，如果设备不见了那么需要将设备踢出去 15 | # devices_str = init_all_devices() 16 | # log.info(f'Now running device count: {len(devices_str)}') 17 | # if device.device_id not in devices_str: 18 | # kill_master_port(device, port=master_port) 19 | # return device 20 | """ 21 | import time 22 | import logging 23 | from adb import Device 24 | from functools import partial 25 | from command import CmdExecute 26 | from multiprocessing import Queue 27 | from multiprocessing import Process 28 | from deploy.cli import deploy_to_remote 29 | from __init__ import __author__, __version__, __site__ 30 | 31 | log = logging.getLogger(__name__) 32 | 33 | 34 | def configure_logging(level): 35 | logging.basicConfig( 36 | level=level, 37 | format='[%(asctime)s %(levelname)s] -> %(message)s', 38 | ) 39 | 40 | 41 | _set_debug_logging = partial(configure_logging, logging.DEBUG) 42 | _set_info_logging = partial(configure_logging, logging.INFO) 43 | 44 | IP_SWITCHING_TIME = 30 * 60 # second 45 | MASTER_PORT_START = 30000 46 | HEALTH_CHECK_TIME = 1 * 60 # second 47 | WAIT_AIRPLANE_MODE_TIME = 8 # second 48 | 49 | 50 | def _init_msg(debug: bool) -> None: 51 | if debug: # set debug level 52 | _set_debug_logging() 53 | else: 54 | _set_info_logging() 55 | 56 | print(f'\nauthor: [{__author__}] site: [{__site__}]') 57 | print(f""" 58 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 59 | ' ███████╗ ██████╗ ██████╗ ██████╗ ██████╗ ██╗ ██╗██╗ ██╗ ' 60 | ' ██╔════╝██╔════╝ ██╔══██╗██╔══██╗██╔═══██╗╚██╗██╔╝╚██╗ ██╔╝ ' 61 | ' █████╗ ██║ ███╗██████╔╝██████╔╝██║ ██║ ╚███╔╝ ╚████╔╝ ' 62 | ' ██╔══╝ ██║ ██║██╔═══╝ ██╔══██╗██║ ██║ ██╔██╗ ╚██╔╝ ' 63 | ' ██║ ╚██████╔╝██║ ██║ ██║╚██████╔╝██╔╝ ██╗ ██║ ' 64 | ' ╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ╚═════╝ ╚═╝ ╚═╝ ╚═╝ ' 65 | {__site__} 66 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 67 | """) 68 | return None 69 | 70 | 71 | def init_all_devices(): 72 | """ 73 | List of devices attached 74 | 1daa96050207 device 75 | 1daa96050207 device 76 | :return: [1daa96050207, 1daa96050207] 77 | """ 78 | devices_string = CmdExecute.execute_command('adb devices') 79 | time.sleep(3) 80 | devices_with_str = devices_string.split('\n')[1:-2] 81 | online_devices = list() 82 | for device_str in devices_with_str: 83 | if 'device' in device_str: 84 | online_devices.append(device_str) 85 | devices = list(map(lambda x: x.split('\t')[0].strip(), online_devices)) 86 | log.info(f'Count: [{len(devices)}] Devices Found') 87 | return devices 88 | 89 | 90 | def _forward_tcp_port(device: Device, local_port, remote_port): 91 | # nginx complex balanced local port 92 | return device.adb.forward_tcp_port(local_port, remote_port) 93 | 94 | 95 | def kill_master_port(device: Device, port: int): 96 | return device.adb.kill_master_process(port) 97 | 98 | 99 | def run_server(device, master_port): 100 | """ 101 | if int(device.adb.check_remote_port(ip, port).strip()) != 0: 102 | The port is already in use 103 | """ 104 | # kill_master_port(device, master_port) 105 | 106 | device.adb.remove_forward(master_port) 107 | device.adb.kill_port_process(device.port) 108 | 109 | # :If airplane mode is opened, first need close airplane mode 110 | if device.airplane_mode_is_open: 111 | device.adb.close_airplane_mode() 112 | time.sleep(WAIT_AIRPLANE_MODE_TIME) 113 | 114 | # :Running Proxy server command 115 | device.adb.running_server(host=device.ip, port=device.port) 116 | # : wait server running 117 | time.sleep(2) 118 | device.initialize_device() 119 | # waite init device 120 | time.sleep(1) 121 | if not device.transfer_port_is_open: 122 | return 123 | 124 | _forward_tcp_port(device, master_port, device.port) 125 | 126 | 127 | def _change_ip(master_port: int, cluster_device: Device, change_ip_queue): 128 | """ 129 | # :Close master port for nginx 130 | # :Get device info 131 | # :Check proxy running port 132 | # :Time to change ip 133 | # :Open master port for nginx 134 | """ 135 | time.sleep(IP_SWITCHING_TIME) 136 | change_ip_queue.put_nowait(cluster_device.device_id) 137 | 138 | cluster_device.adb.remove_forward(master_port) 139 | cluster_device.adb.kill_port_process(cluster_device.port) 140 | cluster_device.initialize_device() 141 | 142 | if not cluster_device.airplane_mode_is_open: 143 | cluster_device.adb.turn_on_airplane_mode() 144 | cluster_device.adb.close_airplane_mode() 145 | 146 | time.sleep(WAIT_AIRPLANE_MODE_TIME) 147 | # if cluster_device.transfer_port_is_open is False: 148 | cluster_device.adb.running_server( 149 | host=cluster_device.ip, 150 | port=cluster_device.port 151 | ) 152 | time.sleep(1) 153 | if cluster_device.adb.bridge_process().strip(): 154 | _forward_tcp_port( 155 | device=cluster_device, 156 | local_port=master_port, 157 | remote_port=cluster_device.port 158 | ) 159 | time.sleep(1) 160 | 161 | if not cluster_device.adb.ping_test(): 162 | cluster_device.adb.remove_forward(port=master_port) 163 | 164 | if not cluster_device.adb.check_local_port(master_port): 165 | cluster_device.adb.remove_forward(port=master_port) 166 | 167 | change_ip_queue.get_nowait() 168 | 169 | 170 | def _health_check(master_port: int, device: Device, change_ip_queue): 171 | # 如果正在切换 IP 那么此设备不会经过健康检测，否则会出现问题 172 | time.sleep(HEALTH_CHECK_TIME) 173 | if not change_ip_queue.empty(): 174 | if device.device_id == change_ip_queue.get_nowait(): 175 | change_ip_queue.put_nowait(device.device_id) 176 | time.sleep(HEALTH_CHECK_TIME) 177 | return None 178 | 179 | change_ip_queue.put_nowait(device.device_id) 180 | time.sleep(HEALTH_CHECK_TIME) 181 | 182 | device.initialize_device() 183 | if not device.transfer_port_is_open: 184 | device.adb.remove_forward(port=master_port) 185 | 186 | if device.airplane_mode_is_open: 187 | device.adb.remove_forward(port=master_port) 188 | 189 | if not device.adb.ping_test(): 190 | device.adb.remove_forward(port=master_port) 191 | 192 | return None 193 | 194 | 195 | class Daemon(object): 196 | def __init__(self, devices: list): 197 | self.devices = devices 198 | self.change_ip_queue = Queue() 199 | 200 | def worker(self, work_func, change_ip_queue): 201 | while True: 202 | for index, device in enumerate(self.devices): 203 | master_port = MASTER_PORT_START + index 204 | work_func(master_port, device, change_ip_queue) 205 | 206 | def run_forever(self): 207 | """Running change ip and health check""" 208 | process = [Process(target=self.worker, args=(_change_ip, self.change_ip_queue))] 209 | 210 | [p.start() for p in process] 211 | [p.join() for p in process] 212 | 213 | 214 | def _deploy_all_device(devices: list) -> None: 215 | for index, device in enumerate(devices): 216 | log.info(f'Deploy device: {device.device_id} {len(devices)}/{index + 1}') 217 | log.info(f'Device: {device.device_id} transfer port running is {device.transfer_port_is_open}') 218 | log.info(f'Device: {device.device_id} transfer airplane mode open is {device.airplane_mode_is_open}') 219 | # :Deploy and run server... 220 | deploy_to_remote(device=device) 221 | master_port = MASTER_PORT_START + index 222 | run_server(device=device, master_port=master_port) 223 | time.sleep(2) 224 | 225 | daemon = Daemon(devices=devices) 226 | daemon.run_forever() 227 | 228 | 229 | def runner(debug: bool = True, ip: str = '0.0.0.0', port: int = 30000) -> None: 230 | """master entry 231 | 1. deploy the application to the phone. 232 | 2. get all device and init all device. 233 | 3. kill all proxy running port 234 | 4. running application and output log. 235 | """ 236 | _init_msg(debug=debug) 237 | devices_str = init_all_devices() 238 | 239 | if not devices_str: 240 | log.warning('No connected mobile phone was found') 241 | exit(1) 242 | 243 | devices = [Device(device_id=device_id, port=port, ip=ip) for device_id in devices_str] 244 | _deploy_all_device(devices) 245 | 246 | log.info(f'FGProxy {__version__} remote running at {ip}:{port}') 247 | 248 | 249 | if __name__ == '__main__': 250 | runner(True, 'ip6-localhost', 10000) 251 | -------------------------------------------------------------------------------- /proxy/bridge.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | """ 3 | Socket bridge // Send & Receive 4 | http.server: https://docs.python.org/3/library/http.server.html 5 | """ 6 | import os 7 | import ssl 8 | import socket 9 | import select 10 | import sys 11 | import threading 12 | import time 13 | from socketserver import ThreadingMixIn 14 | from urllib.parse import urlsplit 15 | 16 | from header import filter_header 17 | import http.client as http_lib 18 | from subprocess import Popen, PIPE 19 | from http.server import BaseHTTPRequestHandler, HTTPServer 20 | 21 | REQUEST_TIMEOUT = 20 22 | DEFAULT_CHARSET = 'UTF-8' 23 | RECEIVE_BUFFER_SIZE = 8192 24 | 25 | 26 | class ThreadingHTTPServer(ThreadingMixIn, HTTPServer): 27 | address_family = socket.AF_INET6 28 | daemon_threads = True 29 | 30 | def handle_error(self, request, client_address): 31 | cls, e = sys.exc_info()[:2] 32 | if cls is socket.error or cls is ssl.SSLError: 33 | pass 34 | else: 35 | return HTTPServer.handle_error(self, request, client_address) 36 | 37 | 38 | def join_with_script_dir(path): 39 | return os.path.join(os.path.dirname(os.path.abspath(__file__)), path) 40 | 41 | 42 | class ProxyBridge(BaseHTTPRequestHandler): 43 | lock = threading.Lock() 44 | cakey = join_with_script_dir('ca.key') 45 | ca_cert = join_with_script_dir('ca.crt') 46 | cert_key = join_with_script_dir('cert.key') 47 | cert_dir = join_with_script_dir('certs/') 48 | timeout = 5 49 | 50 | def __init__(self, *args, **kwargs): 51 | self.tls = threading.local() 52 | self.tls.conns = dict() 53 | self._headers_buffer = list() 54 | super(ProxyBridge, self).__init__(*args, **kwargs) 55 | 56 | def parse_request(self) -> bool: 57 | symbol = super(ProxyBridge, self).parse_request() 58 | self.command = self.command.lower() 59 | return symbol 60 | 61 | def do_connect(self): 62 | """HANDLE HTTPS CONNECT REQUEST 63 | """ 64 | if os.path.isfile(self.cakey) and os.path.isfile(self.ca_cert) and os.path.isfile( 65 | self.cert_key) and os.path.isdir(self.cert_dir): 66 | self.connect_intercept() 67 | else: 68 | self.exec_connect_options() 69 | 70 | def connect_intercept(self): 71 | hostname = self.path.split(':')[0] 72 | cert_path = "%s/%s.crt" % (self.cert_dir.rstrip('/'), hostname) 73 | 74 | with self.lock: 75 | if not os.path.isfile(cert_path): 76 | epoch = "%d" % (time.time() * 1000) 77 | p1 = Popen(["openssl", "req", "-new", "-key", self.cert_key, "-subj", "/CN=%s" % hostname], stdout=PIPE) 78 | p2 = Popen(["openssl", "x509", "-req", "-days", "3650", "-CA", self.ca_cert, "-CAkey", self.cakey, 79 | "-set_serial", epoch, "-out", cert_path], stdin=p1.stdout, stderr=PIPE) 80 | p2.communicate() 81 | 82 | self.wfile.write( 83 | "%s %d %s\r\n" % (self.protocol_version, 200, 'Connection Established') 84 | ) 85 | self.end_headers() 86 | 87 | self.connection = ssl.wrap_socket(self.connection, keyfile=self.cert_key, certfile=cert_path, server_side=True) 88 | self.rfile = self.connection.makefile("rb", self.rbufsize) 89 | self.wfile = self.connection.makefile("wb", self.wbufsize) 90 | 91 | conn_type = self.headers.get('Proxy-Connection', '') 92 | if self.protocol_version == "HTTP/1.1" and conn_type.lower() != 'close': 93 | self.close_connection = False 94 | else: 95 | self.close_connection = True 96 | 97 | def do_get(self): 98 | """Handle all request protocol and all request method 99 | """ 100 | return self.handle_request() 101 | 102 | def handle_request(self) -> None: 103 | req = self 104 | content_length = int(req.headers.get('Content-Length', 0)) 105 | req_body = self.rfile.read(content_length) if content_length else None 106 | 107 | if req.path[0] == '/': 108 | if isinstance(self.connection, ssl.SSLSocket): 109 | req.path = "https://%s%s" % (req.headers['Host'], req.path) 110 | else: 111 | req.path = "http://%s%s" % (req.headers['Host'], req.path) 112 | 113 | u = urlsplit(req.path) 114 | scheme, netloc, path = u.scheme, u.netloc, (u.path + '?' + u.query if u.query else u.path) 115 | assert scheme in ('http', 'https') 116 | if netloc: 117 | req.headers['Host'] = netloc 118 | setattr(req, 'headers', filter_header(req.headers)) 119 | origin = (scheme, netloc) 120 | 121 | try: 122 | if origin not in self.tls.conns: 123 | if scheme == 'https': 124 | self.tls.conns[origin] = http_lib.HTTPSConnection(netloc, timeout=self.timeout) 125 | else: 126 | self.tls.conns[origin] = http_lib.HTTPConnection(netloc, timeout=self.timeout) 127 | 128 | conn = self.tls.conns[origin] 129 | conn.request(self.command.upper(), path, req_body, dict(req.headers)) 130 | res = conn.getresponse() 131 | version_table = {10: 'HTTP/1.0', 11: 'HTTP/1.1'} 132 | setattr(res, 'response_version', version_table[res.version]) 133 | print("Headers: ", res.headers) 134 | if 'Content-Length' not in res.headers and 'no-store' in res.headers.get('Cache-Control', '') \ 135 | and 'Content-Encoding' not in res.headers: 136 | self.relay_streaming(res) 137 | return 138 | 139 | res_body = res.read() 140 | except Exception as e: 141 | if origin in self.tls.conns: 142 | del self.tls.conns[origin] 143 | self.send_error(502, str(e)) 144 | return 145 | 146 | if req_body is False: 147 | self.send_error(403) 148 | return 149 | 150 | setattr(res, 'headers', filter_header(res.headers)) 151 | 152 | self.wfile.write(f'{self.protocol_version} {res.status} {res.reason}\r\n'.encode(DEFAULT_CHARSET)) 153 | for line in res.headers: 154 | self.wfile.write((line + ': ' + res.headers[line] + '\r\n').encode(DEFAULT_CHARSET)) 155 | 156 | # self.end_headers() 157 | self._headers_buffer = [b"\r\n"] 158 | self.flush_headers() 159 | 160 | self.wfile.write(res_body) 161 | self.wfile.flush() 162 | 163 | def write_and_flush_headers(self, headers: list): 164 | self._headers_buffer += headers 165 | self.flush_headers() 166 | 167 | def relay_streaming(self, res): 168 | """查看client 是否断开链接""" 169 | self.wfile.write(f'{self.protocol_version} {res.status} {res.reason}\r\n'.encode(DEFAULT_CHARSET)) 170 | for line in res.headers.headers: 171 | self.wfile.write(line) 172 | self.end_headers() 173 | try: 174 | while True: 175 | chunk = res.read(8192) 176 | if not chunk: 177 | break 178 | self.wfile.write(chunk) 179 | self.wfile.flush() 180 | except socket.error: 181 | print('connection closed by client') 182 | 183 | def exec_connect_options(self): 184 | host, port = self.path.split(':', 1) 185 | port = int(port) or 443 186 | try: 187 | server = socket.create_connection((host, port), timeout=REQUEST_TIMEOUT) 188 | except Exception as e: 189 | return self.send_error(502, str(e)) 190 | 191 | self.send_response(200, 'Connection Established') 192 | self.end_headers() 193 | 194 | conns = [self.connection, server] 195 | self.close_connection = False 196 | while not self.close_connection: 197 | read_list, write_list, error_list = select.select(conns, [], conns, self.timeout) 198 | 199 | if not read_list or error_list: 200 | break 201 | 202 | for read in read_list: 203 | symbol = self.exec_read_fd(read_conn=read, conns=conns) 204 | if not symbol: 205 | break 206 | 207 | def exec_read_fd(self, read_conn, conns) -> bool: 208 | other = conns[1] if read_conn is conns[0] else conns[0] 209 | buffer = read_conn.recv(RECEIVE_BUFFER_SIZE) 210 | if not buffer: 211 | self.close_connection = True 212 | return False 213 | 214 | other.sendall(buffer) 215 | return True 216 | 217 | 218 | def run(server_class=ThreadingHTTPServer, handler_class=ProxyBridge): 219 | host = sys.argv[1] if len(sys.argv) > 1 else '127.0.0.1' 220 | port = int(sys.argv[2]) if len(sys.argv) > 2 else 10000 221 | print(f'server running at {host}:{port}') 222 | server_address = (host, int(port)) 223 | httpd = server_class(server_address, handler_class) 224 | httpd.serve_forever() 225 | 226 | 227 | if __name__ == '__main__': 228 | run() 229 | -------------------------------------------------------------------------------- /proxy/command.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Optional 3 | from abc import ABCMeta, abstractmethod 4 | from collections import namedtuple 5 | 6 | 7 | Cmd = namedtuple('Cmd', 'name command') 8 | 9 | ADB_PREFIX = 'adb -s {} ' 10 | 11 | ADB = { 12 | 'TURN_ON_AIRPLANE_MODE': 'settings put global airplane_mode_on 1', 13 | 'SET_AIRPLANE_MODE_ON': 'am broadcast -a android.intent.action.AIRPLANE_MODE --ez state true', 14 | 'CLOSE_AIRPLANE_MODE': 'settings put global airplane_mode_on 0', 15 | 'SET_AIRPLANE_MODE_OFF': 'am broadcast -a android.intent.action.AIRPLANE_MODE --ez state false', 16 | 'CHECK_AIRPLANE_MODE': 'settings get global airplane_mode_on', 17 | 'PING_BAIDU': 'ping -c 1 baidu.com', 18 | 'FORWARD_DEVICE_PORT': 'forward', 19 | 'ADB_DEVICES': 'devices', 20 | 'ADB_PUSH': 'push', 21 | 'GZ_DECOMPRESS': 'tar -zxf', 22 | 'RM_DIR': 'rm -rf', 23 | 'MK_DIR': 'mkdir', 24 | 'RUN_SERVER': 'python /data/local/tmp/proxy/bridge.py', 25 | 'CHECK_PORT': 'python /data/local/tmp/proxy/network.py', 26 | 'ROOT': 'root', 27 | 'PROCESS_ID': 'netstat -ntlp | grep', 28 | 'KILL_PROCESS': 'kill -9', 29 | 'KILL_MASTER_PORT': 'kill -9 $(lsof -t -i:', 30 | 'PING_TEST': 'ping -c 1 baidu.com', 31 | 'GREP': "ps -ef | grep bridge.py | grep -v 'grep' | awk '{print $2}'" 32 | } 33 | 34 | 35 | Commands = { 36 | 'TURN_ON_AIRPLANE_MODE': Cmd('TURN_ON_AIRPLANE_MODE', ADB['TURN_ON_AIRPLANE_MODE']), 37 | 'SET_AIRPLANE_MODE_ON': Cmd('SET_AIRPLANE_MODE_ON', ADB['SET_AIRPLANE_MODE_ON']), 38 | 'CLOSE_AIRPLANE_MODE': Cmd('CLOSE_AIRPLANE_MODE', ADB['CLOSE_AIRPLANE_MODE']), 39 | 'SET_AIRPLANE_MODE_OFF': Cmd('SET_AIRPLANE_MODE_OFF', ADB['SET_AIRPLANE_MODE_OFF']), 40 | 'CHECK_AIRPLANE_MODE': Cmd('CHECK_AIRPLANE_MODE', ADB['CHECK_AIRPLANE_MODE']), 41 | 'PING_BAIDU': Cmd('PING_BAIDU', ADB['CHECK_AIRPLANE_MODE']), 42 | 'FORWARD_DEVICE_PORT': Cmd('FORWARD_DEVICE_PORT', ADB['FORWARD_DEVICE_PORT']), 43 | 'ADB_DEVICES': Cmd('ADB_DEVICES', ADB['ADB_DEVICES']), 44 | 'ADB_PUSH': Cmd('ADB_PUSH', ADB['ADB_PUSH']), 45 | 'GZ_DECOMPRESS': Cmd('GZ_DECOMPRESS', ADB['GZ_DECOMPRESS']), 46 | 'RM_DIR': Cmd('RM_DIR', ADB['RM_DIR']), 47 | 'MK_DIR': Cmd('MK_DIR', ADB['MK_DIR']), 48 | 'RUN_SERVER': Cmd('RUN_SERVER', ADB['RUN_SERVER']), 49 | 'CHECK_PORT': Cmd('CHECK_PORT', ADB['CHECK_PORT']), 50 | 'ROOT': Cmd('ROOT', ADB['ROOT']), 51 | 'PROCESS_ID': Cmd('PROCESS_ID', ADB['PROCESS_ID']), 52 | 'KILL_PROCESS': Cmd('KILL_PROCESS', ADB['KILL_PROCESS']), 53 | 'KILL_MASTER_PORT': Cmd('KILL_MASTER_PORT', ADB['KILL_MASTER_PORT']), 54 | 'PING_TEST': Cmd('PING_TEST', ADB['PING_TEST']), 55 | 'GREP': Cmd('GREP', ADB['GREP']), 56 | } 57 | 58 | 59 | class Command(metaclass=ABCMeta): 60 | without_shell_list = ['push', 'forward', 'root'] 61 | 62 | def __init__(self, device_id: str = None): 63 | self.device_id = device_id 64 | 65 | @abstractmethod 66 | def gen_cmd(self, name: Optional[str], *args) -> Optional[str]: 67 | """generate command... 68 | """ 69 | pass 70 | 71 | 72 | class AdbCommand(Command): 73 | def __init__(self, device_id: str = None): 74 | super(AdbCommand, self).__init__(device_id=device_id) 75 | self.device_id = device_id 76 | 77 | def gen_cmd(self, name: Optional[str], *args) -> Optional[str]: 78 | """ Types: 79 | # adb -s '...' push ././. ././. 80 | # adb -s '...' shell ... 81 | """ 82 | command = Commands[name].command # gen command 83 | 84 | if command in self.without_shell_list: 85 | cmd = ''.join([ADB_PREFIX.format(self.device_id), ' '.join([command, *args])]) 86 | else: 87 | cmd = ''.join([ADB_PREFIX.format(self.device_id), 'shell ', ' '.join([command, *args])]) 88 | 89 | return cmd 90 | 91 | 92 | class CmdExecute(): 93 | def __init__(self, command: Command): 94 | self.command = command 95 | 96 | def get_state(self): 97 | pass 98 | 99 | def execute(self, name: Optional[str], *args) -> Optional[str]: 100 | cmd = self.command.gen_cmd(name, *args) 101 | print('[Command]:', cmd) 102 | return self.execute_command(command=cmd) 103 | 104 | @staticmethod 105 | def execute_command(command: Optional[str]): 106 | value = os.popen(command).read() 107 | return value 108 | -------------------------------------------------------------------------------- /proxy/deploy/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import tarfile 3 | import zipfile 4 | from adb import Device 5 | 6 | # Default dir path 7 | current_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) 8 | proxy_dir = os.path.join(current_dir, 'proxy') 9 | 10 | 11 | def make_gz(output_filename: str, source_dir): 12 | with tarfile.open(output_filename, "w:gz") as tar: 13 | tar.add(source_dir, arcname=os.path.basename(source_dir)) 14 | 15 | 16 | def make_zip(output_filename: str, source_dir): 17 | zip_file = zipfile.ZipFile(output_filename, 'w') 18 | pre_len = len(os.path.dirname(source_dir)) 19 | for parent, _, filenames in os.walk(source_dir): 20 | for filename in filenames: 21 | path_file = os.path.join(parent, filename) 22 | name = path_file[pre_len:].strip(os.path.sep) # ads path 23 | zip_file.write(path_file, name) 24 | zip_file.close() 25 | 26 | 27 | def push_to_mobile(filepath: str, remote_path: str, device: Device) -> str: 28 | device.adb.push( 29 | filepath=filepath, 30 | remote_path=remote_path 31 | ) 32 | return remote_path 33 | 34 | 35 | def decompress_mobile_file(remote_path: str, remote_file: str, device: Device) -> str: 36 | device.adb.decompress( 37 | remote_path=remote_path, 38 | remote_file=remote_file, 39 | ) 40 | return remote_file 41 | 42 | 43 | def rm_and_mk_dir(remote_path: str, device: Device) -> str: 44 | device.adb.delete_and_create_dir( 45 | remote_path=remote_path 46 | ) 47 | return remote_path 48 | 49 | 50 | def deploy_to_remote(device: Device): 51 | output_filename = os.path.basename(proxy_dir) + '.tar.gz' 52 | make_gz(output_filename, proxy_dir) 53 | 54 | filepath = os.path.join(proxy_dir, output_filename) 55 | mobile_file_path = push_to_mobile( 56 | filepath=filepath, 57 | remote_path='/data/local/tmp/', 58 | device=device 59 | ) 60 | # print(f'*{output_filename}* was pushed to mobile *{mobile_file_path}*') 61 | mobile_compress_file = os.path.join(mobile_file_path, output_filename) 62 | mobile_script_file = os.path.join(mobile_file_path, os.path.basename(proxy_dir)) 63 | 64 | # decompress script file 65 | # rm and mk dir 66 | rm_and_mk_dir(remote_path=mobile_script_file, device=device) 67 | 68 | decompress_mobile_file( 69 | remote_path=mobile_compress_file, 70 | remote_file=mobile_file_path, 71 | device=device 72 | ) 73 | 74 | # clean local compress file 75 | os.popen(f'rm -rf {filepath}') 76 | -------------------------------------------------------------------------------- /proxy/header.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | """ 3 | FgSurfing.api 4 | ~~~~~~~~~~~~ 5 | Use this module to replace the request headers 6 | :copyright: (c) 2021 by Kris 7 | """ 8 | import re 9 | 10 | # :Filter request headers segments 11 | FILTER_HEADER_SEGMENTS = ( 12 | 'connection', 13 | 'keep-alive', 14 | 'proxy-authenticate', 15 | 'proxy-authorization', 16 | 'te', 17 | 'trailers', 18 | 'transfer-encoding', 19 | 'upgrade' 20 | ) 21 | 22 | 23 | def _to_capitalize(word: str): 24 | """ :Header segment to capitalize 25 | """ 26 | return '-'.join([i.capitalize() for i in word.split('-')]) 27 | 28 | 29 | FILTER_HEADER_SEGMENTS_UPPER = tuple([i.upper() for i in FILTER_HEADER_SEGMENTS]) 30 | FILTER_HEADER_SEGMENTS_CAPITALIZE = tuple([_to_capitalize(i) for i in FILTER_HEADER_SEGMENTS]) 31 | 32 | 33 | def filter_header(headers): 34 | """ 35 | :Remove irrelevant request headers to prevent the 36 | :agent from being discovered..................... 37 | """ 38 | for key, upper_key, capitalize_key in zip( 39 | FILTER_HEADER_SEGMENTS, 40 | FILTER_HEADER_SEGMENTS_UPPER, 41 | FILTER_HEADER_SEGMENTS_CAPITALIZE 42 | ): 43 | del headers[key] 44 | del headers[upper_key] 45 | del headers[capitalize_key] 46 | 47 | if 'Accept-Encoding' in headers: 48 | ae = headers['Accept-Encoding'] 49 | filtered_encodings = [x for x in re.split(r',\s*', ae) if x in ('identity', 'gzip', 'x-gzip', 'deflate')] 50 | headers['Accept-Encoding'] = ', '.join(filtered_encodings) 51 | 52 | return headers 53 | -------------------------------------------------------------------------------- /proxy/network.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | """ 3 | Slaver Process 4 | ~~~~~~~~~~~~ 5 | Use this module to replace the request headers 6 | :copyright: (c) 2021 by Kris 7 | """ 8 | import sys 9 | import socket 10 | 11 | 12 | class Network(object): 13 | def check_local_port(self, port: int, host='localhost'): 14 | port_msg = dict() 15 | port_msg['host'] = host 16 | port_msg['port'] = port 17 | return self._check_tcp_port(port_msg) 18 | 19 | @staticmethod 20 | def _check_tcp_port(port_msg: dict, timeout=2): 21 | cs = socket.socket(socket.AF_INET, socket.SOCK_STREAM) 22 | address = (str(port_msg["host"]), int(port_msg["port"])) 23 | cs.settimeout(timeout) 24 | status = cs.connect_ex(address) 25 | return 1 if status == 0 else 0 26 | 27 | def check_remote_port(self, host: str, port: int): 28 | return self.check_local_port(port, host) 29 | 30 | 31 | def check(): 32 | host = 'localhost' 33 | port = int(sys.argv[1]) if len(sys.argv) > 1 else 30000 34 | network = Network() 35 | print(network.check_remote_port(host, port)) 36 | 37 | 38 | if __name__ == '__main__': 39 | check() 40 | -------------------------------------------------------------------------------- /proxy/structure.py: -------------------------------------------------------------------------------- 1 | # _*_ coding: utf-8 _*_ 2 | --------------------------------------------------------------------------------