├── .gitignore ├── __init__.py ├── rab_requirements.txt ├── Dockerfile ├── .github └── workflows │ └── main.yaml ├── LICENSE ├── rab_docker.py ├── rab_anonymity.py ├── rab_linux_command.ini ├── rab_config.ini ├── rab_wingman.py ├── rab_unicode.py ├── rab_hourglass.py ├── rab_redis.py ├── rab_env.py ├── rab_mongodb.py ├── README.md ├── rab_imap.py ├── rab_config.py ├── rab_ip.py ├── rab_websocket.py ├── rab_client.py ├── rab_subscription.py ├── rab_cryptography.py ├── rab_requests.py ├── rab_distributed_system.py ├── rab_logging.py ├── rab_advance.py ├── rab_bot.py ├── rab_rabbitmq.py ├── rab_crontab.py ├── rab_chrome.py ├── rab_proxy.py ├── rab_postgresql.py ├── rab_node.py └── rab_storage.py /.gitignore: -------------------------------------------------------------------------------- 1 | # IDEA 或 VS Code 等开发工具 2 | .idea/ 3 | .vscode/ 4 | 5 | # 日志 6 | **/log 7 | **/rab_log 8 | nohup.out 9 | 10 | # 编译 11 | **/__pycache__ 12 | 13 | # 测试 14 | **/nodes 15 | 16 | # 备份文件 17 | *bak.py -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/steammarket/rab_python_packages/__init__.py 6 | # @DATE: 2021/08/04 Wed 7 | # @TIME: 21:32:09 8 | # 9 | # @DESCRIPTION: 包初始化 -------------------------------------------------------------------------------- /rab_requirements.txt: -------------------------------------------------------------------------------- 1 | cfscrape==2.1.1 2 | configparser==5.0.2 3 | charset-normalizer==2.0.3 4 | docker==5.0.0 5 | minio==7.1.0 6 | pika==1.2.0 7 | psycopg2==2.9.3 8 | pymongo==3.12.1 9 | PyYAML==6.0 10 | redis==3.5.3 11 | requests[socks]>=2.10.0 12 | rsa==4.7.2 13 | selenium==3.141.0 14 | singleton_decorator==1.0.0 15 | six==1.16.0 16 | urllib3==1.25.11 17 | websocket==0.2.1 -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # 基础镜像系统版本为 CentOS:7 2 | FROM centos:7 3 | 4 | # 维护者信息 5 | LABEL maintainer="Rabbir admin@cs.cheap" 6 | 7 | # Docker 内用户切换到 root 8 | USER root 9 | 10 | # 设置时区为东八区 11 | ENV TZ Asia/Shanghai 12 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime > /etc/timezone 13 | 14 | # 安装 Git 和 Python3 15 | WORKDIR /root 16 | RUN yum -y install git 17 | RUN curl -s https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20Python3%20%E7%8E%AF%E5%A2%83/install.sh | bash 18 | 19 | # 在 /root/GitHub 目录下克隆 rab_python_packages 项目 20 | RUN mkdir /root/GitHub 21 | RUN mkdir /root/GitHub/rab_python_packages 22 | WORKDIR /root/GitHub/rab_python_packages 23 | # 将宿主机当前目录下的所有文件拷贝至镜像内的 /root/GitHub/rab_python_packages 文件夹中 24 | COPY . . 25 | 26 | # 配置环境 27 | RUN pip3 install configparser 28 | RUN python3 rab_env.py 29 | RUN python3 rab_env.py rab_chrome 30 | 31 | # 删除无用文件 32 | RUN rm -r chromedriver_linux64.zip 33 | RUN rm -r google-chrome-stable_current_x86_64.rpm -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | # 将项目构建镜像并发布至 Docker Hub 2 | name: CI to Docker Hub 3 | 4 | on: 5 | push: 6 | branches: [ main ] 7 | 8 | jobs: 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | # Check Out 源码 13 | - name: Check Out Repo 14 | uses: actions/checkout@v2 15 | # 登录至 Docker Hub 16 | - name: Login to Docker Hub 17 | uses: docker/login-action@v1 18 | with: 19 | username: ${{ secrets.DOCKER_HUB_USERNAME }} 20 | password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} 21 | # 设置 Buildx 为构建镜像做准备 22 | - name: Set up Docker Buildx 23 | id: buildx 24 | uses: docker/setup-buildx-action@v1 25 | # 构建并发布 26 | - name: Build and push 27 | id: docker_build 28 | uses: docker/build-push-action@v2 29 | with: 30 | context: ./ 31 | file: ./Dockerfile 32 | push: true 33 | tags: ${{ secrets.DOCKER_HUB_USERNAME }}/rab_python_packages:latest 34 | # 完成 35 | - name: Image digest 36 | run: echo ${{ steps.docker_build.outputs.digest }} -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Rabbir 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /rab_docker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_docker.py 6 | # @DATE: 2021/08/07 Sat 7 | # @TIME: 16:03:08 8 | # 9 | # @DESCRIPTION: Docker 容器管理模块 10 | 11 | 12 | import docker 13 | 14 | 15 | # 主 Client 16 | client = docker.from_env() 17 | 18 | 19 | """ 20 | @description: 获取 Client 21 | ------- 22 | @param: 23 | ------- 24 | @return: 25 | """ 26 | def get_client(): 27 | return client 28 | 29 | """ 30 | @description: 获取所有正在允许的 Docker 容器 31 | ------- 32 | @param: 33 | ------- 34 | @return: 35 | """ 36 | def get_containers(image_keyword=None, name_keyword=None): 37 | containers = client.containers.list(all=True) 38 | # 根据镜像名或关键词来筛选 39 | if (image_keyword): 40 | filtered_containers = [] 41 | for container in containers: 42 | if (image_keyword.lower() in str(container.image).lower()): 43 | filtered_containers.append(container) 44 | containers = filtered_containers 45 | # 根据容器名或关键词来筛选 46 | if (name_keyword): 47 | filtered_containers = [] 48 | for container in containers: 49 | if (name_keyword.lower() in str(container.name).lower()): 50 | filtered_containers.append(container) 51 | containers = filtered_containers 52 | # 如果没有筛选条件则直接返回 53 | return containers 54 | 55 | 56 | """ 57 | @description: 单体测试 58 | ------- 59 | @param: 60 | ------- 61 | @return: 62 | """ 63 | if __name__ == "__main__": 64 | for container in get_containers(): 65 | print(container.name) -------------------------------------------------------------------------------- /rab_anonymity.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_anonymity.py 6 | # @DATE: 2022/01/24 Mon 7 | # @TIME: 15:06:30 8 | # 9 | # @DESCRIPTION: 共通匿名模块 10 | 11 | 12 | import random 13 | 14 | 15 | """ 16 | @description: 获取随机的 User-Agent 17 | ------- 18 | @param: 19 | ------- 20 | @return: 21 | """ 22 | def get_random_user_agent(): 23 | # 浏览器信息 24 | user_agents = [ 25 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", 26 | "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", 27 | "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.82 Safari/537.36", 28 | "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.15 Safari/537.36 Core/1.53.3368.400 QQBrowser/9.6.11974.400", 29 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36", 30 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.15 Safari/537.36", 31 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.15 Safari/537.36 Mozilla/5.0 (Windows NT 6.1; WOW64 Trident/7.0; rv:11.0) like Gecko Windows NT 6.1; Trident/5.0; InfoPath.2; QIHU 360EE)", 32 | "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/93.0.4577.63 Safari/537.36 Mozilla/5.0 (Windows NT 6.1; WOW64 Trident/7.0; rv:11.0) like Gecko Windows NT 6.1; Trident/5.0; InfoPath.2; QIHU 360EE)" 33 | ] 34 | return random.choice(user_agents) -------------------------------------------------------------------------------- /rab_linux_command.ini: -------------------------------------------------------------------------------- 1 | # Liunx 命令 2 | [common] 3 | kill_process_by_port=kill -9 $(lsof -t -i:{port_4_replace}) 4 | 5 | # Chrome 模块 6 | [rab_chrome] 7 | gost_start=/usr/local/bin/gost_start.sh 8 | gost_stop=/usr/local/bin/gost_stop.sh 9 | 10 | # 环境模块 11 | [rab_env] 12 | fix_rab_chrome=curl -s https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20Chrome%20%E5%92%8C%20chromedriver/install.sh | bash 13 | fix_rab_chrome_gost=curl -s https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20GOST%20%E5%B9%B6%E5%90%AF%E5%8A%A8%20HTTP%20%E5%92%8C%20SOCKS5%20%E4%BB%A3%E7%90%86%E6%9C%8D%E5%8A%A1/install.sh | bash 14 | fix_rab_chrome_gost_start=wget --timeout=5 --waitretry=2 --tries=3 https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20GOST%20%E5%B9%B6%E5%90%AF%E5%8A%A8%20HTTP%20%E5%92%8C%20SOCKS5%20%E4%BB%A3%E7%90%86%E6%9C%8D%E5%8A%A1/%E5%90%AF%E5%8A%A8%E6%9C%8D%E5%8A%A1/start.sh -O /usr/local/bin/gost_start.sh && chmod +x /usr/local/bin/gost_start.sh 15 | fix_rab_chrome_gost_stop=wget --timeout=5 --waitretry=2 --tries=3 https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20GOST%20%E5%B9%B6%E5%90%AF%E5%8A%A8%20HTTP%20%E5%92%8C%20SOCKS5%20%E4%BB%A3%E7%90%86%E6%9C%8D%E5%8A%A1/%E5%90%AF%E5%8A%A8%E6%9C%8D%E5%8A%A1/stop.sh -O /usr/local/bin/gost_stop.sh && chmod +x /usr/local/bin/gost_stop.sh 16 | 17 | # 订阅节点 18 | [rab_node] 19 | clash_init=wget --timeout=5 --waitretry=2 --tries=3 'https://gitee.com/senjianlu/one-click-scripts/raw/main/CentOS7%20%E4%B8%8B%E4%B8%80%E9%94%AE%E5%AE%89%E8%A3%85%20Clash%20%E5%AE%A2%E6%88%B7%E7%AB%AF/%E9%85%8D%E7%BD%AE%E6%96%87%E4%BB%B6/config.yaml' -O /root/.config/clash/config.yaml 20 | clash_start=/usr/local/bin/clash_start.sh 21 | clash_stop=/usr/local/bin/clash_stop.sh 22 | clash_configure=sed -i "s/{socks-port}/{socks-port_4_replace}/; s#{node-info}#{node-info_4_replace}#;" /root/.config/clash/config.yaml 23 | gost_start=/usr/local/bin/gost_start.sh 24 | gost_stop=/usr/local/bin/gost_stop.sh -------------------------------------------------------------------------------- /rab_config.ini: -------------------------------------------------------------------------------- 1 | # rab_python_packages 配置文件 2 | 3 | # 共用信息 4 | [common] 5 | # 默认使用数据库 6 | database= 7 | # 用户 8 | user= 9 | # 密码 10 | password= 11 | # IP 或域名 12 | host= 13 | # 端口 14 | port= 15 | # 用以更新 SSR 订阅信息和 Telegram 发送信息的通用代理 16 | proxy=[] 17 | 18 | [rab_requests] 19 | # 超时时间 20 | timeout=5 21 | # 最大访问尝试次数 22 | max_retry_num=5 23 | 24 | # RabbitMQ 连接信息 25 | [rab_rabbitmq] 26 | # 用户名 27 | username= 28 | # 密码 29 | password= 30 | # RabbitMQ 地址 31 | host= 32 | # 端口号 33 | port= 34 | # HTTP API 接口地址 35 | api_host= 36 | 37 | # Redis 连接信息 38 | [rab_redis] 39 | # Redis 地址 40 | host= 41 | # 端口号 42 | port= 43 | # 密码 44 | password= 45 | 46 | # PostgreSQL 数据库驱动 47 | [rab_postgresql] 48 | # 批量插入数值 49 | batch_size=500 50 | 51 | # 爬虫代理池信息 52 | [rab_proxy] 53 | # 用以存储代理信息的数据库 54 | proxy_database= 55 | # 用以存储 HTTP 和 SOCKS5 代理信息的表 56 | proxy_table= 57 | # 用以存储自建代理的服务器信息的表 58 | proxy_servers_table= 59 | 60 | # Telegram 机器人信息 61 | [rab_bot] 62 | # 机器人 Token 63 | telegram_token= 64 | # 默认 Chat ID 65 | telegram_chat_id= 66 | 67 | # 简单加密 68 | [rab_cryptography] 69 | # 字符串 70 | chars= 71 | 72 | # 分布式节点模块 73 | [rab_distributed_system] 74 | # 节点 ID 75 | node_id=main 76 | # 节点启动延迟（秒） 77 | node_delay_time=300 78 | 79 | # 日志记录 80 | [rab_logging] 81 | # 记录等级，只要等于或高于这个等级的日志才会被记录 82 | level=INFO 83 | 84 | # 静态资源 85 | [rab_storage] 86 | # MinIO IP 或域名 87 | minio_host= 88 | # MinIO 用户名 89 | minio_user= 90 | # MinIO 密码 91 | minio_password= 92 | # COS 的 IP 或域名 93 | cos_host= 94 | # Nextcloud IP 或域名 95 | nextcloud_host= 96 | # Nextcloud 用户名 97 | nextcloud_user= 98 | # Nextcloud 密码 99 | nextcloud_password= 100 | 101 | # MongoDB 类 102 | [rab_mongodb] 103 | # 用户名 104 | user= 105 | # 密码 106 | password= 107 | # RabbitMQ 地址 108 | host= 109 | # 端口号 110 | port= 111 | 112 | # 订阅节点解析模块 113 | [rab_node] 114 | # 解析方式（subconverter 或 python3） 115 | parse_method=subconverter 116 | # subconverter 的后端地址 117 | subconverter_url= 118 | 119 | # 自用进阶模块 120 | [rab_advance] 121 | # 邮局管理用户名 122 | rapo_username= 123 | # 邮局管理密码 124 | rapo_password= 125 | # 邮局管理面板 IP 或域名 126 | rapo_host= 127 | # 邮局管理面板端口 128 | rapo_port= 129 | # 邮箱域名后缀 130 | rapo_domain= 131 | # 5sim token 132 | ra5_token= -------------------------------------------------------------------------------- /rab_wingman.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_wingman.py 6 | # @DATE: 2021/07/22 Thu 7 | # @TIME: 15:24:59 8 | # 9 | # @DESCRIPTION: 多线程小帮手 10 | 11 | 12 | import sys 13 | import time 14 | sys.path.append("..") if ".." not in sys.path else True 15 | from rab_python_packages import rab_logging 16 | 17 | 18 | # 日志记录 19 | r_logger = rab_logging.r_logger() 20 | 21 | 22 | """ 23 | @description: r_wingman 多线程小帮手类 24 | ------- 25 | @param: 26 | ------- 27 | @return: 28 | """ 29 | class r_wingman: 30 | 31 | """ 32 | @description: 初始化 33 | ------- 34 | @param: 35 | ------- 36 | @return: 37 | """ 38 | def __init__(self): 39 | # 线程字典 40 | self.thread = {} 41 | # 线程当前状态 42 | self.thread_is_over = {} 43 | 44 | """ 45 | @description: 启动全部线程 46 | ------- 47 | @param: 48 | ------- 49 | @return: 50 | """ 51 | def start_all(self): 52 | r_logger.info("开始启动全部线程......") 53 | for thread_key in self.thread.keys(): 54 | self.thread[thread_key].setDaemon(True) 55 | self.thread[thread_key].start() 56 | # 标志线程启动 57 | self.thread_is_over[thread_key] = False 58 | for thread_key in self.thread.keys(): 59 | self.thread[thread_key].join() 60 | 61 | """ 62 | @description: 标志线程结束 63 | ------- 64 | @param: 65 | ------- 66 | @return: 67 | """ 68 | def over(self, thread_key): 69 | r_logger.info("线程 {} 结束。".format(str(thread_key))) 70 | self.thread_is_over[thread_key] = True 71 | 72 | """ 73 | @description: 是否所有线程均结束 74 | ------- 75 | @param: 76 | ------- 77 | @return: 78 | """ 79 | def is_all_over(self): 80 | for thread_key in self.thread.keys(): 81 | if (not self.thread_is_over[thread_key]): 82 | return False 83 | return True 84 | 85 | """ 86 | @description: 等待所有线程结束 87 | ------- 88 | @param: 89 | ------- 90 | @return: 91 | """ 92 | def wait(self): 93 | while True: 94 | if (self.is_all_over()): 95 | r_logger.info("检测到所有线程均已结束！") 96 | return True 97 | else: 98 | r_logger.info("等待所有线程结束中......") 99 | time.sleep(10) -------------------------------------------------------------------------------- /rab_unicode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_unicode.py 6 | # @DATE: 2022/01/16 Sun 7 | # @TIME: 11:06:09 8 | # 9 | # @DESCRIPTION: 共通包 字符判断 10 | 11 | 12 | import re 13 | 14 | 15 | """ 16 | @description: 判断是否为中文 17 | ------- 18 | @param: 19 | ------- 20 | @return: 21 | """ 22 | def is_chinese(char): 23 | if (char >= "\u4e00" and char <= "\u9fa5"): 24 | return True 25 | else: 26 | return False 27 | 28 | """ 29 | @description: 判断是否为英文字符 30 | ------- 31 | @param: 32 | ------- 33 | @return: 34 | """ 35 | def is_alphabet(char): 36 | if ((char >= "\u0041" and char <= "\u005a") 37 | or (char >= "\u0061" and char <= "\u007a")): 38 | return True 39 | else: 40 | return False 41 | 42 | """ 43 | @description: 判断是否为数字 44 | ------- 45 | @param: 46 | ------- 47 | @return: 48 | """ 49 | def is_number(char): 50 | if (char >= "\u0030" and char <= "\u0039"): 51 | return True 52 | else: 53 | return False 54 | 55 | """ 56 | @description: 判断字符串中是否只有数字和字母 57 | ------- 58 | @param: 59 | ------- 60 | @return: 61 | """ 62 | def is_all_alphabet_and_number(_string): 63 | len_after_filter = re.sub( 64 | u"([^\u0041-\u005a\u0061-\u007a\u0030-\u0039])", "", _string) 65 | if (len(len_after_filter) == len(_string)): 66 | return True 67 | else: 68 | return False 69 | 70 | """ 71 | @description: 获取字符串中全角和半角字符的数量 72 | ------- 73 | @param: 74 | ------- 75 | @return: 76 | """ 77 | def get_true_length(_string): 78 | true_length = 0 79 | for char in _string: 80 | if (char >= "\u0000" and char <= "\u0019"): 81 | true_length += 0 82 | elif(char >= "\u0020" and char <= "\u1FFF"): 83 | true_length += 1 84 | elif(char >= "\u2000" and char <= "\uFF60"): 85 | true_length += 2 86 | elif(char >= "\uFF61" and char <= "\uFF9F"): 87 | true_length += 1 88 | else: 89 | true_length += 2 90 | return true_length 91 | 92 | 93 | """ 94 | @description: 单体测试 95 | ------- 96 | @param: 97 | ------- 98 | @return: 99 | """ 100 | if __name__ == "__main__": 101 | _string = "全角字符，。abc0123456,./™★" 102 | print(get_true_length(_string)) 103 | print(is_all_alphabet_and_number(_string)) -------------------------------------------------------------------------------- /rab_hourglass.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_hourglass.py 6 | # @DATE: 2022/01/24 Mon 7 | # @TIME: 20:32:05 8 | # 9 | # @DESCRIPTION: 沙漏计时 10 | 11 | 12 | import time 13 | from threading import Thread 14 | 15 | 16 | """ 17 | @description: 沙漏类 18 | ------- 19 | @param: 20 | ------- 21 | @return: 22 | """ 23 | class r_hourglass(): 24 | 25 | """ 26 | @description: 初始化 27 | ------- 28 | @param: 29 | ------- 30 | @return: 31 | """ 32 | def __init__(self, countdown_seconds, function_2_execute): 33 | self.countdown_seconds = countdown_seconds 34 | self.function_2_execute = function_2_execute 35 | self.last_seconds = 0 36 | self.countdown_thread = None 37 | 38 | """ 39 | @description: 倒计时 40 | ------- 41 | @param: 42 | ------- 43 | @return: 44 | """ 45 | def countdown(self): 46 | while(self.last_seconds > 0): 47 | time.sleep(1) 48 | self.last_seconds = self.last_seconds - 1 49 | print("沙漏剩余 {} 秒......".format( 50 | str(self.last_seconds))) 51 | return self.function_2_execute() 52 | 53 | """ 54 | @description: 沙漏开始 55 | ------- 56 | @param: 57 | ------- 58 | @return: 59 | """ 60 | def start(self): 61 | print("沙漏开始计时！") 62 | self.last_seconds = self.countdown_seconds 63 | self.countdown_thread = Thread(target=self.countdown) 64 | self.countdown_thread.setDaemon(True) 65 | self.countdown_thread.start() 66 | 67 | """ 68 | @description: 沙漏停止 69 | ------- 70 | @param: 71 | ------- 72 | @return: 73 | """ 74 | def stop(self): 75 | print("沙漏停止计时！") 76 | self.last_seconds = 999999 77 | 78 | """ 79 | @description: 重置沙漏 80 | ------- 81 | @param: 82 | ------- 83 | @return: 84 | """ 85 | def reset(self): 86 | print("沙漏被重置！") 87 | self.last_seconds = self.countdown_seconds 88 | 89 | 90 | """ 91 | @description: 单体测试 92 | ------- 93 | @param: 94 | ------- 95 | @return: 96 | """ 97 | if __name__ == "__main__": 98 | def test_function(): 99 | print("我被执行了！") 100 | r_hourglass = r_hourglass(15, test_function) 101 | r_hourglass.start() 102 | time.sleep(20) 103 | -------------------------------------------------------------------------------- /rab_redis.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_redis.py 6 | # @DATE: 2021/10/18 Mon 7 | # @TIME: 17:29:35 8 | # 9 | # @DESCRIPTION: 共通包 Redis 模块 10 | 11 | 12 | import sys 13 | import redis 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_config 16 | from rab_python_packages import rab_logging 17 | 18 | 19 | # 日志记录 20 | r_logger = rab_logging.r_logger() 21 | 22 | 23 | """ 24 | @description: r_redis_driver 类 25 | ------- 26 | @param: 27 | ------- 28 | @return: 29 | """ 30 | class r_redis_driver(): 31 | 32 | """ 33 | @description: 初始化 34 | ------- 35 | @param: 36 | ------- 37 | @return: 38 | """ 39 | def __init__(self, 40 | host=rab_config.load_package_config( 41 | "rab_config.ini", "rab_redis", "host"), 42 | port=rab_config.load_package_config( 43 | "rab_config.ini", "rab_redis", "port"), 44 | password=rab_config.load_package_config( 45 | "rab_config.ini", "rab_redis", "password"), 46 | decode_responses=True): 47 | self.host = host 48 | self.port = port 49 | self.password = password 50 | self.decode_responses = decode_responses 51 | self.connection = None 52 | 53 | """ 54 | @description: 建立连接 55 | ------- 56 | @param: 57 | ------- 58 | @return: 59 | """ 60 | def connect(self): 61 | try: 62 | self.connection = redis.Redis(host=self.host, port=self.port, \ 63 | password=self.password, decode_responses=self.decode_responses, \ 64 | charset="UTF-8", encoding="UTF-8") 65 | return True 66 | except Exception as e: 67 | r_logger.error("Redis 建立连接时出错！") 68 | r_logger.error(e) 69 | return False 70 | 71 | """ 72 | @description: 断开连接 73 | ------- 74 | @param: 75 | ------- 76 | @return: 77 | """ 78 | def disconnect(self): 79 | pass 80 | 81 | 82 | """ 83 | @description: 单体测试 84 | ------- 85 | @param: 86 | ------- 87 | @return: 88 | """ 89 | if __name__ == "__main__": 90 | r_redis_driver = r_redis_driver() 91 | r_redis_driver.connect() 92 | print(r_redis_driver.connection.hget( 93 | "status.project.subproject.module.submodule.method.function", \ 94 | "node_id_01")) -------------------------------------------------------------------------------- /rab_env.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_env.py 6 | # @DATE: 2021/07/15 Thu 7 | # @TIME: 16:31:00 8 | # 9 | # @DESCRIPTION: 运行环境的检测和修复，包括 Docker 镜像的下载和加载等 10 | 11 | 12 | import os 13 | import sys 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_config 16 | 17 | 18 | """ 19 | @description: 环境安装 Yum 20 | ------- 21 | @param: 22 | ------- 23 | @return: 24 | """ 25 | def fix_env_by_yum(): 26 | dependence_list = ["epel-release", "postgresql-devel"] 27 | for dependence in dependence_list: 28 | print("Yum 开始安装：" + dependence) 29 | command = "yum -y install " + dependence 30 | os.system(command) 31 | 32 | """ 33 | @description: 环境安装 pip 34 | ------- 35 | @param: 36 | ------- 37 | @return: 38 | """ 39 | def fix_env_by_pip(): 40 | file_name = "rab_requirements.txt" 41 | file_path = rab_config.find_rab_file(file_name) 42 | print("pip 开始从文件 " + file_path + " 开始安装！") 43 | command = "pip3 install -r " + file_path 44 | os.system(command) 45 | 46 | """ 47 | @description: 安装运行环境 48 | ------- 49 | @param: 50 | ------- 51 | @return: 52 | """ 53 | def fix(moudule_name=None): 54 | # 没有包名的情况下默认安装所有 55 | if (not moudule_name): 56 | fix_env_by_yum() 57 | fix_env_by_pip() 58 | # rab_chrome 模块 59 | elif(moudule_name=="rab_chrome"): 60 | # 下载安装 Chrome 和对应版本的 chromedriver 并建立软连接 61 | fix_rab_chrome_command = rab_config.load_package_config( 62 | "rab_linux_command.ini", "rab_env", "fix_rab_chrome") 63 | os.system(fix_rab_chrome_command) 64 | # 安装 GOST 并下载启动和停止脚本 65 | fix_rab_chrome_gost_command = rab_config.load_package_config( 66 | "rab_linux_command.ini", "rab_env", "fix_rab_chrome_gost") 67 | os.system(fix_rab_chrome_gost_command) 68 | fix_rab_chrome_gost_start_command = rab_config.load_package_config( 69 | "rab_linux_command.ini", "rab_env", "fix_rab_chrome_gost_start") 70 | os.system(fix_rab_chrome_gost_start_command) 71 | fix_rab_chrome_gost_stop_command = rab_config.load_package_config( 72 | "rab_linux_command.ini", "rab_env", "fix_rab_chrome_gost_stop") 73 | os.system(fix_rab_chrome_gost_stop_command) 74 | 75 | 76 | """ 77 | @description: 单体测试 78 | ------- 79 | @param: 80 | ------- 81 | @return: 82 | """ 83 | if __name__ == "__main__": 84 | # 指定安装某个模块的环境 85 | if (len(sys.argv) > 1): 86 | fix(sys.argv[1]) 87 | else: 88 | fix() -------------------------------------------------------------------------------- /rab_mongodb.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_mongodb.py 6 | # @DATE: 2022/03/27 周日 7 | # @TIME: 21:01:48 8 | # 9 | # @DESCRIPTION: 共通包 MongoDB 模块 10 | 11 | 12 | import sys 13 | import pymongo 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_config 16 | from rab_python_packages import rab_logging 17 | 18 | 19 | # 日志记录 20 | r_logger = rab_logging.r_logger() 21 | 22 | 23 | """ 24 | @description: r_mongodb_driver 类 25 | ------- 26 | @param: 27 | ------- 28 | @return: 29 | """ 30 | class r_mongodb_driver(): 31 | 32 | """ 33 | @description: 初始化 34 | ------- 35 | @param: 36 | ------- 37 | @return: 38 | """ 39 | def __init__(self, 40 | user=rab_config.load_package_config( 41 | "rab_config.ini", "rab_mongodb", "user"), 42 | password=rab_config.load_package_config( 43 | "rab_config.ini", "rab_mongodb", "password"), 44 | host=rab_config.load_package_config( 45 | "rab_config.ini", "rab_mongodb", "host"), 46 | port=rab_config.load_package_config( 47 | "rab_config.ini", "rab_mongodb", "port"), 48 | database=None): 49 | # 用户 50 | self.user = user 51 | # 密码 52 | self.password = password 53 | # IP 或域名 54 | self.host = host 55 | # 端口 56 | self.port = port 57 | # 目标数据库 58 | self.database = database 59 | # 连接（客户端） 60 | self.client = None 61 | # 数据库 62 | self.db = None 63 | 64 | """ 65 | @description: 建立数据库连接 66 | ------- 67 | @param: 68 | ------- 69 | @return: 70 | """ 71 | def connect(self): 72 | # 创建连接 73 | self.client = pymongo.MongoClient("mongodb://{host}:{port}/".format( 74 | host=self.host, port=self.port)) 75 | # 连接默认数据库 76 | if (not self.database): 77 | self.db = self.client["admin"] 78 | else: 79 | self.db = self.client[self.database] 80 | # 认证 81 | self.db.authenticate(self.user, self.password) 82 | 83 | """ 84 | @description: 关闭数据库连接 85 | ------- 86 | @param: 87 | ------- 88 | @return: 89 | """ 90 | def close(self): 91 | if (self.client): 92 | self.client.close() 93 | self.client = None 94 | 95 | 96 | """ 97 | @description: 单体测试 98 | ------- 99 | @param: 100 | ------- 101 | @return: 102 | """ 103 | if __name__ == "__main__": 104 | r_mongodb_driver = r_mongodb_driver(database="test") 105 | r_mongodb_driver.connect() 106 | print(r_mongodb_driver.client["test"].list_collection_names()) 107 | r_mongodb_driver.close() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rab_python_packages 2 | 3 | ## 项目介绍 4 | > **2022-02-19 更新** 5 | > 该包由于代码质量低和在后续对开发中不断地增开临时参数、变量，导致腐化得过于严重，不得已只能废弃。 6 | > 重构和更新会在新的仓库进行：[senjianlu/rabbase](https://github.com/senjianlu/rabbase.git)，当前仓库代码仅供参考和思路启发，请不要再依赖了。 7 | 8 | 简单可用的 Python3 模块包。 9 | 在大量简单方法整合起来的模块加持下，即使是初学者也能迅速开展对数据的爬取、整理和存储等工作。 10 | 11 | ## 环境 12 | 环境需求： 13 | 14 | | 系统 | 版本 | 15 | | -----| ---- | 16 | | Linux | CentOS7 | 17 | 18 | | 数据库 | 版本 | 19 | | -----| ---- | 20 | | PostgreSQL | 12 | 21 | 22 | | 语言 | 版本 | 23 | | -----| ---- | 24 | | Python | 3.8.2 | 25 | 26 | | 模块 | 版本 | 27 | | -----| ---- | 28 | | cfscrape | 2.1.1 | 29 | | configparser | 5.0.2 | 30 | | charset-normalizer | 2.0.3 | 31 | | docker | 5.0.0 | 32 | | minio | 7.1.0 | 33 | | pika | 1.2.0 | 34 | | psycopg2 | 2.8.6 | 35 | | PyYAML | 6.0 | 36 | | redis | 3.5.3 | 37 | | requests[socks] | 2.10.0 | 38 | | rsa | 4.7.2 | 39 | | selenium | 3.141.0 | 40 | | singleton_decorator | 1.0.0 | 41 | | six | 1.16.0 | 42 | | urllib3 | 1.25.11 | 43 | | websocket | 0.2.1 | 44 | 45 | | 浏览器 | 版本 | 46 | | ----- | ----- | 47 | | Google Chrome | 92.0.4515.107 | 48 | 49 | | 浏览器驱动 | 版本 | 50 | | ----- | ----- | 51 | | ChromeDriver | 92.0.4515.107 | 52 | 53 | **你有以下两种方式来配置环境：** 54 | 1. 使用命令配置本机环境： 55 | ```bash 56 | cd rab_python_packages 57 | # 配置运行环境 58 | python3 rab_env.py 59 | # 针对 rab_chrome 配置 Selenium 驱动 Chrome 所需的环境 60 | python3 rab_env.py rab_chrome 61 | ``` 62 | 2. 如果你习惯在 Docker 镜像内进行开发作业，那么也可以从 Docker Hub 上拉取已经配置好环境的 [rabbir/rab_python_packages](https://hub.docker.com/r/rabbir/rab_python_packages) 镜像并连接： 63 | ```bash 64 | docker pull rabbir/rab_python_packages:latest 65 | # 后台运行容器 66 | docker run -dit rabbir/rab_python_packages:latest 67 | # 连接镜像为 rabbir/rab_python_packages:latest 的第一个容器 68 | docker attach `docker ps -aq -l --filter ancestor=rabbir/rab_python_packages:latest` 69 | ``` 70 | > 你也可以选择在本地构建 Docker 镜像，使用本项目中的 Dockerfile 以保证在构建过程中环境会一并被配置好： 71 | > ```bash 72 | > cd rab_python_packages 73 | > # rabbir/rab_python_packages:latest 替换为你想要的标签 74 | > docker build -t rabbir/rab_python_packages:latest . 75 | > ``` 76 | 77 | ## 使用方法 78 | 仅使用当前版本（维持爬虫的稳定可用）: 79 | ```bash 80 | git clone https://github.com/senjianlu/rab_python_packages.git 81 | ``` 82 | 依赖此仓库（获取更多的封装方法）: 83 | ```bash 84 | git submodule add https://github.com/senjianlu/rab_python_packages.git 85 | ``` 86 | *注：既存方法的方法名和参数不进行修改，实在需要添加参数会赋予初始值，任何改动以不影响现在的代码作为第一前提。* 87 | Dockerfile 构建镜像： 88 | ```yaml 89 | # 基础镜像系统版本为 rabbir/rab_python_packages:latest 90 | FROM rabbir/rab_python_packages:latest 91 | ... 92 | ... 93 | ``` 94 | 95 | ## 整体结构 96 | ![rab_python_packages](https://raw.githubusercontent.com/senjianlu/imgs/master/20211111083715.png) 97 | 98 | ## 特别鸣谢 99 | - 感谢 [JetBrains](https://www.jetbrains.com/) 为本项目提供的 [PyCharm](https://www.jetbrains.com/pycharm/) 开源许可证 100 | - 感谢 [DigitalOcean](https://www.digitalocean.com/) 为本项目提供测试用服务器 101 | - 感谢 [Amazon Web Services](https://aws.amazon.com/) 为本项目提供免费的静态资源存储服务 -------------------------------------------------------------------------------- /rab_imap.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_imap.py 6 | # @DATE: 2021/06/29 Tue 7 | # @TIME: 13:50:40 8 | # 9 | # @DESCRIPTION: 共通包 IMAP 邮件收发相关类和方法封装 10 | 11 | 12 | import email 13 | import imaplib 14 | 15 | 16 | """ 17 | @description: 从邮件 Message 中获取邮件标题 18 | ------- 19 | @param: 20 | ------- 21 | @return: 22 | """ 23 | def get_mail_title(mail_message): 24 | return mail_message.get("subject") 25 | 26 | """ 27 | @description: 从邮件 Message 中获取邮件内容 28 | ------- 29 | @param: 30 | ------- 31 | @return: 32 | """ 33 | def get_mail_content(mail_message): 34 | mail_content = "" 35 | for row in mail_message.walk(): 36 | if (not row.is_multipart()): 37 | mail_content += row.get_payload(decode=True).decode("ISO-8859-1") 38 | return mail_content 39 | 40 | 41 | """ 42 | @description: IMAP 类 43 | ------- 44 | @param: 45 | ------- 46 | @return: 47 | """ 48 | class r_imap(): 49 | 50 | """ 51 | @description: 初始化 52 | ------- 53 | @param: 54 | ------- 55 | @return: 56 | """ 57 | def __init__(self, imap_url, imap_username, imap_password, imap_port="993"): 58 | # 邮件服务提供商 IMAP 地址 59 | self.url = imap_url 60 | # 端口 61 | self.port = imap_port 62 | # 用户名 63 | self.username = imap_username 64 | # 密码 65 | self.password = imap_password 66 | # IMAP client 67 | self.client = self.login() 68 | 69 | """ 70 | @description: IMAP 登录 71 | ------- 72 | @param: 73 | ------- 74 | @return: 75 | """ 76 | def login(self): 77 | client = imaplib.IMAP4_SSL(self.url, self.port) 78 | client.login(self.username, self.password) 79 | return client 80 | 81 | """ 82 | @description: 获取所有邮件 Message 83 | ------- 84 | @param: 85 | ------- 86 | @return: 87 | """ 88 | def get_all_mails(self): 89 | all_mails = [] 90 | status, count = self.client.select("Inbox") 91 | # 默认搜索所有邮件 92 | status, mail_no_list_data = self.client.search(None, "ALL") 93 | mail_no_list = mail_no_list_data[0].split() 94 | # 对每封邮件进行解析和序列化 95 | for mail_no in mail_no_list: 96 | status, mail_data = self.client.fetch(mail_no, "(RFC822)") 97 | mail_message = email.message_from_string( 98 | mail_data[0][1].decode("UTF-8")) 99 | # 将邮件类转换为 JSON 格式数据 100 | mail = {} 101 | mail["title"] = get_mail_title(mail_message) 102 | mail["content"] = get_mail_content(mail_message) 103 | all_mails.append(mail) 104 | return all_mails 105 | 106 | """ 107 | @description: 根据标题关键词获取指定邮件列表 108 | ------- 109 | @param: 110 | ------- 111 | @return: 112 | """ 113 | def serach_title(self, keyword_4_search): 114 | keyword_4_search = keyword_4_search.lower() 115 | mails = [] 116 | all_mails = self.get_all_mails() 117 | # 循环每封邮件判断标题是否包含指定关键词 118 | for mail in all_mails: 119 | if (keyword_4_search in mail["title"].lower()): 120 | mails.append(mail) 121 | return mails 122 | 123 | 124 | """ 125 | @description: 单体测试 126 | ------- 127 | @param: 128 | ------- 129 | @return: 130 | """ 131 | if __name__ == "__main__": 132 | pass -------------------------------------------------------------------------------- /rab_config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_config.py 6 | # @DATE: 2020/12/29 周二 7 | # @TIME: 19:50:03 8 | # 9 | # @DESCRIPTION: 共通包 配置读取模块 10 | 11 | 12 | import os 13 | import sys 14 | import json 15 | import configparser 16 | 17 | 18 | """ 19 | @description: 查找包内文件 20 | ------- 21 | @param: 22 | ------- 23 | @return: 24 | """ 25 | def find_rab_file(rab_file_name): 26 | paths = [ 27 | # 在项目主目录 28 | "../", 29 | "../../", 30 | # 判断该路径下配置文件是否存在 31 | None, 32 | # 不在的话就在共通包中查找 33 | "rab_python_packages/", 34 | # 或者在上级路径的共通包中查找 35 | "../rab_python_packages/", 36 | "../../rab_python_packages/" 37 | ] 38 | for path in paths: 39 | if (path): 40 | if (os.path.exists(path+rab_file_name)): 41 | return path + rab_file_name 42 | else: 43 | if (os.path.exists(rab_file_name)): 44 | return rab_file_name 45 | 46 | """ 47 | @description: 查找包路径 48 | ------- 49 | @param: 50 | ------- 51 | @return: 52 | """ 53 | def find_rab_python_packages_path(): 54 | paths = [ 55 | None, 56 | "../" 57 | ] 58 | for path in paths: 59 | if (path): 60 | if (os.path.exists(path+"rab_python_packages")): 61 | return path 62 | else: 63 | if (os.path.exists("rab_python_packages")): 64 | return path 65 | 66 | """ 67 | @description: 获取所有配置信息 68 | ------- 69 | @param: file_name 70 | ------- 71 | @return: 72 | """ 73 | def get_config(file_name): 74 | file_path = find_rab_file(file_name) 75 | config = configparser.RawConfigParser() 76 | config.read(file_path, encoding="UTF-8") 77 | return config 78 | 79 | """ 80 | @description: 合理化转换 81 | ------- 82 | @param: 83 | ------- 84 | @return: 85 | """ 86 | def parse_if_need(configuration_item): 87 | parsed_configuration_item = configuration_item 88 | # 如果是 list 形式的 89 | if ("[" in configuration_item and "]" in configuration_item): 90 | configuration_item = configuration_item.lstrip("[").rstrip("]") 91 | list_items = configuration_item.split(",") 92 | parsed_configuration_item = [] 93 | for list_item in list_items: 94 | list_item = list_item.lstrip().lstrip('"').rstrip('"') 95 | parsed_configuration_item.append(list_item) 96 | # 如果是 JSON 形式 97 | elif(configuration_item.strip().startswith("{") 98 | and configuration_item.strip().endswith("}")): 99 | parsed_configuration_item = json.loads(configuration_item) 100 | return parsed_configuration_item 101 | 102 | """ 103 | @description: 读取共通包中的配置文件 104 | ------- 105 | @param: 106 | ------- 107 | @return: 108 | """ 109 | def load_package_config(file_name, configuration_class, configuration_item): 110 | package_config = get_config(file_name) 111 | configuration_value = package_config.get( 112 | configuration_class, configuration_item) 113 | # 对部分值进行合理转换，例如：列表 114 | configuration_value = parse_if_need(configuration_value) 115 | return configuration_value 116 | 117 | 118 | """ 119 | @description: 单体测试 120 | ------- 121 | @param: 122 | ------- 123 | @return: 124 | """ 125 | if __name__ == "__main__": 126 | subscription_urls = load_package_config( 127 | "rab_config.ini", "rab_subscription", "access_test_urls") 128 | print(subscription_urls) 129 | pass -------------------------------------------------------------------------------- /rab_ip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_ip.py 6 | # @DATE: 2021/08/31 Tue 7 | # @TIME: 14:52:20 8 | # 9 | # @DESCRIPTION: 代理或本机 IP 获取方法整合模块 10 | 11 | 12 | import sys 13 | import json 14 | import requests 15 | sys.path.append("..") if (".." not in sys.path) else True 16 | from rab_python_packages import rab_config 17 | from rab_python_packages import rab_logging 18 | 19 | 20 | # 日志记录 21 | r_logger = rab_logging.r_logger() 22 | 23 | 24 | """ 25 | @description: 解析 IP 接口返回的数据 26 | ------- 27 | @param: 28 | ------- 29 | @return: 30 | """ 31 | def parse_ip_info(api_url, r_json): 32 | ip_info = { 33 | "ip": None, 34 | "location": None 35 | } 36 | if (api_url == "http://ip-api.com/json/?lang=zh-CN"): 37 | ip_info["ip"] = r_json["query"] if "query" in r_json else None 38 | ip_info["location"] = r_json["country"] if "country" in r_json else None 39 | elif(api_url == "https://www.ip.cn/api/index?ip=&type=0"): 40 | ip_info["ip"] = r_json["ip"] if "ip" in r_json else None 41 | ip_info["location"] = r_json["address"] if "address" in r_json else None 42 | elif(api_url == "https://ip-api.io/json"): 43 | ip_info["ip"] = r_json["ip"] if "ip" in r_json else None 44 | ip_info["location"] = r_json["country_name"] \ 45 | if "country_name" in r_json else None 46 | elif(api_url == "https://api.ipify.org/?format=json"): 47 | ip_info["ip"] = r_json["ip"] if "ip" in r_json else None 48 | ip_info["location"] = None 49 | return ip_info 50 | 51 | """ 52 | @description: 获取 IP 信息 53 | ------- 54 | @param: 55 | ------- 56 | @return: 57 | """ 58 | def get_ip_info(proxies=None, 59 | timeout=int(rab_config.load_package_config( 60 | "rab_config.ini", "rab_requests", "timeout"))): 61 | api_urls = [ 62 | "http://ip-api.com/json/?lang=zh-CN", 63 | "https://www.ip.cn/api/index?ip=&type=0", 64 | "https://ip-api.io/json", 65 | # 只会返回 IP 的接口 66 | "https://api.ipify.org/?format=json" 67 | ] 68 | for api_url in api_urls: 69 | try: 70 | r = requests.get(api_url, proxies=proxies, timeout=timeout) 71 | r_json = json.loads(r.text) 72 | ip_info = parse_ip_info(api_url, r_json) 73 | # 如果访问成功了有 IP 信息了则退出 74 | if (ip_info["ip"] or ip_info["location"]): 75 | return ip_info 76 | except Exception as e: 77 | r_logger.error("使用 {} 获取 IP 信息出错！".format(api_url)) 78 | r_logger.error(e) 79 | return {"ip": None, "location": None} 80 | 81 | """ 82 | @description: 测试访问 83 | ------- 84 | @param: 85 | ------- 86 | @return: 87 | """ 88 | def test(test_url, 89 | proxies=None, 90 | timeout = int(rab_config.load_package_config( 91 | "rab_config.ini", "rab_requests", "timeout")), 92 | success_status_codes=[200], 93 | error_status_codes=[500]): 94 | success_flg = False 95 | try: 96 | r = requests.get(test_url, proxies=proxies, timeout=timeout) 97 | if (r.status_code in success_status_codes): 98 | r_logger.info( 99 | "测试访问地址：{test_url} 成功！".format(test_url=test_url)) 100 | success_flg = True 101 | else: 102 | r_logger.info( 103 | "测试访问地址：{test_url} 不通过！响应代码：{status_code}".format( 104 | test_url=test_url, status_code=str(r.status_code))) 105 | except Exception as e: 106 | r_logger.info("测试访问地址：{test_url} 出错！错误信息：{e}".format( 107 | test_url=test_url, e=str(e))) 108 | r_logger.info("使用的代理：{proxies}".format(proxies=str(proxies))) 109 | return success_flg 110 | 111 | 112 | """ 113 | @description: 单体测试 114 | ------- 115 | @param: 116 | ------- 117 | @return: 118 | """ 119 | if __name__ == "__main__": 120 | print(get_ip_info()) 121 | -------------------------------------------------------------------------------- /rab_websocket.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_websocket.py 6 | # @DATE: 2021/09/20 Mon 7 | # @TIME: 15:33:54 8 | # 9 | # @DESCRIPTION: 共通包 WebSocket 模块 10 | 11 | 12 | import sys 13 | import time 14 | import cfscrape 15 | import websocket 16 | sys.path.append("..") if (".." not in sys.path) else True 17 | from rab_python_packages import rab_logging 18 | 19 | 20 | # 日志记录 21 | r_logger = rab_logging.r_logger() 22 | 23 | 24 | """ 25 | @description: 获取 Cloudflare 的认证信息 26 | ------- 27 | @param: 28 | ------- 29 | @return: 30 | """ 31 | def get_cloudflare_access_infos(url, user_agent): 32 | access_infos = [] 33 | token, user_agent = cfscrape.get_cookie_string(url, user_agent) 34 | cookie_args = token.split(";") 35 | for cookie_arg in cookie_args: 36 | cookie_arg = cookie_arg.rstrip().lstrip() 37 | access_infos.append(cookie_arg.replace("=", ": ")) 38 | access_infos.append("User-Agent: {}".format(user_agent)) 39 | return access_infos 40 | 41 | """ 42 | @description: WebSocket 类 43 | ------- 44 | @param: 45 | ------- 46 | @return: 47 | """ 48 | class r_websocket(): 49 | 50 | """ 51 | @description: 初始化 52 | ------- 53 | @param: 54 | ------- 55 | @return: 56 | """ 57 | def __init__(self, url, headers, enable_trace=False, origin=None): 58 | # WebSocket 地址 59 | self.url = url 60 | # WebSocket 连接请求头 61 | self.headers = headers 62 | # 是否开启日志 63 | self.enable_trace = enable_trace 64 | # 连接时来源 65 | self.origin = origin 66 | # WebSocket 连接对象 67 | self.ws = None 68 | self.build() 69 | # 发送的消息 70 | self.sent_message = {} 71 | # 收到的消息 72 | self.received_message = {} 73 | 74 | """ 75 | @description: 建立连接 76 | ------- 77 | @param: 78 | ------- 79 | @return: 80 | """ 81 | def build(self): 82 | # 日志记录 83 | websocket.enableTrace(self.enable_trace) 84 | self.ws = websocket.WebSocketApp( 85 | self.url, 86 | header=self.headers, 87 | on_open=lambda ws: self.on_open(ws), 88 | on_message=lambda ws, message: self.on_message(ws, message), 89 | on_error=lambda ws, error: self.on_error(ws, error), 90 | on_close=lambda ws, close_status_code, close_msg: self.on_close( 91 | ws, close_status_code, close_msg)) 92 | 93 | """ 94 | @description: 连接 95 | ------- 96 | @param: 97 | ------- 98 | @return: 99 | """ 100 | def connect(self): 101 | self.ws.run_forever(origin=self.origin) 102 | 103 | """ 104 | @description: 发送消息 105 | ------- 106 | @param: 107 | ------- 108 | @return: 109 | """ 110 | def send(self, message): 111 | r_logger.info("WebSocket 发送消息：{}".format(message)) 112 | self.ws.send(message) 113 | self.sent_message[time.time()] = message 114 | 115 | """ 116 | @description: 关闭连接 117 | ------- 118 | @param: 119 | ------- 120 | @return: 121 | """ 122 | def close(self): 123 | self.ws.close() 124 | 125 | """ 126 | @description: 建立连接时 127 | ------- 128 | @param: 129 | ------- 130 | @return: 131 | """ 132 | def on_open(self, ws): 133 | r_logger.info("WebSocket 连接成功！") 134 | 135 | """ 136 | @description: 收到消息时 137 | ------- 138 | @param: 139 | ------- 140 | @return: 141 | """ 142 | def on_message(self, ws, message): 143 | r_logger.info("WebSocket 收到消息：{}".format( 144 | message if len(message) <= 100 else (message[0:100] + "......"))) 145 | self.received_message[time.time()] = message 146 | 147 | """ 148 | @description: 出错时 149 | ------- 150 | @param: 151 | ------- 152 | @return: 153 | """ 154 | def on_error(self, ws, error): 155 | r_logger.error("WebSocket 出错！") 156 | r_logger.error(error) 157 | 158 | """ 159 | @description: 连接关闭时 160 | ------- 161 | @param: 162 | ------- 163 | @return: 164 | """ 165 | def on_close(self, ws, close_status_code, close_msg): 166 | r_logger.info("WebSocket 连接关闭，状态码：{status_code}，消息：{msg}" \ 167 | .format(status_code=str(close_status_code), msg=str(close_msg))) 168 | 169 | 170 | """ 171 | @description: 单体测试 172 | ------- 173 | @param: 174 | ------- 175 | @return: 176 | """ 177 | if __name__ == "__main__": 178 | print(time.time()) 179 | test_user_agent = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/" \ 180 | + "537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36" 181 | print(get_cloudflare_access_infos("https://google.com", test_user_agent)) -------------------------------------------------------------------------------- /rab_client.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_clinet.py 6 | # @DATE: 2021/09/04 周六 7 | # @TIME: 21:46:33 8 | # 9 | # @DESCRIPTION: 客户端（长连接）模块 10 | 11 | 12 | import sys 13 | import requests 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_logging 16 | 17 | 18 | # 日志记录 19 | r_logger = rab_logging.r_logger() 20 | 21 | 22 | """ 23 | @description: 长连接类 24 | ------- 25 | @param: 26 | ------- 27 | @return: 28 | """ 29 | class r_client(): 30 | 31 | """ 32 | @description: 初始化 33 | ------- 34 | @param: 35 | ------- 36 | @return: 37 | """ 38 | def __init__(self, basic_url=None, auth=None, headers=None, proxies=None): 39 | # 连接 40 | self.client = requests.session() 41 | # 基础 API 地址 42 | self.basic_url = basic_url 43 | # 默认参数 44 | self.auth = auth 45 | self.headers = headers 46 | self.proxies = proxies 47 | # 动作 48 | self.action = {} 49 | 50 | """ 51 | @description: 添加动作 52 | ------- 53 | @param: 54 | ------- 55 | @return: 56 | """ 57 | def add_action(self, 58 | name, 59 | method="GET", 60 | route=None, 61 | url=None, 62 | basic_params=None, 63 | basic_data=None): 64 | self.action[name] = { 65 | "method": method, 66 | "route": route, 67 | "url": url, 68 | "basic_params": basic_params, 69 | "basic_data": basic_data 70 | } 71 | 72 | """ 73 | @description: 执行动作 74 | ------- 75 | @param: 76 | ------- 77 | @return: 78 | """ 79 | def do_action(self, name, _format=None, params=None, data=None): 80 | action_response = None 81 | # 如果有这个动作 82 | if (name in self.action.keys()): 83 | # 获取 API 地址 84 | if (self.action[name]["route"]): 85 | url = self.basic_url + self.action[name]["route"] 86 | else: 87 | url = action[name]["url"] 88 | # 如果有需要 _format 的参数 89 | if (_format): 90 | url = url.format(**_format) 91 | # 参数增加 92 | if (self.action[name]["basic_params"]): 93 | if (params): 94 | basic_params = self.action[name]["basic_params"] 95 | basic_params.update(params) 96 | params = basic_params 97 | # 请求数据增加 98 | if (self.action[name]["basic_data"]): 99 | # POST 字典类型 100 | if (data and type(data) == dict): 101 | basic_data = self.action[name]["basic_data"] 102 | basic_data.update(data) 103 | data = basic_data 104 | # 文件数据类型 105 | else: 106 | data = data 107 | # 请求 108 | # GET 109 | if (self.action[name]["method"].upper() == "GET"): 110 | action_response = self.client.get(url, \ 111 | params=params, \ 112 | data=data, \ 113 | auth=self.auth, \ 114 | headers=self.headers, \ 115 | proxies=self.proxies) 116 | # POST 117 | elif(self.action[name]["method"].upper() == "POST"): 118 | action_response = self.client.post(url, \ 119 | params=params, \ 120 | data=data, \ 121 | auth=self.auth, \ 122 | headers=self.headers, \ 123 | proxies=self.proxies) 124 | # PUT 125 | elif(self.action[name]["method"].upper() == "PUT"): 126 | action_response = requests.put(url, \ 127 | params=params, \ 128 | data=data, \ 129 | auth=self.auth, \ 130 | headers=self.headers, \ 131 | proxies=self.proxies) 132 | # 不常用请求 133 | else: 134 | action_response = requests.request( 135 | self.action[name]["method"].upper(), \ 136 | url, \ 137 | params=params, \ 138 | data=data, \ 139 | auth=self.auth, \ 140 | headers=self.headers, \ 141 | proxies=self.proxies) 142 | else: 143 | r_logger.error("{} 动作尚不存在！".format(name)) 144 | return action_response 145 | 146 | 147 | """ 148 | @description: 单体测试 149 | ------- 150 | @param: 151 | ------- 152 | @return: 153 | """ 154 | if __name__ == "__main__": 155 | pass -------------------------------------------------------------------------------- /rab_subscription.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/Github/rab_python_packages/rab_subscription.py 6 | # @DATE: 2021/05/10 Mon 7 | # @TIME: 14:11:24 8 | # 9 | # @DESCRIPTION: 共通包 Linux 系统下解析订阅 10 | 11 | 12 | import sys 13 | import json 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_config 16 | from rab_python_packages import rab_logging 17 | from rab_python_packages import rab_cryptography 18 | from rab_python_packages import rab_requests 19 | 20 | 21 | # 日志记录 22 | r_logger = rab_logging.r_logger() 23 | 24 | 25 | class Subscription(): 26 | """ 27 | 订阅类 28 | """ 29 | 30 | def __init__(self, kind, url): 31 | """ 32 | 初始化 33 | """ 34 | self.kind = kind 35 | self.url = url 36 | # 是否已经请求 37 | self.is_request = 0 38 | # 原始的请求信息 39 | self.response_content = None 40 | # 是否已经解析 41 | self.is_parse = 0 42 | # 节点数量 43 | self.nodes_count = None 44 | # 节点原始信息列表 45 | self.node_origin_infos = [] 46 | # 节点 ID 列表（PostgreSQL 支持列表格式数据的存取） 47 | self.node_ids = [] 48 | # 节点认证信息 49 | self.node_auth_info = None 50 | # 流量使用模式 51 | self.traffic_mode = None 52 | # 流量限制（单位为 mb） 53 | self.traffic_maximum = None 54 | # 流量已使用（单位为 mb） 55 | self.traffic_used = None 56 | # 流量下次重置日期 57 | self.traffic_next_reset_date = None 58 | # 订阅过期日期 59 | self.expiration_date = None 60 | # 订阅提供商名字 61 | self.provider_name = None 62 | # 订单提供商地址 63 | self.provider_url = None 64 | # 订阅提供商登陆用户名 65 | self.provider_auth_username = None 66 | # 订阅提供商登陆用户名 67 | self.provider_auth_password = None 68 | 69 | 70 | """ 71 | @description: 获取订阅的 Base64 解码前的原始信息 72 | ------- 73 | @param: 74 | ------- 75 | @return: 76 | """ 77 | def get_subscription_origin_info(subscription_url): 78 | try: 79 | # 保险访问 80 | headers = { 81 | "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " \ 82 | + "AppleWebKit/535.36 (KHTML, like Gecko) " \ 83 | + "Chrome/92.0.4515.131 Safari/537.36" 84 | } 85 | e_response = rab_requests.ensure_request( 86 | "GET", subscription_url, headers=headers) 87 | if (e_response): 88 | return e_response.text 89 | else: 90 | r_logger.warn("{} 尝试所有自建代理仍无法获取订阅原始信息！".format( 91 | subscription_url)) 92 | except Exception as e: 93 | r_logger.error("{} 获取订阅原始信息出错！".format( 94 | subscription_url)) 95 | r_logger.error(e) 96 | return None 97 | 98 | """ 99 | @description: 获取订阅的原始信息经过 Base64 解码后的各节点链接 100 | ------- 101 | @param: 102 | ------- 103 | @return: 104 | """ 105 | def get_node_urls(subscription_origin_info): 106 | node_urls = [] 107 | # 尝试对原始信息进行解码 108 | try: 109 | subscription_info = rab_cryptography.b64decode( 110 | subscription_origin_info).decode("UTF-8") 111 | # 分割节点链接并放入结果列表 112 | if (subscription_info): 113 | for row in subscription_info.split("\n"): 114 | if (row.strip()): 115 | node_urls.append(row.strip()) 116 | except Exception as e: 117 | r_logger.error("{} 订阅原始信息 Base64 解码失败！".format( 118 | str(subscription_origin_info))) 119 | r_logger.error(e) 120 | return node_urls 121 | 122 | 123 | """ 124 | @description: 订阅类 125 | ------- 126 | @param: 127 | ------- 128 | @return: 129 | """ 130 | class r_subscription(): 131 | 132 | """ 133 | @description: 初始化 134 | ------- 135 | @param: 136 | ------- 137 | @return: 138 | """ 139 | def __init__(self, subscription_url): 140 | # 订阅地址 141 | self.url = subscription_url 142 | # 订阅原始信息 143 | self.origin_info = get_subscription_origin_info(subscription_url) 144 | # 节点链接 145 | self.node_urls = get_node_urls(self.origin_info) 146 | 147 | 148 | """ 149 | @description: 单体测试 150 | ------- 151 | @param: 152 | ------- 153 | @return: 154 | """ 155 | if __name__ == "__main__": 156 | 157 | from rab_python_packages import rab_postgresql 158 | 159 | r_pgsql_driver=rab_postgresql.r_pgsql_driver(show_column_name=True) 160 | try: 161 | # 获取订阅并解析节点链接 162 | select_sql = """ 163 | SELECT 164 | * 165 | FROM 166 | sa_proxy_subscription 167 | where 168 | is_deleted = 0 169 | """ 170 | select_result = r_pgsql_driver.select(select_sql) 171 | for row in select_result: 172 | with open("node_urls", "a") as f: 173 | for node_url in r_subscription(row["sps_url"]).node_urls: 174 | f.write("{}\n".format(node_url)) 175 | except Exception as e: 176 | r_logger.error("rab_subscription 模块单体测试出错！") 177 | r_logger.error(e) 178 | finally: 179 | r_pgsql_driver.close() -------------------------------------------------------------------------------- /rab_cryptography.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_cryptography.py 6 | # @DATE: 2021/07/24 周六 7 | # @TIME: 13:45:47 8 | # 9 | # @DESCRIPTION: 共同包 简易密码模块 10 | 11 | 12 | import sys 13 | import base64 14 | import datetime 15 | sys.path.append("..") if (".." not in sys.path) else True 16 | from rab_python_packages import rab_config 17 | from rab_python_packages import rab_logging 18 | 19 | 20 | # 日志记录 21 | r_logger = rab_logging.r_logger() 22 | 23 | 24 | """ 25 | @description: 用以订阅信息 Base64 解码的方法 26 | ------- 27 | @param: 28 | ------- 29 | @return: 30 | """ 31 | def b64decode(str_4_b64decode): 32 | str_4_b64decode = str_4_b64decode.replace("_", "/") 33 | str_4_b64decode = str_4_b64decode.replace("-", "+") 34 | # 带 - 需要用专用的 URL Base64 解码 35 | if ("-" in str_4_b64decode): 36 | # 最多加 4 次 "=" 以使原始信息符合 Base64 格式 37 | for i in range(0, 4): 38 | try: 39 | str_after_b64decode = base64.urlsafe_b64decode(str_4_b64decode) 40 | return str_after_b64decode 41 | except Exception: 42 | str_4_b64decode = str_4_b64decode + "=" 43 | # 不包含 - 则试用普通的 Base64 解码即可 44 | else: 45 | # 最多加 4 次 "=" 以使原始信息符合 Base64 格式 46 | for i in range(0, 4): 47 | try: 48 | str_after_b64decode = base64.b64decode(str_4_b64decode) 49 | return str_after_b64decode 50 | except Exception: 51 | str_4_b64decode = str_4_b64decode + "=" 52 | # 解码失败 53 | r_logger.error("Base64 解码失败！待解码内容：{}".format(str_4_b64decode)) 54 | return None 55 | 56 | """ 57 | @description: 获得当天的日期 YYMMDD 格式作为默认加密密钥 58 | ------- 59 | @param: 60 | ------- 61 | @return: 62 | """ 63 | def get_default_key(): 64 | today = datetime.date.today() 65 | return int(today.strftime('%Y%m%d')) 66 | 67 | """ 68 | @description: 将字符串转为数字 69 | ------- 70 | @param: 71 | ------- 72 | @return: 73 | """ 74 | def str_2_int(str_4_2_int): 75 | bytes_4_2_int = str_4_2_int.encode("UTF-8") 76 | return int.from_bytes(bytes_4_2_int, "little") 77 | 78 | """ 79 | @description: 将数字转回字符串 80 | ------- 81 | @param: 82 | ------- 83 | @return: 84 | """ 85 | def int_2_str(int_4_2_str): 86 | bytes_4_2_str = int_4_2_str.to_bytes((int_4_2_str.bit_length()+7)//8, "little") 87 | return bytes_4_2_str.decode("UTF-8") 88 | 89 | """ 90 | @description: 根据加密 KEY 加密字符串 91 | ------- 92 | @param: 93 | ------- 94 | @return: 95 | """ 96 | def encrypt(str_4_encrypt, encrypt_key: int = get_default_key()): 97 | try: 98 | encrypted_str = "" 99 | is_str = False 100 | # 数字可直接加密 101 | if (str(str_4_encrypt).isdigit()): 102 | int_4_encrypt = int(str_4_encrypt) 103 | else: 104 | # 字符串需要在最后加密后的字符上特殊标记 105 | is_str = True 106 | int_4_encrypt = str_2_int(str_4_encrypt) 107 | chars = rab_config.load_package_config( 108 | "rab_config.ini", "rab_cryptography", "chars") 109 | # 数字加密方法 110 | if (not is_str): 111 | # 乘以加密密钥 112 | encrypted_int = int_4_encrypt*encrypt_key 113 | # 位数 114 | encrypted_str = str(len(str(encrypted_int))) + "A" 115 | # 取除数和余数 116 | quotient = pow(10, len(str(encrypted_int))+3) // encrypted_int 117 | remainder = pow(10, len(str(encrypted_int))+3) % encrypted_int 118 | for quotient_char in str(quotient): 119 | encrypted_str += chars[int(quotient_char)] 120 | encrypted_str += "A" 121 | encrypted_str += str(remainder) 122 | # 字符串加密方法 123 | else: 124 | quotient = int_4_encrypt // encrypt_key 125 | remainder = int_4_encrypt % encrypt_key 126 | encrypted_str = str(int_4_encrypt) + "A" \ 127 | + str(quotient)[0] + "A" \ 128 | + str(remainder)[0] + "A" \ 129 | + "1" 130 | # 加密后的字符串长度向 10 凑整 131 | for _ in range(0, 10): 132 | if (len(encrypted_str)%10 == 0): 133 | break 134 | else: 135 | encrypted_str += "I" 136 | return True, encrypted_str, encrypt_key 137 | except Exception as e: 138 | print("字符串：" + str(str_4_encrypt) + " 加密失败！" + str(e)) 139 | return False, None, encrypt_key 140 | 141 | """ 142 | @description: 解密 143 | ------- 144 | @param: 145 | ------- 146 | @return: 147 | """ 148 | def decrypt(str_4_decrypt, decrypt_key: int = get_default_key()): 149 | # 判断是否为用此密码模块加密过的字符 150 | if (len(str_4_decrypt)%10 != 0): 151 | print("字符串：" + str(str_4_decrypt) + " 不符合解密字符串要求！") 152 | return False, None, decrypt_key 153 | try: 154 | decrypted_str = "" 155 | is_str = False 156 | # 获取各个参数 157 | str_4_decrypt = str_4_decrypt.replace("I", "") 158 | encrypted_params = str_4_decrypt.split("A") 159 | # 参数有 4 个的话说明加密前为字符串类型 160 | if (len(encrypted_params) == 4): 161 | is_str = True 162 | # 如果加密前字符串是纯数字 163 | if (not is_str): 164 | chars = rab_config.load_package_config( 165 | "rab_config.ini", "rab_cryptography", "chars") 166 | # 取位数 167 | pow_num = int(encrypted_params[0]) + 3 168 | # 除余数 169 | remainder = int(encrypted_params[2]) 170 | # 除整数 171 | quotient = "" 172 | encrypted_quotient = encrypted_params[1] 173 | for encrypted_quotient_char in encrypted_quotient: 174 | quotient += str(chars.find(encrypted_quotient_char)) 175 | decrypted_int = int( 176 | (pow(10, pow_num)-remainder)/int(quotient)/decrypt_key) 177 | return True, str(decrypted_int), decrypt_key 178 | else: 179 | decrypted_str = int_2_str(int(encrypted_params[0])) 180 | return True, decrypted_str, decrypt_key 181 | except Exception as e: 182 | print("字符串：" + str(str_4_decrypt) + " 解密失败！" + str(e)) 183 | return False, None, decrypt_key 184 | 185 | 186 | """ 187 | @description: 单体测试 188 | ------- 189 | @param: 190 | ------- 191 | @return: 192 | """ 193 | if __name__ == "__main__": 194 | string = "190235" 195 | print("加密前字符串：" + string) 196 | # print(str_2_int(string)) 197 | encrypt_success_flg, encrypted_str, encrypt_key = encrypt(string) 198 | print("加密后字符串：" + encrypted_str) 199 | decrypt_success_flg, decrypted_str, decrypt_key = decrypt(encrypted_str) 200 | print("解密后字符串：" + decrypted_str) -------------------------------------------------------------------------------- /rab_requests.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_requests.py 6 | # @DATE: 2021/07/12 Mon 7 | # @TIME: 14:40:41 8 | # 9 | # @DESCRIPTION: 基于 Requests 的爬虫用优化版本 10 | 11 | 12 | import sys 13 | import json 14 | import urllib 15 | import requests 16 | sys.path.append("..") if (".." not in sys.path) else True 17 | from rab_python_packages import rab_config 18 | from rab_python_packages import rab_logging 19 | from rab_python_packages import rab_proxy 20 | 21 | 22 | # 日志记录 23 | r_logger = rab_logging.r_logger() 24 | 25 | 26 | """ 27 | @description: 根据请求地址和参数拼凑出实际链接 28 | ------- 29 | @param: 30 | ------- 31 | @return: 32 | """ 33 | def get_true_url(url, params): 34 | url_parts = list(urllib.parse.urlparse(url)) 35 | url_parts[4] = urllib.parse.urlencode(params) 36 | return urllib.parse.urlunparse(url_parts) 37 | 38 | """ 39 | @description: 不使用代理无法访问的情况下，尝试所有自建代理进行保险访问 40 | ------- 41 | @param: 42 | ------- 43 | @return: 44 | """ 45 | def ensure_request(method, 46 | url, 47 | params=None, 48 | data=None, 49 | headers=None, 50 | timeout = int(rab_config.load_package_config( 51 | "rab_config.ini", "rab_requests", "timeout")), 52 | success_status_codes=[200], 53 | error_status_codes=[500]): 54 | # 不适用代理的情况下访问 55 | try: 56 | r = requests.request(method.upper(), url, params=params, \ 57 | data=data, headers=headers, timeout=timeout) 58 | if (r.status_code in success_status_codes): 59 | return r 60 | except Exception as e: 61 | r_logger.info("无法在不使用代理的情况下访问：{url}！错误信息：{e}".format( 62 | url=url, e=str(e))) 63 | r_logger.info("不使用代理无法访问的情况下，开始尝试使用代理访问：{}".format(url)) 64 | # 使用自建 SOCKS5 代理进行访问 65 | personal_proxy_infos = rab_proxy.get_personal_proxy_infos() 66 | for proxy_out_ip in personal_proxy_infos["socks5"]: 67 | proxies = rab_proxy.parse_proxy_info( 68 | "socks5", personal_proxy_infos["socks5"][proxy_out_ip]) 69 | try: 70 | r = requests.request(method.upper(), url, params=params, \ 71 | data=data, headers=headers, proxies=proxies, timeout=timeout) 72 | if (r.status_code in success_status_codes): 73 | return r 74 | except Exception as e: 75 | r_logger.info("在使用代理的情况下也无法访问：{url}！错误信息：{e}" \ 76 | .format(url=url, e=str(e))) 77 | r_logger.info("使用的代理：{proxies}".format(proxies=str(proxies))) 78 | r_logger.error( 79 | "所有自建代理均无法访问：{url} 请检查地址或自建代理！".format(url=url)) 80 | return None 81 | 82 | 83 | """ 84 | @description: r_requests 类 85 | ------- 86 | @param: 87 | ------- 88 | @return: 89 | """ 90 | class r_requests(): 91 | 92 | """ 93 | @description: 初始化 94 | ------- 95 | @param: 96 | ------- 97 | @return: 98 | """ 99 | def __init__(self, r_proxy): 100 | # 代理类 101 | self.r_proxy = r_proxy 102 | # 超时时间 103 | self.timeout = int(rab_config.load_package_config( 104 | "rab_config.ini", "rab_requests", "timeout")) 105 | # 最大失败尝试次数 106 | self.max_retry_num = int(rab_config.load_package_config( 107 | "rab_config.ini", "rab_requests", "max_retry_num")) 108 | 109 | """ 110 | @description: Get 方法 111 | ------- 112 | @param: 113 | ------- 114 | @return: 115 | """ 116 | def get(self, 117 | web, 118 | url, 119 | params=None, 120 | headers=None, 121 | proxies=None, 122 | timeout=None): 123 | # 最大尝试次数 124 | for try_no in range(1, self.max_retry_num+1): 125 | # 判断是否传入了代理 126 | if (proxies): 127 | _proxies = proxies 128 | else: 129 | _proxies = self.r_proxy.get(web=web) 130 | # 判断是否传入了超时时间 131 | if (timeout): 132 | pass 133 | else: 134 | timeout = self.timeout 135 | try: 136 | r_r = requests.get(url, params=params, headers=headers, \ 137 | proxies=_proxies, timeout=timeout) 138 | return r_r 139 | except Exception as e: 140 | r_logger.info("第 {try_no} 次访问出错！{e}".format( 141 | try_no=str(try_no), e=str(e))) 142 | r_logger.info("使用的代理：{}".format(str(_proxies))) 143 | continue 144 | # 尝试了最大次数后仍然失败 145 | r_logger.warn("共 {} 次访问出错，达到上限访问结束！".format( 146 | str(self.max_retry_num))) 147 | return None 148 | 149 | """ 150 | @description: Post 方法 151 | ------- 152 | @param: 153 | ------- 154 | @return: 155 | """ 156 | def post(self, 157 | web, 158 | url, 159 | params=None, 160 | headers=None, 161 | data=None, 162 | proxies=None, 163 | timeout=None): 164 | # 最大尝试次数 165 | for try_no in range(1, self.max_retry_num+1): 166 | # 判断是否传入了代理 167 | if (proxies): 168 | _proxies = proxies 169 | else: 170 | _proxies = self.r_proxy.get(web=web) 171 | # 判断是否传入了超时时间 172 | if (timeout): 173 | pass 174 | else: 175 | timeout = self.timeout 176 | try: 177 | r_r = requests.post(url, params=params, headers=headers, \ 178 | data=data, proxies=_proxies, timeout=timeout) 179 | return r_r 180 | except Exception as e: 181 | r_logger.info("第 {try_no} 次 Post 出错！{e}".format( 182 | try_no=str(try_no), e=str(e))) 183 | r_logger.info("使用的代理：{}".format(str(_proxies))) 184 | continue 185 | # 尝试了最大次数后仍然失败 186 | r_logger.warn("共 {} 次 Post 出错，达到上限访问结束！".format( 187 | str(self.max_retry_num))) 188 | return None 189 | 190 | 191 | """ 192 | @description: 单体测试 193 | ------- 194 | @param: 195 | ------- 196 | @return: 197 | """ 198 | if __name__ == "__main__": 199 | # personal_proxy_infos = rab_proxy.get_personal_proxy_infos() 200 | # print(personal_proxy_infos) 201 | # for proxy_out_ip in personal_proxy_infos["socks5"]: 202 | # print(proxy_out_ip) 203 | # print(rab_proxy.parse_proxy_info( 204 | # "socks5", personal_proxy_infos["socks5"][proxy_out_ip])) 205 | # pass 206 | print(get_true_url("http://baidu.com", {"wd": "中文"})) -------------------------------------------------------------------------------- /rab_distributed_system.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_distributed_node.py 6 | # @DATE: 2021/07/25 Sun 7 | # @TIME: 14:54:27 8 | # 9 | # @DESCRIPTION: 分布式系统管理模块 10 | 11 | 12 | import sys 13 | from datetime import datetime, timezone, timedelta 14 | sys.path.append("..") if (".." not in sys.path) else True 15 | from rab_python_packages import rab_config 16 | from rab_python_packages import rab_logging 17 | 18 | 19 | # 日志记录 20 | r_logger = rab_logging.r_logger() 21 | 22 | 23 | """ 24 | @description: 设置当前机器的节点 ID 25 | ------- 26 | @param: 27 | ------- 28 | @return: 29 | """ 30 | def set_node_id(node_id): 31 | package_config = rab_config.get_config("rab_config.ini") 32 | package_config.set("rab_distributed_system", "node_id", node_id) 33 | package_config.write(open(rab_config.find_rab_file("rab_config.ini"), "w")) 34 | return node_id 35 | 36 | """ 37 | @description: 获取当前机器的节点 ID 38 | ------- 39 | @param: 40 | ------- 41 | @return: 42 | """ 43 | def get_node_id(): 44 | node_id = rab_config.load_package_config( 45 | "rab_config.ini", "rab_distributed_system", "node_id") 46 | return node_id 47 | 48 | """ 49 | @description: 获取此节点延迟 50 | ------- 51 | @param: 52 | ------- 53 | @return: 54 | """ 55 | def get_node_delay_time(): 56 | # 获取节点所需要等待的单位时间 57 | node_delay_time = rab_config.load_package_config( 58 | "rab_config.ini", "rab_distributed_system", "node_delay_time") 59 | # 获取节点序号，除开 main 之外均需等待 60 | node_no = get_node_id().split("_")[-1] 61 | if (node_no.lower() == "main"): 62 | return 0 63 | elif(node_no.isdigit()): 64 | return int(node_delay_time)*int(node_no) 65 | else: 66 | return 0 67 | 68 | 69 | """ 70 | @description: 系统状态类 71 | ------- 72 | @param: 73 | ------- 74 | @return: 75 | """ 76 | class r_status(): 77 | 78 | """ 79 | @description: 初始化 80 | ------- 81 | @param: 82 | ------- 83 | @return: 84 | """ 85 | def __init__(self, 86 | project, 87 | subproject, 88 | module=None, 89 | function=None, 90 | status=None, 91 | start_time=None, 92 | over_time=None, 93 | result=None): 94 | self.project = project 95 | self.subproject = subproject 96 | self.module = module 97 | self.function = function 98 | self.status = status 99 | self.start_time = start_time 100 | self.over_time = over_time 101 | self.result = result 102 | 103 | """ 104 | @description: 更新 105 | ------- 106 | @param: 107 | ------- 108 | @return: 109 | """ 110 | def update(self, r_pgsql_driver): 111 | update_sql = """ 112 | INSERT INTO 113 | sa_status( 114 | ss_project, 115 | ss_subproject, 116 | ss_method, 117 | ss_function, 118 | ss_status, 119 | ss_start_time, 120 | ss_over_time, 121 | ss_result, 122 | ss_history 123 | ) VALUES( 124 | %s, %s, %s, %s, %s, 125 | %s, %s, %s, '{}' 126 | ) ON CONFLICT(ss_project, ss_subproject, ss_method, ss_function) 127 | DO UPDATE SET 128 | ss_status = %s, 129 | ss_start_time = %s, 130 | ss_over_time = %s, 131 | ss_result = %s 132 | """ 133 | data = [ 134 | self.project, 135 | self.subproject, 136 | self.module, 137 | self.function, 138 | self.status, 139 | self.start_time, 140 | self.over_time, 141 | self.result, 142 | self.status, 143 | self.start_time, 144 | self.over_time, 145 | self.result 146 | ] 147 | r_pgsql_driver.update(update_sql, data) 148 | 149 | """ 150 | @description: 保存运行结果 151 | ------- 152 | @param: 153 | ------- 154 | @return: 155 | """ 156 | def save(self, r_pgsql_driver): 157 | update_sql = """ 158 | UPDATE 159 | sa_status 160 | SET 161 | ss_history = ss_history || JSONB_BUILD_OBJECT( 162 | TO_CHAR(ss_start_time AT TIME ZONE 'Asia/Shanghai', 163 | 'YYYY-MM-DD HH24:MI:SS'), 164 | ss_result) 165 | WHERE 166 | ss_project = %s 167 | AND ss_subproject = %s 168 | AND ss_method = %s 169 | AND ss_function = %s 170 | """ 171 | data = [self.project, self.subproject, self.module, self.function] 172 | r_pgsql_driver.update(update_sql, data) 173 | 174 | """ 175 | @description: 方法开始 176 | ------- 177 | @param: 178 | ------- 179 | @return: 180 | """ 181 | def start(self, r_pgsql_driver, module=None, function=None): 182 | if (module): 183 | self.module = module 184 | if (function): 185 | self.function = function 186 | self.status = "running" 187 | self.start_time = datetime.now(timezone.utc) 188 | self.over_time = None 189 | self.result = None 190 | self.update(r_pgsql_driver) 191 | r_logger.info("方法 {} 开始！".format(self.function)) 192 | 193 | """ 194 | @description: 方法结束 195 | ------- 196 | @param: 197 | ------- 198 | @return: 199 | """ 200 | def over(self, result, r_pgsql_driver, module=None, function=None): 201 | if (module): 202 | self.module = module 203 | if (function): 204 | self.function = function 205 | self.status = "over" 206 | # self.start_time = None 207 | self.over_time = datetime.now(timezone.utc) 208 | self.result = result 209 | self.update(r_pgsql_driver) 210 | self.save(r_pgsql_driver) 211 | r_logger.info("方法 {} 结束！".format(self.function)) 212 | 213 | """ 214 | @description: 方法出错 215 | ------- 216 | @param: 217 | ------- 218 | @return: 219 | """ 220 | def error(self, result, r_pgsql_driver, module=None, function=None): 221 | if (module): 222 | self.module = module 223 | if (function): 224 | self.function = function 225 | self.status = "error" 226 | # self.start_time = None 227 | self.over_time = datetime.now(timezone.utc) 228 | self.result = result 229 | self.update(r_pgsql_driver) 230 | r_logger.error("方法 {} 出错！".format(self.function)) 231 | 232 | 233 | """ 234 | @description: 单体测试 235 | ------- 236 | @param: 237 | ------- 238 | @return: 239 | """ 240 | if __name__ == "__main__": 241 | # import time 242 | # from rab_python_packages import rab_postgresql 243 | # r_pgsql_driver = rab_postgresql.r_pgsql_driver() 244 | # try: 245 | # r_status = r_status("test_project", "test_subproject", "test_module") 246 | # r_status.start(r_pgsql_driver, function="test_function") 247 | # time.sleep(10) 248 | # r_status.over("test_result", r_pgsql_driver, function="test_function") 249 | # except Exception as e: 250 | # r_logger.error("单体测试出错！") 251 | # r_logger.error(e) 252 | # finally: 253 | # r_pgsql_driver.close() 254 | # 测试修改节点 ID 255 | set_node_id("test_node_id") -------------------------------------------------------------------------------- /rab_logging.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_logging.py 6 | # @DATE: 2020/12/16 Wed 7 | # @TIME: 16:50:21 8 | # 9 | # @DESCRIPTION: 共通包 日志记录模块 10 | 11 | 12 | import os 13 | import sys 14 | import logging 15 | from datetime import datetime 16 | from singleton_decorator import singleton 17 | sys.path.append("..") if (".." not in sys.path) else True 18 | from rab_python_packages import rab_config 19 | 20 | 21 | """ 22 | @description: logger 初始化配置 23 | ------- 24 | @param: 25 | ------- 26 | @return: 27 | """ 28 | def build_logger(): 29 | # 默认打印和记录日志等级为：info 30 | logging.basicConfig(filemode="a", 31 | format="%(asctime)s %(name)s:%(levelname)s:%(message)s", 32 | datefmt="%Y/%m/%d %H:%M:%S", 33 | level=logging.INFO) 34 | logger = logging.getLogger("RAB_LOGGER") 35 | # 在 rab_python_packages 同级目录下创建 rab_logs 日志存放用文件夹 36 | path = rab_config.find_rab_python_packages_path() 37 | # 判断是否存在现有的 rab_logs 日志存放用文件夹 38 | if (path): 39 | rab_logs_path = path + "rab_logs" 40 | else: 41 | rab_logs_path = "rab_logs" 42 | if (not os.path.exists(rab_logs_path)): 43 | os.makedirs(rab_logs_path) 44 | # 以天为单位存储日志 45 | fh = logging.FileHandler( 46 | (rab_logs_path+"/{:%Y%m%d}.log").format(datetime.now())) 47 | # 日志存储格式 48 | formatter = logging.Formatter("%(asctime)s | %(levelname)-8s | %(message)s") 49 | fh.setFormatter(formatter) 50 | logger.addHandler(fh) 51 | return logger 52 | 53 | """ 54 | @description: 获取 logging 的等级对象 55 | ------- 56 | @param: 57 | ------- 58 | @return: 59 | """ 60 | def get_logging_level(level): 61 | if (level.lower() == "debug"): 62 | return logging.DEBUG 63 | elif(level.lower() == "info"): 64 | return logging.INFO 65 | elif(level.lower() == "warn"): 66 | return logging.WARN 67 | elif(level.lower() == "error"): 68 | return logging.ERROR 69 | elif(level.lower() == "critical"): 70 | return logging.CRITICAL 71 | 72 | """ 73 | @description: log 等级比较 74 | ------- 75 | @param: 76 | ------- 77 | @return: 78 | """ 79 | def get_weight(level): 80 | if (level): 81 | level_weight = { 82 | "debug": 1, 83 | "info": 2, 84 | "warn": 3, 85 | "error": 4, 86 | "critical": 5 87 | } 88 | return level_weight[level] 89 | else: 90 | return 0 91 | 92 | 93 | """ 94 | @description: 用以定位实际调用日志记录方法位置的装饰器 95 | ------- 96 | @param: 97 | ------- 98 | @return: 99 | """ 100 | def findcaller(func): 101 | def wrapper(*args): 102 | # 获取调用该函数的文件名、函数名及行号 103 | filename = sys._getframe(1).f_code.co_filename 104 | funcname = sys._getframe(1).f_code.co_name 105 | lineno = sys._getframe(1).f_lineno 106 | # 将原本的入参转变为列表，再把调用者的信息添加到入参列表中 107 | args = list(args) 108 | args.append( 109 | f" {os.path.basename(filename)} | {funcname} | 行：{lineno} | ") 110 | func(*args) 111 | return wrapper 112 | 113 | 114 | """ 115 | @description: r_logger 类 116 | ------- 117 | @param: 118 | ------- 119 | @return: 120 | """ 121 | # 单例模式 122 | @singleton 123 | class r_logger(): 124 | 125 | """ 126 | @description: 初始化 127 | ------- 128 | @param: 129 | ------- 130 | @return: 131 | """ 132 | def __init__(self, 133 | level="info", 134 | logger=build_logger(), 135 | telegarm_notice_level=rab_config.load_package_config( 136 | "rab_config.ini", "rab_logging", "level").lower(), 137 | qq_notice_level=None): 138 | # logger 对象 139 | self.logger = logger 140 | # 修改 logger 对象的日志记录等级 141 | self.logger.setLevel(get_logging_level(level)) 142 | # Telegram 机器人通知等级（大于等于这个等级的都进行通知，None 为不通知） 143 | self.telegarm_notice_level = telegarm_notice_level 144 | # QQ 机器人通知等级 145 | self.qq_notice_level = qq_notice_level 146 | 147 | """ 148 | @description: 机器人通知 149 | ------- 150 | @param: 151 | ------- 152 | @return: 153 | """ 154 | def send_log(self, log_level, log): 155 | # 加上日志等级 156 | log = "| {log_level} | {log}".format( 157 | log_level=log_level.upper().ljust(8, " "), log=log) 158 | # Telegram 机器人通知 159 | if (get_weight(log_level) >= get_weight(self.telegarm_notice_level)): 160 | pass 161 | # QQ 机器人通知 162 | if (get_weight(log_level) >= get_weight(self.qq_notice_level)): 163 | pass 164 | 165 | """ 166 | @description: debug 日志等级方法重写 167 | ------- 168 | @param: 169 | ------- 170 | @return: 171 | """ 172 | @findcaller 173 | def debug(self, log, caller=None): 174 | log = caller + log 175 | self.send_log("debug", log) 176 | self.logger.debug(log) 177 | 178 | """ 179 | @description: info 日志等级方法重写 180 | ------- 181 | @param: 182 | ------- 183 | @return: 184 | """ 185 | @findcaller 186 | def info(self, log, caller=None): 187 | log = caller + log 188 | self.send_log("info", log) 189 | self.logger.info(log) 190 | 191 | """ 192 | @description: warn 日志等级方法重写 193 | ------- 194 | @param: 195 | ------- 196 | @return: 197 | """ 198 | @findcaller 199 | def warn(self, log, caller=None): 200 | log = caller + log 201 | self.send_log("warn", log) 202 | self.logger.warning(log) 203 | 204 | """ 205 | @description: error 日志等级方法重写 206 | ------- 207 | @param: 208 | ------- 209 | @return: 210 | """ 211 | @findcaller 212 | def error(self, log, caller=None): 213 | # 单纯的消息直接记录 214 | if (type(log) == str): 215 | pass 216 | # 如果是错误类 217 | else: 218 | error = log 219 | # 获取错误行号 220 | error_no = str(error.__traceback__.tb_lineno) 221 | # 获取出错的文件 222 | error_file = str(error.__traceback__.tb_frame.f_globals["__file__"]) 223 | # 错误信息 224 | error_msg = str(error).strip() 225 | # 具体日志 226 | log = "出错！文件 {error_file} 第 {error_no} 行，报错：{error_msg}" \ 227 | .format(error_file=error_file, error_no=error_no, \ 228 | error_msg=error_msg) 229 | log = caller + log 230 | self.send_log("error", log) 231 | self.logger.error(log) 232 | 233 | """ 234 | @description: critical 日志等级方法重写 235 | ------- 236 | @param: 237 | ------- 238 | @return: 239 | """ 240 | @findcaller 241 | def critical(self, log, caller=None): 242 | log = caller + log 243 | self.send_log("critical", log) 244 | self.logger.critical(log) 245 | 246 | 247 | """ 248 | @description: 单体测试 249 | ------- 250 | @param: 251 | ------- 252 | @return: 253 | """ 254 | if __name__ == "__main__": 255 | r_logger = r_logger("warn") 256 | r_logger.debug("debug") 257 | r_logger.info("info") 258 | r_logger.warn("warn") 259 | r_logger.error("error") 260 | try: 261 | a = 123 262 | b = 456 263 | import aaa 264 | except Exception as e: 265 | r_logger.error(e) 266 | r_logger.critical("critical") -------------------------------------------------------------------------------- /rab_advance.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_advance.py 6 | # @DATE: 2021/07/12 Mon 7 | # @TIME: 14:51:13 8 | # 9 | # @DESCRIPTION: 自用进阶方法 10 | 11 | 12 | import sys 13 | import json 14 | import time 15 | import base64 16 | import requests 17 | sys.path.append("..") if (".." not in sys.path) else True 18 | from rab_python_packages import rab_config 19 | 20 | 21 | """ 22 | @description: 基于 DirectAdmin 面板的自建邮局类 23 | 文档：https://www.directadmin.com/api.html#showallusers 24 | ------- 25 | @param: 26 | ------- 27 | @return: 28 | """ 29 | class r_a_post_office(): 30 | 31 | """ 32 | @description: 初始化 33 | ------- 34 | @param: 35 | ------- 36 | @return: 37 | """ 38 | def __init__(self, 39 | username=rab_config.load_package_config( 40 | "rab_config.ini", "rab_advance", "rapo_username"), 41 | password=rab_config.load_package_config( 42 | "rab_config.ini", "rab_advance", "rapo_password"), 43 | host=rab_config.load_package_config( 44 | "rab_config.ini", "rab_advance", "rapo_host"), 45 | port=rab_config.load_package_config( 46 | "rab_config.ini", "rab_advance", "rapo_port"), 47 | domain=rab_config.load_package_config( 48 | "rab_config.ini", "rab_advance", "rapo_domain")): 49 | # 初始化 50 | self._username = username 51 | self._password = password 52 | self._host = host 53 | self._port = port 54 | self._domain = domain 55 | 56 | """ 57 | @description: 执行命令 58 | ------- 59 | @param: command, params 60 | ------- 61 | @return: response 62 | """ 63 | def execute_command(self, command, params=None): 64 | # 拼接地址 65 | url = self._host + ":" + str(self._port) + "/" + command 66 | try: 67 | response = requests.post( 68 | url, auth=(self._username, self._password), data=params) 69 | return response 70 | except Exception as e: 71 | print(e) 72 | return None 73 | 74 | """ 75 | @description: 获取当前所有的邮箱 76 | ------- 77 | @param: 78 | ------- 79 | @return: mailboxes 80 | """ 81 | def get_all_mailboxes(self): 82 | # 参数 83 | params = { 84 | "action": "list", 85 | "domain": self._domain 86 | } 87 | # 请求 88 | response = self.execute_command("CMD_API_POP", params) 89 | # 分割和凭借邮箱地址 90 | mailboxes = [] 91 | for prefix_str in response.text.split("&"): 92 | prefix_str = prefix_str.replace("list[]=", "") 93 | mailbox = prefix_str + "@" + self._domain 94 | mailboxes.append(mailbox) 95 | return mailboxes 96 | 97 | """ 98 | @description: 新建邮箱 99 | ------- 100 | @param: mailbox_prefix 101 | ------- 102 | @return: 103 | """ 104 | def create_mailbox(self, mailbox_prefix, mailbox_password): 105 | # 参数 106 | params = { 107 | "action": "create", 108 | "domain": self._domain, 109 | "user": mailbox_prefix, 110 | "passwd": mailbox_password 111 | } 112 | # 请求 113 | response = self.execute_command("CMD_API_POP", params) 114 | if (response.status_code == 200): 115 | return True 116 | else: 117 | print(response.status_code, response.text) 118 | return False 119 | 120 | """ 121 | @description: 删除邮箱 122 | ------- 123 | @param: mailbox_prefix 124 | ------- 125 | @return: 126 | """ 127 | def delete_mailbox(self, mailbox_prefix): 128 | # 参数 129 | params = { 130 | "action": "delete", 131 | "domain": self._domain, 132 | "user": mailbox_prefix 133 | } 134 | # 请求 135 | response = self.execute_command("CMD_API_POP", params) 136 | if (response.status_code == 200): 137 | print(response.status_code, response.text) 138 | return True 139 | else: 140 | print(response.status_code, response.text) 141 | return False 142 | 143 | """ 144 | @description: 5sim 接码类 145 | ------- 146 | @param: 147 | ------- 148 | @return: 149 | """ 150 | class r_a_5sim(): 151 | 152 | """ 153 | @description: 初始化 154 | ------- 155 | @param: 156 | ------- 157 | @return: 158 | """ 159 | def __init__(self, token=rab_config.load_package_config( 160 | "rab_config.ini", "rab_advance", "ra5_token")): 161 | self.token = token 162 | 163 | """ 164 | @description: 执行命令 165 | ------- 166 | @param: 167 | ------- 168 | @return: 169 | """ 170 | def execute_command(self, route="", params=None): 171 | url = "https://5sim.net/v1/user" + route 172 | headers = { 173 | "Authorization": "Bearer {}".format(self.token), 174 | "Accept": "application/json" 175 | } 176 | try: 177 | response = requests.get(url, headers=headers, params=params) 178 | return response 179 | except Exception as e: 180 | print(e) 181 | return None 182 | 183 | """ 184 | @description: 购买手机号 185 | ------- 186 | @param: 187 | ------- 188 | @return: 189 | """ 190 | def buy(self, country, operator, product): 191 | # 路由 192 | route = "/buy/activation/{}/{}/{}".format(country, operator, product) 193 | # 请求 194 | response = self.execute_command(route) 195 | if (response.status_code == 200): 196 | return json.loads(response.text) 197 | else: 198 | print(response.status_code, response.text) 199 | return None 200 | 201 | """ 202 | @description: 获取验证码 203 | ------- 204 | @param: 205 | ------- 206 | @return: 207 | """ 208 | def get_sms(self, order_id): 209 | # 路由 210 | route = "/check/{}".format(str(order_id)) 211 | # 最多等待一分钟 212 | for _ in range(0, 30): 213 | time.sleep(2) 214 | # 请求 215 | response = self.execute_command(route) 216 | if (response.status_code == 200): 217 | sms_list = json.loads(response.text)["sms"] 218 | if (len(sms_list) > 0): 219 | return sms_list[0]["code"] 220 | else: 221 | print(response.status_code, response.text) 222 | return None 223 | print("{} 获取短信超过一分钟，停止！") 224 | return None 225 | 226 | """ 227 | @description: 结束这个订单 228 | ------- 229 | @param: 230 | ------- 231 | @return: 232 | """ 233 | def finish(self, order_id): 234 | # 路由 235 | route = "/finish/{}".format(str(order_id)) 236 | # 请求 237 | response = self.execute_command(route) 238 | if (response.status_code == 200): 239 | return json.loads(response.text) 240 | else: 241 | print(response.status_code, response.text) 242 | return None 243 | 244 | 245 | """ 246 | @description: 单体测试 247 | ------- 248 | @param: 249 | ------- 250 | @return: 251 | """ 252 | if __name__ == "__main__": 253 | r_a_5sim = r_a_5sim() 254 | order_id = r_a_5sim.buy("russia", "any", "other")["id"] 255 | print(r_a_5sim.get_sms(str(order_id))) 256 | print(r_a_5sim.finish(str(order_id))) -------------------------------------------------------------------------------- /rab_bot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_telegram_bot.py 6 | # @DATE: 2020/12/16 Wed 7 | # @TIME: 16:29:07 8 | # 9 | # @DESCRIPTION: 共通包 Telegram Bot 机器人消息推送模块 10 | 11 | 12 | import sys 13 | import json 14 | import requests 15 | sys.path.append("..") if (".." not in sys.path) else True 16 | from rab_python_packages import rab_config 17 | from rab_python_packages import rab_logging 18 | from rab_python_packages import rab_requests 19 | 20 | 21 | # 日志记录 22 | r_logger = rab_logging.r_logger() 23 | 24 | 25 | """ 26 | @description: 判断 Telegram 消息是否过长 27 | ------- 28 | @param: 29 | ------- 30 | @return: 31 | """ 32 | def is_message_text_too_long(message_text): 33 | if (len(message_text) > 4095): 34 | return True 35 | else: 36 | return False 37 | 38 | 39 | """ 40 | @description: r_bot 类 41 | ------- 42 | @param: 43 | ------- 44 | @return: 45 | """ 46 | class r_bot: 47 | 48 | """ 49 | @description: 初始化 50 | ------- 51 | @param: token 52 | ------- 53 | @return: 54 | """ 55 | def __init__(self, 56 | proxies=None, 57 | token=rab_config.load_package_config( 58 | "rab_config.ini", "rab_bot", "telegram_token")): 59 | self.token = token 60 | self.url = "https://api.telegram.org/bot" + self.token 61 | self.proxies = proxies 62 | self.is_connected = False 63 | # 测试连接 64 | self.test() 65 | # 存放收到各用户最后一条消息的 message_id 66 | self.chat_id_last_message_id = {} 67 | 68 | """ 69 | @description: 测试对 Telegram 的访问 70 | ------- 71 | @param: 72 | ------- 73 | @return: 74 | """ 75 | def test(self): 76 | # 先尝试不使用代理或使用传入代理进行访问测试 77 | try: 78 | result = requests.get("https://core.telegram.org/bots", 79 | proxies=self.proxies, timeout=5) 80 | if (result.status_code == 200): 81 | self.is_connected = True 82 | r_logger.info("Telegram Bot 使用代理 {} 访问测试成功！".format( 83 | str(self.proxies))) 84 | return True 85 | else: 86 | self.is_connected = False 87 | r_logger.error("Telegram Bot 使用代理 {} 访问测试失败！".format( 88 | str(self.proxies))) 89 | except Exception as e: 90 | self.is_connected = False 91 | r_logger.error("Telegram Bot 使用代理 {} 访问测试出错！".format( 92 | str(self.proxies))) 93 | r_logger.error(e) 94 | # 如果不能连接则尝试配置文件中的代理 95 | if (not self.is_connected): 96 | r_logger.info("Telegram Bot 开始尝试配置文件中的代理......") 97 | for proxy in rab_config.load_package_config( 98 | "rab_config.ini", "common", "proxy"): 99 | try: 100 | proxies = { 101 | "http": proxy.replace("socks5://", "socks5h://"), 102 | "https": proxy.replace("socks5://", "socks5h://") 103 | } 104 | result = requests.get("https://core.telegram.org/bots", 105 | proxies=proxies, timeout=5) 106 | if (result.status_code == 200): 107 | self.proxies = proxies 108 | self.is_connected = True 109 | r_logger.info("Telegram Bot 开始使用代理：{}".format( 110 | str(proxies))) 111 | return True 112 | else: 113 | r_logger.error("Telegram Bot 无法使用代理：{}".format( 114 | str(proxies))) 115 | except Exception as e: 116 | r_logger.error("Telegram Bot 尝试代理 {} 出错！".format( 117 | str(proxies))) 118 | r_logger.error(e) 119 | r_logger.error("Telegram Bot 尝试所有配置文件中的代理均无法访问！") 120 | return False 121 | 122 | 123 | """ 124 | @description: 获取最新的聊天记录 125 | ------- 126 | @param: 127 | ------- 128 | @return: 129 | """ 130 | def get_latest_messages(self): 131 | # 判断是否能连接 Telegram 服务器 132 | if (self.is_connected): 133 | # 获取聊天记录的 API 地址 134 | get_updates_url = self.url + "/getUpdates" 135 | try: 136 | response = requests.get(get_updates_url, proxies=self.proxies, 137 | verify=False) 138 | messages = json.loads(response.text) 139 | # 继上次获取后的新信息列表 140 | latest_messages = [] 141 | for message in messages["result"]: 142 | # 获取 CHAT ID 143 | chat_id = message["message"]["chat"]["id"] 144 | # 如果第一次收到信息，或 message_id 比既存的还要大 145 | if (chat_id not in self.chat_id_last_message_id.keys() 146 | or (message["message"]["message_id"] 147 | > self.chat_id_last_message_id[chat_id])): 148 | # 时间较早的在队列前面 149 | latest_messages.append(message) 150 | self.chat_id_last_message_id[chat_id] \ 151 | = message["message"]["message_id"] 152 | # 将新消息替换响应中消息列表 153 | messages["result"] = latest_messages 154 | if (len(latest_messages) > 0): 155 | return { 156 | "ok": True, 157 | "result":latest_messages, 158 | "description": "共 {} 条新消息！".format( 159 | len(latest_messages)) 160 | } 161 | else: 162 | return {"ok": True, "result":[], "description": "无新消息！"} 163 | except Exception as e: 164 | return {"ok": False, "description": "出错：{}".format(str(e))} 165 | else: 166 | return {"ok": False, "description": "无法连接到 Telegram 服务器！"} 167 | 168 | """ 169 | @description: 发送信息 170 | ------- 171 | @param: chat_id, message 172 | ------- 173 | @return: 174 | """ 175 | def send_message(self, 176 | message_data, 177 | chat_id=rab_config.load_package_config( 178 | "rab_config.ini", "rab_bot", "telegram_chat_id"), 179 | parse_mode=None): 180 | # 判断是否能连接 Telegram 服务器 181 | if (self.is_connected): 182 | # 发送聊天信息的 API 地址 183 | send_message_url = self.url + "/sendMessage" 184 | headers = { 185 | "Content-Type": "application/json" 186 | } 187 | data = { 188 | "parse_mode": parse_mode, 189 | "chat_id": chat_id, 190 | } 191 | data.update(message_data) 192 | try: 193 | response = requests.post(send_message_url, headers=headers, \ 194 | data=json.dumps(data), proxies=self.proxies, verify=False) 195 | return json.loads(response.text) 196 | except Exception as e: 197 | return {"ok": False, "description": "出错：{}".format(str(e))} 198 | else: 199 | return {"ok": False, "description": "无法连接到 Telegram 服务器！"} 200 | 201 | 202 | """ 203 | @description: 单体测试 204 | ------- 205 | @param: 206 | ------- 207 | @return: 208 | """ 209 | if __name__ == "__main__": 210 | r_bot = r_bot() 211 | # 测试基础功能 212 | print(r_bot.get_latest_messages()) 213 | # 测试表格 214 | table = """ 215 | ``` 216 | | BBB | 217 | | AAA | 218 | | Symbol | Price | Change | 219 | |--------|-------|--------| 220 | | ABC | 20.85 | 1.626 | 221 | | DEF | 78.95 | 0.099 | 222 | | GHI | 23.45 | 0.192 | 223 | | JKL | 98.85 | 0.292 | 224 | ``` 225 | """ 226 | message_data = {"text": table} 227 | print(r_bot.send_message(message_data, parse_mode="Markdown")) -------------------------------------------------------------------------------- /rab_rabbitmq.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_rabbitmq.py 6 | # @DATE: 2021/07/12 Mon 7 | # @TIME: 14:39:42 8 | # 9 | # @DESCRIPTION: 共通包 RabbitMQ 消息队列方法模块 10 | 11 | 12 | import sys 13 | import json 14 | import uuid 15 | import pika 16 | import time 17 | import requests 18 | sys.path.append("..") if (".." not in sys.path) else True 19 | from rab_python_packages import rab_config 20 | from rab_python_packages import rab_logging 21 | 22 | 23 | # 日志记录 24 | r_logger = rab_logging.r_logger() 25 | 26 | 27 | """ 28 | @description: RabbitMQ 对象 29 | ------- 30 | @param: 31 | ------- 32 | @return: 33 | """ 34 | class r_rabbitmq(): 35 | 36 | """ 37 | @description: 初始化 38 | ------- 39 | @param: 40 | ------- 41 | @return: 42 | """ 43 | def __init__(self, 44 | username=rab_config.load_package_config( 45 | "rab_config.ini", "rab_rabbitmq", "username"), 46 | password=rab_config.load_package_config( 47 | "rab_config.ini", "rab_rabbitmq", "password"), 48 | host=rab_config.load_package_config( 49 | "rab_config.ini", "rab_rabbitmq", "host"), 50 | port=rab_config.load_package_config( 51 | "rab_config.ini", "rab_rabbitmq", "port"), 52 | api_host=rab_config.load_package_config( 53 | "rab_config.ini", "rab_rabbitmq", "api_host")): 54 | self.username = username 55 | self.password = password 56 | self.host = host 57 | self.port = port 58 | self.api_host = api_host 59 | self.connection = {} 60 | self.channel = {} 61 | 62 | """ 63 | @description: 建立连接 64 | ------- 65 | @param: 66 | ------- 67 | @return: 68 | """ 69 | def connect(self, connection_name): 70 | auth = pika.PlainCredentials(self.username, self.password) 71 | try: 72 | self.connection[connection_name] = pika.BlockingConnection( 73 | pika.ConnectionParameters(self.host, int(self.port), "/", auth)) 74 | return True 75 | except Exception as e: 76 | r_logger.error( 77 | "RabbitMQ 建立连接 {} 时出错！".format(connection_name)) 78 | r_logger.error(e) 79 | return False 80 | 81 | """ 82 | @description: 建立频道 83 | ------- 84 | @param: 85 | ------- 86 | @return: 87 | """ 88 | def build_channel(self, connection_name, channel_name): 89 | if (channel_name not in self.channel.keys()): 90 | self.channel[channel_name] = self.connection[ 91 | connection_name].channel() 92 | else: 93 | r_logger.warn("RabbitMQ 已经存在频道：{}".format(channel_name)) 94 | 95 | """ 96 | @description: 关闭连接 97 | ------- 98 | @param: 99 | ------- 100 | @return: 101 | """ 102 | def disconnect(self): 103 | # 关闭所有频道 104 | if (self.channel): 105 | for channel_name in self.channel.keys(): 106 | if (self.channel[channel_name] 107 | and self.channel[channel_name].is_open): 108 | try: 109 | self.channel[channel_name].close() 110 | except Exception as e: 111 | r_logger.error( 112 | "RabbitMQ 频道 {} 关闭出错！".format(channel_name)) 113 | r_logger.error(e) 114 | self.channel[channel_name] = None 115 | self.channel = {} 116 | # 关闭所有连接 117 | if (self.connection): 118 | for connection_name in self.connection.keys(): 119 | if (self.connection[connection_name] 120 | and self.connection[connection_name].is_open): 121 | try: 122 | self.connection[connection_name].close() 123 | except Exception as e: 124 | r_logger.error( 125 | "RabbitMQ 连接 {} 关闭出错!".format(connection_name)) 126 | r_logger.error(e) 127 | self.connection[connection_name] = None 128 | self.connection = {} 129 | 130 | """ 131 | @description: 发布 132 | ------- 133 | @param: 134 | ------- 135 | @return: 136 | """ 137 | def publish(self, channel_name, queue, body): 138 | try: 139 | if (type(body) == str): 140 | body = json.loads(body) 141 | # 任务是否有 UUID，没有的话一定要赋予 142 | if ("uuid" not in body.keys()): 143 | uuid = str(uuid.uuid1()) 144 | body["uuid"] = uuid 145 | else: 146 | uuid = body["uuid"] 147 | # 发布 148 | self.channel[channel_name].basic_publish(exchange="", \ 149 | routing_key=queue, body=json.dumps(body), \ 150 | # 消息持久化 151 | properties=pika.BasicProperties(delivery_mode=2)) 152 | return True, uuid 153 | except Exception as e: 154 | r_logger.error( 155 | "RabbitMQ 发布信息时出错，队列：{queue}，消息:{body}".format( 156 | queue=queue, body=str(body))) 157 | r_logger.error(e) 158 | return False, None 159 | 160 | """ 161 | @description: 获取（以 UUID 作为主键） 162 | ------- 163 | @param: 164 | ------- 165 | @return: 166 | """ 167 | def get(self, channel_name, queue, uuid): 168 | # 遍历所有消息 169 | for method, properties, body \ 170 | in self.channel[channel_name].consume(queue): 171 | # 如果 UUID 匹配 172 | if ("uuid" in json.loads(body.decode("UTF-8")).keys() 173 | and uuid == json.loads(body.decode("UTF-8"))["uuid"]): 174 | # 删除这个消息 175 | result = json.loads(body.decode("UTF-8")) 176 | self.channel[channel_name].basic_ack(method.delivery_tag) 177 | return result 178 | # 如果 UUID 不匹配 179 | else: 180 | # 将消息放回队列 181 | self.channel[channel_name].basic_nack(method.delivery_tag) 182 | # 没有结果的情况下返回空 183 | return None 184 | 185 | """ 186 | @description: 获取队列长度 187 | ------- 188 | @param: 189 | ------- 190 | @return: 191 | """ 192 | def get_message_count(self, queue, vhost="%2F"): 193 | try: 194 | url = "{api_host}/api/queues/{vhost}/{queue}".format( 195 | api_host=self.api_host, vhost=vhost, queue=queue) 196 | r = requests.get(url, auth=(self.username, self.password)) 197 | result = json.loads(r.text) 198 | ready = result['messages_ready'] 199 | unacknowledged = result['messages_unacknowledged'] 200 | total = result['messages'] 201 | return ready, unacknowledged, total 202 | except Exception as e: 203 | r_logger.error("RabbitMQ 获取队列长度时出错！") 204 | r_logger.error(e) 205 | return None, None, None 206 | 207 | """ 208 | @description: 判断队列是否为空 209 | ------- 210 | @param: 211 | ------- 212 | @return: 213 | """ 214 | def is_queue_empty(self, queue): 215 | # 获取非缓存的准确队列长度 216 | _ready = None 217 | for _ in range(0, 10): 218 | ready, unacknowledged, total = self.get_message_count( 219 | "reptile_results") 220 | if (ready >= 100): 221 | break 222 | elif(ready == 0): 223 | break 224 | else: 225 | time.sleep(0.5) 226 | r_logger.info("RabbitMQ 开始等待获取准确的队列长度......") 227 | if (_ready == None): 228 | _ready = ready 229 | continue 230 | elif(_ready == ready): 231 | continue 232 | else: 233 | break 234 | # 判断准备队列是否为空 235 | if (ready == 0): 236 | return True 237 | else: 238 | return False 239 | 240 | 241 | """ 242 | @description: 单体测试 243 | ------- 244 | @param: 245 | ------- 246 | @return: 247 | """ 248 | if __name__ == "__main__": 249 | 250 | r_rabbitmq = r_rabbitmq() 251 | r_rabbitmq.connect("test_connection") 252 | r_rabbitmq.build_channel("test_connection", "test_channel") 253 | 254 | # 测试是否能正常推送消息和根据 UUID 获取结果 255 | # try: 256 | # for uuid in range(123450, 123460): 257 | # body = json.dumps({"uuid": str(uuid)}) 258 | # r_rabbitmq.publish("test_channel", "test_quere", body) 259 | # print("获得结果:{}".format( 260 | # r_rabbitmq.get("test_channel", "test_quere", "123456"))) 261 | # except Exception as e: 262 | # print("测试出错：{}".format(str(e))) 263 | # finally: 264 | # r_rabbitmq.disconnect() 265 | 266 | # 测试队列长度是否获取正确 267 | print(r_rabbitmq.get_message_count("test_quere")) -------------------------------------------------------------------------------- /rab_crontab.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_crontab.py 6 | # @DATE: 2021/08/12 Thu 7 | # @TIME: 09:30:35 8 | # 9 | # @DESCRIPTION: 定时任务模块 10 | 11 | 12 | import sys 13 | import time 14 | import calendar 15 | import datetime 16 | sys.path.append("..") if (".." not in sys.path) else True 17 | from rab_python_packages import rab_logging 18 | 19 | 20 | # 日志记录 21 | r_logger = rab_logging.r_logger() 22 | 23 | 24 | """ 25 | @description: 获取上一个整分、整时、整天的时间戳 26 | ------- 27 | @param: 28 | ------- 29 | @return: 30 | """ 31 | def get_last_whole_timestamp(timestamp=int(time.time()), method="minute"): 32 | if (method == "minute"): 33 | return ((timestamp//60)) * 60 34 | elif(method == "hour"): 35 | return ((timestamp//3600)) * 3600 36 | elif(method == "day"): 37 | return ((timestamp//86400)) * 86400 38 | elif(method == "month"): 39 | return int(time.mktime(datetime.date( 40 | datetime.date.today().year, \ 41 | datetime.date.today().month, 1).timetuple())) 42 | else: 43 | return None 44 | 45 | """ 46 | @description: 获取下一个整分、整时、整天的时间戳 47 | ------- 48 | @param: 49 | ------- 50 | @return: 51 | """ 52 | def get_next_whole_timestamp(timestamp=int(time.time()), method="minute"): 53 | if (method == "minute"): 54 | return ((timestamp//60)+1) * 60 55 | elif(method == "hour"): 56 | return ((timestamp//3600)+1) * 3600 57 | elif(method == "day"): 58 | return ((timestamp//86400)+1) * 86400 59 | elif(method == "month"): 60 | return int(time.mktime(datetime.date( 61 | datetime.date.today().year, \ 62 | datetime.date.today().month,1).timetuple())) 63 | else: 64 | return None 65 | 66 | """ 67 | @description: 解析 crontab 时间规则以获取分、时、日 68 | ------- 69 | @param: 70 | ------- 71 | @return: 72 | """ 73 | def parse_crontab_time_setting(crontab_time_setting): 74 | minute = crontab_time_setting[0] 75 | hour = crontab_time_setting[1] 76 | day_of_month = crontab_time_setting[2] 77 | month = crontab_time_setting[3] 78 | day_of_week = crontab_time_setting[4] 79 | # 暂不支持每周和每月固定日期执行 80 | if (month != "*" and day_of_week != "*"): 81 | r_logger.error("定时任务模块暂不支持指定月和指定星期执行！") 82 | return None 83 | # 获取需要定时执行的分钟数 84 | minutes = [] 85 | if (minute != "*"): 86 | # 单个分钟 87 | if (minute.isdigit()): 88 | minutes = [int(minute)] 89 | # 手动设定多个 90 | elif("," in minute): 91 | for m in minute.split(","): 92 | minutes.append(int(m)) 93 | # 每隔固定时间执行 94 | elif("/" in minute): 95 | for m in range(0, 60): 96 | if (m%int(minute.split("/")[1]) == 0): 97 | minutes.append(int(m)) 98 | else: 99 | minutes = list(range(0, 60)) 100 | # 获取需要定时执行的小时数 101 | hours = [] 102 | if (hour != "*"): 103 | # 单个小时 104 | if (hour.isdigit()): 105 | hours = [int(hour)] 106 | # 手动设定多个 107 | elif("," in hour): 108 | for h in hour.split(","): 109 | hours.append(int(h)) 110 | # 每隔固定时间执行 111 | elif("/" in hour): 112 | for h in range(0, 24): 113 | if (h%int(hour.split("/")[1]) == 0): 114 | hours.append(int(h)) 115 | else: 116 | hours = list(range(0, 24)) 117 | # 获取需要定时执行的天数（以月为单位，并且会拓展到下个月 1 日 24 点） 118 | day = day_of_month 119 | days = [] 120 | week, days_num = calendar.monthrange( 121 | datetime.datetime.now().year, datetime.datetime.now().month) 122 | if (day != "*"): 123 | # 单个日期 124 | if (day.isdigit()): 125 | days = [int(day)] 126 | # 手动设定多个 127 | elif("," in day): 128 | for d in day.split(","): 129 | days.append(int(d)) 130 | # 每隔固定时间执行 131 | elif("/" in day): 132 | # 每月 1 日必定符合 133 | days.append(0) 134 | # 循环剩余日期 135 | for d in range(1, days_num+1): 136 | if (d%int(day.split("/")[1]) == 0): 137 | days.append(int(d)) 138 | else: 139 | days = list(range(0, days_num+1)) 140 | return minutes, hours, days 141 | 142 | """ 143 | @description: 获取当月所有满足要求的时间点时间戳 144 | ------- 145 | @param: 146 | ------- 147 | @return: 148 | """ 149 | def get_run_timestamps(crontab_time_setting): 150 | # 先生成所有符合 crontab 要求的时间戳 151 | run_timestamps = [] 152 | minutes, hours, days = parse_crontab_time_setting( 153 | crontab_time_setting) 154 | for day in days: 155 | for hour in hours: 156 | for minute in minutes: 157 | this_timestamp = get_last_whole_timestamp(method="month") \ 158 | + day*86400 + hour*3600 + minute*60 159 | run_timestamps.append(this_timestamp) 160 | return run_timestamps 161 | 162 | 163 | """ 164 | @description: r_crontab 类 165 | ------- 166 | @param: 167 | ------- 168 | @return: 169 | """ 170 | class r_crontab(): 171 | 172 | """ 173 | @description: 初始化 174 | ------- 175 | @param: 176 | ------- 177 | @return: 178 | """ 179 | def __init__(self): 180 | self.task = {} 181 | 182 | """ 183 | @description: 新增任务，默认周期为 30 天 184 | ------- 185 | @param: 186 | ------- 187 | @return: 188 | """ 189 | def add(self, task_name, crontab_time_setting): 190 | self.task[task_name] = { 191 | "crontab_time_setting": crontab_time_setting, 192 | "run_timestamps": get_run_timestamps(crontab_time_setting), 193 | "no_stop": False} 194 | 195 | """ 196 | @description: 续约任务 197 | ------- 198 | @param: 199 | ------- 200 | @return: 201 | """ 202 | def renew(self, task_name): 203 | self.task[task_name]["run_timestamps"] = get_run_timestamps( 204 | self.task[task_name]["crontab_time_setting"]) 205 | 206 | """ 207 | @description: 运行一次任务以记录时间戳 208 | ------- 209 | @param: 210 | ------- 211 | @return: 212 | """ 213 | def run(self, task_name): 214 | self.task[task_name]["last_run_over_timestamp"] = None 215 | self.task[task_name]["last_run_start_timestamp"] = int(time.time()) 216 | 217 | """ 218 | @description: 记录运行结束 219 | ------- 220 | @param: 221 | ------- 222 | @return: 223 | """ 224 | def over(self, task_name): 225 | self.task[task_name]["last_run_over_timestamp"] = int(time.time()) 226 | # 如果接下来剩下少于一个可用的时间戳则进行续约 227 | available_run_timestamps = [] 228 | for run_timestamp in self.task[task_name]["run_timestamps"]: 229 | if (run_timestamp >= int(time.time())): 230 | available_run_timestamps.append(run_timestamp) 231 | if (len(available_run_timestamps) <= 1): 232 | self.renew(task_name) 233 | 234 | """ 235 | @description: 获取下一次运行的时间戳 236 | ------- 237 | @param: 238 | ------- 239 | @return: 240 | """ 241 | def get_next_run_timestamp(self, task_name): 242 | for run_timestamp in self.task[task_name]["run_timestamps"]: 243 | if (run_timestamp > int(time.time())): 244 | return run_timestamp 245 | return None 246 | 247 | """ 248 | @description: 到了运行的时间点 249 | ------- 250 | @param: 251 | ------- 252 | @return: 253 | """ 254 | def is_time_2_run(self, task_name): 255 | # 如果是不间断运行则直接返回 True 256 | if (self.task[task_name]["no_stop"]): 257 | return True 258 | # 如果曾经开始并结束过 259 | if ("last_run_over_timestamp" in self.task[task_name].keys() 260 | and self.task[task_name]["last_run_over_timestamp"]): 261 | # 遍历所有需要运行的时间戳 262 | for run_timestamp_index, run_timestamp in enumerate( 263 | self.task[task_name]["run_timestamps"]): 264 | # 判断当前时间戳是否比上次运行时间点的下一次时间戳还要大 265 | if ((self.task[task_name]["last_run_over_timestamp"] \ 266 | >= run_timestamp) 267 | and (self.task[task_name]["last_run_over_timestamp"] \ 268 | < self.task[task_name]["run_timestamps"][( 269 | run_timestamp_index+1)]) 270 | and (int(time.time()) >= self.task[task_name][ 271 | "run_timestamps"][(run_timestamp_index+1)])): 272 | return True 273 | # 时间戳循环结束返回 False 274 | return False 275 | # 未开始过则直接开始 276 | else: 277 | return True 278 | 279 | """ 280 | @description: 等待下一次运行时间的到来 281 | ------- 282 | @param: 283 | ------- 284 | @return: 285 | """ 286 | def wait(self, task_name): 287 | # 如果是不间断运行则直接返回 True 288 | if (self.task[task_name]["no_stop"]): 289 | return True 290 | next_run_timestamp = self.get_next_run_timestamp(task_name) 291 | r_logger.info("下一次 {task_name} 任务执行的时间：{time}".format( 292 | task_name=task_name, time=time.strftime( 293 | "%Y-%m-%d %H:%M:%S",time.localtime(next_run_timestamp)))) 294 | r_logger.info("等待中......") 295 | while True: 296 | if (int(time.time()) > next_run_timestamp): 297 | return True 298 | else: 299 | time.sleep(1) 300 | 301 | """ 302 | @description: 距上次运行结束是否是新的分钟、小时、天 303 | ------- 304 | @param: 305 | ------- 306 | @return: 307 | """ 308 | def is_new(self, task_name, method="day"): 309 | # 如果没有开始过 310 | if ("last_run_start_timestamp" not in self.task[task_name].keys()): 311 | return True 312 | # 如果还没结束 313 | elif(not "last_run_over_timestamp"): 314 | return False 315 | # 如果开始过则判定是否到了新的时间点 316 | else: 317 | if (method=="minute"): 318 | if ((int(time.time())-get_last_whole_timestamp(self.task[ 319 | task_name]["last_run_start_timestamp"], "minute")) \ 320 | >= 60): 321 | return True 322 | else: 323 | return False 324 | elif (method=="hour"): 325 | if ((int(time.time())-get_last_whole_timestamp(self.task[ 326 | task_name]["last_run_start_timestamp"], "hour")) \ 327 | >= 3600): 328 | return True 329 | else: 330 | return False 331 | elif (method=="day"): 332 | if ((int(time.time())-get_last_whole_timestamp(self.task[ 333 | task_name]["last_run_start_timestamp"], "day")) \ 334 | >= 86400): 335 | return True 336 | else: 337 | return False 338 | 339 | 340 | """ 341 | @description: 单体测试 342 | ------- 343 | @param: 344 | ------- 345 | @return: 346 | """ 347 | if __name__ == "__main__": 348 | r_crontab = r_crontab() 349 | r_crontab.add("test", ["*/3", "*", "*", "*", "*"]) 350 | print(len(r_crontab.task["test"]["run_timestamps"])) 351 | # r_crontab.wait("test") 352 | 353 | # 测试到点执行 354 | # while True: 355 | # time.sleep(10) 356 | # if (r_crontab.is_time_2_run("test")): 357 | # r_crontab.run("test") 358 | # print("run") 359 | # r_crontab.over("test") 360 | # print("wait next: {}".format(r_crontab.get_next_run_timestamp("test"))) -------------------------------------------------------------------------------- /rab_chrome.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_chrome.py 6 | # @DATE: 2021/07/30 Fri 7 | # @TIME: 23:00:08 8 | # 9 | # @DESCRIPTION: 共通包 Chrome 浏览器构建和控制 10 | 11 | 12 | import re 13 | import os 14 | import sys 15 | import time 16 | import requests 17 | import platform 18 | from selenium import webdriver 19 | from selenium.webdriver.chrome.options import Options 20 | from selenium.webdriver.common.desired_capabilities import DesiredCapabilities 21 | sys.path.append("..") if (".." not in sys.path) else True 22 | from rab_python_packages import rab_config 23 | from rab_python_packages import rab_logging 24 | from rab_python_packages import rab_node 25 | 26 | 27 | # 日志记录 28 | r_logger = rab_logging.r_logger() 29 | 30 | 31 | """ 32 | @description: 获取当前操作系统 33 | ------- 34 | @param: 35 | ------- 36 | @return: 37 | """ 38 | def get_system_type(): 39 | if ("windows" in str(platform.platform()).lower()): 40 | return "windows" 41 | else: 42 | return "linux" 43 | 44 | """ 45 | @description: Windows 系统下在指定端口构建 Chrome 浏览器 46 | ------- 47 | @param: 48 | ------- 49 | @return: 50 | """ 51 | def construct_chrome(port, chrome_path, chrome_user_data_path): 52 | try: 53 | build_command = 'start "" "{}" '.format(chrome_path) \ 54 | + '--remote-debugging-port={} '.format(str(port)) \ 55 | + '--user-data-dir="{}"'.format(chrome_user_data_path) 56 | print(build_command) 57 | os.system(build_command) 58 | time.sleep(3) 59 | r_logger.info("Windows 系统下端口 {} 上 Chrome 启动成功！".format( 60 | str(port))) 61 | except Exception as e: 62 | r_logger.error("Windows 系统下端口 {} 上 Chrome 启动失败！".format( 63 | str(port))) 64 | r_logger.error(e) 65 | 66 | """ 67 | @description: 关闭 GOST 68 | ------- 69 | @param: 70 | ------- 71 | @return: 72 | """ 73 | def gost_stop(local_proxy_port): 74 | gost_stop_command = rab_config.load_package_config( 75 | "rab_linux_command.ini", "rab_chrome", "gost_stop") 76 | gost_stop_command += " {}".format(str(local_proxy_port)) 77 | os.system(gost_stop_command) 78 | 79 | """ 80 | @description: 开启 GOST 81 | ------- 82 | @param: 83 | ------- 84 | @return: 85 | """ 86 | def gost_start(local_proxy_port, proxy): 87 | gost_start_command = rab_config.load_package_config( 88 | "rab_linux_command.ini", "rab_chrome", "gost_start") 89 | # 添加启动参数 90 | gost_start_command += " -p {}".format(str(local_proxy_port)) 91 | proxy_info = rab_node.parse_http_or_socks5_node_url(proxy) 92 | gost_start_command += " -A {}".format(str(proxy_info["type"])) 93 | gost_start_command += " -U {}".format(str(proxy_info["username"])) 94 | gost_start_command += " -D {}".format(str(proxy_info["password"])) 95 | gost_start_command += " -H {}".format(str(proxy_info["server"])) 96 | gost_start_command += " -P {}".format(str(proxy_info["port"])) 97 | os.system(gost_start_command) 98 | 99 | 100 | """ 101 | @description: r_chrome 类 102 | ------- 103 | @param: 104 | ------- 105 | @return: 106 | """ 107 | class r_chrome(): 108 | 109 | """ 110 | @description: 初始化 111 | ------- 112 | @param: 113 | ------- 114 | @return: 115 | """ 116 | def __init__(self, 117 | driver=None, 118 | port=None, 119 | proxy=None, 120 | local_proxy_port=1080): 121 | # chromedriver 122 | self.driver = driver 123 | # 浏览器起的端口 124 | self.port = port 125 | # 使用的代理 126 | self.proxy = proxy 127 | # 需要验证代理转发时所用的本地端口 128 | self.local_proxy_port = local_proxy_port 129 | # 需要验证代理转发用 GOST 130 | self.is_gost_used = False 131 | # 操作系统 132 | self.system = get_system_type() 133 | 134 | """ 135 | @description: 构建浏览器 136 | ------- 137 | @param: 138 | ------- 139 | @return: 140 | """ 141 | def build(self, 142 | dev_shm_usage=True, 143 | chrome_path="chrome.exe", 144 | chrome_driver_path="chromedriver", 145 | chrome_user_data_path="C:\selenum\AutomationProfile", 146 | prefs=None): 147 | # 浏览器配置 148 | capabilities = DesiredCapabilities.CHROME 149 | capabilities["goog:loggingPrefs"] = {"browser": "ALL"} 150 | chrome_options = Options() 151 | # Windows 系统 152 | if (self.system == "windows"): 153 | r_logger.info("Windows 系统下 Chrome 构建开始......") 154 | # 如果指定了端口 155 | if (self.port): 156 | # 在指定端口构建浏览器 157 | construct_chrome(self.port, chrome_path, chrome_user_data_path) 158 | # 接管端口上的浏览器 159 | chrome_options.add_experimental_option( 160 | "debuggerAddress", "127.0.0.1:"+str(self.port)) 161 | # 如果没有指定端口 162 | else: 163 | pass 164 | # 默认已经提前将 chromedriver 路径加入环境变量中 165 | chrome_driver = chrome_driver_path 166 | # Linux 系统 167 | elif(self.system == "linux"): 168 | r_logger.info("Linux 系统下 Chrome 构建开始......") 169 | # Linux 无头无界面浏览器配置 170 | chrome_options.add_argument("--headless") 171 | chrome_options.add_argument("--disable-gpu") 172 | chrome_options.add_argument("window-size=1024,768") 173 | chrome_options.add_argument("enable-automation") 174 | chrome_options.add_argument("--disable-infobars") 175 | chrome_options.add_argument("--no-sandbox") 176 | chrome_options.add_argument("--disable-browser-side-navigation") 177 | if (not dev_shm_usage): 178 | chrome_options.add_argument("--disable-dev-shm-usage") 179 | if (prefs): 180 | chrome_options.add_experimental_option("prefs", prefs) 181 | # 无需认证的代理则直接加上属性即可 182 | if(self.proxy): 183 | # 如果需要验证的话 184 | if ("@" in self.proxy): 185 | # 通过 GOST 转发 186 | gost_stop(self.local_proxy_port) 187 | gost_start(self.local_proxy_port, self.proxy) 188 | # 本地代理信息 189 | proxy = "socks5://127.0.0.1:{}".format( 190 | str(self.local_proxy_port)) 191 | self.is_gost_used = True 192 | # 如果不需要验证的话 193 | else: 194 | proxy = self.proxy 195 | # 浏览器代理的配置 196 | chrome_options.add_argument( 197 | "--proxy-server={}".format(proxy)) 198 | r_logger.info("Linux 系统下 Chrome 使用代理：{}，原代理：{}" \ 199 | .format(proxy, self.proxy)) 200 | # 默认已经提前为 chromedriver 建立了软连接 201 | chrome_driver = chrome_driver_path 202 | # Windows 和 Linux 分开配置完成后，建立浏览器 203 | try: 204 | self.driver = webdriver.Chrome(chrome_driver, \ 205 | desired_capabilities=capabilities, options=chrome_options) 206 | r_logger.info("{} 系统下 Chrome 构建成功！".format( 207 | self.system.capitalize())) 208 | return True 209 | except Exception as e: 210 | r_logger.error("{} 系统下 Chrome 构建失败！".format( 211 | self.system.capitalize())) 212 | r_logger.error(e) 213 | return False 214 | 215 | """ 216 | @description: 为无界面浏览器导入 jQuery 217 | ------- 218 | @param: 219 | ------- 220 | @return: 221 | """ 222 | def import_jquery(self): 223 | jquery_js_text = requests.get("https://libs.baidu.com/jquery/2.0.0/" \ 224 | + "jquery.min.js").text 225 | self.driver.execute_script(jquery_js_text) 226 | # 循环 10 秒等待导入成功 227 | for _ in range(0, 10): 228 | test_jquery_js = "$('head').append('

jquery_test

');" 229 | try: 230 | self.driver.execute_script(test_jquery_js) 231 | r_logger.info("无头浏览器 jQuery 加载完成！") 232 | return True 233 | except Exception as e: 234 | r_logger.info("无头浏览器等待 jQuery 加载中......") 235 | time.sleep(1) 236 | r_logger.error("无头浏览器 jQuery 加载失败！") 237 | return False 238 | 239 | """ 240 | @description: 关闭浏览器 241 | ------- 242 | @param: 243 | ------- 244 | @return: 245 | """ 246 | def close(self): 247 | # 关闭 GOST 248 | if (self.is_gost_used): 249 | gost_stop(self.local_proxy_port) 250 | # Windows 系统 251 | if (self.system == "windows"): 252 | r_logger.info("Windows 系统下开始关闭 Chrome......") 253 | # 如果指定了端口 254 | if (self.port): 255 | # 查找对应端口的进程 256 | cmd_command = 'netstat -aon|findstr "{}"'.format(str(self.port)) 257 | cmd_driver = os.popen(cmd_command) 258 | try: 259 | cmd_msg = cmd_driver.read().split("\n")[0] 260 | cmd_msgs = cmd_msg.split(" ") 261 | # 获取端口对应的 PID 用来杀死进程 262 | pid = re.findall(r"\d+\.?\d*", cmd_msgs[-1])[0] 263 | cmd_driver = os.popen( 264 | "taskkill -f /pid {}".format(str(pid))) 265 | # 返回信息 266 | cmd_msg = cmd_driver.read() 267 | if ("成功" in cmd_msg): 268 | r_logger.info("Windows 系统下关闭浏览器成功！") 269 | return True 270 | else: 271 | r_logger.error( 272 | "Windows 系统下关闭浏览器失败！错误：{}".format( 273 | str(cmd_msg))) 274 | except Exception as e: 275 | r_logger.error("Windows 系统下关闭浏览器失败！") 276 | r_logger.error(e) 277 | finally: 278 | cmd_driver.close() 279 | return False 280 | # 如果没有指定端口 281 | else: 282 | self.driver.quit() 283 | r_logger.info("Windows 系统下关闭浏览器成功！") 284 | return True 285 | # Linux 系统 286 | else: 287 | r_logger.info("Linux 系统下开始关闭 Chrome......") 288 | if (self.driver): 289 | self.driver.quit() 290 | r_logger.info("Linux 系统下关闭浏览器成功！") 291 | return True 292 | 293 | """ 294 | @description: 等待元素出现 295 | ------- 296 | @param: 297 | ------- 298 | @return: 299 | """ 300 | def wait_appear(self, xpath, max_wait_time=10): 301 | for _ in range(0, timeout): 302 | try: 303 | element = slef.driver.find_element_by_xpath(xpath) 304 | return True 305 | except Exception as e: 306 | time.sleep(1) 307 | return False 308 | 309 | """ 310 | @description: 重启浏览器 311 | ------- 312 | @param: 313 | ------- 314 | @return: 315 | """ 316 | def restart(self): 317 | # 关闭现有浏览器 318 | if (self.driver): 319 | self.close() 320 | # 新建浏览器 321 | self.build() 322 | 323 | 324 | """ 325 | @description: 单体测试 326 | ------- 327 | @param: 328 | ------- 329 | @return: 330 | """ 331 | if __name__ == "__main__": 332 | # r_chrome = r_chrome() 333 | # r_chrome.build() 334 | # r_chrome.restart() 335 | # r_chrome.driver.get("https://baidu.com") 336 | # time.sleep(3) 337 | # print(r_chrome.driver.page_source) 338 | # r_chrome.close() 339 | proxy = "socks5://127.0.0.1:1080" 340 | gost_start(33333, proxy) -------------------------------------------------------------------------------- /rab_proxy.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_proxy.py 6 | # @DATE: 2021/07/27 Tue 7 | # @TIME: 17:42:23 8 | # 9 | # @DESCRIPTION: 共通包 代理获取模块 10 | 11 | 12 | import os 13 | import sys 14 | import requests 15 | import random 16 | import docker 17 | sys.path.append("..") if (".." not in sys.path) else True 18 | from rab_python_packages import rab_config 19 | from rab_python_packages import rab_logging 20 | from rab_python_packages import rab_ip 21 | from rab_python_packages import rab_postgresql 22 | from rab_python_packages import rab_node 23 | 24 | 25 | # 日志记录 26 | r_logger = rab_logging.r_logger() 27 | 28 | 29 | """ 30 | @description: 获取自建代理 31 | ------- 32 | @param: 33 | ------- 34 | @return: 35 | """ 36 | def get_personal_proxy_infos(location=None): 37 | personal_proxy_infos = {} 38 | for personal_proxy in rab_config.load_package_config( 39 | "rab_config.ini", "common", "proxy"): 40 | proxy_info = rab_node.parse_http_or_socks5_node_url( 41 | personal_proxy) 42 | if (proxy_info["type"] not in personal_proxy_infos.keys()): 43 | personal_proxy_infos[proxy_info["type"]] = {} 44 | personal_proxy_info = { 45 | "host": proxy_info["server"], 46 | "port": proxy_info["port"], 47 | "auth": "{}:{}".format( 48 | proxy_info["username"], proxy_info["password"]), 49 | "access": {}, 50 | "level": 10 51 | } 52 | ip_info = rab_ip.get_ip_info( 53 | parse_proxy_info(proxy_info["type"], personal_proxy_info)) 54 | if (not location or ip_info["location"] == location): 55 | personal_proxy_infos[proxy_info["type"]][ip_info["ip"]] \ 56 | = personal_proxy_info 57 | return personal_proxy_infos 58 | 59 | """ 60 | @description: 获取指定地区下所有可用代理 61 | ------- 62 | @param: 63 | ------- 64 | @return: 65 | """ 66 | @rab_postgresql.change_database(rab_config.load_package_config( 67 | "rab_config.ini", "rab_proxy", "proxy_database")) 68 | def get_proxy_infos(r_pgsql_driver, location=None, level=None): 69 | # 代理信息存储 70 | proxy_infos = { 71 | "reverse": {}, 72 | "http": {}, 73 | "socks5":{} 74 | } 75 | # 如果对代理等级无要求，则从数据库中搜索出其他代理 76 | if (not level or level < 10): 77 | # 搜索所有代理 78 | select_sql = """ 79 | SELECT 80 | * 81 | FROM 82 | (SELECT 83 | sp_ip AS host, 84 | sp_out_ip AS out_ip, 85 | sp_location AS location, 86 | '' AS reverse_proxy_port, 87 | '' AS reverse_auth_info, 88 | sp_http_proxy_port AS http_proxy_port, 89 | sp_http_auth_info AS http_auth_info, 90 | sp_socks5_proxy_port AS socks5_proxy_port, 91 | sp_socks5_auth_info AS socks5_auth_info, 92 | sp_access_info AS access_info, 93 | sp_status AS status, 94 | is_deleted AS is_deleted, 95 | 5 AS level 96 | FROM 97 | sa_proxy 98 | WHERE 99 | 1 = 1 100 | UNION 101 | SELECT 102 | sps_ip AS host, 103 | sps_ip AS out_ip, 104 | sps_location AS location, 105 | sps_reverse_proxy_port AS reverse_proxy_port, 106 | '' AS reverse_auth_info, 107 | sps_http_proxy_port AS http_proxy_port, 108 | sps_http_auth_info AS http_auth_info, 109 | sps_socks5_proxy_port AS socks5_proxy_port, 110 | sps_socks5_auth_info AS socks5_auth_info, 111 | sps_access_info AS access_info, 112 | sps_status AS status, 113 | is_deleted AS is_deleted, 114 | 8 AS level 115 | FROM 116 | sa_proxy_server 117 | WHERE 118 | 1 = 1 119 | ) proxy 120 | WHERE 121 | 1 = 1 122 | AND status = 1 123 | AND NOT is_deleted 124 | """ 125 | # 如果有地区限制，多加一行搜索条件 126 | if (location): 127 | select_filter = """ 128 | AND proxy.location IS NOT NULL 129 | AND proxy.location LIKE '%{}%' 130 | """.format(location) 131 | select_sql += select_filter 132 | # 搜索 133 | select_result = r_pgsql_driver.select(select_sql) 134 | # 乱序搜索结果 135 | random.shuffle(select_result) 136 | # 将搜索出的数据转为代理字典 137 | for row in select_result: 138 | for proxy_method in proxy_infos.keys(): 139 | # 如果端口有服务 140 | if (row[proxy_method+"_proxy_port"]): 141 | # 出口 IP 作为主键 142 | out_ip = row["out_ip"] 143 | # 代理信息 144 | proxy_info = { 145 | "host": row["host"], 146 | "port": row[proxy_method+"_proxy_port"], 147 | "auth": row[proxy_method+"_auth_info"], 148 | "access": row["access_info"], 149 | "level": row["level"] 150 | } 151 | proxy_infos[proxy_method][out_ip] = proxy_info 152 | # 端口无服务说明此代理不存在 153 | else: 154 | pass 155 | return proxy_infos 156 | 157 | """ 158 | @description: 初始化代理使用次数 159 | ------- 160 | @param: 161 | ------- 162 | @return: 163 | """ 164 | def init_ip_usage_counts(proxy_infos): 165 | ip_usage_counts = {} 166 | for proxy_method in proxy_infos.keys(): 167 | for out_ip in proxy_infos[proxy_method]: 168 | for web in proxy_infos[proxy_method][out_ip]["access"].keys(): 169 | # 如果还不存在这个网站对应的列表则进行新建 170 | if (web not in ip_usage_counts.keys()): 171 | ip_usage_counts[web] = {} 172 | # 如果可以访问，从 0 开始计数 173 | if (proxy_infos[proxy_method][out_ip]["access"][web]): 174 | ip_usage_counts[web][out_ip] = 0 175 | return ip_usage_counts 176 | 177 | """ 178 | @description: 根据代理信息和协议将代理可用化 179 | ------- 180 | @param: 181 | ------- 182 | @return: 183 | """ 184 | def parse_proxy_info(proxy_method, proxy_info): 185 | # 反代 186 | if (proxy_method == "reverse"): 187 | return "http://{host}:{port}".format(**proxy_info) 188 | # HTTP 代理 189 | elif(proxy_method == "http"): 190 | return { 191 | "http": "http://{auth}@{host}:{port}".format(**proxy_info), 192 | "https": "http://{auth}@{host}:{port}".format(**proxy_info) 193 | } 194 | # SOCKS5 代理 195 | elif(proxy_method == "socks5"): 196 | return { 197 | "http": "socks5h://{auth}@{host}:{port}".format(**proxy_info), 198 | "https": "socks5h://{auth}@{host}:{port}".format(**proxy_info) 199 | } 200 | 201 | 202 | """ 203 | @description: r_proxy 类 204 | ------- 205 | @param: 206 | ------- 207 | @return: 208 | """ 209 | class r_proxy(): 210 | 211 | """ 212 | @description: 初始化 213 | ------- 214 | @param: 215 | ------- 216 | @return: 217 | """ 218 | def __init__(self, r_pgsql_driver, location=None, level=None): 219 | # 数据库连接 220 | self.r_pgsql_driver = r_pgsql_driver 221 | # 所有代理信息 222 | self.proxy_infos = get_proxy_infos(r_pgsql_driver, location, level) 223 | # 各个网站中各个代理的访问次数 224 | self.ip_usage_counts = init_ip_usage_counts(self.proxy_infos) 225 | 226 | """ 227 | @description: 获取一个可用代理 228 | ------- 229 | @param: 230 | ------- 231 | @return: 232 | """ 233 | def get(self, web, accessable_webs=[], proxy_method="socks5"): 234 | # 测试所有需要验证的网站是否都有访问可行性数据 235 | all_webs = [web] 236 | all_webs.extend(accessable_webs) 237 | for web in all_webs: 238 | if (web not in self.ip_usage_counts.keys()): 239 | r_logger.warn( 240 | "暂时没有针对网站：{} 的代理访问可行性测试！".format(web)) 241 | return None 242 | # 筛选出的 IP 243 | filtered_ips = list(self.ip_usage_counts[web].keys()) 244 | for accessable_web in accessable_webs: 245 | filtered_ips = [ip for ip in filtered_ips if ip in list( 246 | self.ip_usage_counts[accessable_web].keys())] 247 | filtered_ip_usage_counts = { 248 | ip:self.ip_usage_counts[web][ip] for ip in filtered_ips} 249 | # 取出使用次数最少的代理 250 | least_used_ip = min( 251 | filtered_ip_usage_counts, key=filtered_ip_usage_counts.get) 252 | # 将使用次数加 1 253 | self.ip_usage_counts[web][least_used_ip] += 1 254 | # 将这个代理的所有信息根据传入协议转换为直接可用的代理 255 | serviceable_proxy = parse_proxy_info( 256 | proxy_method, self.proxy_infos[proxy_method][least_used_ip]) 257 | return serviceable_proxy 258 | 259 | """ 260 | @description: Ban 掉无法使用的 IP 261 | ------- 262 | @param: 263 | ------- 264 | @return: 265 | """ 266 | def ban(self, out_ip, web=None): 267 | # 如果不传入网站，则说明节点已经不通，直接去除 268 | if (web and web in self.ip_usage_counts.keys()): 269 | if (out_ip in self.ip_usage_counts[web].keys()): 270 | del self.ip_usage_counts[web][out_ip] 271 | else: 272 | pass 273 | # 如果不传入网站，则说明节点已经不通，直接去除 274 | else: 275 | for web in self.ip_usage_counts.keys(): 276 | if (out_ip in self.ip_usage_counts[web].keys()): 277 | del self.ip_usage_counts[web][out_ip] 278 | else: 279 | pass 280 | 281 | """ 282 | @description: 代理访问网站可行性的临时测试 283 | ------- 284 | @param: 285 | ------- 286 | @return: 287 | """ 288 | def temporary_test_access(self, 289 | web, 290 | test_url, 291 | num_2_test=None, 292 | proxy_method="socks5"): 293 | r_logger.info( 294 | "代理对站点的临时访问测试开始！站点名：{web} 地址：{test_url}".format( 295 | web=web, test_url=test_url)) 296 | # 新建这个网站的代理使用次数统计 297 | self.ip_usage_counts[web] = {} 298 | # 如果没有指定测试代理数，默认从代理池中取出一半的代理进行测试，最多选出 10 个 299 | if (not num_2_test 300 | and len(list(self.proxy_infos[proxy_method].keys())) <= 20): 301 | num_2_test = len(list(self.proxy_infos[proxy_method].keys())) // 2 302 | elif(not num_2_test): 303 | num_2_test = 10 304 | r_logger.info("总测试代理数：{}".format(str(num_2_test))) 305 | # 遍历这些代理 306 | for out_ip in random.sample(list( 307 | self.proxy_infos[proxy_method].keys()), num_2_test): 308 | # 如果测试访问通过，记使用次数 1 次 309 | if (rab_ip.test(test_url, proxies=parse_proxy_info( 310 | proxy_method, self.proxy_infos[proxy_method][out_ip]))): 311 | self.ip_usage_counts[web][out_ip] = 1 312 | # 最后如果测试完这个网站无可访问节点，则直接删除次数统计 313 | if (not self.ip_usage_counts[web]): 314 | del self.ip_usage_counts[web] 315 | r_logger.info("测试完成，无可访问 {web} 的代理。".format(web=web)) 316 | else: 317 | r_logger.info("测试完成！访问 {web} 可使用代理的个数：{num}".format( 318 | web=web, num=str(len(self.ip_usage_counts[web].keys())))) 319 | 320 | 321 | """ 322 | @description: 单体测试 323 | ------- 324 | @param: 325 | ------- 326 | @return: 327 | """ 328 | if __name__ == "__main__": 329 | r_pgsql_driver = rab_postgresql.r_pgsql_driver(show_column_name=True) 330 | try: 331 | # 配置文件中的代理 332 | print(get_personal_proxy_infos()) 333 | # 数据库中的代理 334 | r_proxy = r_proxy(r_pgsql_driver, location="香港") 335 | for _ in range(0, 10): 336 | print(r_proxy.get("google")) 337 | except Exception as e: 338 | r_logger.error("rab_proxy.py 单体测试出错！") 339 | r_logger.error(e) 340 | finally: 341 | r_pgsql_driver.close() -------------------------------------------------------------------------------- /rab_postgresql.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_postgresql.py 6 | # @DATE: 2020/12/17 周四 7 | # @TIME: 11:03:40 8 | # 9 | # @DESCRIPTION: 共通包 PostgreSQL 数据库驱动 10 | 11 | 12 | import sys 13 | import datetime 14 | import psycopg2 15 | import psycopg2.extras 16 | sys.path.append("..") if (".." not in sys.path) else True 17 | from rab_python_packages import rab_config 18 | from rab_python_packages import rab_logging 19 | from rab_python_packages import rab_ip 20 | 21 | 22 | # 日志记录 23 | r_logger = rab_logging.r_logger() 24 | 25 | 26 | """ 27 | @description: 切换数据库装饰器 28 | ------- 29 | @param: 30 | ------- 31 | @return: 32 | """ 33 | def change_database(database): 34 | def wrapper(func): 35 | def _wrapper(*args, **kwargs): 36 | func_result = None 37 | # 需要切换数据库的连接 38 | r_pgsql_drivers = {} 39 | before_database = {} 40 | # 获取入参的所有数据库连接 41 | for i, arg in enumerate(args): 42 | if (type(arg).__name__ == "r_pgsql_driver"): 43 | r_pgsql_drivers[str(i)] = arg 44 | before_database[str(i)] = arg.database 45 | if (not r_pgsql_drivers): 46 | r_logger.warn("{} 方法并未传入数据库连接对象！".format( 47 | str(func.__name__))) 48 | # 运行方法 49 | func_result = func(*args, **kwargs) 50 | else: 51 | # 所有连接均切换至目标数据库 52 | for i in r_pgsql_drivers.keys(): 53 | r_pgsql_driver = r_pgsql_drivers[i] 54 | r_logger.info( 55 | "{f_qname} {f_name} 方法 {i} 号数据库 {b_db} -> {db}！" \ 56 | .format(f_qname=str(func.__qualname__), 57 | f_name=str(func.__name__), 58 | i=i, b_db=before_database[i], db=database)) 59 | r_pgsql_driver.change_database(database) 60 | # 运行方法 61 | func_result = func(*args, **kwargs) 62 | # 所有连接均切换回原数据库 63 | for i in r_pgsql_drivers.keys(): 64 | r_pgsql_driver = r_pgsql_drivers[i] 65 | r_logger.info( 66 | "{f_qname} {f_name} 方法 {i} 号数据库 {b_db} <- {db}！" \ 67 | .format(f_qname=str(func.__qualname__), 68 | f_name=str(func.__name__), 69 | i=i, b_db=before_database[i], db=database)) 70 | r_pgsql_driver.change_database(before_database[i]) 71 | return func_result 72 | return _wrapper 73 | return wrapper 74 | 75 | 76 | """ 77 | @description: 获取批量插入数值 78 | ------- 79 | @param: 80 | ------- 81 | @return: batch_size 82 | """ 83 | def get_batch_size(): 84 | return int(rab_config.load_package_config( 85 | "rab_config.ini", "rab_postgresql", "batch_size")) 86 | 87 | 88 | """ 89 | @description: r_pgsql_user 类 90 | ------- 91 | @param: 92 | ------- 93 | @return: 94 | """ 95 | class r_pgsql_user(): 96 | 97 | """ 98 | @description: 初始化 99 | ------- 100 | @param: 101 | ------- 102 | @return: 103 | """ 104 | def __init__(self, user=None, time=None, ip=None): 105 | # 用户名 106 | self.user = user 107 | self.user = self.get_user() 108 | # 更新时间 109 | self.time = None 110 | # IP 111 | self.ip = rab_ip.get_ip_info()["ip"] if not ip else ip 112 | 113 | """ 114 | @description: 获取用户 115 | ------- 116 | @param: 117 | ------- 118 | @return: 119 | """ 120 | def get_user(self): 121 | # 如果当前没有用户名 122 | if (not self.user): 123 | # 获得数据库用户 124 | try: 125 | if (rab_config.load_package_config( 126 | "rab_config.ini", "common", "user")): 127 | self.user = str(rab_config.load_package_config( 128 | "rab_config.ini", "common", "user")) 129 | else: 130 | self.user = "unknown_pgsql_user" 131 | except Exception as e: 132 | self.user = "unknown_pgsql_user" 133 | # 如果当前没有节点备注 134 | if ("(" not in self.user): 135 | # 节点名作为附加 136 | try: 137 | if (rab_config.load_package_config("rab_config.ini", \ 138 | "rab_distributed_system", "node_id")): 139 | self.user += " ({})".format(str( 140 | rab_config.load_package_config("rab_config.ini", \ 141 | "rab_distributed_system", "node_id"))) 142 | except Exception as e: 143 | pass 144 | return self.user 145 | 146 | """ 147 | @description: 获取当前时间 148 | ------- 149 | @param: 150 | ------- 151 | @return: 152 | """ 153 | def get_time(self): 154 | return datetime.datetime.now(datetime.timezone.utc) 155 | 156 | """ 157 | @description: 获取 IP 158 | ------- 159 | @param: 160 | ------- 161 | @return: 162 | """ 163 | def get_ip(self): 164 | return self.ip 165 | 166 | 167 | """ 168 | @description: r_pgsql_driver 类 169 | ------- 170 | @param: 171 | ------- 172 | @return: 173 | """ 174 | class r_pgsql_driver(): 175 | 176 | """ 177 | @description: 初始化 178 | ------- 179 | @param: 180 | ------- 181 | @return: 182 | """ 183 | def __init__(self, 184 | database=rab_config.load_package_config( 185 | "rab_config.ini", "common", "database"), 186 | user=rab_config.load_package_config( 187 | "rab_config.ini", "common", "user"), 188 | password=rab_config.load_package_config( 189 | "rab_config.ini", "common", "password"), 190 | host=rab_config.load_package_config( 191 | "rab_config.ini", "common", "host"), 192 | port=rab_config.load_package_config( 193 | "rab_config.ini", "common", "port"), 194 | show_column_name=False): 195 | # 数据库名 196 | self.database = database 197 | # 用户 198 | self.user = user 199 | # 密码 200 | self.password = password 201 | # IP 或域名 202 | self.host = host 203 | # 端口 204 | self.port = port 205 | # 是否显示列名 206 | self.show_column_name = show_column_name 207 | # 批量插入数 208 | self.batch_size = int(rab_config.load_package_config( 209 | "rab_config.ini", "rab_postgresql", "batch_size")) 210 | # 指针 211 | self.cur = None 212 | # 与数据库之间的连接 213 | self.conn = None 214 | # 用户 215 | self.r_pgsql_user = r_pgsql_user(user=self.user) 216 | 217 | """ 218 | @description: 建立数据库连接 219 | ------- 220 | @param: 221 | ------- 222 | @return: 223 | """ 224 | def connect(self): 225 | # 创建连接对象 226 | self.conn = psycopg2.connect(database=self.database, 227 | user=self.user, 228 | password=self.password, 229 | host=self.host, 230 | port=self.port) 231 | #创建指针对象 232 | if (self.show_column_name): 233 | self.cur = self.conn.cursor( 234 | cursor_factory=psycopg2.extras.RealDictCursor) 235 | else: 236 | self.cur = self.conn.cursor() 237 | 238 | """ 239 | @description: 测试数据库连接是否可用 240 | ------- 241 | @param: 242 | ------- 243 | @return: bool 244 | """ 245 | def test(self): 246 | test_sql = "SELECT 1;" 247 | try: 248 | self.cur.execute(test_sql) 249 | result_list = self.cur.fetchall() 250 | if (result_list): 251 | return True 252 | except Exception: 253 | pass 254 | return False 255 | 256 | """ 257 | @description: 关闭数据库连接 258 | ------- 259 | @param: 260 | ------- 261 | @return: 262 | """ 263 | def close(self): 264 | if (self.cur): 265 | self.cur.close() 266 | if (self.conn): 267 | self.conn.close() 268 | self.cur = None 269 | self.conn = None 270 | 271 | """ 272 | @description: 数据库重连 273 | ------- 274 | @param: 275 | ------- 276 | @return: 277 | """ 278 | def reconnect(self): 279 | self.close() 280 | self.connect() 281 | 282 | """ 283 | @description: 切换数据库 284 | ------- 285 | @param: 286 | ------- 287 | @return: 288 | """ 289 | def change_database(self, database): 290 | select_result = self.select("SELECT current_database();") 291 | if (self.show_column_name): 292 | current_database = select_result[0]["current_database"] 293 | else: 294 | current_database = select_result[0][0] 295 | r_logger.info("当前数据库：{current_database}，目标数据库：{database}" \ 296 | .format(current_database=current_database, database=database)) 297 | if (current_database.strip().lower() == database.strip().lower()): 298 | r_logger.info("无需切换数据库！") 299 | else: 300 | self.close() 301 | self.database = database 302 | self.connect() 303 | r_logger.info("已经切换至数据库 {}！".format(database)) 304 | 305 | """ 306 | @description: 查询语句 307 | ------- 308 | @param: sql, data 309 | ------- 310 | @return: 311 | """ 312 | def select(self, sql, data=None): 313 | # 测试当前连接是否可用，不可用则重连 314 | if (not self.test()): 315 | self.reconnect() 316 | try: 317 | self.cur.execute(sql, data) 318 | select_result = self.cur.fetchall() 319 | r_logger.info("查询成功！") 320 | r_logger.info("SQL 文：{}".format(str(sql))) 321 | r_logger.info("参数：{}".format(str(data))) if data else False 322 | if (len(str(select_result)) > 120): 323 | r_logger.info("结果：{select_result}......行数{row_num}".format( 324 | select_result=str(select_result)[0:100], row_num=str( 325 | len(select_result)))) 326 | else: 327 | r_logger.info("结果：{}".format(str(select_result))) 328 | return select_result 329 | except Exception as e: 330 | r_logger.error("查询失败！") 331 | r_logger.error(e) 332 | r_logger.error("SQL 文：{}".format(str(sql))) 333 | r_logger.error("参数：{}".format(str(data))) if data else False 334 | return [] 335 | 336 | """ 337 | @description: 执行单一 SQL 语句 338 | ------- 339 | @param: 340 | ------- 341 | @return: 342 | """ 343 | def execute(self, sql, data=None): 344 | # 测试当前连接是否可用，不可用则重连 345 | if (not self.test()): 346 | self.reconnect() 347 | try: 348 | self.cur.execute(sql, data) 349 | self.conn.commit() 350 | r_logger.info("SQL 执行成功！") 351 | r_logger.info("SQL 文：{}".format(str(sql))) 352 | r_logger.info("参数：{}".format(str(data))) if data else False 353 | return True 354 | except Exception as e: 355 | r_logger.error("SQL 执行失败！") 356 | r_logger.error(e) 357 | r_logger.error("SQL 文：{}".format(str(sql))) 358 | r_logger.info("参数：{}".format(str(data))) if data else False 359 | return False 360 | 361 | """ 362 | @description: 插入语句 363 | ------- 364 | @param: sql, data 365 | ------- 366 | @return: 367 | """ 368 | def insert(self, sql, data=None): 369 | return self.execute(sql, data) 370 | 371 | """ 372 | @description: 更新语句 373 | ------- 374 | @param: sql, data 375 | ------- 376 | @return: 377 | """ 378 | def update(self, sql, data=None): 379 | return self.execute(sql, data) 380 | 381 | """ 382 | @description: 删除语句 383 | ------- 384 | @param: sql, data 385 | ------- 386 | @return: 387 | """ 388 | def delete(self, sql, data=None): 389 | return self.execute(sql, data) 390 | 391 | """ 392 | @description: 清空表 393 | ------- 394 | @param: table_name 395 | ------- 396 | @return: 397 | """ 398 | def delete_all(self, table): 399 | sql = "DELETE FROM {}".format(table) 400 | return self.execute(sql) 401 | 402 | """ 403 | @description: 根据提供的 SQL 语句处理多条数据 404 | ------- 405 | @param: sql, data 406 | ------- 407 | @return: 408 | """ 409 | def execute_many(self, sql, data): 410 | # 测试当前连接是否可用，不可用则重连 411 | if (not self.test()): 412 | self.reconnect() 413 | try: 414 | # executemany 因为效率问题放弃 415 | # cur.executemany(sql, data) 416 | # 改为 execute_batch 417 | psycopg2.extras.execute_batch( 418 | self.cur, sql, data, page_size=self.batch_size) 419 | self.conn.commit() 420 | r_logger.info("数据插入或更新成功！") 421 | r_logger.info("SQL 文：{}".format(str(sql))) 422 | r_logger.info("数据：{data}......行数：{row_num}".format( 423 | data=str(data)[0:40], row_num=str(len(data)))) 424 | return True 425 | except Exception as e: 426 | r_logger.error("数据插入或更新失败！") 427 | r_logger.error(e) 428 | r_logger.error("SQL 文：{}".format(str(sql))) 429 | r_logger.error("数据：{data}......行数：{row_num}".format( 430 | data=str(data), row_num=str(len(data)))) 431 | return False 432 | 433 | """ 434 | @description: 获取用户信息列表 435 | ------- 436 | @param: 437 | ------- 438 | @return: 439 | """ 440 | def get_user_info(self): 441 | return [ 442 | self.r_pgsql_user.get_user(), 443 | self.r_pgsql_user.get_time(), 444 | self.r_pgsql_user.get_ip() 445 | ] 446 | 447 | 448 | """ 449 | @description: 单体测试 450 | ------- 451 | @param: 452 | ------- 453 | @return: 454 | """ 455 | if __name__ == "__main__": 456 | r_pgsql_driver = r_pgsql_driver(show_column_name=True) 457 | print(r_pgsql_driver.user) 458 | select_sql = "SELECT 1;" 459 | result = r_pgsql_driver.select(select_sql) 460 | for row in result: 461 | print(row) 462 | # print(r_pgsql_driver.r_pgsql_user.get_user()) 463 | # print(r_pgsql_driver.r_pgsql_user.get_time()) 464 | # print(r_pgsql_driver.r_pgsql_user.get_ip()) 465 | r_pgsql_driver.close() -------------------------------------------------------------------------------- /rab_node.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/sub-2-proxy/rab_python_packages/rab_node.py 6 | # @DATE: 2021/11/05 Fri 7 | # @TIME: 14:37:57 8 | # 9 | # @DESCRIPTION: 共通包 Linux 系统下解析订阅的节点 10 | 11 | 12 | import re 13 | import sys 14 | import copy 15 | import json 16 | import yaml 17 | import requests 18 | import urllib.parse 19 | sys.path.append("..") if (".." not in sys.path) else True 20 | from rab_python_packages import rab_config 21 | from rab_python_packages import rab_logging 22 | from rab_python_packages import rab_cryptography 23 | 24 | 25 | # 日志记录 26 | r_logger = rab_logging.r_logger() 27 | 28 | 29 | class Node(): 30 | """ 31 | 代理类 32 | """ 33 | 34 | def __init__(self, 35 | kind, 36 | origin_info, 37 | subscription_id): 38 | """ 39 | 初始化 40 | """ 41 | self.kind = kind 42 | self.protocol = None 43 | # 节点原始信息：VPN 协议为文件内容，机场订阅为 Base64 解码前的节点信息 44 | self.origin_info = origin_info 45 | # 节点信息 46 | self.info = None 47 | # 所属订阅组 48 | self.subscription_id = subscription_id 49 | # 订阅所在服务器 ID 50 | self.server_id = None 51 | # 订阅所在的容器 ID 52 | self.container_id = None 53 | # 是否已经转变为代理 54 | self.is_become_proxy = 0 55 | 56 | """ 57 | @description: 获取指定参数的值 58 | ------- 59 | @param: 60 | ------- 61 | @return: 62 | """ 63 | def get_param_value(param_key, params_str, delimiter="&"): 64 | # 使用指定的分割符对字符串进行分割 65 | param_key_values = params_str.split(delimiter) 66 | for param_key_value in param_key_values: 67 | # 如果参数键相等 68 | if (param_key_value.split("=")[0] == param_key): 69 | return param_key_value.split("=")[1] 70 | # 没有对应参数则返回空 71 | return None 72 | 73 | """ 74 | @description: 解析 HTTP 或 SOCKS5 节点信息（暂无 HTTPS 支持） 75 | ------- 76 | @param: 77 | ------- 78 | @return: 79 | """ 80 | def parse_http_or_socks5_node_url(node_url): 81 | node_info = {} 82 | # === 必要参数 === 83 | if (node_url.lower().startswith("http")): 84 | # HTTP 协议 85 | node_info["type"] = "http" 86 | else: 87 | # SOCKS5 协议 88 | node_info["type"] = "socks5" 89 | node_info_str = node_url.split("://")[1] 90 | # 服务器 IP 或解析域名 91 | if ("@" in node_url): 92 | node_info["server"] = node_info_str.split("@")[1].split(":")[0] 93 | else: 94 | node_info["server"] = node_info_str.split(":")[0] 95 | # 服务器端口 96 | if ("@" in node_url): 97 | node_info["port"] = node_info_str.split("@")[1].split(":")[1] 98 | else: 99 | node_info["port"] = node_info_str.split(":")[1] 100 | # === 进阶参数 === 101 | # 用户名 102 | if ("@" in node_url): 103 | node_info["username"] = node_info_str.split("@")[0].split(":")[0] 104 | else: 105 | node_info["username"] = None 106 | # 密码 107 | if ("@" in node_url): 108 | node_info["password"] = node_info_str.split("@")[0].split(":")[1] 109 | else: 110 | node_info["password"] = None 111 | return node_info 112 | 113 | """ 114 | @description: 解析 SS 节点信息 115 | ------- 116 | @param: 117 | ------- 118 | @return: 119 | """ 120 | def parse_ss_node_url(node_url): 121 | node_info = {} 122 | # === 必要参数 === 123 | # SS 协议 124 | node_info["type"] = "ss" 125 | node_info_str = node_url.replace("ss://", "", 1) 126 | # 部分解码（只有 @ 前的部分需要 Base64 解码） 127 | node_info_str_4_b64decode = node_info_str.split("@")[0] 128 | try: 129 | node_info_str_after_b64decode = rab_cryptography.b64decode( 130 | node_info_str_4_b64decode).decode("UTF-8") 131 | except Exception: 132 | r_logger.error("SS 节点链接部分 Base64 解码失败：{}".format( 133 | node_info_str_4_b64decode)) 134 | return None 135 | # 节点名 136 | node_info["name"] = urllib.parse.unquote( 137 | node_info_str.split("@")[1].split("#")[1]).rstrip("\n") 138 | # 服务器 IP 或解析域名 139 | node_info["server"] = node_info_str.split("@")[1].split("#")[0].split(":")[0] 140 | # 服务器端口 141 | node_info["port"] = node_info_str.split("@")[1].split("#")[0].split(":")[1] 142 | # 加密方式 143 | node_info["cipher"] = node_info_str_after_b64decode.split(":")[0] 144 | # 密码 145 | node_info["password"] = node_info_str_after_b64decode.split(":")[1] 146 | # UDP（默认关闭） 147 | node_info["udp"] = "false" 148 | # === 进阶参数 === 149 | # 混淆插件 plugin 150 | # 暂时未发现有机场配置该参数，无法验证以下方法的正确性，因此注释掉 151 | # if ("plugin=" in node_info_str): 152 | # node_info["plugin"] = get_param_value("plugin", node_info_str) 153 | # # 混淆参数 154 | # node_info["plugin-opts"] = {} 155 | # # 混淆模式 156 | # if ("obfs=" in node_info_str): 157 | # node_info["plugin-opts"]["mode"] = get_param_value( 158 | # "obfs", node_info_str) 159 | # # 混淆所用域名 160 | # if ("obfs-host=" in node_info_str): 161 | # node_info["plugin-opts"]["host"] = get_param_value( 162 | # "obfs-host", node_info_str) 163 | # 返回节点信息 164 | return node_info 165 | 166 | """ 167 | @description: 解析 SSR 节点信息 168 | ------- 169 | @param: 170 | ------- 171 | @return: 172 | """ 173 | def parse_ssr_node_url(node_url): 174 | node_info = {} 175 | # === 必要参数 === 176 | # SSR 协议 177 | node_info["type"] = "ssr" 178 | node_info_str = node_url.replace("ssr://", "", 1) 179 | # 全部解码 180 | try: 181 | node_info_str = rab_cryptography.b64decode( 182 | node_info_str).decode("UTF-8") 183 | except Exception: 184 | r_logger.error("SSR 节点链接 Base64 解码失败：{}".format(node_info_str)) 185 | return None 186 | # 节点名 187 | node_info["name"] = rab_cryptography.b64decode( 188 | node_info_str.split("/?")[1].split("&")[2].split("=")[1]).decode("UTF-8") 189 | # 服务器 IP 或解析域名 190 | node_info["server"] = node_info_str.split("/?")[0].split(":")[0] 191 | # 服务器端口 192 | node_info["port"] = node_info_str.split("/?")[0].split(":")[1] 193 | # 加密方式 194 | node_info["cipher"] = node_info_str.split("/?")[0].split(":")[3] 195 | # 密码 196 | node_info["password"] = rab_cryptography.b64decode( 197 | node_info_str.split("/?")[0].split(":")[5]).decode("UTF-8") 198 | # 混淆插件 199 | node_info["obfs"] = node_info_str.split("/?")[0].split(":")[4] 200 | # 协议 201 | node_info["protocol"] = node_info_str.split("/?")[0].split(":")[2] 202 | # UDP（默认关闭） 203 | node_info["udp"] = "false" 204 | # === 进阶参数 === 205 | # 混淆参数 206 | node_info["obfs-param"] = rab_cryptography.b64decode( 207 | node_info_str.split("/?")[1].split("&")[0].split("=")[1]).decode("UTF-8") 208 | # 协议参数 209 | node_info["protocol"] = node_info_str.split("/?")[0].split(":")[2] 210 | # === 其他参数 === 211 | # 节点备注 212 | node_info["remarks"] = rab_cryptography.b64decode( 213 | node_info_str.split("/?")[1].split("&")[2].split("=")[1]).decode("UTF-8") 214 | # 节点分组 215 | node_info["group"] = rab_cryptography.b64decode( 216 | node_info_str.split("/?")[1].split("&")[3].split("=")[1]).decode("UTF-8") 217 | # 返回节点信息 218 | return node_info 219 | 220 | """ 221 | @description: 解析非标准 JSON 格式的 VMess 节点信息 222 | ------- 223 | @param: 224 | ------- 225 | @return: 226 | """ 227 | def parse_vmess_node_url_ex(node_info_str): 228 | node_info = {} 229 | # VMess 协议 230 | node_info["type"] = "vmess" 231 | # 如果形如 auto:aaaa-bbbb-cccc-dddd@207.46.123.123:12345 样式 232 | search_obj = re.search(u"(.*):(.*)@(.*):(.*)", node_info_str) 233 | if (search_obj): 234 | # 节点名 235 | node_info["name"] = node_info_str.split("@")[1].replace(":", "_") 236 | # 服务器 IP 或解析域名 237 | node_info["server"] = node_info_str.split("@")[1].split(":")[0] 238 | # 服务器端口 239 | node_info["port"] = node_info_str.split("@")[1].split(":")[1] 240 | # 用户 ID 241 | node_info["uuid"] = node_info_str.split("@")[0].split(":")[1] 242 | # 每个请求随机附带的数值的上限（防止同时间戳的请求混杂在一起） 243 | node_info["alterId"] = 0 244 | # 加密方式 245 | node_info["cipher"] = node_info_str.split("@")[0].split(":")[0] 246 | return node_info 247 | # 如果形如 Base64乱码?tfo=1&remark=备注&alterId=0&obfs=websocket 样式 248 | print(node_info_str) 249 | search_obj = re.search(u"(.*)?(.*)", node_info_str) 250 | if (search_obj): 251 | node_info_str_4_b64decode = node_info_str.split("?")[0] 252 | try: 253 | node_info_str_after_b64decode = rab_cryptography.b64decode( 254 | node_info_str_4_b64decode).decode("UTF-8") 255 | except Exception as e: 256 | r_logger.error("VMess 节点链接 Base64 解码失败：{}".format( 257 | node_info_str_4_b64decode)) 258 | return None 259 | # 节点名（如果节点有 remark 参数就使用它的值，否则就自行拼接） 260 | if (get_param_value("remark", node_info_str.split("?")[1], "&")): 261 | node_info["name"] = get_param_value( 262 | "remark", node_info_str.split("?")[1], "&") 263 | else: 264 | node_info["name"] = node_info_str_after_b64decode.split( 265 | "@")[1].replace(":", "_") 266 | # 服务器 IP 或解析域名 267 | node_info["server"] = node_info_str_after_b64decode.split( 268 | "@")[1].split(":")[0] 269 | # 服务器端口 270 | node_info["port"] = node_info_str_after_b64decode.split( 271 | "@")[1].split(":")[1] 272 | # 用户 ID 273 | node_info["uuid"] = node_info_str_after_b64decode.split( 274 | "@")[0].split(":")[1] 275 | # 每个请求随机附带的数值的上限（防止同时间戳的请求混杂在一起） 276 | #（如果节点有 remark 参数就使用它的值，否则就自行拼接） 277 | if (get_param_value("alterId", node_info_str.split("?")[1], "&")): 278 | node_info["alterId"] = get_param_value( 279 | "alterId", node_info_str.split("?")[1], "&") 280 | else: 281 | node_info["alterId"] = 0 282 | # 加密方式 283 | node_info["cipher"] = node_info_str_after_b64decode.split( 284 | "@")[0].split(":")[0] 285 | return node_info 286 | # 均无匹配则返回空 287 | return None 288 | 289 | """ 290 | @description: 解析 VMess 节点信息 291 | ------- 292 | @param: 293 | ------- 294 | @return: 295 | """ 296 | def parse_vmess_node_url(node_url): 297 | node_info = {} 298 | # === 必要参数 === 299 | # VMess 协议 300 | node_info["type"] = "vmess" 301 | node_info_str = node_url.replace("vmess://", "", 1) 302 | # 全部解码 303 | try: 304 | node_info_str = rab_cryptography.b64decode( 305 | node_info_str).decode("UTF-8") 306 | node_info_json = json.loads(node_info_str) 307 | except Exception: 308 | # 尝试用非标准格式解析 309 | node_info = parse_vmess_node_url_ex(node_info_str) 310 | if (node_info and "name" in node_info.keys() and node_info["name"]): 311 | return node_info 312 | else: 313 | r_logger.error( 314 | "VMess 节点链接 Base64 解码失败：{}".format(node_info_str)) 315 | return None 316 | # 节点名 317 | node_info["name"] = u"{}".format(node_info_json["ps"]) 318 | # 服务器 IP 或解析域名 319 | node_info["server"] = node_info_json["add"] 320 | # 服务器端口 321 | node_info["port"] = node_info_json["port"] 322 | # 用户 ID 323 | node_info["uuid"] = node_info_json["id"] 324 | # 每个请求随机附带的数值的上限（防止同时间戳的请求混杂在一起） 325 | node_info["alterId"] = node_info_json["aid"] 326 | # 加密方式 327 | node_info["cipher"] = node_info_json["cipher"] if "cipher" in \ 328 | node_info_json.keys() else "auto" 329 | # === 进阶参数 === 330 | # UDP（默认关闭） 331 | node_info["udp"] = "false" 332 | # TLS（HTTPS） 333 | if ("tls" in node_info_json.keys() and node_info_json["tls"]): 334 | node_info["tls"] = "true" 335 | # 跳过 HTTPS 证书的认证 336 | node_info["skip-cert-verify"] = "false" if "verify_cert" in \ 337 | node_info_json.keys() and node_info_json["verify_cert"] else "true" 338 | # 指定 HTTPS 所通信的域名（应对同一服务上有多个 HTTPS 域名） 339 | node_info["servername"] = node_info_json["servername"] if "sni" in \ 340 | node_info_json.keys() and node_info_json["sni"] else "" 341 | else: 342 | node_info["tls"] = "false" 343 | # 连接方式（默认为 tcp） 344 | node_info["network"] = node_info_json["net"] if "net" in \ 345 | node_info_json.keys() and node_info_json["net"] else "tcp" 346 | # WebSocket 连接模式 347 | if ("net" in node_info_json.keys() and node_info_json["net"] == "ws"): 348 | node_info["network"] = "ws" 349 | # WebSocket 连接参数 350 | node_info["ws-opts"] = {} 351 | # ws 路径 352 | node_info["ws-opts"]["path"] = node_info_json["path"] 353 | # ws 请求头 354 | if ("headerType" in node_info_json.keys() 355 | and str(node_info_json["headerType"]).lower() != "none"): 356 | node_info["ws-opts"]["headers"] = {} 357 | # 暂时未发现有机场限制请求头 358 | # todo... 359 | # ws max-early-data 360 | # 暂时未发现有机场配置该参数 361 | # todo... 362 | # ws early-data-header-name 363 | # 暂时未发现有机场配置该参数 364 | # todo... 365 | # 其他 TCP 模式均默认为 HTTP 连接模式 366 | else: 367 | node_info["network"] = "http" 368 | # 返回节点信息 369 | return node_info 370 | 371 | """ 372 | @description: 使用 subconverter 进行解析 373 | ------- 374 | @param: 375 | ------- 376 | @return: 377 | """ 378 | def parse_node_url_by_subconverter(node_url, subconverter_url): 379 | # 判断转换地址是否带了 /sub，不带则添加 380 | if (not subconverter_url.endswith("/sub")): 381 | if (subconverter_url.endswith("/")): 382 | subconverter_url += "sub" 383 | else: 384 | subconverter_url += "/sub" 385 | # 参数 386 | params = { 387 | "target": "clash", 388 | # 节点链接作为参数时会自动经过 URLEncode 编码 389 | "url": node_url, 390 | "insert": "false", 391 | "emoji": "false", 392 | "list": "true", 393 | "udp": "false", 394 | "tfo": "false", 395 | "scv": "false", 396 | "fdn": "false", 397 | "sort": "false", 398 | "new_name": "true" 399 | } 400 | try: 401 | response = requests.get(subconverter_url, params=params) 402 | if (response): 403 | # 读取 YAML 格式的配置 404 | proxies = yaml.safe_load(response.text)["proxies"][0] 405 | return proxies 406 | else: 407 | r_logger.warn("{} 无法访问 subconverter 后端以解析节点！" \ 408 | .format(node_url)) 409 | except Exception as e: 410 | r_logger.error("{} 访问 subconverter 后端解析节点出错！".format(node_url)) 411 | r_logger.error(e) 412 | return None 413 | 414 | """ 415 | @description: 解析节点链接以获取节点信息 416 | ------- 417 | @param: 418 | ------- 419 | @return: 420 | """ 421 | def parse_node_url(node_url, parse_method): 422 | # 调用 subconverter 后端解析 423 | if (parse_method == "subconverter"): 424 | # subconverter 后端地址 425 | subconverter_url = rab_config.load_package_config( 426 | "rab_config.ini", "rab_node", "subconverter_url") 427 | return parse_node_url_by_subconverter(node_url, subconverter_url) 428 | # 自行解析 429 | else: 430 | try: 431 | if (node_url.startswith("ss://")): 432 | return parse_ss_node_url(node_url) 433 | elif(node_url.startswith("ssr://")): 434 | return parse_ssr_node_url(node_url) 435 | elif(node_url.startswith("vmess://")): 436 | return parse_vmess_node_url(node_url) 437 | elif(node_url.startswith("trojan://")): 438 | # node = parse_trojan_node_url(node_url) 439 | pass 440 | else: 441 | r_logger.warn("自行解析发现未知协议节点：{}".format(node_url)) 442 | except Exception as e: 443 | r_logger.error("自行解析节点链接出错！") 444 | r_logger.error(e) 445 | return None 446 | 447 | """ 448 | @description: 节点类 449 | ------- 450 | @param: 451 | ------- 452 | @return: 453 | """ 454 | class r_node(): 455 | 456 | """ 457 | @description: 初始化 458 | ------- 459 | @param: 460 | ------- 461 | @return: 462 | """ 463 | def __init__(self, 464 | node_url, 465 | subscription_url=None, 466 | parse_method=rab_config.load_package_config( 467 | "rab_config.ini", "rab_node", "parse_method")): 468 | self.url = node_url 469 | self.subscription_url = subscription_url 470 | self.info = parse_node_url(self.url, parse_method) 471 | self.id = self.generate_node_id() 472 | 473 | """ 474 | @description: 生成节点的 ID 475 | ------- 476 | @param: 477 | ------- 478 | @return: 479 | """ 480 | def generate_node_id(self): 481 | if (self.info): 482 | return "_".join([str(value) for value in self.info.values()]) 483 | else: 484 | return None 485 | 486 | """ 487 | @description: 生成节点配置命令 488 | ------- 489 | @param: 490 | ------- 491 | @return: 492 | """ 493 | def generate_node_configure_command(self, socks_port, client_side="clash"): 494 | if (self.info): 495 | # 获取配置命令模板 496 | node_configure_command_template = rab_config.load_package_config( 497 | "rab_linux_command.ini", "rab_node", \ 498 | "{}_configure".format(client_side)) 499 | # 替换 SOCKS5 端口 500 | node_configure_command = node_configure_command_template.replace( 501 | "{socks-port_4_replace}", str(socks_port)) 502 | # 替换节点信息（将节点名改为 node） 503 | _node_info = copy.deepcopy(self.info) 504 | _node_info["name"] = "node" 505 | node_configure_command = node_configure_command.replace( 506 | "{node-info_4_replace}", str(_node_info)) 507 | return node_configure_command 508 | else: 509 | return None 510 | 511 | 512 | """ 513 | @description: 单体测试 514 | ------- 515 | @param: 516 | ------- 517 | @return: 518 | """ 519 | if __name__ == "__main__": 520 | # 对某协议所有节点链接进行解析 521 | node_urls = [] 522 | with open("node_urls", "r") as f: 523 | for row in f.readlines(): 524 | node_urls.append(row) 525 | for node_url in node_urls: 526 | node = r_node(node_url) 527 | print(node.id, node.info) 528 | print(node.generate_node_configure_command(1080)) 529 | 530 | # 对单个节点链接进行解析 531 | # node_url = "" 532 | # node = r_node(node_url) 533 | # print(node.id, node.info) 534 | -------------------------------------------------------------------------------- /rab_storage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding:UTF-8 -*- 3 | # 4 | # @AUTHOR: Rabbir 5 | # @FILE: /root/GitHub/rab_python_packages/rab_storage.py 6 | # @DATE: 2021/07/22 Thu 7 | # @TIME: 13:55:25 8 | # 9 | # @DESCRIPTION: 静态资源上传下载共通模块 10 | 11 | 12 | import os 13 | import sys 14 | import xml 15 | import json 16 | import minio 17 | import urllib 18 | import requests 19 | sys.path.append("..") if (".." not in sys.path) else True 20 | from rab_python_packages import rab_config 21 | from rab_python_packages import rab_logging 22 | from rab_python_packages import rab_client 23 | 24 | 25 | # 日志记录 26 | r_logger = rab_logging.r_logger() 27 | 28 | 29 | """ 30 | @description: 连接至 MinIO 31 | ------- 32 | @param: 33 | ------- 34 | @return: 35 | """ 36 | def minio_connect(): 37 | # MinIO IP 或域名 38 | minio_host = rab_config.load_package_config( 39 | "rab_config.ini", "rab_storage", "minio_host").lower() 40 | # 检查是否需要使用了 HTTPS 41 | if ("https" in minio_host): 42 | minio_secure = True 43 | minio_host = minio_host.replace("https://", "") 44 | elif("http" in minio_host): 45 | minio_secure = False 46 | minio_host = minio_host.replace("http://", "") 47 | # 读取用户名和密码 48 | minio_user = rab_config.load_package_config( 49 | "rab_config.ini", "rab_storage", "minio_user") 50 | minio_password = rab_config.load_package_config( 51 | "rab_config.ini", "rab_storage", "minio_password") 52 | # 拼接为连接信息 53 | minio_conf = { 54 | "endpoint": minio_host, 55 | "access_key": minio_user, 56 | "secret_key": minio_password, 57 | "secure": minio_secure, 58 | "region": "us-east-1" 59 | } 60 | # 尝试连接 61 | r_logger.info("开始连接至 MinIO 存储源站......") 62 | try: 63 | minio_client = minio.Minio(**minio_conf) 64 | minio_buckets = minio_client.list_buckets() 65 | r_logger.info("MinIO 连接成功") 66 | return minio_client 67 | except Exception as e: 68 | r_logger.error("MinIO 尝试出错！") 69 | r_logger.error(e) 70 | return None 71 | 72 | """ 73 | @description: MinIO 文件是否存在 74 | ------- 75 | @param: 76 | ------- 77 | @return: 78 | """ 79 | def minio_exist(file_path, client): 80 | # 存储桶名 81 | bucket_name = file_path.split("/")[0] 82 | # 文件名 83 | file_name = file_path.split("/")[-1] 84 | # 文件在存储桶中的路径 85 | file_prefix = file_path.lstrip(bucket_name+"/").rstrip(file_name) 86 | # 检查文件是否存在 87 | objects = client.list_objects(bucket_name, prefix=file_prefix) 88 | for obj in objects: 89 | # 路径+文件名 == 存储桶中的文件名 90 | if ((file_prefix+file_name) == obj.object_name): 91 | return True, file_path 92 | return False, None 93 | 94 | """ 95 | @description: MinIO 检查文件路径是否安全 96 | ------- 97 | @param: 98 | ------- 99 | @return: 100 | """ 101 | def minio_check(file_path, client): 102 | # MinIO 在上传创建对象时会自动创建没有的路径 103 | return True, file_path 104 | 105 | """ 106 | @description: 上传至 MinIO 107 | ------- 108 | @param: 109 | ------- 110 | @return: 111 | """ 112 | def minio_upload(to_file_path, local_file_path, client): 113 | # 存储桶名 114 | to_bucket_name = to_file_path.split("/")[0] 115 | # 文件名 116 | to_file_name = to_file_path.split("/")[-1] 117 | # 文件在存储桶中的路径 118 | to_file_prefix = to_file_path.lstrip( 119 | to_bucket_name+"/").rstrip(to_file_name) 120 | try: 121 | # 上传文件 122 | client.fput_object(to_bucket_name, (to_file_prefix+to_file_name), \ 123 | local_file_path) 124 | return True, to_file_path 125 | except Exception as e: 126 | r_logger.error("MinIO 上传出错！") 127 | r_logger.error(e) 128 | return False, None 129 | 130 | """ 131 | @description: 从 MinIO 下载 132 | ------- 133 | @param: 134 | ------- 135 | @return: 136 | """ 137 | def minio_download(from_file_path, local_file_path, client): 138 | # 存储桶名 139 | from_bucket_name = from_file_path.split("/")[0] 140 | # 文件名 141 | from_file_name = from_file_path.split("/")[-1] 142 | # 文件在存储桶中的路径 143 | from_file_prefix = from_file_path.lstrip( 144 | from_bucket_name+"/").rstrip(from_file_name) 145 | try: 146 | # 下载文件 147 | client.fget_object(from_bucket_name, \ 148 | (from_file_prefix+from_file_name), local_file_path) 149 | return True, local_file_path 150 | except Exception as e: 151 | r_logger.error("MinIO 下载出错！") 152 | r_logger.error(e) 153 | return False, None 154 | 155 | """ 156 | @description: 从 MinIO 分享 157 | ------- 158 | @param: 159 | ------- 160 | @return: 161 | """ 162 | def minio_share(file_path, client): 163 | # MinIO IP 或域名 164 | minio_host = rab_config.load_package_config( 165 | "rab_config.ini", "rab_storage", "minio_host").lower() 166 | share_url = "{minio_host}/{file_path}".format( 167 | minio_host=minio_host, file_path=file_path) 168 | return True, share_url 169 | 170 | """ 171 | @description: 连接至腾讯云 COS 172 | ------- 173 | @param: 174 | ------- 175 | @return: 176 | """ 177 | def cos_connect(): 178 | # todo... 179 | return None 180 | 181 | """ 182 | @description: COS 文件是否存在 183 | ------- 184 | @param: 185 | ------- 186 | @return: 187 | """ 188 | def cos_exist(file_path, client): 189 | # todo... 190 | return False, None 191 | 192 | """ 193 | @description: COS 检查文件路径是否安全 194 | ------- 195 | @param: 196 | ------- 197 | @return: 198 | """ 199 | def cos_check(file_path, client): 200 | # todo... 201 | return False, None 202 | 203 | """ 204 | @description: 上传至 COS 205 | ------- 206 | @param: 207 | ------- 208 | @return: 209 | """ 210 | def cos_upload(to_file_path, local_file_path, client): 211 | # todo... 212 | return False, None 213 | 214 | """ 215 | @description: 从 COS 下载 216 | ------- 217 | @param: 218 | ------- 219 | @return: 220 | """ 221 | def cos_download(from_file_path, local_file_path, client): 222 | # todo... 223 | return False, None 224 | 225 | """ 226 | @description: 从 COS 分享 227 | ------- 228 | @param: 229 | ------- 230 | @return: 231 | """ 232 | def cos_share(file_path, client): 233 | # todo... 234 | return False, None 235 | 236 | """ 237 | @description: 连接至 Nextcloud 238 | ------- 239 | @param: 240 | ------- 241 | @return: 242 | """ 243 | def nextcloud_connect(): 244 | # Nextcloud IP 或域名 245 | nextcloud_host = rab_config.load_package_config( 246 | "rab_config.ini", "rab_storage", "nextcloud_host").lower() 247 | # 读取用户名和密码 248 | nextcloud_user = rab_config.load_package_config( 249 | "rab_config.ini", "rab_storage", "nextcloud_user") 250 | nextcloud_password = rab_config.load_package_config( 251 | "rab_config.ini", "rab_storage", "nextcloud_password") 252 | # 请求头 253 | headers = { 254 | "OCS-APIRequest": "true", 255 | "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/" \ 256 | + "537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36", 257 | "referrer-policy": "no-referrer" 258 | } 259 | # 建立连接 260 | nextcloud_client = rab_client.r_client(basic_url=nextcloud_host, \ 261 | auth=(nextcloud_user, nextcloud_password), \ 262 | headers=headers) 263 | # 动作：获取路径下的所有文件 264 | nextcloud_client.add_action("list", method="PROPFIND", \ 265 | route="/remote.php/dav/files/{user}/{path}", 266 | basic_params={"format": "json"}) 267 | # 动作：创建一个文件夹 268 | nextcloud_client.add_action("create_folder", method="MKCOL", \ 269 | route="/remote.php/dav/files/{user}/{folder_path}") 270 | # 动作：上传文件 271 | nextcloud_client.add_action("upload", method="PUT", \ 272 | route="/remote.php/dav/files/{user}/{path}") 273 | # 动作：下载文件 274 | nextcloud_client.add_action("download", method="GET", \ 275 | route="/remote.php/dav/files/{user}/{path}") 276 | # 动作：分享文件 277 | nextcloud_client.add_action("create_share", method="POST", \ 278 | route="/ocs/v2.php/apps/files_sharing/api/v1/shares", 279 | basic_params={"format": "json"}) 280 | # 动作：获取所有已分享的文件信息 281 | nextcloud_client.add_action("get_all_shares", method="GET", \ 282 | route="/ocs/v2.php/apps/files_sharing/api/v1/shares", 283 | basic_params={"format": "json"}) 284 | # 动作：获取指定路径下所有已分享的文件信息 285 | nextcloud_client.add_action("get_shares_from_path", method="GET", \ 286 | route="/ocs/v2.php/apps/files_sharing/api/v1/shares", 287 | basic_params={"format": "json", "reshares": "true", "subfiles": "true"}) 288 | return nextcloud_client 289 | 290 | """ 291 | @description: Nextcloud 文件是否存在 292 | ------- 293 | @param: 294 | ------- 295 | @return: 296 | """ 297 | def nextcloud_exist(file_path, client): 298 | # 文件名 299 | file_name = file_path.split("/")[-1] 300 | # 文件路径 301 | file_prefix = file_path.rstrip(file_name) 302 | # 执行查询 303 | nextcloud_response = client.do_action("list", \ 304 | _format={"user": client.auth[0], "path": file_prefix}) 305 | nextcloud_response_str = urllib.parse.unquote( 306 | nextcloud_response.content.decode("UTF-8")) 307 | # 转为 XML 对象 308 | xml_obj = xml.etree.ElementTree.fromstring(nextcloud_response_str) 309 | file_paths = [] 310 | for href in xml_obj.iter("{DAV:}href"): 311 | basic_href="/remote.php/dav/files/{}/".format(client.auth[0]) 312 | file_paths.append(href.text.replace(basic_href, "")) 313 | if (file_path in file_paths): 314 | return True, file_path 315 | else: 316 | return False, None 317 | 318 | """ 319 | @description: Nextcloud 检查文件路径是否安全 320 | ------- 321 | @param: 322 | ------- 323 | @return: 324 | """ 325 | def nextcloud_check(file_path, client): 326 | # 文件名 327 | file_name = file_path.split("/")[-1] 328 | # 文件路径 329 | file_prefix = file_path.rstrip(file_name) 330 | # 循环确认每个父文件夹 331 | _prefix = "" 332 | try: 333 | for file_prefix_part in file_prefix.split("/"): 334 | if (file_prefix_part): 335 | _prefix += "/{}".format(file_prefix_part) 336 | nextcloud_response = client.do_action("create_folder", \ 337 | _format={"user": client.auth[0], "folder_path": _prefix}) 338 | return True, file_path 339 | except Exception as e: 340 | r_logger.error("Nextcloud 新建文件夹 {} 出错！".format(_prefix)) 341 | r_logger.error(e) 342 | return False, None 343 | 344 | """ 345 | @description: 上传至 Nextcloud 346 | ------- 347 | @param: 348 | ------- 349 | @return: 350 | """ 351 | def nextcloud_upload(to_file_path, local_file_path, client): 352 | try: 353 | # 执行上传 354 | nextcloud_response = client.do_action("upload", \ 355 | _format={"user": client.auth[0], "path": to_file_path}, \ 356 | data=open(local_file_path, "rb")) 357 | # 记录日志 358 | r_logger.debug(nextcloud_response.text) 359 | return True, to_file_path 360 | except Exception as e: 361 | r_logger.error("Nextcloud 上传出错！") 362 | r_logger.error(e) 363 | return False, None 364 | 365 | """ 366 | @description: 从 Nextcloud 下载 367 | ------- 368 | @param: 369 | ------- 370 | @return: 371 | """ 372 | def nextcloud_download(from_file_path, local_file_path, client): 373 | try: 374 | # 执行下载 375 | nextcloud_response = client.do_action("download", \ 376 | _format={"user": client.auth[0], "path": to_file_path}, \ 377 | data=open(local_file_path, "rb")) 378 | # 写入文件 379 | with open(local_file_path, "wb") as local_file: 380 | local_file.write(nextcloud_response.content) 381 | return True, local_file_path 382 | except Exception as e: 383 | r_logger.error("Nextcloud 下载出错！") 384 | r_logger.error(e) 385 | return False, None 386 | 387 | """ 388 | @description: 从 Nextcloud 分享 389 | ------- 390 | @param: 391 | ------- 392 | @return: 393 | """ 394 | def nextcloud_share(file_path, client): 395 | # 文件名 396 | file_name = file_path.split("/")[-1] 397 | # 文件路径 398 | file_prefix = file_path.rstrip(file_name) 399 | # 获取当前已经分享的文件 400 | nextcloud_response = client.do_action( 401 | "get_shares_from_path", params={"path": file_prefix}) 402 | # 如果有现场的分享就直接使用 403 | if (json.loads(nextcloud_response.text)["ocs"]["meta"]["status"].lower() 404 | == "ok"): 405 | nextcloud_response_str = nextcloud_response.content.decode("UTF-8") 406 | shares = json.loads(nextcloud_response_str)["ocs"]["data"] 407 | for share in shares: 408 | if (share["path"].lstrip("/") == file_path): 409 | return True, share["url"] 410 | # 文件没有被分享过则新建分享 411 | nextcloud_response = client.do_action("create_share", \ 412 | params={"path": file_path, "shareType": 3}) 413 | # 记录日志 414 | r_logger.debug(nextcloud_response.text) 415 | return True, json.loads( 416 | nextcloud_response.content.decode("UTF-8"))["ocs"]["data"]["url"] 417 | 418 | 419 | """ 420 | @description: r_storage 存储类 421 | ------- 422 | @param: 423 | ------- 424 | @return: 425 | """ 426 | class r_storage(): 427 | 428 | """ 429 | @description: 初始化 430 | ------- 431 | @param: 432 | ------- 433 | @return: 434 | """ 435 | def __init__(self, origins=["MinIO", "COS", "Nextcloud"]): 436 | # 存储源站 437 | self.origins = origins 438 | # 静态资源存储站状态 439 | self.status = {} 440 | # 各个存储站的 Client 441 | self.client = {} 442 | # 开启与各源站间的连接 443 | self.connect() 444 | 445 | """ 446 | @description: 开启与各源站间的连接 447 | ------- 448 | @param: 449 | ------- 450 | @return: 451 | """ 452 | def connect(self): 453 | for origin in self.origins: 454 | # 如果有该存储源的配置文件 455 | host = rab_config.load_package_config( 456 | "rab_config.ini", "rab_storage", "{}_host".format( 457 | origin.lower())).lower() 458 | if (host): 459 | # MinIO 460 | if (origin.lower() == "minio"): 461 | client = minio_connect() 462 | # COS 463 | elif(origin.lower() == "cos"): 464 | client = cos_connect() 465 | # Nextcloud 466 | elif(origin.lower() == "nextcloud"): 467 | client = nextcloud_connect() 468 | # 判断是否连接成功 469 | if (client): 470 | self.client[origin.lower()] = client 471 | self.status[origin.lower()] = True 472 | else: 473 | self.status[origin.lower()] = False 474 | else: 475 | r_logger.error( 476 | "配置文件中没有 {} 存储源站的连接信息！".format(origin)) 477 | 478 | """ 479 | @description: 判断文件是否存在 480 | ------- 481 | @param: 482 | ------- 483 | @return: 484 | """ 485 | def exist(self, file_path): 486 | # 结果 487 | result = {} 488 | # 所有需要判断的存储源 489 | for origin in self.origins: 490 | # 判断是否连接 491 | if (not self.status[origin.lower()]): 492 | r_logger.warn("{} 未连接！".format(origin)) 493 | # 开始判断 494 | result[origin.lower()] = {} 495 | exist_flg = False 496 | _file_path = None 497 | # MinIO 498 | if (origin.lower() == "minio"): 499 | exist_flg, _file_path = minio_exist( 500 | file_path, self.client["minio"]) 501 | # COS 502 | elif(origin.lower() == "cos"): 503 | exist_flg, _file_path = cos_exist( 504 | file_path, self.client["nextcloud"]) 505 | # Nextcloud 506 | elif(origin.lower() == "nextcloud"): 507 | exist_flg, _file_path = nextcloud_exist( 508 | file_path, self.client["nextcloud"]) 509 | # 文件判断结束 510 | result[origin.lower()]["is_exist"] = exist_flg 511 | result[origin.lower()]["file_path"] = _file_path 512 | return result 513 | 514 | """ 515 | @description: 检查文件（文件路径是否存在，不在则新建） 516 | ------- 517 | @param: 518 | ------- 519 | @return: 520 | """ 521 | def check(self, file_path): 522 | # 结果 523 | result = {} 524 | # 所有需要检查的存储源 525 | for origin in self.origins: 526 | result[origin.lower()] = {} 527 | # MinIO 528 | if (origin.lower() == "minio"): 529 | check_flg, _file_path = minio_check(file_path, \ 530 | self.client[origin.lower()]) 531 | # COS 532 | elif(origin.lower() == "cos"): 533 | check_flg, _file_path = cos_check(file_path, \ 534 | self.client[origin.lower()]) 535 | # Nextcloud 536 | elif(origin.lower() == "nextcloud"): 537 | check_flg, _file_path = nextcloud_check(file_path, \ 538 | self.client[origin.lower()]) 539 | # 文件检查完成 540 | result[origin.lower()]["check_flg"] = check_flg 541 | result[origin.lower()]["file_path"] = _file_path 542 | return result 543 | 544 | """ 545 | @description: 上传文件 546 | ------- 547 | @param: 548 | ------- 549 | @return: 550 | """ 551 | def upload(self, to_file_path, local_file_path): 552 | # 结果 553 | result = {} 554 | # 上传前检查文件路径 555 | origin_check = self.check(to_file_path) 556 | # 判断存储源文件是否存在 557 | origin_exist = self.exist(to_file_path) 558 | # 本地文件是否存在 559 | if (not os.path.exists(local_file_path)): 560 | r_logger.warn("{} 文件不存在！".format(local_file_path)) 561 | return False, None 562 | # 所有需要上传的存储源 563 | for origin in self.origins: 564 | result[origin.lower()] = {} 565 | # 文件路径检查未通过 566 | if (not origin_check[origin.lower()]["check_flg"]): 567 | result[origin.lower()]["is_exist"] = False 568 | result[origin.lower()]["upload_flg"] = False 569 | result[origin.lower()]["file_path"] = None 570 | continue 571 | # 存储源是否已存在文件（其中有判断是否连接的过程） 572 | if (origin_exist[origin.lower()]["is_exist"]): 573 | result[origin.lower()]["is_exist"] = True 574 | result[origin.lower()]["upload_flg"] = False 575 | result[origin.lower()]["file_path"] = None 576 | continue 577 | # 开始上传，失败传回空文件路径 578 | upload_flg = False 579 | _to_file_path = None 580 | # MinIO 581 | if (origin.lower() == "minio"): 582 | upload_flg, _to_file_path = minio_upload(to_file_path, \ 583 | local_file_path, self.client[origin.lower()]) 584 | # COS 585 | elif(origin.lower() == "cos"): 586 | upload_flg, _to_file_path = cos_upload(to_file_path, \ 587 | local_file_path, self.client[origin.lower()]) 588 | # Nextcloud 589 | elif(origin.lower() == "nextcloud"): 590 | upload_flg, _to_file_path = nextcloud_upload(to_file_path, \ 591 | local_file_path, self.client[origin.lower()]) 592 | # 文件上传完成 593 | result[origin.lower()]["is_exist"] = False 594 | result[origin.lower()]["upload_flg"] = upload_flg 595 | result[origin.lower()]["file_path"] = _to_file_path 596 | return result 597 | 598 | """ 599 | @description: 下载文件 600 | ------- 601 | @param: 602 | ------- 603 | @return: 604 | """ 605 | def download(self, from_file_path, local_file_path): 606 | # 判断存储源文件是否存在 607 | origin_exist = self.exist(from_file_path) 608 | # 本地文件是否存在 609 | if (os.path.exists(local_file_path)): 610 | r_logger.warn("本地 {} 文件已存在！".format(local_file_path)) 611 | return False, None 612 | # 所有可下载的存储源 613 | for origin in self.origins: 614 | # 存储源是否存在文件（其中有判断是否连接的过程），不存在就跳过 615 | if (not origin_exist[origin.lower()]["is_exist"]): 616 | r_logger.warn("{origin} 源 {from_file_path} 文件不存在！".format( 617 | origin=origin, from_file_path=from_file_path)) 618 | continue 619 | # MinIO 620 | if (origin.lower() == "minio"): 621 | download_flg, _local_file_path = minio_download(from_file_path, \ 622 | local_file_path, self.client[origin.lower()]) 623 | # COS 624 | elif(origin.lower() == "cos"): 625 | download_flg, _local_file_path = cos_download( 626 | from_file_path, local_file_path, self.client[origin.lower()]) 627 | # Nextcloud 628 | elif(origin.lower() == "nextcloud"): 629 | download_flg, _local_file_path = nextcloud_download( 630 | from_file_path, local_file_path, self.client[origin.lower()]) 631 | # 下载成功就返回，失败则使用下一个源 632 | if (download_flg): 633 | return download_flg, _local_file_path 634 | else: 635 | continue 636 | # 所有源都下载失败，返回失败和空路径 637 | return False, None 638 | 639 | """ 640 | @description: 分享文件 641 | ------- 642 | @param: 643 | ------- 644 | @return: 645 | """ 646 | def share(self, file_path): 647 | # 结果 648 | result = {} 649 | # 判断存储源文件是否存在 650 | origin_exist = self.exist(file_path) 651 | # 所有需要分享的文件路径 652 | for origin in self.origins: 653 | result[origin.lower()] = {} 654 | # 存储源是否已存在文件（其中有判断是否连接的过程） 655 | if (not origin_exist[origin.lower()]["is_exist"]): 656 | result[origin.lower()]["share_flg"] = False 657 | result[origin.lower()]["share_url"] = None 658 | continue 659 | # MinIO 660 | if (origin.lower() == "minio"): 661 | share_flg, share_url = minio_share( 662 | file_path, self.client[origin.lower()]) 663 | # COS 664 | elif(origin.lower() == "cos"): 665 | share_flg, share_url = cos_share( 666 | file_path, self.client[origin.lower()]) 667 | # Nextcloud 668 | elif(origin.lower() == "nextcloud"): 669 | share_flg, share_url = nextcloud_share( 670 | file_path, self.client[origin.lower()]) 671 | # 文件分享完成 672 | result[origin.lower()]["share_flg"] = share_flg 673 | result[origin.lower()]["share_url"] = share_url 674 | return result 675 | 676 | 677 | """ 678 | @description: 单体测试 679 | ------- 680 | @param: 681 | ------- 682 | @return: 683 | """ 684 | if __name__ == "__main__": 685 | # 存储类中方法测试 686 | r_storage = r_storage(origins=["MinIO", "Nextcloud"]) 687 | # 检查文件存在 688 | # print(r_storage.exist("test/rab_python_packages.png")) 689 | # 检查文件不存在 690 | # print(r_storage.exist("test/not_exist.png")) 691 | # 检查路径 692 | # print(r_storage.check("test/check_folder.png")) 693 | # 新建路径 694 | # print(r_storage.check("test/new/check_folder.png")) 695 | # 上传文件 696 | # print(r_storage.upload("test/new/README.md", "README.md")) 697 | # print(r_storage.upload("test/new/new/README.md", "README.md")) 698 | # 分享文件 699 | print(r_storage.share("test/new/README.md")) 700 | # 下载文件 701 | # print(r_storage.download("test/new/README.md", "README_new.md")) --------------------------------------------------------------------------------