3 |
4 |
14 |
15 | {% if Count > 0 %}
16 |
5 |
13 |
6 |
12 |
7 |
11 | 8 | 站点资源 9 |
10 |
17 |
39 | {% else %}
40 | {{ OOPS.nodatafound('没有站点', '没有找到任何站点，请正确维护站点信息。') }}
41 | {% endif %}
42 |
--------------------------------------------------------------------------------
/app/indexer/client/_plugins.py:
--------------------------------------------------------------------------------
1 |
2 | from app.plugins import PluginManager
3 | from config import Config
4 |
5 | class PluginsSpider(object):
6 |
7 | # 私有方法
8 | _level = 99
9 | _plugin = {}
10 | _proxy = None
11 | _indexer = None
12 |
13 | def __int__(self, indexer):
14 | self._indexer = indexer
15 | if indexer.proxy:
16 | self._proxy = Config().get_proxies()
17 | self._plugin = PluginManager().get_plugin_apps(self._level).get(self._indexer.parser)
18 |
19 | def status(self, indexer):
20 | try:
21 | plugin = PluginManager().get_plugin_apps(self._level).get(indexer.parser)
22 | return True if plugin else False
23 | except Exception as e:
24 | return False
25 |
26 | def search(self, keyword, indexer, page=0):
27 | try:
28 | result_array = PluginManager().run_plugin_method(pid=indexer.parser, method='search', keyword=keyword, indexer=indexer, page=page)
29 | if not result_array:
30 | return False, []
31 | return True, result_array
32 | except Exception as e:
33 | return False, []
34 |
35 | def sites(self):
36 | result = []
37 | try:
38 | plugins = PluginManager().get_plugin_apps(self._level)
39 | for key in plugins:
40 | if plugins.get(key)['installed']:
41 | result_array = PluginManager().run_plugin_method(pid=plugins.get(key)['id'], method='get_indexers')
42 | if result_array:
43 | result.extend(result_array)
44 | except Exception as e:
45 | pass
46 | return result
--------------------------------------------------------------------------------
/app/helper/progress_helper.py:
--------------------------------------------------------------------------------
1 | from enum import Enum
2 |
3 | from app.utils.commons import singleton
4 | from app.utils.types import ProgressKey
5 |
6 |
7 | @singleton
8 | class ProgressHelper(object):
9 | _process_detail = {}
10 |
11 | def __init__(self):
12 | self._process_detail = {}
13 |
14 | def init_config(self):
15 | pass
16 |
17 | def __reset(self, ptype=ProgressKey.Search):
18 | if isinstance(ptype, Enum):
19 | ptype = ptype.value
20 | self._process_detail[ptype] = {
21 | "enable": False,
22 | "value": 0,
23 | "text": "请稍候..."
24 | }
25 |
26 | def start(self, ptype=ProgressKey.Search):
27 | self.__reset(ptype)
28 | if isinstance(ptype, Enum):
29 | ptype = ptype.value
30 | self._process_detail[ptype]['enable'] = True
31 |
32 | def end(self, ptype=ProgressKey.Search):
33 | if isinstance(ptype, Enum):
34 | ptype = ptype.value
35 | if not self._process_detail.get(ptype):
36 | return
37 | self._process_detail[ptype]['enable'] = False
38 |
39 | def update(self, value=None, text=None, ptype=ProgressKey.Search):
40 | if isinstance(ptype, Enum):
41 | ptype = ptype.value
42 | if not self._process_detail.get(ptype, {}).get('enable'):
43 | return
44 | if value:
45 | self._process_detail[ptype]['value'] = value
46 | if text:
47 | self._process_detail[ptype]['text'] = text
48 |
49 | def get_process(self, ptype=ProgressKey.Search):
50 | if isinstance(ptype, Enum):
51 | ptype = ptype.value
52 | return self._process_detail.get(ptype)
53 |
--------------------------------------------------------------------------------
/app/helper/plugin_helper.py:
--------------------------------------------------------------------------------
1 | from cachetools import cached, TTLCache
2 |
3 | from app.utils import RequestUtils
4 |
5 |
6 | # 2023年08月30日 nastool原作者服务已失效
7 | class PluginHelper:
8 |
9 | @staticmethod
10 | def install(plugin_id):
11 | """
12 | 插件安装统计计数
13 | """
14 | # return RequestUtils(timeout=5).get(f"https://nastool.org/plugin/{plugin_id}/install")
15 | pass
16 |
17 | @staticmethod
18 | def report(plugins):
19 | """
20 | 批量上报插件安装统计数据
21 | """
22 | # return RequestUtils(content_type="application/json",
23 | # timeout=5).post(f"https://nastool.org/plugin/update",
24 | # json={
25 | # "plugins": [
26 | # {
27 | # "plugin_id": plugin,
28 | # "count": 1
29 | # } for plugin in plugins
30 | # ]
31 | # })
32 | return {}
33 |
34 | @staticmethod
35 | @cached(cache=TTLCache(maxsize=1, ttl=3600))
36 | def statistic():
37 | """
38 | 获取插件安装统计数据
39 | """
40 | # ret = RequestUtils(accept_type="application/json",
41 | # timeout=5).get_res("https://nastool.org/plugin/statistic")
42 | # if ret:
43 | # try:
44 | # return ret.json()
45 | # except Exception as e:
46 | # print(e)
47 | # return {}
48 | return {}
49 |
--------------------------------------------------------------------------------
/app/message/message_center.py:
--------------------------------------------------------------------------------
1 | import datetime
2 | import time
3 | from collections import deque
4 |
5 | from app.utils.commons import singleton
6 |
7 |
8 | @singleton
9 | class MessageCenter:
10 | _message_queue = deque(maxlen=50)
11 | _message_index = 0
12 |
13 | def __init__(self):
14 | pass
15 |
16 | def insert_system_message(self, title, content=None):
17 | """
18 | 新增系统消息
19 | :param title: 标题
20 | :param content: 内容
21 | """
22 | title = title.replace("\n", "
18 |
38 |
19 | {% for Site in Sites %}
20 |
23 |
37 |
24 |
34 |
35 | {% endfor %}
36 |
25 |
33 |
26 |
27 |
28 |
29 |
32 | {{ Site.name }}
30 | {{ Site.domain }}
31 | ").strip() if title else "" 23 | content = content.replace("\n", "
").strip() if content else "" 24 | self.__append_message_queue(title, content) 25 | 26 | def __append_message_queue(self, title, content): 27 | """ 28 | 将消息增加到队列 29 | """ 30 | self._message_queue.appendleft({"title": title, 31 | "content": content, 32 | "time": time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(time.time()))}) 33 | 34 | def get_system_messages(self, num=20, lst_time=None): 35 | """ 36 | 查询系统消息 37 | :param num:条数 38 | :param lst_time: 最后时间 39 | """ 40 | if not lst_time: 41 | return list(self._message_queue)[-num:] 42 | else: 43 | ret_messages = [] 44 | for message in list(self._message_queue): 45 | if (datetime.datetime.strptime(message.get("time"), '%Y-%m-%d %H:%M:%S') - datetime.datetime.strptime( 46 | lst_time, '%Y-%m-%d %H:%M:%S')).seconds > 0: 47 | ret_messages.append(message) 48 | else: 49 | break 50 | return ret_messages 51 | -------------------------------------------------------------------------------- /app/message/client/pushdeer.py: -------------------------------------------------------------------------------- 1 | from pypushdeer import PushDeer 2 | 3 | from app.message.client._base import _IMessageClient 4 | from app.utils import StringUtils, ExceptionUtils 5 | 6 | 7 | class PushDeerClient(_IMessageClient): 8 | schema = "pushdeer" 9 | 10 | _server = None 11 | _apikey = None 12 | _client_config = {} 13 | 14 | def __init__(self, config): 15 | self._client_config = config 16 | self.init_config() 17 | 18 | def init_config(self): 19 | if self._client_config: 20 | self._server = StringUtils.get_base_url(self._client_config.get('server')) 21 | self._apikey = self._client_config.get('apikey') 22 | 23 | @classmethod 24 | def match(cls, ctype): 25 | return True if ctype == cls.schema else False 26 | 27 | def send_msg(self, title, text="", image="", url="", user_id=""): 28 | """ 29 | 发送PushDeer消息 30 | :param title: 消息标题 31 | :param text: 消息内容 32 | :param image: 未使用 33 | :param url: 未使用 34 | :param user_id: 未使用 35 | :return: 发送状态、错误信息 36 | """ 37 | if not title and not text: 38 | return False, "标题和内容不能同时为空" 39 | try: 40 | if not self._server or not self._apikey: 41 | return False, "参数未配置" 42 | pushdeer = PushDeer(server=self._server, pushkey=self._apikey) 43 | res = pushdeer.send_markdown(title, desp=text) 44 | if res: 45 | return True, "成功" 46 | else: 47 | return False, "失败" 48 | except Exception as msg_e: 49 | ExceptionUtils.exception_traceback(msg_e) 50 | return False, str(msg_e) 51 | 52 | def send_list_msg(self, **kwargs): 53 | pass 54 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.yml: -------------------------------------------------------------------------------- 1 | name: 问题反馈 2 | description: File a bug report 3 | title: "[错误报告]: 请在此处简单描述你的问题" 4 | labels: ["bug"] 5 | body: 6 | - type: markdown 7 | attributes: 8 | value: | 9 | 请确认以下信息： 10 | 1. 请按此模板提交issues，不按模板提交的问题将直接关闭。 11 | 2. 如果你的问题可以直接在以往 issue 中找到，那么你的 issue 将会被直接关闭。 12 | 3. 提交问题务必描述清楚、附上日志，描述不清导致无法理解和分析的问题会被直接关闭。 13 | - type: checkboxes 14 | id: ensure 15 | attributes: 16 | label: 确认 17 | description: 在提交 issue 之前，请确认你已经阅读并确认以下内容 18 | options: 19 | - label: 我的版本是最新版本，我的版本号与 [version](https://github.com/0xforee/nas-tools/releases/latest) 相同。 20 | required: true 21 | - label: 我已经 [issue](https://github.com/0xforee/nas-tools/issues) 中搜索过，确认我的问题没有被提出过。 22 | required: true 23 | - label: 我已经修改标题，将标题中的 描述 替换为我遇到的问题。 24 | required: true 25 | - type: input 26 | id: version 27 | attributes: 28 | label: 当前程序版本 29 | description: 遇到问题时程序所在的版本号 30 | validations: 31 | required: true 32 | - type: dropdown 33 | id: type 34 | attributes: 35 | label: 问题类型 36 | description: 你在以下哪个部分碰到了问题 37 | options: 38 | - 主程序运行问题 39 | - 插件问题 40 | - Docker或运行环境问题 41 | - 其他问题 42 | validations: 43 | required: true 44 | - type: textarea 45 | id: what-happened 46 | attributes: 47 | label: 问题描述 48 | description: 请详细描述你碰到的问题 49 | placeholder: "问题描述" 50 | validations: 51 | required: true 52 | - type: textarea 53 | id: logs 54 | attributes: 55 | label: 发生问题时系统日志和配置文件 56 | description: 问题出现时，程序运行日志请复制到这里。抓取日志参考：https://github.com/0xforee/nas-tools/wiki/%E5%A6%82%E4%BD%95%E6%8A%93%E5%8F%96%E6%97%A5%E5%BF%97%EF%BC%9F 57 | render: bash 58 | -------------------------------------------------------------------------------- /app/indexer/client/_haidan.py: -------------------------------------------------------------------------------- 1 | from pyquery import PyQuery 2 | from app.utils.exception_utils import ExceptionUtils 3 | import copy 4 | import log 5 | from jinja2 import Template 6 | 7 | from app.indexer.client._spider import TorrentSpider 8 | 9 | 10 | class HaiDanSpider(TorrentSpider): 11 | 12 | def parse(self, request, response): 13 | """ 14 | 解析整个页面 15 | """ 16 | try: 17 | # 获取站点文本 18 | html_text = response.extract() 19 | html_text = self.clean_all_sites_free(html_text) 20 | if not html_text: 21 | self.is_error = True 22 | self.is_complete = True 23 | return 24 | # 解析站点文本对象 25 | html_doc = PyQuery(html_text) 26 | # 种子筛选器 27 | torrents_selector = self.list.get('selector', '') 28 | # 遍历种子html列表 29 | for torn in html_doc(torrents_selector): 30 | group = PyQuery(torn) 31 | for torrent in group('div.torrent_wrap'): 32 | # log.debug(f"[haidan] {torrent}") 33 | torrent_query = PyQuery(torrent) 34 | torrent_info = self.Getinfo(torrent_query) 35 | # use group name + torrent name override 36 | self.Gettitle_with_group(group, torrent_query) 37 | self.torrents_info_array.append(copy.deepcopy(torrent_info)) 38 | if len(self.torrents_info_array) >= int(self.result_num): 39 | break 40 | 41 | except Exception as err: 42 | self.is_error = True 43 | ExceptionUtils.exception_traceback(err) 44 | log.warn(f"【Spider】错误：{self.indexername} {str(err)}") 45 | finally: 46 | self.is_complete = True -------------------------------------------------------------------------------- /diff.md: -------------------------------------------------------------------------------- 1 |  2 | # NAS媒体库管理工具 3 | 4 | 5 | [](https://github.com/0xforee/nas-tools/stargazers) 6 | [](https://github.com/0xforee/nas-tools/network/members) 7 | [](https://github.com/0xforee/nas-tools/issues) 8 | [](https://github.com/0xforee/nas-tools/blob/master/LICENSE.md) 9 | [](https://hub.docker.com/r/0xforee/nas-tools) 10 | [](https://hub.docker.com/r/0xforee/nas-tools) 11 | 12 | Docker：https://hub.docker.com/repository/docker/0xforee/nas-tools 13 | 14 | ## 基于官方主线版本增强 15 | 1. 基于主线最新 3.2.3 版本制作，配置可以完美迁移 16 | 1. 取消了用户认证 17 | 2. 取消新手刷流限制 18 | 1. 刷流增加部分下载能力（拆包），可以灵活配置下载种子的大小和比例，最大化提升刷流效率。（部分站点可能禁止，请慎用！！！） 19 | 2. 刷流增加限免限时检测能力，超过限免，自动降速为 1B/s，防止流量偷跑，默认为关。（目前已测试： mteam 和 hdmayi） 20 | 3. 刷流界面增加信息展示 21 | * 增加展示已保种大小 22 | * 种子明细增加展示种子大小，限免过期时间 23 | 3. 保留了 BT 能力和内置 BT 站点，可以继续索引和下载 BT 磁链和种子文件 24 | 4. 支持 jackett 和 prowlarr 索引器 25 | * 可以通过插件和内置索引器同时使用 26 | 5. 一些入口增加快捷跳转能力，方便将 nastools 作为媒体管理主要入口。具体如下： 27 | * 下载管理-正在下载：增加跳转下载器查看种子详情能力 28 | * 我的媒体库：增加标题跳转媒体库能力 29 | * 索引器：jackett 和 prowlarr 增加跳转各服务能力 30 | * 媒体服务器：每个服务器配置界面增加跳转服务能力 31 | 6. 支持 Mteam 新架构 32 | * 站点配置中配置 api-key 即可 33 | * api-key 从馒头-控制台-实验室-存取令牌 生成获取 34 | * 完美支持站点签到 35 | * 可从控制台登陆设备活动记录中查询到签到（登陆）记录 36 | -------------------------------------------------------------------------------- /tests/meta_recogonize_v3/cases/testcase1.py: -------------------------------------------------------------------------------- 1 | match = { 2 | 'en_name': "Léon", 3 | 'cn_name': "这个杀手不太冷", 4 | "pin_name": "Zhegeshashoubutaileng", 5 | 'year': '1994' 6 | } 7 | 8 | cases = [ 9 | "Léon 1994 2160p UHD Blu-ray HDR10 HEVC TrueHD 7.1 Atmos-CrsS", 10 | "Leon The Professional 1994 EXTENDED 1080p BluRay DTS x264-LEGi0N", 11 | "The Godfather Coda The Death of Michael Corleone 1990 UHD BluRay 2160p 10bit DoVi 5Audio TrueHD 5.1 x2", 12 | "Leon aka The Professional Directors Cut 1994 Bluray 1080p x265 AAC 5.1 2Audios-GREENOTEA", 13 | "Leon 1994 EXTENDED REMASTERED BluRay 1080p 2Audio TrueHD Atmos 7.1 x265 10bit-BeiTai", 14 | "Operation Napoleon 2023 1080p friDay WEB-DL H.264 AAC-PTerWEB", 15 | "Leon 1994 1080p WEB-DL AVC AAC2.0-HDKylinWeb", 16 | "Shake Rattle & Roll Extreme 2023 1080p NF WEB-DL DDP5.1 H.264-HHWEB", 17 | "Operation Napoleon 2023 1080p Blu-ray AVC DTS-HD MA 5.1", 18 | "Leon 1994 WEB-DL 2160p HEVC AAC-ZmWeb", 19 | "Leon The Professional 1994 BluRay 2160p x265 10bit HDR 3Audio mUHD-FRDS", 20 | "Leon The Professional 1994 EXTENDED REMASTERED BluRay 1080p x264 AC3 2Audios-CMCT", 21 | "Leonera 2008 1080p AMZN WEB-DL DDP2.0 H.264-PTerWEB", 22 | "The Godfather Part III 1990 REPACK 2160p UHD BluRay REMUX DV HDR HEVC TrueHD 5.1-FraMeSToR", 23 | "Leon The Professional 1994 Extended 2160p HDR UHD BluRay TrueHD 7.1 Atmos 2Audio x265 10bit-HDS", 24 | "In guerra per amore 2016 WEB-DL 2160p H.265 AAC-HDHWEB", 25 | "Napoleon 2023 2160p iT WEB-DL DDP5.1 Atmos HDR H 265-HHWEB", 26 | "Napoleon 2023 1080p iTunes WEB-DL DDP 5.1 Atmos H.264-CHDWEB", 27 | "Leon 1996 Extended Remastered BluRay 1080p x265 10bit 2Audio MNHD-FRDS", 28 | "Leon the Professional DC EXTENDED 1994 UHD Blu-ray 2160p 10bit HDR 5Audio TrueHDAtmos 7.1 x265-beAst", 29 | ] 30 | 31 | cases_result = [ 32 | 1,1,0,1,1,0,1,0,0,1,1,1,0,0,1,0,0,0,0,1 33 | ] -------------------------------------------------------------------------------- /app/conf/systemconfig.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from app.helper import DictHelper 4 | from app.utils.commons import singleton 5 | from app.utils.types import SystemConfigKey 6 | 7 | 8 | @singleton 9 | class SystemConfig: 10 | # 系统设置 11 | systemconfig = {} 12 | 13 | def __init__(self): 14 | self.dicthelper = DictHelper() 15 | self.init_config() 16 | 17 | def init_config(self): 18 | """ 19 | 缓存系统设置 20 | """ 21 | for item in self.dicthelper.list("SystemConfig"): 22 | if not item: 23 | continue 24 | if self.__is_obj(item.VALUE): 25 | self.systemconfig[item.KEY] = json.loads(item.VALUE) 26 | else: 27 | self.systemconfig[item.KEY] = item.VALUE 28 | 29 | @staticmethod 30 | def __is_obj(obj): 31 | if isinstance(obj, list) or isinstance(obj, dict): 32 | return True 33 | else: 34 | return str(obj).startswith("{") or str(obj).startswith("[") 35 | 36 | def set(self, key: [SystemConfigKey, str], value): 37 | """ 38 | 设置系统设置 39 | """ 40 | if isinstance(key, SystemConfigKey): 41 | key = key.value 42 | # 更新内存 43 | self.systemconfig[key] = value 44 | # 写入数据库 45 | if self.__is_obj(value): 46 | if value is not None: 47 | value = json.dumps(value) 48 | else: 49 | value = '' 50 | self.dicthelper.set("SystemConfig", key, value) 51 | 52 | def get(self, key: [SystemConfigKey, str] = None): 53 | """ 54 | 获取系统设置 55 | """ 56 | if not key: 57 | return self.systemconfig 58 | if isinstance(key, SystemConfigKey): 59 | key = key.value 60 | return self.systemconfig.get(key) 61 | -------------------------------------------------------------------------------- /Q&A.md: -------------------------------------------------------------------------------- 1 | ## 常见问题 2 | 3 | ### 1. 启动inotify报错/无法自动目录同步 4 | 5 | > 问题描述 6 | > 无法启动，日志报inotify instance limit reached、inotify watch limit reached等与inotify相关错误 7 | > 目录同步无法自动同步或只有部份目录正常，但在服务中手动启动可以正常同步 8 | 9 | 解决办法： 10 | * 宿主机上（不是docker容器里），执行以下命令： 11 | ```bash 12 | echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf 13 | echo fs.inotify.max_user_instances=524288 | sudo tee -a /etc/sysctl.conf 14 | sudo sysctl -p 15 | ``` 16 | * 插件-定时目录同步 17 | 18 | ### 2. 启动报错数据库no such column 19 | > 问题描述 20 | > 启动报错数据库no such column 21 | 22 | 解决办法： 23 | * [Sqlite浏览器](https://github.com/sqlitebrowser/sqlitebrowser)打开config文件夹下user.db 24 | * 删除alembic_version表后重启 25 | 26 | ### 3. Nginx-Proxy-Manager无法申请/更新Let's Encrypt证书 27 | > 问题描述 28 | > Nginx-Proxy-Manager无法申请/更新Let's Encrypt证书 29 | 30 | 解决办法： 31 | * 无法通过DNSpod申请, 进入容器，执行命令 32 | ```bash 33 | pip install certbot-dns-dnspod 34 | ``` 35 | 36 | * 无法更新/自动更新 37 | ```bash 38 | 进入容器，执行命令 pip install zope 39 | ``` 40 | 41 | * pip使用代理 42 | ``` 43 | pip install xxx --proxy=http://ip:port 44 | ``` 45 | 46 | ### 4. 消息通知内容无法跳转 47 | > 问题描述 48 | > 消息通知内容无法跳转（包含“点击选择下载”的消息中，没有包含查看详情的入口） 49 | 50 | 解决办法： 51 | * 设置-基础设置-系统-外网访问地址 52 | 53 | ### 5. 电影/电视剧订阅一直队列中 54 | > 问题描述 55 | > 电影/电视剧订阅添加后，一直在队列中 56 | > 需手动刷新订阅开始搜索或订阅 57 | 58 | 解决办法： 59 | * 订阅如启用有订阅站点, 请在设置-基础设置-服务-订阅RSS周期设置启用 60 | * 订阅如启用有搜索站点, 请在设置-基础设置-服务-订阅搜索周期设置启用 61 | * 添加后点击订阅，选择刷新 62 | 63 | ### 6. 目录同步文件重复转移 64 | > 问题描述 65 | > 设置-基础设置-媒体-重命名格式中包含{releaseGroup} 66 | > 文件转移方式为目录同步 67 | > 转移后，出现重复的转移文件（制作组等后缀不同） 68 | 69 | 解决办法： 70 | * 目录同步设置时，目的目录不要在源目录下 71 | 72 | ### 7. 识别转移错误码-1 73 | > 问题描述 74 | > 识别转移错误码-1 75 | 76 | 解决办法： 77 | * 硬链接跨盘，转移前后目录根目录需相同 78 | * 群晖中，不同的共享文件夹会被系统认为是跨盘 79 | 80 | ### 8. 电视剧订阅在完结前自动删除 81 | > 问题描述 82 | > 电视剧订阅在完结前自动删除 83 | 84 | 解决办法： 85 | * TMDB词条未更新集数/下载资源无法识别集数 86 | * 订阅中设置总集数 -------------------------------------------------------------------------------- /web/static/css/nprogress.css: -------------------------------------------------------------------------------- 1 | /* Make clicks pass-through */ 2 | #nprogress { 3 | pointer-events: none; 4 | } 5 | 6 | #nprogress .bar { 7 | background: var(--tblr-primary) !important; 8 | position: fixed; 9 | z-index: 1031; 10 | top: calc(env(safe-area-inset-top) + var(--safe-area-inset-top)); 11 | left: 0; 12 | width: 100%; 13 | height: 2px; 14 | } 15 | 16 | /* Fancy blur effect */ 17 | #nprogress .peg { 18 | display: block; 19 | position: absolute; 20 | right: 0; 21 | width: 5px; 22 | height: 100%; 23 | box-shadow: 0 0 10px var(--tblr-primary), 0 0 5px var(--tblr-primary); 24 | opacity: 1.0; 25 | 26 | -webkit-transform: rotate(0deg) translate(0px, -1px); 27 | -ms-transform: rotate(0deg) translate(0px, -1px); 28 | transform: rotate(0deg) translate(0px, -1px); 29 | } 30 | 31 | /* Remove these to get rid of the spinner */ 32 | #nprogress .spinner { 33 | display: block; 34 | position: fixed; 35 | z-index: 1031; 36 | top: 15px; 37 | right: 15px; 38 | } 39 | 40 | #nprogress .spinner-icon { 41 | width: 18px; 42 | height: 18px; 43 | box-sizing: border-box; 44 | 45 | border: solid 2px transparent; 46 | border-top-color: #29d; 47 | border-left-color: #29d; 48 | border-radius: 50%; 49 | 50 | -webkit-animation: nprogress-spinner 400ms linear infinite; 51 | animation: nprogress-spinner 400ms linear infinite; 52 | } 53 | 54 | .nprogress-custom-parent { 55 | overflow: hidden; 56 | position: relative; 57 | } 58 | 59 | .nprogress-custom-parent #nprogress .spinner, 60 | .nprogress-custom-parent #nprogress .bar { 61 | position: absolute; 62 | } 63 | 64 | @-webkit-keyframes nprogress-spinner { 65 | 0% { -webkit-transform: rotate(0deg); } 66 | 100% { -webkit-transform: rotate(360deg); } 67 | } 68 | @keyframes nprogress-spinner { 69 | 0% { transform: rotate(0deg); } 70 | 100% { transform: rotate(360deg); } 71 | } 72 | 73 | -------------------------------------------------------------------------------- /web/static/components/card/person/index.js: -------------------------------------------------------------------------------- 1 | import { html } from "../../utility/lit-core.min.js"; 2 | import { CustomElement, Golbal } from "../../utility/utility.js"; 3 | 4 | export class PersonCard extends CustomElement { 5 | 6 | static properties = { 7 | person_id: { attribute: "person-id" }, 8 | person_image: { attribute: "person-image" }, 9 | person_name: { attribute: "person-name" }, 10 | person_role: { attribute: "person-role" }, 11 | lazy: {}, 12 | }; 13 | 14 | constructor() { 15 | super(); 16 | this.lazy = "0"; 17 | } 18 | 19 | render() { 20 | return html` 21 |
22 |
42 | `;
43 | }
44 |
45 | }
46 |
47 | window.customElements.define("person-card", PersonCard);
--------------------------------------------------------------------------------
/web/templates/macro/oops.html:
--------------------------------------------------------------------------------
1 |
2 | {% macro nodatafound(title, text) %}
3 |
23 |
41 |
24 |
32 | 34 | ${this.person_name} 35 |
36 | 40 |
4 |
15 | {% endmacro %}
16 |
17 |
18 | {% macro empty(title, text) %}
19 |
5 |
14 |
6 | 
7 |
8 |
13 |
7 | {{ title }}
9 |10 | {{ text }} 11 |
12 |
20 |
31 | {% endmacro %}
32 |
33 |
34 | {% macro systemerror(title, text) %}
35 |
21 |
30 |
22 | 
23 |
24 |
29 |
23 | {{ title }}
25 |26 | {{ text }} 27 |
28 |
36 |
47 | {% endmacro %}
48 |
49 |
50 | {% macro loading() %}
51 |
37 |
46 |
38 | 
39 |
40 |
45 |
39 | {{ title }}
41 |42 | {{ text }} 43 |
44 |
52 |
62 | {% endmacro %}
63 |
--------------------------------------------------------------------------------
/third_party/feapder/feapder/core/handle_failed_requests.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | Created on 2018-08-13 11:43:01
4 | ---------
5 | @summary:
6 | ---------
7 | @author: Boris
8 | @email: boris_liu@foxmail.com
9 | """
10 | import feapder.setting as setting
11 | from feapder.buffer.request_buffer import RequestBuffer
12 | from feapder.db.redisdb import RedisDB
13 | from feapder.network.request import Request
14 | from feapder.utils.log import log
15 |
16 |
17 | class HandleFailedRequests:
18 | def __init__(self, redis_key):
19 | if redis_key.endswith(":z_failed_requests"):
20 | redis_key = redis_key.replace(":z_failed_requests", "")
21 |
22 | self._redisdb = RedisDB()
23 | self._request_buffer = RequestBuffer(redis_key)
24 |
25 | self._table_failed_request = setting.TAB_FAILED_REQUESTS.format(
26 | redis_key=redis_key
27 | )
28 |
29 | def get_failed_requests(self, count=10000):
30 | failed_requests = self._redisdb.zget(self._table_failed_request, count=count)
31 | failed_requests = [eval(failed_request) for failed_request in failed_requests]
32 | return failed_requests
33 |
34 | def reput_failed_requests_to_requests(self):
35 | log.debug("正在重置失败的requests...")
36 | total_count = 0
37 | while True:
38 | try:
39 | failed_requests = self.get_failed_requests()
40 | if not failed_requests:
41 | break
42 |
43 | for request in failed_requests:
44 | request["retry_times"] = 0
45 | request_obj = Request.from_dict(request)
46 | self._request_buffer.put_request(request_obj)
47 |
48 | total_count += 1
49 | except Exception as e:
50 | log.exception(e)
51 |
52 | self._request_buffer.flush()
53 |
54 | log.debug("重置%s条失败requests为待抓取requests" % total_count)
55 |
--------------------------------------------------------------------------------
/app/plugins/modules/_autosignin/pterclub.py:
--------------------------------------------------------------------------------
1 | import json
2 |
3 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler
4 | from app.utils import StringUtils, RequestUtils
5 | from config import Config
6 |
7 |
8 | class PTerClub(_ISiteSigninHandler):
9 | """
10 | 猫签到
11 | """
12 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url
13 | site_url = "pterclub.com"
14 |
15 |
16 | @classmethod
17 | def match(cls, url):
18 | """
19 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可
20 | :param url: 站点Url
21 | :return: 是否匹配，如匹配则会调用该类的signin方法
22 | """
23 | return True if StringUtils.url_equal(url, cls.site_url) else False
24 |
25 | def signin(self, site_info: dict):
26 | """
27 | 执行签到操作
28 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息
29 | :return: 签到结果信息
30 | """
31 | site = site_info.get("name")
32 | site_cookie = site_info.get("cookie")
33 | ua = site_info.get("ua")
34 | proxy = Config().get_proxies() if site_info.get("proxy") else None
35 |
36 | # 签到
37 | sign_res = RequestUtils(cookies=site_cookie,
38 | headers=ua,
39 | proxies=proxy
40 | ).get_res(url="https://pterclub.com/attendance-ajax.php")
41 | if not sign_res or sign_res.status_code != 200:
42 | self.error(f"签到失败，签到接口请求失败")
43 | return False, f'【{site}】签到失败，请检查cookie是否失效'
44 |
45 | sign_dict = json.loads(sign_res.text)
46 | if sign_dict['status'] == '1':
47 | # {"status":"1","data":" (签到已成功300)","message":"
53 |
61 |
54 | 
55 |
56 |
60 |
55 | 57 | 正在加载 58 |
59 |这是您的第237次签到， 48 | # 已连续签到237天。
本次签到获得300克猫粮。
"} 49 | self.info(f"签到成功") 50 | return True, f'【{site}】签到成功' 51 | else: 52 | # {"status":"0","data":"抱歉","message":"您今天已经签到过了，请勿重复刷新。"} 53 | self.info(f"今日已签到") 54 | return True, f'【{site}】今日已签到' 55 | -------------------------------------------------------------------------------- /app/message/client/iyuu.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlencode 2 | 3 | from app.message.client._base import _IMessageClient 4 | from app.utils import RequestUtils, ExceptionUtils 5 | 6 | 7 | class IyuuMsg(_IMessageClient): 8 | schema = "iyuu" 9 | 10 | _token = None 11 | _client_config = {} 12 | 13 | def __init__(self, config): 14 | self._client_config = config 15 | self.init_config() 16 | 17 | def init_config(self): 18 | if self._client_config: 19 | self._token = self._client_config.get('token') 20 | 21 | @classmethod 22 | def match(cls, ctype): 23 | return True if ctype == cls.schema else False 24 | 25 | def send_msg(self, title, text="", image="", url="", user_id=""): 26 | """ 27 | 发送爱语飞飞消息 28 | :param title: 消息标题 29 | :param text: 消息内容 30 | :param image: 未使用 31 | :param url: 未使用 32 | :param user_id: 未使用 33 | """ 34 | if not title and not text: 35 | return False, "标题和内容不能同时为空" 36 | if not self._token: 37 | return False, "参数未配置" 38 | try: 39 | sc_url = "http://iyuu.cn/%s.send?%s" % (self._token, urlencode({"text": title, "desp": text})) 40 | res = RequestUtils().get_res(sc_url) 41 | if res and res.status_code == 200: 42 | ret_json = res.json() 43 | errno = ret_json.get('errcode') 44 | error = ret_json.get('errmsg') 45 | if errno == 0: 46 | return True, error 47 | else: 48 | return False, error 49 | elif res is not None: 50 | return False, f"错误码：{res.status_code}，错误原因：{res.reason}" 51 | else: 52 | return False, "未获取到返回信息" 53 | except Exception as msg_e: 54 | ExceptionUtils.exception_traceback(msg_e) 55 | return False, str(msg_e) 56 | 57 | def send_list_msg(self, **kwargs): 58 | pass 59 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/_base.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import re 3 | from abc import ABCMeta, abstractmethod 4 | 5 | import log 6 | from app.utils import StringUtils 7 | 8 | 9 | class _ISiteSigninHandler(metaclass=ABCMeta): 10 | """ 11 | 实现站点签到的基类，所有站点签到类都需要继承此类，并实现match和signin方法 12 | 实现类放置到sitesignin目录下将会自动加载 13 | """ 14 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 15 | site_url = "" 16 | 17 | @abstractmethod 18 | def match(self, url): 19 | """ 20 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 21 | :param url: 站点Url 22 | :return: 是否匹配，如匹配则会调用该类的signin方法 23 | """ 24 | return True if StringUtils.url_equal(url, self.site_url) else False 25 | 26 | @abstractmethod 27 | def signin(self, site_info: dict): 28 | """ 29 | 执行签到操作 30 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 31 | :return: True|False,签到结果信息 32 | """ 33 | pass 34 | 35 | @staticmethod 36 | def sign_in_result(html_res, regexs): 37 | """ 38 | 判断是否签到成功 39 | """ 40 | html_text = re.sub(r"#\d+", "", re.sub(r"\d+px", "", html_res)) 41 | for regex in regexs: 42 | if re.search(str(regex), html_text): 43 | return True 44 | return False 45 | 46 | def info(self, msg): 47 | """ 48 | 记录INFO日志 49 | :param msg: 日志信息 50 | """ 51 | log.info(f"【Sites】{self.__class__.__name__} - {msg}") 52 | 53 | def warn(self, msg): 54 | """ 55 | 记录WARN日志 56 | :param msg: 日志信息 57 | """ 58 | log.warn(f"【Sites】{self.__class__.__name__} - {msg}") 59 | 60 | def error(self, msg): 61 | """ 62 | 记录ERROR日志 63 | :param msg: 日志信息 64 | """ 65 | log.error(f"【Sites】{self.__class__.__name__} - {msg}") 66 | 67 | def debug(self, msg): 68 | """ 69 | 记录Debug日志 70 | :param msg: 日志信息 71 | """ 72 | log.debug(f"【Sites】{self.__class__.__name__} - {msg}") 73 | -------------------------------------------------------------------------------- /app/utils/time_utils.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | 3 | class TimeUtils: 4 | @staticmethod 5 | def time_difference(last_seen_str): 6 | 7 | try: 8 | if not last_seen_str: 9 | return "" 10 | if not isinstance(last_seen_str, str): 11 | return "" 12 | # 将时间字符串解析为 datetime 对象 13 | last_seen = datetime.strptime(last_seen_str, "%Y-%m-%d %H:%M:%S") 14 | current_time = datetime.now() 15 | 16 | # 计算时间差 17 | time_diff = current_time - last_seen 18 | 19 | # 获取天、小时和分钟 20 | days = time_diff.days 21 | hours, remainder = divmod(time_diff.seconds, 3600) 22 | minutes, seconds = divmod(remainder, 60) 23 | 24 | # 构建返回结果 25 | result_parts = [] 26 | if days > 0: 27 | result_parts.append(f"{days}天") 28 | if hours > 0: 29 | result_parts.append(f"{hours}时") 30 | if minutes > 0 or (days == 0 and hours == 0): # 如果天和小时都为0，仍然显示分钟 31 | result_parts.append(f"{minutes}分") 32 | 33 | return ''.join(result_parts) + "前" 34 | except: 35 | return '' 36 | 37 | @staticmethod 38 | def less_than_days(date_str, target_days): 39 | """ 40 | 小于三十天 41 | """ 42 | try: 43 | if not date_str: 44 | return False 45 | if not isinstance(date_str, str): 46 | return False 47 | # 将时间字符串解析为 datetime 对象 48 | last_seen = datetime.strptime(date_str, "%Y-%m-%d %H:%M:%S") 49 | current_time = datetime.now() 50 | 51 | # 计算时间差 52 | time_diff = current_time - last_seen 53 | 54 | # 获取天、小时和分钟 55 | days = time_diff.days 56 | 57 | if days > target_days: 58 | return False 59 | else: 60 | return True 61 | except: 62 | return False 63 | 64 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/haidan.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class HaiDan(_ISiteSigninHandler): 7 | """ 8 | 海胆签到 9 | """ 10 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 11 | site_url = "haidan.video" 12 | 13 | # 签到成功 14 | _succeed_regex = ['(?<=value=")已经打卡(?=")'] 15 | 16 | @classmethod 17 | def match(cls, url): 18 | """ 19 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 20 | :param url: 站点Url 21 | :return: 是否匹配，如匹配则会调用该类的signin方法 22 | """ 23 | return True if StringUtils.url_equal(url, cls.site_url) else False 24 | 25 | def signin(self, site_info: dict): 26 | """ 27 | 执行签到操作 28 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 29 | :return: 签到结果信息 30 | """ 31 | site = site_info.get("name") 32 | site_cookie = site_info.get("cookie") 33 | ua = site_info.get("ua") 34 | proxy = Config().get_proxies() if site_info.get("proxy") else None 35 | 36 | # 签到 37 | sign_res = RequestUtils(cookies=site_cookie, 38 | headers=ua, 39 | proxies=proxy 40 | ).get_res(url="https://www.haidan.video/signin.php") 41 | if not sign_res or sign_res.status_code != 200: 42 | self.error(f"签到失败，请检查站点连通性") 43 | return False, f'【{site}】签到失败，请检查站点连通性' 44 | 45 | if "login.php" in sign_res.text: 46 | self.error(f"签到失败，cookie失效") 47 | return False, f'【{site}】签到失败，cookie失效' 48 | 49 | sign_status = self.sign_in_result(html_res=sign_res.text, 50 | regexs=self._succeed_regex) 51 | if sign_status: 52 | self.info(f"签到成功") 53 | return True, f'【{site}】签到成功' 54 | 55 | self.error(f"签到失败，签到接口返回 {sign_res.text}") 56 | return False, f'【{site}】签到失败' 57 | -------------------------------------------------------------------------------- /app/message/client/serverchan.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import urlencode 2 | 3 | from app.message.client._base import _IMessageClient 4 | from app.utils import RequestUtils, ExceptionUtils 5 | 6 | 7 | class ServerChan(_IMessageClient): 8 | schema = "serverchan" 9 | 10 | _sckey = None 11 | _client_config = {} 12 | 13 | def __init__(self, config): 14 | self._client_config = config 15 | self.init_config() 16 | 17 | def init_config(self): 18 | if self._client_config: 19 | self._sckey = self._client_config.get('sckey') 20 | 21 | @classmethod 22 | def match(cls, ctype): 23 | return True if ctype == cls.schema else False 24 | 25 | def send_msg(self, title, text="", image="", url="", user_id=""): 26 | """ 27 | 发送ServerChan消息 28 | :param title: 消息标题 29 | :param text: 消息内容 30 | :param image: 未使用 31 | :param url: 未使用 32 | :param user_id: 未使用 33 | """ 34 | if not title and not text: 35 | return False, "标题和内容不能同时为空" 36 | if not self._sckey: 37 | return False, "参数未配置" 38 | try: 39 | sc_url = "https://sctapi.ftqq.com/%s.send?%s" % (self._sckey, urlencode({"title": title, "desp": text})) 40 | res = RequestUtils().get_res(sc_url) 41 | if res and res.status_code == 200: 42 | ret_json = res.json() 43 | errno = ret_json.get('code') 44 | error = ret_json.get('message') 45 | if errno == 0: 46 | return True, error 47 | else: 48 | return False, error 49 | elif res is not None: 50 | return False, f"错误码：{res.status_code}，错误原因：{res.reason}" 51 | else: 52 | return False, "未获取到返回信息" 53 | except Exception as msg_e: 54 | ExceptionUtils.exception_traceback(msg_e) 55 | return False, str(msg_e) 56 | 57 | def send_list_msg(self, **kwargs): 58 | pass 59 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/hhanclub.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class Hhanclub(_ISiteSigninHandler): 7 | """ 8 | 海胆签到 9 | """ 10 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 11 | site_url = "hhanclub.top" 12 | 13 | # 签到成功 14 | _succeed_regex = ["今日签到排名：(\\d+) / \\d+"] 15 | 16 | @classmethod 17 | def match(cls, url): 18 | """ 19 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 20 | :param url: 站点Url 21 | :return: 是否匹配，如匹配则会调用该类的signin方法 22 | """ 23 | return True if StringUtils.url_equal(url, cls.site_url) else False 24 | 25 | def signin(self, site_info: dict): 26 | """ 27 | 执行签到操作 28 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 29 | :return: 签到结果信息 30 | """ 31 | site = site_info.get("name") 32 | site_cookie = site_info.get("cookie") 33 | ua = site_info.get("ua") 34 | proxy = Config().get_proxies() if site_info.get("proxy") else None 35 | 36 | # 签到 37 | sign_res = RequestUtils(cookies=site_cookie, 38 | headers=ua, 39 | proxies=proxy 40 | ).get_res(url="https://hhanclub.top/attendance.php") 41 | if not sign_res or sign_res.status_code != 200: 42 | self.error(f"签到失败，请检查站点连通性") 43 | return False, f'【{site}】签到失败，请检查站点连通性' 44 | 45 | if "rule-tips" not in sign_res.text: 46 | self.error(f"签到失败，cookie失效") 47 | return False, f'【{site}】签到失败，cookie失效' 48 | 49 | sign_status = self.sign_in_result(html_res=sign_res.text, 50 | regexs=self._succeed_regex) 51 | if sign_status: 52 | self.info(f"签到成功") 53 | return True, f'【{site}】签到成功' 54 | 55 | self.error(f"签到失败，签到接口返回 {sign_res.text}") 56 | return False, f'【{site}】签到失败' 57 | -------------------------------------------------------------------------------- /third_party/feapder/feapder/dedup/litefilter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on 2022/9/21 11:28 AM 4 | --------- 5 | @summary: 6 | --------- 7 | @author: Boris 8 | @email: boris_liu@foxmail.com 9 | """ 10 | from typing import List, Union, Set 11 | 12 | from feapder.dedup.basefilter import BaseFilter 13 | 14 | 15 | class LiteFilter(BaseFilter): 16 | def __init__(self): 17 | self.datas: Set[str] = set() 18 | 19 | def add( 20 | self, keys: Union[List[str], str], *args, **kwargs 21 | ) -> Union[List[int], int]: 22 | """ 23 | 24 | Args: 25 | keys: list / 单个值 26 | *args: 27 | **kwargs: 28 | 29 | Returns: 30 | list / 单个值 (如果数据已存在 返回 0 否则返回 1, 可以理解为是否添加成功) 31 | """ 32 | if isinstance(keys, list): 33 | is_add = [] 34 | for key in keys: 35 | if key not in self.datas: 36 | self.datas.add(key) 37 | is_add.append(1) 38 | else: 39 | is_add.append(0) 40 | else: 41 | if keys not in self.datas: 42 | is_add = 1 43 | self.datas.add(keys) 44 | else: 45 | is_add = 0 46 | return is_add 47 | 48 | def get(self, keys: Union[List[str], str]) -> Union[List[int], int]: 49 | """ 50 | 检查数据是否存在 51 | Args: 52 | keys: list / 单个值 53 | 54 | Returns: 55 | list / 单个值 (如果数据已存在 返回 1 否则返回 0) 56 | """ 57 | if isinstance(keys, list): 58 | temp_set = set() 59 | is_exist = [] 60 | for key in keys: 61 | # 数据本身重复或者数据在去重库里 62 | if key in temp_set or key in self.datas: 63 | is_exist.append(1) 64 | else: 65 | is_exist.append(0) 66 | temp_set.add(key) 67 | 68 | return is_exist 69 | else: 70 | return int(keys in self.datas) 71 | -------------------------------------------------------------------------------- /app/sites/siteuserinfo/nexus_rabbit.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import json 3 | 4 | from lxml import etree 5 | 6 | from app.sites.siteuserinfo._base import SITE_BASE_ORDER 7 | from app.sites.siteuserinfo.nexus_php import NexusPhpSiteUserInfo 8 | from app.utils.exception_utils import ExceptionUtils 9 | from app.utils.types import SiteSchema 10 | 11 | 12 | class NexusRabbitSiteUserInfo(NexusPhpSiteUserInfo): 13 | schema = SiteSchema.NexusRabbit 14 | order = SITE_BASE_ORDER + 5 15 | 16 | @classmethod 17 | def match(cls, html_text): 18 | html = etree.HTML(html_text) 19 | if not html: 20 | return False 21 | 22 | printable_text = html.xpath("string(.)") if html else "" 23 | return 'Style by Rabbit' in printable_text 24 | 25 | def _parse_site_page(self, html_text): 26 | super()._parse_site_page(html_text) 27 | self._torrent_seeding_page = f"getusertorrentlistajax.php?page=1&limit=5000000&type=seeding&uid={self.userid}" 28 | self._torrent_seeding_headers = {"Accept": "application/json, text/javascript, */*; q=0.01"} 29 | 30 | def _parse_user_torrent_seeding_info(self, html_text, multi_page=False): 31 | """ 32 | 做种相关信息 33 | :param html_text: 34 | :param multi_page: 是否多页数据 35 | :return: 下页地址 36 | """ 37 | 38 | try: 39 | torrents = json.loads(html_text).get('data') 40 | except Exception as e: 41 | ExceptionUtils.exception_traceback(e) 42 | return 43 | 44 | page_seeding_size = 0 45 | page_seeding_info = [] 46 | 47 | page_seeding = len(torrents) 48 | for torrent in torrents: 49 | seeders = int(torrent.get('seeders', 0)) 50 | size = int(torrent.get('size', 0)) 51 | page_seeding_size += int(torrent.get('size', 0)) 52 | 53 | page_seeding_info.append([seeders, size]) 54 | 55 | self.seeding += page_seeding 56 | self.seeding_size += page_seeding_size 57 | self.seeding_info.extend(page_seeding_info) 58 | -------------------------------------------------------------------------------- /app/sites/site_limiter.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | 4 | class SiteRateLimiter: 5 | def __init__(self, limit_interval: int, limit_count: int, limit_seconds: int): 6 | """ 7 | 限制访问频率 8 | :param limit_interval: 单位时间（秒） 9 | :param limit_count: 单位时间内访问次数 10 | :param limit_seconds: 访问间隔（秒） 11 | """ 12 | self.limit_count = limit_count 13 | self.limit_interval = limit_interval 14 | self.limit_seconds = limit_seconds 15 | self.last_visit_time = 0 16 | self.count = 0 17 | 18 | def check_rate_limit(self) -> (bool, str): 19 | """ 20 | 检查是否超出访问频率控制 21 | :return: 超出返回True，否则返回False，超出时返回错误信息 22 | """ 23 | current_time = time.time() 24 | # 防问间隔时间 25 | if self.limit_seconds: 26 | if current_time - self.last_visit_time < self.limit_seconds: 27 | return True, f"触发流控规则，访问间隔不得小于 {self.limit_seconds} 秒，" \ 28 | f"上次访问时间：{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.last_visit_time))}" 29 | # 单位时间内访问次数 30 | if self.limit_interval and self.limit_count: 31 | if current_time - self.last_visit_time > self.limit_interval: 32 | # 计数清零 33 | self.count = 0 34 | if self.count >= self.limit_count: 35 | return True, f"触发流控规则，{self.limit_interval} 秒内访问次数不得超过 {self.limit_count} 次，" \ 36 | f"上次访问时间：{time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(self.last_visit_time))}" 37 | # 访问计数 38 | self.count += 1 39 | # 更新最后访问时间 40 | self.last_visit_time = current_time 41 | # 未触发流控 42 | return False, "" 43 | 44 | 45 | if __name__ == "__main__": 46 | # 限制 1 分钟内最多访问 10 次，单次访问间隔不得小于 10 秒 47 | site_rate_limit = SiteRateLimiter(10, 60, 10) 48 | 49 | # 模拟访问 50 | for i in range(12): 51 | if site_rate_limit.check_rate_limit(): 52 | print("访问频率超限") 53 | else: 54 | print("访问成功") 55 | time.sleep(3) 56 | -------------------------------------------------------------------------------- /app/media/tmdbv3api/objs/discover.py: -------------------------------------------------------------------------------- 1 | from app.media.tmdbv3api.tmdb import TMDb 2 | 3 | try: 4 | from urllib import urlencode 5 | except ImportError: 6 | from urllib.parse import urlencode 7 | 8 | 9 | class Discover(TMDb): 10 | _urls = { 11 | "movies": "/discover/movie", 12 | "tvs": "/discover/tv" 13 | } 14 | 15 | def discover_movies(self, params, page=1): 16 | """ 17 | Discover movies by different types of data like average rating, number of votes, genres and certifications. 18 | :param params: dict 19 | :param page: int 20 | :return: 21 | """ 22 | if not params: 23 | params = {} 24 | if page: 25 | params.update({"page": page}) 26 | return self._get_obj( 27 | self._call( 28 | self._urls["movies"], 29 | urlencode(params) 30 | ), 31 | "results" 32 | ) 33 | 34 | def discover_movies_pages(self, params): 35 | """ 36 | Discover movies by different types of data like average rating, number of votes, genres and certifications. 37 | :param params: dict 38 | :return: total_pages 39 | """ 40 | if not params: 41 | params = {} 42 | result = self._call( 43 | self._urls["movies"], 44 | urlencode(params) 45 | ) 46 | return result.get("total_pages") or 0 47 | 48 | def discover_tv_shows(self, params, page=1): 49 | """ 50 | Discover TV shows by different types of data like average rating, number of votes, genres, 51 | the network they aired on and air dates. 52 | :param params: dict 53 | :param page: int 54 | :return: 55 | """ 56 | if not params: 57 | params = {} 58 | if page: 59 | params.update({"page": page}) 60 | return self._get_obj( 61 | self._call( 62 | self._urls["tvs"], 63 | urlencode(params) 64 | ), 65 | "results" 66 | ) 67 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/hdcity.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class HDCity(_ISiteSigninHandler): 7 | """ 8 | 城市签到 9 | """ 10 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 11 | site_url = "hdcity.city" 12 | 13 | # 签到成功 14 | _success_text = '本次签到获得魅力' 15 | # 重复签到 16 | _repeat_text = '已签到' 17 | 18 | @classmethod 19 | def match(cls, url): 20 | """ 21 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 22 | :param url: 站点Url 23 | :return: 是否匹配，如匹配则会调用该类的signin方法 24 | """ 25 | return True if StringUtils.url_equal(url, cls.site_url) else False 26 | 27 | def signin(self, site_info: dict): 28 | """ 29 | 执行签到操作 30 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 31 | :return: 签到结果信息 32 | """ 33 | site = site_info.get("name") 34 | site_cookie = site_info.get("cookie") 35 | ua = site_info.get("ua") 36 | proxy = Config().get_proxies() if site_info.get("proxy") else None 37 | 38 | # 获取页面html 39 | html_res = RequestUtils(cookies=site_cookie, 40 | headers=ua, 41 | proxies=proxy 42 | ).get_res(url="https://hdcity.city/sign") 43 | if not html_res or html_res.status_code != 200: 44 | self.error(f"签到失败，请检查站点连通性") 45 | return False, f'【{site}】签到失败，请检查站点连通性' 46 | 47 | if "login" in html_res.text: 48 | self.error(f"签到失败，cookie失效") 49 | return False, f'【{site}】签到失败，cookie失效' 50 | 51 | # 判断是否已签到 52 | # '已连续签到278天，此次签到您获得了100魔力值奖励!' 53 | if self._success_text in html_res.text: 54 | self.info(f"签到成功") 55 | return True, f'【{site}】签到成功' 56 | if self._repeat_text in html_res.text: 57 | self.info(f"今日已签到") 58 | return True, f'【{site}】今日已签到' 59 | self.error(f"签到失败，签到接口返回 {html_res.text}") 60 | return False, f'【{site}】签到失败' 61 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/carpt.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class CarPT(_ISiteSigninHandler): 7 | """ 8 | 车站签到 9 | """ 10 | 11 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 12 | site_url = "carpt.net" 13 | 14 | # 签到成功 15 | _success_text = "签到成功" 16 | _repeat_text = "请不要重复签到哦" 17 | 18 | @classmethod 19 | def match(cls, url): 20 | """ 21 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 22 | :param url: 站点Url 23 | :return: 是否匹配，如匹配则会调用该类的signin方法 24 | """ 25 | return True if StringUtils.url_equal(url, cls.site_url) else False 26 | 27 | def signin(self, site_info: dict): 28 | """ 29 | 执行签到操作 30 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 31 | :return: 签到结果信息 32 | """ 33 | site = site_info.get("name") 34 | site_cookie = site_info.get("cookie") 35 | ua = site_info.get("ua") 36 | proxy = Config().get_proxies() if site_info.get("proxy") else None 37 | 38 | # 获取页面html 39 | html_res = RequestUtils(cookies=site_cookie, 40 | headers=ua, 41 | proxies=proxy 42 | ).get_res(url="https://carpt.net/attendance.php") 43 | if not html_res or html_res.status_code != 200: 44 | self.error(f"签到失败，请检查站点连通性") 45 | return False, f'【{site}】签到失败，请检查站点连通性' 46 | 47 | if "login.php" in html_res.text: 48 | self.error(f"签到失败，cookie失效") 49 | return False, f'【{site}】签到失败，cookie失效' 50 | 51 | # 判断是否已签到 52 | # '已连续签到278天，此次签到您获得了100魔力值奖励!' 53 | if self._success_text in html_res.text: 54 | self.info(f"签到成功") 55 | return True, f'【{site}】签到成功' 56 | if self._repeat_text in html_res.text: 57 | self.info(f"今日已签到") 58 | return True, f'【{site}】今日已签到' 59 | self.error(f"签到失败，签到接口返回 {html_res.text}") 60 | return False, f'【{site}】签到失败' 61 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/hdtime.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class HDTime(_ISiteSigninHandler): 7 | """ 8 | 时光签到 9 | """ 10 | 11 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 12 | site_url = "hdtime.org" 13 | 14 | # 签到成功 15 | _success_text = "签到成功" 16 | _repeat_text = "请不要重复签到哦" 17 | 18 | @classmethod 19 | def match(cls, url): 20 | """ 21 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 22 | :param url: 站点Url 23 | :return: 是否匹配，如匹配则会调用该类的signin方法 24 | """ 25 | return True if StringUtils.url_equal(url, cls.site_url) else False 26 | 27 | def signin(self, site_info: dict): 28 | """ 29 | 执行签到操作 30 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 31 | :return: 签到结果信息 32 | """ 33 | site = site_info.get("name") 34 | site_cookie = site_info.get("cookie") 35 | ua = site_info.get("ua") 36 | proxy = Config().get_proxies() if site_info.get("proxy") else None 37 | 38 | # 获取页面html 39 | html_res = RequestUtils(cookies=site_cookie, 40 | headers=ua, 41 | proxies=proxy 42 | ).get_res(url="https://hdtime.org/attendance.php") 43 | if not html_res or html_res.status_code != 200: 44 | self.error(f"签到失败，请检查站点连通性") 45 | return False, f'【{site}】签到失败，请检查站点连通性' 46 | 47 | if "login.php" in html_res.text: 48 | self.error(f"签到失败，cookie失效") 49 | return False, f'【{site}】签到失败，cookie失效' 50 | 51 | # 判断是否已签到 52 | # '已连续签到278天，此次签到您获得了100魔力值奖励!' 53 | if self._success_text in html_res.text: 54 | self.info(f"签到成功") 55 | return True, f'【{site}】签到成功' 56 | if self._repeat_text in html_res.text: 57 | self.info(f"今日已签到") 58 | return True, f'【{site}】今日已签到' 59 | self.error(f"签到失败，签到接口返回 {html_res.text}") 60 | return False, f'【{site}】签到失败' 61 | -------------------------------------------------------------------------------- /app/plugins/modules/_autosignin/hdfans.py: -------------------------------------------------------------------------------- 1 | from app.plugins.modules._autosignin._base import _ISiteSigninHandler 2 | from app.utils import StringUtils, RequestUtils 3 | from config import Config 4 | 5 | 6 | class HDFans(_ISiteSigninHandler): 7 | """ 8 | hdfans签到 9 | """ 10 | 11 | # 匹配的站点Url，每一个实现类都需要设置为自己的站点Url 12 | site_url = "hdfans.org" 13 | 14 | # 签到成功 15 | _success_text = "签到成功" 16 | _repeat_text = "请不要重复签到哦" 17 | 18 | @classmethod 19 | def match(cls, url): 20 | """ 21 | 根据站点Url判断是否匹配当前站点签到类，大部分情况使用默认实现即可 22 | :param url: 站点Url 23 | :return: 是否匹配，如匹配则会调用该类的signin方法 24 | """ 25 | return True if StringUtils.url_equal(url, cls.site_url) else False 26 | 27 | def signin(self, site_info: dict): 28 | """ 29 | 执行签到操作 30 | :param site_info: 站点信息，含有站点Url、站点Cookie、UA等信息 31 | :return: 签到结果信息 32 | """ 33 | site = site_info.get("name") 34 | site_cookie = site_info.get("cookie") 35 | ua = site_info.get("ua") 36 | proxy = Config().get_proxies() if site_info.get("proxy") else None 37 | 38 | # 获取页面html 39 | html_res = RequestUtils(cookies=site_cookie, 40 | headers=ua, 41 | proxies=proxy 42 | ).get_res(url="https://hdfans.org/attendance.php") 43 | if not html_res or html_res.status_code != 200: 44 | self.error(f"签到失败，请检查站点连通性") 45 | return False, f'【{site}】签到失败，请检查站点连通性' 46 | 47 | if "login.php" in html_res.text: 48 | self.error(f"签到失败，cookie失效") 49 | return False, f'【{site}】签到失败，cookie失效' 50 | 51 | # 判断是否已签到 52 | # '已连续签到278天，此次签到您获得了100魔力值奖励!' 53 | if self._success_text in html_res.text: 54 | self.info(f"签到成功") 55 | return True, f'【{site}】签到成功' 56 | if self._repeat_text in html_res.text: 57 | self.info(f"今日已签到") 58 | return True, f'【{site}】今日已签到' 59 | self.error(f"签到失败，签到接口返回 {html_res.text}") 60 | return False, f'【{site}】签到失败' 61 | -------------------------------------------------------------------------------- /third_party/feapder/feapder/templates/batch_spider_template.tmpl: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | Created on {DATE} 4 | --------- 5 | @summary: 6 | --------- 7 | @author: {USER} 8 | """ 9 | 10 | import feapder 11 | from feapder import ArgumentParser 12 | 13 | 14 | class ${spider_name}(feapder.BatchSpider): 15 | # 自定义数据库，若项目中有setting.py文件，此自定义可删除 16 | __custom_setting__ = dict( 17 | REDISDB_IP_PORTS="localhost:6379", 18 | REDISDB_USER_PASS="", 19 | REDISDB_DB=0, 20 | MYSQL_IP="localhost", 21 | MYSQL_PORT=3306, 22 | MYSQL_DB="", 23 | MYSQL_USER_NAME="", 24 | MYSQL_USER_PASS="", 25 | ) 26 | 27 | def start_requests(self, task): 28 | yield feapder.Request("https://spidertools.cn") 29 | 30 | def parse(self, request, response): 31 | # 提取网站title 32 | print(response.xpath("//title/text()").extract_first()) 33 | # 提取网站描述 34 | print(response.xpath("//meta[@name='description']/@content").extract_first()) 35 | print("网站地址: ", response.url) 36 | 37 | 38 | if __name__ == "__main__": 39 | spider = ${spider_name}( 40 | redis_key="xxx:xxxx", # 分布式爬虫调度信息存储位置 41 | task_table="", # mysql中的任务表 42 | task_keys=["id", "xxx"], # 需要获取任务表里的字段名，可添加多个 43 | task_state="state", # mysql中任务状态字段 44 | batch_record_table="xxx_batch_record", # mysql中的批次记录表 45 | batch_name="xxx(周全)", # 批次名字 46 | batch_interval=7, # 批次周期 天为单位 若为小时 可写 1 / 24 47 | ) 48 | 49 | parser = ArgumentParser(description="${spider_name}爬虫") 50 | 51 | parser.add_argument( 52 | "--start_master", 53 | action="store_true", 54 | help="添加任务", 55 | function=spider.start_monitor_task, 56 | ) 57 | parser.add_argument( 58 | "--start_worker", action="store_true", help="启动爬虫", function=spider.start 59 | ) 60 | 61 | parser.start() 62 | 63 | # 直接启动 64 | # spider.start() # 启动爬虫 65 | # spider.start_monitor_task() # 添加任务 66 | 67 | # 通过命令行启动 68 | # python ${file_name} --start_master # 添加任务 69 | # python ${file_name} --start_worker # 启动爬虫 70 | -------------------------------------------------------------------------------- /third_party/feapder/feapder/dedup/README.md: -------------------------------------------------------------------------------- 1 | # Dedup 2 | 3 | Dedup是feapder大数据去重模块，内置3种去重机制，使用方式一致，可容纳的去重数据量与内存有关。不同于BloomFilter，去重受槽位数量影响，Dedup使用了弹性的去重机制，可容纳海量的数据去重。 4 | 5 | 6 | ## 去重方式 7 | 8 | ### 临时去重 9 | 10 | > 基于redis，支持批量，去重有时效性。去重一万条数据约0.26秒，一亿条数据占用内存约1.43G 11 | 12 | ``` 13 | from feapder.dedup import Dedup 14 | 15 | data = {"xxx": 123, "xxxx": "xxxx"} 16 | datas = ["xxx", "bbb"] 17 | 18 | def test_ExpireFilter(): 19 | dedup = Dedup( 20 | Dedup.ExpireFilter, expire_time=10, redis_url="redis://@localhost:6379/0" 21 | ) 22 | 23 | # 逐条去重 24 | assert dedup.add(data) == 1 25 | assert dedup.get(data) == 1 26 | 27 | # 批量去重 28 | assert dedup.add(datas) == [1, 1] 29 | assert dedup.get(datas) == [1, 1] 30 | ``` 31 | 32 | 33 | ### 内存去重 34 | 35 | > 基于内存，支持批量。去重一万条数据约0.5秒，一亿条数据占用内存约285MB 36 | 37 | ``` 38 | from feapder.dedup import Dedup 39 | 40 | data = {"xxx": 123, "xxxx": "xxxx"} 41 | datas = ["xxx", "bbb"] 42 | 43 | def test_MemoryFilter(): 44 | dedup = Dedup(Dedup.MemoryFilter) # 表名为test 历史数据3秒有效期 45 | 46 | # 逐条去重 47 | assert dedup.add(data) == 1 48 | assert dedup.get(data) == 1 49 | 50 | # 批量去重 51 | assert dedup.add(datas) == [1, 1] 52 | assert dedup.get(datas) == [1, 1] 53 | ``` 54 | 55 | ### 永久去重 56 | 57 | > 基于redis，支持批量，永久去重。 去重一万条数据约3.5秒，一亿条数据占用内存约285MB 58 | 59 | from feapder.dedup import Dedup 60 | 61 | datas = { 62 | "xxx": xxx, 63 | "xxxx": "xxxx", 64 | } 65 | 66 | dedup = Dedup() 67 | 68 | print(dedup) #