├── .gitignore ├── LICENSE ├── README.md ├── papihub-frontend ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── jsconfig.json ├── next.config.js ├── package.json ├── postcss.config.js ├── public │ ├── icons │ │ └── pt │ │ │ ├── chdbits.ico │ │ │ ├── hdsky.ico │ │ │ ├── mteam.ico │ │ │ └── ourbits.ico │ ├── next.svg │ └── vercel.svg ├── src │ ├── app │ │ ├── apidoc │ │ │ ├── layout.js │ │ │ └── page.js │ │ ├── auth │ │ │ ├── layout.js │ │ │ └── login │ │ │ │ └── page.js │ │ ├── components │ │ │ ├── badge │ │ │ │ └── badge.js │ │ │ ├── breadcrumbs │ │ │ │ └── breadcrumbs.js │ │ │ ├── button │ │ │ │ └── button.js │ │ │ ├── footer │ │ │ │ └── footer.js │ │ │ ├── hook-form │ │ │ │ ├── form-provider.js │ │ │ │ ├── input.js │ │ │ │ └── select.js │ │ │ ├── layout │ │ │ │ ├── auth-layout.js │ │ │ │ ├── header │ │ │ │ │ └── main-page-header.js │ │ │ │ ├── main-layout.js │ │ │ │ └── simple-layout.js │ │ │ ├── nav │ │ │ │ ├── desktop │ │ │ │ │ ├── nav-desktop.js │ │ │ │ │ └── nav-user-desktop.js │ │ │ │ └── mobile │ │ │ │ │ └── nav-mobile.js │ │ │ └── page-sections │ │ │ │ └── site │ │ │ │ ├── edit-form.js │ │ │ │ ├── list.js │ │ │ │ └── options.js │ │ ├── favicon.ico │ │ ├── globals.css │ │ ├── layout.js │ │ ├── page.js │ │ └── site │ │ │ ├── add │ │ │ └── page.js │ │ │ ├── layout.js │ │ │ └── page.js │ ├── auth │ │ ├── auth-provider.js │ │ ├── guard │ │ │ ├── auth-guard.js │ │ │ └── guest-guard.js │ │ ├── store │ │ │ └── use-user-store.js │ │ └── utils.js │ ├── context │ │ └── query-provider.js │ ├── hooks │ │ └── use-http.js │ ├── service │ │ └── site-service.js │ └── utils │ │ ├── axios.js │ │ └── format-time.js └── tailwind.config.js ├── papihub ├── .gitignore ├── __init__.py ├── api │ ├── __init__.py │ ├── auth.py │ ├── concurrenttorrentsite.py │ ├── parser │ │ ├── __init__.py │ │ └── htmlparser.py │ ├── sites │ │ ├── __init__.py │ │ └── nexusphp.py │ ├── torrentsite.py │ └── types.py ├── auth.py ├── common │ ├── __init__.py │ ├── customjsonencoder.py │ ├── logging.py │ └── response.py ├── config │ ├── __init__.py │ ├── siteparserconfigloader.py │ └── types.py ├── constants.py ├── databases.py ├── eventbus.py ├── exceptions.py ├── main.py ├── manager │ ├── __init__.py │ └── sitemanager.py ├── models │ ├── __init__.py │ ├── cookiestoremodel.py │ ├── sitemodel.py │ └── usermodel.py ├── routers │ ├── __init__.py │ ├── siterouter.py │ ├── torrentsrouter.py │ └── userrouter.py ├── tasks │ ├── __init__.py │ └── sitetask.py └── utils.py ├── requirements.txt └── tests ├── api ├── __init__.py └── test_nexusphp.py ├── conf └── parser │ ├── mteam.yml │ └── ourbits.yml ├── config ├── __init__.py └── test_siteparserconfigload.py └── test_utils.py /.gitignore: -------------------------------------------------------------------------------- 1 | .vs/ 2 | .vscode/ 3 | .idea/ 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | pip-wheel-metadata/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | notebooks/ 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .envrc 111 | .venv 112 | .venvs 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | 137 | # macOS display setting files 138 | .DS_Store 139 | 140 | # Wandb directory 141 | wandb/ 142 | 143 | # asdf tool versions 144 | .tool-versions 145 | /.ruff_cache/ 146 | 147 | *.pkl 148 | *.bin 149 | 150 | # integration test artifacts 151 | data_map* 152 | \[('_type', 'fake'), ('stop', None)] -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, papihubcom 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PapiHub 2 | 3 | PapiHub = Private API Hub = 私有API中心 4 | 5 | Stars Badge 6 | Forks Badge 7 | Issues Badge 8 | License Badge 9 | 10 | # 背景 11 | 12 | 当前全球BT/PT站点非常多，这些站点的技术架构大多类似，但技术非常古老。如果我们想基于这些资源站点，做一些使用场景的扩展，是找不到合适的开放接口的。本项目的初始目标，既为了解决这个痛点，自动为这些资源站点提供标准化的接口，供我们二次开发，或者DIY一些场景。 13 | 14 | # 安装 15 | 16 | # 使用 17 | 18 | # 开发者调试 19 | 20 | ## 安装依赖 21 | 22 | 项目根目录安装Python依赖 23 | 24 | ```shell 25 | pip install -r requirements.txt 26 | ``` 27 | 28 | 进入前端项目目录，安装前端依赖 29 | 30 | ``` shell 31 | cd papihub-frontend 32 | ``` 33 | 34 | ```yarn install``` or ```npm install``` 35 | 36 | ## 设置环境变量 37 | 38 | 变量名：WORKDIR 39 | 值为你的本地路径，项目运行时路径，此路径会存放项目的数据文件。命令行或者IDE启动，都需要设置此环境变量。 40 | 41 | ## 运行入口 42 | 43 | Python项目的运行入口在 ```papihub/main.py``` 44 | 45 | 前端项目的运行入口在 ```papihub-frontend``` 目录，使用```yarn dev``` 或 ```npm dev``` 运行项目 -------------------------------------------------------------------------------- /papihub-frontend/.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "next/core-web-vitals" 3 | } 4 | -------------------------------------------------------------------------------- /papihub-frontend/.gitignore: -------------------------------------------------------------------------------- 1 | # See https://help.github.com/articles/ignoring-files/ for more about ignoring files. 2 | _static 3 | _static/ 4 | .idea 5 | .vscode 6 | # dependencies 7 | /node_modules 8 | /.pnp 9 | .pnp.js 10 | 11 | # testing 12 | /coverage 13 | 14 | # production 15 | /build 16 | 17 | # misc 18 | .DS_Store 19 | .env 20 | .env.development.local 21 | .env.test.local 22 | .env.production.local 23 | .eslintcache 24 | 25 | npm-debug.log* 26 | yarn-debug.log* 27 | yarn-error.log* 28 | package-lock.json 29 | yarn.lock 30 | stats.html 31 | 32 | .next 33 | .next/* 34 | -------------------------------------------------------------------------------- /papihub-frontend/LICENSE: -------------------------------------------------------------------------------- 1 | BSD 3-Clause License 2 | 3 | Copyright (c) 2023, papihubcom 4 | 5 | Redistribution and use in source and binary forms, with or without 6 | modification, are permitted provided that the following conditions are met: 7 | 8 | 1. Redistributions of source code must retain the above copyright notice, this 9 | list of conditions and the following disclaimer. 10 | 11 | 2. Redistributions in binary form must reproduce the above copyright notice, 12 | this list of conditions and the following disclaimer in the documentation 13 | and/or other materials provided with the distribution. 14 | 15 | 3. Neither the name of the copyright holder nor the names of its 16 | contributors may be used to endorse or promote products derived from 17 | this software without specific prior written permission. 18 | 19 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 22 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 23 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 25 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 26 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 28 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /papihub-frontend/README.md: -------------------------------------------------------------------------------- 1 | This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app). 2 | 3 | ## Getting Started 4 | 5 | First, run the development server: 6 | 7 | ```bash 8 | npm run dev 9 | # or 10 | yarn dev 11 | # or 12 | pnpm dev 13 | ``` 14 | 15 | Open [http://localhost:3000](http://localhost:3000) with your browser to see the result. 16 | 17 | You can start editing the page by modifying `app/page.js`. The page auto-updates as you edit the file. 18 | 19 | This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font. 20 | 21 | ## Learn More 22 | 23 | To learn more about Next.js, take a look at the following resources: 24 | 25 | - [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API. 26 | - [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial. 27 | 28 | You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome! 29 | 30 | ## Deploy on Vercel 31 | 32 | The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js. 33 | 34 | Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details. 35 | -------------------------------------------------------------------------------- /papihub-frontend/jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "paths": { 4 | "@/*": ["./src/*"] 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /papihub-frontend/next.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('next').NextConfig} */ 2 | const API_BASE = process.env.API_BASE; 3 | const nextConfig = { 4 | async rewrites() { 5 | return [ 6 | { 7 | source: '/api/:path*', 8 | destination: `${API_BASE}/api/:path*`, 9 | }, 10 | ] 11 | }, 12 | } 13 | 14 | module.exports = nextConfig 15 | -------------------------------------------------------------------------------- /papihub-frontend/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "papihub-frontend", 3 | "version": "0.1.0", 4 | "private": true, 5 | "scripts": { 6 | "dev": "next dev", 7 | "build": "next build", 8 | "start": "next start", 9 | "lint": "next lint" 10 | }, 11 | "dependencies": { 12 | "@headlessui/react": "^1.7.16", 13 | "@heroicons/react": "^2.0.18", 14 | "@hookform/resolvers": "^3.3.1", 15 | "@tailwindcss/forms": "^0.5.6", 16 | "autoprefixer": "10.4.14", 17 | "axios": "^1.5.0", 18 | "classnames": "^2.3.2", 19 | "date-fns": "^2.30.0", 20 | "eslint": "8.46.0", 21 | "eslint-config-next": "13.4.12", 22 | "next": "13.4.12", 23 | "postcss": "8.4.27", 24 | "prop-types": "^15.8.1", 25 | "react": "^18.2.0", 26 | "react-dom": "^18.2.0", 27 | "react-hook-form": "^7.46.1", 28 | "react-query": "^3.39.3", 29 | "tailwindcss": "3.3.3", 30 | "yup": "^1.2.0", 31 | "zustand": "^4.4.1" 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /papihub-frontend/postcss.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | plugins: { 3 | tailwindcss: {}, 4 | autoprefixer: {}, 5 | }, 6 | } 7 | -------------------------------------------------------------------------------- /papihub-frontend/public/icons/pt/chdbits.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/public/icons/pt/chdbits.ico -------------------------------------------------------------------------------- /papihub-frontend/public/icons/pt/hdsky.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/public/icons/pt/hdsky.ico -------------------------------------------------------------------------------- /papihub-frontend/public/icons/pt/mteam.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/public/icons/pt/mteam.ico -------------------------------------------------------------------------------- /papihub-frontend/public/icons/pt/ourbits.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/public/icons/pt/ourbits.ico -------------------------------------------------------------------------------- /papihub-frontend/public/next.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /papihub-frontend/public/vercel.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/apidoc/layout.js: -------------------------------------------------------------------------------- 1 | import MainLayout from "@/app/components/layout/main-layout"; 2 | 3 | export default function Layout({children}) { 4 | return ( 5 | {children} 6 | ) 7 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/apidoc/page.js: -------------------------------------------------------------------------------- 1 | import MainPageHeader from "@/app/components/layout/header/main-page-header"; 2 | 3 | export default function Page() { 4 | return (
5 | 6 |
) 7 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/auth/layout.js: -------------------------------------------------------------------------------- 1 | import AuthLayout from "@/app/components/layout/auth-layout"; 2 | 3 | export default function Layout({children}) { 4 | return ( 5 | {children} 6 | ) 7 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/auth/login/page.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import Button from "@/app/components/button/button"; 3 | import Input from "@/app/components/hook-form/input"; 4 | import FormProvider from "@/app/components/hook-form/form-provider"; 5 | import {useForm} from "react-hook-form"; 6 | import * as Yup from "yup"; 7 | import {yupResolver} from "@hookform/resolvers/yup"; 8 | import useUserStore from "@/auth/store/use-user-store"; 9 | import {useEffect} from "react"; 10 | import {useRouter, useSearchParams} from "next/navigation"; 11 | import {XCircleIcon} from "@heroicons/react/24/solid"; 12 | 13 | export default function Page() { 14 | const router = useRouter(); 15 | const searchParams = useSearchParams() 16 | 17 | const { 18 | login, 19 | authenticated, 20 | isInitializing, 21 | hasError, 22 | errorMessage 23 | } = useUserStore(); 24 | useEffect(() => { 25 | if (isInitializing) { 26 | return; 27 | } 28 | if (authenticated) { 29 | if (searchParams.get("returnTo")) { 30 | router.replace(searchParams.get("returnTo")); 31 | } else { 32 | router.replace('/'); 33 | } 34 | } 35 | }, [isInitializing, authenticated, router]) 36 | const LoginSchema = Yup.object().shape({ 37 | username: Yup.string().required('必须填写用户名'), 38 | password: Yup.string() 39 | .required('必须填写密码') 40 | .min(6, '密码是一个大于6位的字符串'), 41 | }); 42 | const methods = useForm({ 43 | resolver: yupResolver(LoginSchema), 44 | }); 45 | const { 46 | handleSubmit, 47 | formState: {isSubmitting}, 48 | } = methods; 49 | 50 | const onSubmit = handleSubmit(async (data) => { 51 | login(data.username, data.password); 52 | }); 53 | return (
54 |
56 |
57 | PapiHub 62 |

63 | 登入PapiHub 64 |

65 |
66 |
67 | {hasError &&
68 |
69 |
70 |
72 |
73 |

{errorMessage}

74 |
75 |
76 |
} 77 | 79 | 84 | 90 | 忘记密码? 91 | } 92 | /> 93 |
94 | 95 |
96 | 97 |
98 |
99 |
); 100 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/badge/badge.js: -------------------------------------------------------------------------------- 1 | import classNames from "classnames"; 2 | 3 | const fontColors = { 4 | primary: "text-indigo-400", 5 | info: "text-blue-400", 6 | success: "text-green-400", 7 | error: "text-red-700" 8 | }; 9 | const borderColors = { 10 | primary: "ring-1 ring-inset ring-indigo-400/30", 11 | info: "ring-1 ring-inset ring-blue-400/30", 12 | success: "ring-1 ring-inset ring-green-500/20", 13 | error: "ring-1 ring-inset ring-red-600/10" 14 | } 15 | const transparentBgColors = { 16 | primary: "bg-indigo-400/10", 17 | info: "bg-blue-400/10", 18 | success: "bg-green-500/10", 19 | error: "bg-red-400/10" 20 | } 21 | const bgColor = { 22 | primary: "bg-indigo-50", 23 | info: "bg-blue-50", 24 | success: "bg-green-50", 25 | error: "bg-red-50" 26 | } 27 | const dotColors = { 28 | primary: "fill-indigo-400", 29 | info: "fill-blue-400", 30 | success: "fill-green-400", 31 | error: "fill-red-400" 32 | 33 | }; 34 | export default function Badge({ 35 | color, 36 | dot = false, 37 | fillBgColor = true, 38 | transparentBgColor = true, 39 | withBorder = true, 40 | children 41 | }) { 42 | return ( 53 | {dot && } 58 | {children} 59 | ); 60 | }; -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/breadcrumbs/breadcrumbs.js: -------------------------------------------------------------------------------- 1 | import {ChevronLeftIcon, ChevronRightIcon} from "@heroicons/react/20/solid"; 2 | import Link from "next/link"; 3 | import classNames from "classnames"; 4 | 5 | export default function Breadcrumbs({ 6 | navigation 7 | }) { 8 | return ( 9 |
10 | 19 | 41 |
) 42 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/button/button.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import classNames from "classnames"; 3 | import {useRouter} from "next/navigation"; 4 | 5 | const sizes = { 6 | small: "px-2.5 py-1.5 text-sm", 7 | medium: "px-3 py-2 text-sm", 8 | large: "px-3.5 py-2.5 text-sm", 9 | } 10 | const colors = { 11 | primary: "bg-indigo-500 hover:bg-indigo-400 focus-visible:outline-indigo-500 text-white", 12 | secondary: "bg-white/10 hover:bg-white/20 text-white", 13 | } 14 | export default function Button( 15 | { 16 | type = 'button', 17 | size = 'medium', 18 | color = 'primary', 19 | className = null, 20 | href = null, 21 | onClick, 22 | disabled, 23 | children, 24 | } 25 | ) { 26 | const router = useRouter(); 27 | const handleClick = (e) => { 28 | if (href) { 29 | e.preventDefault(); 30 | router.push(href); 31 | } else if (onClick) { 32 | onClick(e); 33 | } 34 | } 35 | return (); 50 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/footer/footer.js: -------------------------------------------------------------------------------- 1 | const navigation = { 2 | main: [], 3 | social: [ 4 | { 5 | name: 'Twitter', 6 | href: '#', 7 | icon: (props) => ( 8 | 9 | 11 | 12 | ), 13 | }, 14 | { 15 | name: 'GitHub', 16 | href: 'https://github.com/papihubcom/papihub', 17 | icon: (props) => ( 18 | 19 | 24 | 25 | ), 26 | }, 27 | { 28 | name: 'Discord', 29 | href: '#', 30 | icon: (props) => ( 31 | 32 | 34 | 36 | 37 | ), 38 | }, 39 | ], 40 | } 41 | export default function Footer() { 42 | return ( 43 | 72 | ) 73 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/hook-form/form-provider.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import PropTypes from 'prop-types'; 3 | import {FormProvider as Form} from 'react-hook-form'; 4 | 5 | export default function FormProvider({ 6 | className, 7 | children, 8 | onSubmit, 9 | methods 10 | }) { 11 | return ( 12 |
13 | {children}
14 | 15 | ); 16 | } 17 | 18 | FormProvider.propTypes = { 19 | className: PropTypes.string, 20 | children: PropTypes.node, 21 | methods: PropTypes.object, 22 | onSubmit: PropTypes.func, 23 | }; 24 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/hook-form/input.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import PropTypes from 'prop-types'; 3 | import {Controller, useFormContext} from 'react-hook-form'; 4 | import classNames from "classnames"; 5 | import {ExclamationCircleIcon} from "@heroicons/react/20/solid"; 6 | 7 | // ---------------------------------------------------------------------- 8 | 9 | export default function Input({ 10 | label, 11 | name, 12 | helperText, 13 | cornerHint, 14 | type, 15 | className, 16 | ...other 17 | }) { 18 | const {control} = useFormContext(); 19 | 20 | return ( 21 | ( 25 |
26 |
27 | 31 |
32 | {cornerHint} 33 |
34 |
35 |
41 | { 56 | if (type === 'number') { 57 | field.onChange(Number(event.target.value)); 58 | } else { 59 | field.onChange(event.target.value); 60 | } 61 | }} 62 | {...other} 63 | /> 64 | {error &&
66 |
} 69 |
70 | {(helperText || error) &&

75 | {error ? error?.message : helperText} 76 |

} 77 |
78 | )} 79 | /> 80 | ); 81 | } 82 | 83 | Input.propTypes = { 84 | label: PropTypes.string, 85 | cornerHint: PropTypes.object, 86 | helperText: PropTypes.object, 87 | name: PropTypes.string, 88 | type: PropTypes.string, 89 | }; 90 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/hook-form/select.js: -------------------------------------------------------------------------------- 1 | "use client" 2 | import {Fragment} from 'react' 3 | 4 | import classNames from "classnames"; 5 | import {CheckIcon, ChevronUpDownIcon} from "@heroicons/react/24/solid"; 6 | import {Listbox, Transition} from "@headlessui/react"; 7 | import {Controller, useFormContext} from "react-hook-form"; 8 | import {ExclamationCircleIcon} from "@heroicons/react/20/solid"; 9 | 10 | export default function Select({ 11 | name, 12 | label, 13 | cornerHint, 14 | options, 15 | helperText 16 | }) { 17 | const {control} = useFormContext(); 18 | 19 | return ( 20 | ( 24 |
25 |
26 | 30 |
31 | {cornerHint} 32 |
33 |
34 |
40 | 41 | {({open}) => ( 42 | <> 43 |
44 | 46 | 47 | {field.value?.icon} 48 | {field.value?.label} 50 | 51 | 53 | 56 | 57 | 64 | 66 | {options.map((item, index) => ( 67 | 70 | classNames( 71 | active 72 | ? 'bg-indigo-600 text-white' 73 | : 'text-gray-900', 74 | 'relative cursor-default select-none py-2 pl-3 pr-9' 75 | ) 76 | } 77 | value={item} 78 | > 79 | {({selected, active}) => ( 80 | <> 81 |
82 | {item.icon} 83 | 89 | {item.label} 90 | 91 |
92 | 93 | {selected ? ( 94 | 101 | 103 | ) : null} 104 | 105 | )} 106 | 107 | ))} 108 | 109 | 110 |
111 | 112 | )} 113 | {error &&
115 |
} 118 |
119 | {(helperText || error) &&

124 | {error ? error?.message : helperText} 125 |

} 126 |
127 | )} 128 | /> 129 | ) 130 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/layout/auth-layout.js: -------------------------------------------------------------------------------- 1 | import Footer from "@/app/components/footer/footer"; 2 | 3 | export default function AuthLayout({children}) { 4 | return (
5 |
7 | 25 | {/* Footer */} 26 |
27 |
) 28 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/layout/header/main-page-header.js: -------------------------------------------------------------------------------- 1 | import Breadcrumbs from "@/app/components/breadcrumbs/breadcrumbs"; 2 | 3 | export default function MainPageHeader({ 4 | title, 5 | breadcrumbNavs, 6 | actions 7 | }) { 8 | return (
9 | 10 |
11 |
12 |

13 | {title} 14 |

15 |
16 | {actions &&
17 | {actions} 18 |
} 19 |
20 |
); 21 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/layout/main-layout.js: -------------------------------------------------------------------------------- 1 | import SimpleLayout from "@/app/components/layout/simple-layout"; 2 | 3 | const navigation = [ 4 | {name: '首页', href: '/'}, 5 | {name: '网站配置', href: '/site'}, 6 | {name: '使用接口', href: '/apidoc'}, 7 | ] 8 | export default function MainLayout({children}) { 9 | return ( 10 |
12 | 30 | ) 31 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/layout/simple-layout.js: -------------------------------------------------------------------------------- 1 | import NavMobile from "@/app/components/nav/mobile/nav-mobile"; 2 | import NavDesktop from "@/app/components/nav/desktop/nav-desktop"; 3 | import Footer from "@/app/components/footer/footer"; 4 | import NavUserDesktop from "@/app/components/nav/desktop/nav-user-desktop"; 5 | 6 | const navigation = [ 7 | {name: '首页', href: '/'}, 8 | {name: '网站配置', href: '/site'}, 9 | {name: '使用接口', href: '/apidoc'}, 10 | ] 11 | export default function SimpleLayout({children}) { 12 | return (
13 | {/* Header */} 14 |
15 | 35 | 36 |
37 |
38 | {children} 39 |
40 | {/* Footer */} 41 |
42 |
) 43 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/nav/desktop/nav-desktop.js: -------------------------------------------------------------------------------- 1 | import Link from "next/link"; 2 | 3 | export default function NavDesktop({data}) { 4 | return (<> 5 | {data.map((item) => ( 6 | 8 | {item.name} 9 | 10 | ))} 11 | ); 12 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/nav/desktop/nav-user-desktop.js: -------------------------------------------------------------------------------- 1 | "use client" 2 | import {Fragment} from 'react' 3 | 4 | import Link from "next/link"; 5 | import useUserStore from "@/auth/store/use-user-store"; 6 | import {Menu, Transition} from "@headlessui/react"; 7 | import classNames from "classnames"; 8 | import {ChevronDownIcon} from "@heroicons/react/24/solid"; 9 | import {useRouter} from "next/navigation"; 10 | 11 | export default function NavUserDesktop() { 12 | const router = useRouter(); 13 | const {authenticated, user, logout} = useUserStore(); 14 | return (
15 | {!authenticated && 17 | 登录 18 | } 19 | {authenticated && 20 |
21 | 23 | {user?.nickname} 24 | 26 |
27 | 36 | 38 | 39 | {({active}) => ( 40 | { 46 | logout(); 47 | router.push("/"); 48 | }} 49 | > 50 | 退出 51 | 52 | )} 53 | 54 | 55 | 56 | } 57 |
); 58 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/nav/mobile/nav-mobile.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import {Bars3Icon, XMarkIcon} from "@heroicons/react/24/outline"; 3 | import {useState} from "react"; 4 | import {Dialog} from "@headlessui/react"; 5 | export default function NavMobile({data}) { 6 | const [mobileMenuOpen, setMobileMenuOpen] = useState(false) 7 | 8 | return (<> 9 | 17 | 18 |
19 | 20 |
21 | 22 | Your Company 23 | 28 | 29 | 37 |
38 |
39 |
40 |
41 | {data.map((item) => ( 42 | 47 | {item.name} 48 | 49 | ))} 50 |
51 | 59 |
60 |
61 | 62 |
63 | ); 64 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/page-sections/site/edit-form.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import FormProvider from "@/app/components/hook-form/form-provider"; 3 | import Select from "@/app/components/hook-form/select"; 4 | import {Controller, useForm} from "react-hook-form"; 5 | import Input from "@/app/components/hook-form/input"; 6 | import {RadioGroup} from "@headlessui/react"; 7 | import {CheckCircleIcon} from "@heroicons/react/24/solid"; 8 | import classNames from "classnames"; 9 | import {useAddSite, useListParsers} from "@/service/site-service"; 10 | import {useEffect, useState} from "react"; 11 | import * as Yup from "yup"; 12 | import {yupResolver} from "@hookform/resolvers/yup"; 13 | import Button from "@/app/components/button/button"; 14 | import {useRouter} from "next/navigation"; 15 | 16 | const authOptions = [{ 17 | id: 1, 18 | label: "Cookies认证", 19 | desc: "登录站点后，通过浏览器抓包获取。", 20 | value: "cookies" 21 | }, { 22 | id: 2, 23 | label: "登录认证", 24 | desc: "直接使用用户名密码认证，少量站点支持。", 25 | value: "user_auth" 26 | }]; 27 | export default function SiteEditForm() { 28 | const router = useRouter(); 29 | const {data} = useListParsers(); 30 | const {mutate: addSite, isLoading} = useAddSite(); 31 | const [siteOptions, setSiteOptions] = useState([]); 32 | useEffect(() => { 33 | setSiteOptions(data?.data?.map((item) => { 34 | return { 35 | icon: , 37 | label: item.site_name, 38 | value: item.site_id, 39 | } 40 | }) || []); 41 | }, [data]); 42 | const formSchema = Yup.object().shape({ 43 | authType: Yup.string(), 44 | cookies: Yup.string() 45 | .when('authType', ([authType], schema) => 46 | authType === 'cookies' ? schema.required('请填写一个有效的Cookies') 47 | : schema, 48 | ), 49 | username: Yup.string() 50 | .when('authType', ([authType], schema) => 51 | authType === 'user_auth' ? schema.required('用户名必填') : schema, 52 | ), 53 | password: Yup.string() 54 | .when('authType', ([authType], schema) => 55 | authType === 'user_auth' ? schema.required('密码必填') : schema, 56 | ), 57 | }); 58 | const methods = useForm({ 59 | resolver: yupResolver(formSchema), 60 | defaultValues: { 61 | authType: authOptions[0].value 62 | } 63 | }); 64 | const { 65 | setValue, 66 | watch, 67 | control, 68 | register, 69 | handleSubmit, 70 | formState: {isSubmitting}, 71 | } = methods; 72 | const authType = watch("authType"); 73 | const onSubmit = handleSubmit(async (data) => { 74 | const params = { 75 | site_id: data.siteId.value, 76 | auth_type: data.authType, 77 | auth_config: data.authType === "cookies" ? { 78 | cookies: data.cookies 79 | } : { 80 | username: data.username, 81 | password: data.password 82 | } 83 | }; 84 | console.log(params) 85 | addSite(params, { 86 | onSuccess: res => { 87 | const {success, message, data} = res; 88 | if (success) { 89 | //todo 提示成功 90 | router.push("/site"); 91 | } else { 92 | //todo 提示失败 93 | } 94 | } 95 | }); 96 | }); 97 | useEffect(() => { 98 | if (siteOptions && siteOptions.length > 0) { 99 | setValue("siteId", siteOptions[0]) 100 | } 101 | }, [siteOptions]) 102 | return ( 103 | 104 |
105 |
107 |
108 | 181 |
182 | } 183 | {authType === "user_auth" &&
184 |
185 | 187 | 189 |
190 |
} 191 |
192 |
193 |
194 | 195 |
196 | 199 |
200 | ) 201 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/page-sections/site/list.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import SiteOptions from "@/app/components/page-sections/site/options"; 3 | import Badge from "@/app/components/badge/badge"; 4 | import {useListSite} from "@/service/site-service"; 5 | import {fToNow} from "@/utils/format-time"; 6 | 7 | const statusMessage = { 8 | pending: "待检测", 9 | active: "可用", 10 | error: "不可用" 11 | } 12 | const statusColor = { 13 | pending: "info", 14 | active: "success", 15 | error: "error" 16 | } 17 | 18 | export default function SiteList() { 19 | const {data} = useListSite(); 20 | const siteList = data?.data || []; 21 | return (
    23 | {siteList && siteList.map((item) => ( 24 |
  • 26 |
    28 | {item.site_name} 33 |
    {item.site_name}
    35 | 36 |
    37 |
    38 |
    39 |
    上次访问
    40 |
    41 | 44 |
    45 |
    46 |
    47 |
    状态
    48 | 49 | {statusMessage[item.site_status]} 50 | 51 |
    52 |
    53 |
    • 54 | ))} 55 |
); 56 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/components/page-sections/site/options.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import {Menu, Transition} from "@headlessui/react"; 3 | import {EllipsisHorizontalIcon} from "@heroicons/react/20/solid"; 4 | import {Fragment} from "react"; 5 | import classNames from "classnames"; 6 | 7 | export default function SiteOptions() { 8 | return ( 9 | 11 | Open options 12 | 15 | 24 | 26 | 27 | {({active}) => ( 28 | 35 | 详情 36 | 37 | )} 38 | 39 | 40 | {({active}) => ( 41 | 48 | 编辑 49 | 50 | )} 51 | 52 | 53 | {({active}) => ( 54 | 61 | 删除 62 | 63 | )} 64 | 65 | 66 | 67 | ); 68 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/src/app/favicon.ico -------------------------------------------------------------------------------- /papihub-frontend/src/app/globals.css: -------------------------------------------------------------------------------- 1 | @tailwind base; 2 | @tailwind components; 3 | @tailwind utilities; 4 | 5 | :root { 6 | --foreground-rgb: 0, 0, 0; 7 | --background-start-rgb: 214, 219, 220; 8 | --background-end-rgb: 255, 255, 255; 9 | } 10 | 11 | @media (prefers-color-scheme: dark) { 12 | :root { 13 | --foreground-rgb: 255, 255, 255; 14 | --background-start-rgb: 0, 0, 0; 15 | --background-end-rgb: 0, 0, 0; 16 | } 17 | } 18 | 19 | body { 20 | color: rgb(var(--foreground-rgb)); 21 | background: linear-gradient( 22 | to bottom, 23 | transparent, 24 | rgb(var(--background-end-rgb)) 25 | ) 26 | rgb(var(--background-start-rgb)); 27 | } 28 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/layout.js: -------------------------------------------------------------------------------- 1 | import './globals.css' 2 | import {Inter} from 'next/font/google' 3 | import AuthProvider from "@/auth/auth-provider"; 4 | import {QueryProvider} from "@/context/query-provider"; 5 | 6 | const inter = Inter({subsets: ['latin']}) 7 | 8 | export const metadata = { 9 | title: 'PapiHub', 10 | description: 'Generated by create next app', 11 | } 12 | 13 | export default function RootLayout({children}) { 14 | return ( 15 | 16 | 17 | 18 | 19 | {children} 20 | 21 | 22 | 23 | 24 | ) 25 | } 26 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/page.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | 3 | import MainLayout from "@/app/components/layout/main-layout"; 4 | import SimpleLayout from "@/app/components/layout/simple-layout"; 5 | 6 | export default function Home() { 7 | 8 | return ( 9 | 10 |
12 | 46 | 47 | ) 48 | } 49 | -------------------------------------------------------------------------------- /papihub-frontend/src/app/site/add/page.js: -------------------------------------------------------------------------------- 1 | import SiteEditForm from "@/app/components/page-sections/site/edit-form"; 2 | import MainPageHeader from "@/app/components/layout/header/main-page-header"; 3 | 4 | export default function Page() { 5 | return ( 6 |
7 | 20 | 21 |
22 | ) 23 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/site/layout.js: -------------------------------------------------------------------------------- 1 | import MainLayout from "@/app/components/layout/main-layout"; 2 | import {AuthGuard} from "@/auth/guard/auth-guard"; 3 | 4 | export default function Layout({children}) { 5 | return ( 6 | 7 | 8 | {children} 9 | 10 | 11 | ) 12 | } -------------------------------------------------------------------------------- /papihub-frontend/src/app/site/page.js: -------------------------------------------------------------------------------- 1 | import MainPageHeader from "@/app/components/layout/header/main-page-header"; 2 | import Button from "@/app/components/button/button"; 3 | import SiteList from "@/app/components/page-sections/site/list"; 4 | 5 | export const metadata = { 6 | title: '网站配置 | PapiHub', 7 | description: '配置系统内可以抓取访问的站点。', 8 | } 9 | export default function Page() { 10 | 11 | return (
12 | 添加站点} 25 | /> 26 | 27 |
) 28 | } -------------------------------------------------------------------------------- /papihub-frontend/src/auth/auth-provider.js: -------------------------------------------------------------------------------- 1 | "use client" 2 | import useUserStore from "@/auth/store/use-user-store"; 3 | import {useEffect, useState} from "react"; 4 | 5 | export default function AuthProvider({children}) { 6 | const {authenticated, isInitializing, initializeMethod} = useUserStore(); 7 | const [checked, setChecked] = useState(false); 8 | useEffect(() => { 9 | if (!isInitializing) { 10 | return; 11 | } 12 | initializeMethod(); 13 | }, [initializeMethod, isInitializing]); 14 | return ( 15 | <> 16 | {children} 17 | 18 | ); 19 | } -------------------------------------------------------------------------------- /papihub-frontend/src/auth/guard/auth-guard.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import useUserStore from "@/auth/store/use-user-store"; 3 | import {useCallback, useEffect, useState} from "react"; 4 | import {usePathname, useRouter} from "next/navigation"; 5 | 6 | export const AuthGuard = ({children}) => { 7 | const router = useRouter(); 8 | const pathname = usePathname(); 9 | const {authenticated, isInitializing} = useUserStore(); 10 | const [checked, setChecked] = useState(false); 11 | const check = useCallback(async () => { 12 | if (isInitializing) { 13 | return; 14 | } 15 | if (!authenticated) { 16 | const params = new URLSearchParams({returnTo: pathname}).toString(); 17 | router.replace(`/auth/login?${params}`); 18 | } else { 19 | setChecked(true); 20 | } 21 | }, [pathname, authenticated, router, isInitializing]); 22 | useEffect(() => { 23 | check(); 24 | }, [check]); 25 | if (!checked) { 26 | return null; 27 | } 28 | return <>{children}; 29 | } -------------------------------------------------------------------------------- /papihub-frontend/src/auth/guard/guest-guard.js: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub-frontend/src/auth/guard/guest-guard.js -------------------------------------------------------------------------------- /papihub-frontend/src/auth/store/use-user-store.js: -------------------------------------------------------------------------------- 1 | import {create} from "zustand"; 2 | import {setSession} from "@/auth/utils"; 3 | import axios from "@/utils/axios"; 4 | 5 | const useUserStore = create((set, get) => ({ 6 | isInitializing: true, 7 | user: null, 8 | authenticated: false, 9 | hasError: false, 10 | errorMessage: null, 11 | initializeMethod: async () => { 12 | const accessToken = window.localStorage.getItem("accessToken"); 13 | setSession(accessToken); 14 | try { 15 | const response = await axios.get("/api/user/profile"); 16 | const {success, data} = response; 17 | if (success) { 18 | set({user: data, authenticated: true, isInitializing: false}); 19 | } else { 20 | setSession(null) 21 | set({isInitializing: false}); 22 | } 23 | } catch (e) { 24 | setSession(null); 25 | set({isInitializing: false}); 26 | } 27 | }, 28 | login: async (username, password) => { 29 | let formData = new FormData(); 30 | formData.append("username", username); 31 | formData.append("password", password); 32 | const response = await axios.post( 33 | "/api/user/get_token", 34 | formData 35 | ); 36 | const {success, data, message} = response; 37 | if (success) { 38 | setSession(data.access_token); 39 | set({user: data.user, authenticated: true, isInitializing: true}); 40 | } else { 41 | set({hasError: true, errorMessage: message}); 42 | } 43 | }, 44 | logout: () => { 45 | setSession(null); 46 | set({user: null, authenticated: false, isInitializing: true}); 47 | } 48 | })); 49 | 50 | export default useUserStore; -------------------------------------------------------------------------------- /papihub-frontend/src/auth/utils.js: -------------------------------------------------------------------------------- 1 | import axios from "@/utils/axios"; 2 | 3 | const setSession = (accessToken) => { 4 | if (accessToken) { 5 | localStorage.setItem("accessToken", accessToken); 6 | axios.defaults.headers.common.Authorization = `Bearer ${accessToken}`; 7 | // This function below will handle when token is expired 8 | // const { exp } = jwtDecode(accessToken); 9 | // handleTokenExpired(exp); 10 | } else { 11 | localStorage.removeItem("accessToken"); 12 | delete axios.defaults.headers.common.Authorization; 13 | } 14 | }; 15 | export {setSession}; -------------------------------------------------------------------------------- /papihub-frontend/src/context/query-provider.js: -------------------------------------------------------------------------------- 1 | "use client"; 2 | import React from 'react'; 3 | import {QueryClient, QueryClientProvider} from "react-query"; 4 | 5 | export const QueryProvider = ({children}) => { 6 | const queryClient = new QueryClient({ 7 | defaultOptions: { 8 | queries: { 9 | refetchOnWindowFocus: false, 10 | }, 11 | }, 12 | }); 13 | 14 | return ( 15 | 16 | {children} 17 | {/**/} 18 | 19 | ); 20 | 21 | }; 22 | -------------------------------------------------------------------------------- /papihub-frontend/src/hooks/use-http.js: -------------------------------------------------------------------------------- 1 | import axios from "@/utils/axios"; 2 | import { useCallback } from 'react'; 3 | 4 | const useHttp = () => { 5 | return useCallback( 6 | (...[endpoint, config]) => 7 | (config?.method?.toUpperCase() === "POST" 8 | ? axios.post(endpoint, config.params) 9 | : axios.get(endpoint, config)), 10 | [] 11 | ); 12 | }; 13 | 14 | export default useHttp; 15 | 16 | -------------------------------------------------------------------------------- /papihub-frontend/src/service/site-service.js: -------------------------------------------------------------------------------- 1 | import useHttp from "@/hooks/use-http"; 2 | import {useMutation, useQuery} from "react-query"; 3 | 4 | export const useListParsers = (param) => { 5 | const client = useHttp(); 6 | return useQuery(['list_parsers', param], () => 7 | client("/api/site/list_parsers", {params: param}) 8 | ); 9 | }; 10 | export const useListSite = (param) => { 11 | const client = useHttp(); 12 | return useQuery(['list_site', param], () => 13 | client("/api/site/list", {params: param}) 14 | ); 15 | }; 16 | export const useAddSite = (param) => { 17 | const client = useHttp(); 18 | return useMutation( 19 | (params) => 20 | client("/api/site/add", {params: params, method: "POST"}) 21 | ); 22 | }; -------------------------------------------------------------------------------- /papihub-frontend/src/utils/axios.js: -------------------------------------------------------------------------------- 1 | // config 2 | 3 | // ---------------------------------------------------------------------- 4 | 5 | import axios from "axios"; 6 | 7 | const axiosInstance = axios.create({ 8 | validateStatus: function (status) { 9 | return true; // 任何状态码都视为成功 10 | } 11 | }); 12 | 13 | axiosInstance.interceptors.response.use( 14 | (response) => { 15 | const res = response.data; 16 | return {...res, response}; 17 | }, 18 | (error) => Promise.reject( 19 | (error.response && error.response.data) || 'Something went wrong') 20 | ); 21 | 22 | export default axiosInstance; 23 | -------------------------------------------------------------------------------- /papihub-frontend/src/utils/format-time.js: -------------------------------------------------------------------------------- 1 | import {format, formatDistanceToNow, getTime} from 'date-fns'; 2 | import {zhCN} from 'date-fns/locale'; 3 | // ---------------------------------------------------------------------- 4 | 5 | export function fDate(date, newFormat) { 6 | const fm = newFormat || 'dd MMM yyyy'; 7 | 8 | return date ? format(new Date(date), fm) : ''; 9 | } 10 | 11 | export function fDateTime(date, newFormat) { 12 | const fm = newFormat || 'dd MMM yyyy p'; 13 | 14 | return date ? format(new Date(date), fm) : ''; 15 | } 16 | 17 | export function fTimestamp(date) { 18 | return date ? getTime(new Date(date)) : ''; 19 | } 20 | 21 | export function fToNow(date) { 22 | return date 23 | ? formatDistanceToNow(new Date(date), { 24 | addSuffix: true, 25 | locale: zhCN 26 | }) 27 | : ''; 28 | } 29 | -------------------------------------------------------------------------------- /papihub-frontend/tailwind.config.js: -------------------------------------------------------------------------------- 1 | /** @type {import('tailwindcss').Config} */ 2 | module.exports = { 3 | content: [ 4 | './src/pages/**/*.{js,ts,jsx,tsx,mdx}', 5 | './src/components/**/*.{js,ts,jsx,tsx,mdx}', 6 | './src/app/**/*.{js,ts,jsx,tsx,mdx}', 7 | ], 8 | theme: { 9 | extend: { 10 | backgroundImage: { 11 | 'gradient-radial': 'radial-gradient(var(--tw-gradient-stops))', 12 | 'gradient-conic': 13 | 'conic-gradient(from 180deg at 50% 50%, var(--tw-gradient-stops))', 14 | }, 15 | }, 16 | }, 17 | plugins: [ 18 | require('@tailwindcss/forms'), 19 | ], 20 | } 21 | -------------------------------------------------------------------------------- /papihub/.gitignore: -------------------------------------------------------------------------------- 1 | .vs/ 2 | .vscode/ 3 | .idea/ 4 | # Byte-compiled / optimized / DLL files 5 | __pycache__/ 6 | *.py[cod] 7 | *$py.class 8 | 9 | # C extensions 10 | *.so 11 | 12 | # Distribution / packaging 13 | .Python 14 | build/ 15 | develop-eggs/ 16 | dist/ 17 | downloads/ 18 | eggs/ 19 | .eggs/ 20 | lib/ 21 | lib64/ 22 | parts/ 23 | sdist/ 24 | var/ 25 | wheels/ 26 | pip-wheel-metadata/ 27 | share/python-wheels/ 28 | *.egg-info/ 29 | .installed.cfg 30 | *.egg 31 | MANIFEST 32 | 33 | # PyInstaller 34 | # Usually these files are written by a python script from a template 35 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 36 | *.manifest 37 | *.spec 38 | 39 | # Installer logs 40 | pip-log.txt 41 | pip-delete-this-directory.txt 42 | 43 | # Unit test / coverage reports 44 | htmlcov/ 45 | .tox/ 46 | .nox/ 47 | .coverage 48 | .coverage.* 49 | .cache 50 | nosetests.xml 51 | coverage.xml 52 | *.cover 53 | *.py,cover 54 | .hypothesis/ 55 | .pytest_cache/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | db.sqlite3 65 | db.sqlite3-journal 66 | 67 | # Flask stuff: 68 | instance/ 69 | .webassets-cache 70 | 71 | # Scrapy stuff: 72 | .scrapy 73 | 74 | # Sphinx documentation 75 | docs/_build/ 76 | 77 | # PyBuilder 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | notebooks/ 83 | 84 | # IPython 85 | profile_default/ 86 | ipython_config.py 87 | 88 | # pyenv 89 | .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .envrc 111 | .venv 112 | .venvs 113 | env/ 114 | venv/ 115 | ENV/ 116 | env.bak/ 117 | venv.bak/ 118 | 119 | # Spyder project settings 120 | .spyderproject 121 | .spyproject 122 | 123 | # Rope project settings 124 | .ropeproject 125 | 126 | # mkdocs documentation 127 | /site 128 | 129 | # mypy 130 | .mypy_cache/ 131 | .dmypy.json 132 | dmypy.json 133 | 134 | # Pyre type checker 135 | .pyre/ 136 | 137 | # macOS display setting files 138 | .DS_Store 139 | 140 | # Wandb directory 141 | wandb/ 142 | 143 | # asdf tool versions 144 | .tool-versions 145 | /.ruff_cache/ 146 | 147 | *.pkl 148 | *.bin 149 | 150 | # integration test artifacts 151 | data_map* 152 | \[('_type', 'fake'), ('stop', None)] -------------------------------------------------------------------------------- /papihub/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub/__init__.py -------------------------------------------------------------------------------- /papihub/api/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 私有接口的核心实现包 3 | """ 4 | -------------------------------------------------------------------------------- /papihub/api/auth.py: -------------------------------------------------------------------------------- 1 | import abc 2 | 3 | 4 | class Auth(metaclass=abc.ABCMeta): 5 | """ 6 | 站点认证信息类 7 | 不需要认证的站点，无需实现此接口 8 | """ 9 | 10 | @abc.abstractmethod 11 | def auth_with_cookies(self, cookies_str: str): 12 | """ 13 | 通过Cookies完成认证 14 | :param cookies_str: Cookies字符串 15 | :return: 16 | """ 17 | pass 18 | 19 | @abc.abstractmethod 20 | def auth(self, username: str, password: str) -> str: 21 | """ 22 | 通过用户名密码完成认证 23 | :param username: 用户名 24 | :param password: 密码 25 | :return: 返回授权后的cookies字符串 26 | """ 27 | pass 28 | 29 | @abc.abstractmethod 30 | def test_login(self) -> bool: 31 | """ 32 | 测试登录 33 | :return: 34 | """ 35 | pass 36 | -------------------------------------------------------------------------------- /papihub/api/concurrenttorrentsite.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from concurrent.futures import ThreadPoolExecutor 3 | from typing import List, Optional 4 | 5 | from papihub.api.torrentsite import TorrentSite 6 | from papihub.api.types import Torrent 7 | from papihub.exceptions import SiteApiErrorException 8 | 9 | _LOGGER = logging.getLogger(__name__) 10 | 11 | executor = ThreadPoolExecutor(max_workers=50) 12 | 13 | 14 | class ConcurrentTorrentSite: 15 | """ 16 | 并行种子站点操作类 17 | 传入很多站点实例后，开启多线程并行处理 18 | """ 19 | 20 | def __init__(self, torrent_sites: List[TorrentSite]): 21 | if not torrent_sites: 22 | raise SiteApiErrorException('没有可用的站点') 23 | self.torrent_sites = torrent_sites 24 | 25 | def search( 26 | self, 27 | keyword: str, 28 | imdb_id: Optional[str], 29 | cate_level1: Optional[List[str]] 30 | ): 31 | results: List[Torrent] = [] 32 | futures = [executor.submit(lambda: s.search( 33 | keyword=keyword, 34 | imdb_id=imdb_id, 35 | cate_level1_list=cate_level1 36 | )) for s in self.torrent_sites] 37 | for future in futures: 38 | try: 39 | r = future.result() 40 | if not r: 41 | continue 42 | results.extend(r) 43 | except Exception as e: 44 | _LOGGER.error(e) 45 | return results 46 | -------------------------------------------------------------------------------- /papihub/api/parser/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 各种解析器的包 3 | """ 4 | -------------------------------------------------------------------------------- /papihub/api/parser/htmlparser.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import logging 3 | import re 4 | import urllib 5 | 6 | from pyquery import PyQuery 7 | 8 | from papihub.exceptions import ParserFieldException 9 | 10 | _LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | def filter_querystring(value, args): 14 | if value.startswith('?'): 15 | value = value[1:] 16 | elif value.find('?') != -1: 17 | value = value.split('?')[1] 18 | qs = urllib.parse.parse_qs(value) 19 | value = qs.get(args) 20 | if value: 21 | value = value[0] 22 | return value 23 | 24 | 25 | def filter_split(value, args): 26 | arr = value.split(args[0]) 27 | if args[1] < len(arr): 28 | return arr[args[1]] 29 | return None 30 | 31 | 32 | def filter_re_search(value, args): 33 | result = re.search(args[0], value) 34 | if result: 35 | if args[1] <= len(result.groups()): 36 | return result.group(args[1]) 37 | else: 38 | return 39 | return 40 | 41 | 42 | def filter_parse_date_elapsed(value, args) -> datetime: 43 | t = re.match(r'(?:(\d+)日)?(?:(\d+)[時时])?(?:(\d+)分)?', value) 44 | if not t: 45 | return None 46 | now = datetime.datetime.now() 47 | if t.group(1): 48 | now = now + datetime.timedelta(days=int(t.group(1))) 49 | if t.group(2): 50 | now = now + datetime.timedelta(hours=int(t.group(2))) 51 | if t.group(3): 52 | now = now + datetime.timedelta(minutes=int(t.group(3))) 53 | return now 54 | 55 | 56 | def filter_parse_date_elapsed_en(value, args): 57 | if not value: 58 | return 59 | value = str(value).strip() 60 | t = re.match(r'([\d\.]+)\s(seconds|minutes|hours|days|weeks|years)\sago', value) 61 | if not t: 62 | return 63 | now = datetime.datetime.now() 64 | num = t.group(1) 65 | unit = t.group(2) 66 | if unit == 'seconds': 67 | now = now + datetime.timedelta(seconds=float(num)) 68 | elif unit == 'minutes': 69 | now = now + datetime.timedelta(minutes=float(num)) 70 | elif unit == 'hours': 71 | now = now + datetime.timedelta(hours=float(num)) 72 | elif unit == 'days': 73 | now = now + datetime.timedelta(days=float(num)) 74 | elif unit == 'weeks': 75 | now = now + datetime.timedelta(weeks=float(num)) 76 | elif unit == 'years': 77 | now = now + datetime.timedelta(days=float(num) * 365) 78 | return now 79 | 80 | 81 | def filter_regexp(value, args): 82 | return re.sub(args, '', value) 83 | 84 | 85 | def filter_dateparse(value, args): 86 | if not value: 87 | return datetime.datetime.now() 88 | value = str(value) 89 | if isinstance(args, list): 90 | args = args 91 | else: 92 | args = [str(args)] 93 | for f in args: 94 | try: 95 | try: 96 | return datetime.datetime.strptime(value, f) 97 | except ValueError as e: 98 | if value.startswith('今天'): 99 | value = value.replace('今天', datetime.datetime.now().strftime('%Y-%m-%d')) 100 | return filter_dateparse(value, args) 101 | elif value.startswith('昨天'): 102 | value = value.replace('昨天', 103 | (datetime.datetime.now() - datetime.timedelta(days=-1)).strftime('%Y-%m-%d')) 104 | return filter_dateparse(value, args) 105 | raise e 106 | except ValueError as e: 107 | continue 108 | return 109 | 110 | 111 | filter_handler = { 112 | 'lstrip': lambda val, args: str(val).lstrip(str(args[0])), 113 | 'rstrip': lambda val, args: str(val).rstrip(str(args[0])), 114 | 'replace': lambda val, args: val.replace(args[0], args[1]) if val else None, 115 | 'append': lambda val, args: val + args, 116 | 'prepend': lambda val, args: args + val, 117 | 'tolower': lambda val, args: val.lower(), 118 | 'toupper': lambda val, args: val.upper(), 119 | 'split': filter_split, 120 | 'dateparse': filter_dateparse, 121 | 'querystring': filter_querystring, 122 | 're_search': filter_re_search, 123 | 'date_elapsed_parse': filter_parse_date_elapsed, 124 | 'date_en_elapsed_parse': filter_parse_date_elapsed_en, 125 | 'regexp': filter_regexp 126 | } 127 | 128 | 129 | class HtmlParser: 130 | @staticmethod 131 | def _select_value(tag: PyQuery, rule): 132 | val = None 133 | if tag: 134 | if 'attribute' in rule: 135 | attr = tag.attr(rule['attribute']) 136 | if attr: 137 | if isinstance(attr, list): 138 | val = attr[0] 139 | else: 140 | val = attr 141 | elif 'method' in rule: 142 | if rule['method'] == 'next_sibling' and tag: 143 | val = tag[0].tail 144 | elif 'remove' in rule: 145 | remove_tag_name = rule['remove'].split(',') 146 | for rt in remove_tag_name: 147 | tag.remove(rt) 148 | val = tag.text() 149 | elif 'contents' in rule: 150 | idx = rule['contents'] 151 | e = tag.eq(0).contents()[idx] 152 | if hasattr(e, 'text'): 153 | val = e.text 154 | else: 155 | val = str(e) 156 | else: 157 | val = tag.text() 158 | if val: 159 | val = val.strip() 160 | return val 161 | 162 | @staticmethod 163 | def _case_value(r: PyQuery, case): 164 | val = None 165 | for ck in case: 166 | if ck == '*': 167 | val = case[ck] 168 | break 169 | if r(ck): 170 | val = case[ck] 171 | break 172 | return val 173 | 174 | @staticmethod 175 | def _filter_value(value, filters): 176 | if not value: 177 | return value 178 | for f in filters: 179 | if f['name'] in filter_handler: 180 | value = filter_handler[f['name']](value, f.get('args')) 181 | return value 182 | 183 | @staticmethod 184 | def parse_item_fields(item_tag: PyQuery, item_rule, context=None): 185 | if not item_tag: 186 | return {} 187 | self = HtmlParser 188 | values = {} 189 | for key in item_rule: 190 | rule = item_rule[key] 191 | val = None 192 | try: 193 | if 'text' in rule: 194 | if '_template' in rule: 195 | tmpl = rule['_template'] 196 | ctx = {'fields': values, 'now': datetime.datetime.now()} 197 | if context: 198 | ctx.update(context) 199 | val = tmpl.render(ctx) 200 | else: 201 | val = rule.get('text') 202 | elif 'selector' in rule: 203 | val = self._select_value(item_tag(rule['selector']), rule) 204 | elif 'selectors' in rule: 205 | tag_list = item_tag(rule['selectors']) 206 | if rule.get('index'): 207 | if tag_list and rule['index'] < tag_list.length: 208 | tag = tag_list.eq(rule['index']) 209 | val = self._select_value(tag, rule) 210 | else: 211 | val = [] 212 | for i in range(tag_list.length): 213 | val.append(self._select_value(tag_list.eq(i), rule)) 214 | elif 'case' in rule: 215 | val = self._case_value(item_tag, rule['case']) 216 | if 'filters' in rule: 217 | val = self._filter_value(val, rule['filters']) 218 | if not val and 'default_value' in rule: 219 | if '_default_value_template' in rule: 220 | tmpl = rule['_default_value_template'] 221 | ctx = {'fields': values, 'now': datetime.datetime.now(), 'max_time': datetime.datetime.max} 222 | if context: 223 | ctx.update(context) 224 | val = tmpl.render(ctx) 225 | else: 226 | val = rule['default_value'] 227 | if val and 'default_value_format' in rule: 228 | val = datetime.datetime.strptime(val, rule['default_value_format']) 229 | except Exception as e: 230 | raise ParserFieldException(key, e) 231 | values[key] = val 232 | return values 233 | -------------------------------------------------------------------------------- /papihub/api/sites/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/papihub/api/sites/__init__.py -------------------------------------------------------------------------------- /papihub/api/sites/nexusphp.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from http.cookies import SimpleCookie 3 | from typing import Optional, List, Dict 4 | 5 | import httpx 6 | from cssselect import SelectorSyntaxError 7 | from httpx import Timeout 8 | from jinja2 import Template 9 | from pyquery import PyQuery 10 | 11 | from papihub import utils 12 | from papihub.api.auth import Auth 13 | from papihub.api.parser.htmlparser import HtmlParser 14 | from papihub.api.torrentsite import TorrentSite 15 | from papihub.api.types import TorrentDetail, Torrent, TorrentSiteUser, ApiOptions, CateLevel1 16 | from papihub.config.types import TorrentSiteParserConfig 17 | from papihub.constants import BASE_HEADERS, ALL_CATE_LEVEL1 18 | from papihub.exceptions import NotAuthenticatedException, ParserException 19 | 20 | DEFAULT_LOGIN_PATH = '/takelogin.php' 21 | 22 | 23 | class NexusPhp(TorrentSite, Auth): 24 | """ 25 | NexusPHP站点架构解析器，此实现类有以下特点 26 | 1、此类带有站点登录功能 27 | 2、此类遵循NexusPHP站点分类设计，搜索时会自动找到正确的搜索分类进行搜索 28 | """ 29 | 30 | auth_cookies: Optional[Dict[str, str]] = None 31 | auth_headers: Optional[Dict[str, str]] = None 32 | _user: Optional[TorrentSiteUser] = None 33 | _search_paths: List 34 | _search_query: Dict 35 | 36 | def __init__(self, parser_config: TorrentSiteParserConfig, options: Optional[ApiOptions] = None): 37 | self.parser_config = parser_config 38 | if not options: 39 | options = ApiOptions() 40 | options.request_timeout = 20 41 | self.options = options 42 | self._init() 43 | 44 | def _init(self): 45 | """ 46 | 初始化站点配置信息 47 | :return: 48 | """ 49 | for c in self.parser_config.category_mappings: 50 | # id to string 51 | c['id'] = str(c['id']) 52 | self.parser_config.domain = self.parser_config.domain.rstrip('/') 53 | self._init_jinja_template(self.parser_config.torrents.get('fields')) 54 | if self.parser_config.get_list: 55 | self._init_jinja_template(self.parser_config.get_list.get('fields')) 56 | self._search_paths = self._init_search_paths(self.parser_config.search.get('paths'), 57 | self.parser_config.category_mappings) 58 | self._search_query = self._init_search_query(self.parser_config.search.get('query')) 59 | self.auth_headers = BASE_HEADERS.copy() 60 | self.auth_headers['Referer'] = self.parser_config.domain 61 | if self.options.user_agent: 62 | self.auth_headers['User-Agent'] = self.options.user_agent 63 | 64 | @staticmethod 65 | def _init_search_paths(paths_config, category_mappings): 66 | """ 67 | 根据类目的配置，加载搜索配置，把搜索路径及可用的类目组合，方便搜索时查找 68 | :param paths_config: 69 | :param category_mappings: 70 | :return: 71 | """ 72 | paths = [] 73 | for p in paths_config: 74 | obj: dict = dict() 75 | obj['path'] = p.get('path') 76 | cate_ids_config = p.get('categories') 77 | search_cate_ids = [] 78 | if cate_ids_config: 79 | # 如果可用id第一个字符为!，则说明是排除设置模式 80 | if cate_ids_config[0] == '!': 81 | for c in category_mappings: 82 | if (int(c['id']) if c['id'] else 0) not in cate_ids_config: 83 | search_cate_ids.append(str(c['id'])) 84 | else: 85 | search_cate_ids = [str(c) for c in cate_ids_config] 86 | else: 87 | search_cate_ids = [str(c['id']) for c in category_mappings] 88 | obj['categories'] = search_cate_ids 89 | if p.get('method'): 90 | obj['method'] = p.get('method') 91 | else: 92 | obj['method'] = 'get' 93 | paths.append(obj) 94 | return paths 95 | 96 | @staticmethod 97 | def _init_search_query(query_config): 98 | query_tmpl = {} 99 | for key in query_config: 100 | val = query_config[key] 101 | if isinstance(val, str) and val.find('{') != -1: 102 | query_tmpl[key] = Template(val) 103 | else: 104 | query_tmpl[key] = val 105 | return query_tmpl 106 | 107 | @staticmethod 108 | def _init_jinja_template(fields): 109 | """ 110 | 把fields中使用到的jinja2模版提前编译好 111 | :param fields: 112 | :return: 113 | """ 114 | if not fields: 115 | return 116 | if fields: 117 | for key in fields: 118 | rule = fields[key] 119 | if 'text' in rule: 120 | if isinstance(rule['text'], str) and rule['text'].find('{') != -1: 121 | rule['_template'] = Template(rule['text']) 122 | if 'default_value' in rule: 123 | if isinstance(rule['default_value'], str) and rule['default_value'].find('{') != -1: 124 | rule['_default_value_template'] = Template(rule['default_value']) 125 | 126 | def _set_auth_cookies(self, cookie_str: str): 127 | if not cookie_str: 128 | return 129 | cookie = SimpleCookie(cookie_str) 130 | cookies = {} 131 | for key, morsel in cookie.items(): 132 | cookies[key] = morsel.value 133 | self.auth_cookies = cookies 134 | 135 | def _get_cate_level2_ids(self, cate_level1_list: Optional[List[CateLevel1]] = None): 136 | """ 137 | 通过一级大分类，去找到配置好的站点二级小分类，真正搜索时，搜站点二级小分类的编号 138 | :param cate_level1_list: 139 | :return: 140 | """ 141 | if not cate_level1_list: 142 | ids = [] 143 | # 默认不查成人 144 | for c in self.parser_config.category_mappings: 145 | if c['cate_level1'] == CateLevel1.AV.name: 146 | continue 147 | ids.append(c['id']) 148 | return ids 149 | cate_level1_str_arr = [i.name for i in cate_level1_list] 150 | cate2_ids = [] 151 | # 找到一级分类下所有的二级分类编号 152 | for c in self.parser_config.category_mappings: 153 | if c.get('cate_level1') in cate_level1_str_arr or c.get('cate_level1') == '*': 154 | cate2_ids.append(c.get('id')) 155 | return cate2_ids 156 | 157 | def _build_search_path(self, cate_level1_list: Optional[List[CateLevel1]]) -> List[Dict]: 158 | if not cate_level1_list: 159 | cate_level1_list = [x for x in CateLevel1] 160 | input_cate2_ids = set(self._get_cate_level2_ids(cate_level1_list)) 161 | paths = [] 162 | # 根据传入一级分类数据，查找真正要执行的搜索path，一级对应分类 163 | for p in self._search_paths: 164 | cpath = p.copy() 165 | cate_in = list(set(cpath['categories']).intersection(input_cate2_ids)) 166 | if not cate_in: 167 | continue 168 | del cpath['categories'] 169 | if len(cate_in) == len(self.parser_config.category_mappings): 170 | # 如果等于全部，不需要传分类 171 | cpath['query_cates'] = [] 172 | else: 173 | cpath['query_cates'] = cate_in 174 | paths.append(cpath) 175 | return paths 176 | 177 | def _trans_search_cate_id(self, ids): 178 | if not ids: 179 | return ids 180 | id_mapping = self.parser_config.category_id_mapping 181 | if not id_mapping: 182 | return ids 183 | new_ids = [] 184 | for _id in ids: 185 | for mid in id_mapping: 186 | if mid.get('id') == _id: 187 | if isinstance(mid.get('mapping'), list): 188 | new_ids += mid.get('mapping') 189 | else: 190 | new_ids.append(mid.get('mapping')) 191 | new_ids = list(filter(None, new_ids)) 192 | if not new_ids: 193 | return ids 194 | return new_ids 195 | 196 | def _render_querystring(self, query_context: Dict): 197 | qs = '' 198 | for key in self._search_query: 199 | val = self._search_query[key] 200 | if isinstance(val, Template): 201 | val = val.render({'query': query_context}) 202 | if key == '$raw' and val is not None and val != '': 203 | qs += val 204 | elif val is not None and val != '': 205 | qs += f'{key}={val}&' 206 | if qs: 207 | qs = qs.rstrip('&') 208 | return qs 209 | 210 | def _get_response_text(self, response): 211 | if not response: 212 | return 213 | c = response.content 214 | if not c: 215 | return 216 | s = str(response.content, self.parser_config.encoding) 217 | return utils.trim_emoji(s) 218 | 219 | @staticmethod 220 | def _is_login(response) -> bool: 221 | if not response: 222 | return False 223 | if response.url.path.startswith('/login.php'): 224 | return False 225 | return True 226 | 227 | def auth_with_cookies(self, cookies_str: str): 228 | self._set_auth_cookies(cookies_str) 229 | 230 | def auth(self, username: str, password: str) -> str: 231 | with httpx.Client( 232 | headers=self.auth_headers, 233 | cookies=self.auth_cookies, 234 | timeout=Timeout(self.options.request_timeout) if self.options else None, 235 | proxies=self.options.proxies, 236 | follow_redirects=True, 237 | verify=False 238 | ) as client: 239 | if self.parser_config.login and self.parser_config.login.get('path'): 240 | login_path = f'{self.parser_config.domain}{self.parser_config.login.get("path")}' 241 | else: 242 | login_path = f'{self.parser_config.domain}{DEFAULT_LOGIN_PATH}' 243 | login_method = self.parser_config.login.get('method') if self.parser_config.login else 'post' 244 | if login_method == 'get': 245 | res = client.get(f'{login_path}?username={username}&password={password}') 246 | else: 247 | res = client.post(login_path, data={ 248 | 'username': username, 249 | 'password': password 250 | }) 251 | if res.url.path.startswith('/index') and res.history: 252 | cookies_str = res.history[-1].headers.get('Set-Cookie') 253 | self._set_auth_cookies(cookies_str) 254 | return cookies_str 255 | else: 256 | raise NotAuthenticatedException(self.parser_config.site_id, self.parser_config.site_name, 257 | f'{self.parser_config.site_name}登录失败，用户名或密码错误') 258 | 259 | def _parse_user(self, pq: PyQuery) -> Optional[TorrentSiteUser]: 260 | try: 261 | item_tag = pq(self.parser_config.user.get('item')['selector']) 262 | result = HtmlParser.parse_item_fields(item_tag, self.parser_config.user.get('fields')) 263 | return TorrentSiteUser.from_data(result) 264 | except SelectorSyntaxError as e: 265 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 266 | f"{self.parser_config.site_name}解析用户信息使用了错误的CSS选择器：{str(e)}") 267 | except Exception as e: 268 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 269 | f"{self.parser_config.site_name}解析用户信息出现错误：{str(e)}") 270 | 271 | def _copy_to_torrent(self, item: dict) -> Optional[Torrent]: 272 | """ 273 | 把按css选择器解析出来的种子数据，标准化成Torrent对象 274 | """ 275 | if not item: 276 | return None 277 | t = Torrent() 278 | t.site_id = self.parser_config.site_id 279 | t.id = utils.parse_value(str, item.get('id')) 280 | t.name = utils.parse_value(str, item.get('title')) 281 | t.subject = utils.parse_value(str, item.get('description')) 282 | if t.subject: 283 | t.subject = t.subject.strip() 284 | t.free_deadline = item.get('free_deadline') 285 | t.imdb_id = item.get('imdbid') 286 | t.upload_count = utils.parse_value(int, item.get('seeders'), 0) 287 | t.downloading_count = utils.parse_value(int, item.get('leechers'), 0) 288 | t.download_count = utils.parse_value(int, item.get('grabs'), 0) 289 | t.download_url = item.get('download') 290 | if t.download_url and not t.download_url.startswith('http') and not t.download_url.startswith('magnet'): 291 | t.download_url = self.parser_config.domain + t.download_url 292 | t.publish_date = utils.parse_value(datetime.datetime, item.get('date'), datetime.datetime.now()) 293 | t.cate_id = utils.parse_value(str, item.get('category')) 294 | for c in self.parser_config.category_mappings: 295 | cid = t.cate_id 296 | id_mapping = self.parser_config.category_id_mapping 297 | if id_mapping: 298 | for mid in id_mapping: 299 | if str(mid.get('id')) == str(cid): 300 | if isinstance(mid.get('mapping'), list): 301 | cid = mid.get('mapping')[0] 302 | else: 303 | cid = mid.get('mapping') 304 | if str(c.get('id')) == str(cid): 305 | t.cate_level1 = CateLevel1.get_type(c.get('cate_level1')) 306 | break 307 | t.details_url = item.get('details') 308 | if t.details_url: 309 | t.details_url = self.parser_config.domain + t.details_url 310 | t.download_volume_factor = utils.parse_value(float, item.get('downloadvolumefactor'), 1) 311 | t.upload_volume_factor = utils.parse_value(float, item.get('uploadvolumefactor'), 1) 312 | t.size_mb = utils.trans_size_str_to_mb(utils.parse_value(str, item.get('size'), '0')) 313 | t.poster_url = item.get('poster') 314 | t.minimum_ratio = utils.parse_value(float, item.get('minimumratio'), 0.0) 315 | t.minimum_seed_time = utils.parse_value(int, item.get('minimumseedtime'), 0) 316 | if t.poster_url: 317 | if t.poster_url.startswith("./"): 318 | t.poster_url = self.parser_config.domain + t.poster_url[2:] 319 | elif not t.poster_url.startswith("http"): 320 | t.poster_url = self.parser_config.domain + t.poster_url 321 | return t 322 | 323 | def _parse_torrents(self, pq: PyQuery, context: Dict) -> List[Torrent]: 324 | list_rule = self.parser_config.torrents.get('list') 325 | fields_rule = self.parser_config.torrents.get('fields') 326 | if not fields_rule: 327 | return [] 328 | try: 329 | rows = pq(list_rule['selector']) 330 | if not rows: 331 | return [] 332 | result: List[Torrent] = [] 333 | for i in range(rows.length): 334 | tag = rows.eq(i) 335 | result.append(self._copy_to_torrent(HtmlParser.parse_item_fields(tag, fields_rule, context=context))) 336 | return result 337 | except SelectorSyntaxError as e: 338 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 339 | f"{self.parser_config.site_name}种子信息解析使用了错误的CSS选择器：{str(e)}") 340 | except Exception as e: 341 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 342 | f"{self.parser_config.site_name}种子信息解析失败") 343 | 344 | def _copy_torrent_detail(self, item): 345 | if not item: 346 | return 347 | t = TorrentDetail() 348 | t.site_id = self.parser_config.site_id 349 | t.id = item.get_int('id', item.get_value('id')) 350 | t.name = utils.parse_value(str, item.get('title')) 351 | t.subject = utils.parse_value(str, item.get('description')) 352 | if t.subject: 353 | t.subject = t.subject.strip() 354 | t.download_url = utils.parse_value(str, item.get('download')) 355 | if t.download_url and not t.download_url.startswith('http'): 356 | t.download_url = self.parser_config.domain + t.download_url 357 | t.filename = item.get('filename') 358 | t.intro = item.get('intro') 359 | t.publish_date = item.get('date') 360 | return t 361 | 362 | def _parse_detail(self, pq: PyQuery) -> TorrentDetail: 363 | detail_config = self.parser_config.get_detail 364 | if not detail_config: 365 | return 366 | field_rule = detail_config.get('fields') 367 | if not field_rule: 368 | return 369 | try: 370 | item_tag = pq(detail_config.get('item')['selector']) 371 | result = HtmlParser.parse_item_fields(item_tag, field_rule) 372 | return self._copy_torrent_detail(result) 373 | except SelectorSyntaxError as e: 374 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 375 | f"{self.parser_config.site_name}种子详情页解析使用了错误的CSS选择器：{str(e)}") 376 | except Exception as e: 377 | raise ParserException(self.parser_config.site_id, self.parser_config.site_name, 378 | f"{self.parser_config.site_name}种子详情页解析失败") 379 | 380 | def list(self, timeout: Optional[int] = None, cate_level1_list: Optional[List] = None, ) -> List[Torrent]: 381 | if not timeout: 382 | timeout = self.options.request_timeout 383 | list_parser = self.parser_config.get_list 384 | if list_parser: 385 | with httpx.Client( 386 | headers=self.auth_headers, 387 | cookies=self.auth_cookies, 388 | timeout=Timeout(timeout), 389 | proxies=self.options.proxies, 390 | follow_redirects=True, 391 | verify=False 392 | ) as client: 393 | url = f'{self.parser_config.domain}{list_parser.get("path")}' 394 | r = client.get(url) 395 | text = self._get_response_text(r) 396 | if not text: 397 | return [] 398 | pq = PyQuery(text) 399 | if not self._user: 400 | self._user = self._parse_user(pq) 401 | return self._parse_torrents(pq, context={'userinfo': self._user}) 402 | else: 403 | return self.search(cate_level1_list=cate_level1_list if cate_level1_list else ALL_CATE_LEVEL1, 404 | timeout=timeout) 405 | 406 | def get_user(self, refresh=False) -> Optional[TorrentSiteUser]: 407 | url = self.parser_config.user.get('path') 408 | if not url: 409 | return 410 | with httpx.Client( 411 | headers=self.auth_headers, 412 | cookies=self.auth_cookies, 413 | http2=False, 414 | timeout=Timeout(timeout=self.options.request_timeout), 415 | proxies=self.options.proxies, 416 | follow_redirects=True, 417 | verify=False 418 | ) as client: 419 | r = client.get(url) 420 | text = self._get_response_text(r) 421 | pq = PyQuery(text) 422 | return self._parse_user(pq) 423 | 424 | def search(self, keyword: Optional[str] = None, 425 | imdb_id: Optional[str] = None, 426 | cate_level1_list: Optional[List] = None, 427 | free: Optional[bool] = False, 428 | page: Optional[int] = None, 429 | timeout: Optional[int] = None) -> List[Torrent]: 430 | if not self._search_paths: 431 | return [] 432 | paths = self._build_search_path(cate_level1_list) 433 | if not paths: 434 | # 配置文件的分类设置有问题或者真的不存在所需查询分类 435 | return [] 436 | # 构造查询参数的上下文，供配置渲染真实querystring 437 | query_context = {} 438 | if keyword: 439 | query_context['keyword'] = keyword 440 | if imdb_id: 441 | query_context['imdb_id'] = imdb_id 442 | if free: 443 | query_context['free'] = free 444 | else: 445 | query_context['cates'] = [] 446 | if page: 447 | query_context['page'] = page 448 | total_torrents: List[Torrent] = [] 449 | if not timeout: 450 | timeout = self.options.request_timeout 451 | for i, p in enumerate(paths): 452 | if p.get('query_cates'): 453 | query_context['cates'] = self._trans_search_cate_id(p.get('query_cates')) 454 | uri = p.get('path') 455 | qs = self._render_querystring(query_context) 456 | with httpx.Client( 457 | headers=self.auth_headers, 458 | cookies=self.auth_cookies, 459 | timeout=Timeout(timeout), 460 | proxies=self.options.proxies, 461 | follow_redirects=True, 462 | verify=False 463 | ) as client: 464 | if p.get('method') == 'get': 465 | url = f'{self.parser_config.domain}/{uri}?{qs}' 466 | res = client.get(url) 467 | else: 468 | url = f'{self.parser_config.domain}/{uri}' 469 | res = client.post(url, data=qs) 470 | if not self._is_login(res): 471 | raise NotAuthenticatedException(self.parser_config.site_id, self.parser_config.site_name, 472 | f'{self.parser_config.site_name}未授权，无法访问') 473 | text = self._get_response_text(res) 474 | if not text: 475 | continue 476 | pq = PyQuery(text) 477 | if not self._user: 478 | self._user = self._parse_user(pq) 479 | torrents = self._parse_torrents(pq, context={'userinfo': self._user}) 480 | if torrents: 481 | total_torrents += torrents 482 | return total_torrents 483 | 484 | def download_torrent(self, url, filepath): 485 | pass 486 | 487 | def get_detail(self, url) -> Optional[TorrentDetail]: 488 | detail_config = self.parser_config.get_detail 489 | if not detail_config: 490 | return 491 | with httpx.Client( 492 | headers=self.auth_headers, 493 | cookies=self.auth_cookies, 494 | timeout=Timeout(self.options.request_timeout), 495 | proxies=self.options.proxies, 496 | follow_redirects=True, 497 | verify=False 498 | ) as client: 499 | r = client.get(url) 500 | text = self._get_response_text(r) 501 | if not text: 502 | return 503 | pq = PyQuery(text) 504 | return self._parse_detail(pq) 505 | 506 | def test_login(self): 507 | with httpx.Client( 508 | headers=self.auth_headers, 509 | cookies=self.auth_cookies, 510 | timeout=Timeout(self.options.request_timeout) if self.options else None, 511 | proxies=self.options.proxies, 512 | follow_redirects=False, 513 | verify=False 514 | ) as client: 515 | # nexusphp 都有一个用户首页 516 | r = client.get(f'{self.parser_config.domain}/usercp.php') 517 | return r.status_code == 200 518 | -------------------------------------------------------------------------------- /papihub/api/torrentsite.py: -------------------------------------------------------------------------------- 1 | from abc import ABCMeta, abstractmethod 2 | from typing import List, Optional 3 | 4 | from papihub.api.types import Torrent, TorrentDetail, TorrentSiteUser 5 | from papihub.config.types import TorrentSiteParserConfig 6 | 7 | 8 | class TorrentSite(metaclass=ABCMeta): 9 | """ 10 | 种子站点抓取接口 11 | """ 12 | parser_config: TorrentSiteParserConfig 13 | 14 | @abstractmethod 15 | def list(self, timeout=None, cate_level1_list=None) -> List[Torrent]: 16 | """ 17 | 获取种子列表 18 | :param timeout: 超时信息 19 | :param cate_level1_list: 一级种子分类信息 20 | :return: 21 | """ 22 | pass 23 | 24 | @abstractmethod 25 | def get_user(self, refresh=False) -> Optional[TorrentSiteUser]: 26 | """ 27 | 获取用户信息 28 | :param refresh: 29 | :return: 30 | """ 31 | pass 32 | 33 | @abstractmethod 34 | def search( 35 | self, 36 | keyword: Optional[str] = None, 37 | imdb_id: Optional[str] = None, 38 | cate_level1_list: Optional[List] = None, 39 | free: Optional[bool] = False, 40 | page: Optional[int] = None, 41 | timeout: Optional[int] = None 42 | ) -> List[Torrent]: 43 | """ 44 | 搜索种子 45 | :param keyword: 46 | :param imdb_id: 47 | :param cate_level1_list: 48 | :param free: 49 | :param page: 50 | :param timeout: 51 | :return: 52 | """ 53 | pass 54 | 55 | @abstractmethod 56 | def download_torrent(self, url, filepath): 57 | """ 58 | 下载种子 59 | :param url: 60 | :param filepath: 61 | :return: 62 | """ 63 | pass 64 | 65 | @abstractmethod 66 | def get_detail(self, url) -> Optional[TorrentDetail]: 67 | """ 68 | 获取种子详情信息 69 | :param url: 70 | :return: 71 | """ 72 | pass 73 | -------------------------------------------------------------------------------- /papihub/api/types.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | from dataclasses import dataclass 3 | from enum import Enum 4 | from typing import Optional 5 | 6 | from dataclasses_json import dataclass_json 7 | 8 | from papihub import utils 9 | from papihub.utils import trans_size_str_to_mb 10 | 11 | 12 | @dataclass 13 | class ApiOptions: 14 | """ 15 | 接口设置 16 | """ 17 | request_timeout: Optional[int] = None 18 | proxies: Optional[str] = None 19 | user_agent: Optional[str] = None 20 | 21 | 22 | @dataclass_json 23 | @dataclass 24 | class TorrentSiteUser: 25 | """ 26 | 种子站点用户信息 27 | """ 28 | uid: Optional[int] = 0 29 | username: Optional[str] = 'unknown' 30 | user_group: Optional[str] = 'unknown' 31 | share_ratio: Optional[float] = 0 32 | uploaded: Optional[float] = 0 33 | downloaded: Optional[float] = 0 34 | seeding: Optional[int] = 0 35 | leeching: Optional[int] = 0 36 | vip_group: Optional[bool] = False 37 | 38 | @staticmethod 39 | def from_data(result: dict) -> Optional["TorrentSiteUser"]: 40 | if not result: 41 | return None 42 | user = TorrentSiteUser() 43 | user.uid = int(result['uid']) 44 | user.username = result['username'] 45 | user.user_group = result['user_group'] 46 | user.uploaded = trans_size_str_to_mb(str(result['uploaded'])) 47 | user.downloaded = trans_size_str_to_mb(str(result['downloaded'])) 48 | try: 49 | user.seeding = int(result['seeding']) 50 | except Exception as e: 51 | user.seeding = 0 52 | try: 53 | user.leeching = int(result['leeching']) 54 | except Exception as e: 55 | user.leeching = 0 56 | try: 57 | if 'share_ratio' in result: 58 | ss = result['share_ratio'].replace(',', '') 59 | user.share_ratio = float(ss) 60 | else: 61 | if not user.downloaded: 62 | user.share_ratio = float('inf') 63 | else: 64 | user.share_ratio = round(user.uploaded / user.downloaded, 2) 65 | except Exception as e: 66 | user.share_ratio = 0.0 67 | user.vip_group = result['vip_group'] 68 | return user 69 | 70 | 71 | class CateLevel1(str, Enum): 72 | """ 73 | 种子一级分类信息 74 | """ 75 | Movie = 'Movie' 76 | TV = 'TV' 77 | Documentary = 'Documentary' 78 | Anime = 'Anime' 79 | Music = 'Music' 80 | Game = 'Game' 81 | AV = 'AV' 82 | Other = 'Other' 83 | 84 | @staticmethod 85 | def get_type(enum_name: str) -> Optional["CateLevel1"]: 86 | for item in CateLevel1: 87 | if item.name == enum_name: 88 | return item 89 | return None 90 | 91 | 92 | @dataclass_json 93 | @dataclass 94 | class Torrent: 95 | """ 96 | 种子信息（来自列表页） 97 | """ 98 | # 站点编号 99 | site_id: Optional[str] = None 100 | # 种子编号 101 | id: Optional[str] = None 102 | # 种子名称 103 | name: Optional[str] = None 104 | # 种子标题 105 | subject: Optional[str] = None 106 | # 以及类目 107 | cate_level1: Optional[CateLevel1] = None 108 | # 站点类目id 109 | cate_id: Optional[str] = None 110 | # 种子详情页地址 111 | details_url: Optional[str] = None 112 | # 种子下载链接 113 | download_url: Optional[str] = None 114 | # 种子关联的imdbid 115 | imdb_id: Optional[str] = None 116 | # 种子发布时间 117 | publish_date: Optional[datetime.datetime] = None 118 | # 种子大小，转化为mb尺寸 119 | size_mb: Optional[float] = None 120 | # 做种人数 121 | upload_count: Optional[int] = None 122 | # 下载中人数 123 | downloading_count: Optional[int] = None 124 | # 下载完成人数 125 | download_count: Optional[int] = None 126 | # 免费截止时间 127 | free_deadline: Optional[datetime.datetime] = None 128 | # 下载折扣，1为不免费 129 | download_volume_factor: Optional[float] = None 130 | # 做种上传系数，1为正常 131 | upload_volume_factor: Optional[int] = None 132 | minimum_ratio: float = 0 133 | minimum_seed_time: int = 0 134 | # 封面链接 135 | poster_url: Optional[str] = None 136 | 137 | 138 | @dataclass_json 139 | @dataclass 140 | class TorrentDetail: 141 | """ 142 | 种子详情页信息 143 | """ 144 | site_id: Optional[str] = None 145 | name: Optional[str] = None 146 | subject: Optional[str] = None 147 | download_url: Optional[str] = None 148 | filename: Optional[str] = None 149 | intro: Optional[str] = None 150 | publish_date: Optional[datetime.datetime] = None 151 | -------------------------------------------------------------------------------- /papihub/auth.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | from datetime import timedelta 4 | from typing import Annotated, Optional 5 | 6 | from fastapi import Depends, HTTPException 7 | from fastapi.security import OAuth2PasswordBearer 8 | from jose import jwt, JWTError 9 | from passlib.context import CryptContext 10 | from starlette import status 11 | 12 | from papihub.models.usermodel import UserModel 13 | 14 | # 一周过期，单位分钟 15 | ACCESS_TOKEN_EXPIRE_MINUTES = 10080 16 | 17 | # openssl rand -hex 32 生成 secret_key 18 | SECRET_KEY = "4f30cdb27ed3002c32b289371dd5897e73178719432f6b27bbd1747169a6e0b8" 19 | ALGORITHM = "HS256" 20 | 21 | oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/user/get_token") 22 | 23 | # 密码加盐 24 | PWD_SALT = "d3A^FQh8**!q" 25 | pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto") 26 | 27 | 28 | def get_password_hash(password): 29 | return pwd_context.hash(password + PWD_SALT) 30 | 31 | 32 | def verify_password(plain_password, hashed_password): 33 | return pwd_context.verify(plain_password + PWD_SALT, hashed_password) 34 | 35 | 36 | def create_access_token(data: dict, expires_delta: Optional[timedelta] = None) -> str: 37 | if expires_delta: 38 | expire = datetime.datetime.now() + expires_delta 39 | else: 40 | expire = datetime.datetime.now() + timedelta(minutes=60) 41 | data = {**data, "exp": expire} 42 | token = jwt.encode(data, SECRET_KEY, algorithm=ALGORITHM) 43 | return token 44 | 45 | 46 | def get_current_user( 47 | token: Annotated[str, Depends(oauth2_scheme)] 48 | ): 49 | authenticate_value = "Bearer" 50 | credentials_exception = HTTPException( 51 | status_code=status.HTTP_401_UNAUTHORIZED, 52 | detail="验证信息无效", 53 | headers={"WWW-Authenticate": authenticate_value}, 54 | ) 55 | try: 56 | payload = jwt.decode(token, SECRET_KEY, algorithms=ALGORITHM) 57 | username: str = payload.get("sub") 58 | if username is None: 59 | raise credentials_exception 60 | expires = payload.get("exp") 61 | if expires < time.time(): 62 | raise credentials_exception 63 | except JWTError: 64 | raise credentials_exception 65 | user = UserModel.get_by_username(username) 66 | if user is None: 67 | raise credentials_exception 68 | return user 69 | -------------------------------------------------------------------------------- /papihub/common/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 一些公共的，偏工具类的包 3 | """ -------------------------------------------------------------------------------- /papihub/common/customjsonencoder.py: -------------------------------------------------------------------------------- 1 | import json 2 | import datetime 3 | 4 | 5 | class CustomJSONEncoder(json.JSONEncoder): 6 | def default(self, obj): 7 | if isinstance(obj, datetime.datetime): 8 | return obj.strftime("%Y-%m-%d %H:%M:%S") 9 | return super().default(obj) 10 | -------------------------------------------------------------------------------- /papihub/common/logging.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | LOGGING_CONFIG = { 4 | 'version': 1, 5 | 'disable_existing_loggers': False, 6 | 'formatters': { 7 | 'default': { 8 | 'format': '%(asctime)s - %(name)s - %(levelname)s - [%(threadName)s] - %(message)s', 9 | }, 10 | }, 11 | 'handlers': { 12 | 'console': { 13 | 'class': 'logging.StreamHandler', 14 | 'level': 'INFO', 15 | 'formatter': 'default', 16 | }, 17 | 'file': { 18 | 'class': 'logging.handlers.TimedRotatingFileHandler', 19 | 'level': 'INFO', 20 | 'formatter': 'default', 21 | 'filename': f"{os.environ.get('WORKDIR', '/app/papihub')}/logs/app.log", 22 | 'when': 'D', 23 | 'interval': 1, 24 | 'backupCount': 7, 25 | }, 26 | }, 27 | 'loggers': { 28 | '': { # root logger 29 | 'handlers': ['file'], 30 | 'level': 'INFO', 31 | 'propagate': True, 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /papihub/common/response.py: -------------------------------------------------------------------------------- 1 | import json 2 | 3 | from fastapi import status 4 | from fastapi.encoders import jsonable_encoder 5 | from fastapi.responses import JSONResponse, Response 6 | from typing import Union 7 | 8 | from starlette.responses import PlainTextResponse 9 | 10 | from papihub.common.customjsonencoder import CustomJSONEncoder 11 | 12 | 13 | def json_200(data: Union[bool, list, dict, str, None] = None, message: Union[str, None] = None) -> Response: 14 | """ 15 | 返回http_status=200的结果 16 | :param data: 返回结果 17 | :param message: 消息 18 | :return: 19 | """ 20 | if not message: 21 | message = "success" 22 | if data: 23 | if isinstance(data, list): 24 | if len(data) > 0 and 'to_dict' in dir(data[0]): 25 | data = [i.to_dict() for i in data] 26 | elif 'to_dict' in dir(data): 27 | data = data.to_dict() 28 | return PlainTextResponse( 29 | media_type="application/json", 30 | status_code=status.HTTP_200_OK, 31 | content=json.dumps({ 32 | 'success': True, 33 | 'errorCode': 0, 34 | 'message': message, 35 | 'data': data, 36 | }, cls=CustomJSONEncoder), 37 | ) 38 | 39 | 40 | def json_500(data: Union[bool, list, dict, str, None] = None, message: Union[str, None] = None) -> Response: 41 | """ 42 | 返回http_status=500的结果 43 | :param data: 返回结果 44 | :param message: 消息 45 | :return: 46 | """ 47 | if not message: 48 | message = "success" 49 | return JSONResponse( 50 | status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, 51 | content={ 52 | 'success': False, 53 | 'errorCode': 1, 54 | 'message': message, 55 | 'data': data, 56 | } 57 | ) 58 | 59 | 60 | def json_with_status(status_code: int, data: Union[bool, list, dict, str, None] = None, 61 | message: Union[str, None] = None) -> Response: 62 | """ 63 | 返回自定义statuscode的结果 64 | :param data: 返回结果 65 | :param message: 消息 66 | :return: 67 | """ 68 | if not message: 69 | message = "success" 70 | return JSONResponse( 71 | status_code=status_code, 72 | content={ 73 | 'success': False, 74 | 'errorCode': 1, 75 | 'message': message, 76 | 'data': data, 77 | } 78 | ) 79 | -------------------------------------------------------------------------------- /papihub/config/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 配置信息处理包 3 | """ -------------------------------------------------------------------------------- /papihub/config/siteparserconfigloader.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import List, Dict 3 | 4 | import yaml 5 | 6 | from papihub.config.types import TorrentSiteParserConfig, ParserConfig 7 | from papihub.exceptions import ParserConfigErrorException 8 | 9 | 10 | class SiteParserConfigLoader: 11 | """ 12 | 站点适配文件加载器 13 | """ 14 | 15 | def __init__(self, conf_path: str): 16 | if not conf_path or not os.path.exists(conf_path): 17 | raise FileNotFoundError(conf_path, f'站点适配文件路径不存在：{conf_path}') 18 | self.conf_path = conf_path 19 | 20 | def load(self) -> Dict[str, ParserConfig]: 21 | """ 22 | 加载配置目录的所有站点适配文件 23 | :return: 24 | """ 25 | if not self.conf_path: 26 | return {} 27 | parse_configs = {} 28 | for path, dir_list, file_list in os.walk(self.conf_path): 29 | for file_name in file_list: 30 | if os.path.splitext(file_name)[1] != '.yml': 31 | continue 32 | filepath = os.path.join(self.conf_path, file_name) 33 | try: 34 | with open(filepath, 'r', encoding='utf-8') as file: 35 | lines = file.readlines() 36 | parser_config_dict = yaml.safe_load(''.join(lines)) 37 | config_type: str = parser_config_dict.get('config_type') 38 | if config_type == 'torrent_site': 39 | parse_configs[parser_config_dict.get('site_id')] = TorrentSiteParserConfig( 40 | **parser_config_dict) 41 | except Exception as e: 42 | raise ParserConfigErrorException(filepath, 43 | f'站点适配文件错误，请检查文件是否标准yml文件，没有掺杂无效信息：{filepath}', 44 | e) 45 | return parse_configs 46 | -------------------------------------------------------------------------------- /papihub/config/types.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List, Dict 2 | from dataclasses import dataclass 3 | 4 | from dataclasses_json import dataclass_json 5 | 6 | 7 | @dataclass_json 8 | @dataclass 9 | class ParserConfig: 10 | """ 11 | 站点解析配置文件，也可以成为站点适配文件 12 | """ 13 | config_type: Optional[str] = None 14 | # 站点编号 15 | site_id: Optional[str] = None 16 | # 站点名称 17 | site_name: Optional[str] = None 18 | # 站点类型 19 | site_type: Optional[str] = None 20 | # 站点域名 21 | domain: Optional[str] = None 22 | # 站点编码 23 | encoding: Optional[str] = None 24 | 25 | 26 | @dataclass 27 | class TorrentSiteParserConfig(ParserConfig): 28 | """ 29 | 种子站点特有的配置信息 30 | """ 31 | login: Optional[Dict] = None 32 | # 站点类目映射 33 | category_mappings: Optional[List[Dict]] = None 34 | category_id_mapping: Optional[List[Dict]] = None 35 | # 用户信息解析配置 36 | user: Optional[Dict] = None 37 | # 搜索解析配置 38 | search: Optional[Dict] = None 39 | # 独立的种子列表页解析配置 40 | get_list: Optional[Dict] = None 41 | # 标准种子解析配置 42 | torrents: Optional[Dict] = None 43 | get_detail: Optional[Dict] = None 44 | -------------------------------------------------------------------------------- /papihub/constants.py: -------------------------------------------------------------------------------- 1 | """ 2 | 常量信息类 3 | """ 4 | from papihub.api.types import CateLevel1 5 | 6 | # 默认所有种子搜索的一级分类 7 | ALL_CATE_LEVEL1 = [CateLevel1.Movie, 8 | CateLevel1.TV, 9 | CateLevel1.Documentary, 10 | CateLevel1.Anime, 11 | CateLevel1.Music, 12 | CateLevel1.AV, 13 | CateLevel1.Game, 14 | CateLevel1.Other] 15 | 16 | # 全局http请求的默认头 17 | BASE_HEADERS = { 18 | 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36' 19 | } 20 | -------------------------------------------------------------------------------- /papihub/databases.py: -------------------------------------------------------------------------------- 1 | """ 2 | 与数据库有关的操作类 3 | """ 4 | import datetime 5 | import os 6 | 7 | from dataclasses_json import dataclass_json 8 | from sqlalchemy import create_engine, Column, DateTime, String, Integer, Text, select 9 | from sqlalchemy.ext.declarative import declarative_base 10 | from sqlalchemy.orm import Session 11 | 12 | from papihub import utils 13 | 14 | # WORKDIR环境变量文件夹内的db目录，为数据库文件存放目录 15 | db_path = os.path.join(os.environ.get('WORKDIR', os.path.dirname(os.path.abspath(__file__))), 'db') 16 | if not os.path.exists(db_path): 17 | os.makedirs(db_path) 18 | engine = create_engine( 19 | f'sqlite:////{db_path}/main.db?check_same_thread=False&timeout=60' 20 | ) 21 | Base = declarative_base() 22 | 23 | 24 | def create_all(): 25 | """ 26 | 自动初始化数据库引擎和ORM框架 27 | 会自动生成模型定义的结构为数据表 28 | :return: 29 | """ 30 | Base.metadata.create_all(engine) 31 | 32 | 33 | class BaseDBModel(Base): 34 | """ 35 | 数据表基类，每张表的模型类继承此类 36 | """ 37 | __abstract__ = True 38 | __table_args__ = {'extend_existing': True} 39 | gmt_create = Column(DateTime, nullable=False, default=datetime.datetime.now) 40 | gmt_modified = Column(DateTime, nullable=False, default=datetime.datetime.now, onupdate=datetime.datetime.now) 41 | 42 | def get_columns(self): 43 | """ 44 | 返回所有字段对象 45 | :return: 46 | """ 47 | return self.__table__.columns 48 | 49 | @classmethod 50 | def query(cls): 51 | session = Session(bind=engine) 52 | return session.query(cls) 53 | 54 | def get_fields(self): 55 | """ 56 | 返回所有字段 57 | :return: 58 | """ 59 | return self.__dict__ 60 | 61 | def save(self): 62 | """ 63 | 新增 64 | :return: 65 | """ 66 | session = Session(bind=engine) 67 | try: 68 | session.add(self) 69 | session.commit() 70 | except BaseException as e: 71 | session.rollback() 72 | raise 73 | 74 | def update(self): 75 | """ 76 | 新增 77 | :return: 78 | """ 79 | session = Session(bind=engine) 80 | try: 81 | self.gmt_modified = datetime.datetime.now() 82 | session.merge(self) 83 | session.commit() 84 | except: 85 | session.rollback() 86 | raise 87 | 88 | @staticmethod 89 | def save_all(model_list): 90 | """ 91 | 批量新增 92 | :param model_list: 93 | :return: 94 | """ 95 | session = Session(bind=engine) 96 | try: 97 | session.add_all(model_list) 98 | session.commit() 99 | except: 100 | session.rollback() 101 | raise 102 | 103 | def delete(self): 104 | session = Session(bind=engine) 105 | try: 106 | session.commit() 107 | except: 108 | session.rollback() 109 | raise 110 | 111 | def to_dict(self, hidden_fields=None): 112 | """ 113 | Json序列化 114 | :param hidden_fields: 覆盖类属性 hidden_fields 115 | :return: 116 | """ 117 | model_json = {} 118 | if not hidden_fields: 119 | hidden_fields = self.__hidden_fields__ 120 | if not hidden_fields: 121 | hidden_fields = [] 122 | for column in self.__dict__: 123 | if column in hidden_fields: 124 | continue 125 | if hasattr(self, column): 126 | model_json[column] = utils.parse_field_value(getattr(self, column)) 127 | if '_sa_instance_state' in model_json: 128 | del model_json['_sa_instance_state'] 129 | return model_json 130 | -------------------------------------------------------------------------------- /papihub/eventbus.py: -------------------------------------------------------------------------------- 1 | """ 2 | 程序事件管理类 3 | 程序内产生的所有事件，都统一在此处描述 4 | """ 5 | from event_bus import EventBus 6 | 7 | # 全局默认的事件处理器 8 | bus = EventBus() 9 | # 站点初始化事件 10 | EVENT_SITE_INIT = 'site:init' 11 | -------------------------------------------------------------------------------- /papihub/exceptions.py: -------------------------------------------------------------------------------- 1 | """ 2 | 程序内产生的自定义异常 3 | """ 4 | 5 | 6 | class PapiHubException(Exception): 7 | pass 8 | 9 | 10 | class SiteApiErrorException(PapiHubException): 11 | pass 12 | 13 | 14 | class SiteAuthenticationFailureException(PapiHubException): 15 | def __init__(self, site_id: str, site_name: str, *args): 16 | super().__init__(*args) 17 | self.site_id = site_id 18 | self.site_name = site_name 19 | 20 | 21 | class ParserException(PapiHubException): 22 | pass 23 | 24 | 25 | class NotFoundParserException(ParserException): 26 | pass 27 | 28 | 29 | class ParserFieldException(ParserException): 30 | def __init__(self, field_name: str, *args): 31 | super().__init__(*args) 32 | self.field_name = field_name 33 | 34 | 35 | class ParserConfigErrorException(ParserException): 36 | filepath: str 37 | 38 | def __init__(self, filepath: str, *args): 39 | super().__init__(*args) 40 | self.filepath = filepath 41 | 42 | 43 | class NotAuthenticatedException(PapiHubException): 44 | def __init__(self, site_id: str, site_name: str, *args): 45 | super().__init__(*args) 46 | self.site_id = site_id 47 | self.site_name = site_name 48 | -------------------------------------------------------------------------------- /papihub/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | 程序启动入口类 3 | """ 4 | import json 5 | import logging.config 6 | import os 7 | 8 | from fastapi.exceptions import RequestValidationError 9 | from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY 10 | 11 | from papihub.common.logging import LOGGING_CONFIG 12 | 13 | logging.config.dictConfig(LOGGING_CONFIG) 14 | 15 | import inject 16 | 17 | from papihub.config.siteparserconfigloader import SiteParserConfigLoader 18 | from papihub.manager.sitemanager import SiteManager 19 | 20 | import httpx 21 | import uvicorn 22 | from starlette.exceptions import HTTPException 23 | from fastapi import FastAPI 24 | from papihub.databases import create_all 25 | 26 | from papihub.common.response import json_200, json_500, json_with_status 27 | from papihub.routers import torrentsrouter 28 | from papihub.routers import siterouter 29 | from papihub.routers import userrouter 30 | from papihub.models import * 31 | from papihub.tasks import * 32 | 33 | log = logging.getLogger(__name__) 34 | 35 | # 初始化ORM框架 36 | create_all() 37 | 38 | app = FastAPI() 39 | 40 | # 加载所有fastapi的接口路由 41 | app.include_router(torrentsrouter.router) 42 | app.include_router(siterouter.router) 43 | app.include_router(userrouter.router) 44 | 45 | 46 | @app.get("/") 47 | async def root(): 48 | """ 49 | 默认首页 50 | :return: 51 | """ 52 | return json_200(message='papihub server') 53 | 54 | 55 | @app.exception_handler(RequestValidationError) 56 | async def unprocessable_entity_handler(request, exc: RequestValidationError): 57 | return json_with_status( 58 | status_code=422, 59 | message='参数错误', 60 | data=dict(exc.errors()) 61 | ) 62 | 63 | 64 | @app.exception_handler(HTTPException) 65 | async def http_exception_handler(request, exc): 66 | return json_with_status(status_code=exc.status_code, message=exc.detail) 67 | 68 | 69 | @app.exception_handler(httpx.HTTPStatusError) 70 | async def http_status_exception_handler(request, e: httpx.HTTPStatusError): 71 | msg = e.response.json().get('error', {}).get('message') 72 | log.error('http status exception: ' + msg, exc_info=True) 73 | return json_500(message=msg) 74 | 75 | 76 | @app.exception_handler(Exception) 77 | async def universal_exception_handler(request, exc): 78 | log.error('universal_exception_handler', exc_info=True) 79 | return json_500(message=str(exc)) 80 | 81 | 82 | def config(binder): 83 | """ 84 | 依赖注入机制的初始化 85 | 所有通过inject使用的对象，需要提前再此绑定 86 | :param binder: 87 | :return: 88 | """ 89 | loader = SiteParserConfigLoader(conf_path=os.path.join(os.environ.get('WORKDIR'), 'conf', 'parser')) 90 | binder.bind(SiteParserConfigLoader, loader) 91 | binder.bind(SiteManager, SiteManager(loader)) 92 | 93 | 94 | def init_biz_data(): 95 | """ 96 | 初始化业务数据 97 | :return: 98 | """ 99 | from papihub.models.usermodel import UserModel 100 | list_users = UserModel.query().all() 101 | if not list_users: 102 | log.info("初始化用户：admin 密码：papiadmin 请尽快登录后修改密码") 103 | from papihub.auth import get_password_hash 104 | user = UserModel( 105 | nickname="默认管理员", 106 | username='admin', 107 | password=get_password_hash("admin123"), 108 | ) 109 | user.save() 110 | 111 | 112 | if __name__ == "__main__": 113 | # 加载公共全局依赖 114 | inject.configure(config) 115 | init_biz_data() 116 | uvicorn.run(app, host="0.0.0.0", port=os.environ.get("WEB_PORT", 8000)) 117 | -------------------------------------------------------------------------------- /papihub/manager/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | 带有业务逻辑的公共管理器包 3 | """ 4 | -------------------------------------------------------------------------------- /papihub/manager/sitemanager.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from typing import Optional, Dict, List 3 | from papihub import utils 4 | from papihub.api.auth import Auth 5 | from papihub.api.sites.nexusphp import NexusPhp 6 | from papihub.api.torrentsite import TorrentSite 7 | from papihub.config.siteparserconfigloader import SiteParserConfigLoader 8 | from papihub.eventbus import bus, EVENT_SITE_INIT 9 | from papihub.exceptions import NotFoundParserException, ParserException, SiteAuthenticationFailureException 10 | from papihub.models import CookieStoreModel 11 | from papihub.models.sitemodel import AuthType, AuthConfig, SiteModel, SiteStatus, CookieAuthConfig, UserAuthConfig 12 | 13 | _LOGGER = logging.getLogger(__name__) 14 | 15 | 16 | class SiteManager: 17 | """ 18 | 站点管理器 19 | """ 20 | 21 | def __init__(self, site_parser_config_loader: SiteParserConfigLoader): 22 | self.parser_instance: Dict[str, TorrentSite] = {} 23 | self.site_parser_config_loader = site_parser_config_loader 24 | self.parser_config = site_parser_config_loader.load() 25 | self.reload_site_instance() 26 | 27 | def reload_site_instance(self): 28 | """ 29 | 重新加载站点实例 30 | 此处加载站点实例暂时不做强制验证，避免应用初始化过慢 31 | :return: 32 | """ 33 | _LOGGER.info("开始初始化站点配置实例") 34 | site_list = SiteModel.list() 35 | if not site_list: 36 | return 37 | for site in site_list: 38 | self.init_site(site.site_id, test_login=False) 39 | _LOGGER.info(f"站点配置实例初始化完成，共{len(site_list)}个站点") 40 | 41 | def get_instance(self, site_id: List[str]): 42 | """ 43 | 获取站点实例 44 | :param site_id: 站点唯一编号 45 | :return: 46 | """ 47 | if not site_id: 48 | return [] 49 | return [self.parser_instance.get(s) for s in site_id if self.parser_instance.get(s) is not None] 50 | 51 | def add(self, site_id: str, auth_type: AuthType, auth_config: AuthConfig): 52 | """ 53 | 添加站点配置信息 54 | 配置会存储在数据库内，方便后期加载到程序 55 | :param site_id: 站点唯一编号 56 | :param auth_type: 授权类型 57 | :param auth_config: 授权详细配置 58 | :return: 59 | """ 60 | if site_id not in self.parser_config: 61 | raise NotFoundParserException(f'站点编号不存在：{site_id}') 62 | parser_config = self.parser_config[site_id] 63 | site = SiteModel.get_by_site_id(site_id) 64 | if site: 65 | raise Exception(f'站点已存在：{site_id}') 66 | site = SiteModel( 67 | site_id=site_id, 68 | display_name=parser_config.site_name, 69 | auth_type=auth_type.value, 70 | auth_config=auth_config.to_json(), 71 | site_status=SiteStatus.Pending.value 72 | ) 73 | site.save() 74 | # 异步做后续的处理 75 | bus.emit(EVENT_SITE_INIT, site_id=site_id, threads=True) 76 | 77 | def init_site(self, site_id: str, test_login: Optional[bool] = True): 78 | """ 79 | 初始化站信息 80 | 初始化后会将状态同步存储到数据库内 81 | :param site_id: 站点唯一编号 82 | :param test_login: 是否测试登录 83 | :return: 84 | """ 85 | site_model = SiteModel.get_by_site_id(site_id) 86 | if not site_model: 87 | site_model.site_status = SiteStatus.Error.value 88 | site_model.status_message = f'站点不存在：{site_id}' 89 | site_model.update() 90 | raise Exception(site_model.status_message) 91 | parser_config = self.parser_config[site_id] 92 | _LOGGER.info(f'开始初始化站点：{parser_config.site_name}') 93 | torrent_site: Optional[TorrentSite] = None 94 | if 'nexusphp' == parser_config.site_type: 95 | torrent_site = NexusPhp(parser_config) 96 | if not torrent_site: 97 | site_model.site_status = SiteStatus.Error.value 98 | site_model.status_message = f'站点类型不支持：{parser_config.site_type}' 99 | site_model.update() 100 | raise ParserException(site_model.status_message) 101 | 102 | try: 103 | if isinstance(torrent_site, Auth): 104 | if site_model.auth_type == AuthType.Cookies.value: 105 | _LOGGER.info(f'使用现有cookie授权：{parser_config.site_name}') 106 | auth_config: CookieAuthConfig = CookieAuthConfig.from_json(site_model.auth_config) 107 | torrent_site.auth_with_cookies(auth_config.cookies) 108 | if test_login: 109 | if not torrent_site.test_login(): 110 | site_model.site_status = SiteStatus.Error.value 111 | site_model.status_message = f'站点认证配置错误：Cookies无效' 112 | site_model.update() 113 | raise SiteAuthenticationFailureException(site_id, parser_config.site_name, 114 | site_model.status_message) 115 | elif site_model.auth_type == AuthType.UserAuth.value: 116 | _LOGGER.info(f'使用用户名密码授权：{parser_config.site_name}') 117 | cookie_store = CookieStoreModel.get_cookies(site_id) 118 | auth = False 119 | if cookie_store: 120 | _LOGGER.info(f'检测到存在cookie历史，使用现有cookie授权：{parser_config.site_name}') 121 | # 优先用现存cookie授权 122 | torrent_site.auth_with_cookies(cookie_store.cookies) 123 | if test_login: 124 | auth = torrent_site.test_login() 125 | else: 126 | auth = True 127 | if not auth: 128 | _LOGGER.info(f'使用现有cookie授权失败，采用用户名密码授权：{parser_config.site_name}') 129 | # 如果已有cookie授权失败，使用用户名密码授权 130 | auth_config: UserAuthConfig = UserAuthConfig.from_json(site_model.auth_config) 131 | cookie_str = torrent_site.auth(auth_config.username, auth_config.password) 132 | expire_time = utils.parse_cookies_expire_time(cookie_str) 133 | ck_item = CookieStoreModel( 134 | site_id=site_id, 135 | cookies=cookie_str, 136 | expire_time=expire_time 137 | ) 138 | ck_item.save() 139 | else: 140 | site_model.site_status = SiteStatus.Error.value 141 | site_model.status_message = f'站点认证配置错误：{site_model.auth_config}' 142 | site_model.update() 143 | raise SiteAuthenticationFailureException(site_id, parser_config.site_name) 144 | except SiteAuthenticationFailureException as e: 145 | raise e 146 | except Exception as e: 147 | site_model.site_status = SiteStatus.Error.value 148 | site_model.status_message = f'站点认证配置错误：{str(e)}' 149 | site_model.update() 150 | raise SiteAuthenticationFailureException(site_id, parser_config.site_name, e) 151 | self.parser_instance.update({site_id: torrent_site}) 152 | SiteModel.update_status(site_id, SiteStatus.Active) 153 | _LOGGER.info(f'站点初始化完成：{parser_config.site_name}') 154 | -------------------------------------------------------------------------------- /papihub/models/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | ORM框架数据持久层的模型 3 | """ 4 | from .cookiestoremodel import CookieStoreModel -------------------------------------------------------------------------------- /papihub/models/cookiestoremodel.py: -------------------------------------------------------------------------------- 1 | from typing import Optional, List 2 | 3 | from papihub.databases import * 4 | 5 | 6 | class CookieStoreModel(BaseDBModel): 7 | """ 8 | 存储站点用的一些Cookie 9 | """ 10 | __tablename__ = 'cookie_store' 11 | 12 | id = Column(Integer, primary_key=True, autoincrement=True, comment='id') 13 | site_id = Column(String, comment='站点编号', nullable=False) 14 | cookies = Column(String, comment='Cookie字符串', nullable=False) 15 | expire_time = Column(DateTime, nullable=False, default=datetime.datetime.now, comment='过期时间') 16 | 17 | @staticmethod 18 | def get_cookies(site_id: str) -> Optional["CookieStoreModel"]: 19 | """ 20 | 根据站点唯一编号获取存储的cookie信息，只返回首条结果 21 | :param site_id: 22 | :return: 23 | """ 24 | return CookieStoreModel.query().filter( 25 | (CookieStoreModel.site_id == site_id) & (CookieStoreModel.expire_time > datetime.datetime.now())).first() 26 | -------------------------------------------------------------------------------- /papihub/models/sitemodel.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from dataclasses_json import dataclass_json 3 | from enum import Enum 4 | from typing import Optional, List 5 | 6 | from papihub.databases import * 7 | 8 | 9 | @dataclass 10 | class AuthType(Enum): 11 | Cookies = 'cookies' 12 | UserAuth = 'user_auth' 13 | 14 | @staticmethod 15 | def from_str(value: str): 16 | if value == 'cookies': 17 | return AuthType.Cookies 18 | elif value == 'user_auth': 19 | return AuthType.UserAuth 20 | else: 21 | raise NotImplementedError 22 | 23 | 24 | @dataclass_json 25 | @dataclass 26 | class AuthConfig: 27 | user_agent: Optional[str] = None 28 | 29 | 30 | @dataclass_json 31 | @dataclass 32 | class CookieAuthConfig(AuthConfig): 33 | cookies: Optional[str] = None 34 | 35 | 36 | @dataclass_json 37 | @dataclass 38 | class UserAuthConfig(AuthConfig): 39 | username: Optional[str] = None 40 | password: Optional[str] = None 41 | 42 | 43 | @dataclass 44 | class SiteStatus(Enum): 45 | Pending = 'pending' 46 | Active = 'active' 47 | Error = 'error' 48 | 49 | 50 | class SiteModel(BaseDBModel): 51 | """ 52 | 站点信息管理 53 | """ 54 | __tablename__ = 'site' 55 | id = Column(Integer, primary_key=True, autoincrement=True, comment='id') 56 | site_id = Column(String, comment='站点编号', nullable=False) 57 | display_name = Column(String, comment='站点显示名称', nullable=False) 58 | auth_type = Column(String, comment='站点认证类型', nullable=False) 59 | auth_config = Column(String, comment='站点认证配置', nullable=False) 60 | site_status = Column(String, comment='站点状态', nullable=False, default='pending') 61 | status_message = Column(String, comment='状态信息', nullable=True) 62 | last_active_time = Column(DateTime, nullable=False, default=datetime.datetime.now) 63 | 64 | @staticmethod 65 | def list() -> List["SiteModel"]: 66 | """ 67 | 获取所有站点信息 68 | :return: 69 | """ 70 | return SiteModel.query().all() 71 | 72 | @staticmethod 73 | def get_by_site_id(site_id: str) -> "SiteModel": 74 | """ 75 | 根据站点唯一编号获取站点信息 76 | :param site_id: 77 | :return: 78 | """ 79 | return SiteModel.query().filter(SiteModel.site_id == site_id).first() 80 | 81 | @staticmethod 82 | def update_status(site_id: str, status: SiteStatus, message: Optional[str] = None): 83 | """ 84 | 更新站点状态 85 | :param site_id: 站点唯一编号 86 | :param status: 状态码 87 | :param message: 状态相关消息 88 | :return: 89 | """ 90 | site = SiteModel.get_by_site_id(site_id) 91 | if site: 92 | site.site_status = status.value 93 | site.status_message = message 94 | site.update() 95 | -------------------------------------------------------------------------------- /papihub/models/usermodel.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | 3 | from papihub.databases import * 4 | 5 | from papihub.databases import BaseDBModel 6 | 7 | 8 | class UserModel(BaseDBModel): 9 | """ 10 | 存储站点用的一些Cookie 11 | """ 12 | __tablename__ = 'user' 13 | __hidden_fields__ = ['password'] 14 | id = Column(Integer, primary_key=True, autoincrement=True, comment='id') 15 | nickname = Column(String(64), nullable=False) 16 | username = Column(String(64), nullable=False) 17 | password = Column(String(256), nullable=False) 18 | 19 | @staticmethod 20 | def get_by_username(username: str) -> "UserModel": 21 | """ 22 | 根据用户名获取用户信息，只返回首条结果 23 | :param username: 24 | :return: 25 | """ 26 | return UserModel.query().filter(UserModel.username == username).first() 27 | -------------------------------------------------------------------------------- /papihub/routers/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | http接口的控制器 3 | """ 4 | -------------------------------------------------------------------------------- /papihub/routers/siterouter.py: -------------------------------------------------------------------------------- 1 | import inject 2 | from fastapi import APIRouter, Depends 3 | 4 | from papihub.auth import get_current_user 5 | from papihub.common.response import json_200 6 | from pydantic import BaseModel 7 | 8 | from papihub.manager.sitemanager import SiteManager 9 | from papihub.models.sitemodel import AuthType, CookieAuthConfig, UserAuthConfig, SiteModel 10 | from papihub.models.usermodel import UserModel 11 | 12 | router = APIRouter() 13 | 14 | 15 | class AddParam(BaseModel): 16 | """ 17 | 添加站点接口的参数 18 | """ 19 | site_id: str 20 | auth_type: str 21 | auth_config: dict 22 | 23 | 24 | @router.get("/api/site/list_parsers") 25 | def list_parsers(): 26 | site_manager = inject.instance(SiteManager) 27 | return json_200( 28 | message='获取站点解析器配置成功', 29 | data=[{ 30 | 'site_id': x.site_id, 31 | 'site_name': x.site_name, 32 | 'domain': x.domain, 33 | 'site_type': x.site_type, 34 | 'config_type': x.config_type, 35 | 'encoding': x.encoding, 36 | } for x in site_manager.parser_config.values()] 37 | ) 38 | 39 | 40 | @router.get("/api/site/list") 41 | def list(): 42 | site_list = SiteModel.list() 43 | site_manager = inject.instance(SiteManager) 44 | results = [] 45 | for x in site_list: 46 | config = site_manager.parser_config.get(x.site_id) 47 | results.append({ 48 | 'site_id': x.site_id, 49 | 'site_name': x.display_name, 50 | 'auth_type': x.auth_type, 51 | 'site_status': x.site_status, 52 | 'status_message': x.status_message, 53 | 'domain': config.domain, 54 | 'last_active_time': x.last_active_time 55 | }) 56 | return json_200( 57 | message='获取站点配置成功', 58 | data=results 59 | ) 60 | 61 | 62 | @router.post("/api/site/add") 63 | def add(param: AddParam, user: UserModel = Depends(get_current_user)): 64 | """ 65 | 添加站点信息到数据库 66 | :param param: 67 | :return: 68 | """ 69 | site_manager = inject.instance(SiteManager) 70 | auth_type = AuthType.from_str(param.auth_type) 71 | auth_config = None 72 | if auth_type is AuthType.Cookies: 73 | auth_config = CookieAuthConfig.from_dict(param.auth_config) 74 | elif auth_type is AuthType.UserAuth: 75 | auth_config = UserAuthConfig.from_dict(param.auth_config) 76 | site_manager.add( 77 | site_id=param.site_id, 78 | auth_type=auth_type, 79 | auth_config=auth_config 80 | ) 81 | return json_200( 82 | message='添加站点配置成功', 83 | ) 84 | -------------------------------------------------------------------------------- /papihub/routers/torrentsrouter.py: -------------------------------------------------------------------------------- 1 | from typing import List, Optional 2 | 3 | import inject 4 | from fastapi import APIRouter 5 | from pydantic import BaseModel 6 | 7 | from papihub.api.concurrenttorrentsite import ConcurrentTorrentSite 8 | from papihub.api.types import CateLevel1 9 | from papihub.common.response import json_200 10 | from papihub.manager.sitemanager import SiteManager 11 | 12 | router = APIRouter() 13 | 14 | 15 | class SearchParam(BaseModel): 16 | site_id: Optional[List[str]] 17 | keyword: str 18 | imdb_id: Optional[str] = None 19 | cate_level1: Optional[List[CateLevel1]] = None 20 | 21 | 22 | @router.post("/api/torrents/search") 23 | def search(param: SearchParam): 24 | """ 25 | 搜索站点种子信息 26 | :param param: 27 | :return: 28 | """ 29 | site_manager = inject.instance(SiteManager) 30 | cts = ConcurrentTorrentSite( 31 | torrent_sites=site_manager.get_instance(param.site_id) 32 | ) 33 | results = cts.search( 34 | keyword=param.keyword, 35 | imdb_id=param.imdb_id, 36 | cate_level1=param.cate_level1 37 | ) 38 | return json_200(data=results) 39 | -------------------------------------------------------------------------------- /papihub/routers/userrouter.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta 2 | from typing import Optional 3 | 4 | from fastapi import Depends, status, APIRouter 5 | from fastapi.security import OAuth2PasswordRequestForm 6 | 7 | from papihub.auth import create_access_token, verify_password, ACCESS_TOKEN_EXPIRE_MINUTES, get_current_user 8 | from papihub.common.response import json_with_status, json_200 9 | from papihub.models.usermodel import UserModel 10 | 11 | router = APIRouter() 12 | 13 | 14 | def auth(username: str, password: str) -> Optional[UserModel]: 15 | user = UserModel.get_by_username(username) 16 | if not user: 17 | return 18 | # 密码要加盐，防范彩虹表攻击 19 | if not verify_password(password, user.password): 20 | return 21 | return user 22 | 23 | 24 | @router.get("/api/user/profile") 25 | def profile(user: UserModel = Depends(get_current_user)): 26 | return json_200(data=user) 27 | 28 | 29 | @router.post("/api/user/get_token") 30 | def get_token(form: OAuth2PasswordRequestForm = Depends()): 31 | user = auth(form.username, form.password) 32 | if not user: 33 | return json_with_status(status.HTTP_401_UNAUTHORIZED, message='用户名或密码错误') 34 | token = create_access_token({"sub": user.username}, timedelta(minutes=ACCESS_TOKEN_EXPIRE_MINUTES)) 35 | return json_200(data={"access_token": token, "token_type": "bearer"}, message='登录成功') 36 | -------------------------------------------------------------------------------- /papihub/tasks/__init__.py: -------------------------------------------------------------------------------- 1 | from .sitetask import * 2 | -------------------------------------------------------------------------------- /papihub/tasks/sitetask.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | import inject 4 | 5 | from papihub.eventbus import bus, EVENT_SITE_INIT 6 | from papihub.manager.sitemanager import SiteManager 7 | 8 | _LOGGER = logging.getLogger(__name__) 9 | 10 | 11 | @bus.on(EVENT_SITE_INIT) 12 | def on_site_init(site_id: str): 13 | """ 14 | 接收站点初始化事件对站点授权信息进行验证，实例初始化。 15 | :param site_id: 站点唯一编号 16 | :return: 17 | """ 18 | _LOGGER.info("站带初始化事件：%s", site_id) 19 | site_manager = inject.instance(SiteManager) 20 | site_manager.init_site(site_id) 21 | -------------------------------------------------------------------------------- /papihub/utils.py: -------------------------------------------------------------------------------- 1 | """ 2 | 所有的工具函数 3 | """ 4 | import datetime 5 | import decimal 6 | import json 7 | from email.utils import parsedate_to_datetime 8 | from enum import Enum 9 | from http.cookies import SimpleCookie 10 | from typing import _GenericAlias, List, Union, Dict 11 | 12 | import emoji 13 | 14 | 15 | def _list_value(value): 16 | if isinstance(value, str): 17 | if value[0] in ['{', '[']: 18 | return json.loads(value) 19 | else: 20 | return value.split(',') 21 | else: 22 | return list(value) 23 | 24 | 25 | def _dict_value(value): 26 | if isinstance(value, str): 27 | return json.loads(value) 28 | else: 29 | return value 30 | 31 | 32 | def parse_field_value(field_value): 33 | if isinstance(field_value, decimal.Decimal): # Decimal -> float 34 | field_value = round(float(field_value), 2) 35 | elif isinstance(field_value, datetime.datetime): # datetime -> str 36 | field_value = str(field_value) 37 | elif isinstance(field_value, list): 38 | field_value = [parse_field_value(i) for i in field_value] 39 | if hasattr(field_value, 'to_json'): 40 | field_value = field_value.to_json() 41 | elif isinstance(field_value, Enum): 42 | field_value = field_value.name 43 | elif isinstance(field_value, Dict): 44 | val = {} 45 | for key_ in field_value: 46 | val[key_] = parse_field_value(field_value[key_]) 47 | field_value = val 48 | return field_value 49 | 50 | 51 | def parse_value(func, value, default_value=None): 52 | if value is not None: 53 | if func == bool: 54 | if value in (1, True, "1", "true"): 55 | return True 56 | elif value in (0, False, "0", "false"): 57 | return False 58 | else: 59 | raise ValueError(value) 60 | 61 | elif func in (int, float): 62 | try: 63 | if isinstance(value, str): 64 | value = value.replace(',', '') 65 | return func(value) 66 | except ValueError: 67 | return float('nan') 68 | elif func == datetime.datetime: 69 | if isinstance(value, datetime.datetime): 70 | return value 71 | elif isinstance(value, str): 72 | if value: 73 | return datetime.datetime.strptime(value, '%Y-%m-%d %H:%M:%S') 74 | else: 75 | return None 76 | else: 77 | return None 78 | elif func in [Dict, dict]: 79 | return _dict_value(value) 80 | elif func in [List, list]: 81 | return _list_value(value) 82 | elif isinstance(func, _GenericAlias): 83 | if func.__origin__ in [List, list]: 84 | list_ = _list_value(value) 85 | res = [] 86 | for x in list_: 87 | res.append(parse_value(func.__args__[0], x)) 88 | return res 89 | elif func.__origin__ == Union: 90 | return parse_value(func.__args__[0], value) 91 | return func(value) 92 | else: 93 | return default_value 94 | 95 | 96 | def trim_emoji(text): 97 | """ 98 | 去掉字符串中的emoji表情 99 | :param text: 100 | :return: 101 | """ 102 | return emoji.demojize(text) 103 | 104 | 105 | def trans_size_str_to_mb(size: str): 106 | """ 107 | 把一个字符串格式的文件尺寸单位，转换成MB单位的标准数字 108 | :param size: 109 | :return: 110 | """ 111 | if not size: 112 | return 0.0 113 | s = None 114 | u = None 115 | if size.find(' ') != -1: 116 | arr = size.split(' ') 117 | s = arr[0] 118 | u = arr[1] 119 | else: 120 | if size.endswith('GB'): 121 | s = size[0:-2] 122 | u = 'GB' 123 | elif size.endswith('GiB'): 124 | s = size[0:-3] 125 | u = 'GB' 126 | elif size.endswith('MB'): 127 | s = size[0:-2] 128 | u = 'MB' 129 | elif size.endswith('MiB'): 130 | s = size[0:-3] 131 | u = 'MB' 132 | elif size.endswith('KB'): 133 | s = size[0:-2] 134 | u = 'KB' 135 | elif size.endswith('KiB'): 136 | s = size[0:-3] 137 | u = 'KB' 138 | elif size.endswith('TB'): 139 | s = size[0:-2] 140 | u = 'TB' 141 | elif size.endswith('TiB'): 142 | s = size[0:-3] 143 | u = 'TB' 144 | elif size.endswith('PB'): 145 | s = size[0:-2] 146 | u = 'PB' 147 | elif size.endswith('PiB'): 148 | s = size[0:-3] 149 | u = 'PB' 150 | if not s: 151 | return 0.0 152 | if s.find(',') != -1: 153 | s = s.replace(',', '') 154 | return trans_unit_to_mb(float(s), u) 155 | 156 | 157 | def trans_unit_to_mb(size: float, unit: str) -> float: 158 | """ 159 | 按文件大小尺寸规格，转换成MB单位的数字 160 | :param size: 161 | :param unit: 162 | :return: 163 | """ 164 | if unit == 'GB' or unit == 'GiB': 165 | return round(size * 1024, 2) 166 | elif unit == 'MB' or unit == 'MiB': 167 | return round(size, 2) 168 | elif unit == 'KB' or unit == 'KiB': 169 | return round(size / 1024, 2) 170 | elif unit == 'TB' or unit == 'TiB': 171 | return round(size * 1024 * 1024, 2) 172 | elif unit == 'PB' or unit == 'PiB': 173 | return round(size * 1024 * 1024 * 1024, 2) 174 | else: 175 | return size 176 | 177 | 178 | def parse_cookies_expire_time(cookie_str: str): 179 | cookie = SimpleCookie() 180 | cookie.load(cookie_str) 181 | for key, morsel in cookie.items(): 182 | if morsel.get('expires'): 183 | return parsedate_to_datetime(morsel['expires']) 184 | return 185 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | uvicorn[standard] 2 | fastapi 3 | SQLAlchemy 4 | PyYAML 5 | Jinja2 6 | httpx 7 | pyquery==2.0.0 8 | dataclasses-json 9 | inject 10 | event-bus 11 | emoji==2.2.0 12 | pydantic 13 | python-jose[cryptography] 14 | passlib[bcrypt] 15 | python-multipart -------------------------------------------------------------------------------- /tests/api/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/tests/api/__init__.py -------------------------------------------------------------------------------- /tests/api/test_nexusphp.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | from papihub.api.sites.nexusphp import NexusPhp 4 | from papihub.api.types import CateLevel1 5 | from tests.config.test_siteparserconfigload import test_load_parser 6 | 7 | parsers = test_load_parser() 8 | 9 | 10 | def test_search(): 11 | api = NexusPhp(parsers.get('mteam')) 12 | asyncio.run(api.auth('xxx', 'xxx')) 13 | torrents = asyncio.run(api.search(keyword='潜行者', cate_level1_list=[CateLevel1.TV])) 14 | -------------------------------------------------------------------------------- /tests/conf/parser/mteam.yml: -------------------------------------------------------------------------------- 1 | config_type: torrent_site 2 | site_id: mteam 3 | site_name: 馒头 4 | site_type: nexusphp 5 | domain: https://kp.m-team.cc 6 | encoding: UTF-8 7 | 8 | 9 | category_mappings: 10 | - { id: 401, cate_level1: Movie, cate_level2: Movies/SD, cate_level2_desc: "Movie(電影)/SD" } 11 | - { id: 419, cate_level1: Movie, cate_level2: Movies/HD, cate_level2_desc: "Movie(電影)/HD" } 12 | - { id: 420, cate_level1: Movie, cate_level2: Movies/DVD, cate_level2_desc: "Movie(電影)/DVDiSo" } 13 | - { id: 421, cate_level1: Movie, cate_level2: Movies/BluRay, cate_level2_desc: "Movie(電影)/Blu-Ray" } 14 | - { id: 439, cate_level1: Movie, cate_level2: Movies/Other, cate_level2_desc: "Movie(電影)/Remux" } 15 | - { id: 403, cate_level1: TV, cate_level2: TV/SD, cate_level2_desc: "TV Series(影劇/綜藝)/SD" } 16 | - { id: 402, cate_level1: TV, cate_level2: TV/HD, cate_level2_desc: "TV Series(影劇/綜藝)/HD" } 17 | - { id: 435, cate_level1: TV, cate_level2: TV/SD, cate_level2_desc: "TV Series(影劇/綜藝)/DVDiSo" } 18 | - { id: 438, cate_level1: TV, cate_level2: TV/HD, cate_level2_desc: "TV Series(影劇/綜藝)/BD" } 19 | - { id: 404, cate_level1: Documentary, cate_level2: TV/Documentary, cate_level2_desc: "紀錄教育" } 20 | - { id: 405, cate_level1: Anime, cate_level2: TV/Anime, cate_level2_desc: "Anime(動畫)" } 21 | - { id: 407, cate_level1: TV, cate_level2: TV/Sport, cate_level2_desc: "Sports(運動)" } 22 | - { id: 422, cate_level1: Other, cate_level2: PC/0day, cate_level2_desc: "Software(軟體)" } 23 | - { id: 423, cate_level1: Game, cate_level2: PC/Games, cate_level2_desc: "PCGame(PC遊戲)" } 24 | - { id: 427, cate_level1: Other, cate_level2: Books, cate_level2_desc: "eBook(電子書)" } 25 | - { id: 409, cate_level1: Other, cate_level2: Other, cate_level2_desc: "Misc(其他)" } 26 | - { id: 406, cate_level1: Music, cate_level2: Audio/Video, cate_level2_desc: "MV(演唱)" } 27 | - { id: 408, cate_level1: Music, cate_level2: Audio/Other, cate_level2_desc: "Music(AAC/ALAC)" } 28 | - { id: 434, cate_level1: Music, cate_level2: Audio, cate_level2_desc: "Music(無損)" } 29 | - { id: 410, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(有碼)/HD Censored" } 30 | - { id: 429, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(無碼)/HD Uncensored" } 31 | - { id: 424, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(有碼)/SD Censored" } 32 | - { id: 430, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(無碼)/SD Uncensored" } 33 | - { id: 426, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(無碼)/DVDiSo Uncensored" } 34 | - { id: 437, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(有碼)/DVDiSo Censored" } 35 | - { id: 431, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(有碼)/Blu-Ray Censored" } 36 | - { id: 432, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(無碼)/Blu-Ray Uncensored" } 37 | - { id: 436, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "AV(網站)/0Day" } 38 | - { id: 425, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "IV(寫真影集)/Video Collection" } 39 | - { id: 433, cate_level1: AV, cate_level2: XXX, cate_level2_desc: "IV(寫真圖集)/Picture Collection" } 40 | - { id: 411, cate_level1: Game, cate_level2: XXX, cate_level2_desc: "H-Game(遊戲)" } 41 | - { id: 412, cate_level1: Anime, cate_level2: XXX, cate_level2_desc: "H-Anime(動畫)" } 42 | - { id: 413, cate_level1: Anime, cate_level2: XXX, cate_level2_desc: "H-Comic(漫畫)" } 43 | 44 | user: 45 | path: https://kp.m-team.cc/index.php 46 | item: 47 | selector: table[id="info_block"] 48 | fields: 49 | uid: 50 | default_value: 0 51 | selector: a[href^="userdetails.php?id="] 52 | attribute: href 53 | filters: 54 | - name: re_search 55 | args: [ '\d+', 0 ] 56 | user_group: 57 | default_value: '' 58 | selector: a[href^="userdetails.php?id="] 59 | attribute: class 60 | username: 61 | default_value: '未知用户名' 62 | selector: a[href^="userdetails.php?id="] 63 | share_ratio: 64 | default_value: '' 65 | selector: font[class="color_ratio"] 66 | method: next_sibling 67 | filters: 68 | - name: replace 69 | args: [ '無限','inf' ] 70 | uploaded: 71 | default_value: 0 72 | selector: font[class="color_uploaded"] 73 | method: next_sibling 74 | filters: 75 | - name: replace 76 | args: [ '無限','inf' ] 77 | downloaded: 78 | default_value: 0 79 | selector: font[class="color_downloaded"] 80 | method: next_sibling 81 | filters: 82 | - name: replace 83 | args: [ '無限','inf' ] 84 | seeding: 85 | default_value: 0 86 | selector: img[alt="Torrents seeding"] 87 | method: next_sibling 88 | leeching: 89 | default_value: 0 90 | selector: img[alt="Torrents leeching"] 91 | method: next_sibling 92 | vip_group: 93 | case: 94 | a[class^="VIP"]: true 95 | "*": false 96 | search: 97 | paths: 98 | - path: torrents.php 99 | categories: [ "!", 410, 429, 424, 430, 426, 437, 431, 432, 436, 425, 433, 411, 412, 413, 406, 408, 434 ] 100 | - path: adult.php 101 | categories: [ 410, 429, 424, 430, 426, 437, 431, 432, 436, 425, 433, 411 ] 102 | - path: music.php 103 | categories: [ 406, 408, 434 ] 104 | query: 105 | $raw: "{% for c in query.cates %}cat{{c}}=1&{% endfor %}" 106 | search: "{% if query.imdb_id %}{{query.imdb_id}}{%else%}{{query.keyword}}{% endif %}" 107 | # 0 incldead, 1 active, 2 dead 108 | incldead: 1 109 | # 0 all, 1 normal, 2 free, 3 2x, 4 2xfree, 5 50%, 6 2x50%, 7 30% 110 | spstate: "{% if query.free %}2{% else %}0{% endif %}" 111 | # 0 title, 1 descr, 3 uploader, 4 imdburl (searching imdburl does not work with tt1234567, but descr is good) 112 | search_area: "{% if query.imdb_id %}4{% else %}0{%endif%}" 113 | # 0 AND, 1 OR, 2 exact 114 | search_mode: 0 115 | sort: "{{ query.sort }}" 116 | type: "{{ query.type }}" 117 | page: "{{ query.page }}" 118 | 119 | torrents: 120 | list: 121 | selector: table.torrents > tr:has(table.torrentname) 122 | fields: 123 | id: 124 | selector: a[href^="details.php?id="] 125 | attribute: href 126 | filters: 127 | - name: re_search 128 | args: [ '\d+', 0 ] 129 | title_default: 130 | # shortened for long release names 131 | selector: a[href^="details.php?id="] > b 132 | title_optional: 133 | # not available if IMDB tooltips are turned on 134 | optional: true 135 | selector: a[title][href^="details.php?id="] 136 | attribute: title 137 | title: 138 | text: "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}" 139 | category: 140 | selector: a[href^="?cat="] 141 | attribute: href 142 | filters: 143 | - name: querystring 144 | args: cat 145 | details: 146 | selector: a[href^="details.php?id="] 147 | attribute: href 148 | download: 149 | selector: a[href^="download.php?id="] 150 | attribute: href 151 | poster: 152 | selector: img[alt="torrent thumbnail"] 153 | attribute: src 154 | filters: 155 | - name: replace 156 | args: [ "pic/nopic.jpg", "" ] 157 | imdbid: 158 | selector: a[href*="imdb.com/title/tt"] 159 | attribute: href 160 | filters: 161 | - name: re_search 162 | args: [ 'tt\d+',0 ] 163 | size: 164 | selector: td.rowfollow:nth-last-child(6) 165 | grabs: 166 | selector: td.rowfollow:nth-last-child(3) 167 | seeders: 168 | selector: td.rowfollow:nth-last-child(5) 169 | leechers: 170 | selector: td.rowfollow:nth-last-child(4) 171 | date_added: 172 | selector: td.rowfollow:nth-last-child(7) > span[title] 173 | optional: true 174 | attribute: title 175 | date_elapsed: 176 | selector: tr > td > span[title] 177 | attribute: title 178 | optional: true 179 | date: 180 | text: "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_elapsed'] if fields['date_elapsed'] else fields['date_added'] }}{% else %}now{% endif %}" 181 | filters: 182 | - name: dateparse 183 | args: "%Y-%m-%d %H:%M:%S" 184 | downloadvolumefactor: 185 | case: 186 | img.pro_free: 0 187 | img.pro_free2up: 0 188 | img.pro_50pctdown: 0.5 189 | img.pro_50pctdown2up: 0.5 190 | img.pro_30pctdown: 0.3 191 | "*": 1 192 | uploadvolumefactor: 193 | case: 194 | img.pro_50pctdown2up: 2 195 | img.pro_free2up: 2 196 | img.pro_2up: 2 197 | "*": 1 198 | free_deadline: 199 | default_value: "{% if fields['downloadvolumefactor']==0 %}{{max_time}}{% endif%}" 200 | default_value_format: '%Y-%m-%d %H:%M:%S.%f' 201 | selector: span[style="font-weight:normal"] 202 | filters: 203 | - name: re_search 204 | args: [ '(?:限時：\s*)((?:\d+日)?(?:\d+時)?(?:\d+分)?)',1 ] 205 | - name: date_elapsed_parse 206 | description: 207 | selector: table.torrentname > tr > td.embedded 208 | contents: -1 -------------------------------------------------------------------------------- /tests/conf/parser/ourbits.yml: -------------------------------------------------------------------------------- 1 | config_type: torrent_site 2 | site_id: ourbits 3 | site_name: 我堡 4 | site_type: nexusphp 5 | domain: https://ourbits.club/ 6 | encoding: UTF-8 7 | 8 | category_mappings: 9 | - { id: 401, cate_level1: Movie, cate_level2: "Movies", cate_level2_desc: "Movies" } 10 | - { id: 402, cate_level1: Movie, cate_level2: "Movies-3D", cate_level2_desc: "Movies-3D" } 11 | - { id: 419, cate_level1: Music, cate_level2: "Concert", cate_level2_desc: "Concert" } 12 | - { id: 412, cate_level1: TV, cate_level2: "TV-Episode", cate_level2_desc: "TV-Episode" } 13 | - { id: 405, cate_level1: TV, cate_level2: "TV-Pack", cate_level2_desc: "TV-Pack" } 14 | - { id: 413, cate_level1: TV, cate_level2: "TV-Show", cate_level2_desc: "TV-Show" } 15 | - { id: 410, cate_level1: Documentary, cate_level2: "Documentary", cate_level2_desc: "Documentary" } 16 | - { id: 411, cate_level1: Anime, cate_level2: "Animation", cate_level2_desc: "Animation" } 17 | - { id: 415, cate_level1: TV, cate_level2: "Sports", cate_level2_desc: "Sports" } 18 | - { id: 414, cate_level1: Music, cate_level2: "Music-Video", cate_level2_desc: "Music-Video" } 19 | - { id: 414, cate_level1: Music, cate_level2: "Music-Video", cate_level2_desc: "Music-Video" } 20 | - { id: 416, cate_level1: Music, cate_level2: "Music", cate_level2_desc: "Music" } 21 | user: 22 | path: https://ourbits.club/rules.php 23 | item: 24 | selector: table[id="info_block"] 25 | fields: 26 | uid: 27 | default_value: 0 28 | selector: a[href^="userdetails.php?id="] 29 | attribute: href 30 | filters: 31 | - name: re_search 32 | args: [ '\d+', 0 ] 33 | user_group: 34 | default_value: '' 35 | selector: a[href^="userdetails.php?id="] 36 | attribute: class 37 | username: 38 | default_value: '未知用户名' 39 | selector: a[href^="userdetails.php?id="] 40 | share_ratio: 41 | default_value: '' 42 | selector: span[class="color_ratio"] 43 | method: next_sibling 44 | filters: 45 | - name: replace 46 | args: [ '无限','inf' ] 47 | uploaded: 48 | default_value: 0 49 | selector: font[class="color_uploaded"] 50 | method: next_sibling 51 | filters: 52 | - name: replace 53 | args: [ '无限','inf' ] 54 | downloaded: 55 | default_value: 0 56 | selector: font[class="color_downloaded"] 57 | method: next_sibling 58 | filters: 59 | - name: replace 60 | args: [ '无限','inf' ] 61 | seeding: 62 | default_value: 0 63 | selector: img[alt="Torrents seeding"] 64 | method: next_sibling 65 | leeching: 66 | default_value: 0 67 | selector: img[alt="Torrents leeching"] 68 | method: next_sibling 69 | vip_group: 70 | case: 71 | a[class^="VIP"]: true 72 | "*": false 73 | search: 74 | paths: 75 | - path: torrents.php 76 | method: get 77 | query: 78 | $raw: "{% for c in query.cates %}cat{{c}}=1&{% endfor %}" 79 | search: "{% if query.imdb_id %}{{query.imdb_id}}{%else%}{{query.keyword}}{% endif %}" 80 | # 0 incldead, 1 active, 2 dead 81 | incldead: 1 82 | # 0 all, 1 normal, 2 free, 3 2x, 4 2xfree, 5 50%, 6 2x50%, 7 30% 83 | spstate: "{% if query.free %}2{% else %}0{% endif %}" 84 | # 0 title, 1 descr, 3 uploader, 4 imdburl (searching imdburl does not work with tt1234567, but descr is good) 85 | search_area: "{% if query.imdb_id %}4{% else %}0{%endif%}" 86 | # 0 AND, 1 OR, 2 exact 87 | search_mode: 0 88 | sort: "{{ query.sort }}" 89 | type: "{{ query.type }}" 90 | page: "{{ query.page }}" 91 | 92 | torrents: 93 | list: 94 | selector: table.torrents > tr:has(table.torrentname) 95 | fields: 96 | id: 97 | selector: a[href^="details.php?id="] 98 | attribute: href 99 | filters: 100 | - name: re_search 101 | args: [ '\d+', 0 ] 102 | category: 103 | selector: a[href^="?cat="] 104 | attribute: href 105 | filters: 106 | - name: querystring 107 | args: cat 108 | title_default: 109 | selector: a[href^="details.php?id="] 110 | title_optional: 111 | optional: true 112 | selector: a[title][href^="details.php?id="] 113 | attribute: title 114 | title: 115 | text: "{% if fields['title_optional'] %}{{ fields['title_optional'] }}{% else %}{{ fields['title_default'] }}{% endif %}" 116 | details: 117 | selector: a[href^="details.php?id="] 118 | attribute: href 119 | download: 120 | selector: a[href^="download.php?id="] 121 | attribute: href 122 | size: 123 | selectors: td:nth-child(5) 124 | index: 1 125 | grabs: 126 | selector: td:nth-child(8) 127 | seeders: 128 | selector: td:nth-child(6) 129 | leechers: 130 | selector: td:nth-child(7) 131 | date_elapsed: 132 | # time type: time elapsed (default) 133 | selector: td:nth-child(4) > span[title] 134 | attribute: title 135 | optional: true 136 | date_added: 137 | # time added 138 | selector: td:nth-child(4):not(:has(span)) 139 | optional: true 140 | date: 141 | text: "{% if fields['date_elapsed'] or fields['date_added'] %}{{ fields['date_elapsed'] if fields['date_elapsed'] else fields['date_added'] }}{% else %}now{% endif %}" 142 | filters: 143 | - name: dateparse 144 | args: "%Y-%m-%d %H:%M:%S" 145 | downloadvolumefactor: 146 | case: 147 | img.pro_free: 0 148 | img.pro_free2up: 0 149 | img.pro_50pctdown: 0.5 150 | img.pro_50pctdown2up: 0.5 151 | img.pro_30pctdown: 0.3 152 | "*": 1 153 | uploadvolumefactor: 154 | case: 155 | img.pro_50pctdown2up: 2 156 | img.pro_free2up: 2 157 | img.pro_2up: 2 158 | "*": 1 159 | free_deadline: 160 | default_value: "{% if fields['downloadvolumefactor']==0 %}{{max_time}}{% endif%}" 161 | default_value_format: '%Y-%m-%d %H:%M:%S.%f' 162 | selector: 'td[class="embedded"] > b > span[title]' 163 | attribute: title 164 | filters: 165 | - name: dateparse 166 | args: "%Y-%m-%d %H:%M:%S" 167 | tags: 168 | selector: table.torrentname > tr > td.embedded > div:has(a) 169 | tmp_subject: 170 | selector: table.torrentname > tr > td.embedded 171 | remove: div, a 172 | subject: 173 | selector: table.torrentname > tr > td.embedded 174 | contents: -1 175 | description: 176 | text: "{% if fields['tags']%}{{ fields['subject']+' '+fields['tags'] }}{% else %}{{ fields['subject'] }}{% endif %}" 177 | minimumratio: 178 | case: 179 | img.hitandrun: 1 180 | "*": 0 181 | minimumseedtime: 182 | case: 183 | img.hitandrun: 172800 184 | "*": 0 -------------------------------------------------------------------------------- /tests/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/papihubcom/papihub/8a6f72f7e3aba86e93004eb9ea5215bf0d8c58e3/tests/config/__init__.py -------------------------------------------------------------------------------- /tests/config/test_siteparserconfigload.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from papihub.config.siteparserconfigloader import SiteParserConfigLoader 4 | 5 | 6 | def test_load_parser(): 7 | configs = SiteParserConfigLoader(os.path.join(os.path.dirname(os.path.realpath('.')), 'conf', 'parser')).load() 8 | assert len(configs) > 0 9 | return configs 10 | -------------------------------------------------------------------------------- /tests/test_utils.py: -------------------------------------------------------------------------------- 1 | from papihub import utils 2 | 3 | 4 | def test_parse_cookies_expire_time(): 5 | print(utils.parse_cookies_expire_time('session_id=abcd1234; expires=Wed, 09-Jun-2021 10:18:14 GMT')) 6 | --------------------------------------------------------------------------------