├── .babelrc ├── .dockerignore ├── .github └── workflows │ ├── build_macos_release.yml │ └── build_windows_release.yml ├── .gitignore ├── CHANGELOG ├── Dockerfile ├── JAV_HELP.md ├── JavHelper ├── __init__.py ├── app.py ├── cache.py ├── core │ ├── OOF_downloader.py │ ├── __init__.py │ ├── aria2_handler.py │ ├── arzon.py │ ├── backend_translation.py │ ├── deluge_downloader.py │ ├── emby_actors.py │ ├── file_scanner.py │ ├── ini_file.py │ ├── jav321.py │ ├── jav777.py │ ├── jav_scraper.py │ ├── javbus.py │ ├── javdb.py │ ├── javlibrary.py │ ├── local_db.py │ ├── nfo_parser.py │ ├── requester_proxy.py │ ├── tushyraw.py │ ├── utils.py │ └── warashi.py ├── model │ ├── __init__.py │ └── jav_manager.py ├── run.py ├── scripts │ ├── __init__.py │ └── emby_actors.py ├── static │ ├── js │ │ └── webHelper.js │ └── webHelper │ │ ├── HelpDoc.jsx │ │ ├── OofValidator.jsx │ │ ├── configurator.css │ │ ├── configurator.jsx │ │ ├── entry.jsx │ │ ├── fileTable.jsx │ │ ├── i18n.js │ │ ├── idmm_download.css │ │ ├── idmm_download.jsx │ │ ├── javBrowserChecker.css │ │ ├── javBrowserChecker.jsx │ │ ├── javBrowserV2.css │ │ ├── javBrowserV2.jsx │ │ ├── javCardV2.jsx │ │ ├── javMagnetButton.jsx │ │ ├── javSetSearchBGroup.jsx │ │ ├── javTable.jsx │ │ ├── localJavCard.jsx │ │ ├── localJavInfoTabs.jsx │ │ ├── localManager.jsx │ │ ├── localManager │ │ ├── local_jav_card.jsx │ │ ├── local_jav_card_state.jsx │ │ ├── local_manager_app.jsx │ │ ├── local_manager_configurator.jsx │ │ └── local_manager_state.jsx │ │ ├── statButtonGroup.jsx │ │ ├── styling.jsx │ │ ├── urlLimiterInspector.jsx │ │ ├── webHelper.css │ │ └── webHelper.jsx ├── templates │ └── home.html ├── utils.py └── views │ ├── __init__.py │ ├── emby_actress.py │ ├── jav_browser.py │ ├── local_manager.py │ ├── parse_jav.py │ └── scan_directory.py ├── LICENSE ├── README.md ├── build_docker.sh ├── build_macos.sh ├── build_windows.bat ├── demo ├── favicon.png ├── feature1.gif ├── feature2.gif ├── icon.ico ├── javdown_115_1.png ├── javdown_115_2.png ├── javdown_115_3.png ├── javdown_115_4.png ├── javdown_115_add_cookies.gif ├── javdown_115_validate.png ├── javdown_basic_down_1.gif ├── javdown_basic_down_2.png ├── javdown_basic_stat.png ├── javdown_filter.png ├── javdown_infiScroll.gif └── javdown_manual_search.png ├── package.json ├── requirements.txt ├── translation.json ├── version.py ├── webpack.config.js └── webpack.prod.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["react", "es2015"] 3 | } -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | JavHelper/static/nv 4 | node_modules 5 | .idea 6 | dist-python 7 | __pycache__ 8 | test.py 9 | .vscode 10 | .ipynb_checkpoints 11 | *.gz 12 | yarn-error.log 13 | yarn.lock 14 | *.ipynb 15 | *.pyc 16 | *.DS_Store 17 | run.spec 18 | rewrite_his.sh 19 | *.backup 20 | *.db -------------------------------------------------------------------------------- /.github/workflows/build_macos_release.yml: -------------------------------------------------------------------------------- 1 | name: build_macos_release 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | #on: [push] 8 | 9 | jobs: 10 | build: 11 | runs-on: macos-latest 12 | steps: 13 | - uses: actions/checkout@master 14 | - name: Install Python 15 | uses: actions/setup-python@v1 16 | with: 17 | python-version: 3.7.9 18 | 19 | - name: Install Python Packages 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install -r requirements.txt 23 | 24 | - name: build macos pkg 25 | run: | 26 | pyinstaller --onedir \ 27 | --add-data="demo:demo" \ 28 | --add-data="translation.json:." \ 29 | --add-data="README.md:." \ 30 | --add-data="JAV_HELP.md:." \ 31 | --add-data="JavHelper/templates:JavHelper/templates" \ 32 | --add-data="JavHelper/static:JavHelper/static" \ 33 | --add-data="/Users/runner/hostedtoolcache/Python/3.7.9/x64/lib/python3.7/site-packages/cloudscraper:cloudscraper" \ 34 | --hidden-import="js2py" \ 35 | --hidden-import="cloudscraper" \ 36 | --hidden-import="cloudscraper_exception" \ 37 | --exclude-module="FixTk" \ 38 | --exclude-module="tcl" \ 39 | --exclude-module="tk" \ 40 | --exclude-module="_tkinter" \ 41 | --exclude-module="tkinter" \ 42 | --exclude-module="Tkinter" \ 43 | --noconfirm \ 44 | --distpath JAVOneStop_${GITHUB_REF##*/} \ 45 | JavHelper/run.py 46 | 47 | tar -czf Jav_OneStop_macos.tar.gz JAVOneStop_${GITHUB_REF##*/} 48 | 49 | - name: Create MacOS Release 50 | id: create_release_macos 51 | uses: actions/create-release@master 52 | env: 53 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 54 | with: 55 | tag_name: ${{ github.ref }}_macos 56 | release_name: Release ${{ github.ref }} MacOS 57 | draft: false 58 | prerelease: false 59 | 60 | - name: Upload Release Asset MacOS 61 | id: upload-release-asset-macos 62 | uses: actions/upload-release-asset@v1 63 | env: 64 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 65 | with: 66 | upload_url: ${{ steps.create_release_macos.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps 67 | asset_path: ./Jav_OneStop_macos.tar.gz 68 | asset_name: Jav_OneStop_macos.tar.gz 69 | asset_content_type: application/zip 70 | -------------------------------------------------------------------------------- /.github/workflows/build_windows_release.yml: -------------------------------------------------------------------------------- 1 | name: build_windows_releases 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | #on: [push] 8 | 9 | jobs: 10 | build: 11 | runs-on: windows-latest 12 | 13 | steps: 14 | - uses: actions/checkout@master 15 | - name: Install Python 16 | uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.7.9 19 | 20 | - name: Install Python Packages 21 | run: | 22 | python -m pip install --upgrade pip 23 | pip install -r requirements.txt 24 | 25 | - name: build windows pkg 26 | run: | 27 | pyinstaller --onedir --icon "demo\icon.ico" --add-data="demo;demo" --add-data="translation.json;." --add-data="README.md;." --add-data="JAV_HELP.md;." --add-data="JavHelper\templates;JavHelper\templates" --add-data="JavHelper\static;JavHelper\static" --add-data="c:\hostedtoolcache\windows\python\3.7.9\x64\lib\site-packages\cloudscraper;cloudscraper" --hidden-import="js2py" --hidden-import="cloudscraper" --hidden-import="cloudscraper_exception" --exclude-module="FixTk" --exclude-module="tcl" --exclude-module="tk" --exclude-module="_tkinter" --exclude-module="tkinter" --exclude-module="Tkinter" --noconfirm --distpath dist-python JavHelper\run.py 28 | 29 | 7z a -tzip "Jav_OneStop_windows.zip" ".\dist-python\" 30 | 31 | - name: Create Windows Release 32 | id: create_release_windows 33 | uses: actions/create-release@master 34 | env: 35 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 36 | with: 37 | tag_name: ${{ github.ref }}_windows 38 | release_name: Release ${{ github.ref }} Windows 39 | draft: false 40 | prerelease: false 41 | 42 | - name: Upload Release Asset Windows 43 | id: upload-release-asset-windows 44 | uses: actions/upload-release-asset@v1 45 | env: 46 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 47 | with: 48 | upload_url: ${{ steps.create_release_windows.outputs.upload_url }} 49 | asset_path: ./Jav_OneStop_windows.zip 50 | asset_name: Jav_OneStop_windows.zip 51 | asset_content_type: application/zip 52 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | dist 3 | JavHelper/static/nv 4 | node_modules 5 | .idea 6 | dist-python 7 | __pycache__ 8 | test.py 9 | .vscode 10 | .ipynb_checkpoints 11 | *.gz 12 | yarn-error.log 13 | yarn.lock 14 | *.ipynb 15 | *.pyc 16 | settings.ini 17 | 115_cookies.json 18 | *.DS_Store 19 | jav_manager.db 20 | run.spec 21 | rewrite_his.sh 22 | .spec 23 | javlib_cf_cookies.json 24 | dist-python.zip 25 | jav_manager.sqlite -------------------------------------------------------------------------------- /CHANGELOG: -------------------------------------------------------------------------------- 1 | # Changelog 2 | JavOneStop which is a small tool that helps users rename, parse, generate nfo, organize jav video files and 3 | communicate with Emby to add actresses images. 4 | 5 | ## [Unreleased] 6 | 7 | 8 | ## [0.9.0] - 2021-08-21 9 | ### 新增 10 | - 增加deluge磁链下载支持 11 | - 增加tushyraw站点 12 | - 增加javdb的磁链搜索 13 | - 增加zhongziso磁链搜索 14 | ### 改进 / 修复 15 | - 更新nyaa磁链搜索路径 16 | - 默认JAV浏览切换至JavLibrary 17 | - 修复新的115下载失败的逻辑 18 | - 修复下载图片失败 19 | - 改进文件写入逻辑 20 | - 修正部分节点不使用代理 21 | - 修复后缀命名逻辑 22 | - 修复各站点的xpath数据路径 23 | - 修复部分节点不支持cloudflare 24 | - 修复一些刮削失败情况下的处理 25 | ### 移除 26 | - 移除torrentkitty搜索 27 | - 移除添加115任务至Aria2下载器 28 | 29 | ## [0.8.0] - 2020-09-17 30 | ### 新增 31 | - 支持sqlite数据库后端 32 | - 如何从blitz转移数据至sqlite: 33 | - 在配置表单, 数据库类型里保持选中blitz; 请备份已有的blitz数据库 34 | - 访问 127.0.0.1:8009/migrate_to_sqlite 等待完成, 切勿打断 35 | - 进入配置表单, 切换数据库类型至sqlite, 并重启程序 36 | - 将来的新功能将不再测试blitz数据库而只保证支持sqlite 37 | - 增加115限额读取 38 | - 新增jav321网站支持 39 | - 新增本地数据库访问至JAV下载器以用于重新下载 40 | - 添加更新日志至工具首页 41 | - 部分页面现在支持url构建, 用户可以刷新页面而不用从头开始 42 | - 新增全部标记为"想要"的快捷键 43 | - 重写刮削工具已有视频逻辑 44 | - 支持原有的"重写nfo", "重写图片", "更新nfo"功能 45 | - 新增车牌状态"冷冻箱", 用来分辨不在本地但是找不到下载链接的状况 46 | - 新增中英文语言切换 47 | ### 改进 / 修复 48 | - 优化dockerfile构建顺序 49 | - 优化后端结构, 合并各大jav刮削和访问节点 50 | - 改进图片写入逻辑 51 | - 修复jav777车牌处理的问题 52 | - 修复javdb访问问题 53 | - 改进idmm进度工具访问逻辑 54 | - jav777的下载链接现在可以复制车牌至粘贴板 55 | - 改进各种界面比例 56 | 57 | 58 | ## [0.7.6] - 2020-08-17 59 | ### Added 60 | - Able to scrape multiple jav in jav scraper 61 | - Add UI language toggle 62 | ### Updated 63 | - Update docker build script to use local db to reduce build time 64 | - Fix image expired bug when accessing javdb 65 | - Added retry logic when accessing db 66 | - Update javdb url link 67 | - Directory scan now will return files sort by filename 68 | 69 | ## [0.7.5] - 2020-07-28 70 | ### Updated 71 | - Fix pyinstaller issue 72 | - Update Pillow for security 73 | 74 | ## [0.7.4] - 2020-07-25 75 | ### Updated 76 | - Major rewrite with the local manager tab 77 | - Try to fix macos release build 78 | 79 | ## [0.7.2] - 2020-06-08 80 | ### Added 81 | - testing new keyboard shortcut for jav browser 82 | - added javdb to jav browser and parser 83 | ### Updated 84 | - added ignore for cloudflare cookies json 85 | - up the allowable 115 file size from 100 to 200MB 86 | - better error handling for 115 downloader 87 | - no genres will no longer be copied from tags nfo 88 | - update cloudflare scraper handling 89 | - reshape a lot of jav browser layout for small screens 90 | 91 | ## [0.7.1] - 2020-04-17 92 | ### Added 93 | - alpha version of jav777 support as a download source 94 | - add configuration to allow user remove certain string when renaming files 95 | - add UI support for user custom ikoa / dmmc downloader 96 | - add a overall download search to optimize download flow 97 | - add an endpoint for car that requires ikoa credit 98 | ### Updated 99 | - fixed some read and write database issue with car 100 | - upgrade pillow for security reason 101 | - upgrade cloudscraper for newer chanllenges 102 | 103 | ## [0.7.0] - 2020-03-15 104 | ### Added 105 | - readme tab to teach user how to 106 | - toast error messages for misconfigured 115 / aria2 server 107 | ### Updated 108 | - brand new pagination 109 | - better 115 & aria2 downloader logging 110 | - full localization on jav download tool 111 | - magnet sorting based on size and subtitled (flawed though) 112 | - bug fix for not infinite scroll when switch between fully loaded page and new page 113 | - remove cache from reading the source site (since we need newest db stat) 114 | 115 | ## [0.6.4] - 2020-03-13 116 | ### Added 117 | - 115 downloader error message translations 118 | - ui elements to switch between different sources for magnet link search 119 | - add 115 validator when manual validation is necessary 120 | ### Updated 121 | - retry logic for 115 downloader 122 | - fix bug for release date (changes to "premiered" in nfo) 123 | 124 | ## [0.6.3] - 2020-03-09 125 | ### Updated 126 | - fix cloudscraper import issue 127 | 128 | ## [0.6.2] - 2020-03-09 129 | ### Updated 130 | - fix opacity issue 131 | 132 | ## [0.6.1] - 2020-03-09 133 | ### Added 134 | - now user can configure jav sources and priority in the configuration tool 135 | ### Updated 136 | - better loading animation since javlibrary now is very slow 137 | - fixed detailed image tab when browsing javbus 138 | 139 | ## [0.6.0] - 2020-03-08 140 | ### Added 141 | - added javbus support (user needs to manually edit settings.ini for now to add javbus scrape) 142 | ### Updated 143 | - updated the docker related script to resolve slow build issue 144 | - javlibrary now uses cloudscraper to bypass cloudflare 145 | - some localization improvements 146 | ### Removed 147 | - completely removed older javbrower code (now only v2 exists) 148 | 149 | ## [0.5.3] - 2020-03-04 150 | ### Added 151 | - [JavBrowserV2] jav browser now has a detailed image tab for each jav 152 | - add role when writing nfo for better visual 153 | - language now is configurable 154 | - user now can manually choose which data source to use for individual source 155 | ### Updated 156 | - add infinite scroll back to jav browser 157 | - update the migrating logic so now the tool will look for an actual video file instead of just nfo 158 | - small updates to the translation 159 | - fix the pagenation issue where sometimes it won't get updated 160 | - pin log console to the top for better log viewing 161 | 162 | ## [0.5.0] - 2020-02-27 163 | ### Added 164 | - local jav manager 165 | - new scraper class to ease new implementation 166 | - user can configure scraper sources directly in the configure tool 167 | ### Updated 168 | - restructure javlibrary and arzon scraper 169 | - better windows and linux (macos) os path support 170 | - user can configure saved folder structure in the configure tool 171 | - fix a bug to rebuild index when doing db search locally 172 | 173 | ## [0.4.2] - 2020-02-17 174 | ### Added 175 | - docker deployment script 176 | - now use chinese by default 177 | - add tool to configure 115 cookies directly inside of the tool 178 | ### Updated 179 | - 115 download grace failure 180 | - better jav search functions 181 | - alpha mobile view version 182 | 183 | ## [0.4.1] - 2020-02-12 184 | ### Updated 185 | - fix T28 R18 jav scrape 186 | - fix problematic rename for subtitled video 187 | - upgrade pyinstaller version for security reason 188 | - backend only will handle one request at a time to avoid concurrency issue for blitzDB and 115 download 189 | 190 | ## [0.4.0] - 2020-02-12 191 | ### Added 192 | - local blitzDB to handle jav file status look up 193 | - jav manager - 115 - aria2 download support 194 | - flask cache for faster web response 195 | - new function to parse two javlibrary most wanted and best rated pages 196 | - new readme with demo gifs 197 | ### Updated 198 | - nfo parser now by default capitalize "car" 199 | - production webpack react compile to reduce warnings 200 | ### Removed 201 | - no longer support "C" as cd postfix 202 | 203 | ## [0.3.0] - 2020-02-05 204 | ### Added 205 | - warashi scraper which is used for emby actor images 206 | - Handle multiple CD filename postfix 207 | - Handle Chinese subtitle filename postfix 208 | ### Updated 209 | - Fix bug when writing images 210 | - Fix read from ini file so no restart is needed 211 | 212 | ## [0.2.0] - 2019-12-31 213 | ### Updated 214 | - update README for new usage 215 | ### Removed 216 | - remove flaskwebgui package usage 217 | 218 | ## [0.1.0] - 2019-12-29 219 | ### Added 220 | - Basic Architecture for front and back end 221 | - javLibrary parser 222 | - arzon plot parser 223 | - emby actress image upload 224 | - jav file organization and generate nfo -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # Dockerfile 2 | 3 | # FROM directive instructing base image to build upon 4 | FROM python:3.7.3-stretch 5 | 6 | # Config app 7 | WORKDIR /usr/src/app 8 | COPY requirements.txt . 9 | RUN pip install --no-cache-dir -r requirements.txt 10 | 11 | # EXPOSE port 8000 to allow communication to/from server 12 | EXPOSE 8009 13 | 14 | # CMD specifcies the command to execute to start the server running. 15 | WORKDIR /usr/src/app 16 | COPY . . 17 | 18 | # INIT SERVER 19 | CMD ["python", "-m", "JavHelper.run"] 20 | # done! 21 | -------------------------------------------------------------------------------- /JAV_HELP.md: -------------------------------------------------------------------------------- 1 | # JAV Downloader JAV 下载器 上手教程 2 | **千里之行, 始于架构** 3 | 4 | JAV下载器集成Jav网站-115-Aria2下载于一个界面, 需要正确的115_cookies.json和Aria2 5 | 6 | (目前可以设置HTTP代理但是代码并没有测试过) 7 | 8 | ## 设置115 cookies和aria2 9 | ### 配置aria2服务器 10 | 先切换至"配置表单"并选取"本地配置"(默认) 11 | 12 | 13 | 14 | 为了能自动添加下载链接至aria2服务器, 填入下列3个域并点击"提交"按钮保存设置: 15 | * aria2网址: 请参照例子填入, 必须含有"http://"前缀, 不需要结尾的"/" 16 | * aria2端口: aria2服务器的端口号 17 | * aria2 authentication token: 目前只支持token的验证方法(不支持用户名密码验证), 在此域填入aria2服务器设置的验证token 18 | 19 | 20 | 21 | ### 设置115 cookies 22 | 23 | 为了能自动添加磁链至115离线下载, 进行以下步骤: 24 | * 获取115 cookies: 25 | * 登录115网页版 26 | * 使用EditThisCookie Chrome插件复制cookies至系统粘贴板 27 | 28 | 29 | * 填入115 cookies: 30 | * 进入工具网页, 切换至"配置表单"之下的"更新115 Cookies" 31 | 32 | 33 | * 粘贴(Ctrl+V)系统粘贴板内的115 Cookies, 工具将自动保存填入的内容. (自动在目录创建115_cookies.json, 用户也可以手动更改) 34 | 35 | 36 | * 如需更新115 Cookies, 用户亦可以通过相同的操作用"配置表单"更新. 37 | 38 | ## 下载Jav 39 | 40 | ### Jav本地数据库状态列表 41 | JAVOneStop有本地文档储存的数据库, 用以储存Jav相关信息. 对于Jav下载器来说, 最关联的域为"stat", 用于用户识别不同Jav的状态. 数据库假设车牌有唯一性, 一个车牌只能对应一个Jav. 42 | 43 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 |
数据库stat值显示备注
0想要想下载, 在此状态会加载磁链
1已阅不感兴趣, 主要用于过滤
2没想法默认选项, 不会被过滤
3本地Jav已存在于本地, 会被过滤
4下载中已通过工具添加至aria2下载,
会被过滤
81 | 82 | ### 下载Jav教程: 83 | * 选择"JAV下载器" 84 | 85 | 86 | * 下载器默认选择"javbus"作为网站来源; "中文字幕"页面; 并过滤所有非"想要"或者"没想法"的状态 87 | * 如果115或者aria2没有配置, 工具页面将有提示 88 | * 将想要下载的Jav切换至"想要"状态, 工具将加载磁链并显示下载按钮 89 | * 选择想要下载的磁链, 并等待下载添加完成 90 | 91 | 92 | 93 | ### 备注 94 | * 如果磁链下载成功添加至aria2, 对应Jav将自动转换至"下载中"状态 95 | * 主页面默认只显示缩略图(图片尺寸小, 加载迅速和防止被限流); 如需加载全海报, 则可以点击"加载详细海报"按钮, 工具将会加载大图海报 96 | * 用户可以同时下载同一页面多个Jav的磁链, 但不推荐同时下载多个单一Jav里的磁链(第一个磁链添加成功则会切换Jav状态, 用户看不到其他磁链下载结果) 97 | * 用户可以自行切换Jav状态, 切换结果将自动存入数据库 98 | * 工具默认采用"javbus"作为磁链搜索源(最全最快), 用户可以自行切换磁链搜索源, 目前提供"torrrentkitty"和"nyaa"里站. 99 | 100 | 101 | 102 | ## 瀑布流浏览 103 | 主页面支持瀑布流浏览并自动加载多余内容. 104 | 105 | 106 | 107 | 备注: 如果一个页面内容过少可能无法触发自动加载, 请手动增加页数以加载更多内容. 108 | 109 | ## 过滤功能, 来源切换 110 | 111 | ### 过滤器 112 | 本工具默认会开启"想要/没想法"过滤器, 此过滤器将过滤获取的Jav, 并只显示"想要"和"没想法"状态的Jav. 用户可以自行切换成"不过滤"以显示所有Jav. 113 | 114 | ### 来源切换 115 | 本工具目前支持2个来源网站: javbus和javlibrary. 用户可以自行选择切换不同的来源网站. 每个来源网站有不同的来源页面, 具体来源页面参照下表: 116 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 |
来源网站标签来源页面
javbus中文字幕https://www.javbus.com/genre/sub
javbus新话题https://www.javbus.com/
javlibrary最想要http://www.javlibrary.com/cn/vl_mostwanted.php
javlibrary高评价http://www.javlibrary.com/cn/vl_bestrated.php
javlibrary新话题http://www.javlibrary.com/cn/vl_update.php
javlibrary
(未来将更改)		还未下载数据来源于数据库中状态为"想要"的Jav
160 | 161 | 162 | 163 | ## 手动搜索 164 | 本工具亦带有搜索来源网站功能, 可以通过改变"搜索类别"下拉菜单自由切换: 165 | * 番号 166 | * 将想搜索的番号填入"搜索字符"栏并点击"提交"按钮. 167 | * 由于结果将写入数据库, 请使用标准番号, 如IPZ-773 168 | * 字母须大写; 必须带"-"; 169 | * 女优 170 | * 这里的"搜索字符"需要填入来源网站的女优id 171 | * javbus: "三上悠亜" => "okq" (不含引号) 172 | * https://www.javbus.com/star/okq 女优id为"star"之后的字母 173 | * javlibrary: "三上悠亜" => "ayera" 174 | * http://www.javlibrary.com/cn/vl_star.php?s=ayera 女优id为"s="之后的字母 175 | * 分类 176 | * 类似于女优搜索, 这里需要填入的"搜索字符"为分类的id 177 | * javbus: "温泉" => "6j" 178 | * https://www.javbus.com/genre/6j 179 | * javlibrary: "韩国" => "a4hq" 180 | * http://www.javlibrary.com/cn/vl_genre.php?g=a4hq 181 | 182 | 手动搜索结果亦支持瀑布流浏览. 183 | 184 | 185 | 186 | ## 手动验证115 187 | 188 | 当用户大量添加磁链至115离线工具后, 会触发115验证工具. 用户可以选择前往115离线工具页面, 手动添加磁链并验证. 或者, 亦可使用本工具自带的115验证工具进行快速验证. 189 | 190 | 由于iframe无法传递cookies, 要求打开本工具的浏览器必须同时登录进115离线工具. 以下情况无法使用自带的115验证工具: Chrome登录进115离线网页, 但是使用Firefox打开本工具. 191 | 192 | ### 115验证工具使用教程: 193 | 194 | 195 | 196 | * 进入"配置表单", "更新115 Cookies"工具, 并点击"115验证码工具"按钮. 197 | * 在弹出的窗口中通过115验证. 198 | * 验证成功目前不会有任何提示, 用户等待几秒之后就可以退出并继续添加磁链 199 | * 验证失败则弹出窗口内会提示用户重新验证 200 | * 如果弹出窗口显示不了正确的文字, 则用户当前使用的浏览器没有登录115, 请在当前使用的浏览器登入115离线(更新Cookies)并重新尝试 201 | 202 | 203 | 电报反馈: [link](https://t.me/joinchat/PBVbLRfEaXOVFifI2nz3Kg) 204 | -------------------------------------------------------------------------------- /JavHelper/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddd354/JAVOneStop/ba41c2561e44b3782e92c8c3004c9e1ca3097b0a/JavHelper/__init__.py -------------------------------------------------------------------------------- /JavHelper/app.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | import os 3 | from flask import Flask, render_template, jsonify, send_from_directory 4 | from werkzeug.exceptions import HTTPException 5 | from traceback import format_exc, print_exc 6 | 7 | from JavHelper.core.ini_file import recreate_ini, DEFAULT_INI, verify_ini_file, return_default_config_string 8 | # init setting file 9 | if not os.path.isfile(DEFAULT_INI): 10 | print('ini file {} doesn\'t exists, recreate one and apply default settings'.format(DEFAULT_INI)) 11 | recreate_ini(DEFAULT_INI) 12 | # verify all fields exist 13 | verify_ini_file() 14 | 15 | from JavHelper.cache import cache 16 | from JavHelper.views.emby_actress import emby_actress 17 | from JavHelper.views.parse_jav import parse_jav 18 | from JavHelper.views.jav_browser import jav_browser 19 | from JavHelper.views.local_manager import local_manager 20 | from JavHelper.views.scan_directory import directory_scan 21 | from JavHelper.utils import resource_path 22 | 23 | if return_default_config_string('db_type') == 'sqlite': 24 | from JavHelper.model.jav_manager import SqliteJavManagerDB as JavManagerDB 25 | else: 26 | from JavHelper.model.jav_manager import BlitzJavManagerDB as JavManagerDB 27 | 28 | 29 | def create_app(): 30 | # initialize local db and index 31 | _db = JavManagerDB() 32 | _db.create_indexes() 33 | 34 | # create and configure the app 35 | app = Flask(__name__, template_folder='templates') 36 | cache.init_app(app) 37 | 38 | app.register_blueprint(emby_actress) 39 | app.register_blueprint(parse_jav) 40 | app.register_blueprint(jav_browser) 41 | app.register_blueprint(directory_scan) 42 | app.register_blueprint(local_manager) 43 | 44 | app.config['JSON_AS_ASCII'] = False 45 | 46 | # a simple page that says hello 47 | @app.route('/p/') 48 | @app.route('/') 49 | def hello(*args, **kwargs): 50 | return render_template('home.html') 51 | 52 | @app.route('/demo/') 53 | def serve_demo_images(path): 54 | return send_from_directory(resource_path('demo'), path) 55 | 56 | @app.errorhandler(Exception) 57 | def handle_exception(e): 58 | # pass through HTTP errors 59 | if isinstance(e, HTTPException): 60 | return e 61 | 62 | print_exc() 63 | # now you're handling non-HTTP exceptions only 64 | return jsonify({'error': format_exc()}), 500 65 | 66 | return app 67 | -------------------------------------------------------------------------------- /JavHelper/cache.py: -------------------------------------------------------------------------------- 1 | from flask_caching import Cache 2 | 3 | cache = Cache(config={'CACHE_TYPE': 'simple', 'CACHE_DEFAULT_TIMEOUT': 300}) -------------------------------------------------------------------------------- /JavHelper/core/__init__.py: -------------------------------------------------------------------------------- 1 | class JAVNotFoundException(Exception): 2 | pass 3 | 4 | 5 | class IniNotFoundException(Exception): 6 | pass 7 | 8 | 9 | class ActorNotFoundException(Exception): 10 | pass 11 | -------------------------------------------------------------------------------- /JavHelper/core/aria2_handler.py: -------------------------------------------------------------------------------- 1 | import aria2p 2 | 3 | from JavHelper.core.ini_file import return_default_config_string 4 | 5 | def get_aria2(): 6 | return aria2p.API( 7 | aria2p.Client( 8 | host=return_default_config_string('aria_address'), 9 | port=int(return_default_config_string('aria_port') or 0), 10 | secret=return_default_config_string('aria_token') 11 | ) 12 | ) 13 | 14 | def verify_aria2_configs_exist(): 15 | if not return_default_config_string('aria_address') or \ 16 | not int(return_default_config_string('aria_port') or 0) or \ 17 | not return_default_config_string('aria_token'): 18 | return False 19 | else: 20 | return True -------------------------------------------------------------------------------- /JavHelper/core/arzon.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | from copy import deepcopy 4 | from lxml import etree 5 | 6 | from JavHelper.core.jav_scraper import JavScraper 7 | from JavHelper.core import JAVNotFoundException 8 | from JavHelper.core.requester_proxy import return_get_res, return_html_text 9 | 10 | 11 | class ArzonScraper(JavScraper): 12 | def __init__(self, *args, **kwargs): 13 | super(ArzonScraper, self).__init__(*args, **kwargs) 14 | self.source = 'arzon' 15 | self.xpath_dict = { 16 | 'search_field': { 17 | 'title': '//h1/text()', 18 | 'studio': '//tr[td="AVメーカー："]/td[2]/a/text()', 19 | 'premiered': '//tr[td="発売日："]/td[2]/text()', 20 | #'year': processed from release date 21 | 'length': '//tr[td="収録時間："]/td[2]/text()', 22 | 'director': '//tr[td="監督："]/td[2]/a/text()', 23 | 'image': '//a[@data-lightbox="jacket1"]/@href', 24 | #'score':no good source 25 | }, 26 | 'search_list_field': { 27 | 'plot': '//table[@class="item_detail"]//div[@class="item_text"]/text()', 28 | 'all_actress': '//tr[td="AV女優："]/td[2]/a/text()' 29 | }, 30 | } 31 | 32 | def get_site_sessions(self): 33 | session = requests.Session() 34 | session.get('https://www.arzon.jp/index.php?action=adult_customer_agecheck&agecheck=1' 35 | '&redirect=https%3A%2F%2Fwww.arzon.jp%2F', timeout=10) 36 | return session 37 | 38 | def postprocess(self): 39 | if self.jav_obj.get('image'): 40 | # remove invalid image link 41 | if 'noimagepl' in self.jav_obj['image']: 42 | self.jav_obj.pop('image') 43 | 44 | if self.jav_obj.get('plot') and isinstance(self.jav_obj['plot'], list): 45 | _temp = deepcopy(self.jav_obj['plot']) 46 | _temp = ''.join(_temp) 47 | _temp = _temp.replace('\r\n', '') 48 | _temp = _temp.strip(' ') 49 | self.jav_obj['plot'] = deepcopy(_temp) 50 | 51 | if self.jav_obj.get('length') and isinstance(self.jav_obj['length'], str): 52 | _temp = deepcopy(self.jav_obj['length']) 53 | _temp = _temp.replace('\r\n', '') 54 | _temp = _temp.strip(' ') 55 | _temp = str(re.search(r'\d+', _temp).group(0)) 56 | self.jav_obj['length'] = deepcopy(_temp) 57 | 58 | if self.jav_obj.get('premiered') and isinstance(self.jav_obj['premiered'], str): 59 | re_pattern = r'^(\d{4})(\/\d{2}\/\d{2}).*$' 60 | _temp = deepcopy(self.jav_obj['premiered']) 61 | _temp = _temp.replace('\r\n', '') 62 | _temp = _temp.strip(' ') 63 | matched = re.match(re_pattern, _temp) 64 | if matched and len(matched.groups()) == 2: 65 | #import ipdb;ipdb.set_trace() 66 | self.jav_obj['year'] = str(matched.groups()[0]) 67 | self.jav_obj['premiered'] = ''.join(matched.groups()[:1]) 68 | else: 69 | # keep cleaned release date only 70 | self.jav_obj['premiered'] = deepcopy(_temp) 71 | 72 | if self.jav_obj.get('title'): 73 | self.jav_obj['title'] = '{} {}'.format(self.jav_obj['car'], self.jav_obj['title']) 74 | 75 | 76 | def get_single_jav_page(self): 77 | arzon_cookies = self.get_site_sessions().cookies.get_dict() 78 | arz_search_url = 'https://www.arzon.jp/itemlist.html?t=&m=all&s=&q=' + self.car 79 | search_html = return_html_text(arz_search_url, cookies=arzon_cookies) 80 | 81 | AVs = re.findall(r'

dict: 82 | xpath_dict = { 83 | 'title': '//div[@class="thumbnail"]/a/text()', 84 | 'javid': '//div[@class="thumbnail"]/a/@href', # need to extract from link 85 | 'img': '//div[@class="thumbnail"]/a/img/@src', 86 | 'car': '//div[@class="thumbnail"]/a/text()' # need to extract from title 87 | } 88 | xpath_max_page = '//ul[@class="pager"]/li[@class="next"]/a/text()' 89 | max_page = page_num # default value 90 | 91 | # force to get url from ini file each time 92 | #javbus_url = return_config_string(['其他设置', 'javbus网址']) 93 | jav_url = 'https://www.jav321.com/' 94 | set_url = jav_url + page_template.format(page_num=page_num, url_parameter=url_parameter) 95 | print(f'accessing {set_url}') 96 | 97 | res = return_post_res(set_url).content 98 | root = etree.HTML(res) 99 | 100 | jav_objs_raw = defaultlist(dict) 101 | for k, v in xpath_dict.items(): 102 | _values = root.xpath(v) 103 | for _i, _value in enumerate(_values): 104 | # need to extract car from title, reusing file_scanner function 105 | if k == 'car': 106 | # need to separate text with car first 107 | _preprocess = _value.split(' ')[-1] 108 | 109 | # try to extract proper car 110 | try: 111 | name_group = re.search(DEFAULT_FILENAME_PATTERN, _preprocess) 112 | name_digits = name_group.group('digit') 113 | 114 | # only keep 0 under 3 digits 115 | # keep 045, 0830 > 830, 1130, 0002 > 002, 005 116 | if name_digits.isdigit(): 117 | name_digits = str(int(name_digits)) 118 | while len(name_digits) < 3: 119 | name_digits = '0' + name_digits 120 | _value = name_group.group('pre') + '-' + name_digits 121 | except AttributeError as e: 122 | print(f'cannot extract standard car format from {_preprocess} due to {e}') 123 | _value = _preprocess 124 | jav_objs_raw[_i].update({k: _value}) 125 | 126 | try: 127 | _new_max = root.xpath(xpath_max_page) 128 | if len(_new_max) > 0: 129 | max_page = int(max_page) + 1 130 | except: 131 | pass 132 | 133 | # max page override 134 | #if 'type' in page_template: 135 | # max_page = max_page * 100 136 | 137 | return jav_objs_raw, max_page 138 | 139 | 140 | def jav321_search(set_type: str, search_string: str, page_num=1): 141 | def search_by_car(car: str, **kwargs): 142 | car = car.upper() 143 | jav_obj = Jav321Scraper({'car': car}).scrape_jav() 144 | db_conn = JavManagerDB() 145 | 146 | if db_conn.pk_exist(str(jav_obj.get('car'))): 147 | jav_obj.update( 148 | dict( 149 | db_conn.get_by_pk(str(jav_obj.get('car'))) 150 | ) 151 | ) 152 | else: 153 | jav_obj['stat'] = 2 154 | db_conn.upcreate_jav(jav_obj) 155 | 156 | # use the full image (image key) instead of img (much smaller) 157 | jav_obj['img'] = jav_obj.get('image', '') 158 | 159 | return [jav_obj], 1 160 | 161 | def search_for_actress(javlib_actress_code: str, page_num=1): 162 | search_url = 'star/{url_parameter}/{page_num}' 163 | db_conn = JavManagerDB() 164 | 165 | # get actress first page 166 | jav_objs, max_page = jav321_set_page(search_url, 167 | page_num=page_num, 168 | url_parameter=javlib_actress_code 169 | ) 170 | 171 | for jav_obj in jav_objs: 172 | if db_conn.pk_exist(str(jav_obj.get('car'))): 173 | jav_obj.update( 174 | dict( 175 | db_conn.get_by_pk(str(jav_obj.get('car'))) 176 | ) 177 | ) 178 | else: 179 | jav_obj['stat'] = 2 180 | db_conn.upcreate_jav(jav_obj) 181 | 182 | return jav_objs, max_page 183 | 184 | search_map = { 185 | '番号': {'function': search_by_car, 'params': {'car': search_string}}, 186 | '女优': {'function': search_for_actress, 'params': { 187 | 'javlib_actress_code': search_string, 'page_num': page_num}}, 188 | '分类': {'function': jav321_set_page, 'params': 189 | {'page_template': 'genre/{url_parameter}/{page_num}', 190 | 'page_num': page_num, 'url_parameter': search_string} 191 | }, 192 | '系列': {'function': jav321_set_page, 'params': 193 | {'page_template': 'series/{url_parameter}/{page_num}', 194 | 'page_num': page_num, 'url_parameter': search_string} 195 | } 196 | } 197 | 198 | # verify set type 199 | if set_type not in search_map: 200 | raise Exception(BackendTranslation()['no_support_set_search'].format(set_type)) 201 | 202 | jav_objs, max_page = search_map[set_type]['function'](**search_map[set_type]['params']) 203 | return jav_objs, max_page -------------------------------------------------------------------------------- /JavHelper/core/jav777.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | from lxml import etree 3 | import re 4 | from copy import deepcopy 5 | 6 | from JavHelper.core.jav_scraper import JavScraper 7 | from JavHelper.core import JAVNotFoundException 8 | from JavHelper.core.requester_proxy import return_html_text, return_post_res, return_get_res 9 | from JavHelper.core.utils import re_parse_html, re_parse_html_list_field, defaultlist 10 | from JavHelper.core.ini_file import return_config_string 11 | from JavHelper.core.utils import parsed_size_to_int 12 | from JavHelper.core.file_scanner import DEFAULT_FILENAME_PATTERN 13 | 14 | 15 | JAV777_URL = 'https://www.jav777.xyz/' # hard coded for now 16 | 17 | class Jav777Scraper(JavScraper): 18 | def __init__(self, *args, **kwargs): 19 | super(Jav777Scraper, self).__init__(*args, **kwargs) 20 | self.source = 'jav777' 21 | self.xpath_dict = { 22 | 'search_field': { 23 | 'title': '//a[@class="bigImage"]/img/@title', 24 | 'studio': '//p[span="製作商:"]/a/text()', 25 | 'premiered': '//p[span="發行日期:"]/text()', 26 | #'year': processed from release date 27 | 'length': '//p[span="長度:"]/text()', 28 | 'director': '//p[span="導演:"]/a/text()', 29 | 'image': '//a[@class="bigImage"]/img/@src', 30 | #'score': no good source 31 | }, 32 | 'search_list_field': { 33 | #'plot': no good source, 34 | 'all_actress': '//span[@class="genre" and @onmouseover]/a/text()', 35 | 'genres': '//span[@class="genre"]/a[contains(@href, "genre")]/text()' 36 | }, 37 | } 38 | 39 | self.jav_url = JAV777_URL 40 | 41 | def postprocess(self): 42 | if self.jav_obj.get('premiered'): 43 | self.jav_obj['premiered'] = self.jav_obj['premiered'].lstrip(' ') 44 | self.jav_obj['year'] = self.jav_obj['premiered'][:4] 45 | if self.jav_obj.get('image'): 46 | # get rid of https to have consistent format with other sources 47 | self.jav_obj['image'] = self.jav_obj['image'].lstrip('https:').lstrip('http:') 48 | if self.jav_obj.get('length'): 49 | self.jav_obj['length'] = self.jav_obj['length'].lstrip(' ')[:-2] 50 | if self.jav_obj.get('title'): 51 | self.jav_obj['title'] = '{} {}'.format(self.jav_obj['car'], self.jav_obj['title']) 52 | 53 | def get_single_jav_page(self): 54 | """ 55 | This search method is currently NOT DETERMINISTIC! 56 | Example: SW-098 -> has 3 outputs 57 | """ 58 | 59 | # perform search first 60 | # https://www.javbus.com/search/OFJE-235&type=&parent=ce 61 | search_url = self.jav_url + '?s={}'.format(self.car) 62 | print(f'accessing {search_url}') 63 | 64 | jav_search_content = return_get_res(search_url).content 65 | search_root = etree.HTML(jav_search_content) 66 | 67 | search_results = search_root.xpath('//h2[@class="post-title"]/a/@href') 68 | 69 | if not search_results: 70 | raise JAVNotFoundException('{} cannot be found in {}'.format(self.car, self.source)) 71 | 72 | self.total_index = len(search_results) 73 | result_first_url = search_results[self.pick_index] 74 | 75 | return return_get_res(result_first_url).content, self.total_index 76 | 77 | 78 | def jav777_download_search(car: str): 79 | jav_site = Jav777Scraper({'car': car}) 80 | jav_page, _ = jav_site.get_single_jav_page() 81 | 82 | search_root = etree.HTML(jav_page) 83 | search_results = search_root.xpath('//a[@class="exopopclass"]/@href') 84 | title = search_root.xpath('//h1[@class="post-title"]/a/@title') 85 | 86 | if search_results: 87 | return [{'web_link': search_results[0], 'title': title[0], 'size': '--'}] 88 | else: 89 | return [] 90 | 91 | 92 | def jav777_set_page(page_template: str, page_num=1, url_parameter=None, config=None) -> dict: 93 | xpath_dict = { 94 | 'title': '//h2[@class="post-title"]/a/@title', 95 | 'javid': '//div[@class="post-container"]/div/@id', 96 | 'img': '//div[@class="featured-media"]/a/img/@src', 97 | 'car': '//h2[@class="post-title"]/a/@title' 98 | } 99 | xpath_max_page = '//center/a[position() = (last()-1)]/text()' 100 | 101 | # force to get url from ini file each time 102 | jav777_url = JAV777_URL 103 | set_url = jav777_url + page_template.format(page_num=page_num, url_parameter=url_parameter) 104 | print(f'accessing {set_url}') 105 | 106 | res = return_post_res(set_url).content 107 | root = etree.HTML(res) 108 | 109 | jav_objs_raw = defaultlist(dict) 110 | for k, v in xpath_dict.items(): 111 | _values = root.xpath(v) 112 | for _i, _value in enumerate(_values): 113 | # need to extract car from title, reusing file_scanner function 114 | if k == 'car': 115 | # remove hd prefixes 116 | _value = _value.lstrip('(HD)') 117 | 118 | name_group = re.search(DEFAULT_FILENAME_PATTERN, _value) 119 | name_digits = name_group.group('digit') 120 | 121 | # only keep 0 under 3 digits 122 | # keep 045, 0830 > 830, 1130, 0002 > 002, 005 123 | if name_digits.isdigit(): 124 | name_digits = str(int(name_digits)) 125 | while len(name_digits) < 3: 126 | name_digits = '0' + name_digits 127 | _value = name_group.group('pre') + '-' + name_digits 128 | jav_objs_raw[_i].update({k: _value}) 129 | 130 | try: 131 | max_page = root.xpath(xpath_max_page)[0] 132 | except: 133 | max_page = page_num 134 | if not max_page: 135 | max_page = page_num 136 | 137 | return jav_objs_raw, max_page 138 | -------------------------------------------------------------------------------- /JavHelper/core/jav_scraper.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import re 3 | from lxml import etree 4 | 5 | from JavHelper.core import JAVNotFoundException 6 | from JavHelper.core.requester_proxy import return_html_text 7 | 8 | 9 | class JavScraper: 10 | def __init__(self, jav_obj: dict, pick_index=0): 11 | self.jav_obj = jav_obj 12 | self.car = jav_obj['car'].upper() 13 | self.source = 'main_class' 14 | 15 | self.xpath_dict = { 16 | 'search_field': { 17 | }, 18 | 'search_list_field': { 19 | }, 20 | } 21 | self.pick_index = pick_index 22 | self.total_index = 1 # default to just 1 result 23 | 24 | def get_site_sessions(self): 25 | pass 26 | 27 | def get_single_jav_page(self): 28 | pass 29 | 30 | def postprocess(self): 31 | pass 32 | 33 | def scrape_jav(self): 34 | page_content, total_index = self.get_single_jav_page() 35 | #import ipdb; ipdb.set_trace() 36 | self.jav_obj['pick_index'] = self.pick_index 37 | self.jav_obj['total_index'] = total_index 38 | #import ipdb; ipdb.set_trace() 39 | if not page_content: 40 | print(f'cannot find {self.car} in {self.source}') 41 | return self.jav_obj 42 | 43 | root = etree.HTML(page_content) 44 | # search single field 45 | self.jav_obj.update(self.search_single_xpath(self.jav_obj, self.xpath_dict['search_field'], root)) 46 | # search multi field 47 | self.jav_obj.update(self.search_multifield_xpath(self.jav_obj, self.xpath_dict['search_list_field'], root)) 48 | 49 | # run post process 50 | self.postprocess() 51 | 52 | return self.jav_obj 53 | 54 | @staticmethod 55 | def search_single_xpath(update_obj: dict, search_dict: dict, source_root): 56 | for k, v in search_dict.items(): 57 | _temp_v = source_root.xpath(v) 58 | #if k == 'title': import ipdb; ipdb.set_trace() 59 | if len(_temp_v) > 0: 60 | update_obj[k] = _temp_v[0] 61 | 62 | return update_obj 63 | 64 | @staticmethod 65 | def search_multifield_xpath(update_obj: dict, search_dict: dict, source_root): 66 | for k, v in search_dict.items(): 67 | #if k == 'genres': 68 | # import ipdb; ipdb.set_trace() 69 | update_obj[k] = source_root.xpath(v) 70 | 71 | return update_obj 72 | 73 | -------------------------------------------------------------------------------- /JavHelper/core/javbus.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | from lxml import etree 3 | import re 4 | from copy import deepcopy 5 | 6 | from JavHelper.core.jav_scraper import JavScraper 7 | from JavHelper.core import JAVNotFoundException 8 | from JavHelper.core.requester_proxy import return_html_text, return_post_res, return_get_res 9 | from JavHelper.core.utils import re_parse_html, re_parse_html_list_field, defaultlist 10 | from JavHelper.core.ini_file import return_config_string, return_default_config_string 11 | from JavHelper.core.utils import parsed_size_to_int 12 | from JavHelper.core.backend_translation import BackendTranslation 13 | 14 | if return_default_config_string('db_type') == 'sqlite': 15 | from JavHelper.model.jav_manager import SqliteJavManagerDB as JavManagerDB 16 | else: 17 | from JavHelper.model.jav_manager import BlitzJavManagerDB as JavManagerDB 18 | 19 | 20 | class JavBusScraper(JavScraper): 21 | def __init__(self, *args, **kwargs): 22 | super(JavBusScraper, self).__init__(*args, **kwargs) 23 | self.source = 'javbus' 24 | self.xpath_dict = { 25 | 'search_field': { 26 | 'title': '//a[@class="bigImage"]/img/@title', 27 | 'studio': '//p[span="製作商:"]/a/text()', 28 | 'premiered': '//p[span="發行日期:"]/text()', 29 | #'year': processed from release date 30 | 'length': '//p[span="長度:"]/text()', 31 | 'director': '//p[span="導演:"]/a/text()', 32 | 'image': '//a[@class="bigImage"]/img/@src', 33 | #'score': no good source 34 | }, 35 | 'search_list_field': { 36 | #'plot': no good source, 37 | 'all_actress': '//span[@class="genre" and @onmouseover]/a/text()', 38 | 'genres': '//span[@class="genre"]/a[contains(@href, "genre")]/text()' 39 | }, 40 | } 41 | 42 | self.jav_url = return_config_string(['其他设置', 'javbus网址']) 43 | 44 | def postprocess(self): 45 | if self.jav_obj.get('premiered'): 46 | self.jav_obj['premiered'] = self.jav_obj['premiered'].lstrip(' ') 47 | self.jav_obj['year'] = self.jav_obj['premiered'][:4] 48 | if self.jav_obj.get('image'): 49 | # get rid of https to have consistent format with other sources 50 | self.jav_obj['image'] = self.jav_obj['image'].lstrip('https:').lstrip('http:') 51 | # new local image logic 52 | self.jav_obj['image'] = self.jav_url.lstrip('http').lstrip('s://').rstrip('/') + self.jav_obj['image'] 53 | if self.jav_obj.get('length'): 54 | self.jav_obj['length'] = self.jav_obj['length'].lstrip(' ')[:-2] 55 | if self.jav_obj.get('title'): 56 | self.jav_obj['title'] = '{} {}'.format(self.jav_obj['car'], self.jav_obj['title']) 57 | 58 | def get_single_jav_page(self): 59 | """ 60 | This search method is currently NOT DETERMINISTIC! 61 | Example: SW-098 -> has 3 outputs 62 | """ 63 | 64 | # perform search first 65 | # https://www.javbus.com/search/OFJE-235&type=&parent=ce 66 | search_url = self.jav_url + 'search/{}&type=&parent=ce'.format(self.car) 67 | print(f'accessing {search_url}') 68 | 69 | jav_search_content = return_get_res(search_url).content 70 | search_root = etree.HTML(jav_search_content) 71 | 72 | search_results = search_root.xpath('//a[@class="movie-box"]/@href') 73 | 74 | if not search_results: 75 | # sometimes the access will fail, try directly access by car 76 | direct_url = self.jav_url + self.car 77 | print(f'no search result, try direct accessing {search_url}') 78 | jav_search_content = return_get_res(direct_url).content 79 | search_root = etree.HTML(jav_search_content) 80 | 81 | if search_root.xpath('//a[@class="bigImage"]/img/@title'): 82 | search_results = [direct_url] 83 | 84 | if not search_results: 85 | raise JAVNotFoundException('{} cannot be found in javbus'.format(self.car)) 86 | 87 | self.total_index = len(search_results) 88 | result_first_url = search_results[self.pick_index] 89 | 90 | return return_get_res(result_first_url).content, self.total_index 91 | 92 | 93 | def javbus_magnet_search(car: str): 94 | jav_url = return_config_string(['其他设置', 'javbus网址']) 95 | gid_match = r'.*?var gid = (\d*);.*?' 96 | magnet_xpath = { 97 | 'magnet': '//tr/td[position()=1]/a[1]/@href', 98 | 'title': '//tr/td[position()=1]/a[1]/text()', 99 | 'size': '//tr/td[position()=2]/a[1]/text()' 100 | } 101 | main_url_template = jav_url+'{car}' 102 | magnet_url_template = jav_url+'ajax/uncledatoolsbyajax.php?gid={gid}&uc=0' 103 | 104 | res = return_get_res(main_url_template.format(car=car)).text 105 | gid = re.search(gid_match, res).groups()[0] 106 | 107 | res = return_get_res(magnet_url_template.format(gid=gid), headers={'referer': main_url_template.format(car=car)}).content 108 | root = etree.HTML(res.decode('utf-8')) 109 | 110 | magnets = defaultlist(dict) 111 | for k, v in magnet_xpath.items(): 112 | _values = root.xpath(v) 113 | for _i, _value in enumerate(_values): 114 | magnets[_i].update({k: _value.strip('\t').strip('\r').strip('\n').strip()}) 115 | if k == 'size': 116 | magnets[_i].update({'size_sort': parsed_size_to_int(_value.strip('\t').strip('\r').strip('\n').strip())}) 117 | 118 | return magnets 119 | 120 | 121 | def javbus_set_page(page_template: str, page_num=1, url_parameter=None, config=None) -> dict: 122 | xpath_dict = { 123 | 'title': '//div[@class="photo-frame"]/img[not(contains(@src, "actress"))]/@title', 124 | 'javid': '//div[@class="photo-info"]/span/date[1]/text()', 125 | 'img': '//div[@class="photo-frame"]/img[not(contains(@src, "actress"))]/@src', 126 | 'car': '//div[@class="photo-info"]/span/date[1]/text()' 127 | } 128 | xpath_max_page = '//ul[@class="pagination pagination-lg"]/li/a/text()' 129 | max_page = page_num # default value 130 | 131 | # force to get url from ini file each time 132 | javbus_url = return_config_string(['其他设置', 'javbus网址']) 133 | set_url = javbus_url + page_template.format(page_num=page_num, url_parameter=url_parameter) 134 | print(f'accessing {set_url}') 135 | 136 | res = return_post_res(set_url).content 137 | root = etree.HTML(res) 138 | 139 | jav_objs_raw = defaultlist(dict) 140 | for k, v in xpath_dict.items(): 141 | _values = root.xpath(v) 142 | 143 | # new logic for local images 144 | javbus_img_url = javbus_url.lstrip('http').lstrip('s://').rstrip('/') 145 | if k == 'img': 146 | _values = [javbus_img_url +_ind for _ind in _values if 'dmm.co.jp' not in _ind] 147 | 148 | for _i, _value in enumerate(_values): 149 | jav_objs_raw[_i].update({k: _value}) 150 | 151 | try: 152 | _new_max = root.xpath(xpath_max_page)[-2] 153 | if int(_new_max) > int(page_num): 154 | max_page = _new_max 155 | except: 156 | pass 157 | 158 | return jav_objs_raw, max_page 159 | 160 | 161 | def javbus_search(set_type: str, search_string: str, page_num=1): 162 | 163 | def search_by_car(car: str, **kwargs): 164 | car = car.upper() 165 | jav_obj = JavBusScraper({'car': car}).scrape_jav() 166 | db_conn = JavManagerDB() 167 | 168 | if db_conn.pk_exist(str(jav_obj.get('car'))): 169 | jav_obj.update( 170 | dict( 171 | db_conn.get_by_pk(str(jav_obj.get('car'))) 172 | ) 173 | ) 174 | else: 175 | jav_obj['stat'] = 2 176 | db_conn.upcreate_jav(jav_obj) 177 | 178 | # use the full image (image key) instead of img (much smaller) 179 | jav_obj['img'] = jav_obj.get('image', '') 180 | 181 | return [jav_obj], 1 182 | 183 | def search_for_actress(javlib_actress_code: str, page_num=1): 184 | search_url = 'star/{url_parameter}/{page_num}' 185 | db_conn = JavManagerDB() 186 | 187 | # get actress first page 188 | jav_objs, max_page = javbus_set_page(search_url, 189 | page_num=page_num, 190 | url_parameter=javlib_actress_code 191 | ) 192 | 193 | for jav_obj in jav_objs: 194 | if db_conn.pk_exist(str(jav_obj.get('car'))): 195 | jav_obj.update( 196 | dict( 197 | db_conn.get_by_pk(str(jav_obj.get('car'))) 198 | ) 199 | ) 200 | else: 201 | jav_obj['stat'] = 2 202 | db_conn.upcreate_jav(jav_obj) 203 | 204 | return jav_objs, max_page 205 | 206 | search_map = { 207 | '番号': {'function': search_by_car, 'params': {'car': search_string}}, 208 | '女优': {'function': search_for_actress, 'params': { 209 | 'javlib_actress_code': search_string, 'page_num': page_num}}, 210 | '分类': {'function': javbus_set_page, 'params': 211 | {'page_template': 'genre/{url_parameter}/{page_num}', 212 | 'page_num': page_num, 'url_parameter': search_string}}, 213 | } 214 | 215 | # verify set type 216 | if set_type not in search_map: 217 | raise Exception(BackendTranslation()['no_support_set_search'].format(set_type)) 218 | 219 | jav_objs, max_page = search_map[set_type]['function'](**search_map[set_type]['params']) 220 | return jav_objs, max_page 221 | -------------------------------------------------------------------------------- /JavHelper/core/javlibrary.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | from lxml import etree 3 | import re 4 | from copy import deepcopy 5 | import json 6 | 7 | from JavHelper.core.jav_scraper import JavScraper 8 | from JavHelper.core import JAVNotFoundException 9 | from JavHelper.core.requester_proxy import return_html_text, return_post_res, return_get_res 10 | from JavHelper.core.utils import re_parse_html, re_parse_html_list_field, defaultlist 11 | from JavHelper.core.ini_file import return_config_string, return_default_config_string 12 | from JavHelper.core.backend_translation import BackendTranslation 13 | 14 | if return_default_config_string('db_type') == 'sqlite': 15 | from JavHelper.model.jav_manager import SqliteJavManagerDB as JavManagerDB 16 | else: 17 | from JavHelper.model.jav_manager import BlitzJavManagerDB as JavManagerDB 18 | 19 | 20 | LOCAL_CF_COOKIES = 'javlib_cf_cookies.json' 21 | 22 | class JavLibraryScraper(JavScraper): 23 | def __init__(self, *args, **kwargs): 24 | super(JavLibraryScraper, self).__init__(*args, **kwargs) 25 | self.source = 'javlibrary' 26 | self.xpath_dict = { 27 | 'search_field': { 28 | 'title': '//title/text()', 29 | 'studio': '//tr[td="制作商:"]/td[2]/span/a/text()', 30 | 'premiered': '//tr[td="发行日期:"]/td[2]/text()', 31 | #'year': processed from release date 32 | 'length': '//tr[td="长度:"]/td[2]/span/text()', 33 | 'director': '//tr[td="导演:"]/td[2]/text()', 34 | 'image': '//img[@id="video_jacket_img"]/@src', 35 | 'score': '//span[@class="score"]/text()' 36 | }, 37 | 'search_list_field': { 38 | #'plot': no good source, 39 | 'all_actress': '//span[@class="star"]/a/text()', 40 | 'genres': '//span[@class="genre"]/a/text()' 41 | }, 42 | } 43 | 44 | self.jav_url = return_config_string(['其他设置', 'javlibrary网址']) 45 | 46 | def postprocess(self): 47 | if self.jav_obj.get('image'): 48 | # remove invalid image link 49 | if 'noimagepl' in self.jav_obj['image']: 50 | self.jav_obj.pop('image') 51 | 52 | if ' - JAVLibrary' in self.jav_obj['title']: 53 | self.jav_obj['title'] = self.jav_obj['title'].replace(' - JAVLibrary', '') 54 | 55 | if self.jav_obj.get('premiered') and isinstance(self.jav_obj['premiered'], str): 56 | self.jav_obj['year'] = self.jav_obj['premiered'][0:4] 57 | 58 | def get_single_jav_page(self): 59 | """ 60 | This search method is currently NOT DETERMINISTIC! 61 | Example: SW-098 -> has 3 outputs 62 | """ 63 | 64 | # perform search first 65 | lib_search_url = self.jav_url + 'vl_searchbyid.php?keyword=' + self.car 66 | #print(f'accessing {lib_search_url}') 67 | jav_html = return_html_text(lib_search_url, behind_cloudflare=True) 68 | #print('page return ok') 69 | 70 | # 搜索结果的网页，大部分情况就是这个影片的网页，也有可能是多个结果的网页 71 | # 尝试找标题，第一种情况：找得到，就是这个影片的网页 72 | if self.car.upper().startswith('T28'): 73 | # special filter for T28 74 | title_re = re.search(r'((T28-|T-28)\d{1,5}.+?) - JAVLibrary<\/title>', jav_html) 75 | elif self.car.upper().startswith('R18'): 76 | # special filter for T28 77 | title_re = re.search(r'<title>((R18-|R-18)\d{1,5}.+?) - JAVLibrary<\/title>', jav_html) 78 | else: 79 | title_re = re.search(r'<title>([a-zA-Z]{1,6}-\d{1,5}.+?) - JAVLibrary', jav_html) # 匹配处理“标题” 80 | 81 | # 搜索结果就是AV的页面 82 | if title_re: 83 | return return_get_res(lib_search_url, behind_cloudflare=True).content, 1 84 | # 第二种情况：搜索结果可能是两个以上，所以这种匹配找不到标题，None！ 85 | else: # 继续找标题，但匹配形式不同，这是找“可能是多个结果的网页”上的第一个标题 86 | #import ipdb; ipdb.set_trace() 87 | search_results = re.findall(r'v=jav(.+?)" title=".+?-\d+?[a-z]? ', jav_html) 88 | # 搜索有几个结果，用第一个AV的网页，打开它 89 | if search_results: 90 | self.total_index = len(search_results) 91 | result_first_url = self.jav_url + '?v=jav' + search_results[self.pick_index] 92 | return return_get_res(result_first_url, behind_cloudflare=True).content, self.total_index 93 | # 第三种情况：搜索不到这部影片，搜索结果页面什么都没有 94 | else: 95 | raise JAVNotFoundException('{} cannot be found in javlib'.format(self.car)) 96 | 97 | @staticmethod 98 | def load_local_cookies(return_all=False): 99 | raw_cookies = json.load(open(LOCAL_CF_COOKIES, 'r')) 100 | if return_all: 101 | return raw_cookies 102 | else: 103 | return {x['name']: x['value'] for x in raw_cookies} 104 | 105 | @staticmethod 106 | def update_local_cookies(update_dict: dict or list): 107 | with open(LOCAL_CF_COOKIES, 'w') as oof_f: 108 | oof_f.write(json.dumps(update_dict)) 109 | return f'115 cookies updated to local file {LOCAL_CF_COOKIES}' 110 | 111 | def find_max_page(search_str: str): 112 | search_pattern = r'.*?page=(\d*)' 113 | try: 114 | search_result = re.match(search_pattern, search_str).groups()[0] 115 | return str(search_result) 116 | except Exception as e: 117 | print(e) 118 | return None 119 | 120 | def javlib_set_page(page_template: str, page_num=1, url_parameter=None, config=None) -> dict: 121 | xpath_dict = { 122 | 'title': '//*[@class="video"]/a/@title', 123 | 'javid': '//*[@class="video"]/@id', 124 | 'img': '//*[@class="video"]/a/img/@src', 125 | 'car': '//*/div[@class="video"]/a/div[@class="id"]/text()' 126 | } 127 | xpath_max_page = '//*/div[@class="page_selector"]/a[@class="page last"]/@href' 128 | 129 | # force to get url from ini file each time 130 | javlib_url = return_config_string(['其他设置', 'javlibrary网址']) 131 | 132 | lib_url = javlib_url + page_template.format(page_num=page_num, url_parameter=url_parameter) 133 | print(f'accessing {lib_url}') 134 | 135 | res = return_post_res(lib_url, behind_cloudflare=True).content 136 | root = etree.HTML(res) 137 | 138 | jav_objs_raw = defaultlist(dict) 139 | for k, v in xpath_dict.items(): 140 | _values = root.xpath(v) 141 | for _i, _value in enumerate(_values): 142 | jav_objs_raw[_i].update({k: _value}) 143 | 144 | try: 145 | max_page = find_max_page(root.xpath(xpath_max_page)[0]) 146 | except IndexError: 147 | max_page = page_num 148 | 149 | return jav_objs_raw, max_page 150 | 151 | 152 | def javlib_search(set_type: str, search_string: str, page_num=1): 153 | def search_by_car(car: str, **kwargs): 154 | car = car.upper() 155 | jav_obj = JavLibraryScraper({'car': car}).scrape_jav() 156 | db_conn = JavManagerDB() 157 | 158 | if db_conn.pk_exist(str(jav_obj.get('car'))): 159 | jav_obj.update( 160 | dict( 161 | db_conn.get_by_pk(str(jav_obj.get('car'))) 162 | ) 163 | ) 164 | else: 165 | jav_obj['stat'] = 2 166 | db_conn.upcreate_jav(jav_obj) 167 | 168 | # use the full image (image key) instead of img (much smaller) 169 | jav_obj['img'] = jav_obj.get('image', '') 170 | 171 | return [jav_obj], 1 172 | 173 | def search_for_actress(javlib_actress_code: str, page_num=1): 174 | """ 175 | This only support javlibrary actress code 176 | """ 177 | search_url = 'vl_star.php?&mode=&s={url_parameter}&page={page_num}' 178 | db_conn = JavManagerDB() 179 | 180 | # get actress first page 181 | jav_objs, max_page = javlib_set_page(search_url, 182 | page_num=page_num, 183 | url_parameter=javlib_actress_code 184 | ) 185 | 186 | for jav_obj in jav_objs: 187 | if db_conn.pk_exist(str(jav_obj.get('car'))): 188 | jav_obj.update( 189 | dict( 190 | db_conn.get_by_pk(str(jav_obj.get('car'))) 191 | ) 192 | ) 193 | else: 194 | jav_obj['stat'] = 2 195 | db_conn.upcreate_jav(jav_obj) 196 | 197 | return jav_objs, max_page 198 | 199 | search_map = { 200 | '番号': {'function': search_by_car, 'params': {'car': search_string}}, 201 | '女优': {'function': search_for_actress, 'params': { 202 | 'javlib_actress_code': search_string, 'page_num': page_num}}, 203 | '分类': {'function': javlib_set_page, 'params': 204 | {'page_template': 'vl_genre.php?&mode=&g={url_parameter}&page={page_num}', 205 | 'page_num': page_num, 'url_parameter': search_string}}, 206 | } 207 | 208 | # verify set type 209 | if set_type not in search_map: 210 | raise Exception(BackendTranslation()['no_support_set_search'].format(set_type)) 211 | 212 | jav_objs, max_page = search_map[set_type]['function'](**search_map[set_type]['params']) 213 | return jav_objs, max_page -------------------------------------------------------------------------------- /JavHelper/core/local_db.py: -------------------------------------------------------------------------------- 1 | from urllib.parse import parse_qs 2 | from blitzdb.document import DoesNotExist 3 | 4 | from JavHelper.core.ini_file import return_default_config_string, return_config_string 5 | from JavHelper.views.parse_jav import parse_single_jav 6 | 7 | if return_default_config_string('db_type') == 'sqlite': 8 | from JavHelper.model.jav_manager import SqliteJavManagerDB as JavManagerDB 9 | else: 10 | from JavHelper.model.jav_manager import BlitzJavManagerDB as JavManagerDB 11 | 12 | 13 | def local_set_page(page_template: str, page_num=1, url_parameter=None, config=None): 14 | """ 15 | local return func 16 | currently only support stat search 17 | """ 18 | stat_type = parse_qs(page_template) 19 | _stat = str(stat_type.get('stat', [0])[0]) 20 | db = JavManagerDB() 21 | 22 | print(f'searching jav with stat {_stat}') 23 | # search on both stat string and int 24 | s_result, max_page = db.query_on_filter({'stat': int(_stat)}, page=int(page_num)) 25 | for jav_obj in s_result: 26 | # get rid of invalid image url from javdb 27 | if 'jdbimgs' in jav_obj.get('image', ''): 28 | jav_obj.pop('image') 29 | elif 'jdbimgs' in jav_obj.get('img', ''): 30 | jav_obj.pop('img') 31 | 32 | if not jav_obj.get('image') and not jav_obj.get('img'): 33 | # need to refresh db to get image 34 | jav_obj.update(find_images(jav_obj['car'])) 35 | 36 | return s_result, max_page 37 | 38 | def local_multi_search(search_funcs: list, *args, **kwargs): 39 | for search_func in search_funcs: 40 | try: 41 | _rt, _max = search_func(*args, **kwargs) 42 | if len(_rt) > 0: 43 | return _rt, _max 44 | except Exception as e: 45 | print(f'error {e} occurs, continue to next') 46 | 47 | return [], 0 48 | 49 | #------------------------------------ utils --------------------------------------------- 50 | 51 | def find_images(car: str): 52 | db_conn = JavManagerDB() 53 | try: 54 | jav_obj = dict(db_conn.get_by_pk(car)) 55 | except (DoesNotExist, TypeError) as e: 56 | # typeerror to catch dict(None) 57 | jav_obj = {'car': car} 58 | 59 | sources = return_default_config_string('jav_obj_priority').split(',') 60 | 61 | res = parse_single_jav({'car': car}, sources) 62 | 63 | if res != jav_obj: 64 | jav_obj.update(res) 65 | db_conn.upcreate_jav(jav_obj) 66 | return jav_obj -------------------------------------------------------------------------------- /JavHelper/core/nfo_parser.py: -------------------------------------------------------------------------------- 1 | import xml.etree.ElementTree as Et 2 | import os 3 | 4 | 5 | class EmbyNfo: 6 | single_field_mapping = { 7 | 'plot': './/plot', 8 | 'title': './/title', 9 | 'director': './/director', 10 | 'rating': './/rating', 11 | 'year': './/year', 12 | 'premiered': './/premiered', 13 | 'length': './/runtime', 14 | 'studio': './/studio', 15 | 'car': './/id' 16 | } 17 | list_field_mapping = { 18 | 'genres': './/genre', 19 | #'tags': './/tag', 20 | 'all_actress': './/actor/name' 21 | } 22 | 23 | def __init__(self): 24 | self.jav_obj = {} # parsed jav object 25 | 26 | def parse_emby_nfo(self, file_path): 27 | print(file_path) 28 | # record file_name 29 | self.jav_obj['file_name'] = os.path.split(file_path)[1] 30 | 31 | tree = Et.parse(file_path) 32 | for k, v in self.single_field_mapping.items(): 33 | try: 34 | self.jav_obj[k] = tree.find(v).text 35 | except: 36 | pass 37 | 38 | for k, v in self.list_field_mapping.items(): 39 | self.jav_obj[k] = [ele.text for ele in tree.findall(v)] 40 | 41 | if isinstance(self.jav_obj.get('car', None), str): 42 | self.jav_obj['car'] = self.jav_obj['car'].upper() 43 | -------------------------------------------------------------------------------- /JavHelper/core/requester_proxy.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | import requests 3 | import cloudscraper 4 | from time import sleep 5 | 6 | from JavHelper.core.ini_file import return_config_string 7 | 8 | DEFAULT_HEADERS = { 9 | 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36' 10 | } 11 | 12 | def return_post_res(url, data=None, cookies={}, proxies=None, headers=None, encoding='utf-8', behind_cloudflare=False, **kwargs): 13 | #print(f'accessing {url}') 14 | if not headers: 15 | headers = DEFAULT_HEADERS 16 | 17 | # read settings from ini file 18 | use_proxy = return_config_string(['代理', '是否使用代理？']) 19 | 20 | # prioritize passed in proxies 21 | if use_proxy == '是' and not proxies: 22 | proxies = {'https': return_config_string(['代理', '代理IP及端口']), 'http': return_config_string(['代理', '代理IP及端口'])} 23 | else: 24 | pass 25 | #print('not using proxy for requests') 26 | 27 | if behind_cloudflare: 28 | res = cloudflare_post(url, data, cookies=cookies, proxies=proxies, **kwargs) 29 | else: 30 | res = requests.post(url, data, headers=headers, cookies=cookies, proxies=proxies, **kwargs) 31 | res.encoding = encoding 32 | return res 33 | 34 | def return_get_res(url, cookies={}, proxies=None, headers=None, encoding='utf-8', behind_cloudflare=False, **kwargs): 35 | #print(f'accessing {url}') 36 | if not headers: 37 | headers = DEFAULT_HEADERS 38 | 39 | # read settings from ini file 40 | use_proxy = return_config_string(['代理', '是否使用代理？']) 41 | 42 | # prioritize passed in proxies 43 | if use_proxy == '是' and not proxies: 44 | proxies = {'https': return_config_string(['代理', '代理IP及端口']), 'http': return_config_string(['代理', '代理IP及端口'])} 45 | 46 | if behind_cloudflare: 47 | res = cloudflare_get(url, cookies=cookies, proxies=proxies, **kwargs) 48 | else: 49 | res = requests.get(url, headers=headers, cookies=cookies, proxies=proxies, **kwargs) 50 | res.encoding = encoding 51 | return res 52 | 53 | 54 | def return_html_text(url, cookies={}, proxies=None, encoding='utf-8', behind_cloudflare=False): 55 | #print(f'accessing {url}') 56 | # read settings from ini file 57 | use_proxy = return_config_string(['代理', '是否使用代理？']) 58 | 59 | # prioritize passed in proxies 60 | if use_proxy == '是' and not proxies: 61 | proxies = {'https': return_config_string(['代理', '代理IP及端口']), 'http': return_config_string(['代理', '代理IP及端口'])} 62 | 63 | if behind_cloudflare: 64 | res = cloudflare_get(url, cookies=cookies, proxies=proxies) 65 | else: 66 | res = requests.get(url, cookies=cookies, proxies=proxies) 67 | res.encoding = encoding 68 | return res.text 69 | 70 | def cloudflare_get(url, cookies={}, proxies=None, retry=6, **kwargs): 71 | from JavHelper.core.javlibrary import JavLibraryScraper 72 | while retry > 0: 73 | try: 74 | cookies.update(JavLibraryScraper.load_local_cookies()) # update cloudflare cookies when updating 75 | res = cloudscraper.create_scraper().get(url, cookies=cookies, proxies=proxies) 76 | #print(res.text) 77 | return res 78 | #except cloudscraper.exceptions.CloudflareIUAMError: 79 | except Exception as e: 80 | print(f'cloudflare get failed on {e}, retrying {url}') 81 | retry = retry - 1 82 | sleep(5) 83 | 84 | raise Exception(f'cloudflare get {url} failed') 85 | 86 | def cloudflare_post(url, data=None, cookies={}, proxies=None, retry=6, **kwargs): 87 | from JavHelper.core.javlibrary import JavLibraryScraper 88 | while retry > 0: 89 | try: 90 | cookies.update(JavLibraryScraper.load_local_cookies()) # update cloudflare cookies when updating 91 | res = cloudscraper.create_scraper().post(url, data, cookies=cookies, proxies=proxies) 92 | #print(res.text) 93 | return res 94 | #except cloudscraper.exceptions.CloudflareIUAMError: 95 | except Exception as e: 96 | print(f'cloudflare get failed on {e}, retrying {url}') 97 | retry = retry - 1 98 | sleep(5) 99 | 100 | raise Exception(f'cloudflare get {url} failed') -------------------------------------------------------------------------------- /JavHelper/core/utils.py: -------------------------------------------------------------------------------- 1 | import re 2 | import requests 3 | from PIL import Image 4 | import re 5 | 6 | 7 | class CloudFlareError(Exception): 8 | # error exception when cloudflare readout is failing 9 | pass 10 | 11 | def byte_to_MB(some_input): 12 | if isinstance(some_input, int) or str(some_input).isdigit(): 13 | return int(some_input)/1024/1024 14 | else: 15 | return 0 16 | 17 | def parsed_size_to_int(size_str: str): 18 | if 'GB' in size_str or 'GiB' in size_str or 'gb' in size_str: 19 | multiplier = 1000000 20 | elif 'MB' in size_str or 'MiB' in size_str or 'mb' in size_str: 21 | multiplier = 1000 22 | else: 23 | multiplier = 1 24 | 25 | size_int = float(re.search(r'(\d*\.*\d*)', size_str).group()) * multiplier 26 | 27 | return size_int 28 | 29 | class defaultlist(list): 30 | def __init__(self, fx): 31 | self._fx = fx 32 | 33 | def __setitem__(self, index, value): 34 | while len(self) <= index: 35 | self.append(self._fx()) 36 | list.__setitem__(self, index, value) 37 | 38 | def __getitem__(self, index): 39 | while len(self) <= index: 40 | self.append(self._fx()) 41 | return super(defaultlist, self).__getitem__(index) 42 | 43 | 44 | def re_parse_html(config, html_text): 45 | info = {} 46 | for k, v in config.items(): 47 | parsed_info = re.search(v, html_text) 48 | if str(parsed_info) != 'None': 49 | info[k] = parsed_info.group(1) 50 | 51 | return info 52 | 53 | 54 | def re_parse_html_list_field(config, html_text): 55 | info = {} 56 | for k, v in config.items(): 57 | info[k] = re.findall(v, html_text) 58 | 59 | return info -------------------------------------------------------------------------------- /JavHelper/core/warashi.py: -------------------------------------------------------------------------------- 1 | # -*- coding:utf-8 -*- 2 | from lxml import etree 3 | 4 | from JavHelper.core.requester_proxy import return_post_res, return_get_res 5 | from JavHelper.core import ActorNotFoundException 6 | 7 | 8 | class WarashiScraper: 9 | top_url = 'http://warashi-asian-pornstars.fr/' 10 | 11 | def actress_searcher(self, search_str: str): 12 | search_endpoint = 'en/s-12/search' # for female search most likely, s-12 might be changing? 13 | search_url = self.top_url + search_endpoint 14 | res = return_post_res(search_url, data={'recherche_valeur': search_str, 'recherche_critere': 'f'}).content 15 | 16 | root = etree.HTML(res) 17 | search_results = root.xpath('//div[@class="resultat-pornostar correspondance_exacte"]/p/a') 18 | if len(search_results) < 1: 19 | raise ActorNotFoundException(f'cannot find actor {search_str}') 20 | actress_href = search_results[0].get('href') # we only use 1st return 21 | 22 | return return_get_res(self.top_url+actress_href).content 23 | 24 | def get_image_from_actress_page(self, req_content, search_str=''): 25 | root = etree.HTML(req_content) 26 | image_results = root.xpath('//div[@id="pornostar-profil-photos"]/div/figure/a') 27 | if len(image_results) < 1: 28 | # only 1 image avaiable 29 | selected_image = root.xpath('//div[@id="pornostar-profil-photos-0"]/figure/img') 30 | if len(selected_image) < 1: 31 | raise ActorNotFoundException(f'cannot find actor {search_str}') 32 | else: 33 | selected_image = selected_image[0].get('src') 34 | else: 35 | selected_image = image_results[0].get('href') # we choose 2nd one for now 36 | return self.top_url+selected_image[1:] # need to get rid of 1st / 37 | 38 | def return_image_by_name(self, search_str: str): 39 | actress_res = self.actress_searcher(search_str) 40 | return self.get_image_from_actress_page(actress_res, search_str=search_str) 41 | 42 | -------------------------------------------------------------------------------- /JavHelper/model/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddd354/JAVOneStop/ba41c2561e44b3782e92c8c3004c9e1ca3097b0a/JavHelper/model/__init__.py -------------------------------------------------------------------------------- /JavHelper/model/jav_manager.py: -------------------------------------------------------------------------------- 1 | from math import ceil 2 | from time import sleep 3 | import json 4 | 5 | from blitzdb import Document, FileBackend 6 | from blitzdb.document import DoesNotExist 7 | import dataset 8 | import sqlalchemy 9 | 10 | 11 | class JavObj(Document): 12 | pass 13 | 14 | 15 | class SqliteJavManagerDB: 16 | def __init__(self): 17 | retry = 0 18 | while retry < 3: 19 | try: 20 | self.whole_db = dataset.connect('sqlite:///jav_manager.sqlite', engine_kwargs={'connect_args': {'check_same_thread': False}}) 21 | self.jav_db = self.whole_db['jav_obj'] 22 | break 23 | except Exception as e: 24 | print(f'read file sqlite db error {e}, gonna retry') 25 | retry += 1 26 | sleep(5) 27 | 28 | def create_indexes(self): 29 | #print('creating index for stat') 30 | #self.jav_db.create_index(JavObj, 'stat') 31 | #raise NotImplementedError('create_indexes hasn\'t been implemented') 32 | pass 33 | 34 | def rebuild_index(self): 35 | #raise NotImplementedError('rebuild_index hasn\'t been implemented') 36 | #self.jav_db.rebuild_index(self.jav_db.get_collection_for_cls(JavObj), 'stat') 37 | pass 38 | 39 | def bulk_list(self): 40 | return self.jav_db.find() 41 | 42 | def partial_search(self, search_string: str): 43 | rt = self.whole_db.query("SELECT * FROM jav_obj WHERE `car` LIKE '{}%' LIMIT 20".format(search_string)) 44 | return [self.reverse_prepare_obj(x) for x in rt] 45 | 46 | def query_on_filter(self, filter_on: dict, page=1, limit=8): 47 | max_page = self.whole_db.query("SELECT COUNT(*) FROM jav_obj WHERE {}={}".format( 48 | list(filter_on.keys())[0], list(filter_on.values())[0] 49 | )).next() 50 | print(max_page) 51 | rt_max_page = max_page['COUNT(*)'] // limit + 1 52 | 53 | rt = self.whole_db.query("SELECT * FROM jav_obj WHERE {}={} ORDER BY id asc LIMIT {} OFFSET {}".format( 54 | list(filter_on.keys())[0], list(filter_on.values())[0], limit, (page-1)*limit 55 | )) 56 | return [self.reverse_prepare_obj(x) for x in rt], rt_max_page 57 | 58 | @staticmethod 59 | def reverse_prepare_obj(input_obj: dict): 60 | if input_obj is None: 61 | return input_obj 62 | 63 | rt = {} 64 | for k, v in input_obj.items(): 65 | if k.startswith('_l_') and v: 66 | _k = k[3:] 67 | _v = json.loads(v)['list'] 68 | rt[_k] = _v 69 | elif k.startswith('_j_') and v: 70 | _k = k[3:] 71 | _v = json.loads(v) 72 | rt[_k] = _v 73 | elif v is not None: 74 | rt[k] = v 75 | return rt 76 | 77 | @staticmethod 78 | def prepare_obj(input_obj: dict): 79 | rt = {} 80 | for k, v in input_obj.items(): 81 | if isinstance(v, list): 82 | _k = '_l_{}'.format(k) 83 | _v = json.dumps({'list': v}) 84 | elif isinstance(v, dict): 85 | _k = '_j_{}'.format(k) 86 | _v = json.dumps(v) 87 | else: 88 | _k = k 89 | _v = v 90 | rt[_k] = _v 91 | return rt 92 | 93 | def upcreate_jav(self, jav_obj: dict): 94 | # uniform car to upper case 95 | jav_obj['car'] = str(jav_obj['car']).upper() 96 | try: 97 | # cannot use if to avoid 0 problem 98 | jav_obj['stat'] = int(jav_obj['stat']) 99 | except KeyError: 100 | import ipdb; ipdb.set_trace() 101 | jav_obj['stat'] = 2 102 | 103 | # convert nested field to text 104 | jav_obj = self.prepare_obj(jav_obj) 105 | 106 | self.jav_db.upsert(jav_obj, ['car']) 107 | #print(f'written \n {jav_obj} \n') 108 | 109 | def get_by_pk(self, pk: str): 110 | return self.reverse_prepare_obj(self.jav_db.find_one(car=pk.upper())) 111 | 112 | def pk_exist(self, pk: str): 113 | rt = self.jav_db.find_one(car=pk.upper()) 114 | if rt: 115 | return True 116 | else: 117 | return False 118 | 119 | 120 | class BlitzJavManagerDB: 121 | def __init__(self): 122 | retry = 0 123 | while retry < 3: 124 | try: 125 | self.jav_db = FileBackend('jav_manager.db') 126 | break 127 | except Exception as e: 128 | print(f'read file db error {e}, gonna retry') 129 | retry += 1 130 | sleep(5) 131 | 132 | if not self.jav_db: 133 | raise Exception('read local db error') 134 | 135 | def create_indexes(self): 136 | print('creating index for stat') 137 | self.jav_db.create_index(JavObj, 'stat') 138 | 139 | def rebuild_index(self): 140 | self.jav_db.rebuild_index(self.jav_db.get_collection_for_cls(JavObj), 'stat') 141 | 142 | def bulk_list(self): 143 | return self.jav_db.filter(JavObj, {}) 144 | 145 | def partial_search(self, search_string: str): 146 | rt = self.jav_db.filter(JavObj, {'pk': {'$regex': search_string.upper()}})[:20] 147 | return rt 148 | 149 | def query_on_filter(self, filter_on: dict, page=1, limit=8): 150 | rt = self.jav_db.filter(JavObj, filter_on) 151 | rt_max_page = ceil(len(rt)/limit) 152 | rt_list = rt[(page-1)*limit : (page)*limit] 153 | 154 | return [dict(x) for x in rt_list], rt_max_page 155 | 156 | def upcreate_jav(self, jav_obj: dict): 157 | # uniform car to upper case 158 | jav_obj['car'] = str(jav_obj['car']).upper() 159 | # set pk to car 160 | jav_obj['pk'] = jav_obj['car'] 161 | 162 | # pull existing data since this is update function 163 | try: 164 | current_jav_obj = dict(self.get_by_pk(jav_obj['car'])) 165 | #print(f'current dict {current_jav_obj}') 166 | # overwrite current db dict with input dict 167 | current_jav_obj.update(jav_obj) 168 | jav_obj = current_jav_obj 169 | except DoesNotExist: 170 | # set default to no opinion 171 | #0-want, 1-viewed, 2-no opinion 3-local 4-downloading 5-iceboxed 172 | jav_obj.setdefault('stat', 2) 173 | # safety measure to set stat to int 174 | jav_obj['stat'] = int(jav_obj['stat']) 175 | _jav_doc = JavObj(jav_obj) 176 | _jav_doc.save(self.jav_db) 177 | self.jav_db.commit() 178 | #print('writed ', jav_obj) 179 | 180 | def get_by_pk(self, pk: str): 181 | return self.jav_db.get(JavObj, {'pk': pk.upper()}) 182 | 183 | def pk_exist(self, pk: str): 184 | try: 185 | self.jav_db.get(JavObj, {'pk': pk.upper()}) 186 | return True 187 | except DoesNotExist: 188 | return False 189 | 190 | 191 | def migrate_blitz_to_sqlite(): 192 | #from JavHelper.model.jav_manager import migrate_blitz_to_sqlite 193 | b_db = BlitzJavManagerDB() 194 | s_db = SqliteJavManagerDB() 195 | n = 0 196 | 197 | for obj in b_db.bulk_list(): 198 | _obj = dict(obj) 199 | s_db.upcreate_jav(obj) 200 | n += 1 201 | if n % 1000 == 0: 202 | print(f'processed {n}') -------------------------------------------------------------------------------- /JavHelper/run.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from JavHelper.app import create_app 4 | 5 | if __name__ == '__main__': 6 | os.environ['FLASK_ENV'] = 'development' 7 | app = create_app() 8 | app.run(threaded=True, host='0.0.0.0', port=8009) 9 | -------------------------------------------------------------------------------- /JavHelper/scripts/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ddd354/JAVOneStop/ba41c2561e44b3782e92c8c3004c9e1ca3097b0a/JavHelper/scripts/__init__.py -------------------------------------------------------------------------------- /JavHelper/scripts/emby_actors.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import requests 4 | import traceback 5 | import argparse 6 | import json 7 | 8 | from JavHelper.core.ini_file import return_config_string 9 | 10 | 11 | def list_emby_actress(emby_url, api_key): 12 | url = f'{emby_url}emby/Persons?api_key={api_key}' 13 | return requests.get(url).json()['Items'] 14 | 15 | 16 | def post_image_to_actress(actress_id, image_f, emby_url, api_key): 17 | with open(image_f, 'rb') as f: 18 | b6_pic = base64.b64encode(f.read()) # 读取文件内容，转换为base64编码 19 | 20 | url = f'{emby_url}emby/Items/{actress_id}/Images/Primary?api_key={api_key}' 21 | if image_f.endswith('png'): 22 | header = {"Content-Type": 'image/png', } 23 | else: 24 | header = {"Content-Type": 'image/jpeg', } 25 | 26 | requests.post(url=url, data=b6_pic, headers=header) 27 | print(f'successfully post actress ID: {actress_id} image\n') 28 | return 1 29 | 30 | 31 | def send_emby_images(image_folder_path): 32 | # init 33 | num = 0 34 | up_num = 0 35 | 36 | if not os.path.exists(image_folder_path): 37 | print('current path: {}'.format(os.getcwd())) 38 | raise Exception('{} image folder doesn\'t exist, please specify correct path'.format(image_folder_path)) 39 | 40 | emby_url = return_config_string(["emby专用", "网址"]) 41 | api_key = return_config_string(["emby专用", "api id"]) 42 | 43 | # try correct emby url with / 44 | if not emby_url.endswith('/'): 45 | emby_url += '/' 46 | 47 | try: 48 | for actress in list_emby_actress(emby_url, api_key): 49 | num += 1 50 | if num % 500 == 0: 51 | print('have processed', num, '个actress') 52 | 53 | actress_name = actress['Name'] 54 | actress_id = actress['Id'] 55 | res_info = {'log': f'processed 女优：{actress_name}, ID：{actress_id}'} 56 | 57 | if os.path.isfile(os.path.join(image_folder_path, f'{actress_name}.jpg')): 58 | file_path = os.path.join(image_folder_path, f'{actress_name}.jpg') 59 | up_num += post_image_to_actress(actress_id, file_path, emby_url, api_key) 60 | 61 | elif os.path.isfile(os.path.join(image_folder_path, f'{actress_name}.png')): 62 | file_path = os.path.join(image_folder_path, f'{actress_name}.png') 63 | up_num += post_image_to_actress(actress_id, file_path, emby_url, api_key) 64 | 65 | else: 66 | res_info = {'log': f'{actress_name} image file doen\'t exist'} 67 | print(res_info) 68 | 69 | yield json.dumps(res_info)+'\n' 70 | 71 | except requests.exceptions.ConnectionError: 72 | print('emby服务端无法访问，请检查：', emby_url, '\n') 73 | except Exception as err: 74 | traceback.print_exc() 75 | print('发生未知错误，请截图给作者：', emby_url, err) 76 | 77 | print(f'成功upload {up_num} 个女优头像！') 78 | yield json.dumps({'log': f'成功upload {up_num} 个女优头像！'})+'\n' 79 | 80 | 81 | if __name__ == '__main__': 82 | parser = argparse.ArgumentParser(description='Submit AV actress images to emby') 83 | parser.add_argument('--image-folder-path', required=True, help='parsable path to the image folder directory') 84 | parser.add_argument('--ini-name', help='filename for ini file') 85 | 86 | args = parser.parse_args() 87 | send_emby_images(args.image_folder_path, args.ini_name) 88 | -------------------------------------------------------------------------------- /JavHelper/static/webHelper/HelpDoc.jsx: -------------------------------------------------------------------------------- 1 | import React, {useState, useEffect} from 'react'; 2 | import ReactMarkdown from 'react-markdown/with-html'; 3 | import ToggleButtonGroup from 'react-bootstrap/ToggleButtonGroup'; 4 | import ToggleButton from 'react-bootstrap/ToggleButton'; 5 | 6 | import Container from 'react-bootstrap/Container' 7 | import Row from 'react-bootstrap/Row' 8 | import Col from 'react-bootstrap/Col' 9 | 10 | import { useTranslation } from 'react-i18next'; 11 | 12 | const HelpDoc = () => { 13 | const { t, i18n } = useTranslation(); 14 | 15 | const [markdown, setMarkdown] = useState(''); 16 | const [markdown_source, setMarkdownSource]= useState('changelog'); 17 | 18 | // init with github markdown 19 | useEffect(() => { 20 | fetch(`/local_manager/readme?source=${markdown_source}`) 21 | .then(response => response.json()) 22 | .then((jsonData) => { 23 | setMarkdown(jsonData.success); 24 | }) 25 | }, [markdown_source]); 26 | 27 | return ( 28 | 29 | 30 | setMarkdownSource(e)} style={{flexWrap: "wrap", marginLeft: "5px"}}> 32 | 33 | {t('changelog')} 34 | 35 | 36 | {t('main_readme')} 37 | 38 | 39 | {t('javdownloader_readme')} 40 | 41 | 42 | 43 | 44 | 48 | 49 | 50 | ); 51 | } 52 | 53 | export default HelpDoc; 54 | 55 | -------------------------------------------------------------------------------- /JavHelper/static/webHelper/OofValidator.jsx: -------------------------------------------------------------------------------- 1 | import React, { useState } from 'react'; 2 | import Modal from 'react-bootstrap/Modal' 3 | import Button from 'react-bootstrap/Button' 4 | import Iframe from 'react-iframe' 5 | 6 | import { useTranslation } from 'react-i18next'; 7 | 8 | 9 | const OofValidator = () => { 10 | const { t, i18n } = useTranslation(); 11 | 12 | const [show, setShow] = useState(false); 13 | 14 | const handleClose = () => setShow(false); 15 | const handleShow = () => setShow(true); 16 | 17 | return ( 18 |
19 | 22 | 23 | 24 | 25 | {t('oof_validate_instruction')} 26 | 27 | 28 |