├── .gitignore ├── LICENSE ├── README-en.md ├── README.md ├── README.rss.md ├── autoban.py ├── check_config.py ├── common.py ├── config.py.en.example ├── config.py.example ├── filter.py ├── gen_stats.py ├── gzapi.py ├── remove_unregistered.py ├── reseed.py ├── rss.py ├── show_banned.sh ├── show_new_torrent.sh └── torrent_filter.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.txt 2 | *.bak 3 | *.log 4 | *.torrent 5 | __pycache__/ 6 | cache/ 7 | tocheck/ 8 | watch/ 9 | private/ 10 | *.py -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) fishpear 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README-en.md: -------------------------------------------------------------------------------- 1 | - [Gazelle Autodl Script Set](#gazelle-autodl-script-set) 2 | - [Functionalities](#functionalities) 3 | - [Installation Instructions](#installation-instructions) 4 | - [Configuration](#configuration) 5 | - [Which entries should I edit?](#which-entries-should-i-edit) 6 | - [Verify the config](#verify-the-config) 7 | - [Where to find cookies](#where-to-find-cookies) 8 | - [Where to find authkey, torrent_pass](#where-to-find-authkey-torrent_pass) 9 | - [Where to find api_key](#where-to-find-api_key) 10 | - [How to comment / uncomment a piece of code](#how-to-comment--uncomment-a-piece-of-code) 11 | - [Filter by uploader and use tokens by torrent size `filter.py`](#filter-by-uploader-and-use-tokens-by-torrent-size-filterpy) 12 | - [Warning](#warning) 13 | - [Configuration Required](#configuration-required) 14 | - [Run](#run) 15 | - [Mode A: Monitor a directory](#mode-a-monitor-a-directory) 16 | - [Mode B: Call by parameters](#mode-b-call-by-parameters) 17 | - [Parameters](#parameters) 18 | - [some pieces of log (in monitoring mode):](#some-pieces-of-log-in-monitoring-mode) 19 | - [Automatically ban uploaders `autoban.py`](#automatically-ban-uploaders-autobanpy) 20 | - [Rule of banning](#rule-of-banning) 21 | - [Configuration Required](#configuration-required-1) 22 | - [Run](#run-1) 23 | - [Parameters](#parameters-1) 24 | - [A piece of log](#a-piece-of-log) 25 | - [Export deluge statistics `gen_stats.py`](#export-deluge-statistics-gen_statspy) 26 | - [Delete unregistered torrents in deluge `remove_unregistered.py`](#delete-unregistered-torrents-in-deluge-remove_unregisteredpy) 27 | - [Warning](#warning-1) 28 | - [Functionality](#functionality) 29 | - [Run](#run-2) 30 | - [A piece of log：](#a-piece-of-log-1) 31 | - [Crossseeding `reseed.py`](#crossseeding-reseedpy) 32 | - [Configuration Required](#configuration-required-2) 33 | - [Run](#run-3) 34 | - [Mode A: scan all subdirectories](#mode-a-scan-all-subdirectories) 35 | - [Mode B: scan one directory](#mode-b-scan-one-directory) 36 | - [Mode C: crossseed multiple torrent files under a directory](#mode-c-crossseed-multiple-torrent-files-under-a-directory) 37 | - [Mode D: crossseed single torrent file](#mode-d-crossseed-single-torrent-file) 38 | - [Results](#results) 39 | - [Parameters](#parameters-2) 40 | - [a piece of log](#a-piece-of-log-2) 41 | - [Bug report and feature request](#bug-report-and-feature-request) 42 | # Gazelle Autodl Script Set 43 | This set of scripts aims on bringing better autodl experience in gazelle music trackers. 44 | 45 | Redacted, Orpheus and Dicmusic are now supported. 46 | 47 | ## Functionalities 48 | * Filter by uploader. You can customize some filter conditions including a ban-list of uploaders. No restriction on BT client. 49 | * Spend tokens wisely. The filter will spend tokens if the torrent size is within a configured range. No restriction on BT client. RED/OPS is not supported in this feature because of its rule. 50 | * Autoban. Automatically ban an uploader if your ratio is too low. Only deluge is supported. 51 | * Export deluge statistics. 52 | * Delete unregistered torrents in deluge, along with their files. 53 | * Cross-seed. Scan files in a directory and download the "crossseedable" torrents. 54 | 55 | ## Installation Instructions 56 | 57 | Only python3 is supported. To see if your python3 is correctly installed, type in 58 | ``` 59 | python3 --version 60 | ``` 61 | in your console and you'll see your python's version. 62 | 63 | Install python modules (omit sudo if you're root or in Windows) 64 | ``` 65 | sudo pip3 install bencode.py ipython requests datasize deluge-client 66 | ``` 67 | if you don't have root privilege, you can use `--user` 68 | ``` 69 | pip3 install bencode.py ipython requests datasize deluge-client --user 70 | ``` 71 | or things like `virtualenv` (google it) 72 | 73 | Then download this repo: 74 | ``` 75 | git clone https://github.com/qfishpear/fishrss.git 76 | cd fishrss/ 77 | ``` 78 | 79 | ## Configuration 80 | 81 | make a copy of `config.py.en.example` into `config.py` 82 | ``` 83 | cp config.py.en.example config.py 84 | ``` 85 | and edit `config.py` according to the instructions in it. Make sure all files and directories in config have been already created by yourself. 86 | 87 | Relative path is tolerable, but if you're going to run in crontab or something like that, absolute path is highly recommended 88 | 89 | If you're using Windows, use `/` instead of `\` as the separator of path. 90 | 91 | ### Which entries should I edit? 92 | 93 | Different entries in config are required in different scripts. Not all of them are required if you're not going to use every scripts in this repo. However, the following ones are required to be edited in all scripts. (`None` is also regarded as edited) 94 | 95 | * For Redacted, in `CONFIG["red"]`, `"api_cache_dir"`，`"authkey"`, `"torrent_pass"` are required, and one of `"cookies"` and `"api_key"` are required. If you don't edit `"cookies"`, leave it commented. 96 | * For Orpheus, in `CONFIG["ops"]`, `"api_cache_dir"`，`"authkey"`, `"torrent_pass"` are required, and one of `"cookies"` and `"api_key"` are required. If you don't edit `"cookies"`, leave it commented. 97 | * For Dicmusic, in `CONFIG["dic"]`, `"api_cache_dir"`，`"cookies"`, `"authkey"`, `"torrent_pass"` are required. 98 | 99 | If you're going to use other scripts besides `filter.py`, it's highly recommended to fill `api_cache_dir` and create the corresponding directory. Otherwise, same api requests will be sent to server repeatedly if you run the scripts multiple times, which makes it really slow. Some JSON files should be created in that folder as API cache during running. 100 | 101 | The entries required in each script will be explained correspondingly. 102 | 103 | ### Verify the config 104 | To see if you've edited it correctly, run 105 | ``` 106 | python3 check_config.py 107 | ``` 108 | If everything is fine, it should print as below. However, it's not guaranteed to be correct even if the check is passed. 109 | ``` 110 | 2021-04-20 16:01:43,835 - INFO - dic querying action=index 111 | 2021-04-20 16:01:44,616 - INFO - dic logged in successfully, username：fishpear uid: 1132 112 | 2021-04-20 16:01:44,616 - INFO - red querying action=index 113 | 2021-04-20 16:01:44,781 - INFO - red logged in successfully, username：fishpear uid: 50065 114 | 2021-04-20 16:01:44,783 - INFO - ops querying action=index 115 | 2021-04-20 16:01:45,099 - INFO - ops logged in successfully, username：fishpear uid: 21482 116 | 2021-04-20 16:01:45,116 - INFO - deluge is correctly configured 117 | ``` 118 | 119 | ### Where to find cookies 120 | There are multiple ways. I personally recommend using the "editthiscookie" plugin of chrome 121 | ``` 122 | https://chrome.google.com/webstore/detail/editthiscookie/fngmhnnpilhplaeedifhccceomclgfbg 123 | ``` 124 | Open an arbitrary page of red, and copy like this: 125 | ![1.png](https://i.loli.net/2021/04/22/1yoz6rNGcBHU4TF.png) 126 | 127 | ### Where to find authkey, torrent_pass 128 | Copy the downloading link "DL" of an arbitrary torrent, and you can find them in the link. 129 | 130 | ### Where to find api_key 131 | red：open your profile and follow: 132 | ![1.png](https://i.loli.net/2021/04/22/WNk4ZXz7vi1DneP.png) 133 | ops：almost the same as red (and even easier) 134 | 135 | ### How to comment / uncomment a piece of code 136 | In python, commenting means adding `#` in front of a line. 137 | 138 | In sublime and other mainstream text editors, select the piece of code that you want to comment / uncomment and press shortcut `ctrl+/` 139 | 140 | ## Filter by uploader and use tokens by torrent size `filter.py` 141 | 142 | In a word, this script monitors new torrent files in `source_dir`, saves the ones that satisfy given conditions to `dest_dir`, and then spends the tokens according to the torrent size limit. 143 | 144 | ### Warning 145 | * Filtering will slightly increase latency comparing to the raw irssi-autodl and might have influence on the racing performance. 146 | * It won't check if your tokens are used up. If tokens are used up, it will still leave the torrent downloaded. So keep an eye on how many tokens are left if you're short in buffer. 147 | 148 | ### Configuration Required 149 | 150 | * All entries in `CONFIG["filter"]`：`"source_dir"`, `"dest_dir"`, `"default_behavior"` 151 | * For dic/red/ops, fill the following entries in `CONFIG["dic"/"red"/"ops"]` correspondingly: 152 | * all entries in `"filter_config"`: `"name"`, `"banlist"`, `"whitelist"`, `"media"`, `"format"`, `"sizelim"` 153 | * `"token_thresh"` (only for dic) 154 | 155 | ### Run 156 | 157 | #### Mode A: Monitor a directory 158 | Just run 159 | ``` 160 | python3 filter.py 161 | ``` 162 | It will monitor new .torrent files in `source_dir`, saves the ones that satisfy given conditions to `dest_dir`, and spends the tokens according to the torrent size limit. 163 | 164 | To ease the anxiety of a blank screen, it will print a line of "tick" every minute. 165 | 166 | If `config.py` is changed, the `filter.py` should be restarted. Press `ctrl-C` to shut it down and run again. 167 | 168 | Example: the directories to enter in config.py, autodl and deluge: 169 | ![2.JPG](https://i.loli.net/2021/04/22/KZlzXAj8eTEtObu.jpg) 170 | ![2.JPG](https://i.loli.net/2021/04/20/xMldRSFw1A4qs8u.jpg) 171 | ![3.JPG](https://i.loli.net/2021/04/20/x6TZYJGXidgSjOQ.jpg) 172 | 173 | #### Mode B: Call by parameters 174 | 175 | Example 176 | ``` 177 | python3 filter.py --file ./1.torrent 178 | python3 filter.py --url https://redacted.ch/torrents.php?action=download\&id=xxxx\&authkey=xxxx\&torrent_pass=xxx 179 | ``` 180 | 181 | Notice: the login check will be skipped to reduce latency. It's recommended to run `check_config.py` first in order to make sure things are correct. 182 | 183 | If you want to use it with irssi-autodl, edit the Preference->Action like this: 184 | 185 | ![1.JPG](https://i.loli.net/2021/04/21/g9dPteW3ciMmKUj.jpg) 186 | 187 | The above entry should be the absolute path to python3. The under entry should be: 188 | ``` 189 | "/absolute/path/to/filter.py" --file "$(TorrentPathName)" 190 | ``` 191 | Below is also fine but with slightly more latency: 192 | ``` 193 | "/absolute/path/to/filter.py" --url $(TorrentUrl) 194 | ``` 195 | All paths in `config.py` should be absolute if `filter.py` is used in this way. 196 | 197 | ### Parameters 198 | ~~~~ 199 | usage: filter.py [-h] [--url URL] [--file FILE] [--skip-api] [--no-tick] [--force-accept] [--deluge] 200 | 201 | optional arguments: 202 | -h, --help show this help message and exit 203 | --url URL Download url of a torrent. If an url is provided, the code will only run once for the provided url 204 | and exits. 205 | --file FILE path of a torrent file. If a file is provided, the code will only run once for the provided file and 206 | exits. 207 | --skip-api If set, the site api call will be skipped. Notice: the api-only information will be unavailable: 208 | uploader, media and format. Therefore, their filter config must be None otherwise there won't be any 209 | torrent filtered out. 210 | --no-tick If not set, every minute there will be a "tick" shown in the log, in order to save people with 211 | "black screen anxiety" 212 | --force-accept If set, always accept a torrent regardless of filter's setting 213 | --deluge push torrents to deluge by its api directly instead of saving to CONFIG["filter"]["dest_dir"] 214 | ~~~~ 215 | Notice: if you just want the "wisely-use-token" functionality and want a lower latency, add `--skip-api` and `--force-accept` like: 216 | ``` 217 | python3 filter.py --skip-api --force-accept 218 | ``` 219 | or 220 | ``` 221 | "/absolute/path/to/filter.py" --file "$(TorrentPathName)" --skip-api --force-accept 222 | ``` 223 | in irssi-autodl. 224 | 225 | The latter one should ideally have negligible latency increment comparing to "save to watch folder" directly in irssi-autodl. If you add option `--deluge`, it might be even faster. 226 | 227 | ### some pieces of log (in monitoring mode): 228 | ``` 229 | 2021-04-12 18:28:37,392 - INFO - api and filter of RED are set 230 | 2021-04-12 18:28:37,392 - INFO - api and filter of DIC are set 231 | ...... 232 | 2021-04-12 19:04:14,717 - INFO - new torrent from red: xxxxxxxxxxxxxxxxx 233 | 2021-04-12 19:04:14,718 - INFO - red querying action=torrent&id=xxxxxxxxxxxxxxxxx 234 | 2021-04-12 19:04:15,465 - INFO - redfilter: checking 235 | 2021-04-12 19:04:15,465 - INFO - uploader: xxxxxxxxxxxxxxxxx media: CD format: FLAC releasetype: 9 size: 114.9MB 236 | 2021-04-12 19:04:15,465 - INFO - reject: banned user 237 | 2021-04-12 19:04:22,676 - INFO - tick 238 | 2021-04-12 19:04:52,522 - INFO - new torrent from red: xxxxxxxxxxxxxxxxx 239 | 2021-04-12 19:04:52,523 - INFO - red querying action=torrent&id=xxxxxxxxxxxxxxxxx 240 | 2021-04-12 19:04:53,428 - INFO - redfilter: checking 241 | 2021-04-12 19:04:53,428 - INFO - uploader: xxxxxxxxxxxxxxxxx media: CD format: FLAC releasetype: 1 FLAC size: 224.7MB 242 | 2021-04-12 19:04:53,429 - INFO - accept 243 | 2021-04-12 19:05:23,673 - INFO - tick 244 | 2021-04-12 19:06:23,760 - INFO - tick 245 | 2021-04-12 19:07:23,858 - INFO - tick 246 | 2021-04-12 19:08:23,944 - INFO - tick 247 | 2021-04-12 19:09:24,031 - INFO - tick 248 | ``` 249 | 250 | ## Automatically ban uploaders `autoban.py` 251 | This script read the statistics from deluge, add the uploaders who fullfill given conditions to file `CONFIG["red"/"ops"/"dic]["filter_config"]["banlist"]`. 252 | 253 | `autoban.py` runs independently of `filter.py` and only interacts with it by the banlist file. 254 | 255 | ### Rule of banning 256 | 257 | Here is the entries in `config["red"/"ops"/"dic"]["autoban"]` 258 | 259 | * Only consider torrents with progress more than `autoban["ignore"]["min_progress"]` and added no more than `autoban["ignore"]["max_time_added"]` seconds ago. 260 | * For each entry in `autoban["ratio"]`: if an uploader has uploaded no less than `"count"` torrents and your ratio is under "ratiolim"`, it will be added to banlist. 261 | 262 | ### Configuration Required 263 | 264 | * If you want autoban for red/ops/dic, `CONFIG["red"/"ops"/"dic"]["filter_config"]["banlist"]` should be a file. 265 | * All entries in `CONFIG["deluge"]`: `"ip"`, `"port"`, `"username"`, `"password"`. Here `ip` and `port` should be consistent with connection manager. `username` and `password` are the ones for logging into deluge's webui, if you're not asked to enter username and password, they can be arbitrary (but I'm not confident of this)
266 | ![5.JPG](https://i.loli.net/2021/04/16/ZBVay3rjhCPK6Ui.jpg) 267 | * All entries in `config["red"/"ops"/"dic"]["autoban"]` 268 | 269 | ### Run 270 | For first run, add `--init` 271 | ``` 272 | python3 autoban.py --init 273 | ``` 274 | Afterwards, just 275 | ``` 276 | python3 autoban.py 277 | ``` 278 | for each run. 279 | 280 | The script only runs once, i.e. bans once. For contineously running, use crontab/watch. 281 | 282 | Example of using watch: run every 2 minutes 283 | ``` 284 | watch -n 120 python3 autoban.py 285 | ``` 286 | 287 | Notice: `autoban.py` will send API request and might be slow at the first run because of the limitation of API frequency. 288 | 289 | ### Parameters 290 | ~~~~ 291 | usage: autoban.py [-h] [--stats] [--init] [--site {dic,red,ops}] 292 | 293 | optional arguments: 294 | -h, --help show this help message and exit 295 | --init run as initialization. if not set, the autoban logic will ONLY ban an uploader if one of his uploaded torrents is active. Here 296 | "active" is defined by being uploaded in an hour or not completed. 297 | --site {dic,red,ops} if set, only update the banlist of the specified site. 298 | --stats show stats of all the uploaders 299 | ~~~~ 300 | `--stats` can be used if you have question on the ban logic. A piece of log: 301 | ``` 302 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 3 ratio: 1.030 0.678GB/0.658GB 303 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 1 ratio: 0.506 0.218GB/0.430GB 304 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 13 ratio: 0.842 0.658GB/0.781GB 305 | 2021-04-17 09:56:00,778 - INFO - uploader: xxxxxxxxx #torrents: 67 ratio: 0.924 7.557GB/8.180GB 306 | ``` 307 | 308 | ### A piece of log 309 | ``` 310 | 2021-04-14 11:10:17,372 - INFO - autoban: deluge is connected: True 311 | 2021-04-14 11:10:19,876 - INFO - new user banned: ********* #torrents: 1 ratio: 0.000 0.000GB/0.141GB 312 | 2021-04-14 11:10:19,876 - INFO - related torrents: megane panda (眼鏡熊猫) - Natsu No Machi EP (夏の街EP) (2015) [WEB] [FLAC] ratio: 0.000 0.0MB/144.7MB 313 | 2021-04-14 11:10:19,877 - INFO - 39 user banned in total 314 | ``` 315 | 316 | ## Export deluge statistics `gen_stats.py` 317 | 318 | If deluge and API are correctly configured, just run 319 | ``` 320 | python3 gen_stats.py > stats.txt 321 | ``` 322 | will get a table in text file `stats.txt`. It not only contains information in deluge, but also information from API. 323 | 324 | It uses `tab` as separator, so you can copy-paste the content into excel and analysis the data afterwards 325 | 326 | Notice: `gen_stats.py` will send API request and might be slow at the first run because of the limitation of API frequency. 327 | 328 | ## Delete unregistered torrents in deluge `remove_unregistered.py` 329 | 330 | ### Warning 331 | 332 | This script will delete torrents and **files** permanently in deluge, make sure you know its functionality before use it. 333 | 334 | ### Functionality 335 | 336 | delete all the uncompleted torrents in deluge with "Unregistered torrent" in "Tracker Status", along with the **files**. 337 | 338 | Notice: it's not limited to torrents of red/dic/ops. 339 | ### Run 340 | 341 | If deluge and API are correctly configured, just run 342 | ``` 343 | python3 remove_unregistered.py 344 | ``` 345 | The script only runs once, i.e. deletes once. For contineously running, use crontab/watch. 346 | 347 | ### A piece of log： 348 | ``` 349 | 2021-04-14 11:02:13,526 - INFO - remove_unregistered: deluge is connected: True 350 | 2021-04-14 11:02:13,582 - INFO - removing torrent "Atomic Kitten - Feels So Good (2002) [7243 5433722 2]" reason: "xxxxxxxx.xxxx: Error: Unregistered torrent" 351 | 2021-04-14 11:02:13,974 - INFO - removing torrent "VA-Clap-(COUD_11)-12INCH_VINYL-FLAC-199X-YARD" reason: "flacsfor.me: Error: Unregistered torrent" 352 | 2021-04-14 11:02:14,533 - INFO - removing torrent "Headnodic - Tuesday (2002) - WEB FLAC" reason: "flacsfor.me: Error: Unregistered torrent" 353 | ``` 354 | 355 | ## Crossseeding `reseed.py` 356 | 357 | This script scans all sub-directories under a given directory and searchs in the tracker for torrents that can be crossseeded. 358 | 359 | It can also scan torrents under a given directory to see if it can be crossseeded in another tracker. 360 | 361 | Notice: The automatical way of searching can't be as perfect. Therefore it does NOT promise all found torrents can be correctly crossseeded, NOR does it promise all missed directories can not be crossseeded. A minority of torrents might have their files/directories renamed and other unexpected problems. Make sure your BT client does NOT automatically start when you add the scanned torrents. 362 | 363 | ### Configuration Required 364 | 365 | No extra entries in `config.py` needed to fill, except [the mandatory ones](#which-entries-should-i-edit) 366 | 367 | However, if you have some irrelevant files/directories in your music directory, for example, `.DS_Store` in macOS, they will influence the scanning process. Add the files/directories you want to ignore to the `IGNORED_PATH` list in `reseed.py`. 368 | 369 | ### Run 370 | 371 | If you terminated the script during running, the next time you run it, it will skip the sub-directories it scanned before. If you don't want to skip them, delete the `scan_history.txt` under the directory given by `--result-dir`. 372 | 373 | The directory given by `--result-dir` should be created before running the script. 374 | 375 | #### Mode A: scan all subdirectories 376 | For example, if all your music files are downloaded to `~/downloads` and you're trying to crossseed in RED and store the results in folder `~/results`, run: 377 | ``` 378 | python3 reseed.py --site red --dir ~/downloads --result-dir ~/results 379 | ``` 380 | 381 | #### Mode B: scan one directory 382 | 383 | Use `--single-dir` to crossseed for a single directory. 384 | 385 | For example: if there are music files in `~/downloads/Masashi Sada (さだまさし) - さだ丼～新自分風土記III～ (2021) [24-96]/`, run: 386 | ``` 387 | python3 reseed.py --site dic --single-dir ~/downloads/Masashi\ Sada\ \(さだまさし\)\ -\ さだ丼～新自分風土記III～\ \(2021\)\ \[24-96\]/ --result-dir ~/results 388 | ``` 389 | Take care of the escape characters are required when coming up with spaces and some other characters. It's recommended to enter the directory names by `TAB` 390 | 391 | #### Mode C: crossseed multiple torrent files under a directory 392 | 393 | Use `--torrent-dir` to try to crossseed multiple .torrent files: 394 | 395 | For example: if there are .torrent files in `~/torrents`, run: 396 | ``` 397 | python3 reseed.py --site red --torrent-dir ~/torrents --result-dir ~/results 398 | ``` 399 | 400 | #### Mode D: crossseed single torrent file 401 | Use `--single-torrent` to try to crossseed one .torrent files: 402 | 403 | For example: if there is a torrent `~/torrents/The Call - Collected - 2019 (CD - FLAC - Lossless).torrent`, run: 404 | ``` 405 | python3 reseed.py --site red --single-torrent ~/torrents/The\ Call\ -\ Collected\ -\ 2019\ \(CD\ -\ FLAC\ -\ Lossless\).torrent --result-dir ~/results 406 | ``` 407 | 408 | ### Results 409 | 410 | The following things will be generated under the directory given by `--result-dir`: 411 | * `torrent/`, a directory storing all the downloaded .torrent files. All .torrent files will be named by "name-of-crossseeding-directory.torrent". This way of naming may be helpful if the name of directory was changed by the uploader. 412 | * `result_mapping.txt`, the result of crossseeding. Each line contains a (directory / .torrent file) name and a torrent ID, seperated by a TAB (\t). If no torrent is found for crossseeding, the torrent ID would be -1. 413 | * `result_url.txt`, each line contains a download link of "crossseedable" torrents. 414 | * `result_url_undownloaded.txt`. Due to some tracker's limitation of downloading with scripts, some torrents will fail to download. The links of these torrents will be store in this file and you can manually download them. 415 | * `scan_history.txt`, each line contains the absolute path to a directory that has been scanned before. The crossseeding script will ignore the directories written in this file. 416 | 417 | ### Parameters 418 | ``` 419 | usage: reseed.py [-h] (--dir DIR | --single-dir SINGLE_DIR) --site {dic,red,ops,snake} --result-dir RESULT_DIR 420 | [--api-frequency API_FREQUENCY] [--no-download] 421 | 422 | scan a directory to find torrents that can be cross-seeded on given tracker 423 | 424 | optional arguments: 425 | -h, --help show this help message and exit 426 | --dir DIR folder for batch cross-seeding 427 | --single-dir SINGLE_DIR 428 | folder for just one cross-seeding 429 | --torrent-dir TORRENT_DIR 430 | folder containing .torrent files for cross-seeding 431 | --single-torrent SINGLE_TORRENT 432 | one .torrent file for cross-seeding 433 | --site {dic,red,ops,snake} 434 | the tracker to scan for cross-seeding. 435 | --result-dir RESULT_DIR 436 | folder for saving scanned results 437 | --api-frequency API_FREQUENCY 438 | if set, override the default api calling frequency. Unit: number of api call per 10 seconds (must be integer) 439 | --no-download if set, don't download the .torrent files. Only the id of torrents are saved 440 | ``` 441 | 442 | ### a piece of log 443 | ``` 444 | fishpear@sea:~/rss$ python3 reseed.py --site dic --dir ~/downloads --result-dir ~/results 445 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/scan_history.txt 446 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/result_url.txt 447 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/result_mapping.txt 448 | 2021-04-25 16:22:16,800 - INFO - directory automatically created: /home7/fishpear/results/torrents/ 449 | 2021-04-25 16:22:16,800 - INFO - file automatically created: /home7/fishpear/results/result_url_undownloaded.txt 450 | 2021-04-25 16:22:16,800 - INFO - dic querying action=index 451 | 2021-04-25 16:22:17,588 - INFO - dic logged in successfully, username：fishpear uid: 1132 452 | 2021-04-25 16:22:17,632 - INFO - 1797/1797 unscanned folders found in /home7/fishpear/downloads, start scanning for cross-seeding dic 453 | 2021-04-25 16:22:17,632 - INFO - 1/1797 /home7/fishpear/downloads/Eliane Radigue - Backward 454 | 2021-04-25 16:22:17,632 - INFO - dic querying filelist=3+Songs+Of+Milarepa+1+2+Remastered+2021+flac&action=browse 455 | 2021-04-25 16:22:18,329 - INFO - not found 456 | ... 457 | 2021-04-25 16:22:19,711 - INFO - 4/1797 /home7/fishpear/downloads/55 Schubert and Boccherini String Quintets 458 | 2021-04-25 16:22:19,711 - INFO - dic querying filelist=03+Quintet+in+C+Major+for+Two+Violins+Viola+and+Two+Cellos+D+956+III+Scherzo+Presto+Trio+Andante+sostenuto+flac&action=browse 459 | 2021-04-25 16:22:20,406 - INFO - found, torrentid=49506 460 | 2021-04-25 16:22:21,517 - INFO - saving to /home7/fishpear/results/torrents/55 Schubert and Boccherini String Quintets.torrent 461 | ... 462 | ``` 463 | 464 | ## Bug report and feature request 465 | 466 | Bug reports are welcomed: 467 | * Please send the log file (`filter.log` by default) and screenshots to help analysis. 468 | 469 | Feature requests are welcomed: 470 | * Just don't request a filtering condition in `filter.py` if it can be done by irssi-autodl itself. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | English Version of README: [README-en.md](https://github.com/qfishpear/fishrss/blob/main/README-en.md) 2 | 3 | - [Gazelle r种增强脚本](#gazelle-r种增强脚本) 4 | - [功能](#功能) 5 | - [安装依赖](#安装依赖) 6 | - [下载本脚本](#下载本脚本) 7 | - [填写配置信息](#填写配置信息) 8 | - [我应该填写哪些信息](#我应该填写哪些信息) 9 | - [验证config](#验证config) 10 | - [如何获取cookie, authkey, torrent_pass](#如何获取cookie-authkey-torrent_pass) 11 | - [如何获取api_key](#如何获取api_key) 12 | - [怎么编辑配置文件](#怎么编辑配置文件) 13 | - [怎么对一段代码添加注释/去除注释](#怎么对一段代码添加注释去除注释) 14 | - [种子过滤与智能令牌`filter.py`](#种子过滤与智能令牌filterpy) 15 | - [警告](#警告) 16 | - [填写配置信息](#填写配置信息-1) 17 | - [运行](#运行) 18 | - [方式1：监控文件夹](#方式1监控文件夹) 19 | - [方法2：参数调用](#方法2参数调用) 20 | - [参数解释](#参数解释) 21 | - [部分log节选](#部分log节选) 22 | - [自动拉黑`autoban.py`](#自动拉黑autobanpy) 23 | - [拉黑规则](#拉黑规则) 24 | - [填写配置信息](#填写配置信息-2) 25 | - [运行](#运行-1) 26 | - [参数解释](#参数解释-1) 27 | - [部分log节选](#部分log节选-1) 28 | - [deluge数据导出`gen_stats.py`](#deluge数据导出gen_statspy) 29 | - [deluge删除网站上被删种的种子`remove_unregistered.py`](#deluge删除网站上被删种的种子remove_unregisteredpy) 30 | - [警告](#警告-1) 31 | - [功能](#功能-1) 32 | - [运行](#运行-2) 33 | - [扫描文件夹辅种`reseed.py`](#扫描文件夹辅种reseedpy) 34 | - [填写配置信息](#填写配置信息-3) 35 | - [运行](#运行-3) 36 | - [方法1：扫描所有子文件夹](#方法1扫描所有子文件夹) 37 | - [方法2：扫描单个文件夹](#方法2扫描单个文件夹) 38 | - [方法3：扫描种子文件夹](#方法3扫描种子文件夹) 39 | - [方法4：扫描单个种子](#方法4扫描单个种子) 40 | - [辅种结果](#辅种结果) 41 | - [参数解释](#参数解释-2) 42 | - [部分log节选](#部分log节选-2) 43 | - [向我报bug、提需求](#向我报bug提需求) 44 | # Gazelle r种增强脚本 45 | 本脚本集合主要目的是增强gazelle站里r种刷流的体验 46 | 47 | 目前支持海豚、red、ops、毒蛇 48 | 49 | ## 功能 50 | 有以下主要功能 51 | * 种子过滤。对irssi或者别的方式得到的种子进行个性化过滤，目前支持按体积/发行类别/格式来过滤，并支持根据拉黑列表过滤发布者，对bt客户端没有要求 52 | * 智能使用令牌。根据体积限制对符合要求的种子使用令牌。对bt客户端没有要求。由于违反规则RED/OPS不支持此功能 53 | * 自动拉黑。自动拉黑低分享率种子的发布者，仅支持deluge 54 | * deluge数据导出，方便分析刷流情况 55 | * deluge删除网站上被删种的种子（unregistered torrents） 56 | * 扫描文件夹辅种。对bt客户端没有要求。 57 | 58 | 59 | ## 安装依赖 60 | 61 | 本脚本仅支持python3，所以你首先需要安装一个python3的环境，这个怎么搞自行上网搜索，正确安装在之后你打开命令行输入 62 | ``` 63 | python3 --version 64 | ``` 65 | 之后应该能看到python安装的版本信息 66 | 67 | 之后安装本程序依赖的包（win用户/root用户省略sudo）： 68 | ``` 69 | sudo pip3 install bencode.py ipython requests datasize deluge-client 70 | ``` 71 | 如果没有root权限，可以使用`--user`： 72 | ``` 73 | pip3 install bencode.py ipython requests datasize deluge-client --user 74 | ``` 75 | 或者使用virtualenv等手段（请自行上网查阅） 76 | 77 | ## 下载本脚本 78 | 79 | in case你不知道怎么下载： 80 | ``` 81 | git clone https://github.com/qfishpear/fishrss.git 82 | cd fishrss/ 83 | ``` 84 | 85 | ## 填写配置信息 86 | 87 | 首先你需要将`config.py.example`复制一份为`config.py` 88 | ``` 89 | cp config.py.example config.py 90 | ``` 91 | 然后按照`config.py`里面的提示填写，并创建好所有已填写的文件/文件夹。 92 | 93 | 所有路径可以填写相对路径，但是如果要crontab等方式运行，填写绝对路径更为保险 94 | 95 | 填写路径的时候，如果是Windows，依然建议使用左斜杠`/`而非右斜杠`\`作为路径的分隔符，除非你知道自己在写什么。 96 | 97 | ### 我应该填写哪些信息 98 | 99 | 不同的脚本对于配置中需要的信息不一样，如果你不需要全套的脚本则不需要全部填写，以下信息是必须的（填为None也视为一种填写） 100 | 101 | * 如果要使用海豚，至少需要填写`CONFIG["dic"]`里的`"api_cache_dir"`, `"cookies"`，`"authkey"`, `"torrent_pass"` 102 | * 如果要使用red，至少需要填写`CONFIG["red"]`里的`"api_cache_dir"`，`"authkey"`, `"torrent_pass"`, 而`"cookies"`和`"api_key"`两个要填至少一个，如果不填写`"cookies"`，请保持它被注释掉的状态 103 | * 如果要使用ops，至少需要填写`CONFIG["ops"]`里的`"api_cache_dir"`，`"authkey"`, `"torrent_pass"`, 而`"cookies"`和`"api_key"`两个要填至少一个，如果不填写`"cookies"`，请保持它被注释掉的状态 104 | * 如果要使用毒蛇，至少需要填写`CONFIG["snake"]`里的`"api_cache_dir"`，`"cookies"`，`"authkey"`, `"torrent_pass"` 105 | 106 | 除了种子过滤以外如果要使用其他脚本，强烈建议填写`api_cache_dir`并创建对应文件夹，否则多次运行会反反复复向网站发同样的请求导致运行特别慢。脚本运行后此文件夹下应当生成了若干个json文件，是保存的网站api的缓存。 107 | 108 | 各个脚本所需要的信息我会在对应项目下说明。 109 | 110 | ### 验证config 111 | 为了验证config填没填对，可以运行 112 | ``` 113 | python3 check_config.py 114 | ``` 115 | 来检查，正确填写时，应当输出类似以下内容。当然，检查通过不代表config填写完全正确。 116 | ``` 117 | 2021-04-30 22:17:21,335 - INFO - dic querying action=index 118 | 2021-04-30 22:17:22,118 - INFO - dic logged in successfully, username：fishpear uid: 1132 119 | 2021-04-30 22:17:22,119 - INFO - red querying action=index 120 | 2021-04-30 22:17:22,604 - INFO - red logged in successfully, username：fishpear uid: 50065 121 | 2021-04-30 22:17:22,606 - INFO - ops querying action=index 122 | 2021-04-30 22:17:22,888 - INFO - ops logged in successfully, username：fishpear uid: 21482 123 | 2021-04-30 22:17:22,889 - INFO - snake querying action=index 124 | 2021-04-30 22:17:23,210 - INFO - snake logged in successfully, username：fishpear uid: 10084 125 | 2021-04-30 22:17:23,228 - INFO - deluge is correctly configured 126 | ``` 127 | 128 | ### 如何获取cookie, authkey, torrent_pass 129 | 请参考本repo内[README.rss.md](https://github.com/qfishpear/fishrss/blob/main/README.rss.md)内的相关内容 130 | 131 | ### 如何获取api_key 132 | red：打开你的个人设置，然后看下图： 133 | ![4.png](https://i.loli.net/2021/04/16/1Hzdi3YZpVXBtc9.png) 134 | ops：同样打开个人设置，然后照葫芦画瓢 135 | 136 | ### 怎么编辑配置文件 137 | 138 | 对于完全不知道python的小白，你需要一个正常的文本编辑器，这里推荐sublime: 139 | ``` 140 | https://www.sublimetext.com/ 141 | ``` 142 | 用sublime打开配置文件`config.py`，正常来说会识别这是一个python文件，如果没有，请点击右下角 143 | 144 | ![1.JPG](https://i.loli.net/2021/04/16/WMzGr3hAan5kc4m.jpg) 145 | 146 | 并手动选择python 147 | 148 | ![2.JPG](https://i.loli.net/2021/04/16/7JHglirA2sMzemW.jpg) 149 | 150 | 这样sublime就会对代码进行语法高亮，对于你填的格式错误会给予提示 151 | 152 | 在sublime里把某个选项填为None之后应该是这样显示的，不需要加引号： 153 | 154 | ![3.JPG](https://i.loli.net/2021/04/16/vuaDwFWLEdfVOrp.jpg) 155 | ### 怎么对一段代码添加注释/去除注释 156 | python里，注释的意思是在一行代码前面添加井号# 157 | 158 | 在sublime里，如果要注释掉一段代码，或者取消一整段代码的注释，请选中这一段代码并按快捷键`ctrl+/`，其中`/`是问号那个键 159 | 160 | ## 种子过滤与智能令牌`filter.py` 161 | 简单来说本脚本的功能是监控`source_dir`内的种子，将满足设定条件的种子转存到`dest_dir`中，并根据种子体积限制智能使用令牌。 162 | 163 | ### 警告 164 | * 种子过滤会增加延迟，一些情况下会影响刷流，使用前请考虑清楚。 165 | * 本脚本自动使用token的逻辑不会检查你是否还有token，当你网站token已用光之后，r种依然会继续，此时你将会进入不用令牌r种的状态，请做好心理准备。 166 | 167 | ### 填写配置信息 168 | 169 | * `CONFIG["filter"]`下的所有信息：`"source_dir"`, `"dest_dir"`, `"default_behavior"` 170 | * 对于海豚/red/ops/毒蛇，分别填写对应`CONFIG["dic"/"red"/"ops"/"snake"]`的以下内容： 171 | * `"filter_config"`下的所有信息：`"name"`, `"banlist"`, `"whitelist"`, `"media"`, `"format"`, `"sizelim"` 172 | * `"token_thresh"`（仅dic可以填写） 173 | 174 | ### 运行 175 | 176 | #### 方式1：监控文件夹 177 | ``` 178 | python3 filter.py 179 | ``` 180 | 即可，这个脚本会持续监控指定文件夹，然后将满足过滤条件的种子保存到另一个指定文件夹下。监控文件夹下的种子被检查后会被删除。 181 | 182 | 为了缓解黑屏焦虑症，这个脚本每分钟会输出一行"tick" 183 | 184 | 当你修改`config.py`之后，要重新运行，请按`ctrl-c`掐掉原来运行的`filter.py`，再重新运行。 185 | 186 | 例子：用监控文件夹时 config.py, autodl, deluge分别填写的监控目录： 187 | ![1.JPG](https://i.loli.net/2021/04/20/rzybIaVcXJTw5AR.jpg) 188 | ![2.JPG](https://i.loli.net/2021/04/20/xMldRSFw1A4qs8u.jpg) 189 | ![3.JPG](https://i.loli.net/2021/04/20/x6TZYJGXidgSjOQ.jpg) 190 | 191 | #### 方法2：参数调用 192 | 193 | 栗子： 194 | ``` 195 | python3 filter.py --file ./1.torrent 196 | python3 filter.py --url https://redacted.ch/torrents.php?action=download\&id=xxxx\&authkey=xxxx\&torrent_pass=xxx 197 | ``` 198 | 199 | 注意，这样调用会跳过登陆的检查，所以建议先跑一下`check_config.py`确认配置信息没填错 200 | 201 | 如果要使用irssi： 202 | 修改preference->action，像这样： 203 | 204 | ![1.JPG](https://i.loli.net/2021/04/21/g9dPteW3ciMmKUj.jpg) 205 | 206 | 上面填python可执行文件的路径，下面填（如果路径有空格或者一些奇怪字符的话必须像这样加上引号，没有的话不加也行） 207 | ``` 208 | "/absolute/path/to/filter.py" --file "$(TorrentPathName)" 209 | ``` 210 | 另外，填这样也是可以的，但会略慢于上面的： 211 | ``` 212 | "/absolute/path/to/filter.py" --url $(TorrentUrl) 213 | ``` 214 | 215 | 如果要用irssi这样使用本脚本，配置文件内所有路径必须是绝对路径。 216 | 217 | ### 参数解释 218 | 219 | * `--url URL` 从URL添加种子，使用此功能则只运行一次，不再监控文件夹 220 | * `--file FILE` 从文件FILE添加种子，使用此功能则只运行一次，不再监控文件夹 221 | * `--skip-api` 不请求api，注意，如果使用此功能则filter_config中涉及必须api获取的信息(format, media)必须全部是None，否则任何种子都无法通过过滤 222 | * `--no-tick` 不再每分钟输出一行tick 223 | * `--deluge` 直接推送种子到deluge客户端，而不存储到dest_dir 224 | * `--force-accept` 强制接受，忽略filter_config内填的任何内容 225 | 226 | ### 部分log节选 227 | 隐私已去除 228 | ``` 229 | 2021-04-12 18:28:37,392 - INFO - api and filter of RED are set 230 | 2021-04-12 18:28:37,392 - INFO - api and filter of DIC are set 231 | ...... 232 | 2021-04-12 19:04:14,717 - INFO - new torrent from red: xxxxxxxxxxxxxxxxx 233 | 2021-04-12 19:04:14,718 - INFO - red querying action=torrent&id=xxxxxxxxxxxxxxxxx 234 | 2021-04-12 19:04:15,465 - INFO - redfilter: checking 235 | 2021-04-12 19:04:15,465 - INFO - uploader: xxxxxxxxxxxxxxxxx media: CD format: FLAC releasetype: 9 size: 114.9MB 236 | 2021-04-12 19:04:15,465 - INFO - reject: banned user 237 | 2021-04-12 19:04:22,676 - INFO - tick 238 | 2021-04-12 19:04:52,522 - INFO - new torrent from red: xxxxxxxxxxxxxxxxx 239 | 2021-04-12 19:04:52,523 - INFO - red querying action=torrent&id=xxxxxxxxxxxxxxxxx 240 | 2021-04-12 19:04:53,428 - INFO - redfilter: checking 241 | 2021-04-12 19:04:53,428 - INFO - uploader: xxxxxxxxxxxxxxxxx media: CD format: FLAC releasetype: 1 FLAC size: 224.7MB 242 | 2021-04-12 19:04:53,429 - INFO - accept 243 | 2021-04-12 19:05:23,673 - INFO - tick 244 | 2021-04-12 19:06:23,760 - INFO - tick 245 | 2021-04-12 19:07:23,858 - INFO - tick 246 | 2021-04-12 19:08:23,944 - INFO - tick 247 | 2021-04-12 19:09:24,031 - INFO - tick 248 | ``` 249 | 250 | ## 自动拉黑`autoban.py` 251 | 本脚本会读取deluge里种子的信息，将满足设定条件的发种人添加到`CONFIG["red"/"ops"/"dic"/"snake"]["filter_config"]["banlist"]`文件内 252 | 253 | 此脚本和`filter.py`分别独立运行，二者只通过banlist文件进行交互。 254 | 255 | ### 拉黑规则 256 | 257 | 以下为`config["red"/"ops"/"dic"]["autoban"]`下的内容： 258 | 259 | * 只统计种子完成度大于`autoban["ignore"]["min_progress"]`，且距离发种时间少于`autoban["ignore"]["max_time_added"]`的种子 260 | * 对于`autoban["ratio"]`下的任意一个条目：如果某发种人发种数量不少于`"count"`且总ratio低于`"ratiolim"`，则ban掉此发种人 261 | 262 | ### 填写配置信息 263 | 264 | * 要对red/ops/dic进行自动拉黑，则`CONFIG["red"/"ops"/"dic"/"snake"]["filter_config"]["banlist"]`必须已经填写 265 | * `CONFIG["deluge"]`下的所有信息：`"ip"`, `"port"`, `"username"`, `"password"`。其中ip和port应当和connection manager下的信息一致。username和password是deluge登陆webui所输的账号和密码，如果你登录的时候不需要输，可能可以随便填（关于这一点我也不是很确定）。
266 | ![5.JPG](https://i.loli.net/2021/04/16/ZBVay3rjhCPK6Ui.jpg) 267 | * `config["red"/"ops"/"dic"/"snake"]["autoban"]`下的所有信息 268 | 269 | ### 运行 270 | 第一次运行请加个参数`--init` 271 | ``` 272 | python3 autoban.py --init 273 | ``` 274 | 之后每次运行只需要运行 275 | ``` 276 | python3 autoban.py 277 | ``` 278 | 这个只会运行一次，要持续拉黑请自己设定定时运行，如crontab/watch/Windows定时任务等，以watch为例： 279 | 280 | 样例：每2分钟运行一次 281 | ``` 282 | watch -n 120 python3 autoban.py 283 | ``` 284 | 285 | 注意，`autoban.py`运行时会去请求已配置信息的网站获取种子信息，受限于api频率限制第一次运行可能会比较慢 286 | 287 | ### 参数解释 288 | * `--init`：如果**不**添加，那么默认情况下如果一个发种人最近1小时没有发种且没有未完成种子，则无视他。添加之后则会对有满足`"ignore"`过滤条件种子的全部发种人进行自动拉黑。默认情况下的目的是为了防止一些发种人因为`max_time_added`带来的滑动窗口而被ban掉。 289 | * `--site`：可设置为red/ops/dic，只运行指定站点的拉黑逻辑。 290 | * `--stats`：输出统计信息，如果你对ban人的情况有疑问的话可以看一看，部分log节选（隐私已隐藏）： 291 | ``` 292 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 3 ratio: 1.030 0.678GB/0.658GB 293 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 1 ratio: 0.506 0.218GB/0.430GB 294 | 2021-04-17 09:56:00,777 - INFO - uploader: xxxxxxxxx #torrents: 13 ratio: 0.842 0.658GB/0.781GB 295 | 2021-04-17 09:56:00,778 - INFO - uploader: xxxxxxxxx #torrents: 67 ratio: 0.924 7.557GB/8.180GB 296 | ``` 297 | 298 | ### 部分log节选 299 | 隐私已去除 300 | ``` 301 | 2021-04-14 11:10:17,372 - INFO - autoban: deluge is connected: True 302 | 2021-04-14 11:10:19,876 - INFO - new user banned: ********* #torrents: 1 ratio: 0.000 0.000GB/0.141GB 303 | 2021-04-14 11:10:19,876 - INFO - related torrents: megane panda (眼鏡熊猫) - Natsu No Machi EP (夏の街EP) (2015) [WEB] [FLAC] ratio: 0.000 0.0MB/144.7MB 304 | 2021-04-14 11:10:19,877 - INFO - 39 user banned in total 305 | ``` 306 | 307 | ## deluge数据导出`gen_stats.py` 308 | 当你配置好自动拉黑里所说的中deluge的信息和网站api后，直接运行 309 | ``` 310 | python3 gen_stats.py > stats.txt 311 | ``` 312 | 你就会在文本文件`stats.txt`中得到一个数据表格，里面不仅包含deluge的信息，还有种子从网站api内获取的信息 313 | 314 | 这个表格是用tab分割的，所以你只要把文件内容全选复制到excel里即可进行你需要的数据分析 315 | 316 | 注意，`gen_stats.py`运行时会去请求已配置信息的网站获取种子信息，受限于api频率限制第一次运行可能会比较慢 317 | 318 | ## deluge删除网站上被删种的种子`remove_unregistered.py` 319 | 320 | ### 警告 321 | 此脚本会删除deluge内的种子和**文件**，请确认功能后使用！ 322 | 323 | ### 功能 324 | 删除所有deluge里还未下完的、且"Tracker Status"中含有"Unregistered torrent"字样的种子，**以及文件**。 325 | 326 | 注意，如果red/dic/ops/毒蛇之外的种子里的tracker回复了这条信息一样会被删除。 327 | 328 | ### 运行 329 | 330 | 当配置好自动拉黑中所说的和deluge相关的部分后，直接运行 331 | ``` 332 | python3 remove_unregistered.py 333 | ``` 334 | 这个只会运行一次，要持续删种请自己设定定时运行 335 | 336 | 部分log节选： 337 | ``` 338 | 2021-04-14 11:02:13,526 - INFO - remove_unregistered: deluge is connected: True 339 | 2021-04-14 11:02:13,582 - INFO - removing torrent "Atomic Kitten - Feels So Good (2002) [7243 5433722 2]" reason: "xxxxxxxx.xxxx: Error: Unregistered torrent" 340 | 2021-04-14 11:02:13,974 - INFO - removing torrent "VA-Clap-(COUD_11)-12INCH_VINYL-FLAC-199X-YARD" reason: "flacsfor.me: Error: Unregistered torrent" 341 | 2021-04-14 11:02:14,533 - INFO - removing torrent "Headnodic - Tuesday (2002) - WEB FLAC" reason: "flacsfor.me: Error: Unregistered torrent" 342 | ``` 343 | ## 扫描文件夹辅种`reseed.py` 344 | 本脚本会扫描给定文件夹下的所有子文件夹，在指定站点里搜索所有这些文件夹可以辅种的种子。 345 | 346 | 本脚本也可以扫描给定文件夹内的所有种子，并寻找指定站点内对应的可以辅种的种子。 347 | 348 | 注意：由于自动化的搜索方式不能面面俱到，不代表所有搜到的种子都可以正确辅种，也不代表所有没搜到的文件夹无法辅种。少数搜索到的种子可能出现文件名重命名以及其他不可预知的问题，辅种时请在添加前关闭自动开始下载。 349 | 350 | ### 填写配置信息 351 | 除了必填信息以外不需要填写额外的信息。 352 | 353 | ### 运行 354 | 355 | 运行到一半时退出，下次重新执行的时候会跳过上次扫描过的文件夹，如果不想跳过想重新扫描，请删除`--result-dir`所指定的文件夹下的`scan_history.txt`文件。 356 | 357 | 请注意路径名里带空格或者其他奇怪字符的时候要进行转义，推荐使用tab输入这些长目录。 358 | 359 | #### 方法1：扫描所有子文件夹 360 | 361 | 假如你的音乐文件存在文件夹`~/downloads`下，你要为海豚辅种，辅种的所有结果存在文件夹`~/results`下，则运行 362 | ``` 363 | python3 reseed.py --site dic --dir ~/downloads --result-dir ~/results 364 | ``` 365 | 即可。注意`--result-dir`所带的文件夹必须已经创建好。 366 | 367 | #### 方法2：扫描单个文件夹 368 | 369 | 你也可以用`--single-dir`为单个文件夹辅种 370 | 371 | 样例：假如你有一个下好了的种子存在`~/downloads/Masashi Sada (さだまさし) - さだ丼～新自分風土記III～ (2021) [24-96]/ `里，则运行： 372 | ``` 373 | python3 reseed.py --site dic --single-dir ~/downloads/Masashi\ Sada\ \(さだまさし\)\ -\ さだ丼～新自分風土記III～\ \(2021\)\ \[24-96\]/ --result-dir ~/results 374 | ``` 375 | 376 | #### 方法3：扫描种子文件夹 377 | 378 | `--torrent-dir`为文件夹内所有种子文件搜索站点内可辅种的种子 379 | 380 | 样例：假如有若干种子存在`~/torrents`里，则运行 381 | ``` 382 | python3 reseed.py --site dic --torrent-dir ~/torrents --result-dir ~/results 383 | ``` 384 | 385 | #### 方法4：扫描单个种子 386 | 387 | `--single-torrent`为单个种子文件搜索站点内可辅种的种子 388 | 389 | 样例：假如有一个种子`~/torrents/The Call - Collected - 2019 (CD - FLAC - Lossless).torrent`，则运行 390 | 391 | ``` 392 | python3 reseed.py --site dic --single-torrent ~/torrents/The\ Call\ -\ Collected\ -\ 2019\ \(CD\ -\ FLAC\ -\ Lossless\).torrent --result-dir ~/results 393 | ``` 394 | 395 | ### 辅种结果 396 | 397 | `--result-dir`所指定的文件夹下会生成： 398 | * `torrent/`，一个文件夹，里面存了所有下载的.torrent文件，所有.torrent文件的文件名为 "辅种文件夹名.torrent"。注：利用这个命名方式可以方便辅那种只改了文件夹名的种子。 399 | * `result_mapping.txt`，辅种结果，每行为一个tab（制表符）隔开的文件夹名和种子的torrentid，如果没扫到的话torrentid会填为-1。 400 | * `result_url.txt`，每行一个可以辅种的种子的下载链接 401 | * `result_url_undownloaded.txt`，由于流控，部分情况下下载.torrent文件会失败，这些失败了的种子的下载链接会放在这个文件里。 402 | * `scan_history.txt`，曾经扫描过的文件夹，一行一个，你如果运行了一半意外退出了的话，重新运行会跳过这里面记录的扫描过的文件夹 403 | 404 | ### 参数解释 405 | * `--dir` 批量辅种所在音乐文件所在的总文件夹 406 | * `--single-dir` 扫描单个种子辅种时的音乐文件所在文件夹 407 | * `--torrent-dir` 批量扫描种子文件辅种时所有.torrent文件所在文件夹 408 | * `--single-torrent` 扫描单个种子文件辅种的种子路径 409 | * `--site` 对于海豚/RED/OPS/毒蛇分别填dic/red/ops/snake（小写） 410 | * `--result-dir` 存储扫描出的辅种信息的文件夹 411 | * `--api-frequency` api调用频率限制。如果你还有其他使用api的脚本在持续运行，为了不超过api频率限制你可以手动指定api调用频率。单位：次每10秒。 412 | * `--no-download` 只保存辅种信息，不下载.torrent文件 413 | 414 | ### 部分log节选 415 | ``` 416 | fishpear@sea:~/rss$ python3 reseed.py --site dic --dir ~/downloads --result-dir ~/results 417 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/scan_history.txt 418 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/result_url.txt 419 | 2021-04-25 16:22:16,799 - INFO - file automatically created: /home7/fishpear/results/result_mapping.txt 420 | 2021-04-25 16:22:16,800 - INFO - directory automatically created: /home7/fishpear/results/torrents/ 421 | 2021-04-25 16:22:16,800 - INFO - file automatically created: /home7/fishpear/results/result_url_undownloaded.txt 422 | 2021-04-25 16:22:16,800 - INFO - dic querying action=index 423 | 2021-04-25 16:22:17,588 - INFO - dic logged in successfully, username：fishpear uid: 1132 424 | 2021-04-25 16:22:17,632 - INFO - 1797/1797 unscanned folders found in /home7/fishpear/downloads, start scanning for cross-seeding dic 425 | 2021-04-25 16:22:17,632 - INFO - 1/1797 /home7/fishpear/downloads/Eliane Radigue - Backward 426 | 2021-04-25 16:22:17,632 - INFO - dic querying filelist=3+Songs+Of+Milarepa+1+2+Remastered+2021+flac&action=browse 427 | 2021-04-25 16:22:18,329 - INFO - not found 428 | ... 429 | 2021-04-25 16:22:19,711 - INFO - 4/1797 /home7/fishpear/downloads/55 Schubert and Boccherini String Quintets 430 | 2021-04-25 16:22:19,711 - INFO - dic querying filelist=03+Quintet+in+C+Major+for+Two+Violins+Viola+and+Two+Cellos+D+956+III+Scherzo+Presto+Trio+Andante+sostenuto+flac&action=browse 431 | 2021-04-25 16:22:20,406 - INFO - found, torrentid=49506 432 | 2021-04-25 16:22:21,517 - INFO - saving to /home7/fishpear/results/torrents/55 Schubert and Boccherini String Quintets.torrent 433 | ... 434 | ``` 435 | 436 | ## 向我报bug、提需求 437 | 438 | 欢迎向我报bug： 439 | * 请提供log文件（默认为`filter.log`）和相关截图以便分析问题所在 440 | 441 | 欢迎向我提需求： 442 | * 所有涉及到筛选的逻辑（选种，ban人，使用令牌）我写的比较简略，如果有其他需求可以告诉我，不过选种部分的筛选逻辑其实许多完全可以在irssi里筛过了，所以irssi能完成的筛选逻辑就不要提需求了 443 | * 数据导出更多字段，有什么字段你觉得有必要导出的可以告诉我 444 | -------------------------------------------------------------------------------- /README.rss.md: -------------------------------------------------------------------------------- 1 | # rss脚本 2 | 3 | ## 功能简述 4 | * 访问api `https://海豚/ajax.php?action=notifications`来获取r种的种子id，访问api需要cookie鉴权 5 | * 根据种子id和设定的`AUTHKEY`和`TORRENT_PASS`来生成种子链接并下载 6 | * 下载前会检查种子的体积，如果体积大于设定的大小`FL_THRESHOLD`则生成的种子链接中会添加`&usetoken=1`以使用token 7 | * 下载下来的种子会存在设定的文件夹`DOWNLOAD_DIR`内，你的bt客户端应当监控这个目录以实现自动下载，本脚本没有其他任何和bt客户端交互的逻辑 8 | * 程序会记录已经下载过的种子链接在文件`DOWNLOADED_URLS`里，不会重复下载 9 | * 以上功能运行一次代码只会跑一遍，如需持续监控rss请自行配置定时运行，如crontab/watch 10 | 11 | ## 安装依赖 12 | 只支持python3 13 | ``` 14 | pip3 install bencode.py requests 15 | ``` 16 | 17 | ## 填写信息 18 | ### COOKIES 19 | 首先是cookie，注意我这里要求填写的是已经解码为key-value形式的cookie，即你需要填写cookie里`PHPSESSID`和`session`的值，而非编码后放在一起的那个字符串 20 | ``` 21 | COOKIES = {"PHPSESSID": "xxxxxxxxxxxxxxxx", 22 | "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} 23 | ``` 24 | 怎么找到网站上的cookie有多种方式，这里推荐一个chrome插件editthiscookie 25 | ``` 26 | https://chrome.google.com/webstore/detail/editthiscookie/fngmhnnpilhplaeedifhccceomclgfbg?hl=zh-CN 27 | ``` 28 | 安装完此插件之后，打开任意海豚的网页，点击editthiscookie的图标，然后按照下图方式复制cookie 29 | ![](https://i.loli.net/2021/04/13/hcXIKgVbr5mHuED.png) 30 | 31 | ### AUTHKEY, TORRENT_PASS 32 | 这个你去网站里复制一下下载链接即可，里面有 33 | ``` 34 | AUTHKEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 35 | TORRENT_PASS = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 36 | ``` 37 | 38 | ### FL_THRESHOLD 39 | 逻辑：如果体积大于设定的大小`FL_THRESHOLD`则生成的种子链接中会添加`&usetoken=1`以使用token，单位bytes 40 | ``` 41 | FL_THRESHOLD = 100 * 1024**3 # 100GB 42 | ``` 43 | 建议先填个大的值，运行一遍代码，再填你实际需要的体积限制，这个是因为如果之前rss列表里有种子，那么你可能会在上面浪费令牌，而跑过一遍之后这些种子都已经被记录过了，就不会重复下载了 44 | 45 | ### DOWNLOAD_DIR, DOWNLOADED_URLS 46 | DOWNLOAD_DIR 应当填写为你bt客户端监控下载的目录，DOWNLOADED_URLS可以不用改 47 | ``` 48 | DOWNLOAD_DIR = "./watch/" 49 | DOWNLOADED_URLS = "./downloaded_urls.txt" 50 | ``` 51 | 52 | ## 运行代码 53 | ``` 54 | python3 rss.py 55 | ``` 56 | 正确运行时的部分log（隐私已略去）： 57 | ``` 58 | 2021-04-13 10:38:27,382 - INFO - Starting new HTTPS connection (1): xxxxxxxx.xxxx 59 | 2021-04-13 10:38:28,474 - INFO - downloaded file doesn't exist, create new one: ./downloaded_urls.txt 60 | 2021-04-13 10:38:28,474 - INFO - torrent directory doesn't exist, create new one: ./watch/ 61 | 2021-04-13 10:38:28,474 - INFO - 50 torrents in rss result 62 | 2021-04-13 10:38:28,475 - INFO - download https://xxxxxxxx.xxxx/torrents.php?action=download&id=49132&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 63 | 2021-04-13 10:38:28,476 - INFO - Starting new HTTPS connection (1): xxxxxxxx.xxxx 64 | 2021-04-13 10:38:29,563 - INFO - hash=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 65 | 2021-04-13 10:38:29,565 - INFO - download https://xxxxxxxx.xxxx/torrents.php?action=download&id=49131&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 66 | 2021-04-13 10:38:29,566 - INFO - Starting new HTTPS connection (1): xxxxxxxx.xxxx 67 | 2021-04-13 10:38:30,631 - INFO - hash=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 68 | 2021-04-13 10:38:30,632 - INFO - download https://xxxxxxxx.xxxx/torrents.php?action=download&id=49130&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 69 | 2021-04-13 10:38:32,073 - INFO - 3 torrents added 70 | ``` 71 | 此时watch文件夹里有： 72 | ``` 73 | fishpear@sea:~/rss/tmp/watch$ ls 74 | Proc Fiskal - Lothian Buses (2021) [24B-44.1Khz].torrent 75 | Raphael Saadiq - The Way I See It.torrent 76 | Taylor Swift - Fearless - Taylor's Version (2021) {Target Limited Edition} [FLAC].torrent 77 | ``` 78 | downloaded_urls.txt里有（隐私已略去）： 79 | ``` 80 | fishpear@sea:~/rss/tmp$ cat downloaded_urls.txt 81 | downloaded urls: 82 | https://xxxxxxxx.xxxx/torrents.php?action=download&id=49132&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 83 | https://xxxxxxxx.xxxx/torrents.php?action=download&id=49131&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 84 | https://xxxxxxxx.xxxx/torrents.php?action=download&id=49130&authkey=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx&torrent_pass=xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 85 | ``` 86 | 87 | ## 定时运行 88 | 首先老生常谈crontab显然是可以用的，用法自己查，但是坏处就是只能精确到分钟。 89 | 90 | 那我想每30秒r一次怎么搞呢？ 其实linux有个很简单的命令：watch，以下命令表示30秒运行一次 91 | ``` 92 | watch -n 30 python3 rss.py 93 | ``` 94 | 注意，你这个时间间隔不能设的太小不然没有意义，因为海豚服务器本身就很卡请爱护服务器，另外api的话有2秒一次的限制不能高于这个限制不然会被z酱打屁股（大雾）。 -------------------------------------------------------------------------------- /autoban.py: -------------------------------------------------------------------------------- 1 | """ 2 | red自动ban人脚本，仅支持deluge 3 | """ 4 | import os 5 | import traceback 6 | import deluge_client 7 | import time 8 | import base64 9 | import json 10 | import urllib 11 | import argparse 12 | from IPython import embed 13 | from collections import defaultdict 14 | 15 | from config import CONFIG 16 | import common 17 | from common import logger, flush_logger, SITE_CONST 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('--stats', action='store_true', default=False, 21 | help="show stats of all the uploaders") 22 | parser.add_argument('--init', action='store_true', default=False, 23 | help="run as initialization. " 24 | "if not set, the autoban logic will ONLY ban an uploader if one of his uploaded torrents " 25 | "is active. Here \"active\" is defined by being uploaded in an hour or not completed.") 26 | parser.add_argument('--site', default=None, choices=SITE_CONST.keys(), 27 | help="if set, only update the banlist of the specified site.") 28 | args = parser.parse_args() 29 | 30 | DELUGE = CONFIG["deluge"] 31 | client = deluge_client.DelugeRPCClient(DELUGE["ip"], DELUGE["port"], DELUGE["username"], DELUGE["password"]) 32 | client.connect() 33 | logger.info("autoban: deluge is connected: {}".format(client.connected)) 34 | assert client.connected 35 | 36 | now = time.time() 37 | 38 | # 1. 从deluge获取必要信息 39 | tlist = list(client.core.get_torrents_status({}, ["hash", "ratio", "progress", "total_size", "time_added", "comment"]).values()) 40 | logger.info("{} torrents in deluge".format(len(tlist))) 41 | 42 | def work(site): 43 | banlist = CONFIG[site]["filter_config"]["banlist"] 44 | api = common.get_api(site) 45 | AUTOBAN = CONFIG[site]["autoban"] 46 | 47 | # 2. 按uploader分类 48 | uploaders = defaultdict(lambda: list()) 49 | for t in tlist: 50 | comment = t[b"comment"].decode("utf-8") 51 | if common.get_domain_name_from_url(comment) != SITE_CONST[site]["domain"]: 52 | continue 53 | tid = common.get_params_from_url(comment)["torrentid"] 54 | js = api.query_tid(tid) 55 | if js["status"] == "success": 56 | uploaders[js["response"]["torrent"]["username"]].append(t) 57 | uploaders = list(uploaders.items()) 58 | uploaders.sort(key=lambda kv: max([t[b"time_added"] for t in kv[1]])) 59 | 60 | # 3. 根据规则添加ban人 61 | with open(banlist, "r") as f: 62 | bannedusers = set([line.strip() for line in f]) 63 | for uploader, torrents in uploaders: 64 | # 根据规则忽略掉一些种子： 65 | counted_tlist = [] 66 | for t in torrents: 67 | if t[b"progress"] / 100 >= AUTOBAN["ignore"]["min_progress"] and \ 68 | now - t[b"time_added"] < AUTOBAN["ignore"]["max_time_added"]: 69 | counted_tlist.append(t) 70 | if len(counted_tlist) == 0: 71 | continue 72 | total_ul = sum([t[b"total_size"] * t[b"ratio"] * t[b"progress"] / 100 for t in counted_tlist]) 73 | total_size = sum([t[b"total_size"] * t[b"progress"] / 100 for t in counted_tlist]) 74 | ratio = total_ul / total_size 75 | if args.stats: 76 | logger.info("uploader: {} #torrents: {} ratio: {:.3f} {:.3f}GB/{:.3f}GB".format( 77 | uploader, len(counted_tlist), ratio, total_ul / 1024**3, total_size / 1024**3)) 78 | if uploader not in bannedusers: 79 | if not args.init: 80 | # 如果不是作为初始化运行，则忽略最近没有新的活动种子的发种人 81 | # "活动"被定义为还未下载完成，或者添加时间未超过1小时 82 | # 这个功能是为了防止一些人因为"max_time_added"带来的滑动窗口而被ban。 83 | last_complete = max([t[b"time_added"] for t in torrents]) 84 | min_progress = min([t[b"progress"] for t in torrents]) 85 | if min_progress == 100 and now - last_complete > 1 * 60 * 60: 86 | continue 87 | for cond in AUTOBAN["ratio"]: 88 | if len(counted_tlist) >= cond["count"] and ratio < cond["ratiolim"]: 89 | bannedusers.add(uploader) 90 | logger.info("new user banned in {}: {} #torrents: {} ratio: {:.3f} {:.3f}GB/{:.3f}GB".format( 91 | site, uploader, len(counted_tlist), ratio, total_ul / 1024**3, total_size / 1024**3 92 | )) 93 | for t in counted_tlist: 94 | tname = client.core.get_torrent_status(t[b"hash"], ["name"])[b"name"] 95 | logger.info("related torrents: {} ratio: {:.3f} {:.1f}MB/{:.1f}MB".format( 96 | tname.decode("utf-8"), t[b"ratio"], 97 | t[b"ratio"] * t[b"total_size"] * t[b"progress"] / 100 / 1024**2, 98 | t[b"total_size"] * t[b"progress"] / 100 / 1024**2, 99 | )) 100 | break 101 | with open(banlist, "w") as f: 102 | for user in sorted(list(bannedusers)): 103 | if len(user) > 1: 104 | f.write(user+"\n") 105 | logger.info("{}: {} users banned in total".format(site, len(bannedusers))) 106 | 107 | if args.site is None: 108 | for site in SITE_CONST.keys(): 109 | if site in CONFIG.keys() and "autoban" in CONFIG[site].keys(): 110 | common.error_catcher(work, site=site) 111 | else: 112 | common.error_catcher(work, site=args.site) -------------------------------------------------------------------------------- /check_config.py: -------------------------------------------------------------------------------- 1 | import os 2 | from common import get_api, get_filter, logger, SITE_CONST 3 | from config import CONFIG 4 | import traceback 5 | 6 | def check_path(path, name): 7 | if path is not None: 8 | if not os.path.exists(path): 9 | logger.warning("{} does NOT exist, please create:{}".format(name, path)) 10 | 11 | # filter 12 | check_path(CONFIG["filter"]["source_dir"], "source_dir") 13 | check_path(CONFIG["filter"]["dest_dir"], "dest_dir") 14 | # 网站 15 | configured_sites = {} 16 | for site in SITE_CONST.keys(): 17 | if site not in CONFIG.keys(): 18 | logger.warning("{} is not configured".format(site)) 19 | else: 20 | check_path(CONFIG[site]["filter_config"]["banlist"], "{}'s banlist".format(site)) 21 | check_path(CONFIG[site]["filter_config"]["whitelist"], "{}'s whitelist".format(site)) 22 | if "autoban" in CONFIG[site].keys(): 23 | if CONFIG[site]["filter_config"]["banlist"] is None: 24 | logger.warning("{} set \"autoban\" but without banlist") 25 | try: 26 | api = get_api(site) 27 | except: 28 | logger.warning("{} fail to login".format(site)) 29 | logger.info(traceback.format_exc()) 30 | try: 31 | f = get_filter(site) 32 | except: 33 | logger.info("error in {}'s filter_config".format(site)) 34 | logger.info(traceback.format_exc()) 35 | # bt客户端 36 | try: 37 | import deluge_client 38 | DELUGE = CONFIG["deluge"] 39 | de = deluge_client.DelugeRPCClient(DELUGE["ip"], DELUGE["port"], DELUGE["username"], DELUGE["password"]) 40 | de.connect() 41 | assert(de.connected) 42 | logger.info("deluge is correctly configured") 43 | except: 44 | logger.info("can't connect to deluge") 45 | # logger.info(traceback.format_exc()) 46 | -------------------------------------------------------------------------------- /common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import logging 4 | import traceback 5 | import base64 6 | import hashlib 7 | import urllib 8 | import bencode 9 | 10 | # fix gbk problem on Windows 11 | import _locale 12 | _locale._getdefaultlocale = (lambda *args: ['en_US', 'utf8']) 13 | 14 | from torrent_filter import TorrentFilter 15 | from gzapi import REDApi, DICApi, OPSApi, SnakeApi 16 | from config import CONFIG 17 | 18 | # logger相关 19 | LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" 20 | logger = logging.getLogger("logger") 21 | logger.setLevel(logging.INFO) 22 | log_stream = open(CONFIG["log_file"], "a") 23 | filehandler = logging.StreamHandler(log_stream) 24 | filehandler.formatter = logging.Formatter(fmt=LOG_FORMAT) 25 | filehandler.setLevel(logging.INFO) 26 | logger.addHandler(filehandler) 27 | consoleHandler = logging.StreamHandler() 28 | consoleHandler.setFormatter(logging.Formatter(fmt=LOG_FORMAT)) 29 | logger.addHandler(consoleHandler) 30 | def flush_logger(): 31 | log_stream.flush() 32 | os.fsync(log_stream) 33 | sys.stdout.flush() 34 | 35 | # 常数 36 | SITE_CONST = { 37 | "dic":{ 38 | "domain": "dicmusic.club", 39 | "tracker": "tracker.dicmusic.club", 40 | "source": "DICMusic", 41 | }, 42 | "red":{ 43 | "domain": "redacted.ch", 44 | "tracker": "flacsfor.me", 45 | "source": "RED", 46 | }, 47 | "ops":{ 48 | "domain": "orpheus.network", 49 | "tracker": "home.opsfet.ch", 50 | "source": "OPS", 51 | }, 52 | "snake":{ 53 | "domain": "snakepop.art", 54 | "tracker": "announce.snakepop.art", 55 | "source": "Snakepop", 56 | }, 57 | } 58 | # api/filter相关 59 | def get_api(site, **kwargs): 60 | assert site in CONFIG.keys(), "no configuration of {} found".format(site) 61 | assert site in SITE_CONST.keys(), "unsupported site: {}".format(site) 62 | if site == "red": 63 | RED = CONFIG["red"] 64 | api = REDApi( 65 | apikey=RED["api_key"], 66 | cookies=RED["cookies"] if "cookies" in RED.keys() else None, 67 | logger=logger, 68 | cache_dir=RED["api_cache_dir"], 69 | timeout=CONFIG["requests_timeout"], 70 | authkey=RED["authkey"], 71 | torrent_pass=RED["torrent_pass"], 72 | **kwargs, 73 | ) 74 | elif site == "dic": 75 | DIC = CONFIG["dic"] 76 | api = DICApi( 77 | cookies=DIC["cookies"], 78 | logger=logger, 79 | cache_dir=DIC["api_cache_dir"], 80 | timeout=CONFIG["requests_timeout"], 81 | authkey=DIC["authkey"], 82 | torrent_pass=DIC["torrent_pass"], 83 | **kwargs, 84 | ) 85 | elif site == "ops": 86 | OPS = CONFIG["ops"] 87 | api = OPSApi( 88 | apikey=OPS["api_key"], 89 | cookies=OPS["cookies"] if "cookies" in OPS.keys() else None, 90 | logger=logger, 91 | cache_dir=OPS["api_cache_dir"], 92 | timeout=CONFIG["requests_timeout"], 93 | authkey=OPS["authkey"], 94 | torrent_pass=OPS["torrent_pass"], 95 | **kwargs, 96 | ) 97 | elif site == "snake": 98 | SNAKE = CONFIG["snake"] 99 | api = SnakeApi( 100 | cookies=SNAKE["cookies"], 101 | logger=logger, 102 | cache_dir=SNAKE["api_cache_dir"], 103 | timeout=CONFIG["requests_timeout"], 104 | authkey=SNAKE["authkey"], 105 | torrent_pass=SNAKE["torrent_pass"], 106 | **kwargs, 107 | ) 108 | 109 | return api 110 | def get_filter(site): 111 | assert site in CONFIG.keys(), "no configuration of {} found".format(site) 112 | assert site in SITE_CONST.keys(), "unsupported site: {}".format(site) 113 | f = TorrentFilter( 114 | config=CONFIG[site]["filter_config"], 115 | logger=logger, 116 | ) 117 | return f 118 | 119 | # 杂项，通用函数 120 | def get_domain_name_from_url(url): 121 | return urllib.parse.urlparse(url).netloc 122 | def get_params_from_url(url): 123 | return dict(urllib.parse.parse_qsl(urllib.parse.urlparse(url).query)) 124 | def get_info_hash(torrent): 125 | info = torrent["info"] 126 | info_raw = bencode.encode(info) 127 | sha = hashlib.sha1(info_raw) 128 | info_hash = sha.hexdigest() 129 | return info_hash 130 | def get_torrent_site(torrent): 131 | if "source" not in torrent["info"].keys(): 132 | source = None 133 | else: 134 | source = torrent["info"]["source"] 135 | for site, sinfo in SITE_CONST.items(): 136 | if source == sinfo["source"]: 137 | return site 138 | return "unknown" 139 | def get_url_site(url): 140 | domain_name = get_domain_name_from_url(url) 141 | for site, sinfo in SITE_CONST.items(): 142 | if domain_name == sinfo["domain"]: 143 | return site 144 | return "unknown" 145 | def get_tracker_site(url): 146 | domain_name = get_domain_name_from_url(url) 147 | for site, sinfo in SITE_CONST.items(): 148 | if domain_name == sinfo["tracker"]: 149 | return site 150 | return "unknown" 151 | def error_catcher(func, *args, **kwargs): 152 | try: 153 | func(*args, **kwargs) 154 | except KeyboardInterrupt: 155 | logger.info(traceback.format_exc()) 156 | exit(0) 157 | except Exception as _: 158 | logger.info(traceback.format_exc()) 159 | pass -------------------------------------------------------------------------------- /config.py.en.example: -------------------------------------------------------------------------------- 1 | KB = 1024 2 | MB = 1024**2 3 | GB = 1024**3 4 | TB = 1024**4 5 | SECOND = 1 6 | MINUTE = 60 7 | HOUR = 60 * 60 8 | DAY = 24 * 60 * 60 9 | """ 10 | FILTER_CONFIG_TEMPLATE = { 11 | # filter name 12 | "name": "redfilter", 13 | # a text file of banned uploaders. Each line should contain one username. 14 | # if you don't want to ban anyone, let it be "None" 15 | # it will be re-read each time new torrents are found. 16 | "banlist" : "/home7/fishpear/rss/ban_red.txt", 17 | # a text file of whitelisted uploaders. If you don't want to whitelist anyone, let it be "None"。 18 | # notice: this does not indicate a "whitelist" mode, 19 | # it simply bypasses other filter conditions for the whitelisted uploaders. 20 | # it will be re-read each time new torrents are found. 21 | "whitelist" : "/home7/fishpear/rss/white_red.txt", 22 | # accepted media types Must be "None" or a set of string 23 | "media" : set(["CD", "Vinyl", "WEB"]), 24 | # accepted formats. Must be "None" or a set of string 25 | "format": set(["FLAC", "WAV"]), 26 | # range of size. unit: Byte. if you don't want to limit the size, let it be None. Example: 100MB-1024MB 27 | "sizelim": (100 * 1024**2, 1024 * 1024**2), 28 | } 29 | """ 30 | 31 | """ 32 | AUTOBAN_TEMPLATE = { 33 | # conditions to ban users by ratio: 34 | "ratio" : [ 35 | # the following conditions mean: if an uploader has uploaded no less than "count" torrents 36 | # and your ratio is lower than "ratiolim", the uploader will be added to banlist. 37 | # Example: if ratio is lower than 0.2 or the uploader has uploaded 2 torrents and ratio is 38 | # lower than 0.4 or the uploader has uploaded 4 torrents and ratio is lower than 0.55, it will be banned. 39 | {"count":1, "ratiolim":0.2}, 40 | {"count":2, "ratiolim":0.4}, 41 | {"count":4, "ratiolim":0.55}, 42 | ], 43 | # the ratio is only counted for a torrent if it satisfies: 44 | # its progress is no less than "min_progress" and has been added no more than "max_time_added" (unit: Second) 45 | "ignore": { 46 | # Example: only count torrents with at least 30% in progress and uploaded no more than 36 hours: 47 | "min_progress" : 0.3, 48 | "max_time_added" : 36 * HOUR, 49 | }, 50 | } 51 | """ 52 | 53 | # All files and directories should be created by yourself. 54 | CONFIG = { 55 | # the log of all scripts will be here: 56 | "log_file": "./filter.log", 57 | # general configuration of filter 58 | "filter" : { 59 | # the directory to monitor .torrent files. Tools like irssi-autodl should save .torrent files here. 60 | "source_dir" : "./tocheck", 61 | # the filtered torrents will be saved here. This should be the directory monitored by BT clients. 62 | "dest_dir" : "./watch", 63 | # default behavior of a torrent from unconfigured trackers. If you don't want to accept, let it be "reject" 64 | "default_behavior" : "accept", 65 | }, 66 | # Redacted configuration. If you don't want redacted, comment the code between Seperation Line 1 67 | # ========================Seperation Line 1============================= 68 | "red" : { 69 | # a diretory for API request cache. If you don't want it, let it be None 70 | "api_cache_dir" : "~/.cache/fishrss/red", 71 | # cookie/apikey: choose one of them for logging in. The api_key is recommended because 72 | # it halves the API request limitation (5 times per 10 seconds -> 10 times per 10 seconds) 73 | # if you still want to use cookies, uncomment it and let "api_key" to be None 74 | # cookies: see `README.rss.md` for more instructions 75 | # "cookies" : { 76 | # "session": "xxxxxxxxxxxx" 77 | # }, 78 | # api_key: set it in your profile. It is only activated after you save your profile. 79 | "api_key" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 80 | # authkey & torrent_pass: copy a download link of arbitrary torrent, they will be in it 81 | "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 82 | "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 83 | # filter_config: refer to the example and FILTER_CONFIG_TEMPLATE above 84 | "filter_config" : { 85 | "name" : "redfilter", 86 | "banlist" : None, 87 | "whitelist" : None, 88 | "media" : set(["CD"]), 89 | "format" : None, 90 | "sizelim" : (100 * MB, 1024 * MB) 91 | }, 92 | # Staff of RED explicitly explained that using tokens by scripts is against the rule. 93 | # So don't modify "token_thresh". It won't take effect even if you modify it. 94 | "token_thresh": (0, -1), 95 | # autoban: uncomment below if you need autoban script for red. Refer to the example and AUTOBAN_TEMPLATE above 96 | # "autoban" : { 97 | # "ratio" : [ 98 | # {"count":1, "ratiolim":0.2}, 99 | # ], 100 | # "ignore": { 101 | # "min_progress" : 0.3, 102 | # "max_time_added" : 36 * HOUR, 103 | # }, 104 | # }, 105 | }, 106 | # ========================Seperation Line 1============================= 107 | # Dicmusic configuration. If you don't want dicmusic, comment the code between Seperation Line 2(commented by default) 108 | # ========================Seperation Line 2============================= 109 | # "dic":{ 110 | # # a diretory for API request cache. If you don't want it, let it be None 111 | # "api_cache_dir" : "~/.cache/fishrss/dic", 112 | # # cookies: see instruction in `README.rss.md` 113 | # "cookies" : { 114 | # "PHPSESSID" : "xxxxxxxxxxxxxxxxxxxxxxxx", 115 | # "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 116 | # }, 117 | # # authkey & torrent_pass: copy a download link of arbitrary torrent, they will be in it 118 | # "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 119 | # "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 120 | # # filter_config: refer to the example and FILTER_CONFIG_TEMPLATE above 121 | # "filter_config" : { 122 | # "name" : "dicfilter", 123 | # "banlist" : None, 124 | # "whitelist" : None, 125 | # "media" : None, 126 | # "format" : set(["FLAC"]), 127 | # "sizelim" : None, 128 | # }, 129 | # # token_thresh: use token when the torrent size is within the range 130 | # # Example：100MB-2GB. If you don't want to use tokens, let it be an invalid range like (0, -1) 131 | # # Notice: if the tokens are used up, filtered torrents will still be saved to "dest_dir". 132 | # "token_thresh": (100 * MB, 2 * GB), 133 | # # autoban: uncomment below if you need autoban script for dic. Refer to the example and AUTOBAN_TEMPLATE above 134 | # # "autoban" : { 135 | # # "ratio" : [ 136 | # # {"count":1, "ratiolim":0.2}, 137 | # # ], 138 | # # "ignore": { 139 | # # "min_progress" : 0.3, 140 | # # "max_time_added" : 36 * HOUR, 141 | # # }, 142 | # # }, 143 | # }, 144 | # ========================Seperation Line 2============================= 145 | # Orpheus configuration. If you don't want orpheus, comment the code between Seperation Line 3(commented by default) 146 | # ========================Seperation Line 3============================= 147 | # "ops":{ 148 | # # a diretory for API request cache. If you don't want it, let it be None 149 | # "api_cache_dir" : "~/.cache/fishrss/ops", 150 | # # cookie/apikey: choose one of them for logging in. 151 | # # if you want to use cookies, uncomment it and let "api_key" to be None 152 | # # cookies: see `README.rss.md` for more instructions 153 | # # "cookies" : { 154 | # # "session": "xxxxxxxxxxxx" 155 | # # }, 156 | # # api_key: set it in your profile. 157 | # "api_key" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 158 | # # authkey & torrent_pass: copy a download link of arbitrary torrent, they will be in it 159 | # "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 160 | # "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 161 | # # filter_config: refer to the example and FILTER_CONFIG_TEMPLATE above 162 | # "filter_config" : { 163 | # "name" : "opsfilter", 164 | # "banlist" : None, 165 | # "whitelist" : None, 166 | # "media" : None, 167 | # "format" : set(["FLAC"]), 168 | # "sizelim" : (100 * MB, 1 * GB), 169 | # }, 170 | # # Staff of OPS explicitly explained that using tokens by scripts is against the rule. 171 | # # So don't modify "token_thresh". It won't take effect even if you modify it. 172 | # "token_thresh": (0, -1), 173 | # # autoban: uncomment below if you need autoban script for ops. Refer to the example and AUTOBAN_TEMPLATE above 174 | # # "autoban" : { 175 | # # "ratio" : [ 176 | # # {"count":1, "ratiolim":0.2}, 177 | # # ], 178 | # # "ignore": { 179 | # # "min_progress" : 0.3, 180 | # # "max_time_added" : 36 * HOUR, 181 | # # }, 182 | # # }, 183 | # }, 184 | # ========================Seperation Line 3============================= 185 | # deluge api login information 186 | "deluge" : { 187 | "ip": "127.0.0.1", 188 | # this is the port of deluge api but not the port that deluge listens on BT income connections. 189 | # click "connect manager" on webui and it will show this port on "host" 190 | "port": 12345, 191 | "username": "xxxxxxxxxxxxxx", 192 | "password": "xxxxxxxxxxxxxx", 193 | }, 194 | # timeout of requests library's requests, unit: second 195 | # just leave it there if you don't know what it is 196 | "requests_timeout" : 10, 197 | } -------------------------------------------------------------------------------- /config.py.example: -------------------------------------------------------------------------------- 1 | KB = 1024 2 | MB = 1024**2 3 | GB = 1024**3 4 | TB = 1024**4 5 | SECOND = 1 6 | MINUTE = 60 7 | HOUR = 60 * 60 8 | DAY = 24 * 60 * 60 9 | """ 10 | filter_config模板： 11 | FILTER_CONFIG_TEMPLATE = { 12 | # 过滤器名称 13 | "name": "redfilter", 14 | # 存储被ban用户列表的文件，不ban填None，手动编辑此文件时请一行一个用户名 15 | # 每次来新种子的时候会重新读取这个文件，autoban.py和filter.py可以分别独立运行 16 | "banlist" : "/home7/fishpear/rss/ban_red.txt", 17 | # 存储白名单用户列表的文件，不需要填None，手动编辑此文件时请一行一个用户名 18 | # 注意，这个不代表白名单模式，它的意思仅仅是此文件里的用户发的种子会被无条件接受而已 19 | # 每次来新种子的时候会重新读取这个文件 20 | "whitelist" : "/home7/fishpear/rss/white_red.txt", 21 | # 允许的媒体类型，必须是set类型，无限制填None 22 | "media" : set(["CD", "Vinyl", "WEB"]), 23 | # 允许的格式，必须是set类型，无限制填None 24 | "format": set(["FLAC", "WAV"]), 25 | # 体积范围，单位Byte，如果不限制填None，样例：100MB-1024MB 26 | "sizelim": (100 * 1024**2, 1024 * 1024**2), 27 | } 28 | """ 29 | 30 | """ 31 | autoban模板： 32 | AUTOBAN_TEMPLATE = { 33 | # 按ratio ban人的条件 34 | "ratio" : [ 35 | # 以下3个条件表示当某个人发种个数超过count的时候，你的总ratio低于ratiolim则ban掉他 36 | # 样例：ratio低于0.2，或发了至少2个种且ratio低于0.4，或发了4至少4个种且ratio低于0.55，则ban掉他 37 | {"count":1, "ratiolim":0.2}, 38 | {"count":2, "ratiolim":0.4}, 39 | {"count":4, "ratiolim":0.55}, 40 | ], 41 | # 统计ratio时忽略满足以下条件的种子：完成度低于min_progress，或已经发布时间超过max_time_added 42 | "ignore": { 43 | # 样例：只统计至少完成了30%，且时间不超过36小时的种子。 44 | "min_progress" : 0.3, 45 | "max_time_added" : 36 * HOUR, 46 | }, 47 | } 48 | """ 49 | 50 | """ 51 | 种子过滤功能`filter.py`与配置填写相关的逻辑简述： 52 | 太长不看：从source_dir读取种子，满足要求的转存到dest_dir里 53 | 1. 从文件夹CONFIG["filter"]["source_dir"]里读取种子文件，读取完成后删除原种子 54 | 2. 判断种子属于哪个tracker，目前支持red和dic 55 | 3. 根据对应tracker设定的过滤条件CONFIG["red"/"dic"/"ops"]["filter_config"]把满足条件的种子保存到CONFIG["filter"]["dest_dir"]里，不满足条件的会被忽略。 56 | 对于不支持的tracker根据CONFIG["filter"]["default_behavior"]的设定进行保存或忽略。 57 | 文件CONFIG["red"/"dic"/"ops"]["filter_config"]["banlist"]在过滤时实时读取，因此修改此文件之后不需要重新运行`filter.py` 58 | 4. 如果种子符合CONFIG["red"/"dic"/"ops"]["token_thresh"]的体积条件的话，则会使用token 59 | """ 60 | 61 | # 所有填写的文件夹和文件请预先创建好，本程序不会自动创建它们，所有信息没有另外说的都是必填 62 | CONFIG = { 63 | # log文件输出路径，本程序所有输出都会保存一份在这个log里 64 | "log_file": "./filter.log", 65 | # filter的通用设置 66 | "filter" : { 67 | # 从这个文件夹里读取.torrent，此文件夹应为irssi等工具的保存种子的文件夹 68 | "source_dir" : "./tocheck", 69 | # 满足filter条件的种子会被转存到这个文件夹，此文件夹应为bt客户端的监控文件夹 70 | "dest_dir" : "./watch", 71 | # 对于未配置信息的站点r到的种子的行为，默认是接受（即认为是满足条件），如果不要接受请改成"reject" 72 | "default_behavior" : "accept", 73 | }, 74 | # dic 配置信息，如果不需要刷dic请将分割线1内整段注释掉 75 | # ========================分割线1============================= 76 | "dic":{ 77 | # 一个文件夹，保存曾经请求过的api回复，如果不需要可以填None 78 | "api_cache_dir" : "~/.cache/fishrss/dic", 79 | # 关于cookie的填写请参考`README.rss.md`里的描述 80 | "cookies" : { 81 | "PHPSESSID" : "xxxxxxxxxxxxxxxxxxxxxxxx", 82 | "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 83 | }, 84 | # authkey和torrent_pass请随便复制一个种子的下载链接，里面有 85 | "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 86 | "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 87 | # filter_config的填写请参考上面的模板FILTER_CONFIG_TEMPLATE和给出的样例 88 | "filter_config" : { 89 | "name" : "dicfilter", 90 | "banlist" : None, 91 | "whitelist" : None, 92 | "media" : None, 93 | "format" : set(["FLAC"]), 94 | "sizelim" : None, 95 | }, 96 | # 体积在这个区间内时自动使用令牌，样例：100MB-2GB，如果不想使用令牌请填一个非法的区间，比如(0,-1) 97 | # 注意，即使令牌用完，本脚本依然会继续下载种子，此时等于下黑种，请关注令牌余量 98 | "token_thresh": (100 * MB, 2 * GB), 99 | # 自动ban人脚本条件，如果需要对海豚使用该脚本，请取消以下的注释： 100 | # 填写时，请参考上面的模板AUTOBAN_TEMPLATE和给出的样例 101 | # "autoban" : { 102 | # "ratio" : [ 103 | # {"count":1, "ratiolim":0.2}, 104 | # ], 105 | # "ignore": { 106 | # "min_progress" : 0.3, 107 | # "max_time_added" : 36 * HOUR, 108 | # }, 109 | # }, 110 | }, 111 | # ========================分割线1============================= 112 | # red 配置信息，如果不需要刷red请将分割线2内整段注释掉（默认是注释掉的） 113 | # ========================分割线2============================= 114 | # "red" : { 115 | # # 一个文件夹，保存曾经请求过的api回复，如果不需要自动拉黑脚本可以填None 116 | # "api_cache_dir" : "~/.cache/fishrss/red", 117 | # # cookie/apikey二选一用于鉴权，推荐使用api_key，因为可以缩小一半的api请求间隔限制（10秒5次->10秒10次） 118 | # # 如果非要使用cookie的话，请将cookie取消注释，并将api_key填为None 119 | # # 关于cookie的填写请参考`README.rss.md`里的描述 120 | # # "cookies" : { 121 | # # "session": "xxxxxxxxxxxx" 122 | # # }, 123 | # # apikey请在red个人设置页面进行设置，记得勾选"Confirm API Key"并保存之后apikey才生效 124 | # "api_key" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 125 | # # authkey和torrent_pass请随便复制一个种子的下载链接，里面有 126 | # "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 127 | # "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 128 | # # filter_config的填写请参考上面的模板FILTER_CONFIG_TEMPLATE和给出的样例 129 | # "filter_config" : { 130 | # "name" : "redfilter", 131 | # "banlist" : None, 132 | # "whitelist" : None, 133 | # "media" : set(["CD"]), 134 | # "format" : None, 135 | # "sizelim" : (100 * MB, 1024 * MB) 136 | # }, 137 | # # red staff说red规则不允许自动使用token，请不要修改"token_thresh"（修改了也没用） 138 | # "token_thresh": (0, -1), 139 | # # 自动ban人脚本条件，如果需要对red使用该脚本，请取消以下的注释： 140 | # # 填写时，请参考上面的模板AUTOBAN_TEMPLATE和给出的样例 141 | # # "autoban" : { 142 | # # "ratio" : [ 143 | # # {"count":1, "ratiolim":0.2}, 144 | # # ], 145 | # # "ignore": { 146 | # # "min_progress" : 0.3, 147 | # # "max_time_added" : 36 * HOUR, 148 | # # }, 149 | # # }, 150 | # }, 151 | # ========================分割线2============================= 152 | # ops 配置信息，如果不需要刷ops请将分割线3内整段注释掉（默认是注释掉的） 153 | # ========================分割线3============================= 154 | # "ops":{ 155 | # # 保存曾经请求过的api回复，如果不需要可以填None 156 | # "api_cache_dir" : "~/.cache/fishrss/ops", 157 | # # cookie/apikey二选一用于鉴权 158 | # # 如果使用cookie的话，请将cookie取消注释，并将api_key填为None 159 | # # 关于cookie的填写请参考`README.rss.md`里的描述 160 | # # "cookies" : { 161 | # # "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 162 | # # }, 163 | # # apikey请在ops个人设置页面进行设置 164 | # "api_key" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 165 | # # authkey和torrent_pass请随便复制一个种子的下载链接，里面有 166 | # "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 167 | # "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 168 | # # filter_config的填写请参考上面的模板FILTER_CONFIG_TEMPLATE和给出的样例 169 | # "filter_config" : { 170 | # "name" : "opsfilter", 171 | # "banlist" : None, 172 | # "whitelist" : None, 173 | # "media" : None, 174 | # "format" : set(["FLAC"]), 175 | # "sizelim" : (100 * MB, 1 * GB), 176 | # }, 177 | # # ops staff说ops规则不允许自动使用token，请不要修改"token_thresh"（修改了也没用） 178 | # "token_thresh": (0, -1), 179 | # # 自动ban人脚本条件，如果需要对ops使用该脚本，请取消以下的注释： 180 | # # 填写时，请参考上面的模板AUTOBAN_TEMPLATE和给出的样例 181 | # # "autoban" : { 182 | # # "ratio" : [ 183 | # # {"count":1, "ratiolim":0.2}, 184 | # # ], 185 | # # "ignore": { 186 | # # "min_progress" : 0.3, 187 | # # "max_time_added" : 36 * HOUR, 188 | # # }, 189 | # # }, 190 | # }, 191 | # ========================分割线3============================= 192 | # snakepop配置信息，如果不需要刷snakepop请将分割线4内整段注释掉（默认是注释掉的） 193 | # ========================分割线4============================= 194 | # "snake":{ 195 | # # 一个文件夹，保存曾经请求过的api回复，如果不需要可以填None 196 | # "api_cache_dir" : "~/.cache/fishrss/snake", 197 | # # 关于cookie的填写请参考`README.rss.md`里的描述 198 | # "cookies" : { 199 | # "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 200 | # }, 201 | # # authkey和torrent_pass请随便复制一个种子的下载链接，里面有 202 | # "authkey" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 203 | # "torrent_pass" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx", 204 | # # filter_config的填写请参考上面的模板FILTER_CONFIG_TEMPLATE和给出的样例 205 | # "filter_config" : { 206 | # "name" : "snakefilter", 207 | # "banlist" : None, 208 | # "whitelist" : None, 209 | # "media" : None, 210 | # "format" : set(["FLAC"]), 211 | # "sizelim" : None, 212 | # }, 213 | # # 体积在这个区间内时自动使用令牌，样例：100MB-2GB，如果不想使用令牌请填一个非法的区间，比如(0,-1) 214 | # # 注意，即使令牌用完，本脚本依然会继续下载种子，此时等于下黑种，请关注令牌余量 215 | # "token_thresh": (100 * MB, 2 * GB), 216 | # # 自动ban人脚本条件，如果需要对snake使用该脚本，请取消以下的注释： 217 | # # 填写时，请参考上面的模板AUTOBAN_TEMPLATE和给出的样例 218 | # # "autoban" : { 219 | # # "ratio" : [ 220 | # # {"count":1, "ratiolim":0.2}, 221 | # # ], 222 | # # "ignore": { 223 | # # "min_progress" : 0.3, 224 | # # "max_time_added" : 36 * HOUR, 225 | # # }, 226 | # # }, 227 | # }, 228 | # ========================分割线4============================= 229 | # deluge客户端的登陆信息，如果不需要自动ban人脚本可以不填 230 | "deluge" : { 231 | "ip": "127.0.0.1", 232 | #这个端口是deluge api的端口，而不是deluge暴露在外网监听bt连接的端口，在webui上点"connect manager"在host一栏上会显示这个端口 233 | "port": 12345, 234 | "username": "xxxxxxxxxxxxxx", 235 | "password": "xxxxxxxxxxxxxx", 236 | }, 237 | # requests发送请求的超时时间，不知道是啥的可以不改，单位：秒 238 | "requests_timeout" : 10, 239 | } -------------------------------------------------------------------------------- /filter.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import os, sys 5 | import logging 6 | import traceback 7 | import bencode 8 | import hashlib 9 | import urllib 10 | import collections 11 | import base64 12 | import argparse 13 | from multiprocessing.pool import ThreadPool 14 | from IPython import embed 15 | 16 | from config import CONFIG 17 | import common 18 | from common import logger 19 | from torrent_filter import TorrentFilter 20 | import gzapi 21 | 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument("--url", default=None, 24 | help="Download url of a torrent. If an url is provided, " 25 | "the code will only run once for the provided url and exits.") 26 | parser.add_argument("--file", default=None, 27 | help="path of a torrent file. If a file is provided, " 28 | "the code will only run once for the provided file and exits.") 29 | parser.add_argument("--skip-api", action="store_true", default=False, 30 | help="If set, the site api call will be skipped. Notice: the api-only information " 31 | "will be unavailable: uploader, media and format. Therefore, their filter config " 32 | "must be None otherwise there won't be any torrent filtered out.") 33 | parser.add_argument("--no-tick", action="store_true", default=False, 34 | help="If not set, every minute there will be a \"tick\" shown in the log, " 35 | "in order to save people with \"black screen anxiety\"") 36 | parser.add_argument("--force-accept", action="store_true", default=False, 37 | help="If set, always accept a torrent regardless of filter's setting") 38 | parser.add_argument("--deluge", action="store_true", default=False, 39 | help="push torrents to deluge by its api directly instead of saving to CONFIG[\"filter\"][\"dest_dir\"]") 40 | # parser.add_argument("--qbittorrent", action="store_true", default=False, 41 | # help="push to qbittorrent by its api directly") 42 | try: 43 | args = parser.parse_args() 44 | except Exception as e: 45 | logger.info(traceback.format_exc()) 46 | exit(0) 47 | 48 | run_once = args.url is not None or args.file is not None 49 | gzapi.FISH_HEADERS = requests.utils.default_headers() 50 | 51 | if args.deluge: 52 | import deluge_client 53 | DELUGE = CONFIG["deluge"] 54 | # 本机的话这个延迟大概0.01s左右，就不并行了 55 | de = deluge_client.DelugeRPCClient(DELUGE["ip"], DELUGE["port"], DELUGE["username"], DELUGE["password"]) 56 | de.connect() 57 | logger.info("deluge is connected: {}".format(de.connected)) 58 | assert de.connected 59 | 60 | configured_sites = {} 61 | for site in common.SITE_CONST.keys(): 62 | try: 63 | # 如果只运行一次，则不验证登陆以减少延迟 64 | api = common.get_api(site, skip_login=run_once) 65 | tfilter = common.get_filter(site) 66 | logger.info("api and filter of {} are set".format(site)) 67 | configured_sites[site] = {"api":api, "filter":tfilter} 68 | except: 69 | logger.info("api or filter of {} is NOT set".format(site)) 70 | 71 | def handle_accept(torrent): 72 | fname = "{}.torrent".format(common.get_info_hash(torrent)) 73 | if args.deluge: 74 | logger.info("pushing to deluge") 75 | raw = bencode.encode(torrent) 76 | de.core.add_torrent_file(fname, base64.b64encode(raw), {}) 77 | else: 78 | # default: save to dest_dir 79 | save_path = os.path.join(CONFIG["filter"]["dest_dir"], fname) 80 | logger.info("saving to {}".format(save_path)) 81 | with open(save_path, "wb") as f: 82 | f.write(bencode.encode(torrent)) 83 | 84 | def _handle(*, 85 | torrent : collections.OrderedDict, 86 | tinfo : dict, 87 | filter : TorrentFilter, 88 | token_thresh : tuple, 89 | fl_url=None 90 | ): 91 | """ 92 | main handle logic 93 | """ 94 | logger.info("{} torrentid={} infohash={}".format(torrent["info"]["name"], tinfo["tid"], tinfo["hash"])) 95 | check_result = filter.check_tinfo(**tinfo) 96 | if check_result != "accept" and not args.force_accept: 97 | # 不满足条件 98 | logger.info("reject: {}".format(check_result)) 99 | else: 100 | # 满足条件，转存种子 101 | logger.info("accept") 102 | handle_accept(torrent) 103 | # 如果是red/ops，遵守规则不使用令牌 104 | site = common.get_torrent_site(torrent) 105 | if site == "red" or site == "ops": 106 | return 107 | # 根据体积限制使用令牌 108 | if token_thresh is not None and token_thresh[0] < tinfo["size"] and tinfo["size"] < token_thresh[1]: 109 | if fl_url is None: 110 | logger.info("fl url not provided") 111 | else: 112 | logger.info("getting fl:{}".format(fl_url)) 113 | # 因为种子已转存，FL链接下载下来的种子会被丢弃 114 | r = requests.get(fl_url, timeout=CONFIG["requests_timeout"], headers=gzapi.FISH_HEADERS) 115 | try: 116 | # 验证种子合法性 117 | fl_torrent = bencode.decode(r.content) 118 | assert common.get_info_hash(torrent) == common.get_info_hash(fl_torrent) 119 | except: 120 | logger.info("Invalid torrent downloaded from fl_url. It might because you don't have ENOUGH tokens(可能令牌不足？):") 121 | logger.info(traceback.format_exc()) 122 | 123 | def handle_default(torrent): 124 | if args.force_accept: 125 | handle_accept(torrent) 126 | return 127 | source = torrent["info"]["source"] 128 | logger.info("unconfigured source: {}, {} by default".format(source, CONFIG["filter"]["default_behavior"])) 129 | if CONFIG["filter"]["default_behavior"] == "accept": 130 | handle_accept(torrent) 131 | 132 | def handle_gz(*, 133 | torrent, 134 | api_response, 135 | fl_url 136 | ): 137 | """ 138 | congregate torrent information 139 | """ 140 | tinfo = dict() 141 | # update info from torrent 142 | if "comment" in torrent.keys(): 143 | tinfo["tid"] = int(common.get_params_from_url(torrent["comment"])["torrentid"]) 144 | tinfo["size"] = sum([f["length"] for f in torrent["info"]["files"]]) 145 | tinfo["hash"] = common.get_info_hash(torrent) 146 | # update info from api_response 147 | if api_response is not None: 148 | api_tinfo = api_response["response"]["torrent"] 149 | if "tid" in tinfo.keys(): 150 | if tinfo["tid"] != api_tinfo["id"]: 151 | logging.info("torrentid dismatch: {} from comment and {} from api response:".format( 152 | tinfo["tid"], api_tinfo["id"] 153 | )) 154 | del tinfo["tid"] 155 | else: 156 | tinfo["tid"] = api_tinfo["id"] 157 | tinfo["uploader"] = api_tinfo["username"] 158 | tinfo["media"] = api_tinfo["media"] 159 | tinfo["file_format"] = api_tinfo["format"] 160 | if "tid" not in tinfo: 161 | logging.warning("torrentid not found, using 0 as dummy") 162 | tinfo["tid"] = 0 163 | site = common.get_torrent_site(torrent) 164 | _handle( 165 | torrent=torrent, 166 | tinfo=tinfo, 167 | filter=configured_sites[site]["filter"], 168 | token_thresh=CONFIG[site]["token_thresh"], 169 | fl_url=fl_url, 170 | ) 171 | 172 | def handle_file(filepath : str): 173 | with open(filepath, "rb") as f: 174 | raw = f.read() 175 | torrent = bencode.decode(raw) 176 | site = common.get_torrent_site(torrent) 177 | logger.info("new torrent from {}: {}".format(site, filepath)) 178 | if site not in configured_sites.keys(): 179 | handle_default(torrent=torrent) 180 | return 181 | api = configured_sites[site]["api"] 182 | tid = common.get_params_from_url(torrent["comment"])["torrentid"] 183 | try: 184 | fl_url = api.get_fl_url(tid) 185 | except: 186 | fl_url = None 187 | if args.skip_api: 188 | api_response = None 189 | else: 190 | api_response = api.query_tid(tid) 191 | if api_response["status"] != "success": 192 | logger.info("error in response: {}".format(repr(api_response))) 193 | api_response = None 194 | handle_gz( 195 | torrent=torrent, 196 | api_response=api_response, 197 | fl_url=fl_url, 198 | ) 199 | 200 | def handle_url(dl_url : str): 201 | def _call_api(api, tid): 202 | logger.info("calling api: {} tid: {}".format(api.apiname, tid)) 203 | api_response = api.query_tid(tid) 204 | logger.info("api responded") 205 | return api_response 206 | def _download_torrent(dl_url): 207 | logger.info("downloading torrent_file") 208 | r = requests.get(dl_url, timeout=CONFIG["requests_timeout"], headers=gzapi.FISH_HEADERS) 209 | torrent = bencode.decode(r.content) 210 | logger.info("torrent file downloaded") 211 | return torrent 212 | site = common.get_url_site(dl_url) 213 | logger.info("new torrent from {}: {}".format(site, dl_url)) 214 | if site not in configured_sites: 215 | torrent = _download_torrent(dl_url) 216 | handle_default(torrent) 217 | return 218 | tid = common.get_params_from_url(dl_url)["id"] 219 | api = configured_sites[site]["api"] 220 | # call api & download torrent in parallel 221 | pool = ThreadPool(processes=2) 222 | t_dl = pool.apply_async(_download_torrent, args=(dl_url,)) 223 | if not args.skip_api: 224 | t_api = pool.apply_async(_call_api, args=(api, tid)) 225 | pool.close() 226 | pool.join() 227 | if not args.skip_api: 228 | api_response = t_api.get() 229 | else: 230 | api_response = None 231 | try: 232 | fl_url = api.get_fl_url(tid) 233 | except: 234 | fl_url = None 235 | handle_gz( 236 | torrent=t_dl.get(), 237 | api_response=api_response, 238 | fl_url=fl_url, 239 | ) 240 | 241 | def check_dir(): 242 | """ 243 | check new torrents in source_dir 244 | """ 245 | flist = os.listdir(CONFIG["filter"]["source_dir"]) 246 | if len(flist) == 0: 247 | return 248 | for fname in flist: 249 | tpath = os.path.join(CONFIG["filter"]["source_dir"], fname) 250 | if os.path.splitext(fname)[1] == ".torrent": 251 | handle_file(tpath) 252 | os.remove(tpath) 253 | common.flush_logger() 254 | 255 | if args.url is not None: 256 | common.error_catcher(func=handle_url, dl_url=args.url) 257 | elif args.file is not None: 258 | common.error_catcher(func=handle_file, filepath=args.file) 259 | else: 260 | # monitor directory 261 | logger.info("monitoring torrent files in {}".format(CONFIG["filter"]["source_dir"])) 262 | cnt = 0 263 | while True: 264 | common.error_catcher(check_dir) 265 | time.sleep(0.2) 266 | if cnt % 300 == 0 and not args.no_tick: 267 | # 每分钟输出点东西，缓解黑屏焦虑 268 | logger.info("tick") 269 | common.flush_logger() 270 | cnt += 1 -------------------------------------------------------------------------------- /gen_stats.py: -------------------------------------------------------------------------------- 1 | """ 2 | deluge数据统计脚本 3 | """ 4 | import traceback 5 | import deluge_client 6 | import urllib 7 | import functools 8 | from IPython import embed 9 | 10 | from config import CONFIG 11 | import common 12 | from common import logger, flush_logger 13 | configured_sites = {} 14 | for site in common.SITE_CONST.keys(): 15 | try: 16 | # 如果只运行一次，则不验证登陆以减少延迟 17 | api = common.get_api(site) 18 | tfilter = common.get_filter(site) 19 | logger.info("api and filter of {} are set".format(site)) 20 | configured_sites[site] = {"api":api, "filter":tfilter} 21 | except: 22 | logger.info("api or filter of {} is NOT set".format(site)) 23 | 24 | DELUGE = CONFIG["deluge"] 25 | client = deluge_client.DelugeRPCClient(DELUGE["ip"], DELUGE["port"], DELUGE["username"], DELUGE["password"]) 26 | client.connect() 27 | logger.info("gen_stats: deluge is connected: {}".format(client.connected)) 28 | 29 | tlist = list(client.core.get_torrents_status({"state":"Seeding"}, [ 30 | "hash", 31 | "name", 32 | "ratio", 33 | "total_size", 34 | "time_added", 35 | "comment", 36 | "tracker", 37 | ]).values()) 38 | 39 | """ 40 | 从网站api获取更多信息 41 | """ 42 | def gen_extra_info(t): 43 | site = common.get_tracker_site(t[b"tracker"].decode("utf-8")) 44 | if site not in configured_sites.keys(): 45 | # unsupported tracker 46 | return {} 47 | else: 48 | api = configured_sites[site]["api"] 49 | comment = t[b"comment"].decode("utf-8") 50 | tid = urllib.parse.parse_qs(urllib.parse.urlparse(comment).query)["torrentid"][0] 51 | js = api.query_tid(tid) 52 | if js["status"] != "success": 53 | # torrentid is the only available info 54 | return {"torrentid":tid} 55 | response = js["response"] 56 | tinfo = response["torrent"] 57 | ginfo = response["group"] 58 | extra_info = { 59 | "uploader" : tinfo["username"], 60 | "media" : tinfo["media"], 61 | "format" : tinfo["format"], 62 | "encoding" : tinfo["encoding"], 63 | "releaseType" : ginfo["releaseType"], 64 | "remasterYear": tinfo["remasterYear"], 65 | "hasCue" : tinfo["hasCue"], 66 | "hasLog" : tinfo["hasLog"], 67 | "logScore" : tinfo["logScore"], 68 | "torrentid" : tid, 69 | } 70 | return extra_info 71 | 72 | torrent_infos = [] 73 | 74 | for t in tlist: 75 | info = { 76 | "name" : t[b"name"].decode("utf-8"), 77 | "ratio" : t[b"ratio"], 78 | "tracker" : common.get_domain_name_from_url(t[b"tracker"].decode("utf-8")), 79 | "size": t[b"total_size"], 80 | "uploaded": int(t[b"total_size"] * t[b"ratio"]), 81 | "hash": t[b"hash"].decode("utf-8"), 82 | "time_added":t[b"time_added"], 83 | } 84 | extra_info = gen_extra_info(t) 85 | info.update(extra_info) 86 | torrent_infos.append(info) 87 | 88 | first_keys = ["tracker", "ratio", "uploaded", "size"] 89 | last_keys = ["time_added", "hash", "name"] 90 | keys = functools.reduce(lambda x,y: x.union(y), [set(info.keys()) for info in torrent_infos], set()) 91 | keys = first_keys + list(set(keys) - set(first_keys) - set(last_keys)) + last_keys 92 | empty = {key:"" for key in keys} 93 | print("\t".join(keys)) 94 | torrent_infos.sort(key=lambda info:info["time_added"]) 95 | for info in torrent_infos: 96 | complete_info = empty.copy() 97 | complete_info.update(info) 98 | print("\t".join([str(complete_info[key]) for key in keys])) 99 | -------------------------------------------------------------------------------- /gzapi.py: -------------------------------------------------------------------------------- 1 | import time 2 | import os 3 | import requests 4 | import json 5 | import urllib 6 | import traceback 7 | import logging 8 | """ 9 | 所有api参数中的 10 | tid表示torrentid种子id 11 | gid表示groupid种子组id，也即种子链接中id=后面的那个id 12 | uid表示账号id 13 | """ 14 | 15 | # 提供一个计时器，每次wait会等待直到前count次wait之后的interval秒 16 | class Timer(object): 17 | 18 | def __init__(self, count, interval): 19 | self.history = [0 for _ in range(count)] 20 | self.interval = interval 21 | 22 | def wait(self): 23 | t = time.time() 24 | if t - self.history[0] < self.interval: 25 | time.sleep(self.interval - (t - self.history[0]) + 0.1) 26 | t = time.time() 27 | self.history = self.history[1:] + [t,] 28 | 29 | FISH_HEADERS = requests.utils.default_headers() 30 | FISH_HEADERS['User-Agent'] = "FishRSS" 31 | 32 | class GazelleApi(object): 33 | 34 | def __init__(self, *, 35 | logger:logging.Logger, 36 | apiname, 37 | timer, 38 | api_url, 39 | authkey, 40 | torrent_pass, 41 | cache_dir=None, 42 | cookies=None, 43 | headers=None, 44 | timeout=10, 45 | skip_login=False, 46 | # discard 47 | **kwargs 48 | ): 49 | self.logger = logger 50 | self.apiname = apiname 51 | self.cache_dir = cache_dir 52 | self.cookies = cookies 53 | if headers is None: 54 | headers = requests.utils.default_headers() 55 | self.headers = headers 56 | self.timer = timer 57 | self.timeout = timeout 58 | self.api_url = api_url 59 | self.authkey = authkey 60 | self.torrent_pass = torrent_pass 61 | 62 | # limit the max retry to 1 to prohibit retrying from influencing the frequency control 63 | self.sess = requests.Session() 64 | self.sess.mount('https://', requests.adapters.HTTPAdapter(max_retries=1)) 65 | self.sess.mount('http://', requests.adapters.HTTPAdapter(max_retries=1)) 66 | 67 | # sanity check 68 | if cache_dir is not None: 69 | assert os.path.exists(cache_dir), "{}的api_cache_dir文件夹不存在：{}".format(apiname, cache_dir) 70 | else: 71 | self.logger.warning("{}未配置api cache dir".format(apiname)) 72 | 73 | # try login 74 | if not skip_login: 75 | self.login() 76 | 77 | def login(self): 78 | js = self._query(params={"action":"index"}, use_cache=False) 79 | if js["status"] == "failure": 80 | if "error" in js.keys() and js["error"] == "bad credentials": 81 | self.logger.error("{}的鉴权凭证(cookie/apikey)填写不正确: {}".format( 82 | self.apiname, repr(js))) 83 | assert js["error"] != "bad credential" 84 | if js["status"] != "success": 85 | self.logger.error("{}鉴权错误：{}".format(self.apiname, repr(js))) 86 | assert js["status"] == "success" 87 | uinfo = js["response"] 88 | self.username = uinfo["username"] 89 | self.uid = uinfo["id"] 90 | if self.authkey != uinfo["authkey"]: 91 | self.logger.warning("{}的authkey填写错误或过期，应当为(authkey should be) \"{}\"，而不是(but not) \"{}\"。" 92 | "如果只是过期，则这不一定会导致错误，" 93 | "但如果发现脚本运行不正常（比如token无法正常使用）请按照提示修改。".format( 94 | self.apiname, uinfo["authkey"], self.authkey)) 95 | if self.torrent_pass != uinfo["passkey"]: 96 | self.logger.error("{}的torrent_pass填写错误，应当为(torrent_pass should be) \"{}\"，而不是(but not) \"{}\"".format( 97 | self.apiname, uinfo["passkey"], self.torrent_pass)) 98 | assert self.torrent_pass == uinfo["passkey"] 99 | self.logger.info("{} logged in successfully, username：{} uid: {}".format(self.apiname, self.username, self.uid)) 100 | 101 | """ 102 | 此函数仅保证返回是一个dict，且至少含有"status"一个key 103 | """ 104 | def _query(self, params: dict, use_cache=True) -> dict: 105 | if self.cache_dir is not None and use_cache: 106 | fname = "{}.json".format(urllib.parse.urlencode(params).replace("&", "_")) 107 | cache_file = os.path.join(self.cache_dir, fname) 108 | if os.path.exists(cache_file): 109 | with open(cache_file, "r") as f: 110 | return json.load(f) 111 | self.logger.info("{} querying {}".format(self.apiname, urllib.parse.urlencode(params))) 112 | self.timer.wait() 113 | r = self.sess.get( 114 | url=self.api_url, 115 | cookies=self.cookies, 116 | headers=self.headers, 117 | params=params, 118 | timeout=self.timeout 119 | ) 120 | try: 121 | js = json.loads(r.text) 122 | except json.JSONDecodeError: 123 | self.logger.info(r.text) 124 | self.logger.info(traceback.format_exc()) 125 | return {"status": "json decode failure", "raw_response":r.text} 126 | if js["status"] == "failure": 127 | if "error" in js.keys() and js["error"] == "bad credentials": 128 | self.logger.warning("login credentials (cookie/apikey) of {} is wrong: {}".format( 129 | self.apiname, repr(js))) 130 | # 鉴权错误时直接返回结果不保存至cache 131 | return js 132 | if self.cache_dir is not None and use_cache: 133 | with open(cache_file, "w") as f: 134 | json.dump(js, f) 135 | return js 136 | 137 | def query_hash(self, h, **kwargs): 138 | return self._query(params={ 139 | "action": "torrent", 140 | "hash": h.upper(), 141 | }, **kwargs) 142 | 143 | def query_tid(self, tid, **kwargs): 144 | return self._query(params={ 145 | "action": "torrent", 146 | "id": tid, 147 | }, **kwargs) 148 | 149 | def query_gid(self, gid, **kwargs): 150 | return self._query(params={ 151 | "action": "torrentgroup", 152 | "id": gid, 153 | }, **kwargs) 154 | 155 | def query_uploaded(self, uid, **kwargs): 156 | return self._query(params={ 157 | "action":"user_torrents", 158 | "id":uid, 159 | "type":"uploaded", 160 | }) 161 | 162 | def search(self, search_params : dict, **kwargs): 163 | params = search_params.copy() 164 | params["action"] = "browse" 165 | return self._query(params=params, **kwargs) 166 | 167 | def search_torrent_by_filename(self, filename, **kwargs): 168 | def _escape(s): 169 | """ 170 | replace characters to space except alphabet and numbers 171 | """ 172 | s2 = "" 173 | for ch in s: 174 | if ch.isalnum(): 175 | s2 += ch 176 | else: 177 | s2 += " " 178 | s = s2 179 | while s.replace(" ", " ") != s: 180 | s = s.replace(" ", " ") 181 | return s 182 | return self.search(search_params={ 183 | # 由于煞笔的按文件名搜索功能有毒，搜文件名时把文件名中非字母数字的部分全部去掉 184 | "filelist":_escape(filename), 185 | }, **kwargs) 186 | 187 | def get_dl_url(self, tid): 188 | raise NotImplementedError 189 | 190 | def get_fl_url(self, tid): 191 | return self.get_dl_url(tid) + "&usetoken=1" 192 | 193 | class REDApi(GazelleApi): 194 | 195 | def __init__(self, *, apikey=None, **kwargs): 196 | headers = FISH_HEADERS.copy() 197 | self.apikey = apikey 198 | if apikey is not None: 199 | timer = Timer(10, 10.5) 200 | headers["Authorization"] = apikey 201 | else: 202 | timer = Timer(5, 10.5) 203 | super().__init__( 204 | apiname="red", 205 | headers=headers, 206 | timer=timer, 207 | api_url="https://redacted.ch/ajax.php", 208 | **kwargs 209 | ) 210 | 211 | def get_dl_url(self, tid): 212 | return "https://redacted.ch/torrents.php?action=download&id={}&authkey={}&torrent_pass={}".format( 213 | tid, self.authkey, self.torrent_pass) 214 | 215 | # override fl link for RED to abide the rule 216 | def get_fl_url(self, tid): 217 | raise NotImplementedError 218 | 219 | class DICApi(GazelleApi): 220 | 221 | def __init__(self, **kwargs): 222 | super().__init__( 223 | apiname="dic", 224 | headers=FISH_HEADERS, 225 | timer=Timer(5, 10.5), 226 | api_url="https://dicmusic.club/ajax.php", 227 | **kwargs 228 | ) 229 | 230 | def get_dl_url(self, tid): 231 | return "https://dicmusic.club/torrents.php?action=download&id={}&authkey={}&torrent_pass={}".format( 232 | tid, self.authkey, self.torrent_pass) 233 | 234 | class SnakeApi(GazelleApi): 235 | 236 | def __init__(self, **kwargs): 237 | super().__init__( 238 | apiname="snake", 239 | headers=FISH_HEADERS, 240 | timer=Timer(5, 10.5), 241 | api_url="https://snakepop.art/ajax.php", 242 | **kwargs 243 | ) 244 | 245 | def get_dl_url(self, tid): 246 | return "https://snakepop.art/torrents.php?action=download&id={}&authkey={}&torrent_pass={}".format( 247 | tid, self.authkey, self.torrent_pass) 248 | 249 | class OPSApi(GazelleApi): 250 | 251 | def __init__(self, *, apikey=None, **kwargs): 252 | headers = FISH_HEADERS.copy() 253 | self.apikey = apikey 254 | if apikey is not None: 255 | headers["Authorization"] = apikey 256 | super().__init__( 257 | apiname="ops", 258 | headers=headers, 259 | timer=Timer(5, 10.5), 260 | api_url="https://orpheus.network/ajax.php", 261 | **kwargs 262 | ) 263 | 264 | def get_dl_url(self, tid): 265 | return "https://orpheus.network/torrents.php?action=download&id={}&torrent_pass={}".format( 266 | tid, self.torrent_pass) 267 | 268 | # override fl link for OPS to abide the rule 269 | def get_fl_url(self, tid): 270 | raise NotImplementedError -------------------------------------------------------------------------------- /remove_unregistered.py: -------------------------------------------------------------------------------- 1 | """ 2 | 警告，此脚本会删除deluge内的种子和文件！ 3 | 功能：删除所有deluge里"Tracker Status"中含有"Unregistered torrent"字样的种子，以及文件 4 | """ 5 | 6 | import traceback 7 | import deluge_client 8 | from IPython import embed 9 | 10 | from config import CONFIG 11 | from common import logger 12 | 13 | DELUGE = CONFIG["deluge"] 14 | client = deluge_client.DelugeRPCClient(DELUGE["ip"], DELUGE["port"], DELUGE["username"], DELUGE["password"]) 15 | client.connect() 16 | logger.info("remove_unregistered: deluge is connected: {}".format(client.connected)) 17 | 18 | tlist = list(client.core.get_torrents_status({"state":"Downloading"}, ["hash", "name", "tracker_status"]).values()) 19 | cnt = 0 20 | for t in tlist: 21 | if "Unregistered torrent" in t[b"tracker_status"].decode("utf-8"): 22 | logger.info("removing torrent \"{}\" reason: \"{}\"".format( 23 | t[b"name"].decode("utf-8"), t[b"tracker_status"].decode("utf-8"))) 24 | client.core.remove_torrent(t[b"hash"], True) 25 | cnt += 1 26 | logger.info("{} torrents removed".format(cnt)) 27 | -------------------------------------------------------------------------------- /reseed.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import os, sys 5 | import logging 6 | import traceback 7 | import bencode 8 | import hashlib 9 | import urllib 10 | import collections 11 | import base64 12 | import argparse 13 | from typing import Optional 14 | from IPython import embed 15 | 16 | from config import CONFIG 17 | import common 18 | from common import logger 19 | import gzapi 20 | 21 | IGNORED_PATH =[ 22 | "@eaDir", #dummy directory created by Synology 23 | ".DS_Store", #dummy directory created by macOS 24 | ] 25 | 26 | def is_ignored(path): 27 | for ignore_str in IGNORED_PATH: 28 | if ignore_str in path: 29 | return True 30 | return False 31 | 32 | def check_path(path: str, is_file: bool = False, auto_create: bool = False): 33 | if path is not None: 34 | abspath = os.path.abspath(path) 35 | if os.path.exists(abspath): 36 | if is_file and not os.path.isfile(abspath): 37 | logger.warning("path {} must be a file".format(path)) 38 | exit(0) 39 | if not is_file and not os.path.isdir(abspath): 40 | logger.warning("path {} must be a folder".format(path)) 41 | exit(0) 42 | else: 43 | if not auto_create: 44 | logger.warning("path doesn't exist: {} ".format(path)) 45 | exit(0) 46 | else: 47 | if is_file: 48 | logger.info("file automatically created: {}".format(path)) 49 | folder = os.path.split(abspath)[0] 50 | if not os.path.exists(folder): 51 | os.makedirs(folder) 52 | with open(abspath, "w") as _: 53 | pass 54 | else: 55 | logger.info("directory automatically created: {}".format(path)) 56 | os.makedirs(path) 57 | 58 | parser = argparse.ArgumentParser(description=""" 59 | 功能：扫描指定文件夹进行辅种 scan a directory to find torrents that can be cross-seeded on given tracker 60 | """) 61 | group = parser.add_mutually_exclusive_group(required=True) 62 | group.add_argument('--dir', default=None, help="批量辅种的文件夹 folder for batch cross-seeding") 63 | group.add_argument('--single-dir', default=None, help="单个辅种的文件夹 folder for just one cross-seeding") 64 | group.add_argument('--torrent-dir', default=None, help="存储待辅种的种子的文件夹 folder containing .torrent files for cross-seeding") 65 | group.add_argument('--single-torrent', default=None, help="单个辅种的种子文件 .torrent file for cross-seeding") 66 | parser.add_argument('--site', required=True, choices=common.SITE_CONST.keys(), 67 | help="扫描的站点 the tracker to scan for cross-seeding.") 68 | parser.add_argument('--result-dir', required=True, 69 | help="储存扫描结果的文件夹 folder for saving scanned results") 70 | parser.add_argument('--api-frequency', default=None, 71 | help="if set, override the default api calling frequency. Unit: number of api call per 10 seconds (must be integer)") 72 | parser.add_argument("--no-download", action="store_true", default=False, 73 | help="if set, don't download the .torrent files. Only the id of torrents are saved") 74 | if len(sys.argv) == 1: 75 | parser.print_help() 76 | exit(0) 77 | args = parser.parse_args() 78 | 79 | # check and create files/folders 80 | 81 | # basic folders 82 | check_path(args.dir) 83 | check_path(args.single_dir) 84 | check_path(args.torrent_dir) 85 | check_path(args.single_torrent, is_file=True) 86 | check_path(args.result_dir) 87 | 88 | # scanned directories, one absolute path per line 89 | scan_history_path = os.path.join(args.result_dir, "scan_history.txt") 90 | check_path(scan_history_path, is_file=True, auto_create=True) 91 | 92 | # scanned results, one torrent url per line 93 | result_url_path = os.path.join(args.result_dir, "result_url.txt") 94 | check_path(result_url_path, is_file=True, auto_create=True) 95 | 96 | # mapping of scanning results, "{path}\t{result}" for each line 97 | result_map_path = os.path.join(args.result_dir, "result_mapping.txt") 98 | check_path(result_map_path, is_file=True, auto_create=True) 99 | 100 | # folder of downloaded .torrents files of scanning results 101 | result_torrent_path = os.path.join(args.result_dir, "torrents/") 102 | check_path(result_torrent_path, is_file=False, auto_create=True) 103 | 104 | # urls of .torrents files that are unable to download, one torrent url per line 105 | result_url_undownloaded_path = os.path.join(args.result_dir, "result_url_undownloaded.txt") 106 | check_path(result_url_undownloaded_path, is_file=True, auto_create=True) 107 | 108 | GLOBAL = dict() 109 | GLOBAL["found"] = 0 110 | GLOBAL["downloaded"] = 0 111 | GLOBAL["scanned"] = 0 112 | GLOBAL["cnt_dl_fail"] = 0 113 | 114 | def scan(*, flist : list, tsize : int, scan_source : str, api): 115 | GLOBAL["scanned"] += 1 116 | 117 | tid = -1 118 | # search for the files with top 5 longest name 119 | for fname in sorted(flist, key=lambda fname:-len(fname))[:5]: 120 | resp = api.search_torrent_by_filename(fname, use_cache=False) 121 | if resp["status"] != "success": 122 | logger.info("api failure: {}".format(repr(resp))) 123 | continue 124 | torrents = sum([group["torrents"] for group in resp["response"]["results"] if "torrents" in group.keys()], []) 125 | for t in torrents: 126 | if t["size"] == tsize: 127 | tid = t["torrentId"] 128 | break 129 | if tid != -1: 130 | break 131 | # if it is impossible to find a match, just stop: 132 | if len(torrents) == 0 or resp["response"]["pages"] <= 1: 133 | break 134 | 135 | if tid == -1: 136 | logger.info("not found") 137 | with open(result_map_path, "a") as f: 138 | f.write("{}\t{}\n".format(os.path.split(scan_source)[1], tid)) 139 | with open(scan_history_path, "a") as f: 140 | f.write("{}\n".format(scan_source)) 141 | downloaded = False 142 | else: 143 | GLOBAL["found"] += 1 144 | logger.info("found, torrentid={}".format(tid)) 145 | dl_url = api.get_dl_url(tid) 146 | downloaded = False 147 | if not args.no_download: 148 | try: 149 | resp = requests.get(dl_url, headers=gzapi.FISH_HEADERS, timeout=CONFIG["requests_timeout"]) 150 | # check the integrity of torrent 151 | _ = bencode.decode(resp.content) 152 | if os.path.splitext(scan_source)[1] == ".torrent": 153 | # use the source torrent's name as the torrent's name to help crossseeding 154 | tname = os.path.splitext(os.path.split(scan_source)[1])[0] 155 | else: 156 | # use the folder's name as the torrent's name to help crossseeding 157 | tname = os.path.split(scan_source)[1] 158 | tpath = os.path.join(result_torrent_path, "{}.torrent".format(tname)) 159 | logger.info("saving to {}".format(tpath)) 160 | with open(tpath, "wb") as f: 161 | f.write(resp.content) 162 | downloaded = True 163 | GLOBAL["downloaded"] += 1 164 | except: 165 | logger.info("fail to download .torrent file from {}".format(dl_url)) 166 | GLOBAL["cnt_dl_fail"] += 1 167 | if GLOBAL["cnt_dl_fail"] <= 10: 168 | logger.info(traceback.format_exc()) 169 | logger.info("It might because the torrent id {} has reached the " 170 | "limitation of non-browser downloading of {}. " 171 | "The URL of failed downloading will be saved to {}. " 172 | "You can download it from your own browser.".format( 173 | tid, args.site, result_url_undownloaded_path)) 174 | logger.info("下载种子(id {})失败，这可能是因为触发了{}对于非浏览器下载该种子的限制。" 175 | "失败种子的下载链接保存在{}里，你可以用你的浏览器从该链接下载种子".format( 176 | tid, args.site, result_url_undownloaded_path)) 177 | if GLOBAL["cnt_dl_fail"] == 10: 178 | logger.info("suppress further hinting for .torrent file downloading failure") 179 | with open(result_url_path, "a") as f: 180 | f.write("{}\n".format(dl_url)) 181 | with open(result_map_path, "a") as f: 182 | f.write("{}\t{}\n".format(os.path.split(scan_source)[1], tid)) 183 | if not downloaded: 184 | with open(result_url_undownloaded_path, "a") as f: 185 | f.write("{}\n".format(dl_url)) 186 | with open(scan_history_path, "a") as f: 187 | f.write("{}\n".format(scan_source)) 188 | return tid, downloaded 189 | 190 | api = common.get_api(args.site) 191 | if args.api_frequency is not None: 192 | api.timer = gzapi.Timer(args.api_frequency, 10.5) 193 | 194 | def handle_folder(folder: str, api): 195 | tsize = 0 196 | flist = [] 197 | for path, _, files in os.walk(folder): 198 | if not is_ignored(path): 199 | for fname in files: 200 | if not is_ignored(fname): 201 | tsize += os.path.getsize(os.path.join(path, fname)) 202 | flist.append(fname) 203 | scan(flist=flist, tsize=tsize, scan_source=folder, api=api) 204 | 205 | 206 | def handle_torrent(tpath: str, api): 207 | with open(tpath, "rb") as f: 208 | torrent: collections.OrderedDict = bencode.decode(f.read()) 209 | tsize = 0 210 | flist = [] 211 | if "files" not in torrent["info"].keys(): 212 | scan(flist=[torrent["info"]["name"]], 213 | tsize=torrent["info"]["length"], 214 | scan_source=tpath, 215 | api=api) 216 | else: 217 | for f in torrent["info"]["files"]: 218 | tsize += f["length"] 219 | flist.append(f["path"][-1]) 220 | scan(flist=flist, tsize=tsize, scan_source=tpath, api=api) 221 | 222 | if args.single_dir: 223 | folder = os.path.abspath(args.single_dir) 224 | logger.info("scanning {} for cross seeding in {}".format( 225 | folder, args.site)) 226 | handle_folder(folder, api) 227 | elif args.single_torrent: 228 | tpath = os.path.abspath(args.single_torrent) 229 | logger.info("scanning {} for cross seeding in {}".format( 230 | args.single_torrent, args.site)) 231 | handle_torrent(tpath, api) 232 | elif args.dir: 233 | try: 234 | with open(scan_history_path, "r") as f: 235 | scanned_set = set([line.strip() for line in f]) 236 | candidates = [] 237 | parent_folder = os.path.abspath(args.dir) 238 | for f in os.listdir(parent_folder): 239 | path = os.path.join(parent_folder, f) 240 | if os.path.isdir(path): 241 | candidates.append(path) 242 | unscanned_folders = [path for path in candidates if path not in scanned_set] 243 | logger.info("{}/{} unscanned folders found in {}, start scanning for cross-seeding {}".format( 244 | len(unscanned_folders), len(candidates), args.dir, args.site)) 245 | for i, folder in enumerate(unscanned_folders): 246 | logger.info("{}/{} {}".format(i+1, len(unscanned_folders), folder)) 247 | handle_folder(folder, api) 248 | except Exception: 249 | logger.info(traceback.format_exc()) 250 | finally: 251 | logger.info("{} folders scanned, {} torrents found, {} .torrent files downloaded from {}".format( 252 | GLOBAL["scanned"], GLOBAL["found"], GLOBAL["downloaded"], args.site 253 | )) 254 | elif args.torrent_dir: 255 | try: 256 | with open(scan_history_path, "r") as f: 257 | scanned_set = set([line.strip() for line in f]) 258 | candidates = [] 259 | for f in os.listdir(args.torrent_dir): 260 | path = os.path.join(args.torrent_dir, f) 261 | if os.path.isfile(path) and os.path.splitext(path)[1] == ".torrent": 262 | candidates.append(path) 263 | unscanned_torrents = [path for path in candidates if path not in scanned_set] 264 | logger.info("{}/{} unscanned torrents found in {}, start scanning for cross-seeding {}".format( 265 | len(unscanned_torrents), len(candidates), args.torrent_dir, args.site)) 266 | for i, tpath in enumerate(unscanned_torrents): 267 | logger.info("{}/{} {}".format(i+1, len(unscanned_torrents), tpath)) 268 | handle_torrent(tpath, api) 269 | except Exception: 270 | logger.info(traceback.format_exc()) 271 | finally: 272 | logger.info("{} torrents scanned, {} torrents found, {} .torrent files downloaded from {}".format( 273 | GLOBAL["scanned"], GLOBAL["found"], GLOBAL["downloaded"], args.site 274 | )) 275 | -------------------------------------------------------------------------------- /rss.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import hashlib 3 | import logging 4 | import bencode 5 | import traceback 6 | import os 7 | import json 8 | import time 9 | 10 | #============================================================================ 11 | # 仅支持python3，依赖： 12 | # pip3 install bencode.py requests 13 | #============================================================================ 14 | # 这些字段需要你自己填写： 15 | # cookie，从浏览器中复制，只需要cookie中的PHPSESSID和session两个字段 16 | COOKIES = {"PHPSESSID": "xxxxxxxxxxxxxxxx", 17 | "session" : "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx"} 18 | # authkey和torrentpass：你可以从任意一个你的种子的下载链接里获得，长度均为32个字符 19 | AUTHKEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 20 | TORRENT_PASS = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" 21 | # 对于大于FL_THRESHOLD体积的种子使用免费令牌，单位字节Byte 22 | # 建议第一次运行的时候不要修改这个值，这样就不会在老种上使用令牌，之后运行的时候再调小 23 | FL_THRESHOLD = 100 * 1024**3 # 100GB 24 | # 存储rss出来的种子的文件夹： 25 | DOWNLOAD_DIR = "./watch/" 26 | # 存储rss过的种子链接的文件： 27 | DOWNLOADED_URLS = "./downloaded_urls.txt" 28 | #============================================================================ 29 | 30 | 31 | HEADERS = { 32 | 'User-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36' 33 | } 34 | 35 | def get_info_hash(raw): 36 | info = bencode.decode(raw)["info"] 37 | info_raw = bencode.encode(info) 38 | sha = hashlib.sha1(info_raw) 39 | info_hash = sha.hexdigest() 40 | return info_hash 41 | def get_name(raw): 42 | info = bencode.decode(raw)["info"] 43 | return info["name"] 44 | 45 | LOG_FORMAT = "%(asctime)s - %(levelname)s - %(message)s" 46 | logging.basicConfig(level=logging.INFO, format=LOG_FORMAT) 47 | try: 48 | resp = requests.get("https://dicmusic.club/ajax.php?action=notifications", cookies=COOKIES, timeout=10) 49 | tlist = json.loads(resp.text)["response"]["results"] 50 | except: 51 | logging.info("fail to read from RSS url") 52 | logging.info(traceback.format_exc()) 53 | exit() 54 | if not os.path.exists(DOWNLOADED_URLS): 55 | logging.info("downloaded file doesn't exist, create new one: {}".format(DOWNLOADED_URLS)) 56 | with open(DOWNLOADED_URLS, "w") as f: 57 | f.write("downloaded urls:\n") 58 | with open(DOWNLOADED_URLS, "r") as f: 59 | downloaded = set([line.strip() for line in f]) 60 | if not os.path.exists(DOWNLOAD_DIR): 61 | logging.info("torrent directory doesn't exist, create new one: {}".format(DOWNLOAD_DIR)) 62 | os.makedirs(DOWNLOAD_DIR) 63 | logging.info("{} torrents in rss result".format(len(tlist))) 64 | cnt = 0 65 | now = time.time() 66 | for t in tlist[:10]: 67 | tid = t["torrentId"] 68 | dl_url_raw = "https://dicmusic.club/torrents.php?action=download&id={}&authkey={}&torrent_pass={}".format(tid, AUTHKEY, TORRENT_PASS) 69 | if dl_url_raw in downloaded: 70 | continue 71 | if t["size"] > FL_THRESHOLD: 72 | dl_url = dl_url_raw + "&usetoken=1" 73 | else: 74 | dl_url = dl_url_raw 75 | try: 76 | logging.info("download {}".format(dl_url_raw)) 77 | resp = requests.get(dl_url, headers=HEADERS, timeout=120) 78 | raw = resp.content 79 | h = get_info_hash(raw) 80 | logging.info("hash={}".format(h)) 81 | with open(DOWNLOADED_URLS, "a") as f: 82 | f.write("{}\n".format(dl_url_raw)) 83 | with open(os.path.join(DOWNLOAD_DIR, "{}.torrent".format(get_name(raw))), "wb") as f: 84 | f.write(raw) 85 | cnt += 1 86 | except KeyboardInterrupt: 87 | logging.info(traceback.format_exc()) 88 | break 89 | except: 90 | logging.info("fail to download:") 91 | logging.info(traceback.format_exc()) 92 | logging.info("{} torrents added".format(cnt)) 93 | -------------------------------------------------------------------------------- /show_banned.sh: -------------------------------------------------------------------------------- 1 | cat filter.log|grep "new user banned" -A 3 2 | -------------------------------------------------------------------------------- /show_new_torrent.sh: -------------------------------------------------------------------------------- 1 | cat filter.log|grep "new torrent" -A 5|tail -n 100 2 | -------------------------------------------------------------------------------- /torrent_filter.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import os 3 | class TorrentFilter(object): 4 | 5 | FILTER_CONFIG_TEMPLATE = { 6 | "name": "redfilter", 7 | "banlist" : "/home7/fishpear/rss/ban_red.txt", 8 | "whitelist" : "/home7/fishpear/rss/white_red.txt", 9 | "media" : set(["CD", "Vinyl", "WEB"]), 10 | "format": set(["FLAC", "WAV"]), 11 | "sizelim": (100 * 1024**2, 1024**3), 12 | } 13 | 14 | def __init__(self, config, logger:logging.Logger): 15 | self.config = config 16 | self.logger = logger 17 | #sanity check 18 | for key in set(TorrentFilter.FILTER_CONFIG_TEMPLATE.keys()) - set(config.keys()): 19 | assert False, "filter_config配置错误：缺少{}".format(key) 20 | for key in set(config.keys()) - set(TorrentFilter.FILTER_CONFIG_TEMPLATE.keys()): 21 | assert False, "filter_config配置错误：多余的配置项{}".format(key) 22 | if config["banlist"] is not None: 23 | assert os.path.exists(config["banlist"]), "filter_config配置错误：被ban用户列表文件{}不存在".format(config["banlist"]) 24 | assert config["media"] is None or type(config["media"]) is set, \ 25 | "filter_config配置错误：媒体类型(media)应当是集合或者None，而不是{}".format(repr(config["media"])) 26 | assert config["format"] is None or type(config["format"]) is set, \ 27 | "filter_config配置错误：格式类型(format)应当是集合或者None，而不是{}".format(repr(config["media"])) 28 | if config["sizelim"] is not None: 29 | assert type(config["sizelim"]) is tuple and len(config["sizelim"]) == 2, \ 30 | "filter_config配置错误：体积范围(sizelim)应当是一个二元组(x,y)或者None，而不是{}".format(repr(config["sizelim"])) 31 | 32 | def check_tinfo(self, *, 33 | uploader=None, 34 | media=None, 35 | file_format=None, 36 | size=None, 37 | tid=None, 38 | # discarded: 39 | **kwargs 40 | ): 41 | self.logger.info("{}: checking".format(self.config["name"])) 42 | self.logger.info("tid: {} uploader: {} media: {} format: {} size: {:.1f}MB".format( 43 | tid, uploader, media, file_format, size / 1024**2, 44 | )) 45 | if self.config["whitelist"] is not None: 46 | with open(self.config["whitelist"], "r") as f: 47 | whitelisted_users = set([line.strip() for line in f]) 48 | if uploader in whitelisted_users: 49 | self.logger.info("whitelisted uploader: {}".format(uploader)) 50 | return "accept" 51 | if self.config["banlist"] is not None: 52 | with open(self.config["banlist"], "r") as f: 53 | bannedusers = set([line.strip() for line in f]) 54 | if uploader in bannedusers: 55 | return "banned user" 56 | if self.config["media"] is not None: 57 | if media not in self.config["media"]: 58 | return "wrong media" 59 | if self.config["format"] is not None: 60 | if file_format not in self.config["format"]: 61 | return "wrong format" 62 | if self.config["sizelim"] is not None: 63 | if size < self.config["sizelim"][0]: 64 | return "size too small" 65 | if size > self.config["sizelim"][1]: 66 | return "size too big" 67 | return "accept" 68 | 69 | 70 | def check_json_response(self, js: dict): 71 | self.logger.info("{}: checking".format(self.config["name"])) 72 | if js["status"] != "success": 73 | return "error status: {}".format(js["status"]) 74 | tinfo = js["response"]["torrent"] 75 | return self.check_tinfo( 76 | uploader=tinfo["username"], 77 | media=tinfo["media"], 78 | file_format=tinfo["format"], 79 | size=tinfo["size"], 80 | tid=tinfo["id"], 81 | ) --------------------------------------------------------------------------------