├── .gitignore ├── src ├── e621 │ ├── io │ │ ├── tags.txt │ │ ├── parser.rs │ │ ├── mod.rs │ │ └── tag.rs │ ├── tui │ │ └── mod.rs │ ├── mod.rs │ ├── sender │ │ ├── mod.rs │ │ └── entries.rs │ ├── blacklist.rs │ └── grabber.rs ├── main.rs └── program.rs ├── Cargo.toml ├── .github ├── ISSUE_TEMPLATE │ └── bug_report.md └── workflows │ ├── rust.yml │ └── rust-clippy.yml ├── README.md ├── LICENSE └── Cargo.lock /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | **/*.rs.bk 3 | /.idea 4 | /config.json 5 | /tags.txt 6 | /downloads 7 | /login.json 8 | /Testing Bench 9 | /e621_downloader.log 10 | 11 | .vscode/ 12 | -------------------------------------------------------------------------------- /src/e621/io/tags.txt: -------------------------------------------------------------------------------- 1 | # This is the tag file that you will use so the program can know what tags to search. 2 | # If you wish to comment in this file, simply put `#` at the beginning or end of line. 3 | 4 | # Insert tags you wish to download in the appropriate group (remove all example tags and IDs with what you wish to download): 5 | 6 | [artists] 7 | braeburned 8 | 9 | [pools] 10 | 1106 # Title: Taste of the Order 11 | 12 | [sets] 13 | 28495 # Title: Good Picture 14 | 15 | [single-post] 16 | 1662487 # Photonoko. Basic Description: Otters 17 | 18 | [general] 19 | lutrine order:score -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "e621_downloader" 3 | version = "1.7.2" 4 | authors = ["McSib "] 5 | edition = "2021" 6 | 7 | [dependencies] 8 | # These are added to fix security vulnerabilities. 9 | regex = "1.8.1" 10 | smallvec = "1.10.0" 11 | tokio = "1.28.0" 12 | rustls = "0.21.0" 13 | h2 = "0.3.18" 14 | bumpalo = "3.12.1" 15 | remove_dir_all = "0.8.2" 16 | 17 | once_cell = "1.17.1" 18 | base64-url = "2.0.0" 19 | indicatif = "0.17.3" 20 | dialoguer = "0.10.4" 21 | console = "0.15.5" 22 | log = "0.4.17" 23 | simplelog = "0.12.1" 24 | reqwest = { version = "0.11.16", features = ["blocking", "rustls-tls", "json"] } 25 | serde = { version = "1.0.160", features = ["derive"] } 26 | serde_json = "1.0.96" 27 | anyhow = "1.0.70" 28 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: Bug 6 | assignees: McSib 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. Windows, macOS, Linux Distro] 28 | - Version [e.g. v1.7.1] 29 | 30 | **Additional context** 31 | Add any other context about the problem here. 32 | -------------------------------------------------------------------------------- /.github/workflows/rust.yml: -------------------------------------------------------------------------------- 1 | name: Rust 2 | 3 | on: 4 | push: 5 | branches: [ "main", "develop" ] 6 | pull_request: 7 | branches: [ "main", "develop" ] 8 | 9 | env: 10 | CARGO_TERM_COLOR: always 11 | 12 | jobs: 13 | ubuntu_build: 14 | name: Ubuntu Build 15 | runs-on: ubuntu-latest 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Setup Rust Toolchain 19 | uses: actions-rs/toolchain@v1.0.6 20 | with: 21 | toolchain: nightly 22 | - name: Configure OpenSSL 23 | run: sudo apt-get install libssl-dev 24 | - name: Install PKG Config 25 | run: sudo apt-get install pkg-config 26 | - name: Export PKG for OpenSSL 27 | run: export PKG_CONFIG_PATH=/usr/lib/x86_64-linux-gnu/pkgconfig 28 | - name: Build 29 | run: cargo build 30 | - name: Run tests 31 | run: cargo test --verbose 32 | 33 | windows_build: 34 | name: Windows Build 35 | runs-on: windows-latest 36 | steps: 37 | - uses: actions/checkout@v2 38 | - name: Setup Rust Toolchain 39 | uses: actions-rs/toolchain@v1.0.6 40 | with: 41 | toolchain: nightly 42 | - name: Build 43 | run: cargo build 44 | - name: Run tests 45 | run: cargo test --verbose 46 | 47 | mac_build: 48 | name: Mac Build 49 | runs-on: macos-latest 50 | steps: 51 | - uses: actions/checkout@v2 52 | - name: Setup Rust Toolchain 53 | uses: actions-rs/toolchain@v1.0.6 54 | with: 55 | toolchain: nightly 56 | - name: Build 57 | run: cargo build 58 | - name: Run tests 59 | run: cargo test --verbose 60 | 61 | -------------------------------------------------------------------------------- /.github/workflows/rust-clippy.yml: -------------------------------------------------------------------------------- 1 | # This workflow uses actions that are not certified by GitHub. 2 | # They are provided by a third-party and are governed by 3 | # separate terms of service, privacy policy, and support 4 | # documentation. 5 | # rust-clippy is a tool that runs a bunch of lints to catch common 6 | # mistakes in your Rust code and help improve your Rust code. 7 | # More details at https://github.com/rust-lang/rust-clippy 8 | # and https://rust-lang.github.io/rust-clippy/ 9 | 10 | name: rust-clippy analyze 11 | 12 | on: 13 | push: 14 | branches: [ "main", "develop" ] 15 | pull_request: 16 | # The branches below must be a subset of the branches above 17 | branches: [ "main", "develop" ] 18 | schedule: 19 | - cron: '38 11 * * 6' 20 | 21 | jobs: 22 | rust-clippy-analyze: 23 | name: Run rust-clippy analyzing 24 | runs-on: ubuntu-latest 25 | permissions: 26 | contents: read 27 | security-events: write 28 | actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status 29 | steps: 30 | - name: Checkout code 31 | uses: actions/checkout@v2 32 | 33 | - name: Install Rust toolchain 34 | uses: actions-rs/toolchain@16499b5e05bf2e26879000db0c1d13f7e13fa3af #@v1 35 | with: 36 | profile: minimal 37 | toolchain: stable 38 | components: clippy 39 | override: true 40 | 41 | - name: Install required cargo 42 | run: cargo install clippy-sarif sarif-fmt 43 | 44 | - name: Run rust-clippy 45 | run: 46 | cargo clippy 47 | --all-features 48 | --message-format=json | clippy-sarif | tee rust-clippy-results.sarif | sarif-fmt 49 | continue-on-error: true 50 | 51 | - name: Upload analysis results to GitHub 52 | uses: github/codeql-action/upload-sarif@v1 53 | with: 54 | sarif_file: rust-clippy-results.sarif 55 | wait-for-processing: true 56 | -------------------------------------------------------------------------------- /src/main.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | #[macro_use] 18 | extern crate log; 19 | 20 | use std::env::consts::{ 21 | ARCH, DLL_EXTENSION, DLL_PREFIX, DLL_SUFFIX, EXE_EXTENSION, EXE_SUFFIX, FAMILY, OS, 22 | }; 23 | use std::fs::File; 24 | 25 | use anyhow::Error; 26 | use log::LevelFilter; 27 | use simplelog::{ 28 | ColorChoice, CombinedLogger, Config, ConfigBuilder, TermLogger, TerminalMode, WriteLogger, 29 | }; 30 | 31 | use crate::program::Program; 32 | 33 | mod e621; 34 | mod program; 35 | 36 | fn main() -> Result<(), Error> { 37 | initialize_logger(); 38 | log_system_information(); 39 | 40 | let program = Program::new(); 41 | program.run() 42 | } 43 | 44 | /// Initializes the logger with preset filtering. 45 | fn initialize_logger() { 46 | let mut config = ConfigBuilder::new(); 47 | config.add_filter_allow_str("e621_downloader"); 48 | 49 | CombinedLogger::init(vec![ 50 | TermLogger::new( 51 | LevelFilter::Info, 52 | Config::default(), 53 | TerminalMode::Mixed, 54 | ColorChoice::Auto, 55 | ), 56 | WriteLogger::new( 57 | LevelFilter::max(), 58 | config.build(), 59 | File::create("e621_downloader.log").unwrap(), 60 | ), 61 | ]) 62 | .unwrap(); 63 | } 64 | 65 | /// Logs important information about the system being used. 66 | fn log_system_information() { 67 | trace!("Printing system information out into log for debug purposes..."); 68 | trace!("ARCH: \"{}\"", ARCH); 69 | trace!("DLL_EXTENSION: \"{}\"", DLL_EXTENSION); 70 | trace!("DLL_PREFIX: \"{}\"", DLL_PREFIX); 71 | trace!("DLL_SUFFIX: \"{}\"", DLL_SUFFIX); 72 | trace!("EXE_EXTENSION: \"{}\"", EXE_EXTENSION); 73 | trace!("EXE_SUFFIX: \"{}\"", EXE_SUFFIX); 74 | trace!("FAMILY: \"{}\"", FAMILY); 75 | trace!("OS: \"{}\"", OS); 76 | } 77 | -------------------------------------------------------------------------------- /src/e621/tui/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | use std::time::Duration; 19 | 20 | use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle}; 21 | 22 | /// A builder that helps in making a new [ProgressStyle] for use. 23 | pub(crate) struct ProgressStyleBuilder { 24 | /// The [ProgressStyle] being built. 25 | progress_style: ProgressStyle, 26 | } 27 | 28 | impl ProgressStyleBuilder { 29 | /// Sets the template of the progress style. 30 | /// 31 | /// # Arguments 32 | /// 33 | /// * `msg_template`: The template to use. 34 | /// 35 | /// returns: ProgressStyleBuilder 36 | pub(crate) fn template(mut self, msg_template: &str) -> Self { 37 | self.progress_style = self.progress_style.template(msg_template).unwrap(); 38 | self 39 | } 40 | 41 | /// Sets the progress style chars. 42 | /// 43 | /// # Arguments 44 | /// 45 | /// * `chars`: Progress chars to use. 46 | /// 47 | /// returns: ProgressStyleBuilder 48 | pub(crate) fn progress_chars(mut self, chars: &str) -> Self { 49 | self.progress_style = self.progress_style.progress_chars(chars); 50 | self 51 | } 52 | 53 | /// Builds and returns the new [ProgressStyle]. 54 | pub(crate) fn build(self) -> ProgressStyle { 55 | self.progress_style 56 | } 57 | } 58 | 59 | impl Default for ProgressStyleBuilder { 60 | fn default() -> Self { 61 | Self { 62 | progress_style: ProgressStyle::default_bar(), 63 | } 64 | } 65 | } 66 | 67 | /// A builder that helps in initializing and configuring a new [ProgressBar] for use. 68 | pub(crate) struct ProgressBarBuilder { 69 | /// The [ProgressBar] to build. 70 | pub(crate) progress_bar: ProgressBar, 71 | } 72 | 73 | impl ProgressBarBuilder { 74 | /// Creates new instance of the builder. 75 | /// 76 | /// # Arguments 77 | /// 78 | /// * `len`: Total length of the progress bar. 79 | /// 80 | /// returns: ProgressBarBuilder 81 | pub(crate) fn new(len: u64) -> Self { 82 | Self { 83 | progress_bar: ProgressBar::new(len), 84 | } 85 | } 86 | 87 | /// Sets the style of the progress bar to the style given. 88 | /// 89 | /// # Arguments 90 | /// 91 | /// * `progress_style`: The style to set the progress bar to. 92 | /// 93 | /// returns: ProgressBarBuilder 94 | pub(crate) fn style(self, progress_style: ProgressStyle) -> Self { 95 | self.progress_bar.set_style(progress_style); 96 | self 97 | } 98 | 99 | /// Sets the draw target (output) of the progress bar to the target given. 100 | /// 101 | /// # Arguments 102 | /// 103 | /// * `target`: The output draw target. 104 | /// 105 | /// returns: ProgressBarBuilder 106 | pub(crate) fn draw_target(self, target: ProgressDrawTarget) -> Self { 107 | self.progress_bar.set_draw_target(target); 108 | self 109 | } 110 | 111 | /// Resets the progress bar state to update it. 112 | pub(crate) fn reset(self) -> Self { 113 | self.progress_bar.reset(); 114 | self 115 | } 116 | 117 | /// Sets the steady tick's duration to the given duration. 118 | /// 119 | /// # Arguments 120 | /// 121 | /// * `duration`: Steady tick duration. 122 | /// 123 | /// returns: ProgressBarBuilder 124 | pub(crate) fn steady_tick(self, duration: Duration) -> Self { 125 | self.progress_bar.enable_steady_tick(duration); 126 | self 127 | } 128 | 129 | /// Returns the newly built progress bar. 130 | pub(crate) fn build(self) -> ProgressBar { 131 | self.progress_bar 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/program.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::env::current_dir; 18 | use std::fs::write; 19 | use std::path::Path; 20 | 21 | use console::Term; 22 | use anyhow::Error; 23 | 24 | use crate::e621::E621WebConnector; 25 | use crate::e621::io::{Config, emergency_exit, Login}; 26 | use crate::e621::io::tag::{parse_tag_file, TAG_FILE_EXAMPLE, TAG_NAME}; 27 | use crate::e621::sender::RequestSender; 28 | 29 | /// The name of the cargo package. 30 | const NAME: &str = env!("CARGO_PKG_NAME"); 31 | 32 | /// The version of the cargo package. 33 | const VERSION: &str = env!("CARGO_PKG_VERSION"); 34 | 35 | /// The authors who created the package. 36 | const AUTHORS: &str = env!("CARGO_PKG_AUTHORS"); 37 | 38 | /// A program class that handles the flow of the downloader user experience and steps of execution. 39 | pub(crate) struct Program; 40 | 41 | impl Program { 42 | /// Creates a new instance of the program. 43 | pub(crate) fn new() -> Self { 44 | Self 45 | } 46 | 47 | /// Runs the downloader program. 48 | pub(crate) fn run(&self) -> Result<(), Error> { 49 | Term::stdout().set_title("e621 downloader"); 50 | trace!("Starting e621 downloader..."); 51 | trace!("Program Name: {}", NAME); 52 | trace!("Program Version: {}", VERSION); 53 | trace!("Program Authors: {}", AUTHORS); 54 | trace!( 55 | "Program Working Directory: {}", 56 | current_dir() 57 | .expect("Unable to get working directory!") 58 | .to_str() 59 | .unwrap() 60 | ); 61 | 62 | // Check the config file and ensures that it is created. 63 | trace!("Checking if config file exists..."); 64 | if !Config::config_exists() { 65 | trace!("Config file doesn't exist..."); 66 | info!("Creating config file..."); 67 | Config::create_config()?; 68 | } 69 | 70 | // Create tag if it doesn't exist. 71 | trace!("Checking if tag file exists..."); 72 | if !Path::new(TAG_NAME).exists() { 73 | info!("Tag file does not exist, creating tag file..."); 74 | write(TAG_NAME, TAG_FILE_EXAMPLE)?; 75 | trace!("Tag file \"{}\" created...", TAG_NAME); 76 | 77 | emergency_exit( 78 | "The tag file is created, the application will close so you can include \ 79 | the artists, sets, pools, and individual posts you wish to download.", 80 | ); 81 | } 82 | 83 | // Creates connector and requester to prepare for downloading posts. 84 | let login = Login::get(); 85 | trace!("Login information loaded..."); 86 | trace!("Login Username: {}", login.username()); 87 | trace!("Login API Key: {}", "*".repeat(login.api_key().len())); 88 | trace!("Login Download Favorites: {}", login.download_favorites()); 89 | 90 | let request_sender = RequestSender::new(); 91 | let mut connector = E621WebConnector::new(&request_sender); 92 | connector.should_enter_safe_mode(); 93 | 94 | // Parses tag file. 95 | trace!("Parsing tag file..."); 96 | let groups = parse_tag_file(&request_sender)?; 97 | 98 | // Collects all grabbed posts and moves it to connector to start downloading. 99 | if !login.is_empty() { 100 | trace!("Parsing user blacklist..."); 101 | connector.process_blacklist(); 102 | } else { 103 | trace!("Skipping blacklist as user is not logged in..."); 104 | } 105 | 106 | connector.grab_all(&groups); 107 | connector.download_posts(); 108 | 109 | info!("Finished downloading posts!"); 110 | info!("Exiting..."); 111 | 112 | Ok(()) 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /src/e621/io/parser.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use crate::e621::io::emergency_exit; 18 | 19 | /// A parser that's responsible for parsing files character-by-character without any inherit rule. 20 | /// 21 | /// This is a thin blanket for other parsers to use and build rules, allowing for quick and easy 22 | /// parsing for any file. 23 | #[derive(Default)] 24 | pub(crate) struct BaseParser { 25 | /// Current cursor position in the array of characters. 26 | pos: usize, 27 | /// Input used for parsing. 28 | input: String, 29 | /// The current column being parsed. 30 | current_column: usize, 31 | /// The total number of characters in the input. 32 | total_len: usize, 33 | /// The total number of columns in the input. 34 | total_columns: usize, 35 | } 36 | 37 | impl BaseParser { 38 | /// Creates a new `BaseParser` with the given input. 39 | pub(crate) fn new(input: String) -> Self { 40 | let mut parser = BaseParser { 41 | input: input.trim().to_string(), 42 | total_len: input.len(), 43 | ..Default::default() 44 | }; 45 | // total columns is calculated by counting every instance of a newline character. 46 | parser.total_columns = parser.input.matches('\n').count(); 47 | parser 48 | } 49 | 50 | /// Consume and discard zero or more whitespace characters. 51 | pub(crate) fn consume_whitespace(&mut self) { 52 | self.consume_while(char::is_whitespace); 53 | } 54 | 55 | /// /// Consumes characters until `test` returns false. 56 | /// 57 | /// # Arguments 58 | /// 59 | /// * `test`: The function to test against. 60 | /// 61 | /// returns: String 62 | pub(crate) fn consume_while(&mut self, test: F) -> String 63 | where 64 | F: Fn(char) -> bool, 65 | { 66 | let mut result = String::new(); 67 | while !self.eof() && test(self.next_char()) { 68 | result.push(self.consume_char()); 69 | } 70 | 71 | result 72 | } 73 | 74 | /// Returns current char and pushes `self.pos` to the next char. 75 | pub(crate) fn consume_char(&mut self) -> char { 76 | let mut iter = self.get_current_input().char_indices(); 77 | let (_, cur_char) = iter.next().unwrap(); 78 | let (next_pos, next_char) = iter.next().unwrap_or((1, ' ')); 79 | 80 | // If next char is a newline, increment the column count. 81 | if next_char == '\n' || next_char == '\r' { 82 | self.current_column += 1; 83 | } 84 | 85 | self.pos += next_pos; 86 | cur_char 87 | } 88 | 89 | /// Read the current char without consuming it. 90 | pub(crate) fn next_char(&mut self) -> char { 91 | self.get_current_input().chars().next().unwrap() 92 | } 93 | 94 | /// Checks if the current input starts with the given string. 95 | /// 96 | /// # Arguments 97 | /// 98 | /// * `s`: The string to compare the start of the current input with. 99 | /// 100 | /// returns: bool 101 | pub(crate) fn starts_with(&self, s: &str) -> bool { 102 | self.get_current_input().starts_with(s) 103 | } 104 | 105 | /// Gets current input from current `pos` onward. 106 | pub(crate) fn get_current_input(&self) -> &str { 107 | &self.input[self.pos..] 108 | } 109 | 110 | /// Checks whether or not `pos` is at end of file. 111 | pub(crate) fn eof(&self) -> bool { 112 | self.pos >= self.input.len() 113 | } 114 | 115 | /// Reports an error to the parser so that it can exit gracefully. 116 | /// 117 | /// This will print a message to the console through the `error!` macro. 118 | /// After this, it will also attach the current character number and column number to the message. 119 | /// 120 | /// # Arguments 121 | /// 122 | /// * `msg`: Error message to print. 123 | pub(crate) fn report_error(&self, msg: &str) { 124 | error!( 125 | "Error parsing file at character {} (column {}): {msg}", 126 | self.pos, self.current_column 127 | ); 128 | trace!( 129 | "Total characters: {}, total columns: {}", 130 | self.total_len, 131 | self.total_columns 132 | ); 133 | 134 | emergency_exit("Parser error encountered."); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/e621/io/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::fs::{read_to_string, write}; 18 | use std::io; 19 | use std::path::Path; 20 | use std::process::exit; 21 | 22 | use anyhow::Error; 23 | use once_cell::sync::OnceCell; 24 | use serde::{Deserialize, Serialize}; 25 | use serde_json::{from_str, to_string_pretty}; 26 | 27 | pub(crate) mod parser; 28 | pub(crate) mod tag; 29 | 30 | /// Name of the configuration file. 31 | pub(crate) const CONFIG_NAME: &str = "config.json"; 32 | 33 | /// Name of the login file. 34 | pub(crate) const LOGIN_NAME: &str = "login.json"; 35 | 36 | /// Config that is used to do general setup. 37 | #[derive(Serialize, Deserialize, Debug, Clone)] 38 | pub(crate) struct Config { 39 | /// The location of the download directory. 40 | #[serde(rename = "downloadDirectory")] 41 | download_directory: String, 42 | /// The file naming convention (e.g "md5", "id"). 43 | #[serde(rename = "fileNamingConvention")] 44 | naming_convention: String, 45 | } 46 | 47 | static CONFIG: OnceCell = OnceCell::new(); 48 | 49 | impl Config { 50 | /// The location of the download directory. 51 | pub(crate) fn download_directory(&self) -> &str { 52 | &self.download_directory 53 | } 54 | 55 | /// The file naming convention (e.g "md5", "id"). 56 | pub(crate) fn naming_convention(&self) -> &str { 57 | &self.naming_convention 58 | } 59 | 60 | /// Checks config and ensure it isn't missing. 61 | pub(crate) fn config_exists() -> bool { 62 | if !Path::new(CONFIG_NAME).exists() { 63 | trace!("config.json: does not exist!"); 64 | return false; 65 | } 66 | 67 | true 68 | } 69 | 70 | /// Creates config file. 71 | pub(crate) fn create_config() -> Result<(), Error> { 72 | let json = to_string_pretty(&Config::default())?; 73 | write(Path::new(CONFIG_NAME), json)?; 74 | 75 | Ok(()) 76 | } 77 | 78 | /// Get the global instance of the `Config`. 79 | pub(crate) fn get() -> &'static Self { 80 | CONFIG.get_or_init(|| Self::get_config().unwrap()) 81 | } 82 | 83 | /// Loads and returns `config` for quick management and settings. 84 | fn get_config() -> Result { 85 | let mut config: Config = from_str(&read_to_string(CONFIG_NAME).unwrap())?; 86 | config.naming_convention = config.naming_convention.to_lowercase(); 87 | let convention = ["md5", "id"]; 88 | if !convention 89 | .iter() 90 | .any(|e| *e == config.naming_convention.as_str()) 91 | { 92 | error!( 93 | "There is no naming convention {}!", 94 | config.naming_convention 95 | ); 96 | info!("The naming convention can only be [\"md5\", \"id\"]"); 97 | emergency_exit("Naming convention is incorrect!"); 98 | } 99 | 100 | Ok(config) 101 | } 102 | } 103 | 104 | impl Default for Config { 105 | /// The default configuration for `Config`. 106 | fn default() -> Self { 107 | Config { 108 | download_directory: String::from("downloads/"), 109 | naming_convention: String::from("md5"), 110 | } 111 | } 112 | } 113 | 114 | /// `Login` contains all login information for obtaining information about a certain user. 115 | /// This is currently only used for the blacklist. 116 | #[derive(Serialize, Deserialize, Clone)] 117 | pub(crate) struct Login { 118 | /// Username of user. 119 | #[serde(rename = "Username")] 120 | username: String, 121 | /// The password hash (also known as the API key) for the user. 122 | #[serde(rename = "APIKey")] 123 | api_key: String, 124 | /// Whether or not the user wishes to download their favorites. 125 | #[serde(rename = "DownloadFavorites")] 126 | download_favorites: bool, 127 | } 128 | 129 | static LOGIN: OnceCell = OnceCell::new(); 130 | 131 | impl Login { 132 | /// Username of user. 133 | pub(crate) fn username(&self) -> &str { 134 | &self.username 135 | } 136 | 137 | /// The password hash (also known as the API key) for the user. 138 | pub(crate) fn api_key(&self) -> &str { 139 | &self.api_key 140 | } 141 | 142 | /// Whether or not the user wishes to download their favorites. 143 | pub(crate) fn download_favorites(&self) -> bool { 144 | self.download_favorites 145 | } 146 | 147 | /// Gets the global instance of [Login]. 148 | pub(crate) fn get() -> &'static Self { 149 | LOGIN.get_or_init(|| Self::load().unwrap_or_else(|e| { 150 | error!("Unable to load `login.json`. Error: {}", e); 151 | warn!("The program will use default values, but it is highly recommended to check your login.json file to \ 152 | ensure that everything is correct."); 153 | Login::default() 154 | })) 155 | } 156 | 157 | /// Loads the login file or creates one if it doesn't exist. 158 | fn load() -> Result { 159 | let mut login = Login::default(); 160 | let login_path = Path::new(LOGIN_NAME); 161 | if login_path.exists() { 162 | login = from_str(&read_to_string(login_path)?)?; 163 | } else { 164 | login.create_login()?; 165 | } 166 | 167 | Ok(login) 168 | } 169 | 170 | /// Checks if the login user and password is empty. 171 | pub(crate) fn is_empty(&self) -> bool { 172 | if self.username.is_empty() || self.api_key.is_empty() { 173 | return true; 174 | } 175 | 176 | false 177 | } 178 | 179 | /// Creates a new login file. 180 | fn create_login(&self) -> Result<(), Error> { 181 | write(LOGIN_NAME, to_string_pretty(self)?)?; 182 | 183 | info!("The login file was created."); 184 | info!( 185 | "If you wish to use your Blacklist, \ 186 | be sure to give your username and API hash key." 187 | ); 188 | info!( 189 | "Do not give out your API hash unless you trust this software completely, \ 190 | always treat your API hash like your own password." 191 | ); 192 | 193 | Ok(()) 194 | } 195 | } 196 | 197 | impl Default for Login { 198 | /// The default state for the login if none exists. 199 | fn default() -> Self { 200 | Login { 201 | username: String::new(), 202 | api_key: String::new(), 203 | download_favorites: true, 204 | } 205 | } 206 | } 207 | 208 | /// Exits the program after message explaining the error and prompting the user to press `ENTER`. 209 | /// 210 | /// # Arguments 211 | /// 212 | /// * `error`: The error message to print. 213 | pub(crate) fn emergency_exit(error: &str) { 214 | info!("{error}"); 215 | println!("Press ENTER to close the application..."); 216 | 217 | let mut line = String::new(); 218 | io::stdin().read_line(&mut line).unwrap_or_default(); 219 | 220 | exit(0x00FF); 221 | } 222 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # E621 Downloader 2 | ![Release](https://img.shields.io/github/release/McSib/e621_downloader.svg) 3 | ![Commits Since](https://img.shields.io/github/commits-since/McSib/e621_downloader/latest.svg) 4 | ![Stars](https://img.shields.io/github/stars/McSib/e621_downloader.svg) 5 | ![Watchers](https://img.shields.io/github/watchers/McSib/e621_downloader.svg) 6 | ![Forks](https://img.shields.io/github/forks/McSib/e621_downloader.svg) 7 | 8 | ![Maintained?](https://img.shields.io/badge/Maintained%3F-yes-green.svg) 9 | ![Lisence](https://img.shields.io/github/license/McSib/e621_downloader.svg) 10 | ![Downloads](https://img.shields.io/github/downloads/McSib/e621_downloader/total.svg) 11 | 12 | ![Issues Open](https://img.shields.io/github/issues/McSib/e621_downloader.svg) 13 | ![Issues Closed](https://img.shields.io/github/issues-closed/McSib/e621_downloader.svg) 14 | ![PR Open](https://img.shields.io/github/issues-pr/McSib/e621_downloader.svg) 15 | ![PR Closed](https://img.shields.io/github/issues-pr-closed/McSib/e621_downloader.svg) 16 | 17 | [![Rust](https://github.com/McSib/e621_downloader/actions/workflows/rust.yml/badge.svg?branch=active)](https://github.com/McSib/e621_downloader/actions/workflows/rust.yml) 18 | 19 | The `e621_downloader` is a low-level, close-to-hardware program meant to download a large number of images at a fast pace. It can handle bulk posts, single posts, sets, and pools via a custom easy-to-read language that I made. 20 | 21 | Having tested this software extensively with downloading, I managed to download 10,000 posts (which was over 20+ GB) in just two hours, averaging around 20MB/s. 22 | 23 | ### Goal 24 | 25 | The goal of this application is to keep up-to-date with your favorite artist, download pools, grab images from normal, everyday tags, while most of all, staying reliable. 26 | 27 | ## About E621/E926 28 | 29 | E621 is a mature image board replacement for the image board Sidechan. A general audience image board, e926 (formerly e961) complements this site. E621 runs off of the Ouroboros platform, a danbooru styled software specifically designed for the site. 30 | 31 | E621 has over 2,929,253+ images and videos hosted on its platform. 32 | 33 | ## Todo list for future updates 34 | 35 | This list is constantly updated and checked for new features and addons that need to be implemented in the future. My hope with all this work is to have a downloader that will last a long time for the next forseable future, as there are many downloaders like this that have either ceased developement, or found to be too hard or confusing to operate for the average person. 36 | 37 | - [ ] Add a menu system with configuration editing, tag editing, and download configuration built in. 38 | - [ ] Transition the tag file language into Json to integrate easily with the menu system. 39 | - [ ] Update the code to be more sound, structured, and faster. 40 | 41 | # Installation Guide (Windows) 42 | 1. If you are on Windows, simply visit this [link](https://rustup.rs) and install rust and cargo through the installer provided. _You will need GCC or MSVC in order to compile the project, so choose either 1 or 2 for this._ 43 | 1. To get GCC, I would recommend [this](https://winlibs.com) helpful little site which contains the most up to date versions of GCC. Note, however, that you will need to unzip this in a directory you make, and will have to link the bin folder in that directory to your `PATH` for it to work. 44 | 2. For MSVC, just go to this [link](https://visualstudio.microsoft.com/downloads/?q=build+tools) and download the Visual Studio Build Tools (at the very bottom of the page), which will install all the needed binaries without the full Visual Studio IDE. 45 | 46 | 2. Now, once you've done that, you can either clone the GitHub project directly through Git, or download a zip of the latest version. You can download Git from [here](https://git-scm.com/downloads). 47 | - If you choose to use Git, find a proper directory you want the project in, and then type in `git clone https://github.com/McSib/e621_downloader.git` into a console and pressing Enter. This will clone the directory and prepare the project for you to modify or just compile. 48 | 3. No matter what option you chose, you want to open a terminal (CMD or Terminal) and go into the root directory of the project (where Cargo.toml and Cargo.lock are located). Inside this directory, type in `cargo build` or `cargo build --release`. If the program compiles and works, you're good to go. 49 | 50 | # Installation Guide (Arch Linux) 51 | 1. For Arch Linux users, you will need a couple things installed in order to get the project up and running. The first thing you want to do is get the packages required (if you haven't). Run this command to download everything you need. 52 | 53 | ``` 54 | sudo pacman -S rust base-devel openssl git gdb 55 | ``` 56 | 57 | 2. The next thing you will need to do is clone the git repository in a directory of your choosing. 58 | 59 | ``` 60 | git clone https://github.com/McSib/e621_downloader.git 61 | ``` 62 | 63 | 3. From there, go into the newly cloned directory, and see if you can build by running `cargo build --release` or `cargo build`. If it compiled okay, then you are good to go. 64 | 65 | 4. **You can also now download a prebuilt binary of the program on the release page if you just want to use the program with little hassle.** 66 | 67 | # Installation Guide (Debian) 68 | 1. This is very much like the Arch Linux setup with some minor tweaks to the package download command. Instead of the pacman command, enter: `sudo apt install gcc g++ gdb cargo libssl-dev git` and then follow step 2 from the Arch Linux installation forward. 69 | 70 | # FAQ 71 | 72 | ### Why does the program only grab only 1,280 posts with certain tags? 73 | 74 | When a tag passes the limit of **1,500** posts, it is **considered too large a collection for the software to download** as the size of all the files combined will not only put strain on the server, but on the program as well as the system it runs on. The program will opt to download only 5 pages worth of posts to compensate for this hard limit. The pages use the **highest post limit** the e621/e926 servers will allow, which is **320 posts per page**. In total, it will grab **1,280 posts as its maximum**. 75 | 76 | Something to keep a note of, depending on the type of tag, the program will either ignore or use this limit. This is handled low-level by categorizing the tag into two sections: **General** and **Special**. 77 | 78 | General will force the program to use the 1,280 post limit. The tags that register under this flag are as such: **General** (this is basic tags, such as `fur`, `smiling`, `open_mouth`), **Copyright** (any form of copyrighted media should always be considered too large to download in full), **Species** (since species are very close to general in terms of number of posts they can hold, it will be treated as such), and **Character in special cases** (when a character has greater than 1,500 posts tied to them, it will be considered a General tag to avoid longer wait times while downloading). 79 | 80 | Tags that register under the Special flag are as such: **Artist** (generally, if you are grabbing an artist's work directly, you plan to grab all their work for archiving purposes. Thus, it will always be considered Special), and **Character** (if the amount of posts tied to the character is below 1,500, it will be considered a Special tag and the program will download _all_ posts with the character in it). 81 | 82 | This system is more complex than what I have explained so far, but in a basic sense, this is how the downloading function works with tags directly. These checks and grabs happen with a tight-knit relationship that is carried with the parser and the downloader. The parser will help grab the number of posts and also categorize the tags to their correct spots while the downloader focuses on using these tag types to grab and download their posts correctly. 83 | 84 | Hopefully, this explains how and why the limit is there. 85 | 86 | # Notice for users using the new version (1.6.0 and newer) 87 | If you are not logged into e621, a filter (almost like a global blacklist) is applied. This blacklist will nullify any posts that fall under its settings. So, if you notice images you're trying to download aren't showing up, log in and then download it, otherwise, this filter will continue blacklisting them. 88 | 89 | # Notice for users using a VPN 90 | I have had a recurring "bug" that has shown in my issues the last couple of months, and they tend to crop up right after a new release, so I am going to supply a new notice for those using VPNs to prevent this becoming issue spam. There are users who are experiencing crashes consistently when parsing, obtaining blacklist, or downloading. It is an issue that is consistent, and each person thus far have been using a VPN with no other noticeable cause linked. After a multitude of testing, I have concluded that users using VPNs will occasionally have either e621 directly or Cloudflare prompt for a captcha, or a test for whether you are a robot. Since my program does not support GUI, or no tangible way of handling that, it will crash immediately. I have looked for fixes to this issue and have yet to find anything. So, if you are using a VPN, be warned, this can happen. The current work around for this issue is switching locations in the VPN (if you have that feature) or disabling the VPN altogether (if you have that option). I understand it is annoying, and can be a pain, but this is all I can do until I come across a fix. Sorry for the inconvenience, and apologies if you are some of the users experiencing this issue. -------------------------------------------------------------------------------- /src/e621/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::cell::RefCell; 18 | use std::fs::{create_dir_all, write}; 19 | use std::path::PathBuf; 20 | use std::rc::Rc; 21 | use std::time::Duration; 22 | 23 | use anyhow::Context; 24 | use dialoguer::Confirm; 25 | use indicatif::{ProgressBar, ProgressDrawTarget}; 26 | 27 | use crate::e621::blacklist::Blacklist; 28 | use crate::e621::grabber::{Grabber, Shorten}; 29 | use crate::e621::io::tag::Group; 30 | use crate::e621::io::{Config, Login}; 31 | use crate::e621::sender::entries::UserEntry; 32 | use crate::e621::sender::RequestSender; 33 | use crate::e621::tui::{ProgressBarBuilder, ProgressStyleBuilder}; 34 | 35 | pub(crate) mod blacklist; 36 | pub(crate) mod grabber; 37 | pub(crate) mod io; 38 | pub(crate) mod sender; 39 | pub(crate) mod tui; 40 | 41 | /// A web connector that manages how the API is called (through the [RequestSender]), how posts are grabbed 42 | /// (through [Grabber]), and how the posts are downloaded. 43 | pub(crate) struct E621WebConnector { 44 | /// The sender used for all API calls. 45 | request_sender: RequestSender, 46 | /// The config which is modified when grabbing posts. 47 | download_directory: String, 48 | /// Progress bar that displays the current progress in downloading posts. 49 | progress_bar: ProgressBar, 50 | /// Grabber which is responsible for grabbing posts. 51 | grabber: Grabber, 52 | /// The user's blacklist. 53 | blacklist: Rc>, 54 | } 55 | 56 | impl E621WebConnector { 57 | /// Creates instance of `Self` for grabbing and downloading posts. 58 | pub(crate) fn new(request_sender: &RequestSender) -> Self { 59 | E621WebConnector { 60 | request_sender: request_sender.clone(), 61 | download_directory: Config::get().download_directory().to_string(), 62 | progress_bar: ProgressBar::hidden(), 63 | grabber: Grabber::new(request_sender.clone(), false), 64 | blacklist: Rc::new(RefCell::new(Blacklist::new(request_sender.clone()))), 65 | } 66 | } 67 | 68 | /// Gets input and enters safe depending on user choice. 69 | pub(crate) fn should_enter_safe_mode(&mut self) { 70 | trace!("Prompt for safe mode..."); 71 | let confirm_prompt = Confirm::new() 72 | .with_prompt("Should enter safe mode?") 73 | .show_default(true) 74 | .default(false) 75 | .interact() 76 | .with_context(|| { 77 | error!("Failed to setup confirmation prompt!"); 78 | "Terminal unable to set up confirmation prompt..." 79 | }) 80 | .unwrap(); 81 | 82 | trace!("Safe mode decision: {confirm_prompt}"); 83 | if confirm_prompt { 84 | self.request_sender.update_to_safe(); 85 | self.grabber.set_safe_mode(true); 86 | } 87 | } 88 | 89 | /// Processes the blacklist and tokenizes for use when grabbing posts. 90 | pub(crate) fn process_blacklist(&mut self) { 91 | let username = Login::get().username(); 92 | let user: UserEntry = self 93 | .request_sender 94 | .get_entry_from_appended_id(username, "user"); 95 | if let Some(blacklist_tags) = user.blacklisted_tags { 96 | if !blacklist_tags.is_empty() { 97 | let blacklist = self.blacklist.clone(); 98 | blacklist 99 | .borrow_mut() 100 | .parse_blacklist(blacklist_tags) 101 | .cache_users(); 102 | self.grabber.set_blacklist(blacklist); 103 | } 104 | } 105 | } 106 | 107 | /// Creates `Grabber` and grabs all posts before returning a tuple containing all general posts and single posts 108 | /// (posts grabbed by its ID). 109 | /// 110 | /// # Arguments 111 | /// 112 | /// * `groups`: The groups to grab from. 113 | pub(crate) fn grab_all(&mut self, groups: &[Group]) { 114 | trace!("Grabbing posts..."); 115 | self.grabber.grab_favorites(); 116 | self.grabber.grab_posts_by_tags(groups); 117 | } 118 | 119 | /// Saves image to download directory. 120 | fn save_image(&self, file_path: &str, bytes: &[u8]) { 121 | write(file_path, bytes) 122 | .with_context(|| { 123 | error!("Failed to save image!"); 124 | "A downloaded image was unable to be saved..." 125 | }) 126 | .unwrap(); 127 | trace!("Saved {file_path}..."); 128 | } 129 | 130 | /// Removes invalid characters from directory path. 131 | /// 132 | /// # Arguments 133 | /// 134 | /// * `dir_name`: Directory name to remove invalid chars from. 135 | /// 136 | /// returns: String 137 | fn remove_invalid_chars(&self, dir_name: &str) -> String { 138 | dir_name 139 | .chars() 140 | .map(|e| match e { 141 | '?' | ':' | '*' | '<' | '>' | '\"' | '|' => '_', 142 | _ => e, 143 | }) 144 | .collect() 145 | } 146 | 147 | /// Processes `PostSet` and downloads all posts from it. 148 | fn download_collection(&mut self) { 149 | for collection in self.grabber.posts().iter() { 150 | let collection_name = collection.name(); 151 | let collection_category = collection.category(); 152 | let collection_posts = collection.posts(); 153 | let collection_count = collection_posts.len(); 154 | let short_collection_name = collection.shorten("..."); 155 | 156 | #[cfg(unix)] 157 | let static_path: PathBuf = [ 158 | &self.download_directory, 159 | collection.category(), 160 | &self.remove_invalid_chars(collection_name), 161 | ] 162 | .iter() 163 | .collect(); 164 | 165 | #[cfg(windows)] 166 | let mut static_path: PathBuf = [ 167 | &self.download_directory, 168 | collection.category(), 169 | &self.remove_invalid_chars(collection_name), 170 | ] 171 | .iter() 172 | .collect(); 173 | 174 | // This is put here to attempt to shorten the length of the path if it passes window's 175 | // max path length. 176 | #[cfg(windows)] 177 | const MAX_PATH: usize = 260; // Defined in Windows documentation. 178 | 179 | #[cfg(windows)] 180 | let start_path_len = static_path.as_os_str().len(); 181 | 182 | #[cfg(windows)] 183 | if start_path_len >= MAX_PATH { 184 | static_path = [ 185 | &self.download_directory, 186 | collection_category, 187 | &self.remove_invalid_chars(&collection.shorten('_')), 188 | ] 189 | .iter() 190 | .collect(); 191 | 192 | let new_len = static_path.as_os_str().len(); 193 | if new_len >= MAX_PATH { 194 | error!("Path is too long and crosses the {MAX_PATH} char limit.\ 195 | Please relocate the program to a directory closer to the root drive directory."); 196 | trace!("Path length: {new_len}"); 197 | } 198 | } 199 | 200 | trace!("Printing Collection Info:"); 201 | trace!("Collection Name: \"{collection_name}\""); 202 | trace!("Collection Category: \"{collection_category}\""); 203 | trace!("Collection Post Length: \"{collection_count}\""); 204 | trace!( 205 | "Static file path for this collection: \"{}\"", 206 | static_path.to_str().unwrap() 207 | ); 208 | 209 | for post in collection_posts { 210 | let file_path: PathBuf = [ 211 | &static_path.to_str().unwrap().to_string(), 212 | &self.remove_invalid_chars(post.name()), 213 | ] 214 | .iter() 215 | .collect(); 216 | 217 | if file_path.exists() { 218 | self.progress_bar 219 | .set_message("Duplicate found: skipping... "); 220 | self.progress_bar.inc(post.file_size() as u64); 221 | continue; 222 | } 223 | 224 | self.progress_bar 225 | .set_message(format!("Downloading: {short_collection_name} ")); 226 | 227 | let parent_path = file_path.parent().unwrap(); 228 | create_dir_all(parent_path) 229 | .with_context(|| { 230 | error!("Could not create directories for images!"); 231 | format!( 232 | "Directory path unable to be created...\nPath: \"{}\"", 233 | parent_path.to_str().unwrap() 234 | ) 235 | }) 236 | .unwrap(); 237 | 238 | let bytes = self 239 | .request_sender 240 | .download_image(post.url(), post.file_size()); 241 | self.save_image(file_path.to_str().unwrap(), &bytes); 242 | self.progress_bar.inc(post.file_size() as u64); 243 | } 244 | 245 | trace!("Collection {collection_name} is finished downloading..."); 246 | } 247 | } 248 | 249 | /// Initializes the progress bar for downloading process. 250 | /// 251 | /// # Arguments 252 | /// 253 | /// * `len`: The total bytes to download. 254 | fn initialize_progress_bar(&mut self, len: u64) { 255 | self.progress_bar = ProgressBarBuilder::new(len) 256 | .style( 257 | ProgressStyleBuilder::default() 258 | .template("{msg} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} {binary_bytes_per_sec} {eta}") 259 | .progress_chars("=>-") 260 | .build()) 261 | .draw_target(ProgressDrawTarget::stderr()) 262 | .reset() 263 | .steady_tick(Duration::from_secs(1)) 264 | .build(); 265 | } 266 | 267 | /// Downloads tuple of general posts and single posts. 268 | pub(crate) fn download_posts(&mut self) { 269 | // Initializes the progress bar for downloading. 270 | let length = self.get_total_file_size(); 271 | trace!("Total file size for all images grabbed is {length}KB"); 272 | self.initialize_progress_bar(length); 273 | self.download_collection(); 274 | self.progress_bar.finish_and_clear(); 275 | } 276 | 277 | /// Gets the total size (in KB) of every post image to be downloaded. 278 | fn get_total_file_size(&self) -> u64 { 279 | self.grabber 280 | .posts() 281 | .iter() 282 | .map(|e| e.posts().iter().map(|f| f.file_size() as u64).sum::()) 283 | .sum() 284 | } 285 | } 286 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 Alexander 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /src/e621/io/tag.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::fs::read_to_string; 18 | 19 | use anyhow::{Context, Error}; 20 | 21 | use crate::e621::io::emergency_exit; 22 | use crate::e621::io::parser::BaseParser; 23 | use crate::e621::sender::entries::TagEntry; 24 | use crate::e621::sender::RequestSender; 25 | 26 | /// Constant of the tag file's name. 27 | pub(crate) const TAG_NAME: &str = "tags.txt"; 28 | 29 | /// An example file for newly created tag files. 30 | pub(crate) const TAG_FILE_EXAMPLE: &str = include_str!("tags.txt"); 31 | 32 | /// A tag that can be either general or special. 33 | #[derive(Debug, Clone, PartialOrd, PartialEq, Eq)] 34 | pub(crate) enum TagSearchType { 35 | /// A general tag that is used for everything except artist and sometimes character (depending on the amount of posts tied to it) 36 | General, 37 | /// A special tag that is searched differently from general tags (artist and characters). 38 | Special, 39 | /// This is used only if the type of tag is `2` or its greater than `5`. 40 | None, 41 | } 42 | 43 | /// The type a tag can be. 44 | #[derive(Debug, Clone, PartialEq, Eq)] 45 | pub(crate) enum TagType { 46 | Pool, 47 | Set, 48 | General, 49 | Artist, 50 | Post, 51 | Unknown, 52 | } 53 | 54 | /// A tag that contains its name, search type, and tag type. 55 | #[derive(Debug, Clone, PartialEq, Eq)] 56 | pub(crate) struct Tag { 57 | /// The name of the tag. 58 | name: String, 59 | /// The search type of the tag. 60 | search_type: TagSearchType, 61 | /// The tag type of the tag. 62 | tag_type: TagType, 63 | } 64 | 65 | impl Tag { 66 | fn new(tag: &str, category: TagSearchType, tag_type: TagType) -> Self { 67 | Tag { 68 | name: String::from(tag), 69 | search_type: category, 70 | tag_type, 71 | } 72 | } 73 | 74 | /// The name of the tag. 75 | pub(crate) fn name(&self) -> &str { 76 | &self.name 77 | } 78 | 79 | /// The search type of the tag. 80 | pub(crate) fn search_type(&self) -> &TagSearchType { 81 | &self.search_type 82 | } 83 | 84 | /// The tag type of the tag. 85 | pub(crate) fn tag_type(&self) -> &TagType { 86 | &self.tag_type 87 | } 88 | } 89 | 90 | impl Default for Tag { 91 | fn default() -> Self { 92 | Tag { 93 | name: String::new(), 94 | search_type: TagSearchType::None, 95 | tag_type: TagType::Unknown, 96 | } 97 | } 98 | } 99 | 100 | /// Group object generated from parsed code. 101 | #[derive(Debug, Clone)] 102 | pub(crate) struct Group { 103 | /// The name of group. 104 | name: String, 105 | /// A [Vec] containing all the tags parsed. 106 | tags: Vec, 107 | } 108 | 109 | impl Group { 110 | pub(crate) fn new(name: String) -> Self { 111 | Group { 112 | name, 113 | tags: Vec::new(), 114 | } 115 | } 116 | 117 | /// The name of group. 118 | pub(crate) fn name(&self) -> &str { 119 | &self.name 120 | } 121 | 122 | /// A [Vec] containing all the tags parsed. 123 | pub(crate) fn tags(&self) -> &Vec { 124 | &self.tags 125 | } 126 | } 127 | 128 | /// Parses the tag file and returns the serialized form of it. 129 | /// 130 | /// # Arguments 131 | /// 132 | /// * `request_sender`: The sender to use for the API call (this is used for tag and alias checks). 133 | /// 134 | /// returns: Result, Error> 135 | pub(crate) fn parse_tag_file(request_sender: &RequestSender) -> Result, Error> { 136 | TagParser { 137 | parser: BaseParser::new( 138 | read_to_string(TAG_NAME) 139 | .with_context(|| { 140 | error!("Unable to read tag file!"); 141 | "Possible I/O block when trying to read tag file..." 142 | }) 143 | .unwrap(), 144 | ), 145 | request_sender: request_sender.clone(), 146 | } 147 | .parse_groups() 148 | } 149 | 150 | /// Identifier to help categorize tags. 151 | pub(crate) struct TagIdentifier { 152 | /// Request sender for making any needed API calls. 153 | request_sender: RequestSender, 154 | } 155 | 156 | impl TagIdentifier { 157 | /// Creates new identifier. 158 | fn new(request_sender: RequestSender) -> Self { 159 | TagIdentifier { request_sender } 160 | } 161 | 162 | /// Identifies tags to ensure they exist. 163 | /// 164 | /// # Arguments 165 | /// 166 | /// * `tags`: Tags to id. 167 | /// * `request_sender`: The sender to use for the API calls. 168 | /// 169 | /// returns: Tag 170 | fn id_tag(tags: &str, request_sender: RequestSender) -> Tag { 171 | let identifier = TagIdentifier::new(request_sender); 172 | identifier.search_for_tag(tags) 173 | } 174 | 175 | /// Search for tag on e621. 176 | /// 177 | /// # Arguments 178 | /// 179 | /// * `tags`: Tags to search for. 180 | /// 181 | /// returns: Tag 182 | fn search_for_tag(&self, tags: &str) -> Tag { 183 | // Splits the tags and cycles through each one, checking if they are valid and searchable tags 184 | // If the tag isn't searchable, the tag will default and consider itself invalid. Which will 185 | // then be filtered through the last step. 186 | let mut map = tags 187 | .split(' ') 188 | .map(|e| { 189 | let temp = e.trim_start_matches('-'); 190 | match self.request_sender.get_tags_by_name(temp).first() { 191 | Some(entry) => self.create_tag(tags, entry), 192 | None => { 193 | if let Some(alias_tag) = self.get_tag_from_alias(temp) { 194 | self.create_tag(tags, &alias_tag) 195 | } else if temp.contains(':') { 196 | Tag::default() 197 | } else { 198 | self.exit_tag_failure(temp); 199 | unreachable!(); 200 | } 201 | } 202 | } 203 | }) 204 | .filter(|e| *e != Tag::default()); 205 | 206 | // Tries to return any tag in the map with category special, return the last element otherwise. 207 | // If returning the last element fails, assume the tag is syntax only and default. 208 | map.find(|e| e.search_type == TagSearchType::Special) 209 | .unwrap_or_else(|| { 210 | map.last() 211 | .unwrap_or_else(|| Tag::new(tags, TagSearchType::General, TagType::General)) 212 | }) 213 | } 214 | 215 | /// Checks if the tag is an alias and searches for the tag it is aliased to, returning it. 216 | /// 217 | /// # Arguments 218 | /// 219 | /// * `tag`: Alias to check for. 220 | /// 221 | /// returns: Option 222 | fn get_tag_from_alias(&self, tag: &str) -> Option { 223 | let entry = match self.request_sender.query_aliases(tag) { 224 | Some(e) => e.first().unwrap().clone(), 225 | None => { 226 | return None; 227 | } 228 | }; 229 | 230 | // Is there possibly a way to make this better? 231 | Some( 232 | self.request_sender 233 | .get_tags_by_name(&entry.consequent_name) 234 | .first() 235 | .unwrap() 236 | .clone(), 237 | ) 238 | } 239 | 240 | /// Emergency exits if a tag isn't identified. 241 | /// 242 | /// # Arguments 243 | /// 244 | /// * `tag`: Tag to log for. 245 | fn exit_tag_failure(&self, tag: &str) { 246 | error!("{tag} is invalid!"); 247 | info!("The tag may be a typo, be sure to double check and ensure that the tag is correct."); 248 | emergency_exit(format!("The server API call was unable to find tag: {tag}!").as_str()); 249 | } 250 | 251 | /// Processes the tag type and creates the appropriate tag for it. 252 | /// 253 | /// # Arguments 254 | /// 255 | /// * `tags`: Tags to use. 256 | /// * `tag_entry`: The tag entry related to the tags. 257 | /// 258 | /// returns: Tag 259 | fn create_tag(&self, tags: &str, tag_entry: &TagEntry) -> Tag { 260 | let tag_type = tag_entry.to_tag_type(); 261 | let category = match tag_type { 262 | TagType::General => { 263 | const CHARACTER_CATEGORY: u8 = 4; 264 | if tag_entry.category == CHARACTER_CATEGORY { 265 | if tag_entry.post_count > 1500 { 266 | TagSearchType::General 267 | } else { 268 | TagSearchType::Special 269 | } 270 | } else { 271 | TagSearchType::General 272 | } 273 | } 274 | TagType::Artist => TagSearchType::Special, 275 | _ => unreachable!(), 276 | }; 277 | 278 | Tag::new(tags, category, tag_type) 279 | } 280 | } 281 | 282 | /// Parser that reads a tag file and parses the tags. 283 | struct TagParser { 284 | /// Low-level parser for parsing raw data. 285 | parser: BaseParser, 286 | /// Request sender for any needed API calls. 287 | request_sender: RequestSender, 288 | } 289 | 290 | impl TagParser { 291 | /// Parses each group with all tags tied to them before returning a vector with all groups in it. 292 | pub(crate) fn parse_groups(&mut self) -> Result, Error> { 293 | let mut groups: Vec = Vec::new(); 294 | loop { 295 | self.parser.consume_whitespace(); 296 | if self.parser.eof() { 297 | break; 298 | } 299 | 300 | if self.check_and_parse_comment() { 301 | continue; 302 | } 303 | 304 | if self.parser.starts_with("[") { 305 | groups.push(self.parse_group()); 306 | } else { 307 | self.parser.report_error("Tags must be in groups!"); 308 | } 309 | } 310 | 311 | Ok(groups) 312 | } 313 | 314 | /// Parses a group and all tags tied to it before returning the result. 315 | fn parse_group(&mut self) -> Group { 316 | assert_eq!(self.parser.consume_char(), '['); 317 | let group_name = self.parser.consume_while(valid_group); 318 | assert_eq!(self.parser.consume_char(), ']'); 319 | 320 | let mut group = Group::new(group_name); 321 | self.parse_tags(&mut group); 322 | 323 | group 324 | } 325 | 326 | /// Parses all tags for a group and stores it. 327 | /// 328 | /// # Arguments 329 | /// 330 | /// * `group`: The group to parse. 331 | fn parse_tags(&mut self, group: &mut Group) { 332 | let mut tags: Vec = Vec::new(); 333 | loop { 334 | self.parser.consume_whitespace(); 335 | if self.check_and_parse_comment() { 336 | continue; 337 | } 338 | 339 | if self.parser.starts_with("[") { 340 | break; 341 | } 342 | 343 | if self.parser.eof() { 344 | break; 345 | } 346 | 347 | tags.push(self.parse_tag(group.name())); 348 | } 349 | 350 | group.tags = tags; 351 | } 352 | 353 | /// Parses a single tag and identifies it before returning the result. 354 | /// 355 | /// # Arguments 356 | /// 357 | /// * `group_name`: Group name to parse the tag for. 358 | /// 359 | /// returns: Tag 360 | fn parse_tag(&mut self, group_name: &str) -> Tag { 361 | match group_name { 362 | "artists" | "general" => { 363 | let tag = self.parser.consume_while(valid_tag); 364 | TagIdentifier::id_tag(tag.trim(), self.request_sender.clone()) 365 | } 366 | e => { 367 | let temp_char = self.parser.next_char(); 368 | if !char::is_ascii_digit(&temp_char) && temp_char != '#' { 369 | panic!("Invalid tag type! Pools, sets, and single-post tags must be a number!"); 370 | } 371 | 372 | let tag = self.parser.consume_while(valid_id); 373 | let tag_type = match e { 374 | "pools" => TagType::Pool, 375 | "sets" => TagType::Set, 376 | "single-post" => TagType::Post, 377 | _ => { 378 | self.parser.report_error("Unknown tag type!"); 379 | TagType::Unknown 380 | } 381 | }; 382 | 383 | Tag::new(tag.trim(), TagSearchType::Special, tag_type) 384 | } 385 | } 386 | } 387 | 388 | /// Checks if next character is comment identifier and parses it if it is. 389 | fn check_and_parse_comment(&mut self) -> bool { 390 | if self.parser.starts_with("#") { 391 | self.parse_comment(); 392 | return true; 393 | } 394 | 395 | false 396 | } 397 | 398 | /// Skips over comment. 399 | fn parse_comment(&mut self) { 400 | self.parser.consume_while(valid_comment); 401 | } 402 | } 403 | 404 | /// Validates character for tag. 405 | /// 406 | /// # Arguments 407 | /// 408 | /// * `c`: The character to check. 409 | /// 410 | /// returns: bool 411 | fn valid_tag(c: char) -> bool { 412 | match c { 413 | ' '..='\"' | '$'..='~' => true, 414 | // This will check for any special characters in the validator. 415 | _ => { 416 | if c != '#' { 417 | return c.is_alphanumeric(); 418 | } 419 | 420 | false 421 | } 422 | } 423 | } 424 | 425 | /// Validates character for id. 426 | /// 427 | /// # Arguments 428 | /// 429 | /// * `c`: The character to check. 430 | /// 431 | /// returns: bool 432 | fn valid_id(c: char) -> bool { 433 | c.is_ascii_digit() 434 | } 435 | 436 | /// Validates character for group 437 | /// 438 | /// # Arguments 439 | /// 440 | /// * `c`: The character to check. 441 | /// 442 | /// returns: bool 443 | fn valid_group(c: char) -> bool { 444 | matches!(c, 'A'..='Z' | 'a'..='z' | '-') 445 | } 446 | 447 | /// Validates character for comment. 448 | /// 449 | /// # Arguments 450 | /// 451 | /// * `c`: The character to check. 452 | /// 453 | /// returns: bool 454 | fn valid_comment(c: char) -> bool { 455 | match c { 456 | ' '..='~' => true, 457 | _ => c.is_alphanumeric(), 458 | } 459 | } 460 | -------------------------------------------------------------------------------- /src/e621/sender/mod.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::any::type_name; 18 | use std::cell::RefCell; 19 | use std::collections::HashMap; 20 | use std::rc::Rc; 21 | use std::time::Duration; 22 | 23 | use anyhow::{Context, Result}; 24 | use reqwest::blocking::{Client, RequestBuilder, Response}; 25 | use reqwest::header::{AUTHORIZATION, USER_AGENT}; 26 | use serde::de::DeserializeOwned; 27 | use serde_json::{from_value, Value}; 28 | 29 | use crate::e621::io::{emergency_exit, Login}; 30 | use crate::e621::sender::entries::{AliasEntry, BulkPostEntry, PostEntry, TagEntry}; 31 | 32 | pub(crate) mod entries; 33 | 34 | /// Creates a hashmap through similar syntax of the `vec` macro. 35 | /// 36 | /// # Arguments 37 | /// * `x`: Represents multiple tuples being passed as parameter. 38 | /// 39 | /// # Example 40 | /// 41 | /// ```rust 42 | /// # use std::collections::hashmap; 43 | /// 44 | /// let hashmap = hashmap![("Testing", "testing"), ("Example", "example")]; 45 | /// 46 | /// assert_eq!(hashmap["Testing"], String::from("testing")); 47 | /// ``` 48 | #[macro_export] 49 | macro_rules! hashmap { 50 | ( $( $x:expr ),* ) => { 51 | { 52 | let mut hash_map: HashMap = HashMap::new(); 53 | $( 54 | let (calling_name, url) = $x; 55 | hash_map.insert(String::from(calling_name), String::from(url)); 56 | )* 57 | 58 | hash_map 59 | } 60 | }; 61 | } 62 | 63 | /// Default user agent value. 64 | const USER_AGENT_VALUE: &str = concat!( 65 | env!("CARGO_PKG_NAME"), 66 | "/", 67 | env!("CARGO_PKG_VERSION"), 68 | " (by ", 69 | env!("CARGO_PKG_AUTHORS"), 70 | " on e621)" 71 | ); 72 | 73 | /// A reference counted client used for all searches by the [Grabber], [Blacklist], [E621WebConnector], etc. 74 | struct SenderClient { 75 | /// [Client] wrapped in a [Rc] so only one instance of the client exists. This will prevent an overabundance of 76 | /// clients in the code. 77 | client: Rc, 78 | /// The base64 encrypted username and password of the user. This is passed only through the [AUTHORIZATION] header 79 | /// of the request and is a highly secured method of login through client. 80 | auth: Rc, 81 | } 82 | 83 | impl SenderClient { 84 | /// Creates root client. 85 | fn new(auth: String) -> Self { 86 | trace!("SenderClient initializing with USER_AGENT_VALUE \"{USER_AGENT_VALUE}\""); 87 | 88 | SenderClient { 89 | client: Rc::new(SenderClient::build_client()), 90 | auth: Rc::new(auth), 91 | } 92 | } 93 | 94 | /// Runs client through a builder to give it required settings. 95 | /// Cookies aren't stored in the client, TCP_NODELAY is on, and timeout is changed from 30 seconds to 60. 96 | fn build_client() -> Client { 97 | Client::builder() 98 | .use_rustls_tls() 99 | .http2_prior_knowledge() 100 | .tcp_keepalive(Duration::from_secs(30)) 101 | .tcp_nodelay(true) 102 | .timeout(Duration::from_secs(60)) 103 | .build() 104 | .unwrap_or_else(|_| Client::new()) 105 | } 106 | 107 | /// A wrapping function that acts the exact same as `self.client.get` but will instead attach the user agent header 108 | /// before returning the [RequestBuilder]. This will ensure that all requests sent have the proper user agent info. 109 | /// 110 | /// # Arguments 111 | /// 112 | /// * `url`: The url to request. 113 | /// 114 | /// returns: RequestBuilder 115 | pub(crate) fn get(&self, url: &str) -> RequestBuilder { 116 | self.client.get(url).header(USER_AGENT, USER_AGENT_VALUE) 117 | } 118 | 119 | /// This is the same as `self.get(url)` but will attach the authorization header with username and API hash. 120 | /// 121 | /// # Arguments 122 | /// 123 | /// * `url`: The url to request. 124 | /// 125 | /// returns: RequestBuilder 126 | pub(crate) fn get_with_auth(&self, url: &str) -> RequestBuilder { 127 | if self.auth.is_empty() { 128 | self.get(url) 129 | } else { 130 | self.get(url).header(AUTHORIZATION, self.auth.as_str()) 131 | } 132 | } 133 | } 134 | 135 | impl Clone for SenderClient { 136 | /// Creates a new instance of SenderClient, but clones the [Rc] of the root client, ensuring that all requests are 137 | /// going to the same client. 138 | fn clone(&self) -> Self { 139 | SenderClient { 140 | client: Rc::clone(&self.client), 141 | auth: Rc::clone(&self.auth), 142 | } 143 | } 144 | } 145 | 146 | /// A sender that handles direct calls to the API. 147 | /// 148 | /// This acts as a safety layer to ensure calls to the API are less error prone. 149 | pub(crate) struct RequestSender { 150 | /// The client that will be used to send all requests. 151 | /// 152 | /// Even though the [SenderClient] isn't wrapped in a [Rc], the main client inside of it is, this will ensure that 153 | /// all request are only sent through one client. 154 | client: SenderClient, 155 | /// All available urls to use with the sender. 156 | urls: Rc>>, 157 | } 158 | 159 | impl RequestSender { 160 | pub(crate) fn new() -> Self { 161 | let login = Login::get(); 162 | let auth = if login.is_empty() { 163 | String::new() 164 | } else { 165 | base64_url::encode(format!("{}:{}", login.username(), login.api_key()).as_str()) 166 | }; 167 | 168 | RequestSender { 169 | client: SenderClient::new(auth), 170 | urls: Rc::new(RefCell::new(RequestSender::initialize_url_map())), 171 | } 172 | } 173 | 174 | /// Initializes all the urls that will be used by the sender. 175 | fn initialize_url_map() -> HashMap { 176 | hashmap![ 177 | ("posts", "https://e621.net/posts.json"), 178 | ("pool", "https://e621.net/pools/"), 179 | ("set", "https://e621.net/post_sets/"), 180 | ("single", "https://e621.net/posts/"), 181 | ("blacklist", "https://e621.net/users/"), 182 | ("tag", "https://e621.net/tags/"), 183 | ("tag_bulk", "https://e621.net/tags.json"), 184 | ("alias", "https://e621.net/tag_aliases.json"), 185 | ("user", "https://e621.net/users/") 186 | ] 187 | } 188 | 189 | /// If the client authenticated or not. 190 | pub(crate) fn is_authenticated(&self) -> bool { 191 | !self.client.auth.is_empty() 192 | } 193 | 194 | /// Updates all the urls from e621 to e926. 195 | pub(crate) fn update_to_safe(&mut self) { 196 | self.urls 197 | .borrow_mut() 198 | .iter_mut() 199 | .for_each(|(_, value)| *value = value.replace("e621", "e926")); 200 | } 201 | 202 | /// If a request failed, this will output what type of error it is before exiting. 203 | /// 204 | /// # Arguments 205 | /// 206 | /// * `error`: The type of error thrown. 207 | fn output_error(&self, error: &reqwest::Error) { 208 | error!( 209 | "Error occurred from sent request. \ 210 | Error: {error}", 211 | ); 212 | trace!("Url where error occurred: {:#?}", error.url()); 213 | 214 | if let Some(status) = error.status() { 215 | let code = status.as_u16(); 216 | trace!("The response code from the server was: {code}"); 217 | 218 | const SERVER_INTERNAL: u16 = 500; 219 | const SERVER_RATE_LIMIT: u16 = 503; 220 | const CLIENT_FORBIDDEN: u16 = 403; 221 | const CLIENT_THROTTLED: u16 = 421; 222 | match code { 223 | SERVER_INTERNAL => { 224 | error!( 225 | "There was an error that happened internally in the servers, \ 226 | please try using the downloader later until the issue is solved." 227 | ); 228 | } 229 | SERVER_RATE_LIMIT => { 230 | error!( 231 | "Server could not handle the request, or the downloader has \ 232 | exceeded the rate-limit. Contact the developer immediately about this \ 233 | issue." 234 | ); 235 | } 236 | CLIENT_FORBIDDEN => { 237 | error!( 238 | "The client was forbidden from accessing the api, contact the \ 239 | developer immediately if this error occurs." 240 | ); 241 | } 242 | CLIENT_THROTTLED => { 243 | error!( 244 | "The user is throttled, thus the request is unsuccessful. \ 245 | Contact the developer immediately if this error occurs." 246 | ); 247 | } 248 | _ => { 249 | error!("Response code couldn't be posted..."); 250 | } 251 | } 252 | } 253 | 254 | emergency_exit("To prevent the program from crashing, it will do an emergency exit."); 255 | } 256 | 257 | /// Gets the response from a sent request and checks to ensure it was successful. 258 | /// 259 | /// # Arguments 260 | /// 261 | /// * `result`: The result to check. 262 | /// 263 | /// returns: Response 264 | fn check_response(&self, result: Result) -> Response { 265 | match result { 266 | Ok(response) => response, 267 | Err(ref error) => { 268 | self.output_error(error); 269 | unreachable!() 270 | } 271 | } 272 | } 273 | 274 | /// Sends request to download image. 275 | /// 276 | /// # Arguments 277 | /// 278 | /// * `url`: The url to the file to download. 279 | /// * `file_size`: The file size of the file. 280 | /// 281 | /// returns: Vec 282 | pub(crate) fn download_image(&self, url: &str, file_size: i64) -> Vec { 283 | let mut image_response = self.check_response(self.client.get(url).send()); 284 | let mut image_bytes: Vec = Vec::with_capacity(file_size as usize); 285 | image_response 286 | .copy_to(&mut image_bytes) 287 | .with_context(|| "Failed to download image!".to_string()) 288 | .unwrap(); 289 | 290 | image_bytes 291 | } 292 | 293 | /// Appends base url with id/name before ending with `.json`. 294 | /// 295 | /// # Arguments 296 | /// 297 | /// * `url`: The url to change. 298 | /// * `append`: The id/name. 299 | /// 300 | /// returns: String 301 | pub(crate) fn append_url(&self, url: &str, append: &str) -> String { 302 | format!("{url}{append}.json") 303 | } 304 | 305 | /// Gets entry by type `T`, this is used for every request where the url needs to be appended to. 306 | /// 307 | /// # Arguments 308 | /// 309 | /// * `id`: The id to search for. 310 | /// * `url_type_key`: The type of url to use. 311 | /// 312 | /// returns: T 313 | pub(crate) fn get_entry_from_appended_id(&self, id: &str, url_type_key: &str) -> T 314 | where 315 | T: DeserializeOwned, 316 | { 317 | let value: Value = self 318 | .check_response( 319 | self.client 320 | .get_with_auth(&self.append_url(&self.urls.borrow()[url_type_key], id)) 321 | .send(), 322 | ) 323 | .json() 324 | .with_context(|| { 325 | format!( 326 | "Json was unable to deserialize to \"{}\"!\n\ 327 | url_type_key: {}\n\ 328 | id: {}", 329 | type_name::(), 330 | url_type_key, 331 | id 332 | ) 333 | }) 334 | .unwrap(); 335 | 336 | let value = match url_type_key { 337 | "single" => value 338 | .get("post") 339 | .unwrap_or_else(|| { 340 | emergency_exit(&format!( 341 | "Post was not found! Post ID ({}) is invalid or post was deleted.", 342 | id 343 | )); 344 | unreachable!() 345 | }) 346 | .to_owned(), 347 | _ => value, 348 | }; 349 | 350 | from_value(value) 351 | .with_context(|| { 352 | error!("Could not convert entry to type \"{}\"!", type_name::()); 353 | "Unexpected error occurred when trying to perform conversion from value to entry \ 354 | type above." 355 | .to_string() 356 | }) 357 | .unwrap() 358 | } 359 | 360 | /// Performs a bulk search for posts using tags to filter the response. 361 | /// 362 | /// # Arguments 363 | /// 364 | /// * `searching_tag`: The tags for filtering. 365 | /// * `page`: The page to search for. 366 | /// 367 | /// returns: BulkPostEntry 368 | pub(crate) fn bulk_search(&self, searching_tag: &str, page: u16) -> BulkPostEntry { 369 | debug!("Downloading page {page} of tag {searching_tag}"); 370 | 371 | self.check_response( 372 | self.client 373 | .get_with_auth(&self.urls.borrow()["posts"]) 374 | .query(&[ 375 | ("tags", searching_tag), 376 | ("page", &format!("{page}")), 377 | ("limit", &320.to_string()), 378 | ]) 379 | .send(), 380 | ) 381 | .json() 382 | .with_context(|| { 383 | error!( 384 | "Unable to deserialize json to \"{}\"!", 385 | type_name::>() 386 | ); 387 | "Failed to perform bulk search...".to_string() 388 | }) 389 | .unwrap() 390 | } 391 | 392 | /// Gets tags by their name. 393 | /// 394 | /// # Arguments 395 | /// 396 | /// * `tag`: The name of the tag. 397 | /// 398 | /// returns: Vec 399 | pub(crate) fn get_tags_by_name(&self, tag: &str) -> Vec { 400 | let result: Value = self 401 | .check_response( 402 | self.client 403 | .get(&self.urls.borrow()["tag_bulk"]) 404 | .query(&[("search[name]", tag)]) 405 | .send(), 406 | ) 407 | .json() 408 | .with_context(|| { 409 | format!( 410 | "Json was unable to deserialize to \"{}\"!\n\ 411 | url_type_key: tag_bulk\n\ 412 | tag: {}", 413 | type_name::(), 414 | tag 415 | ) 416 | }) 417 | .unwrap(); 418 | if result.is_object() { 419 | vec![] 420 | } else { 421 | from_value::>(result) 422 | .with_context(|| { 423 | error!( 424 | "Unable to deserialize Value to \"{}\"!", 425 | type_name::>() 426 | ); 427 | "Failed to perform bulk search...".to_string() 428 | }) 429 | .unwrap() 430 | } 431 | } 432 | 433 | /// Queries aliases and returns response. 434 | /// 435 | /// # Arguments 436 | /// 437 | /// * `tag`: The alias to search for. 438 | /// 439 | /// returns: Option> 440 | /// 441 | /// # Examples 442 | /// 443 | /// ``` 444 | /// 445 | /// ``` 446 | pub(crate) fn query_aliases(&self, tag: &str) -> Option> { 447 | let result = self 448 | .check_response( 449 | self.client 450 | .get(&self.urls.borrow()["alias"]) 451 | .query(&[ 452 | ("commit", "Search"), 453 | ("search[name_matches]", tag), 454 | ("search[order]", "status"), 455 | ]) 456 | .send(), 457 | ) 458 | .json::>(); 459 | 460 | match result { 461 | Ok(e) => Some(e), 462 | Err(e) => { 463 | trace!("No alias was found for {tag}..."); 464 | trace!("Printing trace message for why None was returned..."); 465 | trace!("{}", e.to_string()); 466 | None 467 | } 468 | } 469 | } 470 | } 471 | 472 | impl Clone for RequestSender { 473 | fn clone(&self) -> Self { 474 | RequestSender { 475 | client: self.client.clone(), 476 | urls: Rc::clone(&self.urls), 477 | } 478 | } 479 | } 480 | -------------------------------------------------------------------------------- /src/e621/sender/entries.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use serde::{Deserialize, Serialize}; 18 | 19 | use crate::e621::io::tag::TagType; 20 | 21 | /// GET return of alias entry for e621/e926. 22 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 23 | pub(crate) struct AliasEntry { 24 | /// Alias ID. 25 | pub(crate) id: i64, 26 | /// Alias name. 27 | pub(crate) antecedent_name: String, 28 | /// Reason for the alias. 29 | pub(crate) reason: String, 30 | /// ID of the creator of the alias. 31 | pub(crate) creator_id: i64, 32 | /// The date the alias was created. 33 | pub(crate) created_at: Option, 34 | /// Forum post id tied to the request for the alias to be approved. 35 | pub(crate) forum_post_id: Option, 36 | /// The date for when the alias was updated. 37 | pub(crate) updated_at: Option, 38 | /// Forum topic ID for the thread where the request for alias approval was created. 39 | pub(crate) forum_topic_id: Option, 40 | /// Original tag name. 41 | pub(crate) consequent_name: String, 42 | /// Current status of the alias. 43 | /// Can be `approved`, `active`, `pending`, `deleted`, `retired`, `processing`, and `queued`. 44 | /// 45 | /// # Error 46 | /// Optionally, there can also be an `error` prompt with the following format: 47 | /// `"error: cannot update a new record"` 48 | /// ## Reason for Error 49 | /// This is probably an internal error with the server, and while it is exceptionally rare, 50 | /// there is still a probability. 51 | pub(crate) status: String, 52 | /// The amount of post the aliased tag is tied to. 53 | pub(crate) post_count: i64, 54 | /// ID of the user that approved the alias. 55 | pub(crate) approver_id: Option, 56 | } 57 | 58 | /// GET return of tag entry for e621/e926. 59 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 60 | pub(crate) struct TagEntry { 61 | /// Id of the tag. 62 | pub(crate) id: i64, 63 | /// Name of the tag. 64 | pub(crate) name: String, 65 | /// Amount of posts that uses the tag. 66 | pub(crate) post_count: i64, 67 | /// Related tags that this tag is commonly paired with. 68 | pub(crate) related_tags: String, 69 | /// Most recent date the `related_tags` was updated. 70 | pub(crate) related_tags_updated_at: String, 71 | /// The type of tag it is. 72 | /// 73 | /// This tag can be the following types: 74 | /// - `0`: General; 75 | /// - `1`: Artist; 76 | /// - `2`: Nil (This used to be something, but was removed); 77 | /// - `3`: Copyright; 78 | /// - `4`: Character; 79 | /// - `5`: Species; 80 | pub(crate) category: u8, 81 | /// Whether or not the tag is locked. 82 | pub(crate) is_locked: bool, 83 | /// The date the tag was created. 84 | pub(crate) created_at: String, 85 | /// The date the tag was updated. 86 | pub(crate) updated_at: String, 87 | } 88 | 89 | impl TagEntry { 90 | /// Constrains the `TagType` enum to a tags type specifically. 91 | /// 92 | /// This can only be `TagType::General` or `TagType::Artist`. 93 | pub(crate) fn to_tag_type(&self) -> TagType { 94 | match self.category { 95 | // `0`: General; `3`: Copyright; `5`: Species; `4`: Character; `6`: Invalid; 96 | // `7`: Meta; `8`: Lore; 97 | 0 | 3..=8 => TagType::General, 98 | // `1`: Artist; 99 | 1 => TagType::Artist, 100 | _ => unreachable!(), 101 | } 102 | } 103 | } 104 | 105 | /// Wrapper struct that holds the return of bulk searches. 106 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 107 | pub(crate) struct BulkPostEntry { 108 | /// All posts in the bulk. 109 | pub(crate) posts: Vec, 110 | } 111 | 112 | /// GET return of post entry for e621/e926. 113 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 114 | pub(crate) struct PostEntry { 115 | /// The ID number of the post. 116 | pub(crate) id: i64, 117 | /// The time the post was created in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 118 | pub(crate) created_at: String, 119 | /// The time the post was last updated in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 120 | pub(crate) updated_at: Option, 121 | /// The main image of the post. 122 | pub(crate) file: File, 123 | /// The preview image of the post. 124 | pub(crate) preview: Preview, 125 | /// The sample image of the post. 126 | pub(crate) sample: Sample, 127 | /// The score of the post. 128 | pub(crate) score: Score, 129 | /// The tags tied to the post. 130 | pub(crate) tags: Tags, 131 | /// An array of tags that are locked on the post. 132 | pub(crate) locked_tags: Vec, 133 | /// An ID that increases for every post alteration on E6 (explained below) 134 | /// 135 | /// `change_seq` is a number that is increased every time a post is changed on the site. 136 | /// It gets updated whenever a post has any of these values change: 137 | /// 138 | /// - `tag_string` 139 | /// - `source` 140 | /// - `description` 141 | /// - `rating` 142 | /// - `md5` 143 | /// - `parent_id` 144 | /// - `approver_id` 145 | /// - `is_deleted` 146 | /// - `is_pending` 147 | /// - `is_flagged` 148 | /// - `is_rating_locked` 149 | /// - `is_pending` 150 | /// - `is_flagged` 151 | /// - `is_rating_locked` 152 | pub(crate) change_seq: i64, 153 | /// All the flags that could be raised on the post. 154 | pub(crate) flags: Flags, 155 | /// The post’s rating. Either `s`, `q` or `e`. 156 | pub(crate) rating: String, 157 | /// How many people have favorited the post. 158 | pub(crate) fav_count: i64, 159 | /// The source field of the post. 160 | pub(crate) sources: Vec, 161 | /// An array of Pool IDs that the post is a part of. 162 | pub(crate) pools: Vec, 163 | /// The relationships of the post. 164 | pub(crate) relationships: Relationships, 165 | /// The ID of the user that approved the post, if available. 166 | pub(crate) approver_id: Option, 167 | /// The ID of the user that uploaded the post. 168 | pub(crate) uploader_id: i64, 169 | /// The post’s description. 170 | pub(crate) description: String, 171 | /// The count of comments on the post. 172 | pub(crate) comment_count: i64, 173 | /// If provided auth credentials, will return if the authenticated user has favorited the post or not. 174 | /// HTTP Basic Auth is recommended over `login` and `api_key` parameters in the URL. 175 | pub(crate) is_favorited: bool, 176 | } 177 | 178 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 179 | pub(crate) struct File { 180 | /// The width of the post. 181 | pub(crate) width: i64, 182 | /// The height of the post. 183 | pub(crate) height: i64, 184 | /// The file’s extension. 185 | pub(crate) ext: String, 186 | /// The size of the file in bytes. 187 | pub(crate) size: i64, 188 | /// The md5 of the file. 189 | pub(crate) md5: String, 190 | /// The URL where the file is hosted on E6 191 | pub(crate) url: Option, 192 | } 193 | 194 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 195 | pub(crate) struct Preview { 196 | /// The width of the post preview. 197 | pub(crate) width: i64, 198 | /// The height of the post preview. 199 | pub(crate) height: i64, 200 | /// The URL where the preview file is hosted on E6 201 | pub(crate) url: Option, 202 | } 203 | 204 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 205 | pub(crate) struct Sample { 206 | /// If the post has a sample/thumbnail or not. 207 | pub(crate) has: Option, 208 | /// The width of the post sample. 209 | pub(crate) height: i64, 210 | /// The height of the post sample. 211 | pub(crate) width: i64, 212 | /// The URL where the sample file is hosted on E6. 213 | pub(crate) url: Option, 214 | } 215 | 216 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 217 | pub(crate) struct Score { 218 | /// The number of times voted up. 219 | pub(crate) up: i64, 220 | /// A negative number representing the number of times voted down. 221 | pub(crate) down: i64, 222 | /// The total score (up + down). 223 | pub(crate) total: i64, 224 | } 225 | 226 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 227 | pub(crate) struct Tags { 228 | /// An array of all the `general` tags on the post. 229 | pub(crate) general: Vec, 230 | /// An array of all the `species` tags on the post. 231 | pub(crate) species: Vec, 232 | /// An array of all the `character` tags on the post. 233 | pub(crate) character: Vec, 234 | /// An array of all the `copyright` tags on the post. 235 | pub(crate) copyright: Vec, 236 | /// An array of all the `artist` tags on the post. 237 | pub(crate) artist: Vec, 238 | /// An array of all the `invalid` tags on the post. 239 | pub(crate) invalid: Vec, 240 | /// An array of all the `lore` tags on the post. 241 | pub(crate) lore: Vec, 242 | /// An array of all the `meta` tags on the post. 243 | pub(crate) meta: Vec, 244 | } 245 | 246 | impl Tags { 247 | /// Consumes and combines all of the tags into a single array. 248 | pub(crate) fn combine_tags(self) -> Vec { 249 | vec![ 250 | self.general, 251 | self.species, 252 | self.character, 253 | self.copyright, 254 | self.artist, 255 | self.invalid, 256 | self.lore, 257 | self.meta, 258 | ] 259 | .into_iter() 260 | .flatten() 261 | .collect() 262 | } 263 | } 264 | 265 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 266 | pub(crate) struct Flags { 267 | /// If the post is pending approval. 268 | pub(crate) pending: bool, 269 | /// If the post is flagged for deletion. 270 | pub(crate) flagged: bool, 271 | /// If the post has it’s notes locked. 272 | pub(crate) note_locked: bool, 273 | /// If the post’s status has been locked. 274 | pub(crate) status_locked: Option, 275 | /// If the post’s rating has been locked. 276 | pub(crate) rating_locked: bool, 277 | /// If the post has been deleted. 278 | pub(crate) deleted: bool, 279 | } 280 | 281 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 282 | pub(crate) struct Relationships { 283 | /// The ID of the post’s parent, if it has one. 284 | pub(crate) parent_id: Option, 285 | /// If the post has child posts. 286 | pub(crate) has_children: bool, 287 | pub(crate) has_active_children: bool, 288 | /// A list of child post IDs that are linked to the post, if it has any. 289 | pub(crate) children: Vec, 290 | } 291 | 292 | /// GET return of set entry for e621/e926. 293 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 294 | pub(crate) struct SetEntry { 295 | /// The ID of the set. 296 | pub(crate) id: i64, 297 | /// The time the pool was created in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 298 | pub(crate) created_at: String, 299 | /// The time the pool was updated in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 300 | pub(crate) updated_at: String, 301 | /// The ID of the user that created the set. 302 | pub(crate) creator_id: i64, 303 | /// If the set is public and visible. 304 | pub(crate) is_public: bool, 305 | /// The name of the set. 306 | pub(crate) name: String, 307 | /// The short name of the set. 308 | pub(crate) shortname: String, 309 | /// The description of the set. 310 | pub(crate) description: String, 311 | /// The amount of posts in the set. 312 | pub(crate) post_count: i64, 313 | /// If the set will transfer its post on delete. 314 | pub(crate) transfer_on_delete: bool, 315 | /// An array group of posts in the pool. 316 | pub(crate) post_ids: Vec, 317 | } 318 | 319 | /// GET return of pool entry for e621/e926. 320 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 321 | pub(crate) struct PoolEntry { 322 | /// The ID of the pool. 323 | pub(crate) id: i64, 324 | /// The name of the pool. 325 | pub(crate) name: String, 326 | /// The time the pool was created in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 327 | pub(crate) created_at: String, 328 | /// The time the pool was updated in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 329 | pub(crate) updated_at: String, 330 | /// The ID of the user that created the pool. 331 | pub(crate) creator_id: i64, 332 | /// The description of the pool. 333 | pub(crate) description: String, 334 | /// If the pool is active and still getting posts added. 335 | pub(crate) is_active: bool, 336 | /// Can be `series` or `collection`. 337 | pub(crate) category: String, 338 | /// An array group of posts in the pool. 339 | pub(crate) post_ids: Vec, 340 | /// The name of the user that created the pool. 341 | pub(crate) creator_name: String, 342 | /// The amount of posts in the pool. 343 | pub(crate) post_count: i64, 344 | } 345 | 346 | /// GET return of user entry for e621/e926. 347 | #[derive(Default, Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] 348 | pub(crate) struct UserEntry { 349 | /// The amount of wiki changes made by the user. 350 | pub(crate) wiki_page_version_count: i64, 351 | /// The amount of artist changes made by the user. 352 | pub(crate) artist_version_count: i64, 353 | /// The amount of pool changes made by the user. 354 | pub(crate) pool_version_count: i64, 355 | /// The amount of post changes made by the user. 356 | pub(crate) forum_post_count: i64, 357 | /// Count of comments posted by the user. 358 | pub(crate) comment_count: i64, 359 | /// Count of flags done by the user. 360 | pub(crate) flag_count: i64, 361 | /// The amount of positive feedback given by the user. 362 | pub(crate) positive_feedback_count: i64, 363 | /// The amount of neutral feedback given by the user. 364 | pub(crate) neutral_feedback_count: i64, 365 | /// The amount of negative feedback given by the user. 366 | pub(crate) negative_feedback_count: i64, 367 | /// Upload limit of the user. 368 | pub(crate) upload_limit: i64, 369 | /// ID of the user. 370 | pub(crate) id: i64, 371 | /// The time the pool was created in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 372 | pub(crate) created_at: String, 373 | /// Name of the user. 374 | pub(crate) name: String, 375 | /// Level of the user. 376 | pub(crate) level: i64, 377 | /// Base upload limit of the user. 378 | pub(crate) base_upload_limit: i64, 379 | /// Count of posts uploaded by the user. 380 | pub(crate) post_upload_count: i64, 381 | /// Count of posts updated by the user. 382 | pub(crate) post_update_count: i64, 383 | /// Count of notes updated by the user. 384 | pub(crate) note_update_count: i64, 385 | /// If user is banned or not. 386 | pub(crate) is_banned: bool, 387 | /// Whether or not the user can approve posts. 388 | pub(crate) can_approve_posts: bool, 389 | /// Whether or not uploading posts affect the post limit. 390 | pub(crate) can_upload_free: bool, 391 | /// The string of the user's current level. 392 | pub(crate) level_string: String, 393 | /// Whether or not avatars should be shown. 394 | pub(crate) show_avatars: Option, 395 | /// Whether or not the blacklist should block avatars. 396 | pub(crate) blacklist_avatars: Option, 397 | /// Whether or not the blacklist should block users. 398 | pub(crate) blacklist_users: Option, 399 | /// Whether or not a post's description should be collapsed initially. 400 | pub(crate) description_collapsed_initially: Option, 401 | /// Whether or not comments should be hidden. 402 | pub(crate) hide_comments: Option, 403 | /// Whether or not hidden comments should be shown. 404 | pub(crate) show_hidden_comments: Option, 405 | /// Whether or not to show post statistics. 406 | pub(crate) show_post_statistics: Option, 407 | /// Whether or not the user has mail. 408 | pub(crate) has_mail: Option, 409 | /// Whether or not the user will receive email notifications. 410 | pub(crate) receive_email_notifications: Option, 411 | /// Whether or not keyboard navigation is on/off. 412 | pub(crate) enable_keyboard_navigation: Option, 413 | /// Whether or not privacy mode is enabled. 414 | pub(crate) enable_privacy_mode: Option, 415 | /// Whether or not usernames should be styled. 416 | pub(crate) style_usernames: Option, 417 | /// Whether auto complete should be on or off. 418 | pub(crate) enable_auto_complete: Option, 419 | /// Whether or not searches should be saved. 420 | pub(crate) has_saved_searches: Option, 421 | /// Whether or not thumbnails should be cropped. 422 | pub(crate) disable_cropped_thumbnails: Option, 423 | /// Whether or not mobile gestures should be on or off. 424 | pub(crate) disable_mobile_gestures: Option, 425 | /// Whether or not safe mode is on/off. 426 | pub(crate) enable_safe_mode: Option, 427 | /// Whether or not responsive mode is disabled. 428 | pub(crate) disable_responsive_mode: Option, 429 | /// Whether or not post tooltips is disabled. 430 | pub(crate) disable_post_tooltips: Option, 431 | /// Whether or not the user can't flag. 432 | pub(crate) no_flagging: Option, 433 | /// Whether or not the user can't give feedback. 434 | pub(crate) no_feedback: Option, 435 | /// Whether or not dmail is disabled. 436 | pub(crate) disable_user_dmails: Option, 437 | /// Whether or not compact uploader is enabled. 438 | pub(crate) enable_compact_uploader: Option, 439 | /// The time the pool was updated in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 440 | pub(crate) updated_at: Option, 441 | /// The user's email. 442 | pub(crate) email: Option, 443 | /// The time the user was last logged in in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 444 | pub(crate) last_logged_in_at: Option, 445 | /// The time the last forum the user read in the format of `YYYY-MM-DDTHH:MM:SS.MS+00:00`. 446 | pub(crate) last_forum_read_at: Option, 447 | /// Recent tags searched by the user. 448 | pub(crate) recent_tags: Option, 449 | /// Comment threshold of the user. 450 | pub(crate) comment_threshold: Option, 451 | /// Default image size of the user. 452 | pub(crate) default_image_size: Option, 453 | /// Favorite tags the user has. 454 | pub(crate) favorite_tags: Option, 455 | /// The user's blacklist tags. 456 | pub(crate) blacklisted_tags: Option, 457 | /// The time zone of the user. 458 | pub(crate) time_zone: Option, 459 | /// The post count per page. 460 | pub(crate) per_page: Option, 461 | /// Custom style/theme of E6. 462 | pub(crate) custom_style: Option, 463 | /// Count of all the user's favorites. 464 | pub(crate) favorite_count: Option, 465 | /// The API regen multiplier. 466 | pub(crate) api_regen_multiplier: Option, 467 | /// The API burst limit. 468 | pub(crate) api_burst_limit: Option, 469 | /// The remaining API limit. 470 | pub(crate) remaining_api_limit: Option, 471 | /// The statement given while being in timeout. 472 | pub(crate) statement_timeout: Option, 473 | /// The limit for how many times a user can favorite. 474 | pub(crate) favorite_limit: Option, 475 | /// The maximum tag query limit, the amount amount of tags a user can search. 476 | pub(crate) tag_query_limit: Option, 477 | } 478 | -------------------------------------------------------------------------------- /src/e621/blacklist.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | use std::cmp::Ordering; 19 | 20 | use anyhow::Context; 21 | 22 | use crate::e621::io::parser::BaseParser; 23 | use crate::e621::sender::entries::{PostEntry, UserEntry}; 24 | use crate::e621::sender::RequestSender; 25 | 26 | /// Root token which contains all the tokens of the blacklist. 27 | #[derive(Default, Debug)] 28 | struct RootToken { 29 | /// The total [LineToken]s from the root. 30 | lines: Vec, 31 | } 32 | 33 | /// A line token that contains all collected [`TagToken`]s from a parsed line. 34 | #[derive(Debug, Default)] 35 | struct LineToken { 36 | /// Total [TagToken] in the line. 37 | tags: Vec, 38 | } 39 | 40 | impl LineToken { 41 | fn new(tags: Vec) -> Self { 42 | LineToken { tags } 43 | } 44 | } 45 | 46 | /// Enum that contains each possible option from `rating:` being in blacklist. 47 | #[derive(Debug, PartialEq)] 48 | enum Rating { 49 | /// No rating. 50 | None, 51 | /// Safe rating. 52 | Safe, 53 | /// Questionable rating. 54 | Questionable, 55 | /// Explicit rating. 56 | Explicit, 57 | } 58 | 59 | /// A enum that contains what type the [TagToken] is. 60 | /// 61 | /// The tag can be seen as four types: [Rating](TagType::Rating), [Id](TagType::Id), [User](TagType::User), and 62 | /// [None](TagType::None). 63 | #[derive(Debug)] 64 | enum TagType { 65 | /// A post rating type. 66 | Rating(Rating), 67 | /// A post id type. 68 | Id(Option), 69 | /// A user type. 70 | User(Option), 71 | /// The blacklisted score 72 | Score(Ordering, i32), 73 | /// No type. 74 | None, 75 | } 76 | 77 | /// Tag token that contains essential information about what is blacklisted. 78 | #[derive(Debug)] 79 | struct TagToken { 80 | /// If the tag is negated or not 81 | negated: bool, 82 | /// If the tag is a rating, this will hold the exact the rating it is 83 | tag_type: TagType, 84 | /// The tag (value for special tags) 85 | name: String, 86 | } 87 | 88 | impl Default for TagToken { 89 | fn default() -> Self { 90 | TagToken { 91 | negated: false, 92 | tag_type: TagType::None, 93 | name: String::new(), 94 | } 95 | } 96 | } 97 | 98 | /// Parser that reads a tag file and parses the tags. 99 | #[derive(Default)] 100 | struct BlacklistParser { 101 | /// The base parser which parses the blacklist character by character. 102 | base_parser: BaseParser, 103 | } 104 | 105 | impl BlacklistParser { 106 | fn new(blacklist: String) -> Self { 107 | trace!("Initializing blacklist parser..."); 108 | BlacklistParser { 109 | base_parser: BaseParser::new(blacklist), 110 | } 111 | } 112 | 113 | /// Parses the entire blacklist. 114 | fn parse_blacklist(&mut self) -> RootToken { 115 | trace!("Parsing blacklist..."); 116 | let mut lines: Vec = Vec::new(); 117 | loop { 118 | self.base_parser.consume_whitespace(); 119 | if self.base_parser.eof() { 120 | break; 121 | } 122 | 123 | lines.push(self.parse_line()); 124 | } 125 | 126 | trace!("Parsed blacklist..."); 127 | 128 | RootToken { lines } 129 | } 130 | 131 | /// Parses each tag and collects them into a [`LineToken`]. 132 | fn parse_line(&mut self) -> LineToken { 133 | let mut tags: Vec = Vec::new(); 134 | loop { 135 | if self.base_parser.starts_with("\n") { 136 | assert_eq!(self.base_parser.consume_char(), '\n'); 137 | break; 138 | } 139 | 140 | self.base_parser.consume_whitespace(); 141 | if self.base_parser.eof() { 142 | break; 143 | } 144 | 145 | tags.push(self.parse_tag()); 146 | } 147 | 148 | LineToken::new(tags) 149 | } 150 | 151 | /// Checks if tag is negated. 152 | fn is_tag_negated(&self) -> bool { 153 | self.base_parser.starts_with("-") 154 | } 155 | 156 | /// Parses tag and runs through basic identification before returning it as a [`TagToken`]. 157 | fn parse_tag(&mut self) -> TagToken { 158 | let mut token = TagToken::default(); 159 | if self.is_tag_negated() { 160 | assert_eq!(self.base_parser.consume_char(), '-'); 161 | token.negated = true; 162 | } 163 | 164 | token.name = self.base_parser.consume_while(valid_tag).to_lowercase(); 165 | 166 | // This will be considered a special tag if it contains the syntax of one. 167 | if !self.base_parser.eof() && self.base_parser.next_char() == ':' { 168 | self.parse_special_tag(&mut token); 169 | } 170 | 171 | token 172 | } 173 | 174 | /// Parses special tag and updates token with the appropriate type and value. 175 | /// 176 | /// # Arguments 177 | /// 178 | /// * `token`: The special [TagToken] to parse. 179 | /// 180 | /// returns: () 181 | /// 182 | /// # Errors 183 | /// 184 | /// An error can occur if 1) the `assert_eq` fails in its check or 2) if the [TagToken] name is not any of the matched 185 | /// values. 186 | fn parse_special_tag(&mut self, token: &mut TagToken) { 187 | assert_eq!(self.base_parser.consume_char(), ':'); 188 | match token.name.as_str() { 189 | "rating" => { 190 | let rating_string = self.base_parser.consume_while(valid_rating); 191 | token.tag_type = TagType::Rating(self.get_rating(&rating_string)); 192 | } 193 | "id" => { 194 | token.tag_type = TagType::Id(Some( 195 | self.base_parser 196 | .consume_while(valid_id) 197 | .parse::() 198 | .unwrap_or_default(), 199 | )); 200 | } 201 | "user" => { 202 | token.tag_type = TagType::User(Some(self.base_parser.consume_while(valid_user))); 203 | } 204 | "score" => { 205 | let ordering = self.get_ordering(); 206 | let score = self.base_parser.consume_while(valid_score); 207 | token.tag_type = TagType::Score( 208 | ordering, 209 | score.parse::().unwrap(), 210 | ); 211 | } 212 | _ => { 213 | self.base_parser.report_error( 214 | format!("Unknown special tag identifier: {}", token.name).as_str(), 215 | ); 216 | } 217 | }; 218 | } 219 | 220 | /// Checks the value and create a new [Rating] from it. 221 | /// 222 | /// # Arguments 223 | /// 224 | /// * `value`: The value to check. 225 | /// 226 | /// returns: Rating 227 | fn get_rating(&self, value: &str) -> Rating { 228 | match value.to_lowercase().as_str() { 229 | "safe" | "s" => Rating::Safe, 230 | "questionable" | "q" => Rating::Questionable, 231 | "explicit" | "e" => Rating::Explicit, 232 | _ => Rating::None, 233 | } 234 | } 235 | 236 | /// Gets the ordering of the score. 237 | fn get_ordering(&mut self) -> Ordering { 238 | let order = self.base_parser.consume_while(valid_ordering); 239 | match order.as_str() { 240 | "<" => Ordering::Less, 241 | ">=" => Ordering::Greater, // This is greater than or equal, but ordering has no combination for that. 242 | _ => Ordering::Equal // Defaults to equal (e.g nothing happens). 243 | } 244 | } 245 | } 246 | 247 | /// Validates character for tag. 248 | /// 249 | /// # Arguments 250 | /// 251 | /// * `c`: The character to check. 252 | /// 253 | /// returns: bool 254 | fn valid_tag(c: char) -> bool { 255 | match c { 256 | '!'..='9' | ';'..='~' => true, 257 | // This will check for any special characters in the validator. 258 | _ => { 259 | if c != ':' { 260 | return c.is_alphanumeric(); 261 | } 262 | 263 | false 264 | } 265 | } 266 | } 267 | 268 | /// Validates character for user. 269 | /// 270 | /// # Arguments 271 | /// 272 | /// * `c`: The character to check. 273 | /// 274 | /// returns: bool 275 | fn valid_user(c: char) -> bool { 276 | match c { 277 | '!'..='9' | ';'..='~' => true, 278 | // This will check for any special characters in the validator. 279 | _ => { 280 | if c != ':' { 281 | return c.is_alphanumeric(); 282 | } 283 | 284 | false 285 | } 286 | } 287 | } 288 | 289 | /// Validates character for rating. 290 | /// 291 | /// # Arguments 292 | /// 293 | /// * `c`: The character to check. 294 | /// 295 | /// returns: bool 296 | fn valid_rating(c: char) -> bool { 297 | c.is_ascii_alphabetic() 298 | } 299 | 300 | /// Validates character for ordering. 301 | /// 302 | /// # Arguments 303 | /// 304 | /// * `c`: The character to check. 305 | /// 306 | /// returns: bool 307 | fn valid_ordering(c: char) -> bool { 308 | matches!(c, '<' | '>' | '=') 309 | } 310 | 311 | /// Validates character for score. 312 | /// 313 | /// # Arguments 314 | /// 315 | /// * `c`: The character to check. 316 | /// 317 | /// returns: bool 318 | fn valid_score(c: char) -> bool { 319 | c.is_ascii_digit() 320 | } 321 | 322 | /// Validates character for id. 323 | /// 324 | /// # Arguments 325 | /// 326 | /// * `c`: The character to check. 327 | /// 328 | /// returns: bool 329 | fn valid_id(c: char) -> bool { 330 | c.is_ascii_digit() 331 | } 332 | 333 | /// A worker that checks and flags post based on a tag predicate, typically from the user's blacklist. 334 | /// 335 | /// It works by comparing and removing any grabbed post that matches with all of the tags in a `LineToken`. 336 | /// The worker works with the supplied syntax and rules on e621's main site listed [here](https://e621.net/help/show/blacklist). 337 | /// This ensures that the client-side blacklist works exactly the same as the server-side blacklist. 338 | #[derive(Default)] 339 | struct FlagWorker { 340 | /// The number of flags raised by the worker 341 | flags: i16, 342 | /// The number of negated flags raised by the worker 343 | negated_flags: i16, 344 | /// The margin of how many flags that should be raised before a post is determined to be blacklisted 345 | margin: i16, 346 | /// The margin of how many negated flags that should be raised before a post is determined to be safe 347 | negated_margin: i16, 348 | /// Whether the post is flagged or not 349 | flagged: bool, 350 | } 351 | 352 | impl FlagWorker { 353 | /// Sets margin for how many flags need to be raised before the post is either blacklisted or considered safe. 354 | /// 355 | /// # Arguments 356 | /// 357 | /// * `tags`: The tags to calibrate flags for. 358 | fn set_flag_margin(&mut self, tags: &[TagToken]) { 359 | for tag in tags { 360 | if tag.negated { 361 | if let TagType::Score(_, _) = tag.tag_type { 362 | // This is done because e621's blacklist itself doesn't handle scores that are negated, at 363 | // least from my testing. 364 | continue; 365 | } 366 | 367 | self.negated_margin += 1; 368 | } else { 369 | self.margin += 1; 370 | } 371 | } 372 | } 373 | 374 | /// Flags post based on blacklisted rating. 375 | /// 376 | /// # Arguments 377 | /// 378 | /// * `rating`: The blacklisted rating. 379 | /// * `post`: The post to check against. 380 | /// * `negated`: Whether the blacklisted rating is negated or not (this will determine if the rating whitelists the 381 | /// post or adds towards removing it from the download pool). 382 | fn flag_rating(&mut self, rating: &Rating, post: &PostEntry, negated: bool) { 383 | // A nice tuple hack to get around some massive nesting. 384 | match (rating, post.rating.as_str()) { 385 | (Rating::Safe, "s") | (Rating::Questionable, "q") | (Rating::Explicit, "e") => { 386 | self.raise_flag(negated); 387 | } 388 | (_, _) => {} 389 | } 390 | } 391 | 392 | /// Raises the flag and blacklists the post if its ID matches with the blacklisted ID. 393 | /// 394 | /// # Arguments 395 | /// 396 | /// * `id`: The blacklisted id to compare. 397 | /// * `post_id`: The post id to check against. 398 | /// * `negated`: Whether the blacklisted rating is negated or not (this will determine if the rating whitelists the 399 | /// post or adds towards removing it from the download pool). 400 | fn flag_id(&mut self, id: i64, post_id: i64, negated: bool) { 401 | if post_id == id { 402 | self.raise_flag(negated); 403 | } 404 | } 405 | 406 | /// Raises the flag and blacklists the post if the user who uploaded it is blacklisted. 407 | /// 408 | /// # Arguments 409 | /// 410 | /// * `user_id`: The blacklisted user id. 411 | /// * `uploader_id`: The user id to check against. 412 | /// * `negated`: Whether the blacklisted rating is negated or not (this will determine if the rating whitelists the 413 | /// post or adds towards removing it from the download pool). 414 | fn flag_user(&mut self, user_id: i64, uploader_id: i64, negated: bool) { 415 | if user_id == uploader_id { 416 | self.raise_flag(negated); 417 | } 418 | } 419 | 420 | /// Flags post based on it's score. 421 | /// 422 | /// # Arguments 423 | /// 424 | /// * `ordering`: The ordering of the score blacklisted (e.g <, >=). 425 | /// * `score`: The score to check and blacklist. 426 | /// * `post_score`: The post score to check against. 427 | fn flag_score(&mut self, ordering: &Ordering, score: &i32, post_score: i64, negated: bool) { 428 | match ordering { 429 | Ordering::Less => { 430 | if post_score < *score as i64 { 431 | self.raise_flag(negated); 432 | } 433 | } 434 | Ordering::Greater => { 435 | if post_score >= *score as i64 { 436 | self.raise_flag(negated); 437 | } 438 | } 439 | _ => {} 440 | } 441 | } 442 | 443 | /// Checks if a single post is blacklisted. 444 | /// 445 | /// # Arguments 446 | /// 447 | /// * `post`: The post to check. 448 | /// * `blacklist_line`: The blacklist tags to check the post against. 449 | fn check_post(&mut self, post: &PostEntry, blacklist_line: &LineToken) { 450 | let post_tags = post.tags.clone().combine_tags(); 451 | for tag in &blacklist_line.tags { 452 | match &tag.tag_type { 453 | TagType::Rating(rating) => { 454 | self.flag_rating(rating, post, tag.negated); 455 | } 456 | TagType::Id(id) => { 457 | if let Some(blacklisted_id) = id { 458 | self.flag_id(*blacklisted_id, post.id, tag.negated); 459 | } 460 | } 461 | TagType::User(_) => { 462 | let user_id = tag 463 | .name 464 | .parse::() 465 | .with_context(|| { 466 | format!("Failed to parse blacklisted user id: {}!", tag.name) 467 | }) 468 | .unwrap(); 469 | self.flag_user(user_id, post.uploader_id, tag.negated); 470 | } 471 | TagType::Score(ordering, score) => { 472 | self.flag_score(ordering, score, post.score.total, tag.negated); 473 | } 474 | TagType::None => { 475 | if post_tags.iter().any(|e| e == tag.name.as_str()) { 476 | self.raise_flag(tag.negated); 477 | } 478 | } 479 | } 480 | } 481 | 482 | if self.is_negated_margin_met() { 483 | self.flagged = false; 484 | } else if self.is_margin_met() { 485 | self.flagged = true; 486 | } 487 | } 488 | 489 | /// Returns true if the negated flags equals the negated margin, false otherwise. 490 | fn is_negated_margin_met(&self) -> bool { 491 | self.negated_margin != 0 && self.negated_flags == self.negated_margin 492 | } 493 | 494 | /// Returns true if the total flags equals the margin, false otherwise. 495 | fn is_margin_met(&self) -> bool { 496 | self.flags == self.margin 497 | } 498 | 499 | /// Raises either the `negated_flags` or `flags` by one depending on the value of `negated`. 500 | /// 501 | /// # Arguments 502 | /// 503 | /// * `negated`: The tag's negation. 504 | fn raise_flag(&mut self, negated: bool) { 505 | if negated { 506 | self.negated_flags += 1; 507 | } else { 508 | self.flags += 1; 509 | } 510 | } 511 | 512 | /// Returns if the flag is raised or not. 513 | fn is_flagged(&self) -> bool { 514 | self.flagged 515 | } 516 | } 517 | 518 | /// Blacklist that holds all of the blacklist entries. 519 | /// These entries will be looped through a parsed before being used for filtering posts that are blacklisted. 520 | pub(crate) struct Blacklist { 521 | /// The blacklist parser which parses the blacklist and tokenizes it. 522 | blacklist_parser: BlacklistParser, 523 | /// All of the blacklist tokens after being parsed. 524 | blacklist_tokens: RootToken, 525 | /// Request sender used for getting user information. 526 | request_sender: RequestSender, 527 | } 528 | 529 | impl Blacklist { 530 | pub(crate) fn new(request_sender: RequestSender) -> Self { 531 | Blacklist { 532 | blacklist_parser: BlacklistParser::default(), 533 | blacklist_tokens: RootToken::default(), 534 | request_sender, 535 | } 536 | } 537 | 538 | /// Parses the user blacklist. 539 | /// 540 | /// # Arguments 541 | /// 542 | /// * `user_blacklist`: The user blacklist to parse 543 | /// 544 | /// returns: &mut Blacklist 545 | pub(crate) fn parse_blacklist(&mut self, user_blacklist: String) -> &mut Blacklist { 546 | self.blacklist_parser = BlacklistParser::new(user_blacklist); 547 | self.blacklist_tokens = self.blacklist_parser.parse_blacklist(); 548 | self 549 | } 550 | 551 | /// Caches user id into the tag name for quicker access during the blacklist checks. 552 | pub(crate) fn cache_users(&mut self) { 553 | let tags: Vec<&mut TagToken> = self 554 | .blacklist_tokens 555 | .lines 556 | .iter_mut() 557 | .flat_map(|e| &mut e.tags) 558 | .collect(); 559 | for tag in tags { 560 | if let TagType::User(Some(username)) = &tag.tag_type { 561 | let user: UserEntry = self 562 | .request_sender 563 | .get_entry_from_appended_id(username, "user"); 564 | tag.name = format!("{}", user.id); 565 | } 566 | } 567 | } 568 | 569 | /// Checks if the blacklist is empty. 570 | pub(crate) fn is_empty(&self) -> bool { 571 | self.blacklist_tokens.lines.is_empty() 572 | } 573 | 574 | /// Filters through a set of posts, only retaining posts that aren't blacklisted. 575 | /// 576 | /// # Arguments 577 | /// 578 | /// * `posts`: Posts to filter through. 579 | /// 580 | /// returns: u16 581 | pub(crate) fn filter_posts(&self, posts: &mut Vec) -> u16 { 582 | let mut filtered: u16 = 0; 583 | for blacklist_line in &self.blacklist_tokens.lines { 584 | posts.retain(|e| { 585 | let mut flag_worker = FlagWorker::default(); 586 | flag_worker.set_flag_margin(&blacklist_line.tags); 587 | flag_worker.check_post(e, blacklist_line); 588 | if flag_worker.is_flagged() { 589 | filtered += 1; 590 | } 591 | 592 | // This inverses the flag to make sure it retains what isn't flagged and disposes of 593 | // what is flagged. 594 | !flag_worker.is_flagged() 595 | }); 596 | } 597 | 598 | match filtered.cmp(&1) { 599 | Ordering::Less => trace!("No posts filtered..."), 600 | Ordering::Equal => trace!("Filtered {filtered} post with blacklist..."), 601 | Ordering::Greater => trace!("Filtered {filtered} posts with blacklist..."), 602 | } 603 | 604 | filtered 605 | } 606 | } 607 | -------------------------------------------------------------------------------- /src/e621/grabber.rs: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2022 McSib 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | use std::cell::RefCell; 18 | use std::cmp::Ordering; 19 | use std::rc::Rc; 20 | 21 | use crate::e621::blacklist::Blacklist; 22 | use crate::e621::io::tag::{Group, Tag, TagSearchType, TagType}; 23 | use crate::e621::io::{emergency_exit, Config, Login}; 24 | use crate::e621::sender::entries::{PoolEntry, PostEntry, SetEntry}; 25 | use crate::e621::sender::RequestSender; 26 | 27 | /// A trait for implementing a conversion function for turning a type into a [Vec] of the same type 28 | /// 29 | /// This can be used as a simple wrapping function for flattening an internal array of the type. 30 | pub(crate) trait NewVec { 31 | fn new_vec(value: T) -> Vec 32 | where 33 | Self: Sized; 34 | } 35 | 36 | /// A collection of values taken from a [PostEntry]. 37 | pub(crate) struct GrabbedPost { 38 | /// The url that leads to the file to download. 39 | url: String, 40 | /// The name of the file to download. 41 | name: String, 42 | /// The size of the file to download. 43 | file_size: i64, 44 | } 45 | 46 | impl GrabbedPost { 47 | /// The url that leads to the file to download. 48 | pub(crate) fn url(&self) -> &str { 49 | &self.url 50 | } 51 | 52 | /// The name of the file to download. 53 | pub(crate) fn name(&self) -> &str { 54 | &self.name 55 | } 56 | 57 | /// The size of the file to download. 58 | pub(crate) fn file_size(&self) -> i64 { 59 | self.file_size 60 | } 61 | } 62 | 63 | impl NewVec> for GrabbedPost { 64 | /// Creates a new [Vec] of type [GrabbedPost] from Vec of type [PostEntry] 65 | /// 66 | /// # Arguments 67 | /// 68 | /// * `vec`: The vector to be consumed and converted. 69 | /// 70 | /// returns: Vec 71 | fn new_vec(vec: Vec) -> Vec { 72 | vec.into_iter() 73 | .map(|e| GrabbedPost::from((e, Config::get().naming_convention()))) 74 | .collect() 75 | } 76 | } 77 | 78 | impl NewVec<(Vec, &str)> for GrabbedPost { 79 | /// Creates a new [Vec] of type [GrabbedPost] from tuple contains types ([PostEntry], &str) 80 | /// 81 | /// Compared to the other overload, this version sets the name of the [GrabbedPost] and numbers them. 82 | /// 83 | /// # Arguments 84 | /// 85 | /// * `(vec, pool_name)`: A tuple containing the posts and the name of the pool associated with them. 86 | /// 87 | /// returns: Vec 88 | fn new_vec((vec, pool_name): (Vec, &str)) -> Vec { 89 | vec.iter() 90 | .enumerate() 91 | .map(|(i, e)| GrabbedPost::from((e, pool_name, (i + 1) as u16))) 92 | .collect() 93 | } 94 | } 95 | 96 | impl From<(&PostEntry, &str, u16)> for GrabbedPost { 97 | /// Creates [GrabbedPost] from tuple of types (&[PostEntry], &str, u16) 98 | /// 99 | /// # Arguments 100 | /// 101 | /// * `(post, name, current_page)`: A tuple containing the post, name, and current page number of post. 102 | /// 103 | /// returns: GrabbedPost 104 | fn from((post, name, current_page): (&PostEntry, &str, u16)) -> Self { 105 | GrabbedPost { 106 | url: post.file.url.clone().unwrap(), 107 | name: format!("{} Page_{:05}.{}", name, current_page, post.file.ext), 108 | file_size: post.file.size, 109 | } 110 | } 111 | } 112 | 113 | impl From<(PostEntry, &str)> for GrabbedPost { 114 | /// Creates [GrabbedPost] from tuple of types ([PostEntry], &str) 115 | /// 116 | /// # Arguments 117 | /// 118 | /// * `(post, name_convention)`: A tuple containing the post, and naming convention of post. 119 | /// 120 | /// returns: GrabbedPost 121 | fn from((post, name_convention): (PostEntry, &str)) -> Self { 122 | match name_convention { 123 | "md5" => GrabbedPost { 124 | url: post.file.url.clone().unwrap(), 125 | name: format!("{}.{}", post.file.md5, post.file.ext), 126 | file_size: post.file.size, 127 | }, 128 | "id" => GrabbedPost { 129 | url: post.file.url.clone().unwrap(), 130 | name: format!("{}.{}", post.id, post.file.ext), 131 | file_size: post.file.size, 132 | }, 133 | _ => { 134 | emergency_exit("Incorrect naming convention!"); 135 | GrabbedPost { 136 | url: String::new(), 137 | name: String::new(), 138 | file_size: 0, 139 | } 140 | } 141 | } 142 | } 143 | } 144 | 145 | /// A trait for the shorten function, it allows for generic types to be the parameter. 146 | pub(crate) trait Shorten { 147 | /// Shortens a string by replacing a portion of it with a delimiter of type `T` and then returning the new string. 148 | fn shorten(&self, delimiter: T) -> String; 149 | } 150 | 151 | /// A set of posts with category and name. 152 | pub(crate) struct PostCollection { 153 | /// The name of the set. 154 | name: String, 155 | /// The category of the set. 156 | category: String, 157 | /// The posts in the set. 158 | posts: Vec, 159 | } 160 | 161 | impl PostCollection { 162 | /// Creates a new post collection. 163 | /// 164 | /// # Arguments 165 | /// 166 | /// * `name`: Name of collection. 167 | /// * `category`: Category of collection. 168 | /// * `posts`: Posts for collection. 169 | /// 170 | /// returns: PostCollection 171 | pub(crate) fn new(name: &str, category: &str, posts: Vec) -> Self { 172 | PostCollection { 173 | name: name.to_string(), 174 | category: category.to_string(), 175 | posts, 176 | } 177 | } 178 | 179 | /// The name of the set. 180 | pub(crate) fn name(&self) -> &str { 181 | &self.name 182 | } 183 | 184 | /// The category of the set. 185 | pub(crate) fn category(&self) -> &str { 186 | &self.category 187 | } 188 | 189 | /// The posts in the set. 190 | pub(crate) fn posts(&self) -> &Vec { 191 | &self.posts 192 | } 193 | } 194 | 195 | impl Shorten<&str> for PostCollection { 196 | /// Shortens [PostCollection] name if it's greater than 25 characters and attaches the delimiter at the end. 197 | /// 198 | /// # Arguments 199 | /// 200 | /// * `delimiter`: What to replace the excess characters with. 201 | /// 202 | /// returns: String 203 | fn shorten(&self, delimiter: &str) -> String { 204 | if self.name.len() >= 25 { 205 | let mut short_name = self.name[0..25].to_string(); 206 | short_name.push_str(delimiter); 207 | short_name 208 | } else { 209 | self.name.to_string() 210 | } 211 | } 212 | } 213 | 214 | impl Shorten for PostCollection { 215 | /// Shortens [PostCollection] name if it's greater than 25 characters and attaches the delimiter at the end. 216 | /// 217 | /// # Arguments 218 | /// 219 | /// * `delimiter`: What to replace the excess characters with. 220 | /// 221 | /// returns: String 222 | fn shorten(&self, delimiter: char) -> String { 223 | if self.name.len() >= 25 { 224 | let mut short_name = self.name[0..25].to_string(); 225 | short_name.push(delimiter); 226 | short_name 227 | } else { 228 | self.name.to_string() 229 | } 230 | } 231 | } 232 | 233 | impl From<(&SetEntry, Vec)> for PostCollection { 234 | /// Creates [PostCollection] from tuple of types (&[SetEntry], [Vec]<[GrabbedPost]>). 235 | /// 236 | /// # Arguments 237 | /// 238 | /// * `(set, posts)`: The set and posts to make [PostCollection] from. 239 | /// 240 | /// returns: PostCollection 241 | fn from((set, posts): (&SetEntry, Vec)) -> Self { 242 | PostCollection::new(&set.name, "Sets", posts) 243 | } 244 | } 245 | 246 | /// The total amount of pages the general search can search for. 247 | const POST_SEARCH_LIMIT: u8 = 5; 248 | 249 | /// Is a collector that grabs posts, categorizes them, and prepares them for the downloader to use in downloading. 250 | pub(crate) struct Grabber { 251 | /// All grabbed posts. 252 | posts: Vec, 253 | /// `RequestSender` for sending API calls. 254 | request_sender: RequestSender, 255 | /// Blacklist used to throwaway posts that contain tags the user may not want. 256 | blacklist: Option>>, 257 | /// Is grabber in safe mode or not 258 | safe_mode: bool, 259 | } 260 | 261 | impl Grabber { 262 | /// Creates a grabber for searching and grabbing posts. 263 | /// 264 | /// # Arguments 265 | /// 266 | /// * `request_sender`: The client to perform the searches. 267 | /// * `safe_mode`: Which mode the grabber will operate under. 268 | /// 269 | /// returns: Grabber 270 | pub(crate) fn new(request_sender: RequestSender, safe_mode: bool) -> Self { 271 | Grabber { 272 | posts: vec![PostCollection::new("Single Posts", "", Vec::new())], 273 | request_sender, 274 | blacklist: None, 275 | safe_mode, 276 | } 277 | } 278 | 279 | /// All grabbed posts. 280 | pub(crate) fn posts(&self) -> &Vec { 281 | &self.posts 282 | } 283 | 284 | /// Sets the blacklist. 285 | /// 286 | /// # Arguments 287 | /// 288 | /// * `blacklist`: The new blacklist 289 | pub(crate) fn set_blacklist(&mut self, blacklist: Rc>) { 290 | if !blacklist.borrow_mut().is_empty() { 291 | self.blacklist = Some(blacklist); 292 | } 293 | } 294 | 295 | /// Sets safe mode. 296 | /// 297 | /// If set true, the grabber will go into safe mode and grab only safe posts, 298 | /// false will grab questionable and explicit posts. 299 | /// 300 | /// # Arguments 301 | /// 302 | /// * `mode`: Which mode to run in 303 | pub(crate) fn set_safe_mode(&mut self, mode: bool) { 304 | self.safe_mode = mode; 305 | } 306 | 307 | /// Grabs favorites from the user's favorites 308 | pub(crate) fn grab_favorites(&mut self) { 309 | let login = Login::get(); 310 | if !login.username().is_empty() && login.download_favorites() { 311 | let tag = format!("fav:{}", login.username()); 312 | let posts = self.search(&tag, &TagSearchType::Special); 313 | self.posts 314 | .push(PostCollection::new(&tag, "", GrabbedPost::new_vec(posts))); 315 | info!( 316 | "{} grabbed!", 317 | console::style(format!("\"{tag}\"")).color256(39).italic() 318 | ); 319 | } 320 | } 321 | 322 | /// Grabs new posts by the given tag. 323 | /// 324 | /// # Arguments 325 | /// 326 | /// * `groups`: The group of tags to search for. 327 | pub(crate) fn grab_posts_by_tags(&mut self, groups: &[Group]) { 328 | let tags: Vec<&Tag> = groups.iter().flat_map(|e| e.tags()).collect(); 329 | for tag in tags { 330 | self.grab_by_tag_type(tag); 331 | } 332 | } 333 | 334 | /// Returns the single post [PostCollection]. 335 | fn single_post_collection(&mut self) -> &mut PostCollection { 336 | self.posts.first_mut().unwrap() // It is guaranteed that the first collection is the single post collection. 337 | } 338 | 339 | /// Adds a single post to the single post [PostCollection]. 340 | /// 341 | /// # Arguments 342 | /// 343 | /// * `entry`: The entry to add to the collection. 344 | /// * `id`: The id that's used for debugging. 345 | /// 346 | /// # Warning 347 | /// 348 | /// This function will not add the single post provided if it has no direct valid URL. 349 | fn add_single_post(&mut self, entry: PostEntry, id: i64) { 350 | match entry.file.url { 351 | None => warn!( 352 | "Post with ID {} has no URL!", 353 | console::style(format!("\"{id}\"")).color256(39).italic() 354 | ), 355 | Some(_) => { 356 | let grabbed_post = GrabbedPost::from((entry, Config::get().naming_convention())); 357 | self.single_post_collection().posts.push(grabbed_post); 358 | info!( 359 | "Post with ID {} grabbed!", 360 | console::style(format!("\"{id}\"")).color256(39).italic() 361 | ); 362 | } 363 | } 364 | } 365 | 366 | /// Searches and grabs post based on the tag given. 367 | /// 368 | /// # Arguments 369 | /// 370 | /// * `tag`: The tag to search for. 371 | fn grab_by_tag_type(&mut self, tag: &Tag) { 372 | match tag.tag_type() { 373 | TagType::Pool => self.grab_pool(tag), 374 | TagType::Set => self.grab_set(tag), 375 | TagType::Post => self.grab_post(tag), 376 | TagType::General | TagType::Artist => self.grab_general(tag), 377 | TagType::Unknown => unreachable!(), 378 | }; 379 | } 380 | 381 | /// Grabs general posts based on the given tag. 382 | /// 383 | /// # Arguments 384 | /// 385 | /// * `tag`: The tag to search for. 386 | fn grab_general(&mut self, tag: &Tag) { 387 | let posts = self.get_posts_from_tag(tag); 388 | self.posts.push(PostCollection::new( 389 | tag.name(), 390 | "General Searches", 391 | GrabbedPost::new_vec(posts), 392 | )); 393 | info!( 394 | "{} grabbed!", 395 | console::style(format!("\"{}\"", tag.name())) 396 | .color256(39) 397 | .italic() 398 | ); 399 | } 400 | 401 | /// Grabs single post based on the given tag. 402 | /// 403 | /// # Arguments 404 | /// 405 | /// * `tag`: The tag to search for. 406 | fn grab_post(&mut self, tag: &Tag) { 407 | let entry: PostEntry = self 408 | .request_sender 409 | .get_entry_from_appended_id(tag.name(), "single"); 410 | let id = entry.id; 411 | 412 | if self.safe_mode { 413 | match entry.rating.as_str() { 414 | "s" => { 415 | self.add_single_post(entry, id); 416 | } 417 | _ => { 418 | info!( 419 | "Skipping Post: {} due to being explicit or questionable", 420 | console::style(format!("\"{id}\"")).color256(39).italic() 421 | ); 422 | } 423 | } 424 | } else { 425 | self.add_single_post(entry, id); 426 | } 427 | } 428 | 429 | /// Grabs a set based on the given tag. 430 | /// 431 | /// # Arguments 432 | /// 433 | /// * `tag`: The tag to search for. 434 | fn grab_set(&mut self, tag: &Tag) { 435 | let entry: SetEntry = self 436 | .request_sender 437 | .get_entry_from_appended_id(tag.name(), "set"); 438 | 439 | // Grabs posts from IDs in the set entry. 440 | let posts = self.search(&format!("set:{}", entry.shortname), &TagSearchType::Special); 441 | self.posts 442 | .push(PostCollection::from((&entry, GrabbedPost::new_vec(posts)))); 443 | 444 | info!( 445 | "{} grabbed!", 446 | console::style(format!("\"{}\"", entry.name)) 447 | .color256(39) 448 | .italic() 449 | ); 450 | } 451 | 452 | /// Grabs pool based on the given tag. 453 | /// 454 | /// # Arguments 455 | /// 456 | /// * `tag`: The tag to search for. 457 | fn grab_pool(&mut self, tag: &Tag) { 458 | let mut entry: PoolEntry = self 459 | .request_sender 460 | .get_entry_from_appended_id(tag.name(), "pool"); 461 | let name = &entry.name; 462 | let mut posts = self.search(&format!("pool:{}", entry.id), &TagSearchType::Special); 463 | 464 | // Updates entry post ids in case any posts were filtered in the search. 465 | entry 466 | .post_ids 467 | .retain(|id| posts.iter().any(|post| post.id == *id)); 468 | 469 | // Sorts the pool to the original order given by entry. 470 | Self::sort_pool_by_id(&entry, &mut posts); 471 | 472 | self.posts.push(PostCollection::new( 473 | name, 474 | "Pools", 475 | GrabbedPost::new_vec((posts, name.as_ref())), 476 | )); 477 | 478 | info!( 479 | "{} grabbed!", 480 | console::style(format!("\"{name}\"")).color256(39).italic() 481 | ); 482 | } 483 | 484 | /// Sorts a pool by id based on the supplied [PoolEntry]. 485 | /// 486 | /// # Arguments 487 | /// 488 | /// * `entry`: The [PoolEntry] to check ids against 489 | /// * `posts`: The [PostEntry] array to sort 490 | fn sort_pool_by_id(entry: &PoolEntry, posts: &mut [PostEntry]) { 491 | for (i, id) in entry.post_ids.iter().enumerate() { 492 | if posts[i].id != *id { 493 | let correct_index = posts.iter().position(|e| e.id == *id).unwrap(); 494 | posts.swap(i, correct_index); 495 | } 496 | } 497 | } 498 | 499 | /// Searches and grabs posts using the given tag. 500 | /// 501 | /// # Arguments 502 | /// 503 | /// * `tag`: The tag to use for the search. 504 | /// 505 | /// returns: Vec 506 | fn get_posts_from_tag(&self, tag: &Tag) -> Vec { 507 | self.search(tag.name(), tag.search_type()) 508 | } 509 | 510 | /// Performs a search where it grabs posts. 511 | /// 512 | /// Depending on the given [TagSearchType], the way posts are grabs will be different. 513 | /// - [General](TagSearchType::General) will search through pages only up to the [POST_SEARCH_LIMIT] 514 | /// - [Special](TagSearchType::Special) will search repeatedly until there are no pages left to grab. 515 | /// 516 | /// # Arguments 517 | /// 518 | /// * `searching_tag`: The tag used for the search. 519 | /// * `tag_search_type`: The type of search to happen. 520 | /// 521 | /// returns: Vec 522 | fn search(&self, searching_tag: &str, tag_search_type: &TagSearchType) -> Vec { 523 | let mut posts: Vec = Vec::new(); 524 | let mut filtered = 0; 525 | let mut invalid_posts = 0; 526 | match tag_search_type { 527 | TagSearchType::General => { 528 | posts = Vec::with_capacity(320 * POST_SEARCH_LIMIT as usize); 529 | self.general_search(searching_tag, &mut posts, &mut filtered, &mut invalid_posts); 530 | } 531 | TagSearchType::Special => { 532 | self.special_search(searching_tag, &mut posts, &mut filtered, &mut invalid_posts); 533 | } 534 | TagSearchType::None => {} 535 | } 536 | 537 | if filtered > 0 { 538 | info!( 539 | "Filtered {} total blacklisted posts from search...", 540 | console::style(filtered).cyan().italic() 541 | ); 542 | } 543 | 544 | if invalid_posts > 0 { 545 | info!( 546 | "Filtered {} total invalid posts from search...", 547 | console::style(invalid_posts).cyan().italic() 548 | ); 549 | } 550 | 551 | posts 552 | } 553 | 554 | /// Performs a special search to grab posts. 555 | /// 556 | /// The difference between special/general searches are this. 557 | /// - Special searches aim to keep grabbing posts until there are not posts left to grab. 558 | /// - General searches aim to grab only a few pages of posts (commonly 320 posts per page). You can refer to the 559 | /// [POST_SEARCH_LIMIT] for the current search limit of the general search. 560 | /// 561 | /// # Arguments 562 | /// 563 | /// * `searching_tag`: The tag to search for. 564 | /// * `posts`: The posts [Vec] to add searched posts into. 565 | /// * `filtered`: The total amount of posts filtered. 566 | /// * `invalid_posts`: The total amount of posts invalid by the [Blacklist]. 567 | fn special_search( 568 | &self, 569 | searching_tag: &str, 570 | posts: &mut Vec, 571 | filtered: &mut u16, 572 | invalid_posts: &mut u16, 573 | ) { 574 | let mut page = 1; 575 | 576 | loop { 577 | let mut searched_posts = self.request_sender.bulk_search(searching_tag, page).posts; 578 | if searched_posts.is_empty() { 579 | break; 580 | } 581 | 582 | *filtered += self.filter_posts_with_blacklist(&mut searched_posts); 583 | *invalid_posts += Self::remove_invalid_posts(&mut searched_posts); 584 | 585 | searched_posts.reverse(); 586 | posts.append(&mut searched_posts); 587 | page += 1; 588 | } 589 | } 590 | 591 | /// Performs a general search to grab posts. 592 | /// 593 | /// The difference between special/general searches are this. 594 | /// - Special searches aim to keep grabbing posts until there are not posts left to grab. 595 | /// - General searches aim to grab only a few pages of posts (commonly 320 posts per page). You can refer to the 596 | /// [POST_SEARCH_LIMIT] for the current search limit of the general search. 597 | /// 598 | /// # Arguments 599 | /// 600 | /// * `searching_tag`: The tag to search for. 601 | /// * `posts`: The posts [Vec] to add searched posts into. 602 | /// * `filtered`: The total amount of posts filtered. 603 | /// * `invalid_posts`: The total amount of posts invalid by the [Blacklist]. 604 | fn general_search( 605 | &self, 606 | searching_tag: &str, 607 | posts: &mut Vec, 608 | filtered: &mut u16, 609 | invalid_posts: &mut u16, 610 | ) { 611 | for page in 1..POST_SEARCH_LIMIT { 612 | let mut searched_posts: Vec = self 613 | .request_sender 614 | .bulk_search(searching_tag, page as u16) 615 | .posts; 616 | if searched_posts.is_empty() { 617 | break; 618 | } 619 | 620 | *filtered += self.filter_posts_with_blacklist(&mut searched_posts); 621 | *invalid_posts += Self::remove_invalid_posts(&mut searched_posts); 622 | 623 | searched_posts.reverse(); 624 | posts.append(&mut searched_posts); 625 | } 626 | } 627 | 628 | /// Checks through posts and removes any that violets the blacklist. 629 | /// 630 | /// # Arguments 631 | /// 632 | /// * `posts`: The posts to check 633 | /// 634 | /// returns: u16 635 | fn filter_posts_with_blacklist(&self, posts: &mut Vec) -> u16 { 636 | if self.request_sender.is_authenticated() { 637 | if let Some(ref blacklist) = self.blacklist { 638 | return blacklist.borrow_mut().filter_posts(posts); 639 | } 640 | } 641 | 642 | 0 643 | } 644 | 645 | /// Removes invalid posts (e.g posts with no urls, or invalid properties). 646 | /// 647 | /// Sometimes, even if a post is available, the url for it isn't; To handle this, the [Vec]<[PostEntry]> will retain only 648 | /// the posts that has an available url. So far, the only check needed is the url check, since if the url is [None], 649 | /// the entire post is [None]. 650 | /// 651 | /// # Arguments 652 | /// 653 | /// * `posts`: Posts to check. 654 | /// 655 | /// returns: u16 656 | fn remove_invalid_posts(posts: &mut Vec) -> u16 { 657 | let mut invalid_posts = 0; 658 | posts.retain(|e| { 659 | if !e.flags.deleted && e.file.url.is_some() { 660 | true 661 | } else { 662 | invalid_posts += 1; 663 | false 664 | } 665 | }); 666 | 667 | Self::log_invalid_posts(&invalid_posts); 668 | 669 | invalid_posts 670 | } 671 | 672 | /// Traces invalid posts to the log file. 673 | /// 674 | /// # Arguments 675 | /// 676 | /// * `invalid_posts`: The total count of invalid posts. 677 | fn log_invalid_posts(invalid_posts: &u16) { 678 | match invalid_posts.cmp(&1) { 679 | Ordering::Less => {} 680 | Ordering::Equal => { 681 | trace!( 682 | "A post was filtered for being invalid (due to the user not being logged in)" 683 | ); 684 | trace!("A post was filtered by e621..."); 685 | } 686 | Ordering::Greater => { 687 | trace!("{} posts were filtered for being invalid (due to the user not being logged in)", invalid_posts); 688 | trace!("{} posts had to be filtered by e621/e926...", invalid_posts,); 689 | } 690 | } 691 | } 692 | } 693 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "aho-corasick" 7 | version = "1.0.1" 8 | source = "registry+https://github.com/rust-lang/crates.io-index" 9 | checksum = "67fc08ce920c31afb70f013dcce1bfc3a3195de6a228474e45e1f145b36f8d04" 10 | dependencies = [ 11 | "memchr", 12 | ] 13 | 14 | [[package]] 15 | name = "aligned" 16 | version = "0.4.1" 17 | source = "registry+https://github.com/rust-lang/crates.io-index" 18 | checksum = "80a21b9440a626c7fc8573a9e3d3a06b75c7c97754c2949bc7857b90353ca655" 19 | dependencies = [ 20 | "as-slice", 21 | ] 22 | 23 | [[package]] 24 | name = "anyhow" 25 | version = "1.0.71" 26 | source = "registry+https://github.com/rust-lang/crates.io-index" 27 | checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" 28 | 29 | [[package]] 30 | name = "as-slice" 31 | version = "0.2.1" 32 | source = "registry+https://github.com/rust-lang/crates.io-index" 33 | checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" 34 | dependencies = [ 35 | "stable_deref_trait", 36 | ] 37 | 38 | [[package]] 39 | name = "autocfg" 40 | version = "1.1.0" 41 | source = "registry+https://github.com/rust-lang/crates.io-index" 42 | checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" 43 | 44 | [[package]] 45 | name = "base64" 46 | version = "0.21.0" 47 | source = "registry+https://github.com/rust-lang/crates.io-index" 48 | checksum = "a4a4ddaa51a5bc52a6948f74c06d20aaaddb71924eab79b8c97a8c556e942d6a" 49 | 50 | [[package]] 51 | name = "base64-url" 52 | version = "2.0.0" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "9c5b0a88aa36e9f095ee2e2b13fb8c5e4313e022783aedacc123328c0084916d" 55 | dependencies = [ 56 | "base64", 57 | ] 58 | 59 | [[package]] 60 | name = "bitflags" 61 | version = "1.3.2" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" 64 | 65 | [[package]] 66 | name = "bumpalo" 67 | version = "3.12.1" 68 | source = "registry+https://github.com/rust-lang/crates.io-index" 69 | checksum = "9b1ce199063694f33ffb7dd4e0ee620741495c32833cde5aa08f02a0bf96f0c8" 70 | 71 | [[package]] 72 | name = "bytes" 73 | version = "1.4.0" 74 | source = "registry+https://github.com/rust-lang/crates.io-index" 75 | checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" 76 | 77 | [[package]] 78 | name = "cc" 79 | version = "1.0.79" 80 | source = "registry+https://github.com/rust-lang/crates.io-index" 81 | checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" 82 | 83 | [[package]] 84 | name = "cfg-if" 85 | version = "1.0.0" 86 | source = "registry+https://github.com/rust-lang/crates.io-index" 87 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 88 | 89 | [[package]] 90 | name = "console" 91 | version = "0.15.5" 92 | source = "registry+https://github.com/rust-lang/crates.io-index" 93 | checksum = "c3d79fbe8970a77e3e34151cc13d3b3e248aa0faaecb9f6091fa07ebefe5ad60" 94 | dependencies = [ 95 | "encode_unicode", 96 | "lazy_static", 97 | "libc", 98 | "unicode-width", 99 | "windows-sys 0.42.0", 100 | ] 101 | 102 | [[package]] 103 | name = "core-foundation" 104 | version = "0.9.3" 105 | source = "registry+https://github.com/rust-lang/crates.io-index" 106 | checksum = "194a7a9e6de53fa55116934067c844d9d749312f75c6f6d0980e8c252f8c2146" 107 | dependencies = [ 108 | "core-foundation-sys", 109 | "libc", 110 | ] 111 | 112 | [[package]] 113 | name = "core-foundation-sys" 114 | version = "0.8.4" 115 | source = "registry+https://github.com/rust-lang/crates.io-index" 116 | checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" 117 | 118 | [[package]] 119 | name = "cvt" 120 | version = "0.1.2" 121 | source = "registry+https://github.com/rust-lang/crates.io-index" 122 | checksum = "d2ae9bf77fbf2d39ef573205d554d87e86c12f1994e9ea335b0651b9b278bcf1" 123 | dependencies = [ 124 | "cfg-if", 125 | ] 126 | 127 | [[package]] 128 | name = "dialoguer" 129 | version = "0.10.4" 130 | source = "registry+https://github.com/rust-lang/crates.io-index" 131 | checksum = "59c6f2989294b9a498d3ad5491a79c6deb604617378e1cdc4bfc1c1361fe2f87" 132 | dependencies = [ 133 | "console", 134 | "shell-words", 135 | "tempfile", 136 | "zeroize", 137 | ] 138 | 139 | [[package]] 140 | name = "e621_downloader" 141 | version = "1.7.2" 142 | dependencies = [ 143 | "anyhow", 144 | "base64-url", 145 | "bumpalo", 146 | "console", 147 | "dialoguer", 148 | "h2", 149 | "indicatif", 150 | "log", 151 | "once_cell", 152 | "regex", 153 | "remove_dir_all", 154 | "reqwest", 155 | "rustls 0.21.1", 156 | "serde", 157 | "serde_json", 158 | "simplelog", 159 | "smallvec", 160 | "tokio", 161 | ] 162 | 163 | [[package]] 164 | name = "encode_unicode" 165 | version = "0.3.6" 166 | source = "registry+https://github.com/rust-lang/crates.io-index" 167 | checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" 168 | 169 | [[package]] 170 | name = "encoding_rs" 171 | version = "0.8.32" 172 | source = "registry+https://github.com/rust-lang/crates.io-index" 173 | checksum = "071a31f4ee85403370b58aca746f01041ede6f0da2730960ad001edc2b71b394" 174 | dependencies = [ 175 | "cfg-if", 176 | ] 177 | 178 | [[package]] 179 | name = "errno" 180 | version = "0.3.1" 181 | source = "registry+https://github.com/rust-lang/crates.io-index" 182 | checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" 183 | dependencies = [ 184 | "errno-dragonfly", 185 | "libc", 186 | "windows-sys 0.48.0", 187 | ] 188 | 189 | [[package]] 190 | name = "errno-dragonfly" 191 | version = "0.1.2" 192 | source = "registry+https://github.com/rust-lang/crates.io-index" 193 | checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" 194 | dependencies = [ 195 | "cc", 196 | "libc", 197 | ] 198 | 199 | [[package]] 200 | name = "fastrand" 201 | version = "1.9.0" 202 | source = "registry+https://github.com/rust-lang/crates.io-index" 203 | checksum = "e51093e27b0797c359783294ca4f0a911c270184cb10f85783b118614a1501be" 204 | dependencies = [ 205 | "instant", 206 | ] 207 | 208 | [[package]] 209 | name = "fnv" 210 | version = "1.0.7" 211 | source = "registry+https://github.com/rust-lang/crates.io-index" 212 | checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" 213 | 214 | [[package]] 215 | name = "foreign-types" 216 | version = "0.3.2" 217 | source = "registry+https://github.com/rust-lang/crates.io-index" 218 | checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" 219 | dependencies = [ 220 | "foreign-types-shared", 221 | ] 222 | 223 | [[package]] 224 | name = "foreign-types-shared" 225 | version = "0.1.1" 226 | source = "registry+https://github.com/rust-lang/crates.io-index" 227 | checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" 228 | 229 | [[package]] 230 | name = "form_urlencoded" 231 | version = "1.1.0" 232 | source = "registry+https://github.com/rust-lang/crates.io-index" 233 | checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8" 234 | dependencies = [ 235 | "percent-encoding", 236 | ] 237 | 238 | [[package]] 239 | name = "fs_at" 240 | version = "0.1.6" 241 | source = "registry+https://github.com/rust-lang/crates.io-index" 242 | checksum = "0504bab20f4487fdf1c20ed48e3e32c7951827a778cd3dfded1768f90b6abb0a" 243 | dependencies = [ 244 | "aligned", 245 | "cfg-if", 246 | "cvt", 247 | "libc", 248 | "nix", 249 | "smart-default", 250 | "windows-sys 0.48.0", 251 | ] 252 | 253 | [[package]] 254 | name = "futures-channel" 255 | version = "0.3.28" 256 | source = "registry+https://github.com/rust-lang/crates.io-index" 257 | checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" 258 | dependencies = [ 259 | "futures-core", 260 | ] 261 | 262 | [[package]] 263 | name = "futures-core" 264 | version = "0.3.28" 265 | source = "registry+https://github.com/rust-lang/crates.io-index" 266 | checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" 267 | 268 | [[package]] 269 | name = "futures-io" 270 | version = "0.3.28" 271 | source = "registry+https://github.com/rust-lang/crates.io-index" 272 | checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" 273 | 274 | [[package]] 275 | name = "futures-sink" 276 | version = "0.3.28" 277 | source = "registry+https://github.com/rust-lang/crates.io-index" 278 | checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" 279 | 280 | [[package]] 281 | name = "futures-task" 282 | version = "0.3.28" 283 | source = "registry+https://github.com/rust-lang/crates.io-index" 284 | checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" 285 | 286 | [[package]] 287 | name = "futures-util" 288 | version = "0.3.28" 289 | source = "registry+https://github.com/rust-lang/crates.io-index" 290 | checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" 291 | dependencies = [ 292 | "futures-core", 293 | "futures-io", 294 | "futures-task", 295 | "memchr", 296 | "pin-project-lite", 297 | "pin-utils", 298 | "slab", 299 | ] 300 | 301 | [[package]] 302 | name = "h2" 303 | version = "0.3.18" 304 | source = "registry+https://github.com/rust-lang/crates.io-index" 305 | checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" 306 | dependencies = [ 307 | "bytes", 308 | "fnv", 309 | "futures-core", 310 | "futures-sink", 311 | "futures-util", 312 | "http", 313 | "indexmap", 314 | "slab", 315 | "tokio", 316 | "tokio-util", 317 | "tracing", 318 | ] 319 | 320 | [[package]] 321 | name = "hashbrown" 322 | version = "0.12.3" 323 | source = "registry+https://github.com/rust-lang/crates.io-index" 324 | checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" 325 | 326 | [[package]] 327 | name = "hermit-abi" 328 | version = "0.2.6" 329 | source = "registry+https://github.com/rust-lang/crates.io-index" 330 | checksum = "ee512640fe35acbfb4bb779db6f0d80704c2cacfa2e39b601ef3e3f47d1ae4c7" 331 | dependencies = [ 332 | "libc", 333 | ] 334 | 335 | [[package]] 336 | name = "hermit-abi" 337 | version = "0.3.1" 338 | source = "registry+https://github.com/rust-lang/crates.io-index" 339 | checksum = "fed44880c466736ef9a5c5b5facefb5ed0785676d0c02d612db14e54f0d84286" 340 | 341 | [[package]] 342 | name = "http" 343 | version = "0.2.9" 344 | source = "registry+https://github.com/rust-lang/crates.io-index" 345 | checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" 346 | dependencies = [ 347 | "bytes", 348 | "fnv", 349 | "itoa", 350 | ] 351 | 352 | [[package]] 353 | name = "http-body" 354 | version = "0.4.5" 355 | source = "registry+https://github.com/rust-lang/crates.io-index" 356 | checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" 357 | dependencies = [ 358 | "bytes", 359 | "http", 360 | "pin-project-lite", 361 | ] 362 | 363 | [[package]] 364 | name = "httparse" 365 | version = "1.8.0" 366 | source = "registry+https://github.com/rust-lang/crates.io-index" 367 | checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" 368 | 369 | [[package]] 370 | name = "httpdate" 371 | version = "1.0.2" 372 | source = "registry+https://github.com/rust-lang/crates.io-index" 373 | checksum = "c4a1e36c821dbe04574f602848a19f742f4fb3c98d40449f11bcad18d6b17421" 374 | 375 | [[package]] 376 | name = "hyper" 377 | version = "0.14.26" 378 | source = "registry+https://github.com/rust-lang/crates.io-index" 379 | checksum = "ab302d72a6f11a3b910431ff93aae7e773078c769f0a3ef15fb9ec692ed147d4" 380 | dependencies = [ 381 | "bytes", 382 | "futures-channel", 383 | "futures-core", 384 | "futures-util", 385 | "h2", 386 | "http", 387 | "http-body", 388 | "httparse", 389 | "httpdate", 390 | "itoa", 391 | "pin-project-lite", 392 | "socket2", 393 | "tokio", 394 | "tower-service", 395 | "tracing", 396 | "want", 397 | ] 398 | 399 | [[package]] 400 | name = "hyper-rustls" 401 | version = "0.23.2" 402 | source = "registry+https://github.com/rust-lang/crates.io-index" 403 | checksum = "1788965e61b367cd03a62950836d5cd41560c3577d90e40e0819373194d1661c" 404 | dependencies = [ 405 | "http", 406 | "hyper", 407 | "rustls 0.20.8", 408 | "tokio", 409 | "tokio-rustls", 410 | ] 411 | 412 | [[package]] 413 | name = "hyper-tls" 414 | version = "0.5.0" 415 | source = "registry+https://github.com/rust-lang/crates.io-index" 416 | checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" 417 | dependencies = [ 418 | "bytes", 419 | "hyper", 420 | "native-tls", 421 | "tokio", 422 | "tokio-native-tls", 423 | ] 424 | 425 | [[package]] 426 | name = "idna" 427 | version = "0.3.0" 428 | source = "registry+https://github.com/rust-lang/crates.io-index" 429 | checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6" 430 | dependencies = [ 431 | "unicode-bidi", 432 | "unicode-normalization", 433 | ] 434 | 435 | [[package]] 436 | name = "indexmap" 437 | version = "1.9.3" 438 | source = "registry+https://github.com/rust-lang/crates.io-index" 439 | checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" 440 | dependencies = [ 441 | "autocfg", 442 | "hashbrown", 443 | ] 444 | 445 | [[package]] 446 | name = "indicatif" 447 | version = "0.17.3" 448 | source = "registry+https://github.com/rust-lang/crates.io-index" 449 | checksum = "cef509aa9bc73864d6756f0d34d35504af3cf0844373afe9b8669a5b8005a729" 450 | dependencies = [ 451 | "console", 452 | "number_prefix", 453 | "portable-atomic", 454 | "unicode-width", 455 | ] 456 | 457 | [[package]] 458 | name = "instant" 459 | version = "0.1.12" 460 | source = "registry+https://github.com/rust-lang/crates.io-index" 461 | checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" 462 | dependencies = [ 463 | "cfg-if", 464 | ] 465 | 466 | [[package]] 467 | name = "io-lifetimes" 468 | version = "1.0.10" 469 | source = "registry+https://github.com/rust-lang/crates.io-index" 470 | checksum = "9c66c74d2ae7e79a5a8f7ac924adbe38ee42a859c6539ad869eb51f0b52dc220" 471 | dependencies = [ 472 | "hermit-abi 0.3.1", 473 | "libc", 474 | "windows-sys 0.48.0", 475 | ] 476 | 477 | [[package]] 478 | name = "ipnet" 479 | version = "2.7.2" 480 | source = "registry+https://github.com/rust-lang/crates.io-index" 481 | checksum = "12b6ee2129af8d4fb011108c73d99a1b83a85977f23b82460c0ae2e25bb4b57f" 482 | 483 | [[package]] 484 | name = "itoa" 485 | version = "1.0.6" 486 | source = "registry+https://github.com/rust-lang/crates.io-index" 487 | checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6" 488 | 489 | [[package]] 490 | name = "js-sys" 491 | version = "0.3.61" 492 | source = "registry+https://github.com/rust-lang/crates.io-index" 493 | checksum = "445dde2150c55e483f3d8416706b97ec8e8237c307e5b7b4b8dd15e6af2a0730" 494 | dependencies = [ 495 | "wasm-bindgen", 496 | ] 497 | 498 | [[package]] 499 | name = "lazy_static" 500 | version = "1.4.0" 501 | source = "registry+https://github.com/rust-lang/crates.io-index" 502 | checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" 503 | 504 | [[package]] 505 | name = "libc" 506 | version = "0.2.142" 507 | source = "registry+https://github.com/rust-lang/crates.io-index" 508 | checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317" 509 | 510 | [[package]] 511 | name = "linux-raw-sys" 512 | version = "0.3.6" 513 | source = "registry+https://github.com/rust-lang/crates.io-index" 514 | checksum = "b64f40e5e03e0d54f03845c8197d0291253cdbedfb1cb46b13c2c117554a9f4c" 515 | 516 | [[package]] 517 | name = "log" 518 | version = "0.4.17" 519 | source = "registry+https://github.com/rust-lang/crates.io-index" 520 | checksum = "abb12e687cfb44aa40f41fc3978ef76448f9b6038cad6aef4259d3c095a2382e" 521 | dependencies = [ 522 | "cfg-if", 523 | ] 524 | 525 | [[package]] 526 | name = "memchr" 527 | version = "2.5.0" 528 | source = "registry+https://github.com/rust-lang/crates.io-index" 529 | checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" 530 | 531 | [[package]] 532 | name = "mime" 533 | version = "0.3.17" 534 | source = "registry+https://github.com/rust-lang/crates.io-index" 535 | checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" 536 | 537 | [[package]] 538 | name = "mio" 539 | version = "0.8.6" 540 | source = "registry+https://github.com/rust-lang/crates.io-index" 541 | checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" 542 | dependencies = [ 543 | "libc", 544 | "log", 545 | "wasi", 546 | "windows-sys 0.45.0", 547 | ] 548 | 549 | [[package]] 550 | name = "native-tls" 551 | version = "0.2.11" 552 | source = "registry+https://github.com/rust-lang/crates.io-index" 553 | checksum = "07226173c32f2926027b63cce4bcd8076c3552846cbe7925f3aaffeac0a3b92e" 554 | dependencies = [ 555 | "lazy_static", 556 | "libc", 557 | "log", 558 | "openssl", 559 | "openssl-probe", 560 | "openssl-sys", 561 | "schannel", 562 | "security-framework", 563 | "security-framework-sys", 564 | "tempfile", 565 | ] 566 | 567 | [[package]] 568 | name = "nix" 569 | version = "0.26.2" 570 | source = "registry+https://github.com/rust-lang/crates.io-index" 571 | checksum = "bfdda3d196821d6af13126e40375cdf7da646a96114af134d5f417a9a1dc8e1a" 572 | dependencies = [ 573 | "bitflags", 574 | "cfg-if", 575 | "libc", 576 | "static_assertions", 577 | ] 578 | 579 | [[package]] 580 | name = "normpath" 581 | version = "1.1.1" 582 | source = "registry+https://github.com/rust-lang/crates.io-index" 583 | checksum = "ec60c60a693226186f5d6edf073232bfb6464ed97eb22cf3b01c1e8198fd97f5" 584 | dependencies = [ 585 | "windows-sys 0.48.0", 586 | ] 587 | 588 | [[package]] 589 | name = "num_cpus" 590 | version = "1.15.0" 591 | source = "registry+https://github.com/rust-lang/crates.io-index" 592 | checksum = "0fac9e2da13b5eb447a6ce3d392f23a29d8694bff781bf03a16cd9ac8697593b" 593 | dependencies = [ 594 | "hermit-abi 0.2.6", 595 | "libc", 596 | ] 597 | 598 | [[package]] 599 | name = "num_threads" 600 | version = "0.1.6" 601 | source = "registry+https://github.com/rust-lang/crates.io-index" 602 | checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" 603 | dependencies = [ 604 | "libc", 605 | ] 606 | 607 | [[package]] 608 | name = "number_prefix" 609 | version = "0.4.0" 610 | source = "registry+https://github.com/rust-lang/crates.io-index" 611 | checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" 612 | 613 | [[package]] 614 | name = "once_cell" 615 | version = "1.17.1" 616 | source = "registry+https://github.com/rust-lang/crates.io-index" 617 | checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3" 618 | 619 | [[package]] 620 | name = "openssl" 621 | version = "0.10.52" 622 | source = "registry+https://github.com/rust-lang/crates.io-index" 623 | checksum = "01b8574602df80f7b85fdfc5392fa884a4e3b3f4f35402c070ab34c3d3f78d56" 624 | dependencies = [ 625 | "bitflags", 626 | "cfg-if", 627 | "foreign-types", 628 | "libc", 629 | "once_cell", 630 | "openssl-macros", 631 | "openssl-sys", 632 | ] 633 | 634 | [[package]] 635 | name = "openssl-macros" 636 | version = "0.1.1" 637 | source = "registry+https://github.com/rust-lang/crates.io-index" 638 | checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" 639 | dependencies = [ 640 | "proc-macro2", 641 | "quote", 642 | "syn 2.0.15", 643 | ] 644 | 645 | [[package]] 646 | name = "openssl-probe" 647 | version = "0.1.5" 648 | source = "registry+https://github.com/rust-lang/crates.io-index" 649 | checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" 650 | 651 | [[package]] 652 | name = "openssl-sys" 653 | version = "0.9.87" 654 | source = "registry+https://github.com/rust-lang/crates.io-index" 655 | checksum = "8e17f59264b2809d77ae94f0e1ebabc434773f370d6ca667bd223ea10e06cc7e" 656 | dependencies = [ 657 | "cc", 658 | "libc", 659 | "pkg-config", 660 | "vcpkg", 661 | ] 662 | 663 | [[package]] 664 | name = "percent-encoding" 665 | version = "2.2.0" 666 | source = "registry+https://github.com/rust-lang/crates.io-index" 667 | checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e" 668 | 669 | [[package]] 670 | name = "pin-project-lite" 671 | version = "0.2.9" 672 | source = "registry+https://github.com/rust-lang/crates.io-index" 673 | checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" 674 | 675 | [[package]] 676 | name = "pin-utils" 677 | version = "0.1.0" 678 | source = "registry+https://github.com/rust-lang/crates.io-index" 679 | checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" 680 | 681 | [[package]] 682 | name = "pkg-config" 683 | version = "0.3.26" 684 | source = "registry+https://github.com/rust-lang/crates.io-index" 685 | checksum = "6ac9a59f73473f1b8d852421e59e64809f025994837ef743615c6d0c5b305160" 686 | 687 | [[package]] 688 | name = "portable-atomic" 689 | version = "0.3.19" 690 | source = "registry+https://github.com/rust-lang/crates.io-index" 691 | checksum = "26f6a7b87c2e435a3241addceeeff740ff8b7e76b74c13bf9acb17fa454ea00b" 692 | 693 | [[package]] 694 | name = "proc-macro2" 695 | version = "1.0.56" 696 | source = "registry+https://github.com/rust-lang/crates.io-index" 697 | checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435" 698 | dependencies = [ 699 | "unicode-ident", 700 | ] 701 | 702 | [[package]] 703 | name = "quote" 704 | version = "1.0.26" 705 | source = "registry+https://github.com/rust-lang/crates.io-index" 706 | checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" 707 | dependencies = [ 708 | "proc-macro2", 709 | ] 710 | 711 | [[package]] 712 | name = "redox_syscall" 713 | version = "0.3.5" 714 | source = "registry+https://github.com/rust-lang/crates.io-index" 715 | checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" 716 | dependencies = [ 717 | "bitflags", 718 | ] 719 | 720 | [[package]] 721 | name = "regex" 722 | version = "1.8.1" 723 | source = "registry+https://github.com/rust-lang/crates.io-index" 724 | checksum = "af83e617f331cc6ae2da5443c602dfa5af81e517212d9d611a5b3ba1777b5370" 725 | dependencies = [ 726 | "aho-corasick", 727 | "memchr", 728 | "regex-syntax", 729 | ] 730 | 731 | [[package]] 732 | name = "regex-syntax" 733 | version = "0.7.1" 734 | source = "registry+https://github.com/rust-lang/crates.io-index" 735 | checksum = "a5996294f19bd3aae0453a862ad728f60e6600695733dd5df01da90c54363a3c" 736 | 737 | [[package]] 738 | name = "remove_dir_all" 739 | version = "0.8.2" 740 | source = "registry+https://github.com/rust-lang/crates.io-index" 741 | checksum = "23895cfadc1917fed9c6ed76a8c2903615fa3704f7493ff82b364c6540acc02b" 742 | dependencies = [ 743 | "aligned", 744 | "cfg-if", 745 | "cvt", 746 | "fs_at", 747 | "lazy_static", 748 | "libc", 749 | "normpath", 750 | "windows-sys 0.45.0", 751 | ] 752 | 753 | [[package]] 754 | name = "reqwest" 755 | version = "0.11.17" 756 | source = "registry+https://github.com/rust-lang/crates.io-index" 757 | checksum = "13293b639a097af28fc8a90f22add145a9c954e49d77da06263d58cf44d5fb91" 758 | dependencies = [ 759 | "base64", 760 | "bytes", 761 | "encoding_rs", 762 | "futures-core", 763 | "futures-util", 764 | "h2", 765 | "http", 766 | "http-body", 767 | "hyper", 768 | "hyper-rustls", 769 | "hyper-tls", 770 | "ipnet", 771 | "js-sys", 772 | "log", 773 | "mime", 774 | "native-tls", 775 | "once_cell", 776 | "percent-encoding", 777 | "pin-project-lite", 778 | "rustls 0.20.8", 779 | "rustls-pemfile", 780 | "serde", 781 | "serde_json", 782 | "serde_urlencoded", 783 | "tokio", 784 | "tokio-native-tls", 785 | "tokio-rustls", 786 | "tower-service", 787 | "url", 788 | "wasm-bindgen", 789 | "wasm-bindgen-futures", 790 | "web-sys", 791 | "webpki-roots", 792 | "winreg", 793 | ] 794 | 795 | [[package]] 796 | name = "ring" 797 | version = "0.16.20" 798 | source = "registry+https://github.com/rust-lang/crates.io-index" 799 | checksum = "3053cf52e236a3ed746dfc745aa9cacf1b791d846bdaf412f60a8d7d6e17c8fc" 800 | dependencies = [ 801 | "cc", 802 | "libc", 803 | "once_cell", 804 | "spin", 805 | "untrusted", 806 | "web-sys", 807 | "winapi", 808 | ] 809 | 810 | [[package]] 811 | name = "rustix" 812 | version = "0.37.18" 813 | source = "registry+https://github.com/rust-lang/crates.io-index" 814 | checksum = "8bbfc1d1c7c40c01715f47d71444744a81669ca84e8b63e25a55e169b1f86433" 815 | dependencies = [ 816 | "bitflags", 817 | "errno", 818 | "io-lifetimes", 819 | "libc", 820 | "linux-raw-sys", 821 | "windows-sys 0.48.0", 822 | ] 823 | 824 | [[package]] 825 | name = "rustls" 826 | version = "0.20.8" 827 | source = "registry+https://github.com/rust-lang/crates.io-index" 828 | checksum = "fff78fc74d175294f4e83b28343315ffcfb114b156f0185e9741cb5570f50e2f" 829 | dependencies = [ 830 | "log", 831 | "ring", 832 | "sct", 833 | "webpki", 834 | ] 835 | 836 | [[package]] 837 | name = "rustls" 838 | version = "0.21.1" 839 | source = "registry+https://github.com/rust-lang/crates.io-index" 840 | checksum = "c911ba11bc8433e811ce56fde130ccf32f5127cab0e0194e9c68c5a5b671791e" 841 | dependencies = [ 842 | "log", 843 | "ring", 844 | "rustls-webpki", 845 | "sct", 846 | ] 847 | 848 | [[package]] 849 | name = "rustls-pemfile" 850 | version = "1.0.2" 851 | source = "registry+https://github.com/rust-lang/crates.io-index" 852 | checksum = "d194b56d58803a43635bdc398cd17e383d6f71f9182b9a192c127ca42494a59b" 853 | dependencies = [ 854 | "base64", 855 | ] 856 | 857 | [[package]] 858 | name = "rustls-webpki" 859 | version = "0.100.1" 860 | source = "registry+https://github.com/rust-lang/crates.io-index" 861 | checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" 862 | dependencies = [ 863 | "ring", 864 | "untrusted", 865 | ] 866 | 867 | [[package]] 868 | name = "ryu" 869 | version = "1.0.13" 870 | source = "registry+https://github.com/rust-lang/crates.io-index" 871 | checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041" 872 | 873 | [[package]] 874 | name = "schannel" 875 | version = "0.1.21" 876 | source = "registry+https://github.com/rust-lang/crates.io-index" 877 | checksum = "713cfb06c7059f3588fb8044c0fad1d09e3c01d225e25b9220dbfdcf16dbb1b3" 878 | dependencies = [ 879 | "windows-sys 0.42.0", 880 | ] 881 | 882 | [[package]] 883 | name = "sct" 884 | version = "0.7.0" 885 | source = "registry+https://github.com/rust-lang/crates.io-index" 886 | checksum = "d53dcdb7c9f8158937a7981b48accfd39a43af418591a5d008c7b22b5e1b7ca4" 887 | dependencies = [ 888 | "ring", 889 | "untrusted", 890 | ] 891 | 892 | [[package]] 893 | name = "security-framework" 894 | version = "2.8.2" 895 | source = "registry+https://github.com/rust-lang/crates.io-index" 896 | checksum = "a332be01508d814fed64bf28f798a146d73792121129962fdf335bb3c49a4254" 897 | dependencies = [ 898 | "bitflags", 899 | "core-foundation", 900 | "core-foundation-sys", 901 | "libc", 902 | "security-framework-sys", 903 | ] 904 | 905 | [[package]] 906 | name = "security-framework-sys" 907 | version = "2.8.0" 908 | source = "registry+https://github.com/rust-lang/crates.io-index" 909 | checksum = "31c9bb296072e961fcbd8853511dd39c2d8be2deb1e17c6860b1d30732b323b4" 910 | dependencies = [ 911 | "core-foundation-sys", 912 | "libc", 913 | ] 914 | 915 | [[package]] 916 | name = "serde" 917 | version = "1.0.160" 918 | source = "registry+https://github.com/rust-lang/crates.io-index" 919 | checksum = "bb2f3770c8bce3bcda7e149193a069a0f4365bda1fa5cd88e03bca26afc1216c" 920 | dependencies = [ 921 | "serde_derive", 922 | ] 923 | 924 | [[package]] 925 | name = "serde_derive" 926 | version = "1.0.160" 927 | source = "registry+https://github.com/rust-lang/crates.io-index" 928 | checksum = "291a097c63d8497e00160b166a967a4a79c64f3facdd01cbd7502231688d77df" 929 | dependencies = [ 930 | "proc-macro2", 931 | "quote", 932 | "syn 2.0.15", 933 | ] 934 | 935 | [[package]] 936 | name = "serde_json" 937 | version = "1.0.96" 938 | source = "registry+https://github.com/rust-lang/crates.io-index" 939 | checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" 940 | dependencies = [ 941 | "itoa", 942 | "ryu", 943 | "serde", 944 | ] 945 | 946 | [[package]] 947 | name = "serde_urlencoded" 948 | version = "0.7.1" 949 | source = "registry+https://github.com/rust-lang/crates.io-index" 950 | checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" 951 | dependencies = [ 952 | "form_urlencoded", 953 | "itoa", 954 | "ryu", 955 | "serde", 956 | ] 957 | 958 | [[package]] 959 | name = "shell-words" 960 | version = "1.1.0" 961 | source = "registry+https://github.com/rust-lang/crates.io-index" 962 | checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" 963 | 964 | [[package]] 965 | name = "simplelog" 966 | version = "0.12.1" 967 | source = "registry+https://github.com/rust-lang/crates.io-index" 968 | checksum = "acee08041c5de3d5048c8b3f6f13fafb3026b24ba43c6a695a0c76179b844369" 969 | dependencies = [ 970 | "log", 971 | "termcolor", 972 | "time", 973 | ] 974 | 975 | [[package]] 976 | name = "slab" 977 | version = "0.4.8" 978 | source = "registry+https://github.com/rust-lang/crates.io-index" 979 | checksum = "6528351c9bc8ab22353f9d776db39a20288e8d6c37ef8cfe3317cf875eecfc2d" 980 | dependencies = [ 981 | "autocfg", 982 | ] 983 | 984 | [[package]] 985 | name = "smallvec" 986 | version = "1.10.0" 987 | source = "registry+https://github.com/rust-lang/crates.io-index" 988 | checksum = "a507befe795404456341dfab10cef66ead4c041f62b8b11bbb92bffe5d0953e0" 989 | 990 | [[package]] 991 | name = "smart-default" 992 | version = "0.7.1" 993 | source = "registry+https://github.com/rust-lang/crates.io-index" 994 | checksum = "0eb01866308440fc64d6c44d9e86c5cc17adfe33c4d6eed55da9145044d0ffc1" 995 | dependencies = [ 996 | "proc-macro2", 997 | "quote", 998 | "syn 2.0.15", 999 | ] 1000 | 1001 | [[package]] 1002 | name = "socket2" 1003 | version = "0.4.9" 1004 | source = "registry+https://github.com/rust-lang/crates.io-index" 1005 | checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" 1006 | dependencies = [ 1007 | "libc", 1008 | "winapi", 1009 | ] 1010 | 1011 | [[package]] 1012 | name = "spin" 1013 | version = "0.5.2" 1014 | source = "registry+https://github.com/rust-lang/crates.io-index" 1015 | checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" 1016 | 1017 | [[package]] 1018 | name = "stable_deref_trait" 1019 | version = "1.2.0" 1020 | source = "registry+https://github.com/rust-lang/crates.io-index" 1021 | checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" 1022 | 1023 | [[package]] 1024 | name = "static_assertions" 1025 | version = "1.1.0" 1026 | source = "registry+https://github.com/rust-lang/crates.io-index" 1027 | checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" 1028 | 1029 | [[package]] 1030 | name = "syn" 1031 | version = "1.0.109" 1032 | source = "registry+https://github.com/rust-lang/crates.io-index" 1033 | checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" 1034 | dependencies = [ 1035 | "proc-macro2", 1036 | "quote", 1037 | "unicode-ident", 1038 | ] 1039 | 1040 | [[package]] 1041 | name = "syn" 1042 | version = "2.0.15" 1043 | source = "registry+https://github.com/rust-lang/crates.io-index" 1044 | checksum = "a34fcf3e8b60f57e6a14301a2e916d323af98b0ea63c599441eec8558660c822" 1045 | dependencies = [ 1046 | "proc-macro2", 1047 | "quote", 1048 | "unicode-ident", 1049 | ] 1050 | 1051 | [[package]] 1052 | name = "tempfile" 1053 | version = "3.5.0" 1054 | source = "registry+https://github.com/rust-lang/crates.io-index" 1055 | checksum = "b9fbec84f381d5795b08656e4912bec604d162bff9291d6189a78f4c8ab87998" 1056 | dependencies = [ 1057 | "cfg-if", 1058 | "fastrand", 1059 | "redox_syscall", 1060 | "rustix", 1061 | "windows-sys 0.45.0", 1062 | ] 1063 | 1064 | [[package]] 1065 | name = "termcolor" 1066 | version = "1.1.3" 1067 | source = "registry+https://github.com/rust-lang/crates.io-index" 1068 | checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755" 1069 | dependencies = [ 1070 | "winapi-util", 1071 | ] 1072 | 1073 | [[package]] 1074 | name = "time" 1075 | version = "0.3.20" 1076 | source = "registry+https://github.com/rust-lang/crates.io-index" 1077 | checksum = "cd0cbfecb4d19b5ea75bb31ad904eb5b9fa13f21079c3b92017ebdf4999a5890" 1078 | dependencies = [ 1079 | "itoa", 1080 | "libc", 1081 | "num_threads", 1082 | "serde", 1083 | "time-core", 1084 | "time-macros", 1085 | ] 1086 | 1087 | [[package]] 1088 | name = "time-core" 1089 | version = "0.1.0" 1090 | source = "registry+https://github.com/rust-lang/crates.io-index" 1091 | checksum = "2e153e1f1acaef8acc537e68b44906d2db6436e2b35ac2c6b42640fff91f00fd" 1092 | 1093 | [[package]] 1094 | name = "time-macros" 1095 | version = "0.2.8" 1096 | source = "registry+https://github.com/rust-lang/crates.io-index" 1097 | checksum = "fd80a657e71da814b8e5d60d3374fc6d35045062245d80224748ae522dd76f36" 1098 | dependencies = [ 1099 | "time-core", 1100 | ] 1101 | 1102 | [[package]] 1103 | name = "tinyvec" 1104 | version = "1.6.0" 1105 | source = "registry+https://github.com/rust-lang/crates.io-index" 1106 | checksum = "87cc5ceb3875bb20c2890005a4e226a4651264a5c75edb2421b52861a0a0cb50" 1107 | dependencies = [ 1108 | "tinyvec_macros", 1109 | ] 1110 | 1111 | [[package]] 1112 | name = "tinyvec_macros" 1113 | version = "0.1.1" 1114 | source = "registry+https://github.com/rust-lang/crates.io-index" 1115 | checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" 1116 | 1117 | [[package]] 1118 | name = "tokio" 1119 | version = "1.28.0" 1120 | source = "registry+https://github.com/rust-lang/crates.io-index" 1121 | checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" 1122 | dependencies = [ 1123 | "autocfg", 1124 | "bytes", 1125 | "libc", 1126 | "mio", 1127 | "num_cpus", 1128 | "pin-project-lite", 1129 | "socket2", 1130 | "windows-sys 0.48.0", 1131 | ] 1132 | 1133 | [[package]] 1134 | name = "tokio-native-tls" 1135 | version = "0.3.1" 1136 | source = "registry+https://github.com/rust-lang/crates.io-index" 1137 | checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" 1138 | dependencies = [ 1139 | "native-tls", 1140 | "tokio", 1141 | ] 1142 | 1143 | [[package]] 1144 | name = "tokio-rustls" 1145 | version = "0.23.4" 1146 | source = "registry+https://github.com/rust-lang/crates.io-index" 1147 | checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" 1148 | dependencies = [ 1149 | "rustls 0.20.8", 1150 | "tokio", 1151 | "webpki", 1152 | ] 1153 | 1154 | [[package]] 1155 | name = "tokio-util" 1156 | version = "0.7.8" 1157 | source = "registry+https://github.com/rust-lang/crates.io-index" 1158 | checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" 1159 | dependencies = [ 1160 | "bytes", 1161 | "futures-core", 1162 | "futures-sink", 1163 | "pin-project-lite", 1164 | "tokio", 1165 | "tracing", 1166 | ] 1167 | 1168 | [[package]] 1169 | name = "tower-service" 1170 | version = "0.3.2" 1171 | source = "registry+https://github.com/rust-lang/crates.io-index" 1172 | checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" 1173 | 1174 | [[package]] 1175 | name = "tracing" 1176 | version = "0.1.37" 1177 | source = "registry+https://github.com/rust-lang/crates.io-index" 1178 | checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" 1179 | dependencies = [ 1180 | "cfg-if", 1181 | "pin-project-lite", 1182 | "tracing-core", 1183 | ] 1184 | 1185 | [[package]] 1186 | name = "tracing-core" 1187 | version = "0.1.30" 1188 | source = "registry+https://github.com/rust-lang/crates.io-index" 1189 | checksum = "24eb03ba0eab1fd845050058ce5e616558e8f8d8fca633e6b163fe25c797213a" 1190 | dependencies = [ 1191 | "once_cell", 1192 | ] 1193 | 1194 | [[package]] 1195 | name = "try-lock" 1196 | version = "0.2.4" 1197 | source = "registry+https://github.com/rust-lang/crates.io-index" 1198 | checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" 1199 | 1200 | [[package]] 1201 | name = "unicode-bidi" 1202 | version = "0.3.13" 1203 | source = "registry+https://github.com/rust-lang/crates.io-index" 1204 | checksum = "92888ba5573ff080736b3648696b70cafad7d250551175acbaa4e0385b3e1460" 1205 | 1206 | [[package]] 1207 | name = "unicode-ident" 1208 | version = "1.0.8" 1209 | source = "registry+https://github.com/rust-lang/crates.io-index" 1210 | checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4" 1211 | 1212 | [[package]] 1213 | name = "unicode-normalization" 1214 | version = "0.1.22" 1215 | source = "registry+https://github.com/rust-lang/crates.io-index" 1216 | checksum = "5c5713f0fc4b5db668a2ac63cdb7bb4469d8c9fed047b1d0292cc7b0ce2ba921" 1217 | dependencies = [ 1218 | "tinyvec", 1219 | ] 1220 | 1221 | [[package]] 1222 | name = "unicode-width" 1223 | version = "0.1.10" 1224 | source = "registry+https://github.com/rust-lang/crates.io-index" 1225 | checksum = "c0edd1e5b14653f783770bce4a4dabb4a5108a5370a5f5d8cfe8710c361f6c8b" 1226 | 1227 | [[package]] 1228 | name = "untrusted" 1229 | version = "0.7.1" 1230 | source = "registry+https://github.com/rust-lang/crates.io-index" 1231 | checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" 1232 | 1233 | [[package]] 1234 | name = "url" 1235 | version = "2.3.1" 1236 | source = "registry+https://github.com/rust-lang/crates.io-index" 1237 | checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643" 1238 | dependencies = [ 1239 | "form_urlencoded", 1240 | "idna", 1241 | "percent-encoding", 1242 | ] 1243 | 1244 | [[package]] 1245 | name = "vcpkg" 1246 | version = "0.2.15" 1247 | source = "registry+https://github.com/rust-lang/crates.io-index" 1248 | checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" 1249 | 1250 | [[package]] 1251 | name = "want" 1252 | version = "0.3.0" 1253 | source = "registry+https://github.com/rust-lang/crates.io-index" 1254 | checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" 1255 | dependencies = [ 1256 | "log", 1257 | "try-lock", 1258 | ] 1259 | 1260 | [[package]] 1261 | name = "wasi" 1262 | version = "0.11.0+wasi-snapshot-preview1" 1263 | source = "registry+https://github.com/rust-lang/crates.io-index" 1264 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 1265 | 1266 | [[package]] 1267 | name = "wasm-bindgen" 1268 | version = "0.2.84" 1269 | source = "registry+https://github.com/rust-lang/crates.io-index" 1270 | checksum = "31f8dcbc21f30d9b8f2ea926ecb58f6b91192c17e9d33594b3df58b2007ca53b" 1271 | dependencies = [ 1272 | "cfg-if", 1273 | "wasm-bindgen-macro", 1274 | ] 1275 | 1276 | [[package]] 1277 | name = "wasm-bindgen-backend" 1278 | version = "0.2.84" 1279 | source = "registry+https://github.com/rust-lang/crates.io-index" 1280 | checksum = "95ce90fd5bcc06af55a641a86428ee4229e44e07033963a2290a8e241607ccb9" 1281 | dependencies = [ 1282 | "bumpalo", 1283 | "log", 1284 | "once_cell", 1285 | "proc-macro2", 1286 | "quote", 1287 | "syn 1.0.109", 1288 | "wasm-bindgen-shared", 1289 | ] 1290 | 1291 | [[package]] 1292 | name = "wasm-bindgen-futures" 1293 | version = "0.4.34" 1294 | source = "registry+https://github.com/rust-lang/crates.io-index" 1295 | checksum = "f219e0d211ba40266969f6dbdd90636da12f75bee4fc9d6c23d1260dadb51454" 1296 | dependencies = [ 1297 | "cfg-if", 1298 | "js-sys", 1299 | "wasm-bindgen", 1300 | "web-sys", 1301 | ] 1302 | 1303 | [[package]] 1304 | name = "wasm-bindgen-macro" 1305 | version = "0.2.84" 1306 | source = "registry+https://github.com/rust-lang/crates.io-index" 1307 | checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" 1308 | dependencies = [ 1309 | "quote", 1310 | "wasm-bindgen-macro-support", 1311 | ] 1312 | 1313 | [[package]] 1314 | name = "wasm-bindgen-macro-support" 1315 | version = "0.2.84" 1316 | source = "registry+https://github.com/rust-lang/crates.io-index" 1317 | checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" 1318 | dependencies = [ 1319 | "proc-macro2", 1320 | "quote", 1321 | "syn 1.0.109", 1322 | "wasm-bindgen-backend", 1323 | "wasm-bindgen-shared", 1324 | ] 1325 | 1326 | [[package]] 1327 | name = "wasm-bindgen-shared" 1328 | version = "0.2.84" 1329 | source = "registry+https://github.com/rust-lang/crates.io-index" 1330 | checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" 1331 | 1332 | [[package]] 1333 | name = "web-sys" 1334 | version = "0.3.61" 1335 | source = "registry+https://github.com/rust-lang/crates.io-index" 1336 | checksum = "e33b99f4b23ba3eec1a53ac264e35a755f00e966e0065077d6027c0f575b0b97" 1337 | dependencies = [ 1338 | "js-sys", 1339 | "wasm-bindgen", 1340 | ] 1341 | 1342 | [[package]] 1343 | name = "webpki" 1344 | version = "0.22.0" 1345 | source = "registry+https://github.com/rust-lang/crates.io-index" 1346 | checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" 1347 | dependencies = [ 1348 | "ring", 1349 | "untrusted", 1350 | ] 1351 | 1352 | [[package]] 1353 | name = "webpki-roots" 1354 | version = "0.22.6" 1355 | source = "registry+https://github.com/rust-lang/crates.io-index" 1356 | checksum = "b6c71e40d7d2c34a5106301fb632274ca37242cd0c9d3e64dbece371a40a2d87" 1357 | dependencies = [ 1358 | "webpki", 1359 | ] 1360 | 1361 | [[package]] 1362 | name = "winapi" 1363 | version = "0.3.9" 1364 | source = "registry+https://github.com/rust-lang/crates.io-index" 1365 | checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" 1366 | dependencies = [ 1367 | "winapi-i686-pc-windows-gnu", 1368 | "winapi-x86_64-pc-windows-gnu", 1369 | ] 1370 | 1371 | [[package]] 1372 | name = "winapi-i686-pc-windows-gnu" 1373 | version = "0.4.0" 1374 | source = "registry+https://github.com/rust-lang/crates.io-index" 1375 | checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" 1376 | 1377 | [[package]] 1378 | name = "winapi-util" 1379 | version = "0.1.5" 1380 | source = "registry+https://github.com/rust-lang/crates.io-index" 1381 | checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" 1382 | dependencies = [ 1383 | "winapi", 1384 | ] 1385 | 1386 | [[package]] 1387 | name = "winapi-x86_64-pc-windows-gnu" 1388 | version = "0.4.0" 1389 | source = "registry+https://github.com/rust-lang/crates.io-index" 1390 | checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" 1391 | 1392 | [[package]] 1393 | name = "windows-sys" 1394 | version = "0.42.0" 1395 | source = "registry+https://github.com/rust-lang/crates.io-index" 1396 | checksum = "5a3e1820f08b8513f676f7ab6c1f99ff312fb97b553d30ff4dd86f9f15728aa7" 1397 | dependencies = [ 1398 | "windows_aarch64_gnullvm 0.42.2", 1399 | "windows_aarch64_msvc 0.42.2", 1400 | "windows_i686_gnu 0.42.2", 1401 | "windows_i686_msvc 0.42.2", 1402 | "windows_x86_64_gnu 0.42.2", 1403 | "windows_x86_64_gnullvm 0.42.2", 1404 | "windows_x86_64_msvc 0.42.2", 1405 | ] 1406 | 1407 | [[package]] 1408 | name = "windows-sys" 1409 | version = "0.45.0" 1410 | source = "registry+https://github.com/rust-lang/crates.io-index" 1411 | checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" 1412 | dependencies = [ 1413 | "windows-targets 0.42.2", 1414 | ] 1415 | 1416 | [[package]] 1417 | name = "windows-sys" 1418 | version = "0.48.0" 1419 | source = "registry+https://github.com/rust-lang/crates.io-index" 1420 | checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" 1421 | dependencies = [ 1422 | "windows-targets 0.48.0", 1423 | ] 1424 | 1425 | [[package]] 1426 | name = "windows-targets" 1427 | version = "0.42.2" 1428 | source = "registry+https://github.com/rust-lang/crates.io-index" 1429 | checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" 1430 | dependencies = [ 1431 | "windows_aarch64_gnullvm 0.42.2", 1432 | "windows_aarch64_msvc 0.42.2", 1433 | "windows_i686_gnu 0.42.2", 1434 | "windows_i686_msvc 0.42.2", 1435 | "windows_x86_64_gnu 0.42.2", 1436 | "windows_x86_64_gnullvm 0.42.2", 1437 | "windows_x86_64_msvc 0.42.2", 1438 | ] 1439 | 1440 | [[package]] 1441 | name = "windows-targets" 1442 | version = "0.48.0" 1443 | source = "registry+https://github.com/rust-lang/crates.io-index" 1444 | checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" 1445 | dependencies = [ 1446 | "windows_aarch64_gnullvm 0.48.0", 1447 | "windows_aarch64_msvc 0.48.0", 1448 | "windows_i686_gnu 0.48.0", 1449 | "windows_i686_msvc 0.48.0", 1450 | "windows_x86_64_gnu 0.48.0", 1451 | "windows_x86_64_gnullvm 0.48.0", 1452 | "windows_x86_64_msvc 0.48.0", 1453 | ] 1454 | 1455 | [[package]] 1456 | name = "windows_aarch64_gnullvm" 1457 | version = "0.42.2" 1458 | source = "registry+https://github.com/rust-lang/crates.io-index" 1459 | checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" 1460 | 1461 | [[package]] 1462 | name = "windows_aarch64_gnullvm" 1463 | version = "0.48.0" 1464 | source = "registry+https://github.com/rust-lang/crates.io-index" 1465 | checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" 1466 | 1467 | [[package]] 1468 | name = "windows_aarch64_msvc" 1469 | version = "0.42.2" 1470 | source = "registry+https://github.com/rust-lang/crates.io-index" 1471 | checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" 1472 | 1473 | [[package]] 1474 | name = "windows_aarch64_msvc" 1475 | version = "0.48.0" 1476 | source = "registry+https://github.com/rust-lang/crates.io-index" 1477 | checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" 1478 | 1479 | [[package]] 1480 | name = "windows_i686_gnu" 1481 | version = "0.42.2" 1482 | source = "registry+https://github.com/rust-lang/crates.io-index" 1483 | checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" 1484 | 1485 | [[package]] 1486 | name = "windows_i686_gnu" 1487 | version = "0.48.0" 1488 | source = "registry+https://github.com/rust-lang/crates.io-index" 1489 | checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" 1490 | 1491 | [[package]] 1492 | name = "windows_i686_msvc" 1493 | version = "0.42.2" 1494 | source = "registry+https://github.com/rust-lang/crates.io-index" 1495 | checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" 1496 | 1497 | [[package]] 1498 | name = "windows_i686_msvc" 1499 | version = "0.48.0" 1500 | source = "registry+https://github.com/rust-lang/crates.io-index" 1501 | checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" 1502 | 1503 | [[package]] 1504 | name = "windows_x86_64_gnu" 1505 | version = "0.42.2" 1506 | source = "registry+https://github.com/rust-lang/crates.io-index" 1507 | checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" 1508 | 1509 | [[package]] 1510 | name = "windows_x86_64_gnu" 1511 | version = "0.48.0" 1512 | source = "registry+https://github.com/rust-lang/crates.io-index" 1513 | checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" 1514 | 1515 | [[package]] 1516 | name = "windows_x86_64_gnullvm" 1517 | version = "0.42.2" 1518 | source = "registry+https://github.com/rust-lang/crates.io-index" 1519 | checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" 1520 | 1521 | [[package]] 1522 | name = "windows_x86_64_gnullvm" 1523 | version = "0.48.0" 1524 | source = "registry+https://github.com/rust-lang/crates.io-index" 1525 | checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" 1526 | 1527 | [[package]] 1528 | name = "windows_x86_64_msvc" 1529 | version = "0.42.2" 1530 | source = "registry+https://github.com/rust-lang/crates.io-index" 1531 | checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" 1532 | 1533 | [[package]] 1534 | name = "windows_x86_64_msvc" 1535 | version = "0.48.0" 1536 | source = "registry+https://github.com/rust-lang/crates.io-index" 1537 | checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" 1538 | 1539 | [[package]] 1540 | name = "winreg" 1541 | version = "0.10.1" 1542 | source = "registry+https://github.com/rust-lang/crates.io-index" 1543 | checksum = "80d0f4e272c85def139476380b12f9ac60926689dd2e01d4923222f40580869d" 1544 | dependencies = [ 1545 | "winapi", 1546 | ] 1547 | 1548 | [[package]] 1549 | name = "zeroize" 1550 | version = "1.6.0" 1551 | source = "registry+https://github.com/rust-lang/crates.io-index" 1552 | checksum = "2a0956f1ba7c7909bfb66c2e9e4124ab6f6482560f6628b5aaeba39207c9aad9" 1553 | --------------------------------------------------------------------------------