├── py.typed ├── version.txt ├── SECURITY.md ├── src ├── utils │ ├── mod.rs │ ├── time.rs │ └── py_to_json.rs ├── network │ ├── mod.rs │ ├── ssl_verify.rs │ ├── proxy_config.rs │ └── http_version.rs ├── request │ ├── mod.rs │ ├── config.rs │ ├── request_item.rs │ ├── concurrency.rs │ └── executor.rs ├── debug.rs └── lib.rs ├── docs └── images │ └── timeout_performance_comparison.png ├── sync_version.py ├── .github ├── ISSUE_TEMPLATE │ ├── feature_request.md │ └── bug_report.md └── workflows │ ├── performance_benchmark.yml │ ├── cross-platform-test.yml │ └── build.yml ├── Cargo.toml ├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── pyproject.toml ├── rusty_req.pyi ├── CONTRIBUTING.md ├── CODE_OF_CONDUCT.md ├── tests └── performance_test.py ├── README.zh.md └── README.md /py.typed: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /version.txt: -------------------------------------------------------------------------------- 1 | 0.4.25 -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | ... 6 | -------------------------------------------------------------------------------- /src/utils/mod.rs: -------------------------------------------------------------------------------- 1 | pub mod py_to_json; 2 | pub mod time; 3 | 4 | pub use py_to_json::py_to_json; 5 | pub use time::format_datetime; 6 | -------------------------------------------------------------------------------- /docs/images/timeout_performance_comparison.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/KAY53N/rusty-req/HEAD/docs/images/timeout_performance_comparison.png -------------------------------------------------------------------------------- /src/utils/time.rs: -------------------------------------------------------------------------------- 1 | use std::time::SystemTime; 2 | use chrono::{DateTime, Local}; 3 | 4 | pub fn format_datetime(time: SystemTime) -> String { 5 | let datetime: DateTime = time.into(); 6 | datetime.format("%Y-%m-%d %H:%M:%S").to_string() 7 | } 8 | -------------------------------------------------------------------------------- /src/network/mod.rs: -------------------------------------------------------------------------------- 1 | // src/network/mod.rs 2 | pub mod http_version; 3 | pub mod proxy_config; 4 | pub mod ssl_verify; // 新增 5 | 6 | // 重新导出，方便外部使用 7 | pub use http_version::HttpVersion; 8 | pub use proxy_config::ProxyConfig; 9 | pub use ssl_verify::SslVerify; // 新增导出 -------------------------------------------------------------------------------- /src/network/ssl_verify.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{pyclass, pymethods}; 2 | 3 | #[pyclass] 4 | #[derive(Clone, Debug)] 5 | pub struct SslVerify(bool); 6 | 7 | #[pymethods] 8 | impl SslVerify { 9 | #[new] 10 | fn new(verify: bool) -> Self { 11 | SslVerify(verify) 12 | } 13 | 14 | // 添加一个方法获取内部 bool 值 15 | pub fn get(&self) -> bool { 16 | self.0 17 | } 18 | } -------------------------------------------------------------------------------- /src/request/mod.rs: -------------------------------------------------------------------------------- 1 | // request/mod.rs 2 | 3 | pub mod request_item; 4 | pub mod executor; 5 | pub mod concurrency; 6 | pub mod config; 7 | 8 | // 重新导出，方便上层直接使用 9 | pub use request_item::RequestItem; 10 | pub use executor::{execute_single_request, fetch_single, fetch_requests}; 11 | pub use concurrency::{execute_with_select_all, execute_with_join_all}; 12 | pub use config::set_global_proxy; 13 | -------------------------------------------------------------------------------- /src/request/config.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{pyfunction, PyAny, PyResult, Python}; 2 | use crate::{ProxyConfig, GLOBAL_PROXY}; 3 | 4 | #[pyfunction] 5 | pub fn set_global_proxy<'py>(py: Python<'py>, proxy: ProxyConfig) -> PyResult<&'py PyAny> { 6 | pyo3_asyncio::tokio::future_into_py(py, async move { 7 | let mut global = GLOBAL_PROXY.lock().await; 8 | *global = Some(proxy); 9 | Ok(()) 10 | }) 11 | } 12 | -------------------------------------------------------------------------------- /sync_version.py: -------------------------------------------------------------------------------- 1 | import toml 2 | 3 | # 读取 version.txt 4 | with open("version.txt", "r") as f: 5 | version = f.read().strip() 6 | 7 | # 更新 Cargo.toml 中的 [package].version 8 | cargo = toml.load("Cargo.toml") 9 | cargo['package']['version'] = version 10 | with open("Cargo.toml", "w") as f: 11 | toml.dump(cargo, f) 12 | 13 | # 更新 pyproject.toml 中的 [project].version 14 | pyproject = toml.load("pyproject.toml") 15 | pyproject['project']['version'] = version # ✅ 改这里 16 | with open("pyproject.toml", "w") as f: 17 | toml.dump(pyproject, f) 18 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "[Feature] Title for your feature suggestion" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of the problem you're facing. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen and how it should work. 15 | 16 | **Proposed API/Interface** (Optional) 17 | If applicable, describe how you envision the new feature's API: 18 | ... 19 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "rusty-req" 3 | version = "0.4.25" 4 | edition = "2021" 5 | license = "MIT" 6 | 7 | [dependencies] 8 | serde_json = "1.0" 9 | chrono = "0.4" 10 | futures = "0.3" 11 | once_cell = "1.18" 12 | url = "2.5.4" 13 | rustc_version = "0.4.1" 14 | 15 | [dependencies.pyo3] 16 | version = "0.20" 17 | features = [ "extension-module", "abi3-py39",] 18 | 19 | [dependencies.pyo3-asyncio] 20 | version = "0.20" 21 | features = [ "tokio-runtime",] 22 | 23 | [dependencies.tokio] 24 | version = "1.0" 25 | features = [ "full",] 26 | 27 | [dependencies.reqwest] 28 | version = "0.11" 29 | features = [ "json", "brotli", "gzip", "deflate", "stream", "native-tls",] 30 | 31 | [package.metadata.maturin] 32 | supported-python-versions = [ "3.9", "3.10", "3.11", "3.12", "3.13",] 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Generated by Cargo 2 | # will have compiled files and executables 3 | debug 4 | target 5 | 6 | # These are backup files generated by rustfmt 7 | **/*.rs.bk 8 | 9 | # MSVC Windows builds of rustc generate these, which store debugging information 10 | *.pdb 11 | 12 | # Generated by cargo mutants 13 | # Contains mutation testing data 14 | **/mutants.out*/ 15 | 16 | # RustRover 17 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 18 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 19 | # and can be added to the global gitignore or merged into this file. For a more nuclear 20 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 21 | .idea 22 | tests/.idea 23 | Cargo.lock -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "[Bug] Title explaining the bug" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Bug Description** 11 | A clear and concise description of the bug. 12 | 13 | **Steps to Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. 16 | 2. 17 | 3. 18 | 19 | **Expected Behavior** 20 | A clear description of what you expected to happen. 21 | 22 | **Actual Behavior** 23 | A description of what actually happened. 24 | 25 | **Environment Information** 26 | - OS: [e.g. Windows 11, Ubuntu 22.04, macOS Ventura] 27 | - Python Version: [e.g. Python 3.9, 3.10, 3.11] 28 | - Rusty-req Version: [e.g. 0.1.0] 29 | - Dependency Versions: [e.g. requests, aiohttp, etc.] 30 | 31 | **Code Example** 32 | ```python 33 | import rusty_req 34 | 35 | # Minimal code example to reproduce the issue 36 | client = rusty_req.Client() 37 | response = client.get("https://api.example.com/data") 38 | print(response.text) 39 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [0.4.25] - 2025.09.25 4 | ### Added 5 | - 完善pyproject 6 | - 更新README 7 | 8 | ## [0.4.21] - 2025.09.23 9 | ### Added 10 | - 添加Python类型存根文件 11 | 12 | ## [0.4.20] - 2025.09.22 13 | ### Added 14 | - ProxyConfig增加trust_env参数，指定是否忽略环境变量 15 | ### Fixed 16 | - Windows安装不成功问题修复 17 | 18 | ## [0.3.85] - 2025.09.15 19 | ### Added 20 | - 指定是否对SSL证书验证 21 | 22 | ## [0.3.83] - 2025.09.11 23 | ### Added 24 | - 增加指定请求的http协议版本 25 | - 新增http_version参数支持，提供完整的HTTP协议版本控制，包括AUTO/HTTP1_ONLY/HTTP2/HTTP2_PRIOR_KNOWLEDGE选项 26 | ### Changed 27 | - 代码结构拆分 28 | ### Fixed 29 | - 代理设置的用户名和密码支持 30 | 31 | ## [0.3.65] - 2025-09-05 32 | ### Added 33 | - 增加代理设置 34 | ### Optimized 35 | - set_debug 方法增加日志记录和控制台输出两种方式 36 | ### Changed 37 | - 中文README 38 | 39 | ## [0.3.2] - 2025-08-11 40 | ### Changed 41 | - 这个版本扩展了对更多 Python 版本的支持，确保不同系统和环境的兼容性。 42 | 43 | 44 | ## [0.3.0] - 2025-08-08 45 | ### Optimized 46 | - fetch_requests方法增加ConcurrencyMode参数 47 | 48 | ## [0.2.8] - 2025-08-08 49 | ### Added 50 | - 增加fetch_single方法 -------------------------------------------------------------------------------- /src/utils/py_to_json.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use pyo3::types::{PyDict, PyList}; 3 | use serde_json::{Value}; 4 | 5 | pub fn py_to_json(py: Python, obj: &PyAny) -> PyResult { 6 | if let Ok(b) = obj.extract::() { return Ok(Value::Bool(b)); } 7 | if let Ok(s) = obj.extract::() { return Ok(Value::String(s)); } 8 | if let Ok(i) = obj.extract::() { return Ok(Value::Number(i.into())); } 9 | if let Ok(f) = obj.extract::() { return Ok(Value::Number(serde_json::Number::from_f64(f).unwrap_or(0.into()))); } 10 | if let Ok(list) = obj.downcast::() { 11 | let mut vec = Vec::new(); 12 | for i in list.iter() { vec.push(py_to_json(py, i)?); } 13 | return Ok(Value::Array(vec)); 14 | } 15 | if let Ok(dict) = obj.downcast::() { 16 | let mut map = serde_json::Map::new(); 17 | for (k,v) in dict.iter() { map.insert(k.to_string(), py_to_json(py, v)?); } 18 | return Ok(Value::Object(map)); 19 | } 20 | Ok(Value::String(obj.to_string())) 21 | } 22 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2025 KAY53N 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /.github/workflows/performance_benchmark.yml: -------------------------------------------------------------------------------- 1 | name: Performance Benchmark 2 | 3 | on: 4 | push: 5 | branches: [ main, develop ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | jobs: 10 | performance-test: 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - uses: actions/checkout@v4 15 | 16 | - name: Start httpbin Docker service 17 | run: | 18 | docker run -d --name httpbin -p 8080:80 kennethreitz/httpbin:latest 19 | # Wait for service to start 20 | sleep 10 21 | # Verify service is running 22 | curl -f http://localhost:8080/status/200 23 | 24 | - name: Set up Python environment 25 | uses: actions/setup-python@v4 26 | with: 27 | python-version: '3.9' 28 | 29 | - name: Install dependencies 30 | run: | 31 | python -m pip install --upgrade pip 32 | pip install rusty-req aiohttp httpx requests psutil 33 | 34 | - name: Run performance tests 35 | run: python tests/performance_test.py 36 | env: 37 | HTTPBIN_URL: http://localhost:8080 38 | 39 | - name: Stop httpbin service 40 | run: | 41 | docker stop httpbin 42 | docker rm httpbin 43 | 44 | -------------------------------------------------------------------------------- /src/request/request_item.rs: -------------------------------------------------------------------------------- 1 | use pyo3::prelude::*; 2 | use pyo3::types::PyDict; 3 | use crate::network::{HttpVersion, ProxyConfig, SslVerify}; 4 | 5 | #[pyclass] 6 | #[derive(Clone)] 7 | pub struct RequestItem { 8 | #[pyo3(get, set)] 9 | pub url: String, 10 | #[pyo3(get, set)] 11 | pub method: Option, 12 | #[pyo3(get, set)] 13 | pub params: Option>, 14 | #[pyo3(get, set)] 15 | pub timeout: Option, 16 | #[pyo3(get, set)] 17 | pub tag: Option, 18 | #[pyo3(get, set)] 19 | pub headers: Option>, 20 | #[pyo3(get, set)] 21 | pub proxy: Option, 22 | #[pyo3(get, set)] 23 | pub http_version: Option, 24 | #[pyo3(get, set)] 25 | pub ssl_verify: Option, 26 | } 27 | 28 | #[pymethods] 29 | impl RequestItem { 30 | #[new] 31 | fn new( 32 | url: String, 33 | method: Option, 34 | params: Option>, 35 | timeout: Option, 36 | tag: Option, 37 | headers: Option>, 38 | proxy: Option, 39 | http_version: Option, 40 | ssl_verify: Option, 41 | ) -> Self { 42 | Self { url, method, params, timeout, tag, headers, proxy, http_version, ssl_verify } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = [ "maturin>=1.9.2",] 3 | build-backend = "maturin" 4 | 5 | [project] 6 | name = "rusty-req" 7 | version = "0.4.25" 8 | description = "⚡ High-performance async HTTP client in Rust with Python bindings for blazing-fast batch requests." 9 | readme = "README.md" 10 | requires-python = ">=3.9" 11 | classifiers = [ "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Rust", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: MacOS :: MacOS X", "Topic :: Internet :: WWW/HTTP", "Topic :: Software Development :: Libraries :: Python Modules", "Topic :: System :: Networking", "Framework :: AsyncIO", "Intended Audience :: Developers", "License :: OSI Approved :: MIT License",] 12 | [[project.authors]] 13 | name = "KAY53N" 14 | email = "kaysen820@gmail.com" 15 | 16 | [project.license] 17 | text = "MIT" 18 | 19 | [project.urls] 20 | Homepage = "https://github.com/KAY53N/rusty-req" 21 | Repository = "https://github.com/KAY53N/rusty-req" 22 | "Bug Tracker" = "https://github.com/KAY53N/rusty-req/issues" 23 | Documentation = "https://github.com/KAY53N/rusty-req#readme" 24 | Changelog = "https://github.com/KAY53N/rusty-req/blob/main/CHANGELOG.md" 25 | 26 | [tool.maturin] 27 | bindings = "pyo3" 28 | module-name = "rusty_req" 29 | include = [ "rusty_req.pyi", "README.md", "README.zh.md", "CHANGELOG.md",] 30 | -------------------------------------------------------------------------------- /src/debug.rs: -------------------------------------------------------------------------------- 1 | use std::fs::OpenOptions; 2 | use std::io::Write; 3 | use std::path::Path; 4 | use std::sync::RwLock; 5 | use once_cell::sync::Lazy; 6 | use pyo3::pyfunction; 7 | use reqwest::StatusCode; 8 | use serde_json::Value; 9 | 10 | #[derive(Clone)] 11 | enum DebugTarget { 12 | Console, 13 | File(String), 14 | } 15 | 16 | #[derive(Clone)] 17 | struct DebugConfig { 18 | enabled: bool, 19 | target: DebugTarget, 20 | } 21 | 22 | static DEBUG_CONFIG: Lazy> = Lazy::new(|| { 23 | RwLock::new(DebugConfig { enabled: false, target: DebugTarget::Console }) 24 | }); 25 | 26 | #[pyfunction] 27 | pub fn set_debug(enabled: bool, target: Option) { 28 | let mut cfg = DEBUG_CONFIG.write().unwrap(); 29 | cfg.enabled = enabled; 30 | cfg.target = match target { 31 | Some(t) if t.to_lowercase() == "console" || t.is_empty() => DebugTarget::Console, 32 | Some(t) => { 33 | let path = Path::new(&t); 34 | if path.is_dir() { DebugTarget::File(path.join("debug.log").to_string_lossy().to_string()) } 35 | else { DebugTarget::File(t) } 36 | }, 37 | None => DebugTarget::Console, 38 | }; 39 | } 40 | 41 | pub fn debug_log( 42 | method: &str, 43 | tag: &str, 44 | url: &str, 45 | status: StatusCode, 46 | headers: &serde_json::Map, 47 | response: &Value, 48 | proxy: Option<&str>, 49 | proxy_auth: Option<&str>, 50 | ) { 51 | if !DEBUG_CONFIG.read().unwrap().enabled { return; } 52 | 53 | let mut msg = format!("\n==== [{}] ====\nMethod: {}\nURL: {}\nStatus: {}\n", tag, method, url, status); 54 | msg.push_str(&format!("Headers: {:?}\nResponse: {}\n", headers, response)); 55 | if let Some(p) = proxy { msg.push_str(&format!("Proxy: {}\n", p)); } 56 | if let Some(auth) = proxy_auth { msg.push_str(&format!("Proxy Auth: {}\n", auth)); } 57 | 58 | match &DEBUG_CONFIG.read().unwrap().target { 59 | DebugTarget::Console => println!("{}", msg), 60 | DebugTarget::File(path) => { let _ = OpenOptions::new().create(true).append(true).open(path).map(|mut f| writeln!(f, "{}", msg)); } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/lib.rs: -------------------------------------------------------------------------------- 1 | #![allow(non_local_definitions)] 2 | 3 | mod network; 4 | mod request; 5 | mod debug; 6 | mod utils; 7 | 8 | use std::process::Command; 9 | use pyo3::prelude::*; 10 | use once_cell::sync::Lazy; 11 | use tokio::sync::Mutex; 12 | use reqwest::Client; 13 | pub use network::{HttpVersion, ProxyConfig}; 14 | pub use request::{RequestItem, fetch_single, fetch_requests, set_global_proxy}; 15 | pub use crate::debug::set_debug; 16 | pub use request::concurrency::ConcurrencyMode; 17 | use crate::network::SslVerify; 18 | 19 | pub static DEFAULT_USER_AGENT: Lazy = Lazy::new(|| { 20 | let rust_version = Command::new("rustc") 21 | .arg("--version") 22 | .output() 23 | .ok() 24 | .and_then(|output| String::from_utf8(output.stdout).ok()) 25 | .map(|v| v.trim().to_string()) 26 | .unwrap_or_else(|| "unknown".to_string()); 27 | 28 | format!("Rust/{} rusty-req/{}", rust_version, env!("CARGO_PKG_VERSION")) 29 | }); 30 | 31 | pub static GLOBAL_CLIENT: Lazy> = Lazy::new(|| { 32 | Mutex::new( 33 | Client::builder() 34 | .timeout(std::time::Duration::from_secs(30)) 35 | .gzip(true) 36 | .brotli(true) 37 | .deflate(true) 38 | .user_agent(&*DEFAULT_USER_AGENT) // 复用静态变量 39 | .build() 40 | .expect("Failed to create HTTP client"), 41 | ) 42 | }); 43 | 44 | // 移除单独的 DEFAULT_USER_AGENT 定义 45 | pub static GLOBAL_PROXY: Lazy>> = Lazy::new(|| Mutex::new(None)); 46 | 47 | #[pymodule] 48 | fn rusty_req(_py: Python, m: &PyModule) -> PyResult<()> { 49 | // 添加版本信息 50 | m.add("__version__", env!("CARGO_PKG_VERSION"))?; 51 | 52 | // 暴露类 53 | m.add_class::()?; 54 | m.add_class::()?; 55 | m.add_class::()?; 56 | m.add_class::()?; 57 | m.add_class::()?; 58 | 59 | // 暴露函数 60 | use pyo3::wrap_pyfunction; 61 | m.add_function(wrap_pyfunction!(set_debug, m)?)?; 62 | m.add_function(wrap_pyfunction!(fetch_single, m)?)?; 63 | m.add_function(wrap_pyfunction!(fetch_requests, m)?)?; 64 | m.add_function(wrap_pyfunction!(set_global_proxy, m)?)?; 65 | 66 | Ok(()) 67 | } 68 | -------------------------------------------------------------------------------- /src/network/proxy_config.rs: -------------------------------------------------------------------------------- 1 | use pyo3::{pyclass, pymethods}; 2 | 3 | #[pyclass] 4 | #[derive(Clone)] 5 | pub struct ProxyConfig { 6 | #[pyo3(get, set)] 7 | pub http: Option, 8 | #[pyo3(get, set)] 9 | pub https: Option, 10 | #[pyo3(get, set)] 11 | pub all: Option, 12 | #[pyo3(get, set)] 13 | pub no_proxy: Option>, 14 | #[pyo3(get, set)] 15 | pub username: Option, 16 | #[pyo3(get, set)] 17 | pub password: Option, 18 | #[pyo3(get, set)] 19 | pub trust_env: Option, 20 | } 21 | 22 | #[pymethods] 23 | impl ProxyConfig { 24 | #[new] 25 | #[pyo3(signature = (http=None, https=None, all=None, no_proxy=None, username=None, password=None, trust_env=None))] 26 | fn new( 27 | http: Option, 28 | https: Option, 29 | all: Option, 30 | no_proxy: Option>, 31 | username: Option, 32 | password: Option, 33 | trust_env: Option, // 新增参数，默认为None 34 | ) -> Self { 35 | Self { 36 | http, 37 | https, 38 | all, 39 | no_proxy, 40 | username, 41 | password, 42 | trust_env, 43 | } 44 | } 45 | 46 | #[staticmethod] 47 | #[pyo3(signature = (proxy_url, username=None, password=None, trust_env=None))] 48 | fn from_url( 49 | proxy_url: String, 50 | username: Option, 51 | password: Option, 52 | trust_env: Option // 新增参数 53 | ) -> Self { 54 | Self { 55 | http: None, 56 | https: None, 57 | all: Some(proxy_url), 58 | no_proxy: None, 59 | username, 60 | password, 61 | trust_env, 62 | } 63 | } 64 | 65 | #[staticmethod] 66 | #[pyo3(signature = (http=None, https=None, username=None, password=None, trust_env=None))] 67 | fn from_dict( 68 | http: Option, 69 | https: Option, 70 | username: Option, 71 | password: Option, 72 | trust_env: Option // 新增参数 73 | ) -> Self { 74 | Self { 75 | http, 76 | https, 77 | all: None, 78 | no_proxy: None, 79 | username, 80 | password, 81 | trust_env, 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /src/request/concurrency.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::time::Duration; 3 | use crate::request::RequestItem; 4 | use crate::request::executor::execute_single_request; 5 | use futures::future::join_all; 6 | use pyo3::{pyclass, pymethods}; 7 | use reqwest::Client; 8 | 9 | #[pyclass] 10 | #[derive(Clone, PartialEq)] 11 | pub enum ConcurrencyMode { 12 | #[pyo3(name = "SELECT_ALL")] 13 | SelectAll, 14 | #[pyo3(name = "JOIN_ALL")] 15 | JoinAll, 16 | } 17 | 18 | #[pymethods] 19 | impl ConcurrencyMode { 20 | #[new] 21 | fn new() -> Self { 22 | ConcurrencyMode::SelectAll 23 | } 24 | 25 | #[classattr] 26 | const SELECT_ALL: ConcurrencyMode = ConcurrencyMode::SelectAll; 27 | 28 | #[classattr] 29 | const JOIN_ALL: ConcurrencyMode = ConcurrencyMode::JoinAll; 30 | 31 | fn __str__(&self) -> String { 32 | match self { 33 | ConcurrencyMode::SelectAll => "SELECT_ALL".to_string(), 34 | ConcurrencyMode::JoinAll => "JOIN_ALL".to_string(), 35 | } 36 | } 37 | 38 | fn __repr__(&self) -> String { 39 | format!("ConcurrencyMode.{}", self.__str__()) 40 | } 41 | } 42 | 43 | pub async fn execute_with_select_all( 44 | requests: Vec, 45 | total_duration: Duration, 46 | base_client: Option, 47 | ) -> Vec> { 48 | let futures = requests.into_iter().map(|req| { 49 | let client = base_client.clone(); 50 | async move { 51 | match tokio::time::timeout(total_duration, execute_single_request(req, client)).await { 52 | Ok(result) => result, 53 | Err(_) => { 54 | let mut timeout_result = HashMap::new(); 55 | timeout_result.insert("http_status".to_string(), "0".to_string()); 56 | timeout_result 57 | } 58 | } 59 | } 60 | }); 61 | 62 | join_all(futures).await 63 | } 64 | 65 | pub async fn execute_with_join_all( 66 | requests: Vec, 67 | total_duration: Duration, 68 | base_client: Option, 69 | ) -> Vec> { 70 | let mut results = Vec::with_capacity(requests.len()); 71 | 72 | for req in requests { 73 | match tokio::time::timeout(total_duration, execute_single_request(req, base_client.clone())).await { 74 | Ok(result) => results.push(result), 75 | Err(_) => { 76 | let mut timeout_result = HashMap::new(); 77 | timeout_result.insert("http_status".to_string(), "0".to_string()); 78 | results.push(timeout_result); 79 | } 80 | } 81 | } 82 | 83 | results 84 | } 85 | -------------------------------------------------------------------------------- /src/network/http_version.rs: -------------------------------------------------------------------------------- 1 | // src/http_version.rs 2 | use pyo3::{pyclass, pymethods, PyResult}; 3 | use pyo3::exceptions::PyValueError; 4 | use reqwest::ClientBuilder; 5 | 6 | #[pyclass] 7 | #[derive(Clone, PartialEq, Debug)] 8 | pub enum HttpVersion { 9 | #[pyo3(name = "AUTO")] 10 | Auto, // 自动协商（默认） 11 | #[pyo3(name = "HTTP1_ONLY")] 12 | Http1Only, // 仅使用 HTTP/1.1 13 | #[pyo3(name = "HTTP2")] 14 | Http2, // 优先尝试 HTTP/2，可回退到 HTTP/1.1 15 | #[pyo3(name = "HTTP2_PRIOR_KNOWLEDGE")] 16 | Http2PriorKnowledge, // 强制 HTTP/2（无回落） 17 | } 18 | 19 | #[pymethods] 20 | impl HttpVersion { 21 | #[new] 22 | fn new() -> Self { 23 | HttpVersion::Auto 24 | } 25 | 26 | // 类属性常量 27 | #[classattr] 28 | const AUTO: HttpVersion = HttpVersion::Auto; 29 | 30 | #[classattr] 31 | const HTTP1_ONLY: HttpVersion = HttpVersion::Http1Only; 32 | 33 | #[classattr] 34 | const HTTP2: HttpVersion = HttpVersion::Http2; 35 | 36 | #[classattr] 37 | const HTTP2_PRIOR_KNOWLEDGE: HttpVersion = HttpVersion::Http2PriorKnowledge; 38 | 39 | // 字符串表示 40 | fn __str__(&self) -> &'static str { 41 | match self { 42 | HttpVersion::Auto => "AUTO", 43 | HttpVersion::Http1Only => "HTTP1_ONLY", 44 | HttpVersion::Http2 => "HTTP2", 45 | HttpVersion::Http2PriorKnowledge => "HTTP2_PRIOR_KNOWLEDGE", 46 | } 47 | } 48 | 49 | fn __repr__(&self) -> String { 50 | format!("HttpVersion.{}", self.__str__()) 51 | } 52 | 53 | // 从字符串创建 54 | #[staticmethod] 55 | fn from_str(s: &str) -> PyResult { 56 | match s.to_uppercase().as_str() { 57 | "AUTO" | "" => Ok(HttpVersion::Auto), 58 | "HTTP1" | "HTTP1.1" | "HTTP1_ONLY" => Ok(HttpVersion::Http1Only), 59 | "HTTP2" => Ok(HttpVersion::Http2), 60 | "HTTP2_PRIOR_KNOWLEDGE" | "FORCE_HTTP2" | "HTTP2_ONLY" => Ok(HttpVersion::Http2PriorKnowledge), 61 | _ => Err(PyValueError::new_err( 62 | format!("Invalid HTTP version: '{}'. Valid values: AUTO, HTTP1_ONLY, HTTP2, HTTP2_PRIOR_KNOWLEDGE", s) 63 | )), 64 | } 65 | } 66 | 67 | // 获取描述信息 68 | fn description(&self) -> &'static str { 69 | match self { 70 | HttpVersion::Auto => "Automatically negotiate the best HTTP version", 71 | HttpVersion::Http1Only => "Use only HTTP/1.1 (no HTTP/2)", 72 | HttpVersion::Http2 => "Prefer HTTP/2, fallback to HTTP/1.1 if needed", 73 | HttpVersion::Http2PriorKnowledge => "Force HTTP/2 without fallback (server must support HTTP/2)", 74 | } 75 | } 76 | 77 | // 检查是否支持 HTTP/2 78 | fn supports_http2(&self) -> bool { 79 | match self { 80 | HttpVersion::Auto | HttpVersion::Http2 | HttpVersion::Http2PriorKnowledge => true, 81 | HttpVersion::Http1Only => false, 82 | } 83 | } 84 | 85 | // 检查是否强制 HTTP/2 86 | fn is_http2_forced(&self) -> bool { 87 | matches!(self, HttpVersion::Http2PriorKnowledge) 88 | } 89 | } 90 | 91 | impl HttpVersion { 92 | // 转换为 reqwest 配置（这个方法不需要 #[pymethods] 标记） 93 | pub(crate) fn apply_to_builder(&self, builder: ClientBuilder) -> ClientBuilder { 94 | match self { 95 | HttpVersion::Auto => builder, 96 | HttpVersion::Http1Only => builder.http1_only(), 97 | HttpVersion::Http2 => builder, 98 | HttpVersion::Http2PriorKnowledge => builder.http2_prior_knowledge(), 99 | } 100 | } 101 | } -------------------------------------------------------------------------------- /.github/workflows/cross-platform-test.yml: -------------------------------------------------------------------------------- 1 | name: Cross Platform Test 2 | 3 | on: 4 | workflow_dispatch: 5 | inputs: 6 | python-version: 7 | description: '要测试的 Python 版本' 8 | required: true 9 | default: '3.10' 10 | type: choice 11 | options: 12 | - '3.9' 13 | - '3.10' 14 | - '3.11' 15 | - '3.12' 16 | - '3.13' 17 | 18 | platform: 19 | description: '要测试的平台' 20 | required: true 21 | default: 'ubuntu-latest' 22 | type: choice 23 | options: 24 | - 'ubuntu-latest' 25 | - 'windows-latest' 26 | - 'macos-latest' 27 | - 'all-platforms' 28 | 29 | jobs: 30 | # 多平台测试 31 | test-all: 32 | if: ${{ github.event.inputs.platform == 'all-platforms' }} 33 | runs-on: ${{ matrix.os }} 34 | strategy: 35 | matrix: 36 | os: [ubuntu-latest, windows-latest, macos-latest] 37 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 38 | 39 | steps: 40 | - name: Checkout repository 41 | uses: actions/checkout@v4 42 | 43 | - name: Set up Python 44 | uses: actions/setup-python@v5 45 | with: 46 | python-version: ${{ matrix.python-version }} 47 | 48 | - name: Display platform info 49 | run: | 50 | echo "Running on: ${{ matrix.os }}" 51 | echo "Python version: ${{ matrix.python-version }}" 52 | python --version 53 | python -c "import platform; print(f'Platform: {platform.platform()}')" 54 | 55 | - name: Install rusty-req 56 | run: | 57 | pip install --upgrade pip 58 | pip install rusty-req --no-cache-dir 59 | 60 | - name: Verify installation 61 | run: | 62 | python -c " 63 | try: 64 | import rusty_req 65 | print('rusty-req imported successfully') 66 | if hasattr(rusty_req, '__version__'): 67 | print(f'Version: {rusty_req.__version__}') 68 | else: 69 | print('Module loaded but no version attribute') 70 | # 只显示不以双下划线开头的公共属性 71 | public_attrs = [attr for attr in dir(rusty_req) if not attr.startswith('_')] 72 | print(f'Public attributes: {public_attrs}') 73 | except ImportError as e: 74 | print(f'Import failed: {e}') 75 | exit(1) 76 | except Exception as e: 77 | print(f'Error: {e}') 78 | exit(1) 79 | " 80 | 81 | # 单平台测试 82 | test-single: 83 | if: ${{ github.event.inputs.platform != 'all-platforms' }} 84 | runs-on: ${{ github.event.inputs.platform }} 85 | 86 | steps: 87 | - name: Checkout repository 88 | uses: actions/checkout@v4 89 | 90 | - name: Set up Python 91 | uses: actions/setup-python@v5 92 | with: 93 | python-version: ${{ github.event.inputs.python-version }} 94 | 95 | - name: Display platform info 96 | run: | 97 | echo "Running on: ${{ github.event.inputs.platform }}" 98 | echo "Python version: ${{ github.event.inputs.python-version }}" 99 | python --version 100 | python -c "import platform; print(f'Platform: {platform.platform()}')" 101 | 102 | - name: Install rusty-req 103 | run: | 104 | pip install --upgrade pip 105 | pip install rusty-req --no-cache-dir 106 | 107 | - name: Verify installation 108 | run: | 109 | python -c " 110 | try: 111 | import rusty_req 112 | print('rusty-req imported successfully') 113 | if hasattr(rusty_req, '__version__'): 114 | print(f'Version: {rusty_req.__version__}') 115 | else: 116 | print('Module loaded but no version attribute') 117 | # 只显示不以双下划线开头的公共属性 118 | public_attrs = [attr for attr in dir(rusty_req) if not attr.startswith('_')] 119 | print(f'Public attributes: {public_attrs}') 120 | except ImportError as e: 121 | print(f'Import failed: {e}') 122 | exit(1) 123 | except Exception as e: 124 | print(f'Error: {e}') 125 | exit(1) 126 | " 127 | -------------------------------------------------------------------------------- /rusty_req.pyi: -------------------------------------------------------------------------------- 1 | """ 2 | Type stubs for rusty-req library. 3 | This file provides auto-completion and type hints for IDEs. 4 | """ 5 | 6 | from typing import Optional, Dict, Any, List, Union 7 | import asyncio 8 | 9 | class ProxyConfig: 10 | """Proxy configuration for requests.""" 11 | 12 | def __init__( 13 | self, 14 | http: Optional[str] = None, 15 | https: Optional[str] = None, 16 | all: Optional[str] = None, 17 | no_proxy: Optional[List[str]] = None, 18 | username: Optional[str] = None, 19 | password: Optional[str] = None, 20 | trust_env: Optional[bool] = None 21 | ) -> None: ... 22 | 23 | class HttpVersion: 24 | """HTTP version enumeration.""" 25 | 26 | # 枚举值 27 | AUTO: str 28 | HTTP1_1: str 29 | HTTP2: str 30 | 31 | class ConcurrencyMode: 32 | """Concurrency mode enumeration.""" 33 | 34 | # 枚举值 35 | SELECT_ALL: str 36 | JOIN_ALL: str 37 | 38 | class SslVerify: 39 | """SSL verification configuration.""" 40 | 41 | def __init__(self, verify: bool = True) -> None: ... 42 | 43 | class RequestItem: 44 | """Represents a single HTTP request.""" 45 | 46 | def __init__( 47 | self, 48 | url: str, 49 | method: str, 50 | params: Optional[Dict[str, Any]] = None, 51 | headers: Optional[Dict[str, str]] = None, 52 | tag: str = "", 53 | timeout: float = 30.0, 54 | ssl_verify: bool = True, 55 | http_version: Optional[HttpVersion] = None, 56 | proxy: Optional[ProxyConfig] = None 57 | ) -> None: ... 58 | 59 | async def fetch_single( 60 | url: str, 61 | method: Optional[str] = None, 62 | params: Optional[Dict[str, Any]] = None, 63 | timeout: Optional[float] = None, 64 | headers: Optional[Dict[str, str]] = None, 65 | tag: Optional[str] = None, 66 | proxy: Optional[ProxyConfig] = None, 67 | http_version: Optional[HttpVersion] = None, 68 | ssl_verify: Optional[bool] = None 69 | ) -> Dict[str, Any]: 70 | """ 71 | Send a single asynchronous HTTP request. 72 | 73 | Args: 74 | url: The target URL to send the request to 75 | method: HTTP method (GET, POST, PUT, DELETE). Defaults to GET 76 | params: Request parameters. For GET/DELETE: URL query parameters. 77 | For POST/PUT/PATCH: JSON body 78 | timeout: Request timeout in seconds. Defaults to 30.0 79 | headers: Custom HTTP headers 80 | tag: Arbitrary tag to identify the request 81 | proxy: Proxy configuration for this request 82 | http_version: HTTP version preference 83 | ssl_verify: SSL certificate verification. Defaults to True 84 | 85 | Returns: 86 | Dictionary containing response data with keys: 87 | - http_status: HTTP status code 88 | - response: Response content and headers 89 | - meta: Metadata including processing time and tag 90 | - exception: Exception information if request failed 91 | """ 92 | ... 93 | 94 | async def fetch_requests( 95 | requests: List[RequestItem], 96 | total_timeout: Optional[float] = None, 97 | mode: Optional[ConcurrencyMode] = None 98 | ) -> List[Dict[str, Any]]: 99 | """ 100 | Send multiple HTTP requests concurrently. 101 | 102 | Args: 103 | requests: List of RequestItem objects 104 | total_timeout: Global timeout for the entire batch 105 | mode: Concurrency strategy (SELECT_ALL or JOIN_ALL) 106 | 107 | Returns: 108 | List of response dictionaries with the same structure as fetch_single 109 | """ 110 | ... 111 | 112 | def set_debug(enabled: bool, log_file: Optional[str] = None) -> None: 113 | """ 114 | Enable or disable debug mode. 115 | 116 | Args: 117 | enabled: Whether to enable debug mode 118 | log_file: Optional log file path for writing debug logs 119 | """ 120 | ... 121 | 122 | async def set_global_proxy(proxy: ProxyConfig) -> None: 123 | """ 124 | Set global proxy configuration for all requests. 125 | 126 | Args: 127 | proxy: Proxy configuration 128 | """ 129 | ... 130 | 131 | # Response type definitions (基于你的实际返回结构) 132 | ResponseHeaders = Dict[str, str] 133 | 134 | class ResponseContent: 135 | """Response content structure.""" 136 | headers: ResponseHeaders 137 | content: str 138 | 139 | class RequestMeta: 140 | """Request metadata.""" 141 | process_time: str 142 | request_time: str 143 | tag: Optional[str] 144 | 145 | class RequestException: 146 | """Exception information.""" 147 | type: str 148 | message: str 149 | 150 | class SingleResponse: 151 | """Structure of a single response.""" 152 | http_status: int 153 | response: ResponseContent 154 | meta: RequestMeta 155 | exception: Optional[RequestException] 156 | 157 | # Module-level attributes 158 | __version__: str -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing Guidelines 2 | 3 | First off, thank you for considering contributing to `Rusty-req`. This guide details all the general information that one should know before contributing to the project. 4 | Please stick as close as possible to the guidelines. That way, we ensure that you have a smooth experience contributing to this project. 5 | 6 | ### General Rules: 7 | 8 | These are, in general, rules that you should be following while contributing to an Open-Source project : 9 | 10 | - Be Nice, Be Respectful (BNBR) 11 | - Check if the Issue you created, exists or not. 12 | - While creating a new issue, make sure you describe the issue clearly. 13 | - Make proper commit messages and document your PR well. 14 | - Always add comments in your Code and explain it at points if possible, add Doctest. 15 | - Always create a Pull Request from a Branch; Never from the Main. 16 | - Follow proper code conventions because writing clean code is important. 17 | - Issues would be assigned on a "First Come, First Served" basis. 18 | - Do mention (@KAY53N) the project maintainer if your PR isn't reviewed within a few days. 19 | 20 | ## First time contributors: 21 | 22 | Pushing files in your own repository is easy, but how to contribute to someone else's project? If you have the same question, then below are the steps that you can follow 23 | to make your first contribution in this repository. 24 | 25 | ### Pull Request 26 | 27 | **1.** The very first step includes forking the project. Click on the `fork` button as shown below to fork the project. 28 |


29 | 30 | **2.** Clone the forked repository. Open up the GitBash/Command Line and type 31 | 32 | ``` 33 | git clone https://github.com//Rusty-req.git 34 | ``` 35 | 36 | **3.** Navigate to the project directory. 37 | 38 | ``` 39 | cd Rusty-req 40 | ``` 41 | 42 | **4.** Add a reference to the original repository. 43 | 44 | ``` 45 | git remote add upstream https://github.com/KAY53N/rusty-req.git 46 | ``` 47 | 48 | **5.** See latest changes to the repo using 49 | 50 | ``` 51 | git remote -v 52 | ``` 53 | 54 | **6.** Create a new branch. 55 | 56 | ``` 57 | git checkout -b 58 | ``` 59 | 60 | **7.** Always take a pull from the upstream repository to your main branch to keep it even with the main project. This will save you from frequent merge conflicts. 61 | 62 | ``` 63 | git pull upstream main 64 | ``` 65 | 66 | **8.** You can make the required changes now. Make appropriate commits with proper commit messages. 67 | 68 | **9.** Add and then commit your changes. 69 | 70 | ``` 71 | git add . 72 | ``` 73 | 74 | ``` 75 | git commit -m "" 76 | ``` 77 | 78 | **10.** Push your local branch to the remote repository. 79 | 80 | ``` 81 | git push -u origin 82 | ``` 83 | 84 | **11.** Once you have pushed the changes to your repository, go to your forked repository. Click on the `Compare & pull request` button as shown below. 85 |


86 | 87 | **12.** The image below is what the new page would look like. Give a proper title to your PR and describe the changes made by you in the description box.(Note - Sometimes there are PR templates which are to be filled as instructed.) 88 |


89 | 90 | **13.** Open a pull request by clicking the `Create pull request` button. 91 | 92 | `Voila, you have made your first contribution to this project` 93 | 94 | ## Issue 95 | 96 | - Issues can be used to keep track of bugs, enhancements, or other requests. Creating an issue to let the project maintainers know about the changes you are planning to make before raising a PR is a good open-source practice. 97 |
98 | 99 | Let's walk through the steps to create an issue: 100 | 101 | **1.** On GitHub, navigate to the main page of the repository. [Here](https://github.com/KAY53N/rusty-req.git) in this case. 102 | 103 | **2.** Under your repository name, click on the `Issues` button. 104 |


105 | 106 | **3.** Click on the `New issue` button. 107 |


108 | 109 | **4.** Select one of the Issue Templates to get started. 110 |


111 | 112 | **5.** Fill in the appropriate `Title` and `Issue description` and click on `Submit new issue`. 113 |


114 | 115 | ### Tutorials that may help you: 116 | 117 | - [Git & GitHub Tutorial](https://www.youtube.com/watch?v=RGOj5yH7evk) 118 | - [Resolve merge conflict](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/resolving-a-merge-conflict-on-github) 119 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | kaysen820@gmail.com. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | tags: 6 | - 'v*' 7 | release: 8 | types: [created] 9 | workflow_dispatch: 10 | inputs: 11 | version: 12 | description: "要发布的版本号 (例如 v0.1.0)" 13 | required: false 14 | default: "" 15 | 16 | jobs: 17 | build-linux: 18 | runs-on: ubuntu-latest 19 | strategy: 20 | matrix: 21 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 22 | container: 23 | image: quay.io/pypa/manylinux_2_28_x86_64 24 | steps: 25 | - uses: actions/checkout@v4 26 | 27 | - name: Install system dependencies 28 | run: | 29 | dnf install -y openssl-devel pkgconfig 30 | 31 | - name: Install Rust 32 | run: | 33 | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile minimal 34 | source "$HOME/.cargo/env" 35 | rustup target add x86_64-unknown-linux-gnu 36 | 37 | - name: Set up Python & Build Wheel 38 | run: | 39 | source "$HOME/.cargo/env" 40 | PYTHON_VERSION=${{ matrix.python-version }} 41 | PYTHON_TAG="cp${PYTHON_VERSION//./}-cp${PYTHON_VERSION//./}" 42 | PYTHON_DIR="" 43 | for dir in /opt/python/*; do 44 | base=$(basename "$dir") 45 | if [[ "$base" == "${PYTHON_TAG}t" ]]; then 46 | PYTHON_DIR="$dir" 47 | break 48 | elif [[ "$base" == "$PYTHON_TAG" ]]; then 49 | PYTHON_DIR="$dir" 50 | fi 51 | done 52 | if [[ -z "$PYTHON_DIR" ]]; then 53 | echo "❌ 没找到解释器目录"; ls -1 /opt/python; exit 1 54 | fi 55 | PYTHON_BIN="${PYTHON_DIR}/bin" 56 | $PYTHON_BIN/pip install --upgrade pip maturin==1.9.2 57 | OUTPUT_DIR="/tmp/wheels/linux-py${PYTHON_VERSION}" 58 | mkdir -p "$OUTPUT_DIR" 59 | cargo clean 60 | $PYTHON_BIN/maturin build --release --manylinux 2_28 --interpreter $PYTHON_BIN/python --out "$OUTPUT_DIR" 61 | 62 | - name: Verify wheel integrity (Linux) 63 | run: | 64 | PYTHON_VERSION=${{ matrix.python-version }} 65 | PYTHON_TAG="cp${PYTHON_VERSION//./}-cp${PYTHON_VERSION//./}" 66 | PYTHON_DIR="" 67 | for dir in /opt/python/*; do 68 | base=$(basename "$dir") 69 | if [[ "$base" == "${PYTHON_TAG}t" ]]; then 70 | PYTHON_DIR="$dir" 71 | break 72 | elif [[ "$base" == "$PYTHON_TAG" ]]; then 73 | PYTHON_DIR="$dir" 74 | fi 75 | done 76 | PYTHON_BIN="${PYTHON_DIR}/bin" 77 | $PYTHON_BIN/pip install wheel 78 | for wheel in /tmp/wheels/linux-py${{ matrix.python-version }}/*.whl; do 79 | echo "检查: $wheel" 80 | # 使用 unpack 来验证 wheel 文件完整性 81 | $PYTHON_BIN/python -m wheel unpack "$wheel" -d /tmp/test_wheel 82 | rm -rf /tmp/test_wheel 83 | # 检查文件类型 84 | file "$wheel" 85 | # 检查 ZIP 文件完整性 86 | $PYTHON_BIN/python -c "import zipfile; zf = zipfile.ZipFile('$wheel'); print('ZIP 文件验证通过:', zf.testzip() is None)" 87 | done 88 | 89 | - name: Upload Linux wheels 90 | uses: actions/upload-artifact@v4 91 | with: 92 | name: wheels-linux-py${{ matrix.python-version }} 93 | path: /tmp/wheels/linux-py${{ matrix.python-version }}/*.whl 94 | 95 | build-macos: 96 | runs-on: macos-latest 97 | strategy: 98 | matrix: 99 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 100 | target: ["x86_64-apple-darwin", "aarch64-apple-darwin"] 101 | max-parallel: 3 102 | steps: 103 | - uses: actions/checkout@v4 104 | 105 | - name: Setup Rust 106 | uses: dtolnay/rust-toolchain@stable 107 | with: 108 | targets: ${{ matrix.target }} 109 | 110 | - name: Setup Python 111 | uses: actions/setup-python@v4 112 | with: 113 | python-version: ${{ matrix.python-version }} 114 | 115 | - name: Install maturin 116 | run: pip install maturin==1.9.2 117 | 118 | - name: Build wheel 119 | run: | 120 | OUTPUT_DIR="dist/macos-${{ matrix.target }}-py${{ matrix.python-version }}" 121 | mkdir -p $OUTPUT_DIR 122 | cargo clean 123 | maturin build --release --target ${{ matrix.target }} --interpreter python --out $OUTPUT_DIR 124 | 125 | - name: Verify wheel integrity (macOS) 126 | run: | 127 | pip install wheel 128 | for wheel in dist/macos-${{ matrix.target }}-py${{ matrix.python-version }}/*.whl; do 129 | echo "检查: $wheel" 130 | # 使用 unpack 替代 verify 131 | python -m wheel unpack "$wheel" -d /tmp/test_wheel 132 | rm -rf /tmp/test_wheel 133 | # 检查文件信息 134 | file "$wheel" 135 | # 检查 ZIP 完整性 136 | python -c "import zipfile; zf = zipfile.ZipFile('$wheel'); print('ZIP 文件验证通过:', zf.testzip() is None)" 137 | done 138 | 139 | - name: Upload macOS wheels 140 | uses: actions/upload-artifact@v4 141 | with: 142 | name: wheels-macos-${{ matrix.target }}-py${{ matrix.python-version }} 143 | path: dist/macos-${{ matrix.target }}-py${{ matrix.python-version }}/*.whl 144 | 145 | build-windows: 146 | runs-on: windows-latest 147 | strategy: 148 | matrix: 149 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 150 | steps: 151 | - uses: actions/checkout@v4 152 | 153 | - name: Setup Rust 154 | uses: dtolnay/rust-toolchain@stable 155 | 156 | - name: Setup Python 157 | uses: actions/setup-python@v4 158 | with: 159 | python-version: ${{ matrix.python-version }} 160 | 161 | - name: Install maturin 162 | run: pip install maturin==1.9.2 163 | 164 | - name: Build wheel (Windows) 165 | shell: pwsh 166 | run: | 167 | $OUTPUT_DIR = "dist\windows-py${{ matrix.python-version }}" 168 | New-Item -ItemType Directory -Path $OUTPUT_DIR -Force 169 | cargo clean 170 | # 构建 wheel 171 | maturin build --release --interpreter python --out $OUTPUT_DIR --strip 172 | 173 | # 修复 ZIP 文件：重新打包以确保格式正确 174 | Get-ChildItem "$OUTPUT_DIR\*.whl" | ForEach-Object { 175 | Write-Host "修复 ZIP 文件: $($_.FullName)" 176 | $tempDir = "temp_repack" 177 | # 解压并重新打包 178 | Expand-Archive -Path $_.FullName -DestinationPath $tempDir -Force 179 | Remove-Item -Force $_.FullName 180 | Compress-Archive -Path "$tempDir\*" -DestinationPath $_.FullName -CompressionLevel Optimal 181 | Remove-Item -Recurse -Force $tempDir 182 | } 183 | 184 | - name: Verify wheel integrity (Windows) 185 | shell: pwsh 186 | run: | 187 | pip install wheel 188 | $wheels = Get-ChildItem "dist\windows-py${{ matrix.python-version }}\*.whl" 189 | foreach ($wheel in $wheels) { 190 | Write-Host "检查: $($wheel.FullName)" 191 | # 使用 unpack 验证 192 | python -m wheel unpack $wheel.FullName -d test_wheel 193 | Remove-Item -Recurse -Force test_wheel -ErrorAction SilentlyContinue 194 | 195 | # 检查文件类型 196 | file $wheel.FullName 197 | 198 | # 使用 PowerShell 的 Expand-Archive 来验证 ZIP 文件 199 | try { 200 | Expand-Archive -Path $wheel.FullName -DestinationPath test_zip -Force 201 | Write-Host "✅ ZIP 验证通过" 202 | Remove-Item -Recurse -Force test_zip -ErrorAction SilentlyContinue 203 | } catch { 204 | Write-Host "❌ ZIP 验证失败: $($_.Exception.Message)" 205 | } 206 | } 207 | 208 | - name: Upload Windows wheels 209 | uses: actions/upload-artifact@v4 210 | with: 211 | name: wheels-windows-py${{ matrix.python-version }} 212 | path: dist/windows-py${{ matrix.python-version }}/*.whl 213 | 214 | publish: 215 | needs: [build-linux, build-macos, build-windows] 216 | runs-on: ubuntu-latest 217 | if: success() && (startsWith(github.ref, 'refs/tags/') || github.event_name == 'release') 218 | 219 | steps: 220 | - uses: actions/checkout@v4 221 | 222 | - name: Download Linux artifacts 223 | uses: actions/download-artifact@v4 224 | with: 225 | path: linux_wheels 226 | pattern: wheels-linux-* 227 | 228 | - name: Download macOS artifacts 229 | uses: actions/download-artifact@v4 230 | with: 231 | path: macos_wheels 232 | pattern: wheels-macos-* 233 | 234 | - name: Download Windows artifacts 235 | uses: actions/download-artifact@v4 236 | with: 237 | path: windows_wheels 238 | pattern: wheels-windows-* 239 | 240 | - name: Install twine and verification tools 241 | run: | 242 | pip install twine wheel 243 | 244 | - name: List all downloaded wheels 245 | run: | 246 | echo "=== 所有下载的 wheel 文件 ===" 247 | find . -name "*.whl" -exec ls -la {} \; 248 | echo "总文件数: $(find . -name "*.whl" | wc -l)" 249 | 250 | - name: Verify all wheels 251 | run: | 252 | echo "=== 验证 Linux wheels ===" 253 | for wheel in $(find linux_wheels -name "*.whl"); do 254 | if python -m wheel unpack "$wheel" -d /tmp/test_linux 2>/dev/null; then 255 | rm -rf /tmp/test_linux 256 | echo "✅ $wheel 验证通过" 257 | else 258 | echo "❌ $wheel 验证失败" 259 | fi 260 | done 261 | 262 | echo "=== 验证 macOS wheels ===" 263 | for wheel in $(find macos_wheels -name "*.whl"); do 264 | if python -m wheel unpack "$wheel" -d /tmp/test_macos 2>/dev/null; then 265 | rm -rf /tmp/test_macos 266 | echo "✅ $wheel 验证通过" 267 | else 268 | echo "❌ $wheel 验证失败" 269 | fi 270 | done 271 | 272 | echo "=== 验证 Windows wheels ===" 273 | for wheel in $(find windows_wheels -name "*.whl"); do 274 | if python -m wheel unpack "$wheel" -d /tmp/test_windows 2>/dev/null; then 275 | rm -rf /tmp/test_windows 276 | echo "✅ $wheel 验证通过" 277 | else 278 | echo "❌ $wheel 验证失败，跳过上传" 279 | fi 280 | done 281 | 282 | - name: Upload to PyPI 283 | env: 284 | TWINE_USERNAME: __token__ 285 | TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }} 286 | run: | 287 | echo "=== 上传 Linux wheels ===" 288 | for wheel in $(find linux_wheels -name "*.whl"); do 289 | echo "上传: $wheel" 290 | twine upload --skip-existing "$wheel" --verbose 291 | done 292 | 293 | echo "=== 上传 macOS wheels ===" 294 | for wheel in $(find macos_wheels -name "*.whl"); do 295 | echo "上传: $wheel" 296 | twine upload --skip-existing "$wheel" --verbose 297 | done 298 | 299 | echo "=== 上传验证通过的 Windows wheels ===" 300 | for wheel in $(find windows_wheels -name "*.whl"); do 301 | # 再次验证确保文件完好 302 | if python -m wheel unpack "$wheel" -d /tmp/upload_test 2>/dev/null; then 303 | rm -rf /tmp/upload_test 304 | echo "上传: $wheel" 305 | twine upload --skip-existing "$wheel" --verbose 306 | else 307 | echo "跳过验证失败的 Windows wheel: $wheel" 308 | fi 309 | done 310 | 311 | - name: Cleanup 312 | run: | 313 | # 清理可能的临时文件 314 | rm -rf /tmp/test_* /tmp/upload_test -------------------------------------------------------------------------------- /src/request/executor.rs: -------------------------------------------------------------------------------- 1 | use std::collections::HashMap; 2 | use std::time::{Duration, SystemTime}; 3 | use pyo3::prelude::*; 4 | use pyo3::types::{PyDict, PyList}; 5 | use reqwest::{Client, Proxy}; 6 | use crate::request::{execute_with_join_all, execute_with_select_all, RequestItem}; 7 | use crate::network::{HttpVersion}; 8 | use serde_json::Value; 9 | use url::Url; 10 | use crate::{ConcurrencyMode, ProxyConfig, DEFAULT_USER_AGENT, GLOBAL_CLIENT, GLOBAL_PROXY}; 11 | use crate::debug::debug_log; 12 | use crate::utils::{format_datetime, py_to_json}; 13 | 14 | pub(crate) async fn create_reqwest_client( 15 | request_url: &str, 16 | proxy_config: &Option, 17 | http_version: &HttpVersion, 18 | ssl_verify: bool, 19 | ) -> Result> { 20 | let mut builder = Client::builder() 21 | .timeout(Duration::from_secs(30)) 22 | .gzip(true) 23 | .brotli(true) 24 | .deflate(true) 25 | .user_agent(&*DEFAULT_USER_AGENT); // 复用同一个静态变量 26 | 27 | builder = http_version.apply_to_builder(builder); 28 | 29 | if !ssl_verify { 30 | builder = builder 31 | .danger_accept_invalid_certs(true) 32 | .danger_accept_invalid_hostnames(true); 33 | } 34 | 35 | // 检查是否信任环境变量，默认为 true 36 | let trust_env = proxy_config 37 | .as_ref() 38 | .and_then(|config| config.trust_env) 39 | .unwrap_or(true); 40 | 41 | // 如果不信任环境变量，禁用自动代理检测 42 | if !trust_env { 43 | builder = builder.no_proxy(); 44 | } 45 | 46 | if let Some(config) = proxy_config { // 解包 Option 47 | if let Some(all_proxy) = &config.all { 48 | let proxy_url = match (&config.username, &config.password) { 49 | (Some(user), Some(pass)) => { 50 | let mut url_parsed = Url::parse(all_proxy)?; 51 | let _ = url_parsed.set_username(user); 52 | let _ = url_parsed.set_password(Some(pass)); 53 | url_parsed.to_string() 54 | } 55 | (Some(user), None) => { 56 | let mut url_parsed = Url::parse(all_proxy)?; 57 | let _ = url_parsed.set_username(user); 58 | url_parsed.to_string() 59 | } 60 | _ => all_proxy.clone(), 61 | }; 62 | builder = builder.proxy(Proxy::all(&proxy_url)?); 63 | } else { 64 | // 如果没有 all_proxy，则根据 scheme 判断 65 | let parsed = Url::parse(request_url)?; // 使用请求的 url 来判断 scheme 66 | match parsed.scheme() { 67 | "http" => { 68 | if let Some(http_proxy) = &config.http { 69 | builder = builder.proxy(Proxy::http(http_proxy)?); 70 | } 71 | } 72 | "https" => { 73 | if let Some(https_proxy) = &config.https { 74 | builder = builder.proxy(Proxy::https(https_proxy)?); 75 | } 76 | } 77 | _ => {} 78 | } 79 | } 80 | } 81 | 82 | Ok(builder.build()?) 83 | } 84 | 85 | pub async fn execute_single_request(req: RequestItem, _base_client: Option) -> HashMap { 86 | let mut result = HashMap::new(); 87 | result.insert("response".to_string(), String::new()); 88 | 89 | let start = SystemTime::now(); 90 | let http_version = req.http_version.clone().unwrap_or(HttpVersion::Auto); 91 | 92 | // 获取代理配置，优先使用请求中的，否则使用全局的 93 | let proxy_config = if req.proxy.is_some() { 94 | req.proxy.clone() 95 | } else { 96 | GLOBAL_PROXY.lock().await.clone() 97 | }; 98 | 99 | // 获取 ssl_verify 布尔值，如果为 None 则默认 true 100 | let ssl_verify_bool = req.ssl_verify.unwrap_or(true); 101 | 102 | // 总是创建一个新的客户端，确保 ssl_verify 和 proxy 配置生效 103 | let client = match create_reqwest_client(&req.url, &proxy_config, &http_version, ssl_verify_bool).await { 104 | Ok(c) => c, 105 | Err(e) => { 106 | result.insert("http_status".to_string(), "0".to_string()); 107 | let mut exc = serde_json::Map::new(); 108 | exc.insert("type".to_string(), Value::String("ClientBuildError".to_string())); 109 | exc.insert("message".to_string(), Value::String(format!("Failed to build reqwest client: {}", e))); 110 | result.insert("exception".to_string(), Value::Object(exc).to_string()); 111 | 112 | let mut meta = serde_json::Map::new(); 113 | meta.insert("request_time".to_string(), Value::String("".to_string())); 114 | meta.insert("process_time".to_string(), Value::String("0.0000".to_string())); 115 | if let Some(tag) = req.tag.clone() { meta.insert("tag".to_string(), Value::String(tag)); } 116 | result.insert("meta".to_string(), Value::Object(meta).to_string()); 117 | return result; 118 | } 119 | }; 120 | 121 | // 客户端创建成功后，继续原有的请求逻辑 122 | let method = req.method.clone().unwrap_or_else(|| "GET".to_string()).to_uppercase(); 123 | let method = method.parse::().unwrap_or(reqwest::Method::GET); 124 | 125 | let mut request_builder = client.request(method.clone(), &req.url); 126 | let timeout = Duration::from_secs_f64(req.timeout.unwrap_or(30.0).max(3.0)); 127 | request_builder = request_builder.timeout(timeout); 128 | 129 | // headers 130 | let mut headers_to_add = Vec::new(); 131 | if let Some(py_headers) = &req.headers { 132 | Python::with_gil(|py| { 133 | if let Ok(dict) = py_headers.as_ref(py).downcast::() { 134 | for (k, v) in dict.iter() { 135 | if let (Ok(k_str), Ok(v_str)) = (k.extract::(), v.extract::()) { 136 | if let (Ok(h_name), Ok(h_val)) = ( 137 | reqwest::header::HeaderName::from_bytes(k_str.as_bytes()), 138 | reqwest::header::HeaderValue::from_str(&v_str), 139 | ) { headers_to_add.push((h_name, h_val)); } 140 | } 141 | } 142 | } 143 | }); 144 | } 145 | for (name, value) in headers_to_add { request_builder = request_builder.header(name, value); } 146 | 147 | if let Some(params_dict) = &req.params { 148 | request_builder = Python::with_gil(|py| { 149 | let mut inner_request_builder = request_builder; 150 | if let Ok(dict) = params_dict.as_ref(py).downcast::() { 151 | if let Ok(json) = py_to_json(py, dict) { 152 | match method { 153 | reqwest::Method::GET | reqwest::Method::DELETE => { 154 | if let Some(obj) = json.as_object() { 155 | let query_pairs: Vec<(String, String)> = obj.iter() 156 | .map(|(k, v)| (k.clone(), v.to_string().trim_matches('"').to_string())) 157 | .collect(); 158 | let query_refs: Vec<(&str, &str)> = query_pairs.iter().map(|(k,v)| (k.as_str(), v.as_str())).collect(); 159 | inner_request_builder = inner_request_builder.query(&query_refs); 160 | } 161 | } 162 | _ => { inner_request_builder = inner_request_builder.json(&json); } 163 | } 164 | } 165 | } 166 | inner_request_builder 167 | }); 168 | } 169 | 170 | let tag = req.tag.clone().unwrap_or_else(|| "no-tag".to_string()); 171 | 172 | match tokio::time::timeout(timeout, request_builder.send()).await { 173 | Ok(Ok(res)) => { 174 | let status = res.status(); 175 | result.insert("http_status".to_string(), status.as_u16().to_string()); 176 | 177 | // 生成 headers_map 178 | let headers_map: serde_json::Map = res.headers().iter() 179 | .map(|(k, v)| (k.to_string(), Value::String(v.to_str().unwrap_or("").to_string()))) 180 | .collect(); 181 | 182 | // 读取响应 183 | let text = res.text().await.unwrap_or_else(|e| format!("Failed to read response text: {}", e)); 184 | 185 | // response 对象 186 | let response = serde_json::json!({ 187 | "headers": headers_map, 188 | "content": text 189 | }); 190 | 191 | // 插入 result 192 | result.insert("response".to_string(), response.to_string()); 193 | 194 | // debug_log 调用 195 | debug_log( 196 | &method.to_string(), 197 | &tag, 198 | &req.url, 199 | status, 200 | &headers_map, 201 | &response, 202 | proxy_config.as_ref().and_then(|p| p.all.as_deref()), 203 | proxy_config.as_ref().and_then(|p| { 204 | if p.username.is_some() { 205 | Some("with authentication") 206 | } else { 207 | None 208 | } 209 | }).map(|s| s), 210 | ); 211 | 212 | if !status.is_success() { 213 | let mut exc = serde_json::Map::new(); 214 | exc.insert("type".to_string(), Value::String("HttpStatusError".to_string())); 215 | exc.insert("message".to_string(), Value::String(format!("HTTP status error: {}", status.as_u16()))); 216 | result.insert("exception".to_string(), Value::Object(exc).to_string()); 217 | } else { 218 | result.insert("exception".to_string(), "{}".to_string()); 219 | } 220 | } 221 | Ok(Err(e)) => { 222 | result.insert("http_status".to_string(), "0".to_string()); 223 | let mut exc = serde_json::Map::new(); 224 | exc.insert("type".to_string(), Value::String("HttpError".to_string())); 225 | exc.insert("message".to_string(), Value::String(format!("Request error: {}", e))); 226 | result.insert("exception".to_string(), Value::Object(exc).to_string()); 227 | result.insert("response".to_string(), serde_json::json!({"headers":{}, "content":""}).to_string()); 228 | } 229 | Err(_) => { 230 | result.insert("http_status".to_string(), "0".to_string()); 231 | let mut exc = serde_json::Map::new(); 232 | exc.insert("type".to_string(), Value::String("Timeout".to_string())); 233 | exc.insert("message".to_string(), Value::String(format!("Request timeout after {:.2} seconds", timeout.as_secs_f64()))); 234 | result.insert("exception".to_string(), Value::Object(exc).to_string()); 235 | result.insert("response".to_string(), serde_json::json!({"headers":{}, "content":""}).to_string()); 236 | } 237 | } 238 | 239 | // meta 信息 240 | let end = SystemTime::now(); 241 | let process_time = end.duration_since(start).unwrap_or(Duration::from_secs(0)).as_secs_f64(); 242 | let start_str = format_datetime(start); 243 | let end_str = format_datetime(end); 244 | let mut meta = serde_json::Map::new(); 245 | meta.insert("request_time".to_string(), Value::String(format!("{} -> {}", start_str, end_str))); 246 | meta.insert("process_time".to_string(), Value::String(format!("{:.4}", process_time))); 247 | if let Some(tag) = req.tag.clone() { meta.insert("tag".to_string(), Value::String(tag)); } 248 | result.insert("meta".to_string(), Value::Object(meta).to_string()); 249 | 250 | result 251 | } 252 | 253 | #[pyfunction] 254 | pub fn fetch_single<'py>( 255 | py: Python<'py>, 256 | url: String, 257 | method: Option, 258 | params: Option>, 259 | timeout: Option, 260 | headers: Option>, 261 | tag: Option, 262 | proxy: Option, 263 | http_version: Option, 264 | ssl_verify: Option, 265 | ) -> PyResult<&'py PyAny> { 266 | // 这里直接调用 execute_single_request 异步包装 267 | pyo3_asyncio::tokio::future_into_py(py, async move { 268 | let req = RequestItem { url, method, params, timeout, tag, headers, proxy, http_version, ssl_verify }; 269 | let result = execute_single_request(req, None).await; 270 | Python::with_gil(|py| -> PyResult> { 271 | let dict = PyDict::new(py); 272 | dict.set_item("response", &result["response"])?; 273 | dict.set_item("http_status", result.get("http_status").unwrap_or(&"0".to_string()))?; 274 | dict.set_item("meta", result.get("meta").unwrap_or(&"{}".to_string()))?; 275 | dict.set_item("exception", result.get("exception").unwrap_or(&"{}".to_string()))?; 276 | Ok(dict.into_py(py)) 277 | }) 278 | }) 279 | } 280 | 281 | /// 异步批量请求函数 282 | #[pyfunction] 283 | pub fn fetch_requests<'py>( 284 | py: Python<'py>, 285 | requests: Vec, 286 | total_timeout: Option, 287 | mode: Option, 288 | ) -> PyResult<&'py PyAny> { 289 | pyo3_asyncio::tokio::future_into_py(py, async move { 290 | let total_duration = Duration::from_secs_f64(total_timeout.unwrap_or(30.0)); 291 | let mode = mode.unwrap_or(ConcurrencyMode::SelectAll); 292 | let base_client = Some(GLOBAL_CLIENT.lock().await.clone()); 293 | 294 | let final_results = match mode { 295 | ConcurrencyMode::SelectAll => { 296 | execute_with_select_all(requests, total_duration, base_client).await 297 | } 298 | ConcurrencyMode::JoinAll => { 299 | execute_with_join_all(requests, total_duration, base_client).await 300 | } 301 | }; 302 | 303 | Python::with_gil(|py| -> PyResult { 304 | let py_list = PyList::empty(py); 305 | for res in final_results { 306 | let dict = PyDict::new(py); 307 | dict.set_item("response", &res["response"])?; 308 | 309 | if let Some(http_status_str) = res.get("http_status") { 310 | if let Ok(http_status_int) = http_status_str.parse::() { 311 | dict.set_item("http_status", http_status_int)?; 312 | } else { 313 | dict.set_item("http_status", http_status_str)?; 314 | } 315 | } 316 | 317 | let meta_json_str = res.get("meta").map(|s| s.as_str()).unwrap_or("{}"); 318 | let meta_pyobj = py.import("json")?.call_method1("loads", (meta_json_str,))?; 319 | dict.set_item("meta", meta_pyobj)?; 320 | 321 | if let Some(exc_str) = res.get("exception") { 322 | let exc_obj = py.import("json")?.call_method1("loads", (exc_str,))?; 323 | dict.set_item("exception", exc_obj)?; 324 | } 325 | 326 | py_list.append(dict)?; 327 | } 328 | Ok(py_list.into_py(py)) 329 | }) 330 | }) 331 | } 332 | -------------------------------------------------------------------------------- /tests/performance_test.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import time 3 | import json 4 | import psutil 5 | import rusty_req 6 | from rusty_req import ConcurrencyMode 7 | from typing import Dict, Any 8 | import aiohttp 9 | import httpx 10 | import requests 11 | from concurrent.futures import ThreadPoolExecutor 12 | import os 13 | 14 | # Global settings 15 | GLOBAL_CONCURRENCY = 800 # Updated concurrency 16 | TOTAL_TIMEOUT = 4.0 # Total timeout for all requests 17 | REQUEST_TIMEOUT = 3.5 # Timeout per request 18 | 19 | class PerformanceTest: 20 | def __init__(self): 21 | self.test_results = {} 22 | self.httpbin_url = os.getenv('HTTPBIN_URL', 'http://localhost:8080') 23 | print(f"🌐 Using httpbin service URL: {self.httpbin_url}") 24 | 25 | async def cooldown(self, seconds: int = 10): 26 | print(f"⏳ Cooling down for {seconds} seconds...") 27 | await asyncio.sleep(seconds) 28 | 29 | async def test_httpbin_connectivity(self): 30 | print("🔍 Testing httpbin connectivity...") 31 | try: 32 | response = await rusty_req.fetch_single( 33 | url=f"{self.httpbin_url}/status/200", 34 | method="GET", 35 | timeout=REQUEST_TIMEOUT 36 | ) 37 | print(f"📝 httpbin response details: {response}") 38 | 39 | http_status = response.get("http_status") 40 | if isinstance(http_status, str): 41 | http_status = int(http_status) if http_status.isdigit() else 0 42 | 43 | exception = response.get("exception", {}) 44 | if isinstance(exception, str): 45 | try: 46 | exception = json.loads(exception) 47 | except json.JSONDecodeError: 48 | exception = {} 49 | 50 | has_error = exception.get("type") is not None 51 | 52 | if http_status == 200 and not has_error: 53 | print("✅ httpbin connectivity OK") 54 | return True 55 | else: 56 | print(f"❌ httpbin abnormal response - status: {http_status}, error: {exception}") 57 | return False 58 | except Exception as e: 59 | print(f"❌ httpbin connectivity failed: {e}") 60 | return False 61 | 62 | async def test_rusty_req_batch(self, num_requests: int = GLOBAL_CONCURRENCY, delay: float = 1.0) -> Dict[str, Any]: 63 | print(f"🚀 Testing rusty-req batch ({num_requests} requests, {delay}s delay)...") 64 | requests_list = [ 65 | rusty_req.RequestItem( 66 | url=f"{self.httpbin_url}/delay/{delay}", 67 | method="GET", 68 | timeout=REQUEST_TIMEOUT, 69 | tag=f"batch-req-{i}", 70 | ) 71 | for i in range(num_requests) 72 | ] 73 | 74 | process = psutil.Process() 75 | start_memory = process.memory_info().rss / 1024 / 1024 76 | 77 | start_time = time.perf_counter() 78 | responses = await rusty_req.fetch_requests( 79 | requests_list, 80 | total_timeout=TOTAL_TIMEOUT, 81 | mode=ConcurrencyMode.SELECT_ALL 82 | ) 83 | end_time = time.perf_counter() 84 | end_memory = process.memory_info().rss / 1024 / 1024 85 | 86 | successful = 0 87 | failed = 0 88 | for r in responses: 89 | http_status = r.get("http_status") 90 | if isinstance(http_status, str): 91 | http_status = int(http_status) if http_status.isdigit() else 0 92 | 93 | exception = r.get("exception", {}) 94 | if isinstance(exception, str): 95 | try: 96 | exception = json.loads(exception) if exception else {} 97 | except json.JSONDecodeError: 98 | exception = {} 99 | 100 | has_error = exception.get("type") is not None 101 | if http_status == 200 and not has_error: 102 | successful += 1 103 | else: 104 | failed += 1 105 | 106 | total_time = end_time - start_time 107 | 108 | return { 109 | "library": "rusty-req", 110 | "mode": "batch", 111 | "total_requests": num_requests, 112 | "successful": successful, 113 | "failed": failed, 114 | "success_rate": (successful / num_requests) * 100, 115 | "total_time": total_time, 116 | "requests_per_second": num_requests / total_time, 117 | "memory_usage": end_memory - start_memory, 118 | "avg_response_time": total_time / num_requests 119 | } 120 | 121 | async def test_httpx_async(self, num_requests: int = GLOBAL_CONCURRENCY, delay: float = 1.0) -> Dict[str, Any]: 122 | print(f"🚀 Testing httpx async ({num_requests} requests, {delay}s delay)...") 123 | start_time = time.perf_counter() 124 | successful = 0 125 | failed = 0 126 | 127 | timeout = httpx.Timeout(REQUEST_TIMEOUT) 128 | async with httpx.AsyncClient(timeout=timeout) as client: 129 | tasks = [client.get(f"{self.httpbin_url}/delay/{delay}") for _ in range(num_requests)] 130 | try: 131 | responses = await asyncio.wait_for( 132 | asyncio.gather(*tasks, return_exceptions=True), 133 | timeout=TOTAL_TIMEOUT 134 | ) 135 | except asyncio.TimeoutError: 136 | print("⏱ Total timeout exceeded, counting remaining requests as failed") 137 | responses = [] 138 | 139 | for response in responses: 140 | if isinstance(response, Exception): 141 | failed += 1 142 | elif hasattr(response, 'status_code') and response.status_code == 200: 143 | successful += 1 144 | else: 145 | failed += 1 146 | 147 | total_time = min(time.perf_counter() - start_time, TOTAL_TIMEOUT) 148 | failed += max(0, num_requests - (successful + failed)) 149 | 150 | return { 151 | "library": "httpx", 152 | "mode": "async", 153 | "total_requests": num_requests, 154 | "successful": successful, 155 | "failed": failed, 156 | "success_rate": (successful / num_requests) * 100, 157 | "total_time": total_time, 158 | "requests_per_second": num_requests / total_time, 159 | "avg_response_time": total_time / num_requests 160 | } 161 | 162 | async def test_aiohttp(self, num_requests: int = GLOBAL_CONCURRENCY, delay: float = 1.0) -> Dict[str, Any]: 163 | print(f"🚀 Testing aiohttp ({num_requests} requests, {delay}s delay)...") 164 | start_time = time.perf_counter() 165 | successful = 0 166 | failed = 0 167 | 168 | timeout = aiohttp.ClientTimeout(total=REQUEST_TIMEOUT) 169 | async with aiohttp.ClientSession(timeout=timeout) as session: 170 | tasks = [session.get(f"{self.httpbin_url}/delay/{delay}") for _ in range(num_requests)] 171 | try: 172 | responses = await asyncio.wait_for( 173 | asyncio.gather(*tasks, return_exceptions=True), 174 | timeout=TOTAL_TIMEOUT 175 | ) 176 | except asyncio.TimeoutError: 177 | print("⏱ Total timeout exceeded, counting remaining requests as failed") 178 | responses = [] 179 | 180 | for response in responses: 181 | if isinstance(response, Exception): 182 | failed += 1 183 | else: 184 | try: 185 | if hasattr(response, 'status') and response.status == 200: 186 | successful += 1 187 | else: 188 | failed += 1 189 | finally: 190 | if hasattr(response, 'close'): 191 | response.close() 192 | 193 | total_time = min(time.perf_counter() - start_time, TOTAL_TIMEOUT) 194 | failed += max(0, num_requests - (successful + failed)) 195 | 196 | return { 197 | "library": "aiohttp", 198 | "mode": "async", 199 | "total_requests": num_requests, 200 | "successful": successful, 201 | "failed": failed, 202 | "success_rate": (successful / num_requests) * 100, 203 | "total_time": total_time, 204 | "requests_per_second": num_requests / total_time, 205 | "avg_response_time": total_time / num_requests 206 | } 207 | 208 | def test_requests_sync(self, num_requests: int = GLOBAL_CONCURRENCY, delay: float = 1.0) -> Dict[str, Any]: 209 | print(f"🚀 Testing requests sync ({num_requests} requests, {delay}s delay)...") 210 | 211 | def make_request(): 212 | try: 213 | response = requests.get(f"{self.httpbin_url}/delay/{delay}", timeout=REQUEST_TIMEOUT) 214 | return response.status_code == 200 215 | except Exception: 216 | return False 217 | 218 | start_time = time.perf_counter() 219 | with ThreadPoolExecutor(max_workers=min(GLOBAL_CONCURRENCY, num_requests)) as executor: 220 | results = list(executor.map(lambda _: make_request(), range(num_requests))) 221 | end_time = time.perf_counter() 222 | 223 | successful = sum(results) 224 | failed = num_requests - successful 225 | total_time = min(end_time - start_time, TOTAL_TIMEOUT) 226 | failed += max(0, num_requests - (successful + failed)) 227 | 228 | return { 229 | "library": "requests", 230 | "mode": "sync_threaded", 231 | "total_requests": num_requests, 232 | "successful": successful, 233 | "failed": failed, 234 | "success_rate": (successful / num_requests) * 100, 235 | "total_time": total_time, 236 | "requests_per_second": num_requests / total_time, 237 | "avg_response_time": total_time / num_requests 238 | } 239 | 240 | async def run_comprehensive_test(self): 241 | print("=" * 60) 242 | print("🎯 Start performance benchmark") 243 | print("=" * 60) 244 | 245 | if not await self.test_httpbin_connectivity(): 246 | print("❌ httpbin service not available, aborting tests") 247 | return {} 248 | 249 | rusty_req.set_debug(False) 250 | results = {} 251 | 252 | try: 253 | # rusty-req batch 254 | print("\n📊 Rusty-Req batch performance test") 255 | results["rusty_req_batch"] = await self.test_rusty_req_batch() 256 | await self.cooldown(30) 257 | 258 | # httpx async 259 | print("\n📊 httpx performance test") 260 | try: 261 | results["httpx_async"] = await self.test_httpx_async() 262 | except Exception as e: 263 | print(f"⚠️ httpx test failed: {e}") 264 | await self.cooldown(30) 265 | 266 | # aiohttp async 267 | print("\n📊 aiohttp performance test") 268 | try: 269 | results["aiohttp"] = await self.test_aiohttp() 270 | except Exception as e: 271 | print(f"⚠️ aiohttp test failed: {e}") 272 | await self.cooldown(30) 273 | 274 | # requests sync 275 | print("\n📊 requests performance test") 276 | try: 277 | results["requests_sync"] = self.test_requests_sync() 278 | except Exception as e: 279 | print(f"⚠️ requests test failed: {e}") 280 | await self.cooldown(30) 281 | 282 | except Exception as e: 283 | print(f"❌ Error during tests: {e}") 284 | import traceback 285 | traceback.print_exc() 286 | 287 | return results 288 | 289 | def print_results(self, results: Dict[str, Any]): 290 | print("\n" + "=" * 80) 291 | print("📋 Benchmark Report") 292 | print("=" * 80) 293 | 294 | for test_name, result in results.items(): 295 | print(f"\n📊 {result['library']} ({result.get('mode', 'default')}):") 296 | print(f" Total Requests: {result['total_requests']}") 297 | print(f" Successful: {result['successful']}") 298 | print(f" Failed: {result['failed']}") 299 | print(f" Success Rate: {result['success_rate']:.1f}%") 300 | print(f" Total Time: {result['total_time']:.2f} s") 301 | print(f" Throughput: {result['requests_per_second']:.1f} req/s") 302 | print(f" Avg Response Time: {result['avg_response_time']*1000:.1f} ms") 303 | if 'memory_usage' in result: 304 | print(f" Memory Usage: {result['memory_usage']:.1f} MB") 305 | 306 | # Ranking: success rate high -> req/s high 307 | performance_data = [] 308 | for result in results.values(): 309 | if 'requests_per_second' in result: 310 | performance_data.append( 311 | (f"{result['library']}({result.get('mode', 'default')})", 312 | result['success_rate'], 313 | result['requests_per_second']) 314 | ) 315 | 316 | performance_data.sort(key=lambda x: (-x[1], -x[2])) # success_rate desc, req/s desc 317 | print(f"\n🏆 Ranking (by success rate, then throughput):") 318 | for i, (lib, success_rate, rps) in enumerate(performance_data, 1): 319 | print(f" {i}. {lib}: {rps:.1f} req/s (Success Rate: {success_rate:.1f}%)") 320 | 321 | async def main(): 322 | tester = PerformanceTest() 323 | results = await tester.run_comprehensive_test() 324 | if not results: 325 | print("❌ Benchmark failed, no results generated") 326 | return 327 | 328 | tester.print_results(results) 329 | 330 | timestamp = time.strftime("%Y%m%d_%H%M%S") 331 | filename = f"rusty_req_benchmark_{timestamp}.json" 332 | with open(filename, "w", encoding="utf-8") as f: 333 | json.dump(results, f, indent=2, ensure_ascii=False) 334 | print(f"\n💾 Results saved to {filename}") 335 | 336 | if __name__ == "__main__": 337 | try: 338 | import rusty_req 339 | except ImportError as e: 340 | print(f"❌ Missing dependency: {e}") 341 | print("Please install: pip install rusty-req") 342 | exit(1) 343 | 344 | optional_deps = [] 345 | try: 346 | import aiohttp 347 | optional_deps.append("aiohttp") 348 | except ImportError: 349 | print("⚠️ aiohttp not installed, skipping related benchmark") 350 | 351 | try: 352 | import httpx 353 | optional_deps.append("httpx") 354 | except ImportError: 355 | print("⚠️ httpx not installed, skipping related benchmark") 356 | 357 | try: 358 | import requests 359 | optional_deps.append("requests") 360 | except ImportError: 361 | print("⚠️ requests not installed, skipping related benchmark") 362 | 363 | try: 364 | import psutil 365 | optional_deps.append("psutil") 366 | except ImportError: 367 | print("⚠️ psutil not installed, skipping memory monitoring") 368 | 369 | if optional_deps: 370 | print(f"✅ Optional dependencies installed: {', '.join(optional_deps)}") 371 | 372 | asyncio.run(main()) 373 | -------------------------------------------------------------------------------- /README.zh.md: -------------------------------------------------------------------------------- 1 | # rusty-req 2 | 3 | [![PyPI version](https://img.shields.io/pypi/v/rusty-req)](https://pypi.org/project/rusty-req/) 4 | [![PyPI downloads](https://img.shields.io/pypi/dm/rusty-req)](https://pypi.org/project/rusty-req/) 5 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 6 | [![Python versions](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/downloads/) 7 | [![GitHub issues](https://img.shields.io/github/issues/KAY53N/rusty-req)](https://github.com/KAY53N/rusty-req/issues) 8 | [![Build Status](https://github.com/KAY53N/rusty-req/actions/workflows/build.yml/badge.svg)](https://github.com/KAY53N/rusty-req/actions/workflows/build.yml) 9 | [![Cross Platform Test](https://github.com/KAY53N/rusty-req/actions/workflows/cross-platform-test.yml/badge.svg)](https://github.com/KAY53N/rusty-req/actions/workflows/cross-platform-test.yml) 10 | 11 | 基于 Rust 和 Python 的高性能异步请求库... 12 | 13 | 一个基于 Rust 和 Python 的高性能异步请求库，适用于需要高吞吐量并发 HTTP 请求的场景。核心并发逻辑使用 Rust 实现，并通过 [PyO3](https://pyo3.rs/) 和 [maturin](https://github.com/PyO3/maturin) 封装为 Python 模块，将 Rust 的性能优势与 Python 的易用性结合。 14 | 15 | ### 🌐 [English](README.md) | [中文](README.zh.md) 16 | 17 | ## 🚀 功能特性 18 | 19 | - **双模式请求**：支持批量并发请求（`fetch_requests`）和单个异步请求（`fetch_single`）。 20 | - **高性能**：使用 Rust、Tokio，并共享 `reqwest` 客户端以最大化吞吐量。 21 | - **高度可定制**：支持自定义请求头、参数/请求体、每个请求的超时及标签。 22 | - **灵活的并发模式**：可选择 `SELECT_ALL`（默认，按完成顺序返回结果）或 `JOIN_ALL`（等待所有请求完成再返回）。 23 | - **智能响应处理**：自动解压 `gzip`、`brotli` 和 `deflate` 编码的响应。 24 | - **全局超时控制**：批量请求可设置 `total_timeout` 防止挂起。 25 | - **详细结果**：每个响应包含 HTTP 状态、响应体、元信息（如处理时间）及异常信息。 26 | - **调试模式**：可选调试模式 (`set_debug(True)`) 打印详细请求/响应日志。 27 | 28 | ## 🔧 安装 29 | 30 | ```bash 31 | pip install rusty-req 32 | ``` 33 | 或从源码构建： 34 | ``` 35 | # 编译 Rust 代码并生成 .whl 文件 36 | maturin build --release 37 | 38 | # 安装生成的 wheel 39 | pip install target/wheels/rusty_req-*.whl 40 | ``` 41 | 42 | ## 开发与调试 43 | ``` 44 | cargo watch -s "maturin develop" 45 | ``` 46 | 47 | ## ⚙️ 代理配置 & 调试 48 | 49 | ### 1. 使用代理 50 | 51 | 如果需要通过代理访问外部网络，可以创建 `ProxyConfig` 对象并设置为全局代理： 52 | 53 | ```python 54 | import asyncio 55 | import rusty_req 56 | 57 | async def proxy_example(): 58 | # 创建 ProxyConfig 对象 59 | proxy = rusty_req.ProxyConfig( 60 | http="http://127.0.0.1:7890", 61 | https="http://127.0.0.1:7890" 62 | ) 63 | 64 | # 设置全局代理（所有请求都会使用该代理） 65 | await rusty_req.set_global_proxy(proxy) 66 | 67 | # 发起请求（将自动通过代理） 68 | resp = await rusty_req.fetch_single(url="https://httpbin.org/get") 69 | print(resp) 70 | 71 | if __name__ == "__main__": 72 | asyncio.run(proxy_example()) 73 | ``` 74 | 75 | ### 2. 调试日志 76 | 77 | `set_debug` 用于启用调试模式，支持 **控制台输出** 和 **日志文件记录**： 78 | 79 | ```python 80 | import rusty_req 81 | 82 | # 仅在控制台打印调试信息 83 | rusty_req.set_debug(True) 84 | 85 | # 同时打印到控制台并写入日志文件 86 | rusty_req.set_debug(True, "logs/debug.log") 87 | 88 | # 关闭调试模式 89 | rusty_req.set_debug(False) 90 | ``` 91 | 92 | ## 📦 使用示例 93 | ### 1. 单个请求 (`fetch_single`) 94 | 适合单个异步请求并等待结果的场景。 95 | 96 | ```python 97 | import asyncio 98 | import pprint 99 | import rusty_req 100 | 101 | async def single_request_example(): 102 | """示例：使用 fetch_single 发起 POST 请求""" 103 | print("🚀 正在向 httpbin.org 发送单个 POST 请求...") 104 | 105 | rusty_req.set_debug(True) # 开启调试模式 106 | 107 | response = await rusty_req.fetch_single( 108 | url="https://httpbin.org/post", 109 | method="POST", 110 | params={"user_id": 123, "source": "example"}, 111 | headers={"X-Client-Version": "1.0"}, 112 | tag="my-single-post" 113 | ) 114 | 115 | print("\n✅ 请求完成，响应如下：") 116 | pprint.pprint(response) 117 | 118 | if __name__ == "__main__": 119 | asyncio.run(single_request_example()) 120 | 121 | ``` 122 | 123 | ### 2. 批量请求 (`fetch_requests`) 124 | 125 | 适合高并发场景或压力测试。 126 | ```python 127 | import asyncio 128 | import time 129 | import rusty_req 130 | from rusty_req import ConcurrencyMode 131 | 132 | async def batch_requests_example(): 133 | """示例：100 个并发请求，设置全局超时""" 134 | requests = [ 135 | rusty_req.RequestItem( 136 | url="https://httpbin.org/delay/2", 137 | method="GET", 138 | timeout=2.9, # 每个请求的超时 139 | tag=f"test-req-{i}", 140 | ) 141 | for i in range(100) 142 | ] 143 | 144 | rusty_req.set_debug(False) # 关闭调试日志 145 | 146 | print("🚀 开始 100 个并发请求...") 147 | start_time = time.perf_counter() 148 | 149 | responses = await rusty_req.fetch_requests( 150 | requests, 151 | total_timeout=3.0, # 批量请求全局超时 152 | mode=ConcurrencyMode.SELECT_ALL 153 | ) 154 | 155 | total_time = time.perf_counter() - start_time 156 | 157 | success_count = 0 158 | failed_count = 0 159 | for r in responses: 160 | if r.get("exception") and r["exception"].get("type"): 161 | failed_count += 1 162 | else: 163 | success_count += 1 164 | 165 | print("\n📊 压力测试结果：") 166 | print(f"⏱️ 总耗时: {total_time:.2f}s") 167 | print(f"✅ 成功请求数: {success_count}") 168 | print(f"⚠️ 超时或失败请求数: {failed_count}") 169 | 170 | if __name__ == "__main__": 171 | asyncio.run(batch_requests_example()) 172 | 173 | ``` 174 | 175 | ### 3. 并发模式对比 (`SELECT_ALL` vs `JOIN_ALL`) 176 | 177 | `fetch_requests` 函数支持两种强大的并发策略，选择合适的策略对于构建健壮的应用非常关键。 178 | 179 | - **`ConcurrencyMode.SELECT_ALL`（默认）：尽力收集模式** 180 | 该模式按照“先完成先返回”或“尽力而为”的原则工作，目标是在指定的 `total_timeout` 时间内尽可能多地收集成功结果。 181 | - 请求一完成就立即返回结果。 182 | - 如果达到 `total_timeout`，会优雅地返回已经完成的请求结果，同时将仍在等待的请求标记为超时。 183 | - **单个请求失败不会影响其他请求。** 184 | 185 | - **`ConcurrencyMode.JOIN_ALL`：事务性模式（全有或全无）** 186 | 该模式将整个批次视为一个原子事务，要求更严格。 187 | - 会等待 **所有** 提交的请求完成后再处理结果。 188 | - 然后对结果进行检查。 189 | - **成功情况**：仅当 *每一个请求都成功* 时，才会返回完整的成功结果列表。 190 | - **失败情况**：如果 *任意一个请求失败*（例如单个超时、网络错误或非 2xx 状态码），整个批次将被视为失败，并返回一个列表，其中 **每个请求都被标记为全局失败**。 191 | 192 | ### 4. 超时性能对比 193 | 194 | 在相同测试条件下（全局超时 3 秒，每个请求超时 2.6 秒，httpbin 延迟 2.3 秒），我们对比了不同库的性能： 195 | 196 | | 库 / 框架 | 请求总数 | 成功数 | 超时数 | 成功率 | 实际总耗时 | 说明 / 特点 | 197 | |--------------------|---------|--------|--------|--------|--------|------------| 198 | | **Rusty-req** | 1000 | 1000 | 0 | 100.0% | 2.56s | 高并发下性能稳定；可以精确控制每个请求和全局超时 | 199 | | **httpx** | 1000 | 0 | 0 | 0.0% | 26.77s | 超时参数未生效，整体性能异常 | 200 | | **aiohttp** | 1000 | 100 | 900 | 10.0% | 2.66s | 单请求超时有效，但全局超时控制不足 | 201 | | **requests** | 1000 | 1000 | 0 | 100.0% | 3.45s | 同步阻塞模式，不适合大规模并发请求 | 202 | 203 | 关键结论： 204 | - **Rusty-req** 可以在严格的全局超时限制下完成任务，同时保持高并发和稳定性。 205 | - 传统异步库在全局超时和极高并发场景下表现欠佳。 206 | - 同步库如 `requests` 虽然能得到正确结果，但不适合大规模并发请求。 207 | 208 | ![超时性能对比](https://raw.githubusercontent.com/KAY53N/rusty-req/main/docs/images/timeout_performance_comparison.png) 209 | 210 | --- 211 | 212 | ### 快速对比 213 | 214 | | 方面 | `ConcurrencyMode.SELECT_ALL`（默认） | `ConcurrencyMode.JOIN_ALL` | 215 | | :---------------------- | :---------------------------------------------------------------- | :----------------------------------------------------------------- | 216 | | **失败处理** | **宽容**。单个请求失败不会影响其他成功请求。 | **严格 / 原子**。单个请求失败会导致整个批次失败。 | 217 | | **主要使用场景** | 最大化吞吐量；尽可能获取更多数据。 | 任务必须全部成功或全部失败（例如事务操作）。 | 218 | | **结果顺序** | 按完成时间返回（最快的先返回）。 | 按原提交顺序返回。 | 219 | | **何时获取结果** | 请求完成即返回，逐个获取。 | 所有请求完成并验证后一次性返回。 | 220 | 221 | --- 222 | 223 | ### 代码示例 224 | 225 | 下面的示例清楚地演示了两种模式的行为差异。 226 | 227 | ```python 228 | import asyncio 229 | import rusty_req 230 | from rusty_req import ConcurrencyMode 231 | 232 | async def concurrency_modes_example(): 233 | """演示 SELECT_ALL 和 JOIN_ALL 模式的区别。""" 234 | # 注意：这里使用一个返回 500 的接口以触发失败。 235 | requests = [ 236 | rusty_req.RequestItem(url="https://httpbin.org/delay/2", tag="should_succeed"), 237 | rusty_req.RequestItem(url="https://httpbin.org/status/500", tag="will_fail"), 238 | rusty_req.RequestItem(url="https://httpbin.org/delay/1", tag="should_also_succeed"), 239 | ] 240 | 241 | # --- 1. 测试 SELECT_ALL --- 242 | print("--- 🚀 测试 SELECT_ALL（尽力收集模式） ---") 243 | results_select = await rusty_req.fetch_requests( 244 | requests, 245 | mode=ConcurrencyMode.SELECT_ALL, 246 | total_timeout=3.0 247 | ) 248 | 249 | print("结果:") 250 | for res in results_select: 251 | tag = res.get("meta", {}).get("tag") 252 | status = res.get("http_status") 253 | err_type = res.get("exception", {}).get("type") 254 | print(f" - Tag: {tag}, Status: {status}, Exception: {err_type}") 255 | 256 | print("\n" + "="*50 + "\n") 257 | 258 | # --- 2. 测试 JOIN_ALL --- 259 | print("--- 🚀 测试 JOIN_ALL（全有或全无模式） ---") 260 | results_join = await rusty_req.fetch_requests( 261 | requests, 262 | mode=ConcurrencyMode.JOIN_ALL, 263 | total_timeout=3.0 264 | ) 265 | 266 | print("结果:") 267 | for res in results_join: 268 | tag = res.get("meta", {}).get("tag") 269 | status = res.get("http_status") 270 | err_type = res.get("exception", {}).get("type") 271 | print(f" - Tag: {tag}, Status: {status}, Exception: {err_type}") 272 | 273 | if __name__ == "__main__": 274 | asyncio.run(concurrency_modes_example()) 275 | ``` 276 | 上述脚本的预期输出： 277 | 278 | ``` 279 | --- 🚀 测试 SELECT_ALL（尽力收集模式） --- 280 | 结果: 281 | - Tag: should_also_succeed, Status: 200, Exception: None 282 | - Tag: will_fail, Status: 500, Exception: HttpStatusError 283 | - Tag: should_succeed, Status: 200, Exception: None 284 | 285 | ================================================== 286 | 287 | --- 🚀 测试 JOIN_ALL（全有或全无模式） --- 288 | 结果: 289 | - Tag: should_succeed, Status: 0, Exception: GlobalTimeout 290 | - Tag: will_fail, Status: 0, Exception: GlobalTimeout 291 | - Tag: should_also_succeed, Status: 0, Exception: GlobalTimeout 292 | ``` 293 | 294 | ## 🧱 数据结构 295 | 296 | ### `RequestItem` 参数 297 | 298 | | 字段 | 类型 | 必填 | 描述 | 299 | |:---------------|:----------------| :--: |:----------------------------------------------------------------------| 300 | | `url` | `str` | ✅ | 目标 URL 地址。 | 301 | | `method` | `str` | ✅ | HTTP 请求方法。 | 302 | | `params` | `dict` / `None` | 否 | 对于 GET/DELETE 请求，会转换为 URL 查询参数；对于 POST/PUT/PATCH 请求，会作为 JSON body 发送。 | 303 | | `headers` | `dict` / `None` | 否 | 自定义 HTTP 请求头。 | 304 | | `tag` | `str` | 否 | 用于标记请求或索引响应的任意字符串标签。 | 305 | | `http_version` | `str` | 否 | 指定的http版本，默认行为是“Auto”，优先尝试 HTTP/2，如果不支持则回退 HTTP/1.1 | 306 | | `ssl_verify` | `bool` | 否 | **SSL 证书验证** (默认 `True` 启用验证，设为 `False` 可禁用以支持自签名证书) | 307 | | `timeout` | `float` | ✅ | 单个请求的超时时间（秒），默认 30 秒。 | 308 | 309 | --- 310 | 311 | ### `ProxyConfig` 参数 312 | 313 | | 字段 | 类型 | 必填 | 描述 | 314 | |:------------|:---------------------|:--------|:--------------------------------------------------------------------| 315 | | `http` | `str` / `None` | 否 | HTTP 请求使用的代理地址（例如：`http://127.0.0.1:8080`）。 | 316 | | `https` | `str` / `None` | 否 | HTTPS 请求使用的代理地址。 | 317 | | `all` | `str` / `None` | 否 | 同时应用于所有协议的代理地址，会覆盖 `http` 和 `https`。 | 318 | | `no_proxy` | `List[str]` / `None` | 否 | 不使用代理的主机名或 IP 列表。 | 319 | | `username` | `str` / `None` | 否 | 可选的代理认证用户名。 | 320 | | `password` | `str` / `None` | 否 | 可选的代理认证密码。 | 321 | | `trust_env` | `bool` / `None` | 否 | 是否信任系统环境变量中的代理配置（如 `HTTP_PROXY`、`NO_PROXY`）。 | 322 | 323 | --- 324 | 325 | ### `fetch_requests` 参数 326 | 327 | | 字段 | 类型 | 必填 | 描述 | 328 | | :--------------- | :-------------------- | :--: | :--------------------------------------------------------------------------------------- | 329 | | `requests` | `List[RequestItem]` | ✅ | 待并发执行的 `RequestItem` 列表。 | 330 | | `total_timeout` | `float` | 否 | 整个批量请求的全局超时时间（秒）。 | 331 | | `mode` | `ConcurrencyMode` | 否 | 并发策略。`SELECT_ALL`（默认）为尽力收集模式，`JOIN_ALL` 为原子执行模式（全有或全无）。详见第 3 节。 | 332 | 333 | --- 334 | 335 | ### `fetch_single` 参数 336 | 337 | | 字段 | 类型 | 必填 | 描述 | 338 | |:--------------|:--------------------|:--------|:------------------------------------------------------------------------------------------------------------| 339 | | `url` | `str` | ✅ | 目标请求的 URL。 | 340 | | `method` | `str` / `None` | 否 | HTTP 请求方法，例如 `"GET"`、`"POST"`，默认可由客户端自行处理。 | 341 | | `params` | `dict` / `None` | 否 | 请求参数。对于 GET/DELETE 请求，会被转换为 URL 查询参数；对于 POST/PUT/PATCH 请求，会作为 JSON body 发送。 | 342 | | `timeout` | `float` / `None` | 否 | 当前请求的超时时间（秒），默认值可为 30 秒。 | 343 | | `headers` | `dict` / `None` | 否 | 自定义 HTTP 请求头。 | 344 | | `tag` | `str` / `None` | 否 | 任意标签，用于标识或索引请求响应。 | 345 | | `proxy` | `ProxyConfig` / `None` | 否 | 可选代理配置，若提供则应用于此请求。 | 346 | | `http_version`| `HttpVersion` / `None` | 否 | HTTP 版本选择，通常支持 `"Auto"`（尝试 HTTP/2，失败回退 HTTP/1.1）、`"1.1"`、`"2"` 等。 | 347 | | `ssl_verify` | `bool` / `None` | 否 | 是否验证 SSL 证书，默认 `True`，若为 `False` 则忽略自签名证书验证。 | 348 | 349 | --- 350 | 351 | ### 响应字典格式 352 | 353 | `fetch_single` 和 `fetch_requests` 返回的结果都为字典（或字典列表），结构统一。 354 | 355 | #### 成功响应示例： 356 | 357 | ```json 358 | { 359 | "http_status": 200, 360 | "response": { 361 | "headers": { 362 | "access-control-allow-credentials": "true", 363 | "access-control-allow-origin": "*", 364 | "connection": "keep-alive", 365 | "content-length": "314", 366 | "content-type": "application/json", 367 | "date": "Wed, 10 Sep 2025 03:15:31 GMT", 368 | "server": "gunicorn/19.9.0" 369 | }, 370 | "content": "{\"data\":\"...\", \"headers\":{\"...\"}}" 371 | }, 372 | "meta": { 373 | "process_time": "2.0846", 374 | "request_time": "2025-09-10 11:22:46 -> 2025-09-10 11:22:48", 375 | "tag": "req-0" 376 | }, 377 | "exception": {} 378 | } 379 | ``` 380 | 381 | #### 失败响应示例（例如超时）： 382 | ```json 383 | { 384 | "http_status": 0, 385 | "response": { 386 | "headers": { 387 | "access-control-allow-credentials": "true", 388 | "access-control-allow-origin": "*", 389 | "connection": "keep-alive", 390 | "content-length": "314", 391 | "content-type": "application/json", 392 | "date": "Wed, 10 Sep 2025 03:15:31 GMT", 393 | "server": "gunicorn/19.9.0" 394 | }, 395 | "content": "" 396 | }, 397 | "meta": { 398 | "process_time": "3.0012", 399 | "request_time": "2025-08-08 03:15:05 -> 2025-08-08 03:15:08", 400 | "tag": "test-req-50" 401 | }, 402 | "exception": { 403 | "type": "Timeout", 404 | "message": "Request timeout after 3.00 seconds" 405 | } 406 | } 407 | ``` 408 | 409 | --- 410 | 411 | ## 更新日志 412 | 413 | 查看详细更新内容请访问 [CHANGELOG](CHANGELOG.md) 414 | 415 | ## Star History 416 | 417 | [![Star History Chart](https://api.star-history.com/svg?repos=KAY53N/rusty-req&type=Date)](https://www.star-history.com/#KAY53N/rusty-req&Date) 418 | 419 | 420 | ## 📄 许可证 421 | 本项目采用 [MIT License](https://opensource.org/license/MIT). 422 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rusty-req 2 | 3 | [![PyPI version](https://img.shields.io/pypi/v/rusty-req)](https://pypi.org/project/rusty-req/) 4 | [![PyPI downloads](https://img.shields.io/pypi/dm/rusty-req)](https://pypi.org/project/rusty-req/) 5 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) 6 | [![Python versions](https://img.shields.io/badge/python-3.9%2B-blue)](https://www.python.org/downloads/) 7 | [![GitHub issues](https://img.shields.io/github/issues/KAY53N/rusty-req)](https://github.com/KAY53N/rusty-req/issues) 8 | [![Build Status](https://github.com/KAY53N/rusty-req/actions/workflows/build.yml/badge.svg)](https://github.com/KAY53N/rusty-req/actions/workflows/build.yml) 9 | [![Cross Platform Test](https://github.com/KAY53N/rusty-req/actions/workflows/cross-platform-test.yml/badge.svg)](https://github.com/KAY53N/rusty-req/actions/workflows/cross-platform-test.yml) 10 | 11 | A high-performance asynchronous request library based on Rust and Python, suitable for scenarios that require high-throughput concurrent HTTP requests. It implements the core concurrent logic in Rust and packages it into a Python module using [PyO3](https://pyo3.rs/) and [maturin](https://github.com/PyO3/maturin), combining Rust's performance with Python's ease of use. 12 | 13 | --- 14 | 15 | ### 🌐 [English](README.md) | [中文](README.zh.md) 16 | 17 | ## 🚀 Features 18 | 19 | - **Dual Request Modes**: Supports both batch concurrent requests (`fetch_requests`) and single asynchronous requests (`fetch_single`). 20 | - **High Performance**: Built with Rust, Tokio, and a shared `reqwest` client for maximum throughput. 21 | - **Highly Customizable**: Allows custom headers, parameters/body, per-request timeouts, and tags. 22 | - **Flexible Concurrency Modes**: Choose between `SELECT_ALL` (default, get results as they complete) and `JOIN_ALL` (wait for all requests to finish) to fit your use case. 23 | - **Smart Response Handling**: Automatically decompresses `gzip`, `brotli`, and `deflate` encoded responses. 24 | - **Global Timeout Control**: Use `total_timeout` in batch requests to prevent hangs. 25 | - **Detailed Results**: Each response includes the HTTP status, body, metadata (like processing time), and any exceptions. 26 | - **Debug Mode**: An optional debug mode (`set_debug(True)`) prints detailed request/response information. 27 | 28 | ## 🔧 Installation 29 | 30 | ```bash 31 | pip install rusty-req 32 | ``` 33 | Or build from source: 34 | ``` 35 | # This will compile the Rust code and create a .whl file 36 | maturin build --release 37 | 38 | # Install from the generated wheel 39 | pip install target/wheels/rusty_req-*.whl 40 | ``` 41 | 42 | ## Development & Debugging 43 | ``` 44 | cargo watch -s "maturin develop" 45 | ``` 46 | 47 | ## ⚙️ Proxy Configuration & Debug 48 | 49 | ### 1. Using Proxy 50 | 51 | If you need to access external networks through a proxy, create a `ProxyConfig` object and set it as a global proxy: 52 | 53 | ```python 54 | import asyncio 55 | import rusty_req 56 | 57 | async def proxy_example(): 58 | # Create ProxyConfig object 59 | proxy = rusty_req.ProxyConfig( 60 | http="http://127.0.0.1:7890", 61 | https="http://127.0.0.1:7890" 62 | ) 63 | 64 | # Set global proxy (all requests will use this proxy) 65 | await rusty_req.set_global_proxy(proxy) 66 | 67 | # Send request (will go through proxy automatically) 68 | resp = await rusty_req.fetch_single(url="https://httpbin.org/get") 69 | print(resp) 70 | 71 | if __name__ == "__main__": 72 | asyncio.run(proxy_example()) 73 | ``` 74 | 75 | ### 2. Debug Logging 76 | 77 | `set_debug` enables debug mode, supporting **console output** and **log file writing**: 78 | 79 | ```python 80 | import rusty_req 81 | 82 | # Print debug logs to console only 83 | rusty_req.set_debug(True) 84 | 85 | # Print to console and write to log file 86 | rusty_req.set_debug(True, "logs/debug.log") 87 | 88 | # Disable debug mode 89 | rusty_req.set_debug(False) 90 | ``` 91 | 92 | ## 📦 Example Usage 93 | ### 1. Fetching a Single Request (`fetch_single`) 94 | Perfect for making a single asynchronous call and awaiting its result. 95 | 96 | ```python 97 | import asyncio 98 | import pprint 99 | import rusty_req 100 | 101 | async def single_request_example(): 102 | """Demonstrates how to use fetch_single for a POST request.""" 103 | print("🚀 Fetching a single POST request to httpbin.org...") 104 | 105 | # Enable debug mode to see detailed logs in the console 106 | rusty_req.set_debug(True) 107 | 108 | response = await rusty_req.fetch_single( 109 | url="https://httpbin.org/post", 110 | method="POST", 111 | params={"user_id": 123, "source": "example"}, 112 | headers={"X-Client-Version": "1.0"}, 113 | tag="my-single-post" 114 | ) 115 | 116 | print("\n✅ Request finished. Response:") 117 | pprint.pprint(response) 118 | 119 | if __name__ == "__main__": 120 | asyncio.run(single_request_example()) 121 | ``` 122 | 123 | ### 2. Fetching Batch Requests (`fetch_requests`) 124 | 125 | The core feature for handling a large number of requests concurrently. This example simulates a simple load test. 126 | ```python 127 | import asyncio 128 | import time 129 | import rusty_req 130 | from rusty_req import ConcurrencyMode 131 | 132 | async def batch_requests_example(): 133 | """Demonstrates 100 concurrent requests with a global timeout.""" 134 | requests = [ 135 | rusty_req.RequestItem( 136 | url="https://httpbin.org/delay/2", # This endpoint waits 2 seconds 137 | method="GET", 138 | timeout=2.9, # Per-request timeout, should succeed 139 | tag=f"test-req-{i}", 140 | ) 141 | for i in range(100) 142 | ] 143 | 144 | # Disable debug logs for cleaner output 145 | rusty_req.set_debug(False) 146 | 147 | print("🚀 Starting 100 concurrent requests...") 148 | start_time = time.perf_counter() 149 | 150 | # Set a global timeout of 3.0 seconds. Some requests will be cut off. 151 | responses = await rusty_req.fetch_requests( 152 | requests, 153 | total_timeout=3.0, 154 | mode=ConcurrencyMode.SELECT_ALL # Explicitly use SELECT_ALL mode 155 | ) 156 | 157 | total_time = time.perf_counter() - start_time 158 | 159 | # --- Process results --- 160 | success_count = 0 161 | failed_count = 0 162 | for r in responses: 163 | # Check the 'exception' field to see if the request was successful 164 | if r.get("exception") and r["exception"].get("type"): 165 | failed_count += 1 166 | else: 167 | success_count += 1 168 | 169 | print("\n📊 Load Test Summary:") 170 | print(f"⏱️ Total time taken: {total_time:.2f}s") 171 | print(f"✅ Successful requests: {success_count}") 172 | print(f"⚠️ Failed or timed-out requests: {failed_count}") 173 | 174 | if __name__ == "__main__": 175 | asyncio.run(batch_requests_example()) 176 | ``` 177 | 178 | ### 3. Understanding Concurrency Modes (`SELECT_ALL` vs `JOIN_ALL`) 179 | 180 | The `fetch_requests` function supports two powerful concurrency strategies. Choosing the right one is key to building robust applications. 181 | 182 | - **`ConcurrencyMode.SELECT_ALL` (Default): Best-Effort Collector** 183 | This mode operates on a "first come, first served" or "best-effort" basis. It aims to collect as many successful results as possible within the given `total_timeout`. 184 | - It returns results as soon as they complete. 185 | - If the `total_timeout` is reached, it gracefully returns all the requests that have already succeeded, while marking any still-pending requests as timed out. 186 | - **A failure in one request does not affect others.** 187 | 188 | - **`ConcurrencyMode.JOIN_ALL`: Transactional (All-or-Nothing)** 189 | This mode treats the entire batch of requests as a single, atomic transaction. It is much stricter. 190 | - It waits for **all** submitted requests to complete first. 191 | - It then inspects the results. 192 | - **Success Case**: Only if *every single request was successful* will it return the complete list of successful results. 193 | - **Failure Case**: If *even one request fails* for any reason (e.g., its individual timeout, a network error, or a non-2xx status code), this mode will discard all results and return a list where **every request is marked as a global failure.** 194 | 195 | ### 4. Timeout Performance Comparison 196 | 197 | Under the same test conditions (global timeout 3s, per-request timeout 2.6s, httpbin delay 2.3s), we compared the performance of different libraries: 198 | 199 | | Library / Framework | Total Requests | Successful | Timed Out | Success Rate | Actual Total Time | Notes / Description | 200 | |--------------------|----------------|------------|-----------|--------------|-------------------|-------------------| 201 | | **Rusty-req** | 1000 | 1000 | 0 | 100.0% | 2.56s | Stable performance under high concurrency; precise control of per-request and total timeouts | 202 | | **httpx** | 1000 | 0 | 0 | 0.0% | 26.77s | Timeout parameters did not take effect; overall performance abnormal | 203 | | **aiohttp** | 1000 | 100 | 900 | 10.0% | 2.66s | Per-request timeout effective, but global timeout control insufficient | 204 | | **requests** | 1000 | 1000 | 0 | 100.0% | 3.45s | Synchronous blocking mode; not suitable for large-scale concurrent requests | 205 | 206 | Key takeaways: 207 | - **Rusty-req** can complete tasks within strict global timeout limits while maintaining high concurrency and stability. 208 | - Traditional asynchronous libraries struggle with global timeout enforcement and extreme concurrency scenarios. 209 | - Synchronous libraries like `requests` produce correct results but are not scalable for large-scale concurrent requests. 210 | 211 | ![Timeout Performance Comparison](https://raw.githubusercontent.com/KAY53N/rusty-req/main/docs/images/timeout_performance_comparison.png) 212 | 213 | --- 214 | 215 | ### Quick Comparison 216 | 217 | | Aspect | `ConcurrencyMode.SELECT_ALL` (Default) | `ConcurrencyMode.JOIN_ALL` | 218 | | :-------------------- | :------------------------------------------------------------------- | :------------------------------------------------------------------ | 219 | | **Failure Handling** | **Tolerant**. One failure does not affect other successful requests. | **Strict / Atomic**. One failure causes the entire batch to fail. | 220 | | **Primary Use Case** | Maximizing throughput; getting as much data as possible. | Tasks that must succeed or fail as a single unit (e.g., transactions). | 221 | | **Result Order** | By completion time (fastest first). | By original submission order. | 222 | | **"When do I get results?"** | As they complete, one by one. | All at once, only after every request has finished and been validated. | 223 | 224 | --- 225 | 226 | ### Code Example 227 | 228 | The example below clearly demonstrates the difference in behavior. 229 | 230 | ```python 231 | import asyncio 232 | import rusty_req 233 | from rusty_req import ConcurrencyMode 234 | 235 | async def concurrency_modes_example(): 236 | """Demonstrates the difference between SELECT_ALL and JOIN_ALL modes.""" 237 | # Note: We are using an endpoint that returns 500 to force a failure. 238 | requests = [ 239 | rusty_req.RequestItem(url="https://httpbin.org/delay/2", tag="should_succeed"), 240 | rusty_req.RequestItem(url="https://httpbin.org/status/500", tag="will_fail"), 241 | rusty_req.RequestItem(url="https://httpbin.org/delay/1", tag="should_also_succeed"), 242 | ] 243 | 244 | # --- 1. Test SELECT_ALL --- 245 | print("--- 🚀 Testing SELECT_ALL (Best-Effort) ---") 246 | results_select = await rusty_req.fetch_requests( 247 | requests, 248 | mode=ConcurrencyMode.SELECT_ALL, 249 | total_timeout=3.0 250 | ) 251 | 252 | print("Results:") 253 | for res in results_select: 254 | tag = res.get("meta", {}).get("tag") 255 | status = res.get("http_status") 256 | err_type = res.get("exception", {}).get("type") 257 | print(f" - Tag: {tag}, Status: {status}, Exception: {err_type}") 258 | 259 | print("\n" + "="*50 + "\n") 260 | 261 | # --- 2. Test JOIN_ALL --- 262 | print("--- 🚀 Testing JOIN_ALL (All-or-Nothing) ---") 263 | results_join = await rusty_req.fetch_requests( 264 | requests, 265 | mode=ConcurrencyMode.JOIN_ALL, 266 | total_timeout=3.0 267 | ) 268 | 269 | print("Results:") 270 | for res in results_join: 271 | tag = res.get("meta", {}).get("tag") 272 | status = res.get("http_status") 273 | err_type = res.get("exception", {}).get("type") 274 | print(f" - Tag: {tag}, Status: {status}, Exception: {err_type}") 275 | 276 | if __name__ == "__main__": 277 | asyncio.run(concurrency_modes_example()) 278 | ``` 279 | The expected output from the script above: 280 | 281 | ``` 282 | --- 🚀 Testing SELECT_ALL (Best-Effort) --- 283 | Results: 284 | - Tag: should_also_succeed, Status: 200, Exception: None 285 | - Tag: will_fail, Status: 500, Exception: HttpStatusError 286 | - Tag: should_succeed, Status: 200, Exception: None 287 | 288 | ================================================== 289 | 290 | --- 🚀 Testing JOIN_ALL (All-or-Nothing) --- 291 | Results: 292 | - Tag: should_succeed, Status: 0, Exception: GlobalTimeout 293 | - Tag: will_fail, Status: 0, Exception: GlobalTimeout 294 | - Tag: should_also_succeed, Status: 0, Exception: GlobalTimeout 295 | ``` 296 | 297 | ## 🧱 Data Structures 298 | 299 | ### `RequestItem` Parameters 300 | 301 | | Field | Type | Required | Description | 302 | |:------------------|:----------------|:--------:|:-----------------------------------------------------------------------------------------------------------------------------------------------| 303 | | `url` | `str` | ✅ | The target URL. | 304 | | `method` | `str` | ✅ | The HTTP method. | 305 | | `params` | `dict` / `None` | No | For GET/DELETE, converted to URL query parameters. For POST/PUT/PATCH, sent as a JSON body. | 306 | | `headers` | `dict` / `None` | No | Custom HTTP headers. | 307 | | `tag` | `str` | No | An arbitrary tag to help identify or index the response. | 308 | | `http_version` | `str` | No | The default behavior when the HTTP version is set to “Auto” is to attempt HTTP/2 first, and fall back to HTTP/1.1 if HTTP/2 is not supported. | 309 | | `ssl_verify` | `bool` | No | **SSL certificate verification** (default `True`, set `False` to disable for self-signed certificates) | 310 | | `timeout` | `float` | ✅ | Timeout for this individual request in seconds. Defaults to 30s. | 311 | 312 | --- 313 | 314 | ### `ProxyConfig` Parameters 315 | 316 | | Field | Type | Required | Description | 317 | |:------------|:---------------------|:--------:|:----------------------------------------------------------------------------| 318 | | `http` | `str` / `None` | No | Proxy URL for HTTP requests (e.g. `http://127.0.0.1:8080`). | 319 | | `https` | `str` / `None` | No | Proxy URL for HTTPS requests. | 320 | | `all` | `str` / `None` | No | A single proxy URL applied to all schemes (overrides `http`/`https`). | 321 | | `no_proxy` | `List[str]` / `None` | No | List of hostnames/IPs to exclude from proxying. | 322 | | `username` | `str` / `None` | No | Optional proxy authentication username. | 323 | | `password` | `str` / `None` | No | Optional proxy authentication password. | 324 | | `trust_env` | `bool` / `None` | No | Whether to respect system environment variables (`HTTP_PROXY`, `NO_PROXY`). | 325 | 326 | --- 327 | 328 | ### `fetch_requests` Parameters 329 | 330 | | Field | Type | Required | Description | 331 | | :-------------- | :-------------------- | :------: | :------------------------------------------------------------------------------------------------------ | 332 | | `requests` | `List[RequestItem]` | ✅ | A list of `RequestItem` objects to be executed concurrently. | 333 | | `total_timeout` | `float` | No | A global timeout in seconds for the entire batch operation. | 334 | | `mode` | `ConcurrencyMode` | No | The concurrency strategy. `SELECT_ALL` (default) for best-effort collection. `JOIN_ALL` for atomic (all-or-nothing) execution. See Section 3 for a detailed comparison.| 335 | 336 | --- 337 | 338 | ### `fetch_single` Parameters 339 | 340 | | Field | Type | Required | Description | 341 | |:--------------|:--------------------|:--------:|:----------------------------------------------------------------------------------------------------------------| 342 | | `url` | `str` | ✅ | The target request URL. | 343 | | `method` | `str` / `None` | No | HTTP method, e.g., `"GET"`, `"POST"`. If not provided, the client may handle defaults. | 344 | | `params` | `dict` / `None` | No | Request parameters. For GET/DELETE, converted to URL query parameters; for POST/PUT/PATCH, sent as JSON body. | 345 | | `timeout` | `float` / `None` | No | Timeout for this request in seconds. Defaults to 30s. | 346 | | `headers` | `dict` / `None` | No | Custom HTTP request headers. | 347 | | `tag` | `str` / `None` | No | Arbitrary tag to help identify or index the response. | 348 | | `proxy` | `ProxyConfig` / `None` | No | Optional proxy configuration. Applied to this request if provided. | 349 | | `http_version`| `HttpVersion` / `None` | No | HTTP version choice, usually supports `"Auto"` (try HTTP/2, fallback to HTTP/1.1), `"1.1"`, `"2"`, etc. | 350 | | `ssl_verify` | `bool` / `None` | No | Whether to verify SSL certificates. Defaults to `True`; set `False` to ignore self-signed certificates. | 351 | 352 | --- 353 | 354 | ### Response Dictionary Format 355 | 356 | Both `fetch_single` and `fetch_requests` return a dictionary (or a list of dictionaries) with a consistent structure. 357 | 358 | #### Example of a successful response: 359 | 360 | ```json 361 | { 362 | "http_status": 200, 363 | "response": { 364 | "headers": { 365 | "access-control-allow-credentials": "true", 366 | "access-control-allow-origin": "*", 367 | "connection": "keep-alive", 368 | "content-length": "314", 369 | "content-type": "application/json", 370 | "date": "Wed, 10 Sep 2025 03:15:31 GMT", 371 | "server": "gunicorn/19.9.0" 372 | }, 373 | "content": "{\"data\":\"...\", \"headers\":{\"...\"}}" 374 | }, 375 | "meta": { 376 | "process_time": "2.0846", 377 | "request_time": "2025-09-10 11:22:46 -> 2025-09-10 11:22:48", 378 | "tag": "req-0" 379 | }, 380 | "exception": {} 381 | } 382 | ``` 383 | 384 | #### Example of a failed response (e.g., timeout): 385 | ```json 386 | { 387 | "http_status": 0, 388 | "response": { 389 | "headers": { 390 | "access-control-allow-credentials": "true", 391 | "access-control-allow-origin": "*", 392 | "connection": "keep-alive", 393 | "content-length": "314", 394 | "content-type": "application/json", 395 | "date": "Wed, 10 Sep 2025 03:15:31 GMT", 396 | "server": "gunicorn/19.9.0" 397 | }, 398 | "content": "" 399 | }, 400 | "meta": { 401 | "process_time": "3.0012", 402 | "request_time": "2025-08-08 03:15:05 -> 2025-08-08 03:15:08", 403 | "tag": "test-req-50" 404 | }, 405 | "exception": { 406 | "type": "Timeout", 407 | "message": "Request timeout after 3.00 seconds" 408 | } 409 | } 410 | ``` 411 | 412 | ## Changelog 413 | 414 | For a detailed list of changes, see the [CHANGELOG](CHANGELOG.md) 415 | 416 | --- 417 | 418 | ## Star History 419 | 420 | [![Star History Chart](https://api.star-history.com/svg?repos=KAY53N/rusty-req&type=Date)](https://www.star-history.com/#KAY53N/rusty-req&Date) 421 | 422 | ## 📄 License 423 | This project is licensed under the [MIT License](https://opensource.org/license/MIT). 424 | --------------------------------------------------------------------------------