├── .github └── workflows │ └── cronet.yml ├── .gitignore ├── .pre-commit-config.yaml ├── LICENSE ├── README.md ├── TODO ├── chrome_release.py ├── libcronet ├── build.sh ├── build_wheels.sh ├── install_deps.sh └── proxy_support_81a06fb873a9b386848719cf9f93e59579fb5d4b.patch ├── pyproject.toml ├── setup.py ├── src └── cronet │ ├── __init__.py │ ├── _cronet.c │ └── cronet.py └── tests ├── __init__.py ├── conftest.py ├── server.py └── test_cronet.py /.github/workflows/cronet.yml: -------------------------------------------------------------------------------- 1 | name: Build linux 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build_wheels: 7 | name: Build wheels for linux 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Maximize build space 11 | uses: easimon/maximize-build-space@master 12 | with: 13 | root-reserve-mb: 10000 14 | swap-size-mb: 1024 15 | remove-dotnet: 'true' 16 | - name: Checkout 17 | uses: actions/checkout@v4 18 | - name: Cache Cronet 19 | id: cache-cronet 20 | uses: actions/cache@v4 21 | with: 22 | path: cronet_build 23 | key: ${{ runner.os }}-cronet2 24 | - name: Build Cronet 25 | if: steps.cache-cronet.outputs.cache-hit != 'true' 26 | run: docker run -t -e PLAT=manylinux_2_28_x86_64 -e CHROMIUM=134.0.6998.165 -e VERSION=0.1.7 -v `pwd`:/app quay.io/pypa/manylinux_2_28_x86_64 /app/libcronet/build.sh 27 | - name: Build wheels 28 | run: docker run -t -e PLAT=manylinux_2_28_x86_64 -e CHROMIUM=134.0.6998.165 -e VERSION=0.1.7 -v `pwd`:/app quay.io/pypa/manylinux_2_28_x86_64 /app/libcronet/build_wheels.sh 29 | - name: Upload artifact 30 | uses: actions/upload-artifact@v4 31 | with: 32 | name: wheels 33 | path: ./wheelhouse/*.whl 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # poetry 98 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 102 | #poetry.lock 103 | 104 | # pdm 105 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 106 | #pdm.lock 107 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 108 | # in version control. 109 | # https://pdm.fming.dev/#use-with-ide 110 | .pdm.toml 111 | 112 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 113 | __pypackages__/ 114 | 115 | # Celery stuff 116 | celerybeat-schedule 117 | celerybeat.pid 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | env.bak/ 129 | venv.bak/ 130 | 131 | # Spyder project settings 132 | .spyderproject 133 | .spyproject 134 | 135 | # Rope project settings 136 | .ropeproject 137 | 138 | # mkdocs documentation 139 | /site 140 | 141 | # mypy 142 | .mypy_cache/ 143 | .dmypy.json 144 | dmypy.json 145 | 146 | # Pyre type checker 147 | .pyre/ 148 | 149 | # pytype static type analyzer 150 | .pytype/ 151 | 152 | # Cython debug symbols 153 | cython_debug/ 154 | 155 | # PyCharm 156 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 157 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 158 | # and can be added to the global gitignore or merged into this file. For a more nuclear 159 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 160 | #.idea/ 161 | 162 | .vscode/ 163 | dev_build.sh 164 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/pre-commit/pre-commit-hooks 3 | rev: v3.2.0 4 | hooks: 5 | - id: trailing-whitespace 6 | - id: end-of-file-fixer 7 | - id: check-yaml 8 | - id: check-added-large-files 9 | - repo: https://github.com/astral-sh/ruff-pre-commit 10 | rev: v0.3.3 11 | hooks: 12 | - id: ruff 13 | args: [ '--fix', '--select', 'I'] 14 | - id: ruff-format 15 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 Lucas Moauro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # python-cronet 2 | 3 | 4 | `python-cronet` is a library to use Chromium's network stack from Python. 5 | 6 | **The library is currently in alpha stage.** 7 | 8 | # What is Cronet 9 | 10 | *[Cronet](https://chromium.googlesource.com/chromium/src/+/master/components/cronet/) is the networking stack of Chromium put into a library for use on Android. 11 | It offers an easy-to-use, high performance, standards-compliant, and secure way to perform HTTP requests.* 12 | 13 | The Chromium team also provides a native version of the library(not officially supported) which allows you to use 14 | it in desktop/server operating systems like Linux, macOS and Windows. 15 | 16 | The main benefits of using cronet as an HTTP client are: 17 | - You get to use the same high quality code that runs on Chromium. 18 | - Support for the latest protocols like QUIC and compression formats. 19 | - Concurrency support by performing asynchronous requests. 20 | - Has the same TLS fingerprint as Chrome, meaning that Cloudflare and other bot detection systems can't block your requests based on it. 21 | - It's much more lightweight on system resources compared to headless Chrome(although it doesn't support executing javascript). 22 | 23 | ## Installation 24 | 25 | **For the time being the only supported platform is linux-x86-64. The plan is to also support windows and macOS.** 26 | 27 | `pip install python-cronet` 28 | 29 | # Example usage 30 | 31 | The library provides an asynchronous API: 32 | 33 | ```!python 34 | import asyncio 35 | import cronet 36 | 37 | async def main(): 38 | with cronet.Cronet() as cr: 39 | # GET request 40 | response = await cr.get("https://httpbin.org/get") 41 | print(f"GET request: {response.url}, {response.status_code}") 42 | print(f"Response JSON: {response.json()}") 43 | 44 | # GET request with query parameters 45 | params = {"param1": "value1", "param2": "value2"} 46 | response = await cr.get("https://httpbin.org/get", params=params) 47 | print(f"GET request with params: {response.url}, {response.status_code}") 48 | print(f"Response JSON: {response.json()}") 49 | 50 | # POST request with form data 51 | data = {"key1": "value1", "key2": "value2"} 52 | response = await cr.post("https://httpbin.org/post", data=data) 53 | print(f"POST request: {response.url}, {response.status_code}") 54 | print(f"Response JSON: {response.json()}") 55 | 56 | # PUT request with JSON data 57 | json_data = {"key1": "value1", "key2": "value2"} 58 | response = await cr.put("https://httpbin.org/put", json=json_data) 59 | print(f"PUT request: {response.url}, {response.status_code}") 60 | print(f"Response JSON: {response.json()}") 61 | 62 | # PATCH request with custom headers 63 | headers = {"X-Custom-Header": "MyValue"} 64 | response = await cr.patch("https://httpbin.org/patch", headers=headers) 65 | print(f"PATCH request: {response.url}, {response.status_code}") 66 | print(f"Response JSON: {response.json()}") 67 | 68 | # DELETE request 69 | response = await cr.delete("https://httpbin.org/delete") 70 | print(f"DELETE request: {response.url}, {response.status_code}") 71 | print(f"Response JSON: {response.json()}") 72 | 73 | 74 | asyncio.run(main()) 75 | ``` 76 | 77 | -------------------------------------------------------------------------------- /TODO: -------------------------------------------------------------------------------- 1 | - Proxy support 2 | - Run tests in gh 3 | - Smart decoding of responses 4 | - Stealth mode? 5 | - Better error handling in c module 6 | - Automate release 7 | - Windows build 8 | - macOS build 9 | -------------------------------------------------------------------------------- /chrome_release.py: -------------------------------------------------------------------------------- 1 | import json 2 | import urllib.request 3 | 4 | url = "https://chromiumdash.appspot.com/fetch_releases?channel=Stable&platform=&num=1&offset=0" 5 | response = urllib.request.urlopen(url) 6 | data = response.read().decode("utf-8") 7 | 8 | json_data = json.loads(data) 9 | 10 | for release in json_data: 11 | platform = release.get("platform", "Unknown") 12 | version = release.get("version", "Unknown") 13 | print(f"Platform: {platform}, Version: {version}") 14 | -------------------------------------------------------------------------------- /libcronet/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | cd /app 6 | mkdir -p out 7 | rm -rf out/* 8 | cd out 9 | 10 | git clone --depth 1 https://chromium.googlesource.com/chromium/tools/depot_tools.git 11 | export PATH="$(pwd)/depot_tools:$PATH" 12 | git clone -b $CHROMIUM --depth=2 https://chromium.googlesource.com/chromium/src 13 | 14 | cd src 15 | git apply /app/libcronet/proxy_support_*.patch 16 | cd .. 17 | echo 'solutions = [ 18 | { 19 | "name": "src", 20 | "url": "https://chromium.googlesource.com/chromium/src.git", 21 | "managed": False, 22 | "custom_deps": {}, 23 | "custom_vars": {}, 24 | }, 25 | ]' > .gclient 26 | 27 | gclient sync --no-history --nohooks 28 | 29 | pipx install pip 30 | pip3 install psutil 31 | 32 | /app/libcronet/install_deps.sh 33 | 34 | gclient runhooks 35 | 36 | cd src/components/cronet 37 | gn gen out/Cronet/ --args='target_os="linux" is_debug=false is_component_build=false' 38 | ninja -C out/Cronet cronet_package 39 | 40 | cp -r out/Cronet/cronet/ /app/cronet_build 41 | 42 | cd /app 43 | rm -rf out 44 | -------------------------------------------------------------------------------- /libcronet/build_wheels.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | /app/libcronet/install_deps.sh 6 | 7 | cd /app 8 | mv /app/cronet_build /tmp 9 | cp /tmp/cronet_build/libcronet*so /lib64 10 | cp /tmp/cronet_build/include/*.h /usr/local/include 11 | 12 | 13 | function repair_wheel() { 14 | local python_version="$1" 15 | local wheel_version="$2" 16 | 17 | python${python_version} -m pip uninstall --no-input auditwheel 18 | python${python_version} -m pip install git+https://github.com/lagenar/auditwheel.git 19 | python${python_version} -m build 20 | python${python_version} -m auditwheel repair --plat manylinux_2_28_x86_64 "dist/python_cronet-${VERSION}-cp${wheel_version}-cp${wheel_version}-linux_x86_64.whl" 21 | } 22 | 23 | 24 | repair_wheel "3.8" "38" 25 | repair_wheel "3.9" "39" 26 | repair_wheel "3.10" "310" 27 | repair_wheel "3.11" "311" 28 | repair_wheel "3.12" "312" 29 | repair_wheel "3.13" "313" 30 | 31 | 32 | mv /tmp/cronet_build /app 33 | -------------------------------------------------------------------------------- /libcronet/install_deps.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | yum -y install git bzip2 tar pkgconfig atk-devel alsa-lib-devel \ 4 | bison binutils brlapi-devel bluez-libs-devel bzip2-devel cairo-devel \ 5 | cups-devel dbus-devel dbus-glib-devel expat-devel fontconfig-devel \ 6 | freetype-devel gcc-c++ glib2-devel glibc.i686 gperf glib2-devel \ 7 | gtk3-devel java-1.*.0-openjdk-devel libatomic libcap-devel libffi-devel \ 8 | libgcc.i686 libjpeg-devel libstdc++.i686 libX11-devel libXScrnSaver-devel \ 9 | libXtst-devel libxkbcommon-x11-devel ncurses-compat-libs nspr-devel nss-devel \ 10 | pam-devel pango-devel pciutils-devel pulseaudio-libs-devel zlib.i686 httpd \ 11 | mod_ssl php php-cli xorg-x11-server-Xvfb 12 | -------------------------------------------------------------------------------- /libcronet/proxy_support_81a06fb873a9b386848719cf9f93e59579fb5d4b.patch: -------------------------------------------------------------------------------- 1 | diff --git a/components/cronet/cronet_context.cc b/components/cronet/cronet_context.cc 2 | index 7fa99a4a88..ddaed57319 100644 3 | --- a/components/cronet/cronet_context.cc 4 | +++ b/components/cronet/cronet_context.cc 5 | @@ -15,6 +15,7 @@ 6 | #include 7 | 8 | #include "base/base64.h" 9 | +#include "base/command_line.h" 10 | #include "base/files/file_path.h" 11 | #include "base/files/file_util.h" 12 | #include "base/files/scoped_file.h" 13 | @@ -63,6 +64,8 @@ 14 | #include "net/url_request/url_request_context_builder.h" 15 | #include "net/url_request/url_request_context_getter.h" 16 | #include "net/url_request/url_request_interceptor.h" 17 | +#include "net/proxy_resolution/proxy_config_service.h" 18 | +#include "net/proxy_resolution/proxy_config.h" 19 | 20 | #if BUILDFLAG(ENABLE_REPORTING) 21 | #include "net/network_error_logging/network_error_logging_service.h" 22 | @@ -206,6 +209,7 @@ CronetContext::CronetContext( 23 | (context_config->load_disable_cache ? net::LOAD_DISABLE_CACHE : 0) | 24 | (context_config->enable_brotli ? net::LOAD_CAN_USE_SHARED_DICTIONARY 25 | : 0)), 26 | + proxy_rules_(context_config->proxy_rules), 27 | network_tasks_( 28 | new NetworkTasks(std::move(context_config), std::move(callback))), 29 | network_task_runner_(network_task_runner) { 30 | @@ -251,12 +255,42 @@ CronetContext::NetworkTasks::~NetworkTasks() { 31 | net::NetworkChangeNotifier::RemoveNetworkObserver(this); 32 | } 33 | 34 | + 35 | +class ProxyConfigServiceCustom : public net::ProxyConfigService { 36 | + public: 37 | + ProxyConfigServiceCustom(const std::string& proxy_rules):proxy_rules_(proxy_rules) {} 38 | + void AddObserver(Observer* observer) override {} 39 | + void RemoveObserver(Observer* observer) override {} 40 | + ConfigAvailability GetLatestProxyConfig( 41 | + net::ProxyConfigWithAnnotation* config) override { 42 | + 43 | + auto proxy_config = net::ProxyConfig(); 44 | + proxy_config.proxy_rules().ParseFromString(proxy_rules_); 45 | + auto annotation = net::DefineNetworkTrafficAnnotation("test", "test"); 46 | + *config = net::ProxyConfigWithAnnotation(proxy_config, annotation); 47 | + return CONFIG_VALID; 48 | + } 49 | + 50 | + private: 51 | + const std::string proxy_rules_; 52 | +}; 53 | + 54 | + 55 | void CronetContext::InitRequestContextOnInitThread() { 56 | DCHECK(OnInitThread()); 57 | // Cannot create this inside Initialize because Android requires this to be 58 | // created on the JNI thread. 59 | - auto proxy_config_service = 60 | - cronet::CreateProxyConfigService(GetNetworkTaskRunner()); 61 | + static const char* const commandline_argv[] = {"cronet", nullptr}; 62 | + base::CommandLine::Init(sizeof(commandline_argv) / sizeof(*commandline_argv) - 1, commandline_argv); 63 | + 64 | + std::unique_ptr proxy_config_service; 65 | + if (!proxy_rules_.empty()) { 66 | + proxy_config_service = 67 | + std::make_unique(proxy_rules_); 68 | + } else { 69 | + proxy_config_service = 70 | + cronet::CreateProxyConfigService(GetNetworkTaskRunner()); 71 | + } 72 | g_net_log.Get().EnsureInitializedOnInitThread(); 73 | GetNetworkTaskRunner()->PostTask( 74 | FROM_HERE, 75 | diff --git a/components/cronet/cronet_context.h b/components/cronet/cronet_context.h 76 | index b5163c84b0..a19b25964f 100644 77 | --- a/components/cronet/cronet_context.h 78 | +++ b/components/cronet/cronet_context.h 79 | @@ -379,6 +379,8 @@ class CronetContext { 80 | // File thread should be destroyed last. 81 | std::unique_ptr file_thread_; 82 | 83 | + const std::string proxy_rules_; 84 | + 85 | // |network_tasks_| is owned by |this|. It is created off the network thread, 86 | // but invoked and destroyed on network thread. 87 | raw_ptr network_tasks_; 88 | diff --git a/components/cronet/cronet_global_state_stubs.cc b/components/cronet/cronet_global_state_stubs.cc 89 | index e1162a786c..d508cb6cc4 100644 90 | --- a/components/cronet/cronet_global_state_stubs.cc 91 | +++ b/components/cronet/cronet_global_state_stubs.cc 92 | @@ -13,7 +13,6 @@ 93 | #include "base/task/thread_pool.h" 94 | #include "base/task/thread_pool/thread_pool_instance.h" 95 | #include "net/proxy_resolution/configured_proxy_resolution_service.h" 96 | -#include "net/proxy_resolution/proxy_config_service.h" 97 | 98 | // This file provides minimal "stub" implementations of the Cronet global-state 99 | // functions for the native library build, sufficient to have cronet_tests and 100 | diff --git a/components/cronet/native/cronet.idl b/components/cronet/native/cronet.idl 101 | index 8e83cb0cf7..3be2a64b8e 100644 102 | --- a/components/cronet/native/cronet.idl 103 | +++ b/components/cronet/native/cronet.idl 104 | @@ -511,6 +511,8 @@ struct EngineParams { 105 | */ 106 | string user_agent; 107 | 108 | + string proxy_rules; 109 | + 110 | /** 111 | * Sets a default value for the Accept-Language header value for UrlRequests 112 | * created by this engine. Explicitly setting the Accept-Language header 113 | diff --git a/components/cronet/native/engine.cc b/components/cronet/native/engine.cc 114 | index c35c4dbfc3..91afa00273 100644 115 | --- a/components/cronet/native/engine.cc 116 | +++ b/components/cronet/native/engine.cc 117 | @@ -152,6 +152,7 @@ Cronet_RESULT Cronet_EngineImpl::StartWithParams( 118 | context_config_builder.experimental_options = params->experimental_options; 119 | context_config_builder.bypass_public_key_pinning_for_local_trust_anchors = 120 | params->enable_public_key_pinning_bypass_for_local_trust_anchors; 121 | + context_config_builder.proxy_rules = params->proxy_rules; 122 | if (!isnan(params->network_thread_priority)) { 123 | context_config_builder.network_thread_priority = 124 | params->network_thread_priority; 125 | diff --git a/components/cronet/native/generated/cronet.idl_c.h b/components/cronet/native/generated/cronet.idl_c.h 126 | index 988e6efacb..3c17921c69 100644 127 | --- a/components/cronet/native/generated/cronet.idl_c.h 128 | +++ b/components/cronet/native/generated/cronet.idl_c.h 129 | @@ -795,6 +795,9 @@ CRONET_EXPORT 130 | void Cronet_EngineParams_user_agent_set(Cronet_EngineParamsPtr self, 131 | const Cronet_String user_agent); 132 | CRONET_EXPORT 133 | +void Cronet_EngineParams_proxy_rules_set(Cronet_EngineParamsPtr self, 134 | + const Cronet_String proxy_rules); 135 | +CRONET_EXPORT 136 | void Cronet_EngineParams_accept_language_set( 137 | Cronet_EngineParamsPtr self, 138 | const Cronet_String accept_language); 139 | @@ -845,6 +848,9 @@ CRONET_EXPORT 140 | Cronet_String Cronet_EngineParams_user_agent_get( 141 | const Cronet_EngineParamsPtr self); 142 | CRONET_EXPORT 143 | +Cronet_String Cronet_EngineParams_proxy_rules_get( 144 | + const Cronet_EngineParamsPtr self); 145 | +CRONET_EXPORT 146 | Cronet_String Cronet_EngineParams_accept_language_get( 147 | const Cronet_EngineParamsPtr self); 148 | CRONET_EXPORT 149 | diff --git a/components/cronet/native/generated/cronet.idl_impl_struct.cc b/components/cronet/native/generated/cronet.idl_impl_struct.cc 150 | index b9120ff8c2..e62eac2e7f 100644 151 | --- a/components/cronet/native/generated/cronet.idl_impl_struct.cc 152 | +++ b/components/cronet/native/generated/cronet.idl_impl_struct.cc 153 | @@ -249,6 +249,12 @@ void Cronet_EngineParams_user_agent_set(Cronet_EngineParamsPtr self, 154 | self->user_agent = user_agent; 155 | } 156 | 157 | +void Cronet_EngineParams_proxy_rules_set(Cronet_EngineParamsPtr self, 158 | + const Cronet_String proxy_rules) { 159 | + DCHECK(self); 160 | + self->proxy_rules = proxy_rules; 161 | +} 162 | + 163 | void Cronet_EngineParams_accept_language_set( 164 | Cronet_EngineParamsPtr self, 165 | const Cronet_String accept_language) { 166 | @@ -342,6 +348,12 @@ Cronet_String Cronet_EngineParams_user_agent_get( 167 | return self->user_agent.c_str(); 168 | } 169 | 170 | +Cronet_String Cronet_EngineParams_proxy_rules_get( 171 | + const Cronet_EngineParamsPtr self) { 172 | + DCHECK(self); 173 | + return self->proxy_rules.c_str(); 174 | +} 175 | + 176 | Cronet_String Cronet_EngineParams_accept_language_get( 177 | const Cronet_EngineParamsPtr self) { 178 | DCHECK(self); 179 | diff --git a/components/cronet/native/generated/cronet.idl_impl_struct.h b/components/cronet/native/generated/cronet.idl_impl_struct.h 180 | index badb341ce3..cef7ccb4a9 100644 181 | --- a/components/cronet/native/generated/cronet.idl_impl_struct.h 182 | +++ b/components/cronet/native/generated/cronet.idl_impl_struct.h 183 | @@ -82,6 +82,7 @@ struct Cronet_EngineParams { 184 | 185 | bool enable_check_result = true; 186 | std::string user_agent; 187 | + std::string proxy_rules; 188 | std::string accept_language; 189 | std::string storage_path; 190 | bool enable_quic = true; 191 | diff --git a/components/cronet/url_request_context_config.cc b/components/cronet/url_request_context_config.cc 192 | index 5fe1caf483..f83e8a201a 100644 193 | --- a/components/cronet/url_request_context_config.cc 194 | +++ b/components/cronet/url_request_context_config.cc 195 | @@ -264,6 +264,7 @@ URLRequestContextConfig::URLRequestContextConfig( 196 | const std::string& storage_path, 197 | const std::string& accept_language, 198 | const std::string& user_agent, 199 | + const std::string& proxy_rules, 200 | base::Value::Dict experimental_options, 201 | std::unique_ptr mock_cert_verifier, 202 | bool enable_network_quality_estimator, 203 | @@ -278,6 +279,7 @@ URLRequestContextConfig::URLRequestContextConfig( 204 | storage_path(storage_path), 205 | accept_language(accept_language), 206 | user_agent(user_agent), 207 | + proxy_rules(proxy_rules), 208 | mock_cert_verifier(std::move(mock_cert_verifier)), 209 | enable_network_quality_estimator(enable_network_quality_estimator), 210 | bypass_public_key_pinning_for_local_trust_anchors( 211 | @@ -304,6 +306,7 @@ URLRequestContextConfig::CreateURLRequestContextConfig( 212 | const std::string& storage_path, 213 | const std::string& accept_language, 214 | const std::string& user_agent, 215 | + const std::string& proxy_rules, 216 | const std::string& unparsed_experimental_options, 217 | std::unique_ptr mock_cert_verifier, 218 | bool enable_network_quality_estimator, 219 | @@ -321,7 +324,7 @@ URLRequestContextConfig::CreateURLRequestContextConfig( 220 | } 221 | return base::WrapUnique(new URLRequestContextConfig( 222 | enable_quic, enable_spdy, enable_brotli, http_cache, http_cache_max_size, 223 | - load_disable_cache, storage_path, accept_language, user_agent, 224 | + load_disable_cache, storage_path, accept_language, user_agent, proxy_rules, 225 | std::move(experimental_options).value(), std::move(mock_cert_verifier), 226 | enable_network_quality_estimator, 227 | bypass_public_key_pinning_for_local_trust_anchors, 228 | @@ -828,7 +831,7 @@ std::unique_ptr 229 | URLRequestContextConfigBuilder::Build() { 230 | return URLRequestContextConfig::CreateURLRequestContextConfig( 231 | enable_quic, enable_spdy, enable_brotli, http_cache, http_cache_max_size, 232 | - load_disable_cache, storage_path, accept_language, user_agent, 233 | + load_disable_cache, storage_path, accept_language, user_agent, proxy_rules, 234 | experimental_options, std::move(mock_cert_verifier), 235 | enable_network_quality_estimator, 236 | bypass_public_key_pinning_for_local_trust_anchors, 237 | diff --git a/components/cronet/url_request_context_config.h b/components/cronet/url_request_context_config.h 238 | index 3ce700f953..2487a1d7b1 100644 239 | --- a/components/cronet/url_request_context_config.h 240 | +++ b/components/cronet/url_request_context_config.h 241 | @@ -127,6 +127,8 @@ struct URLRequestContextConfig { 242 | // User-Agent request header field. 243 | const std::string user_agent; 244 | 245 | + const std::string proxy_rules; 246 | + 247 | // Certificate verifier for testing. 248 | std::unique_ptr mock_cert_verifier; 249 | 250 | @@ -199,6 +201,9 @@ struct URLRequestContextConfig { 251 | const std::string& accept_language, 252 | // User-Agent request header field. 253 | const std::string& user_agent, 254 | + 255 | + const std::string& proxy_rules, 256 | + 257 | // JSON encoded experimental options. 258 | const std::string& unparsed_experimental_options, 259 | // MockCertVerifier to use for testing purposes. 260 | @@ -233,6 +238,9 @@ struct URLRequestContextConfig { 261 | const std::string& accept_language, 262 | // User-Agent request header field. 263 | const std::string& user_agent, 264 | + 265 | + const std::string& proxy_rules, 266 | + 267 | // Parsed experimental options. 268 | base::Value::Dict experimental_options, 269 | // MockCertVerifier to use for testing purposes. 270 | @@ -301,6 +309,9 @@ struct URLRequestContextConfigBuilder { 271 | std::string accept_language = ""; 272 | // User-Agent request header field. 273 | std::string user_agent = ""; 274 | + 275 | + std::string proxy_rules = ""; 276 | + 277 | // Experimental options encoded as a string in a JSON format containing 278 | // experiments and their corresponding configuration options. The format 279 | // is a JSON object with the name of the experiment as the key, and the 280 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [project] 6 | name = "python-cronet" 7 | version = "0.1.7" 8 | authors = [ 9 | { name="Lucas Moauro" }, 10 | ] 11 | description = "Python wrapper for Chromium's http library" 12 | readme = "README.md" 13 | requires-python = ">=3.8" 14 | classifiers = [ 15 | "Programming Language :: Python :: 3", 16 | "License :: OSI Approved :: MIT License", 17 | "Operating System :: OS Independent", 18 | ] 19 | dependencies = ["multidict"] 20 | 21 | [project.optional-dependencies] 22 | test = ["pytest", "aiohttp", "pytest-asyncio"] 23 | 24 | [tool.pytest.ini_options] 25 | pythonpath = ["src"] 26 | 27 | [tool.setuptools.packages.find] 28 | where = ["src"] 29 | 30 | [project.urls] 31 | Homepage = "https://github.com/lagenar/python-cronet" 32 | Issues = "https://github.com/lagenar/python-cronet/issues" 33 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import Extension, setup 2 | 3 | include_dirs = ["src/cronet/build/include"] 4 | 5 | setup( 6 | package_data={"cronet": ["build/include/*.h"]}, 7 | include_package_data=True, 8 | ext_modules=[ 9 | Extension( 10 | name="cronet._cronet", 11 | include_dirs=include_dirs, 12 | libraries=["cronet.134.0.6998.165"], 13 | sources=["src/cronet/_cronet.c"], 14 | ), 15 | ], 16 | ) 17 | -------------------------------------------------------------------------------- /src/cronet/__init__.py: -------------------------------------------------------------------------------- 1 | from .cronet import * 2 | -------------------------------------------------------------------------------- /src/cronet/_cronet.c: -------------------------------------------------------------------------------- 1 | #define PY_SSIZE_T_CLEAN 2 | #include 3 | #include 4 | #include 5 | 6 | #include "Python.h" 7 | #include "cronet_c.h" 8 | 9 | 10 | #define DEBUG 1 11 | 12 | #if DEBUG 13 | #define LOG(message) printf("DEBUG: %s\n", message) 14 | #else 15 | #define LOG(message) 16 | #endif 17 | 18 | #undef Py_Is 19 | #undef Py_IsNone 20 | 21 | // Export Py_Is(), Py_IsNone() as regular functions 22 | // for the stable ABI. 23 | int Py_Is(PyObject *x, PyObject *y) 24 | { 25 | return (x == y); 26 | } 27 | 28 | int Py_IsNone(PyObject *x) 29 | { 30 | return Py_Is(x, Py_None); 31 | } 32 | 33 | 34 | #define RUNNABLES_MAX_SIZE 10000 35 | 36 | typedef struct { 37 | Cronet_RunnablePtr arr[RUNNABLES_MAX_SIZE]; 38 | int front; 39 | int rear; 40 | int size; 41 | pthread_mutex_t mutex; 42 | } Runnables; 43 | 44 | void runnables_init(Runnables* q) { 45 | q->front = 0; 46 | q->rear = 0; 47 | q->size = 0; 48 | pthread_mutex_init(&q->mutex, NULL); 49 | } 50 | 51 | void runnables_destroy(Runnables* q) { 52 | pthread_mutex_destroy(&q->mutex); 53 | } 54 | 55 | int runnables_enqueue(Runnables* q, Cronet_RunnablePtr runnable) { 56 | int status = 0; 57 | 58 | pthread_mutex_lock(&q->mutex); 59 | if (q->size == RUNNABLES_MAX_SIZE) { 60 | status = -1; 61 | } else { 62 | q->arr[q->rear] = runnable; 63 | q->rear = (q->rear + 1) % RUNNABLES_MAX_SIZE; 64 | q->size++; 65 | } 66 | pthread_mutex_unlock(&q->mutex); 67 | 68 | return status; 69 | } 70 | 71 | Cronet_RunnablePtr runnables_dequeue(Runnables* q) { 72 | Cronet_RunnablePtr runnable = NULL; 73 | 74 | pthread_mutex_lock(&q->mutex); 75 | if (q->size > 0) { 76 | runnable = q->arr[q->front]; 77 | q->front = (q->front + 1) % RUNNABLES_MAX_SIZE; 78 | q->size--; 79 | } 80 | pthread_mutex_unlock(&q->mutex); 81 | 82 | return runnable; 83 | } 84 | 85 | /* saves the cronet runnable to execute next for a single executor 86 | cronet runables are the atomic steps of a request. 87 | */ 88 | typedef struct { 89 | Runnables* runnables; 90 | pthread_mutex_t mutex; 91 | pthread_cond_t new_item; 92 | volatile bool should_stop; 93 | } ExecutorContext; 94 | 95 | typedef struct { 96 | Cronet_UrlRequestCallbackPtr callback; 97 | PyObject *py_callback; 98 | bool allow_redirects; 99 | } RequestContext; 100 | 101 | 102 | typedef struct { 103 | size_t upload_size; 104 | size_t upload_bytes_read; 105 | const char *content; 106 | } UploadProviderContext; 107 | 108 | void RequestContext_destroy(RequestContext* ctx) 109 | { 110 | if (ctx->callback) { 111 | Cronet_UrlRequestCallback_Destroy(ctx->callback); 112 | } 113 | if (ctx->py_callback) { 114 | Py_DECREF(ctx->py_callback); 115 | } 116 | free(ctx); 117 | } 118 | 119 | /* callback passed to cronet to schedule a runnable. 120 | gets the executor context and updates the runnable. 121 | */ 122 | void execute_runnable(Cronet_ExecutorPtr executor, 123 | Cronet_RunnablePtr runnable) { 124 | ExecutorContext *run = 125 | (ExecutorContext *)Cronet_Executor_GetClientContext(executor); 126 | 127 | pthread_mutex_lock(&run->mutex); 128 | // TODO: check return value 129 | runnables_enqueue(run->runnables, runnable); 130 | pthread_cond_signal(&run->new_item); 131 | pthread_mutex_unlock(&run->mutex); 132 | } 133 | 134 | 135 | /* executor thread that waits for runnables and starts them. 136 | */ 137 | void *process_requests(void *executor_context) { 138 | ExecutorContext *ctx = (ExecutorContext *)executor_context; 139 | struct timespec ts; 140 | while (!ctx->should_stop) { 141 | 142 | pthread_mutex_lock(&ctx->mutex); 143 | clock_gettime(CLOCK_REALTIME, &ts); 144 | ts.tv_nsec += 100; 145 | 146 | int res = pthread_cond_timedwait(&ctx->new_item, &ctx->mutex, &ts); 147 | if (res == 0 || res == ETIMEDOUT) { 148 | Cronet_RunnablePtr runnable = runnables_dequeue(ctx->runnables); 149 | if (runnable != NULL) { 150 | Cronet_Runnable_Run(runnable); 151 | Cronet_Runnable_Destroy(runnable); 152 | } 153 | } 154 | pthread_mutex_unlock(&ctx->mutex); 155 | } 156 | return NULL; 157 | } 158 | 159 | 160 | void on_redirect_received(Cronet_UrlRequestCallbackPtr callback, 161 | Cronet_UrlRequestPtr request, 162 | Cronet_UrlResponseInfoPtr info, 163 | Cronet_String newLocationUrl) { 164 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 165 | const char *url = Cronet_UrlResponseInfo_url_get(info); 166 | int status_code = Cronet_UrlResponseInfo_http_status_code_get(info); 167 | int headers_size = Cronet_UrlResponseInfo_all_headers_list_size(info); 168 | PyGILState_STATE gstate; 169 | gstate = PyGILState_Ensure(); 170 | //PyObject *headers = PyDict_New(); 171 | PyObject *headers = PyList_New((Py_ssize_t)0); 172 | for (int i=0; i < headers_size; i++) { 173 | Cronet_HttpHeaderPtr header = Cronet_UrlResponseInfo_all_headers_list_at(info, i); 174 | const char *key = Cronet_HttpHeader_name_get(header); 175 | const char *value = Cronet_HttpHeader_value_get(header); 176 | 177 | PyObject *py_key = PyUnicode_FromStringAndSize(key, strlen(key)); 178 | PyObject *py_value = PyUnicode_FromStringAndSize(value, strlen(value)); 179 | PyObject *py_header = PyTuple_New((Py_ssize_t)2); 180 | PyTuple_SetItem(py_header, (Py_ssize_t)0, py_key); 181 | PyTuple_SetItem(py_header, (Py_ssize_t)1, py_value); 182 | PyList_Append(headers, py_header); 183 | //PyDict_SetItemString(headers, key, item); 184 | } 185 | PyObject_CallMethod(ctx->py_callback, "on_redirect_received", 186 | "ssiO", url, newLocationUrl, status_code, headers); 187 | PyGILState_Release(gstate); 188 | 189 | if (ctx->allow_redirects) { 190 | Cronet_UrlRequest_FollowRedirect(request); 191 | } else { 192 | Cronet_UrlRequest_Cancel(request); 193 | } 194 | } 195 | 196 | 197 | void on_response_started(Cronet_UrlRequestCallbackPtr callback, 198 | Cronet_UrlRequestPtr request, 199 | Cronet_UrlResponseInfoPtr info) { 200 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 201 | int status_code = Cronet_UrlResponseInfo_http_status_code_get(info); 202 | int headers_size = Cronet_UrlResponseInfo_all_headers_list_size(info); 203 | const char *url = Cronet_UrlResponseInfo_url_get(info); 204 | PyGILState_STATE gstate; 205 | gstate = PyGILState_Ensure(); 206 | //PyObject *headers = PyDict_New(); 207 | PyObject *headers = PyList_New((Py_ssize_t)0); 208 | for (int i=0; i < headers_size; i++) { 209 | Cronet_HttpHeaderPtr header = Cronet_UrlResponseInfo_all_headers_list_at(info, i); 210 | const char *key = Cronet_HttpHeader_name_get(header); 211 | const char *value = Cronet_HttpHeader_value_get(header); 212 | 213 | PyObject *py_key = PyUnicode_FromStringAndSize(key, strlen(key)); 214 | PyObject *py_value = PyUnicode_FromStringAndSize(value, strlen(value)); 215 | PyObject *py_header = PyTuple_New((Py_ssize_t)2); 216 | PyTuple_SetItem(py_header, (Py_ssize_t)0, py_key); 217 | PyTuple_SetItem(py_header, (Py_ssize_t)1, py_value); 218 | PyList_Append(headers, py_header); 219 | //PyDict_SetItemString(headers, key, item); 220 | } 221 | PyObject_CallMethod(ctx->py_callback, "on_response_started", "siO", 222 | url, status_code, headers); 223 | Py_DECREF(headers); 224 | PyGILState_Release(gstate); 225 | Cronet_BufferPtr buffer = Cronet_Buffer_Create(); 226 | Cronet_Buffer_InitWithAlloc(buffer, 32 * 1024); 227 | Cronet_UrlRequest_Read(request, buffer); 228 | } 229 | 230 | 231 | void on_read_completed(Cronet_UrlRequestCallbackPtr callback, 232 | Cronet_UrlRequestPtr request, 233 | Cronet_UrlResponseInfoPtr info, Cronet_BufferPtr buffer, 234 | uint64_t bytesRead) { 235 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 236 | const char *buf_data = (const char*)Cronet_Buffer_GetData(buffer); 237 | PyGILState_STATE gstate; 238 | gstate = PyGILState_Ensure(); 239 | PyObject *data = PyBytes_FromStringAndSize(buf_data, bytesRead); 240 | PyObject_CallMethod(ctx->py_callback, "on_read_completed", "O", data); 241 | Py_DECREF(data); 242 | PyGILState_Release(gstate); 243 | Cronet_UrlRequest_Read(request, buffer); 244 | } 245 | 246 | 247 | void on_succeeded(Cronet_UrlRequestCallbackPtr callback, 248 | Cronet_UrlRequestPtr request, 249 | Cronet_UrlResponseInfoPtr info) { 250 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 251 | 252 | PyGILState_STATE gstate; 253 | gstate = PyGILState_Ensure(); 254 | PyObject_CallMethod(ctx->py_callback, "on_succeeded", NULL); 255 | RequestContext_destroy(ctx); 256 | PyGILState_Release(gstate); 257 | 258 | Cronet_UrlRequest_Destroy(request); 259 | } 260 | 261 | 262 | void on_failed(Cronet_UrlRequestCallbackPtr callback, 263 | Cronet_UrlRequestPtr request, Cronet_UrlResponseInfoPtr info, 264 | Cronet_ErrorPtr error) { 265 | 266 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 267 | const char *error_msg = Cronet_Error_message_get(error); 268 | 269 | PyGILState_STATE gstate; 270 | gstate = PyGILState_Ensure(); 271 | PyObject_CallMethod(ctx->py_callback, "on_failed", "s", error_msg); 272 | RequestContext_destroy(ctx); 273 | PyGILState_Release(gstate); 274 | 275 | Cronet_UrlRequest_Destroy(request); 276 | } 277 | 278 | 279 | void on_canceled(Cronet_UrlRequestCallbackPtr callback, 280 | Cronet_UrlRequestPtr request, Cronet_UrlResponseInfoPtr info) { 281 | RequestContext *ctx = (RequestContext*)Cronet_UrlRequest_GetClientContext(request); 282 | 283 | PyGILState_STATE gstate; 284 | gstate = PyGILState_Ensure(); 285 | PyObject_CallMethod(ctx->py_callback, "on_canceled", NULL); 286 | RequestContext_destroy(ctx); 287 | PyGILState_Release(gstate); 288 | 289 | Cronet_UrlRequest_Destroy(request); 290 | } 291 | 292 | 293 | int64_t request_content_length(Cronet_UploadDataProviderPtr self) 294 | { 295 | UploadProviderContext *ctx = 296 | (UploadProviderContext *)Cronet_UploadDataProvider_GetClientContext(self); 297 | return ctx->upload_size; 298 | } 299 | 300 | 301 | void request_content_read(Cronet_UploadDataProviderPtr self, 302 | Cronet_UploadDataSinkPtr upload_data_sink, 303 | Cronet_BufferPtr buffer) 304 | { 305 | size_t buffer_size = Cronet_Buffer_GetSize(buffer); 306 | UploadProviderContext *ctx = 307 | (UploadProviderContext *)Cronet_UploadDataProvider_GetClientContext(self); 308 | 309 | size_t offset = ctx->upload_bytes_read; 310 | size_t rem = ctx->upload_size - ctx->upload_bytes_read; 311 | size_t size = buffer_size >= rem ? rem : buffer_size; 312 | memcpy(Cronet_Buffer_GetData(buffer), ctx->content + offset, size); 313 | ctx->upload_bytes_read += size; 314 | Cronet_UploadDataSink_OnReadSucceeded(upload_data_sink, size, false); 315 | } 316 | 317 | 318 | void request_content_rewind(Cronet_UploadDataProviderPtr self, 319 | Cronet_UploadDataSinkPtr upload_data_sink) 320 | { 321 | } 322 | 323 | 324 | void request_content_close(Cronet_UploadDataProviderPtr self) 325 | { 326 | free(Cronet_UploadDataProvider_GetClientContext(self)); 327 | } 328 | 329 | 330 | typedef struct { 331 | PyObject_HEAD 332 | Cronet_EnginePtr engine; 333 | Cronet_ExecutorPtr executor; 334 | ExecutorContext executor_context; 335 | pthread_t executor_thread; 336 | } CronetEngineObject; 337 | 338 | 339 | static void CronetEngine_dealloc(CronetEngineObject *self) { 340 | self->executor_context.should_stop = true; 341 | pthread_join(self->executor_thread, NULL); 342 | runnables_destroy(self->executor_context.runnables); 343 | pthread_cond_destroy(&self->executor_context.new_item); 344 | pthread_mutex_destroy(&self->executor_context.mutex); 345 | Cronet_Executor_Destroy(self->executor); 346 | Cronet_Engine_Shutdown(self->engine); 347 | Cronet_Engine_Destroy(self->engine); 348 | Py_TYPE(self)->tp_free((PyObject *)self); 349 | } 350 | 351 | 352 | static int CronetEngine_init(CronetEngineObject *self, PyObject *args, PyObject *kwds) { 353 | self->engine = Cronet_Engine_Create(); 354 | bool engine_running = false; 355 | Cronet_EngineParamsPtr engine_params = Cronet_EngineParams_Create(); 356 | if (!self->engine || !engine_params) { 357 | PyErr_SetString(PyExc_RuntimeError, "Could not create engine"); 358 | goto fail; 359 | } 360 | PyObject *py_proxy_rules = NULL; 361 | if (!PyArg_ParseTuple(args, "O", &py_proxy_rules)) { 362 | goto fail; 363 | } 364 | if (!Py_IsNone(py_proxy_rules)) { 365 | const char *proxy_rules = PyUnicode_AsUTF8(py_proxy_rules); 366 | if (!proxy_rules) { 367 | goto fail; 368 | } 369 | Cronet_EngineParams_proxy_rules_set(engine_params, proxy_rules); 370 | LOG(proxy_rules); 371 | } 372 | Cronet_EngineParams_http_cache_mode_set( 373 | engine_params, Cronet_EngineParams_HTTP_CACHE_MODE_DISABLED); 374 | Cronet_EngineParams_enable_quic_set(engine_params, false); 375 | Cronet_EngineParams_user_agent_set(engine_params, "python-cronet"); 376 | 377 | Cronet_RESULT res = Cronet_Engine_StartWithParams(self->engine, engine_params); 378 | if (res < 0) { 379 | PyErr_Format(PyExc_RuntimeError, "Could not start engine(%d)", res); 380 | goto fail; 381 | } 382 | engine_running = true; 383 | Cronet_EngineParams_Destroy(engine_params); 384 | engine_params = NULL; 385 | self->executor = Cronet_Executor_CreateWith(&execute_runnable); 386 | if (!self->executor) { 387 | PyErr_SetString(PyExc_RuntimeError, "Could not create executor"); 388 | goto fail; 389 | } 390 | Runnables* runnables = (Runnables*)malloc(sizeof(Runnables)); 391 | runnables_init(runnables); 392 | if (!runnables) { 393 | abort(); 394 | } 395 | self->executor_context = (ExecutorContext){ 396 | .runnables = runnables, 397 | .mutex = PTHREAD_MUTEX_INITIALIZER, 398 | .new_item = PTHREAD_COND_INITIALIZER, 399 | .should_stop = false, 400 | }; 401 | Cronet_Executor_SetClientContext(self->executor, 402 | (void *)&self->executor_context); 403 | pthread_t executor_t; 404 | if (pthread_create(&executor_t, NULL, process_requests, 405 | (void *)&self->executor_context) != 0) { 406 | PyErr_SetString(PyExc_RuntimeError, "Could not start executor thread"); 407 | goto fail; 408 | } 409 | self->executor_thread = executor_t; 410 | LOG("Started cronet engine"); 411 | return 0; 412 | 413 | fail: 414 | if (engine_running) { 415 | Cronet_Engine_Shutdown(self->engine); 416 | } 417 | if (self->executor) { 418 | Cronet_Executor_Destroy(self->executor); 419 | } 420 | if (engine_params) { 421 | Cronet_EngineParams_Destroy(engine_params); 422 | } 423 | if (self->engine) { 424 | Cronet_Engine_Destroy(self->engine); 425 | } 426 | return -1; 427 | } 428 | 429 | 430 | static PyObject *CronetEngine_request(CronetEngineObject *self, PyObject *args) { 431 | PyObject *py_callback = NULL; 432 | if (!PyArg_ParseTuple(args, "O", &py_callback)) { 433 | return NULL; 434 | } 435 | PyObject *py_request = PyObject_GetAttrString(py_callback, "request"); 436 | if (!py_request) { 437 | return NULL; 438 | } 439 | PyObject* url = PyObject_GetAttrString(py_request, "url"); 440 | if (!url) { 441 | return NULL; 442 | } 443 | PyObject* method = PyObject_GetAttrString(py_request, "method"); 444 | if (!method) { 445 | return NULL; 446 | } 447 | PyObject* content = PyObject_GetAttrString(py_request, "content"); 448 | if (!content) { 449 | return NULL; 450 | } 451 | PyObject* headers = PyObject_GetAttrString(py_request, "headers"); 452 | if (!headers) { 453 | return NULL; 454 | } 455 | 456 | const char *c_url = PyUnicode_AsUTF8(url); 457 | if (!c_url) { 458 | return NULL; 459 | } 460 | const char *c_method = PyUnicode_AsUTF8(method); 461 | if (!c_method) { 462 | return NULL; 463 | } 464 | char *c_content = NULL; 465 | if (!Py_IsNone(content)) { 466 | c_content = PyBytes_AsString(content); 467 | if (!c_content) { 468 | return NULL; 469 | } 470 | } 471 | 472 | Cronet_UrlRequestPtr request = Cronet_UrlRequest_Create(); 473 | Cronet_UrlRequestParamsPtr request_params = Cronet_UrlRequestParams_Create(); 474 | Cronet_UrlRequestParams_http_method_set(request_params, c_method); 475 | 476 | if (!Py_IsNone(headers)) { 477 | PyObject *items = PyDict_Items(headers); 478 | if (!items) { 479 | return NULL; 480 | } 481 | Py_ssize_t size = PyList_Size(items); 482 | for (Py_ssize_t i = 0; i < size; i++) { 483 | PyObject *item = PyList_GetItem(items, i); 484 | if (!item) { 485 | return NULL; 486 | } 487 | PyObject *key_obj = PyTuple_GetItem(item, 0); 488 | if (!key_obj) { 489 | return NULL; 490 | } 491 | PyObject* value_obj = PyTuple_GetItem(item, 1); 492 | if (!value_obj) { 493 | return NULL; 494 | } 495 | const char* key = PyUnicode_AsUTF8(key_obj); 496 | if (!key) { 497 | return NULL; 498 | } 499 | const char* value = PyUnicode_AsUTF8(value_obj); 500 | if (!value) { 501 | return NULL; 502 | } 503 | Cronet_HttpHeaderPtr request_header = Cronet_HttpHeader_Create(); 504 | Cronet_HttpHeader_name_set(request_header, key); 505 | Cronet_HttpHeader_value_set(request_header, value); 506 | Cronet_UrlRequestParams_request_headers_add(request_params, request_header); 507 | } 508 | } 509 | 510 | PyObject *py_allow_redirects = PyObject_GetAttrString(py_request, "allow_redirects"); 511 | if (!py_allow_redirects) { 512 | return NULL; 513 | } 514 | int allow_redirects = PyObject_IsTrue(py_allow_redirects); 515 | if (allow_redirects == -1) { 516 | return NULL; 517 | } 518 | 519 | Py_INCREF(py_callback); 520 | Cronet_UrlRequestCallbackPtr callback = Cronet_UrlRequestCallback_CreateWith( 521 | &on_redirect_received, &on_response_started, &on_read_completed, 522 | &on_succeeded, &on_failed, &on_canceled); 523 | 524 | RequestContext *ctx = (RequestContext*)malloc(sizeof(RequestContext)); 525 | if (!ctx) { 526 | abort(); 527 | } 528 | ctx->callback = NULL; 529 | ctx->allow_redirects = (bool)allow_redirects; 530 | ctx->py_callback = py_callback; 531 | if (c_content) { 532 | UploadProviderContext *upload_ctx = 533 | (UploadProviderContext*)malloc(sizeof(UploadProviderContext)); 534 | if (!upload_ctx) { 535 | abort(); 536 | } 537 | upload_ctx->content = c_content; 538 | upload_ctx->upload_size = strlen(c_content); 539 | upload_ctx->upload_bytes_read = 0; 540 | Cronet_UploadDataProviderPtr data_provider = 541 | Cronet_UploadDataProvider_CreateWith(&request_content_length, 542 | &request_content_read, 543 | &request_content_rewind, 544 | &request_content_close); 545 | 546 | Cronet_UploadDataProvider_SetClientContext(data_provider, (void*)upload_ctx); 547 | Cronet_UrlRequestParams_upload_data_provider_set(request_params, data_provider); 548 | } 549 | 550 | Cronet_UrlRequest_SetClientContext(request, (void*)ctx); 551 | Cronet_UrlRequest_InitWithParams(request, self->engine, c_url, request_params, 552 | callback, self->executor); 553 | Cronet_UrlRequestParams_Destroy(request_params); 554 | PyObject *capsule = PyCapsule_New(request, NULL, NULL); 555 | if (!capsule) { 556 | Py_DECREF(py_callback); 557 | RequestContext_destroy(ctx); 558 | Cronet_UrlRequest_Destroy(request); 559 | return NULL; 560 | } 561 | Cronet_UrlRequest_Start(request); 562 | return capsule; 563 | } 564 | 565 | 566 | static PyObject *CronetEngine_cancel(CronetEngineObject *self, PyObject *args) { 567 | PyObject *capsule = NULL; 568 | if (!PyArg_ParseTuple(args, "O", &capsule)) { 569 | return NULL; 570 | } 571 | void *request = PyCapsule_GetPointer(capsule, NULL); 572 | if (!request) { 573 | return NULL; 574 | } 575 | Cronet_UrlRequest_Cancel((Cronet_UrlRequestPtr)request); 576 | Py_RETURN_NONE; 577 | } 578 | 579 | 580 | static PyMethodDef CronetEngine_methods[] = { 581 | {"request", (PyCFunction)CronetEngine_request, METH_VARARGS, 582 | "Run a request"}, 583 | {"cancel", (PyCFunction)CronetEngine_cancel, METH_VARARGS, 584 | "Cancel a request"}, 585 | {NULL} /* Sentinel */ 586 | }; 587 | 588 | 589 | static PyTypeObject CronetEngineType = { 590 | .ob_base = PyVarObject_HEAD_INIT(NULL, 0).tp_name = "_cronet.CronetEngine", 591 | .tp_doc = PyDoc_STR("Cronet engine"), 592 | .tp_basicsize = sizeof(CronetEngineObject), 593 | .tp_itemsize = 0, 594 | .tp_flags = Py_TPFLAGS_DEFAULT, 595 | .tp_new = PyType_GenericNew, 596 | .tp_init = (initproc)CronetEngine_init, 597 | .tp_dealloc = (destructor)CronetEngine_dealloc, 598 | .tp_methods = CronetEngine_methods, 599 | }; 600 | 601 | 602 | static PyModuleDef cronetmodule = { 603 | .m_base = PyModuleDef_HEAD_INIT, 604 | .m_name = "cronet", 605 | .m_doc = "", 606 | .m_size = -1, 607 | }; 608 | 609 | PyMODINIT_FUNC PyInit__cronet(void) { 610 | PyObject *m; 611 | if (PyType_Ready(&CronetEngineType) < 0) 612 | return NULL; 613 | 614 | m = PyModule_Create(&cronetmodule); 615 | if (m == NULL) 616 | return NULL; 617 | 618 | Py_INCREF(&CronetEngineType); 619 | if (PyModule_AddObject(m, "CronetEngine", (PyObject *)&CronetEngineType) < 0) { 620 | Py_DECREF(&CronetEngineType); 621 | Py_DECREF(m); 622 | return NULL; 623 | } 624 | 625 | return m; 626 | } 627 | -------------------------------------------------------------------------------- /src/cronet/cronet.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import concurrent.futures 3 | import json 4 | from dataclasses import dataclass 5 | from functools import cached_property 6 | from typing import Any, Optional, Union 7 | from urllib.parse import ParseResult, parse_qs, urlencode, urlparse 8 | 9 | from multidict import CIMultiDict 10 | 11 | from . import _cronet 12 | 13 | 14 | class CronetException(Exception): 15 | pass 16 | 17 | 18 | URLParams = Union[str, dict[str, Any], tuple[str, Any]] 19 | 20 | 21 | @dataclass 22 | class Request: 23 | url: str 24 | method: str 25 | headers: dict[str, str] 26 | content: bytes 27 | allow_redirects: bool = True 28 | 29 | def add_url_params(self, params: URLParams) -> None: 30 | if not params: 31 | return 32 | 33 | request_params = {} 34 | if isinstance(params, str): 35 | request_params = parse_qs(params) 36 | else: 37 | items = tuple() 38 | if isinstance(params, dict): 39 | items = params.items() 40 | elif isinstance(params, tuple): 41 | items = params 42 | else: 43 | raise TypeError("Received object of invalid type for `params`") 44 | for k, v in items: 45 | request_params.setdefault(k, []) 46 | request_params[k].append(str(v)) 47 | 48 | parsed_url = urlparse(self.url) 49 | if request_params: 50 | query = parsed_url.query 51 | url_params = parse_qs(query) 52 | for k, v in url_params.items(): 53 | request_params.setdefault(k, []) 54 | request_params[k].extend(v) 55 | 56 | parsed_url = ParseResult( 57 | scheme=parsed_url.scheme, 58 | netloc=parsed_url.netloc, 59 | path=parsed_url.path, 60 | params=parsed_url.params, 61 | query=urlencode(request_params, doseq=True), 62 | fragment=parsed_url.fragment, 63 | ) 64 | 65 | self.url = parsed_url.geturl() 66 | 67 | def set_form_data(self, data: dict[str, str]): 68 | self.content = urlencode(data).encode("utf8") 69 | self.headers["Content-Type"] = "application/x-www-form-urlencoded" 70 | 71 | def set_json_data(self, data: Any): 72 | self.content = json.dumps(data).encode("utf8") 73 | self.headers["Content-Type"] = "application/json" 74 | 75 | 76 | @dataclass 77 | class Response: 78 | status_code: int 79 | headers: CIMultiDict 80 | url: str 81 | content: Optional[bytes] 82 | 83 | @cached_property 84 | def text(self): 85 | return self.content.decode("utf8") 86 | 87 | def json(self) -> Any: 88 | return json.loads(self.text) 89 | 90 | 91 | class RequestCallback: 92 | def __init__( 93 | self, request: Request, future: Union[asyncio.Future, concurrent.futures.Future] 94 | ): 95 | self._response = None 96 | self._response_content = bytearray() 97 | self._future = future 98 | self.request = request 99 | 100 | def _set_result(self, result: Any): 101 | if isinstance(self._future, concurrent.futures.Future): 102 | self._future.set_result(result) 103 | elif isinstance(self._future, asyncio.Future): 104 | self._future.get_loop().call_soon_threadsafe( 105 | self._future.set_result, result 106 | ) 107 | 108 | def _set_exception(self, exc: Exception): 109 | if isinstance(self._future, concurrent.futures.Future): 110 | self._future.set_exception(exc) 111 | elif isinstance(self._future, asyncio.Future): 112 | self._future.get_loop().call_soon_threadsafe( 113 | self._future.set_exception, exc 114 | ) 115 | 116 | def on_redirect_received( 117 | self, 118 | url: str, 119 | new_location: str, 120 | status_code: int, 121 | headers: list[tuple[str, str]], 122 | ): 123 | self._response = Response( 124 | url=url, status_code=status_code, headers=CIMultiDict(headers), content=b"" 125 | ) 126 | if not self.request.allow_redirects: 127 | self._set_result(self._response) 128 | 129 | def on_response_started( 130 | self, url: str, status_code: int, headers: list[tuple[str, str]] 131 | ): 132 | self._response = Response( 133 | url=url, status_code=status_code, headers=CIMultiDict(headers), content=None 134 | ) 135 | 136 | def on_read_completed(self, data: bytes): 137 | self._response_content.extend(data) 138 | 139 | def on_succeeded(self): 140 | self._response.content = bytes(self._response_content) 141 | self._set_result(self._response) 142 | 143 | def on_failed(self, error: str): 144 | self._set_exception(CronetException(error)) 145 | 146 | def on_canceled(self): 147 | self._set_result(self._response) 148 | 149 | 150 | class Cronet: 151 | def __init__(self, proxy_settings: Optional[str] = None): 152 | self._engine = None 153 | self.proxy_settings = proxy_settings 154 | 155 | def __enter__(self): 156 | self.start() 157 | return self 158 | 159 | def __exit__(self, *args): 160 | self.stop() 161 | 162 | def start(self): 163 | if not self._engine: 164 | self._engine = _cronet.CronetEngine(self.proxy_settings) 165 | 166 | def stop(self): 167 | if self._engine: 168 | del self._engine 169 | 170 | async def request( 171 | self, 172 | method: str, 173 | url: str, 174 | *, 175 | params: Optional[URLParams] = None, 176 | data: Optional[dict[str, str]] = None, 177 | content: Optional[bytes] = None, 178 | json: Optional[Any] = None, 179 | headers: Optional[dict[str, str]] = None, 180 | allow_redirects: bool = True, 181 | timeout: float = 10.0, 182 | ) -> Response: 183 | if len([c_arg for c_arg in [data, content, json] if c_arg]) > 1: 184 | raise ValueError("Only one of `data`, `content` and `json` can be provided") 185 | 186 | self.start() 187 | req = Request( 188 | method=method, 189 | url=url, 190 | content=content, 191 | headers=headers or {}, 192 | allow_redirects=allow_redirects, 193 | ) 194 | if params: 195 | req.add_url_params(params) 196 | if data: 197 | req.set_form_data(data) 198 | elif json: 199 | req.set_json_data(json) 200 | request_future = asyncio.Future() 201 | callback = RequestCallback(req, request_future) 202 | cronet_req = self._engine.request(callback) 203 | timeout_task = asyncio.create_task(asyncio.sleep(timeout)) 204 | done, pending = await asyncio.wait( 205 | [request_future, timeout_task], return_when=asyncio.FIRST_COMPLETED 206 | ) 207 | for task in pending: 208 | task.cancel() 209 | 210 | if request_future in done: 211 | return request_future.result() 212 | else: 213 | self._engine.cancel(cronet_req) 214 | raise TimeoutError() 215 | 216 | async def get(self, url: str, **kwargs) -> Response: 217 | return await self.request("GET", url, **kwargs) 218 | 219 | async def post(self, url: str, **kwargs) -> Response: 220 | return await self.request("POST", url, **kwargs) 221 | 222 | async def put(self, url: str, **kwargs) -> Response: 223 | return await self.request("PUT", url, **kwargs) 224 | 225 | async def patch(self, url: str, **kwargs) -> Response: 226 | return await self.request("PATCH", url, **kwargs) 227 | 228 | async def delete(self, url: str, **kwargs) -> Response: 229 | return await self.request("DELETE", url, **kwargs) 230 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/lagenar/python-cronet/d3ccfa2ce0be405c86655ad26fad930a9c7dbaf1/tests/__init__.py -------------------------------------------------------------------------------- /tests/conftest.py: -------------------------------------------------------------------------------- 1 | import http.client 2 | import os 3 | import subprocess 4 | import sys 5 | import time 6 | 7 | import cronet 8 | import pytest 9 | 10 | 11 | def wait_server_ready(): 12 | attempts = 5 13 | while True: 14 | try: 15 | conn = http.client.HTTPConnection("127.0.0.1", 8080) 16 | conn.request("GET", "/") 17 | except Exception: 18 | attempts -= 1 19 | if not attempts: 20 | raise 21 | else: 22 | time.sleep(0.5) 23 | continue 24 | else: 25 | break 26 | 27 | 28 | @pytest.fixture(scope="session") 29 | def aiohttp_server(): 30 | process = subprocess.Popen((sys.executable, "tests/server.py")) 31 | wait_server_ready() 32 | yield 33 | process.terminate() 34 | process.wait() 35 | 36 | 37 | @pytest.fixture() 38 | def cronet_client(): 39 | with cronet.Cronet() as cr: 40 | yield cr 41 | -------------------------------------------------------------------------------- /tests/server.py: -------------------------------------------------------------------------------- 1 | import base64 2 | 3 | from aiohttp import web 4 | 5 | 6 | async def status_code(request: web.Request): 7 | return web.Response(text="", status=int(request.match_info["status_code"])) 8 | 9 | 10 | async def echo(request: web.Request): 11 | content = await request.read() 12 | post_data = await request.post() 13 | data = { 14 | "headers": dict(request.headers), 15 | "url": str(request.rel_url), 16 | "method": request.method, 17 | "base64_content": base64.b64encode(content).decode("utf8"), 18 | "post_data": dict(post_data), 19 | } 20 | return web.json_response(data) 21 | 22 | 23 | async def redirect(request: web.Request): 24 | return web.Response(status=301, headers={"location": "/echo"}) 25 | 26 | 27 | app = web.Application() 28 | app.add_routes( 29 | [ 30 | web.get(r"/status_code/{status_code:\d+}", status_code), 31 | web.get("/echo", echo), 32 | web.post("/echo", echo), 33 | web.get("/redirect", redirect), 34 | ] 35 | ) 36 | 37 | 38 | if __name__ == "__main__": 39 | web.run_app(app) 40 | -------------------------------------------------------------------------------- /tests/test_cronet.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import json 3 | 4 | import cronet 5 | import pytest 6 | 7 | BASE_URL = "http://127.0.0.1:8080" 8 | 9 | 10 | @pytest.mark.asyncio 11 | async def test_status_code(aiohttp_server, cronet_client): 12 | response = await cronet_client.request("GET", f"{BASE_URL}/status_code/200") 13 | assert response.status_code == 200 14 | response = await cronet_client.request("GET", f"{BASE_URL}/status_code/404") 15 | assert response.status_code == 404 16 | 17 | 18 | @pytest.mark.asyncio 19 | async def test_send_headers(aiohttp_server, cronet_client): 20 | headers = { 21 | "a": "1", 22 | "b": "2", 23 | "User-Agent": "cronet", 24 | "Accept-Encoding": "*", 25 | "Host": "127.0.0.1:8080", 26 | "Connection": "keep-alive", 27 | } 28 | response = await cronet_client.request("GET", f"{BASE_URL}/echo", headers=headers) 29 | assert response.json()["headers"] == headers 30 | 31 | 32 | @pytest.mark.asyncio 33 | async def test_case_insentive_headers(aiohttp_server, cronet_client): 34 | response = await cronet_client.request("GET", f"{BASE_URL}/echo") 35 | assert response.headers["content-type"] == "application/json; charset=utf-8" 36 | assert response.headers["Content-Type"] == "application/json; charset=utf-8" 37 | 38 | 39 | @pytest.mark.asyncio 40 | async def test_send_params(aiohttp_server, cronet_client): 41 | response = await cronet_client.request( 42 | "GET", f"{BASE_URL}/echo", params={"test1": "1", "test2": "2"} 43 | ) 44 | assert response.url == f"{BASE_URL}/echo?test1=1&test2=2" 45 | assert response.json()["url"] == "/echo?test1=1&test2=2" 46 | 47 | 48 | @pytest.mark.asyncio 49 | async def test_send_content(aiohttp_server, cronet_client): 50 | request_content = b"\xe8\xad\x89\xe6\x98\x8e" * 3550 51 | response = await cronet_client.request( 52 | "GET", f"{BASE_URL}/echo", content=request_content 53 | ) 54 | content = json.loads(response.text)["base64_content"] 55 | assert base64.b64decode(content) == request_content 56 | 57 | 58 | @pytest.mark.asyncio 59 | async def test_send_form_data(aiohttp_server, cronet_client): 60 | data = {"email": "test@example.com", "password": "test"} 61 | response = await cronet_client.request("POST", f"{BASE_URL}/echo", data=data) 62 | response_data = json.loads(response.text) 63 | assert response_data["post_data"] == data 64 | assert ( 65 | response_data["headers"]["Content-Type"] == "application/x-www-form-urlencoded" 66 | ) 67 | 68 | 69 | @pytest.mark.asyncio 70 | async def test_send_json_data(aiohttp_server, cronet_client): 71 | data = {"form": {"email": "test@example.com", "password": "test"}} 72 | response = await cronet_client.request("POST", f"{BASE_URL}/echo", json=data) 73 | response_data = json.loads(response.text) 74 | json_data = base64.b64decode(response_data["base64_content"]) 75 | assert json.loads(json_data) == data 76 | assert response_data["headers"]["Content-Type"] == "application/json" 77 | 78 | 79 | @pytest.mark.asyncio 80 | async def test_redirect(aiohttp_server, cronet_client): 81 | response = await cronet_client.request( 82 | "GET", f"{BASE_URL}/redirect", allow_redirects=True 83 | ) 84 | assert response.url == f"{BASE_URL}/echo" 85 | assert response.status_code == 200 86 | 87 | response = await cronet_client.request( 88 | "GET", f"{BASE_URL}/redirect", allow_redirects=False 89 | ) 90 | assert response.status_code == 301 91 | assert response.url == f"{BASE_URL}/redirect" 92 | assert response.headers["location"] == "/echo" 93 | --------------------------------------------------------------------------------